diff --git a/include/providers/base_provider.hpp b/include/providers/base_provider.hpp index 86616f4b..e1b2dc85 100644 --- a/include/providers/base_provider.hpp +++ b/include/providers/base_provider.hpp @@ -95,6 +95,17 @@ protected: return fm_; } + [[nodiscard]] virtual auto remove_directory_impl(const std::string &api_path) + -> api_error = 0; + + [[nodiscard]] virtual auto remove_file_impl(const std::string &api_path) + -> api_error = 0; + + [[nodiscard]] virtual auto upload_file_impl(const std::string &api_path, + const std::string &source_path, + stop_type &stop_requested) + -> api_error = 0; + public: [[nodiscard]] auto create_directory_clone_source_meta(const std::string &source_api_path, @@ -151,6 +162,12 @@ public: [[nodiscard]] auto is_file_writeable(const std::string &api_path) const -> bool override; + [[nodiscard]] auto remove_directory(const std::string &api_path) + -> api_error override; + + [[nodiscard]] auto remove_file(const std::string &api_path) + -> api_error override; + [[nodiscard]] auto remove_item_meta(const std::string &api_path, const std::string &key) -> api_error override; @@ -168,6 +185,11 @@ public: i_file_manager *mgr) -> bool override; void stop() override; + + [[nodiscard]] auto upload_file(const std::string &api_path, + const std::string &source_path, + stop_type &stop_requested) + -> api_error override; }; } // namespace repertory diff --git a/include/providers/s3/s3_provider.hpp b/include/providers/s3/s3_provider.hpp index a8cc425e..f8c1a8fe 100644 --- a/include/providers/s3/s3_provider.hpp +++ b/include/providers/s3/s3_provider.hpp @@ -80,6 +80,17 @@ protected: directory_item_list &list) const -> api_error override; + [[nodiscard]] auto remove_directory_impl(const std::string &api_path) + -> api_error override; + + [[nodiscard]] auto remove_file_impl(const std::string &api_path) + -> api_error override; + + [[nodiscard]] auto upload_file_impl(const std::string &api_path, + const std::string &source_path, + stop_type &stop_requested) + -> api_error override; + public: [[nodiscard]] auto get_directory_item_count(const std::string &api_path) const -> std::uint64_t override; @@ -118,12 +129,6 @@ public: stop_type &stop_requested) -> api_error override; - [[nodiscard]] auto remove_directory(const std::string &api_path) - -> api_error override; - - [[nodiscard]] auto remove_file(const std::string &api_path) - -> api_error override; - [[nodiscard]] auto rename_file(const std::string &from_api_path, const std::string &to_api_path) -> api_error override; @@ -132,11 +137,6 @@ public: i_file_manager *mgr) -> bool override; void stop() override; - - [[nodiscard]] auto upload_file(const std::string &api_path, - const std::string &source_path, - stop_type &stop_requested) - -> api_error override; }; } // namespace repertory diff --git a/include/providers/sia/sia_provider.hpp b/include/providers/sia/sia_provider.hpp index 8f3fe510..6e6349eb 100644 --- a/include/providers/sia/sia_provider.hpp +++ b/include/providers/sia/sia_provider.hpp @@ -58,6 +58,17 @@ protected: directory_item_list &list) const -> api_error override; + [[nodiscard]] auto remove_directory_impl(const std::string &api_path) + -> api_error override; + + [[nodiscard]] auto remove_file_impl(const std::string &api_path) + -> api_error override; + + [[nodiscard]] auto upload_file_impl(const std::string &api_path, + const std::string &source_path, + stop_type &stop_requested) + -> api_error override; + public: [[nodiscard]] auto get_directory_item_count(const std::string &api_path) const -> std::uint64_t override; @@ -96,12 +107,6 @@ public: stop_type &stop_requested) -> api_error override; - [[nodiscard]] auto remove_directory(const std::string &api_path) - -> api_error override; - - [[nodiscard]] auto remove_file(const std::string &api_path) - -> api_error override; - [[nodiscard]] auto rename_file(const std::string &from_api_path, const std::string &to_api_path) -> api_error override; @@ -110,11 +115,6 @@ public: i_file_manager *mgr) -> bool override; void stop() override; - - [[nodiscard]] auto upload_file(const std::string &api_path, - const std::string &source_path, - stop_type &stop_requested) - -> api_error override; }; } // namespace repertory diff --git a/src/providers/base_provider.cpp b/src/providers/base_provider.cpp index 7b5603bc..50c3e5e4 100644 --- a/src/providers/base_provider.cpp +++ b/src/providers/base_provider.cpp @@ -563,6 +563,97 @@ void base_provider::remove_deleted_files() { } } +auto base_provider::remove_file(const std::string &api_path) -> api_error { + const auto notify_end = [&api_path](api_error error) -> api_error { + if (error == api_error::success) { + event_system::instance().raise(api_path); + } else { + event_system::instance().raise( + api_path, api_error_to_string(error)); + } + + return error; + }; + + const auto *const function_name = __FUNCTION__; + + const auto remove_file_meta = [this, &api_path, &function_name, + ¬ify_end]() -> api_error { + api_meta_map meta{}; + auto res = get_item_meta(api_path, meta); + + auto res2 = get_db()->Delete(rocksdb::WriteOptions(), api_path); + if (not res2.ok()) { + utils::error::raise_api_path_error(function_name, api_path, res2.code(), + "failed to remove file"); + return notify_end(api_error::error); + } + + return notify_end(res); + }; + + bool exists{}; + auto res = is_directory(api_path, exists); + if (res != api_error::success) { + return notify_end(res); + } + if (exists) { + exists = false; + return notify_end(api_error::directory_exists); + } + + res = is_file(api_path, exists); + if (res != api_error::success) { + return notify_end(res); + } + if (not exists) { + event_system::instance().raise( + api_path, api_error_to_string(api_error::item_not_found)); + return remove_file_meta(); + } + + res = remove_file_impl(api_path); + if (res != api_error::success) { + return notify_end(res); + } + + return remove_file_meta(); +} + +auto base_provider::remove_directory(const std::string &api_path) -> api_error { + const auto notify_end = [&api_path](api_error error) -> api_error { + if (error == api_error::success) { + event_system::instance().raise(api_path); + } else { + event_system::instance().raise( + api_path, api_error_to_string(error)); + } + return error; + }; + + bool exists{}; + auto res = is_directory(api_path, exists); + if (res != api_error::success) { + return notify_end(res); + } + if (not exists) { + return notify_end(api_error::item_not_found); + } + + res = remove_directory_impl(api_path); + if (res != api_error::success) { + return notify_end(res); + } + + auto status = get_db()->Delete(rocksdb::WriteOptions(), api_path); + if (not status.ok()) { + utils::error::raise_api_path_error(__FUNCTION__, api_path, status.code(), + "failed to remove directory"); + return notify_end(api_error::error); + } + + return notify_end(api_error::success); +} auto base_provider::remove_item_meta(const std::string &api_path, const std::string &key) -> api_error { api_meta_map meta{}; @@ -687,4 +778,24 @@ void base_provider::stop() { polling::instance().remove_callback("check_deleted"); db_.reset(); } + +auto base_provider::upload_file(const std::string &api_path, + const std::string &source_path, + stop_type &stop_requested) -> api_error { + event_system::instance().raise(api_path, source_path); + + const auto notify_end = [&api_path, + &source_path](api_error error) -> api_error { + event_system::instance().raise(api_path, source_path, + error); + return error; + }; + try { + return notify_end(upload_file_impl(api_path, source_path, stop_requested)); + } catch (const std::exception &e) { + utils::error::raise_error(__FUNCTION__, e, "exception occurred"); + } + + return notify_end(api_error::error); +} } // namespace repertory diff --git a/src/providers/s3/s3_provider.cpp b/src/providers/s3/s3_provider.cpp index 8c1acc2c..2ec62be5 100644 --- a/src/providers/s3/s3_provider.cpp +++ b/src/providers/s3/s3_provider.cpp @@ -727,34 +727,16 @@ auto s3_provider::read_file_bytes(const std::string &api_path, std::size_t size, return api_error::error; } -auto s3_provider::remove_directory(const std::string &api_path) -> api_error { - const auto notify_end = [&api_path](api_error error) -> api_error { - if (error == api_error::success) { - event_system::instance().raise(api_path); - } else { - event_system::instance().raise( - api_path, api_error_to_string(error)); - } - return error; - }; - - bool exists{}; - auto res = is_directory(api_path, exists); - if (res != api_error::success) { - return res; - } - if (not exists) { - return notify_end(api_error::item_not_found); - } - +auto s3_provider::remove_directory_impl(const std::string &api_path) + -> api_error { const auto cfg = get_config().get_s3_config(); const auto is_encrypted = not cfg.encryption_token.empty(); std::string key; if (is_encrypted) { - res = get_item_meta(api_path, META_KEY, key); + auto res = get_item_meta(api_path, META_KEY, key); if (res != api_error::success) { - return notify_end(res); + return res; } } @@ -772,7 +754,7 @@ auto s3_provider::remove_directory(const std::string &api_path) -> api_error { utils::error::raise_api_path_error(__FUNCTION__, api_path, api_error::comm_error, "failed to remove directory"); - return notify_end(api_error::comm_error); + return api_error::comm_error; } if ((response_code < http_error_codes::ok || @@ -780,76 +762,21 @@ auto s3_provider::remove_directory(const std::string &api_path) -> api_error { response_code != http_error_codes::not_found) { utils::error::raise_api_path_error(__FUNCTION__, api_path, response_code, "failed to remove directory"); - return notify_end(api_error::comm_error); + return api_error::comm_error; } - auto status = get_db()->Delete(rocksdb::WriteOptions(), api_path); - if (not status.ok()) { - utils::error::raise_api_path_error(__FUNCTION__, api_path, status.code(), - "failed to remove directory"); - return notify_end(api_error::error); - } - - return notify_end(api_error::success); + return api_error::success; } -auto s3_provider::remove_file(const std::string &api_path) -> api_error { - const auto notify_end = [&api_path](api_error error) -> api_error { - if (error == api_error::success) { - event_system::instance().raise(api_path); - } else { - event_system::instance().raise( - api_path, api_error_to_string(error)); - } - - return error; - }; - - const auto *const function_name = __FUNCTION__; - - const auto remove_file_meta = [this, &api_path, &function_name, - ¬ify_end]() -> api_error { - api_meta_map meta{}; - auto res = get_item_meta(api_path, meta); - - auto res2 = get_db()->Delete(rocksdb::WriteOptions(), api_path); - if (not res2.ok()) { - utils::error::raise_api_path_error(function_name, api_path, res2.code(), - "failed to remove file"); - return notify_end(api_error::error); - } - - return notify_end(res); - }; - - bool exists{}; - auto res = is_directory(api_path, exists); - if (res != api_error::success) { - return notify_end(res); - } - if (exists) { - exists = false; - return notify_end(api_error::directory_exists); - } - - res = is_file(api_path, exists); - if (res != api_error::success) { - return notify_end(res); - } - if (not exists) { - event_system::instance().raise( - api_path, api_error_to_string(api_error::item_not_found)); - return remove_file_meta(); - } - +auto s3_provider::remove_file_impl(const std::string &api_path) -> api_error { const auto cfg = get_config().get_s3_config(); const auto is_encrypted = not cfg.encryption_token.empty(); std::string key; if (is_encrypted) { - res = get_item_meta(api_path, META_KEY, key); + auto res = get_item_meta(api_path, META_KEY, key); if (res != api_error::success) { - return notify_end(res); + return res; } } @@ -866,7 +793,7 @@ auto s3_provider::remove_file(const std::string &api_path) -> api_error { if (not get_comm().make_request(del, response_code, stop_requested)) { utils::error::raise_api_path_error( __FUNCTION__, api_path, api_error::comm_error, "failed to remove file"); - return notify_end(api_error::comm_error); + return api_error::comm_error; } if ((response_code < http_error_codes::ok || @@ -874,10 +801,10 @@ auto s3_provider::remove_file(const std::string &api_path) -> api_error { response_code != http_error_codes::not_found) { utils::error::raise_api_path_error(__FUNCTION__, api_path, response_code, "failed to remove file"); - return notify_end(api_error::comm_error); + return api_error::comm_error; } - return remove_file_meta(); + return api_error::success; } auto s3_provider::rename_file(const std::string & /* from_api_path */, @@ -898,65 +825,51 @@ void s3_provider::stop() { event_system::instance().raise("s3_provider"); } -auto s3_provider::upload_file(const std::string &api_path, - const std::string &source_path, - stop_type &stop_requested) -> api_error { - event_system::instance().raise(api_path, source_path); - - const auto notify_end = [&api_path, - &source_path](api_error error) -> api_error { - event_system::instance().raise(api_path, source_path, - error); - return error; - }; - try { - std::uint64_t file_size{}; - if (utils::file::is_file(source_path) && - not utils::file::get_file_size(source_path, file_size)) { - return notify_end(api_error::comm_error); - } - - const auto cfg = get_config().get_s3_config(); - const auto is_encrypted = not cfg.encryption_token.empty(); - - std::string key; - if (is_encrypted) { - auto res = get_item_meta(api_path, META_KEY, key); - if (res != api_error::success) { - return notify_end(res); - } - } - - const auto object_name = - utils::path::create_api_path(is_encrypted ? key : api_path); - - curl::requests::http_put_file put_file{}; - put_file.aws_service = "aws:amz:" + cfg.region + ":s3"; - put_file.path = object_name; - put_file.source_path = source_path; - - if (is_encrypted && file_size > 0U) { - static stop_type no_stop{false}; - - put_file.reader = std::make_shared( - object_name, source_path, no_stop, cfg.encryption_token, -1); - } - - long response_code{}; - if (not get_comm().make_request(put_file, response_code, stop_requested)) { - return notify_end(api_error::comm_error); - } - - if (response_code != http_error_codes::ok) { - return notify_end(api_error::comm_error); - } - - return notify_end(api_error::success); - } catch (const std::exception &e) { - utils::error::raise_error(__FUNCTION__, e, "exception occurred"); +auto s3_provider::upload_file_impl(const std::string &api_path, + const std::string &source_path, + stop_type &stop_requested) -> api_error { + std::uint64_t file_size{}; + if (utils::file::is_file(source_path) && + not utils::file::get_file_size(source_path, file_size)) { + return api_error::comm_error; } - return notify_end(api_error::error); + const auto cfg = get_config().get_s3_config(); + const auto is_encrypted = not cfg.encryption_token.empty(); + + std::string key; + if (is_encrypted) { + auto res = get_item_meta(api_path, META_KEY, key); + if (res != api_error::success) { + return res; + } + } + + const auto object_name = + utils::path::create_api_path(is_encrypted ? key : api_path); + + curl::requests::http_put_file put_file{}; + put_file.aws_service = "aws:amz:" + cfg.region + ":s3"; + put_file.path = object_name; + put_file.source_path = source_path; + + if (is_encrypted && file_size > 0U) { + static stop_type no_stop{false}; + + put_file.reader = std::make_shared( + object_name, source_path, no_stop, cfg.encryption_token, -1); + } + + long response_code{}; + if (not get_comm().make_request(put_file, response_code, stop_requested)) { + return api_error::comm_error; + } + + if (response_code != http_error_codes::ok) { + return api_error::comm_error; + } + + return api_error::success; } } // namespace repertory diff --git a/src/providers/sia/sia_provider.cpp b/src/providers/sia/sia_provider.cpp index c4bd05cd..cd3630c8 100644 --- a/src/providers/sia/sia_provider.cpp +++ b/src/providers/sia/sia_provider.cpp @@ -43,6 +43,7 @@ auto sia_provider::create_directory_impl(const std::string &api_path, api_meta_map & /* meta */) -> api_error { curl::requests::http_put_file put_file{}; + put_file.allow_timeout = true; put_file.path = "/api/worker/objects" + api_path + "/"; long response_code{}; @@ -538,26 +539,8 @@ auto sia_provider::read_file_bytes(const std::string &api_path, return res; } -auto sia_provider::remove_directory(const std::string &api_path) -> api_error { - const auto notify_end = [&api_path](api_error error) -> api_error { - if (error == api_error::success) { - event_system::instance().raise(api_path); - } else { - event_system::instance().raise( - api_path, api_error_to_string(error)); - } - return error; - }; - - bool exists{}; - auto res = is_directory(api_path, exists); - if (res != api_error::success) { - return res; - } - if (not exists) { - return notify_end(api_error::item_not_found); - } - +auto sia_provider::remove_directory_impl(const std::string &api_path) + -> api_error { curl::requests::http_delete del{}; del.allow_timeout = true; del.path = "/api/bus/objects" + api_path + "/"; @@ -568,71 +551,19 @@ auto sia_provider::remove_directory(const std::string &api_path) -> api_error { utils::error::raise_api_path_error(__FUNCTION__, api_path, api_error::comm_error, "failed to remove directory"); - return notify_end(api_error::comm_error); + return api_error::comm_error; } if (response_code != http_error_codes::ok) { utils::error::raise_api_path_error(__FUNCTION__, api_path, response_code, "failed to remove directory"); - return notify_end(api_error::comm_error); + return api_error::comm_error; } - auto res2 = get_db()->Delete(rocksdb::WriteOptions(), api_path); - if (not res2.ok()) { - utils::error::raise_api_path_error(__FUNCTION__, api_path, res2.code(), - "failed to remove directory"); - return notify_end(api_error::error); - } - - return notify_end(api_error::success); + return api_error::success; } -auto sia_provider::remove_file(const std::string &api_path) -> api_error { - const auto notify_end = [&api_path](api_error error) -> api_error { - if (error == api_error::success) { - event_system::instance().raise(api_path); - } else { - event_system::instance().raise( - api_path, api_error_to_string(error)); - } - - return error; - }; - - const auto remove_file_meta = [this, &api_path, ¬ify_end]() -> api_error { - api_meta_map meta{}; - auto res = get_item_meta(api_path, meta); - - auto res2 = get_db()->Delete(rocksdb::WriteOptions(), api_path); - if (not res2.ok()) { - utils::error::raise_api_path_error(__FUNCTION__, api_path, res2.code(), - "failed to remove file"); - return notify_end(api_error::error); - } - - return notify_end(res); - }; - - bool exists{}; - auto res = is_directory(api_path, exists); - if (res != api_error::success) { - return notify_end(res); - } - if (exists) { - exists = false; - return notify_end(api_error::directory_exists); - } - - res = is_file(api_path, exists); - if (res != api_error::success) { - return notify_end(res); - } - if (not exists) { - event_system::instance().raise( - api_path, api_error_to_string(api_error::item_not_found)); - return remove_file_meta(); - } - +auto sia_provider::remove_file_impl(const std::string &api_path) -> api_error { curl::requests::http_delete del{}; del.allow_timeout = true; del.path = "/api/bus/objects" + api_path; @@ -642,17 +573,17 @@ auto sia_provider::remove_file(const std::string &api_path) -> api_error { if (not get_comm().make_request(del, response_code, stop_requested)) { utils::error::raise_api_path_error( __FUNCTION__, api_path, api_error::comm_error, "failed to remove file"); - return notify_end(api_error::comm_error); + return api_error::comm_error; } if (response_code != http_error_codes::ok && response_code != http_error_codes::not_found) { utils::error::raise_api_path_error(__FUNCTION__, api_path, response_code, "failed to remove file"); - return notify_end(api_error::comm_error); + return api_error::comm_error; } - return remove_file_meta(); + return api_error::success; } auto sia_provider::rename_file(const std::string & /*from_api_path*/, @@ -673,44 +604,27 @@ void sia_provider::stop() { event_system::instance().raise("sia_provider"); } -auto sia_provider::upload_file(const std::string &api_path, - const std::string &source_path, - stop_type &stop_requested) -> api_error { - event_system::instance().raise(api_path, source_path); +auto sia_provider::upload_file_impl(const std::string &api_path, + const std::string &source_path, + stop_type &stop_requested) -> api_error { + curl::requests::http_put_file put_file{}; + put_file.path = "/api/worker/objects" + api_path; + put_file.source_path = source_path; - const auto notify_end = [&api_path, - &source_path](api_error error) -> api_error { - event_system::instance().raise(api_path, source_path, - error); - return error; - }; - - try { - curl::requests::http_put_file put_file{}; - put_file.path = "/api/worker/objects" + api_path; - put_file.source_path = source_path; - - long response_code{}; - if (not get_comm().make_request(put_file, response_code, stop_requested)) { - utils::error::raise_api_path_error(__FUNCTION__, api_path, source_path, - api_error::comm_error, - "failed to upload file"); - return notify_end(api_error::comm_error); - } - - if (response_code != http_error_codes::ok) { - utils::error::raise_api_path_error(__FUNCTION__, api_path, source_path, - response_code, - "failed to upload file"); - return notify_end(api_error::comm_error); - } - - return notify_end(api_error::success); - } catch (const std::exception &e) { - utils::error::raise_api_path_error(__FUNCTION__, api_path, source_path, e, + long response_code{}; + if (not get_comm().make_request(put_file, response_code, stop_requested)) { + utils::error::raise_api_path_error(__FUNCTION__, api_path, source_path, + api_error::comm_error, "failed to upload file"); + return api_error::comm_error; } - return notify_end(api_error::error); + if (response_code != http_error_codes::ok) { + utils::error::raise_api_path_error(__FUNCTION__, api_path, source_path, + response_code, "failed to upload file"); + return api_error::comm_error; + } + + return api_error::success; } } // namespace repertory