From e4289295e86db3bd1b4c89e44c41723d74a6d5fd Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Thu, 19 Dec 2024 08:33:51 -0600 Subject: [PATCH] refactor --- .../providers/encrypt/encrypt_provider.hpp | 5 + .../include/providers/s3/s3_provider.hpp | 7 ++ .../include/providers/sia/sia_provider.hpp | 7 ++ .../src/providers/base_provider.cpp | 20 ++-- .../providers/encrypt/encrypt_provider.cpp | 26 +++-- .../src/providers/s3/s3_provider.cpp | 94 ++++++++++--------- .../src/providers/sia/sia_provider.cpp | 23 ++--- 7 files changed, 105 insertions(+), 77 deletions(-) diff --git a/repertory/librepertory/include/providers/encrypt/encrypt_provider.hpp b/repertory/librepertory/include/providers/encrypt/encrypt_provider.hpp index 6588b35b..da68ba8f 100644 --- a/repertory/librepertory/include/providers/encrypt/encrypt_provider.hpp +++ b/repertory/librepertory/include/providers/encrypt/encrypt_provider.hpp @@ -54,6 +54,7 @@ private: private: app_config &config_; + encrypt_config encrypt_config_; private: std::unique_ptr db_{nullptr}; @@ -73,6 +74,10 @@ private: const std::string &source_path)> callback) const -> api_error; + [[nodiscard]] auto get_encrypt_config() const -> const encrypt_config & { + return encrypt_config_; + } + auto process_directory_entry(const utils::file::i_fs_item &dir_entry, const encrypt_config &cfg, std::string &api_path) const -> bool; diff --git a/repertory/librepertory/include/providers/s3/s3_provider.hpp b/repertory/librepertory/include/providers/s3/s3_provider.hpp index 03bf805e..343a3244 100644 --- a/repertory/librepertory/include/providers/s3/s3_provider.hpp +++ b/repertory/librepertory/include/providers/s3/s3_provider.hpp @@ -46,6 +46,9 @@ public: auto operator=(const s3_provider &) -> s3_provider & = delete; auto operator=(s3_provider &&) -> s3_provider & = delete; +private: + s3_config s3_config_; + private: [[nodiscard]] auto add_if_not_found(api_file &file, const std::string &object_name) const @@ -78,6 +81,10 @@ private: std::optional token = std::nullopt) const -> bool; + [[nodiscard]] auto get_s3_config() const -> const s3_config & { + return s3_config_; + } + protected: [[nodiscard]] auto create_directory_impl(const std::string &api_path, api_meta_map &meta) diff --git a/repertory/librepertory/include/providers/sia/sia_provider.hpp b/repertory/librepertory/include/providers/sia/sia_provider.hpp index 4a05d30a..ce411312 100644 --- a/repertory/librepertory/include/providers/sia/sia_provider.hpp +++ b/repertory/librepertory/include/providers/sia/sia_provider.hpp @@ -45,6 +45,9 @@ public: auto operator=(const sia_provider &) -> sia_provider & = delete; auto operator=(sia_provider &&) -> sia_provider & = delete; +private: + sia_config sia_config_; + private: [[nodiscard]] auto get_object_info(const std::string &api_path, json &object_info) const -> api_error; @@ -52,6 +55,10 @@ private: [[nodiscard]] auto get_object_list(const std::string &api_path, nlohmann::json &object_list) const -> bool; + [[nodiscard]] auto get_sia_config() const -> const auto & { + return sia_config_; + } + protected: [[nodiscard]] auto create_directory_impl(const std::string &api_path, api_meta_map &meta) diff --git a/repertory/librepertory/src/providers/base_provider.cpp b/repertory/librepertory/src/providers/base_provider.cpp index 9abd8cf8..5f489cdf 100644 --- a/repertory/librepertory/src/providers/base_provider.cpp +++ b/repertory/librepertory/src/providers/base_provider.cpp @@ -456,7 +456,7 @@ void base_provider::process_removed_files(std::deque removed_list, REPERTORY_USES_FUNCTION_NAME(); auto orphaned_directory = - utils::path::combine(config_.get_data_directory(), {"orphaned"}); + utils::path::combine(get_config().get_data_directory(), {"orphaned"}); for (const auto &item : removed_list) { if (stop_requested) { return; @@ -670,8 +670,10 @@ void base_provider::remove_unmatched_source_files( return; } + const auto &cfg = get_config(); + auto source_list = - utils::file::directory{config_.get_cache_directory()}.get_files(); + utils::file::directory{cfg.get_cache_directory()}.get_files(); for (const auto &source_file : source_list) { if (stop_requested) { return; @@ -684,15 +686,15 @@ void base_provider::remove_unmatched_source_files( } auto reference_time = - source_file->get_time(config_.get_eviction_uses_accessed_time() + source_file->get_time(cfg.get_eviction_uses_accessed_time() ? utils::file::time_type::accessed : utils::file::time_type::modified); if (not reference_time.has_value()) { continue; } - auto delay = (config_.get_eviction_delay_mins() * 60UL) * - utils::time::NANOS_PER_SECOND; + auto delay = + (cfg.get_eviction_delay_mins() * 60UL) * utils::time::NANOS_PER_SECOND; if ((reference_time.value() + static_cast(delay)) >= utils::time::get_time_now()) { continue; @@ -736,17 +738,19 @@ auto base_provider::start(api_item_added_callback api_item_added, auto online{false}; auto unmount_requested{false}; { + const auto &cfg = get_config(); + repertory::event_consumer consumer( "unmount_requested", [&unmount_requested](const event &) { unmount_requested = true; }); for (std::uint16_t idx = 0U; not online && not unmount_requested && - (idx < config_.get_online_check_retry_secs()); + (idx < cfg.get_online_check_retry_secs()); ++idx) { online = is_online(); if (not online) { event_system::instance().raise( - config_.get_host_config().host_name_or_ip, - config_.get_host_config().api_port); + cfg.get_host_config().host_name_or_ip, + cfg.get_host_config().api_port); std::this_thread::sleep_for(1s); } } diff --git a/repertory/librepertory/src/providers/encrypt/encrypt_provider.cpp b/repertory/librepertory/src/providers/encrypt/encrypt_provider.cpp index 005f721d..ab8383fd 100644 --- a/repertory/librepertory/src/providers/encrypt/encrypt_provider.cpp +++ b/repertory/librepertory/src/providers/encrypt/encrypt_provider.cpp @@ -38,7 +38,8 @@ const std::string file_table = "file"; } // namespace namespace repertory { -encrypt_provider::encrypt_provider(app_config &config) : config_(config) {} +encrypt_provider::encrypt_provider(app_config &config) + : config_(config), encrypt_config_(config.get_encrypt_config()) {} auto encrypt_provider::create_api_file(const std::string &api_path, bool directory, @@ -115,7 +116,8 @@ auto encrypt_provider::do_fs_operation( std::function callback) const -> api_error { - auto cfg = config_.get_encrypt_config(); + const auto &cfg = get_encrypt_config(); + std::string source_path{api_path}; if (api_path != "/" && not utils::encryption::decrypt_file_path( cfg.encryption_token, source_path)) { @@ -327,9 +329,9 @@ auto encrypt_provider::get_file_list(api_file_list &list, -> api_error { REPERTORY_USES_FUNCTION_NAME(); - auto cfg = config_.get_encrypt_config(); - try { + const auto &cfg = get_encrypt_config(); + for (const auto &dir_entry : utils::file::directory{cfg.path}.get_items()) { std::string api_path{}; if (process_directory_entry(*dir_entry.get(), cfg, api_path)) { @@ -501,9 +503,8 @@ auto encrypt_provider::get_item_meta(const std::string &api_path, } auto encrypt_provider::get_total_drive_space() const -> std::uint64_t { - auto total_space = - utils::file::get_total_drive_space(config_.get_encrypt_config().path); - return total_space.value_or(0U); + return utils::file::get_total_drive_space(get_encrypt_config().path) + .value_or(0U); } auto encrypt_provider::get_total_item_count() const -> std::uint64_t { @@ -521,7 +522,7 @@ auto encrypt_provider::get_total_item_count() const -> std::uint64_t { auto encrypt_provider::get_used_drive_space() const -> std::uint64_t { auto free_space = - utils::file::get_free_drive_space(config_.get_encrypt_config().path); + utils::file::get_free_drive_space(get_encrypt_config().path); return free_space.has_value() ? get_total_drive_space() - free_space.value() : 0U; } @@ -585,10 +586,7 @@ auto encrypt_provider::is_file_writeable(const std::string & /*api_path*/) const } auto encrypt_provider::is_online() const -> bool { - return utils::file::directory{ - config_.get_encrypt_config().path, - } - .exists(); + return utils::file::directory{get_encrypt_config().path}.exists(); } auto encrypt_provider::process_directory_entry( @@ -744,7 +742,7 @@ auto encrypt_provider::read_file_bytes(const std::string &api_path, auto file_size{opt_size.value()}; - auto cfg = config_.get_encrypt_config(); + const auto &cfg = get_encrypt_config(); unique_recur_mutex_lock reader_lookup_lock(reader_lookup_mtx_); @@ -848,7 +846,7 @@ auto encrypt_provider::start(api_item_added_callback /*api_item_added*/, fmt::format("failed to get root|{}", api_error_to_string(result))); } - auto cfg_path = utils::path::absolute(config_.get_encrypt_config().path); + auto cfg_path = utils::path::absolute(get_encrypt_config().path); if (result == api_error::success) { auto cur_path = utils::path::absolute(source_path); #if defined(_WIN32) diff --git a/repertory/librepertory/src/providers/s3/s3_provider.cpp b/repertory/librepertory/src/providers/s3/s3_provider.cpp index 90c796a0..ed24813c 100644 --- a/repertory/librepertory/src/providers/s3/s3_provider.cpp +++ b/repertory/librepertory/src/providers/s3/s3_provider.cpp @@ -39,12 +39,11 @@ namespace repertory { s3_provider::s3_provider(app_config &config, i_http_comm &comm) - : base_provider(config, comm) { - get_comm().enable_s3_path_style(config.get_s3_config().use_path_style); -} + : base_provider(config, comm) {} -auto s3_provider::add_if_not_found( - api_file &file, const std::string &object_name) const -> api_error { +auto s3_provider::add_if_not_found(api_file &file, + const std::string &object_name) const + -> api_error { api_meta_map meta{}; if (get_item_meta(file.api_path, meta) == api_error::item_not_found) { auto err = create_path_directories( @@ -70,7 +69,7 @@ auto s3_provider::convert_api_date(std::string_view date) -> std::uint64_t { utils::string::split(date_parts.at(1U), 'Z', true).at(0U)) * 1000000UL; - struct tm tm1 {}; + struct tm tm1{}; #if defined(_WIN32) utils::time::strptime(date_time.c_str(), "%Y-%m-%dT%T", &tm1); return nanos + utils::time::windows_time_t_to_unix_time(_mkgmtime(&tm1)); @@ -85,7 +84,7 @@ auto s3_provider::create_directory_impl(const std::string &api_path, api_meta_map &meta) -> api_error { REPERTORY_USES_FUNCTION_NAME(); - auto cfg = get_config().get_s3_config(); + const auto &cfg = get_s3_config(); auto is_encrypted = not cfg.encryption_token.empty(); stop_type stop_requested{false}; @@ -138,7 +137,8 @@ auto s3_provider::create_file_extra(const std::string &api_path, api_meta_map &meta) -> api_error { REPERTORY_USES_FUNCTION_NAME(); - if (not get_config().get_s3_config().encryption_token.empty()) { + const auto &cfg = get_s3_config(); + if (not cfg.encryption_token.empty()) { std::string encrypted_file_path; auto res = get_item_meta(utils::path::get_parent_api_path(api_path), META_KEY, encrypted_file_path); @@ -150,7 +150,7 @@ auto s3_provider::create_file_extra(const std::string &api_path, data_buffer result; utils::encryption::encrypt_data( - get_config().get_s3_config().encryption_token, + cfg.encryption_token, *(utils::string::split(api_path, '/', false).end() - 1U), result); meta[META_KEY] = utils::path::create_api_path( @@ -161,15 +161,17 @@ auto s3_provider::create_file_extra(const std::string &api_path, return api_error::success; } -auto s3_provider::create_path_directories( - const std::string &api_path, const std::string &key) const -> api_error { +auto s3_provider::create_path_directories(const std::string &api_path, + const std::string &key) const + -> api_error { REPERTORY_USES_FUNCTION_NAME(); if (api_path == "/") { return api_error::success; } - auto encryption_token = get_config().get_s3_config().encryption_token; + const auto &cfg = get_s3_config(); + auto encryption_token = cfg.encryption_token; auto is_encrypted = not encryption_token.empty(); auto path_parts = utils::string::split(api_path, '/', false); @@ -179,8 +181,6 @@ auto s3_provider::create_path_directories( return api_error::error; } - auto cfg = get_config().get_s3_config(); - std::string cur_key{'/'}; std::string cur_path{'/'}; for (std::size_t idx = 0U; idx < path_parts.size(); ++idx) { @@ -244,7 +244,7 @@ auto s3_provider::decrypt_object_name(std::string &object_name) const auto parts = utils::string::split(object_name, '/', false); for (auto &&part : parts) { if (not utils::encryption::decrypt_file_name( - get_config().get_s3_config().encryption_token, part)) { + get_s3_config().encryption_token, part)) { return api_error::decryption_error; } } @@ -258,7 +258,7 @@ auto s3_provider::get_directory_item_count(const std::string &api_path) const REPERTORY_USES_FUNCTION_NAME(); try { - auto cfg = get_config().get_s3_config(); + const auto &cfg = get_s3_config(); auto is_encrypted = not cfg.encryption_token.empty(); std::string key; if (is_encrypted) { @@ -330,11 +330,12 @@ auto s3_provider::get_directory_item_count(const std::string &api_path) const return 0U; } -auto s3_provider::get_directory_items_impl( - const std::string &api_path, directory_item_list &list) const -> api_error { +auto s3_provider::get_directory_items_impl(const std::string &api_path, + directory_item_list &list) const + -> api_error { REPERTORY_USES_FUNCTION_NAME(); - auto cfg = get_config().get_s3_config(); + const auto &cfg = get_s3_config(); auto is_encrypted = not cfg.encryption_token.empty(); auto ret = api_error::success; @@ -476,8 +477,8 @@ auto s3_provider::get_directory_items_impl( return ret; } -auto s3_provider::get_file(const std::string &api_path, - api_file &file) const -> api_error { +auto s3_provider::get_file(const std::string &api_path, api_file &file) const + -> api_error { REPERTORY_USES_FUNCTION_NAME(); try { @@ -516,8 +517,8 @@ auto s3_provider::get_file(const std::string &api_path, return api_error::error; } -auto s3_provider::get_file_list(api_file_list &list, - std::string &marker) const -> api_error { +auto s3_provider::get_file_list(api_file_list &list, std::string &marker) const + -> api_error { REPERTORY_USES_FUNCTION_NAME(); std::string response_data; @@ -559,8 +560,7 @@ auto s3_provider::get_file_list(api_file_list &list, continue; } - auto is_encrypted = - not get_config().get_s3_config().encryption_token.empty(); + auto is_encrypted = not get_s3_config().encryption_token.empty(); if (is_encrypted) { auto err = decrypt_object_name(api_path); if (err != api_error::success) { @@ -593,8 +593,9 @@ auto s3_provider::get_file_list(api_file_list &list, return grab_more ? api_error::more_data : api_error::success; } -auto s3_provider::get_last_modified( - bool directory, const std::string &api_path) const -> std::uint64_t { +auto s3_provider::get_last_modified(bool directory, + const std::string &api_path) const + -> std::uint64_t { bool is_encrypted{}; std::string object_name; head_object_result result{}; @@ -604,13 +605,14 @@ auto s3_provider::get_last_modified( : utils::time::get_time_now(); } -auto s3_provider::get_object_info( - bool directory, const std::string &api_path, bool &is_encrypted, - std::string &object_name, head_object_result &result) const -> api_error { +auto s3_provider::get_object_info(bool directory, const std::string &api_path, + bool &is_encrypted, std::string &object_name, + head_object_result &result) const + -> api_error { REPERTORY_USES_FUNCTION_NAME(); try { - auto cfg = get_config().get_s3_config(); + const auto &cfg = get_s3_config(); is_encrypted = not cfg.encryption_token.empty(); std::string key; @@ -655,13 +657,15 @@ auto s3_provider::get_object_info( return api_error::error; } -auto s3_provider::get_object_list( - std::string &response_data, long &response_code, - std::optional delimiter, std::optional prefix, - std::optional token) const -> bool { +auto s3_provider::get_object_list(std::string &response_data, + long &response_code, + std::optional delimiter, + std::optional prefix, + std::optional token) const + -> bool { curl::requests::http_get get{}; get.allow_timeout = true; - get.aws_service = "aws:amz:" + get_config().get_s3_config().region + ":s3"; + get.aws_service = "aws:amz:" + get_s3_config().region + ":s3"; get.path = '/'; get.query["list-type"] = "2"; if (delimiter.has_value() && not delimiter.value().empty()) { @@ -687,8 +691,8 @@ auto s3_provider::get_total_drive_space() const -> std::uint64_t { return std::numeric_limits::max() / std::int64_t(2); } -auto s3_provider::is_directory(const std::string &api_path, - bool &exists) const -> api_error { +auto s3_provider::is_directory(const std::string &api_path, bool &exists) const + -> api_error { REPERTORY_USES_FUNCTION_NAME(); exists = false; @@ -715,8 +719,8 @@ auto s3_provider::is_directory(const std::string &api_path, return api_error::error; } -auto s3_provider::is_file(const std::string &api_path, - bool &exists) const -> api_error { +auto s3_provider::is_file(const std::string &api_path, bool &exists) const + -> api_error { REPERTORY_USES_FUNCTION_NAME(); exists = false; @@ -753,7 +757,7 @@ auto s3_provider::read_file_bytes(const std::string &api_path, std::size_t size, REPERTORY_USES_FUNCTION_NAME(); try { - auto cfg = get_config().get_s3_config(); + const auto &cfg = get_s3_config(); auto is_encrypted = not cfg.encryption_token.empty(); std::string key; if (is_encrypted) { @@ -859,7 +863,7 @@ auto s3_provider::remove_directory_impl(const std::string &api_path) -> api_error { REPERTORY_USES_FUNCTION_NAME(); - auto cfg = get_config().get_s3_config(); + const auto &cfg = get_s3_config(); auto is_encrypted = not cfg.encryption_token.empty(); std::string key; @@ -901,7 +905,7 @@ auto s3_provider::remove_directory_impl(const std::string &api_path) auto s3_provider::remove_file_impl(const std::string &api_path) -> api_error { REPERTORY_USES_FUNCTION_NAME(); - auto cfg = get_config().get_s3_config(); + const auto &cfg = get_s3_config(); auto is_encrypted = not cfg.encryption_token.empty(); std::string key; @@ -949,6 +953,8 @@ auto s3_provider::rename_file(const std::string & /* from_api_path */, auto s3_provider::start(api_item_added_callback api_item_added, i_file_manager *mgr) -> bool { event_system::instance().raise("s3_provider"); + s3_config_ = get_config().get_s3_config(); + get_comm().enable_s3_path_style(s3_config_.use_path_style); return base_provider::start(api_item_added, mgr); } @@ -972,7 +978,7 @@ auto s3_provider::upload_file_impl(const std::string &api_path, file_size = opt_size.value(); } - auto cfg = get_config().get_s3_config(); + const auto &cfg = get_s3_config(); auto is_encrypted = not cfg.encryption_token.empty(); std::string key; diff --git a/repertory/librepertory/src/providers/sia/sia_provider.cpp b/repertory/librepertory/src/providers/sia/sia_provider.cpp index 17704e64..33d454ab 100644 --- a/repertory/librepertory/src/providers/sia/sia_provider.cpp +++ b/repertory/librepertory/src/providers/sia/sia_provider.cpp @@ -37,7 +37,7 @@ #include "utils/utils.hpp" namespace { -[[nodiscard]] auto get_bucket(repertory::sia_config cfg) -> std::string { +[[nodiscard]] auto get_bucket(const repertory::sia_config &cfg) -> std::string { repertory::utils::string::trim(cfg.bucket); if (cfg.bucket.empty()) { return "default"; @@ -68,7 +68,7 @@ auto sia_provider::create_directory_impl(const std::string &api_path, curl::requests::http_put_file put_file{}; put_file.allow_timeout = true; put_file.path = "/api/worker/objects" + api_path + "/"; - put_file.query["bucket"] = get_bucket(get_config().get_sia_config()); + put_file.query["bucket"] = get_bucket(get_sia_config()); long response_code{}; stop_type stop_requested{}; @@ -289,7 +289,7 @@ auto sia_provider::get_object_info(const std::string &api_path, curl::requests::http_get get{}; get.allow_timeout = true; get.path = "/api/bus/objects" + api_path; - get.query["bucket"] = get_bucket(get_config().get_sia_config()); + get.query["bucket"] = get_bucket(get_sia_config()); get.response_handler = [&object_info](const data_buffer &data, long response_code) { @@ -330,7 +330,7 @@ auto sia_provider::get_object_list(const std::string &api_path, curl::requests::http_get get{}; get.allow_timeout = true; get.path = "/api/bus/objects" + api_path + "/"; - get.query["bucket"] = get_bucket(get_config().get_sia_config()); + get.query["bucket"] = get_bucket(get_sia_config()); get.response_handler = [&object_list](const data_buffer &data, long response_code) { @@ -364,7 +364,7 @@ auto sia_provider::get_total_drive_space() const -> std::uint64_t { curl::requests::http_get get{}; get.allow_timeout = true; get.path = "/api/autopilot/config"; - get.query["bucket"] = get_bucket(get_config().get_sia_config()); + get.query["bucket"] = get_bucket(get_sia_config()); json config_data{}; get.response_handler = [&config_data](const data_buffer &data, @@ -465,7 +465,7 @@ auto sia_provider::is_online() const -> bool { curl::requests::http_get get{}; get.allow_timeout = true; get.path = "/api/bus/consensus/state"; - get.query["bucket"] = get_bucket(get_config().get_sia_config()); + get.query["bucket"] = get_bucket(get_sia_config()); json state_data{}; get.response_handler = [&state_data](const data_buffer &data, @@ -506,7 +506,7 @@ auto sia_provider::read_file_bytes(const std::string &api_path, curl::requests::http_get get{}; get.path = "/api/worker/objects" + api_path; - get.query["bucket"] = get_bucket(get_config().get_sia_config()); + get.query["bucket"] = get_bucket(get_sia_config()); get.range = {{ offset, offset + size - 1U, @@ -561,7 +561,7 @@ auto sia_provider::remove_directory_impl(const std::string &api_path) curl::requests::http_delete del{}; del.allow_timeout = true; del.path = "/api/bus/objects" + api_path + "/"; - del.query["bucket"] = get_bucket(get_config().get_sia_config()); + del.query["bucket"] = get_bucket(get_sia_config()); long response_code{}; stop_type stop_requested{}; @@ -587,7 +587,7 @@ auto sia_provider::remove_file_impl(const std::string &api_path) -> api_error { curl::requests::http_delete del{}; del.allow_timeout = true; del.path = "/api/bus/objects" + api_path; - del.query["bucket"] = get_bucket(get_config().get_sia_config()); + del.query["bucket"] = get_bucket(get_sia_config()); long response_code{}; stop_type stop_requested{}; @@ -619,7 +619,7 @@ auto sia_provider::rename_file(const std::string &from_api_path, {"mode", "single"}, }); post.path = "/api/bus/objects/rename"; - post.query["bucket"] = get_bucket(get_config().get_sia_config()); + post.query["bucket"] = get_bucket(get_sia_config()); long response_code{}; stop_type stop_requested{}; @@ -644,6 +644,7 @@ auto sia_provider::rename_file(const std::string &from_api_path, auto sia_provider::start(api_item_added_callback api_item_added, i_file_manager *mgr) -> bool { event_system::instance().raise("sia_provider"); + sia_config_ = get_config().get_sia_config(); return base_provider::start(api_item_added, mgr); } @@ -660,7 +661,7 @@ auto sia_provider::upload_file_impl(const std::string &api_path, curl::requests::http_put_file put_file{}; put_file.path = "/api/worker/objects" + api_path; - put_file.query["bucket"] = get_bucket(get_config().get_sia_config()); + put_file.query["bucket"] = get_bucket(get_sia_config()); put_file.source_path = source_path; long response_code{};