From ea8eb53a342ee68023131ceceeef9772903df0c0 Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Mon, 2 Dec 2024 18:26:14 -0600 Subject: [PATCH] fixes --- .../include/providers/base_provider.hpp | 141 ++++++++---------- .../src/providers/base_provider.cpp | 46 +++--- .../src/providers/s3/s3_provider.cpp | 65 ++++---- 3 files changed, 113 insertions(+), 139 deletions(-) diff --git a/repertory/librepertory/include/providers/base_provider.hpp b/repertory/librepertory/include/providers/base_provider.hpp index 1129c4d2..4f25734c 100644 --- a/repertory/librepertory/include/providers/base_provider.hpp +++ b/repertory/librepertory/include/providers/base_provider.hpp @@ -47,21 +47,20 @@ private: std::atomic stop_requested_{false}; private: - void remove_deleted_files(bool source_only); + void remove_deleted_files(); protected: - [[nodiscard]] static auto create_api_file(std::string path, std::string key, - std::uint64_t size, - std::uint64_t file_time) - -> api_file; + [[nodiscard]] static auto + create_api_file(std::string path, std::string key, std::uint64_t size, + std::uint64_t file_time) -> api_file; [[nodiscard]] static auto create_api_file(std::string path, std::uint64_t size, api_meta_map &meta) -> api_file; - [[nodiscard]] virtual auto create_directory_impl(const std::string &api_path, - api_meta_map &meta) - -> api_error = 0; + [[nodiscard]] virtual auto + create_directory_impl(const std::string &api_path, + api_meta_map &meta) -> api_error = 0; [[nodiscard]] virtual auto create_file_extra(const std::string & /* api_path */, @@ -73,8 +72,8 @@ protected: return api_item_added_; } - [[nodiscard]] auto get_api_item_added() const - -> const api_item_added_callback & { + [[nodiscard]] auto + get_api_item_added() const -> const api_item_added_callback & { return api_item_added_; } @@ -100,22 +99,20 @@ protected: return fm_; } - [[nodiscard]] virtual auto remove_directory_impl(const std::string &api_path) - -> api_error = 0; + [[nodiscard]] virtual auto + remove_directory_impl(const std::string &api_path) -> api_error = 0; - [[nodiscard]] virtual auto remove_file_impl(const std::string &api_path) - -> api_error = 0; + [[nodiscard]] virtual auto + remove_file_impl(const std::string &api_path) -> api_error = 0; - [[nodiscard]] virtual auto upload_file_impl(const std::string &api_path, - const std::string &source_path, - stop_type &stop_requested) - -> api_error = 0; + [[nodiscard]] virtual auto + upload_file_impl(const std::string &api_path, const std::string &source_path, + stop_type &stop_requested) -> api_error = 0; public: - [[nodiscard]] auto - create_directory_clone_source_meta(const std::string &source_api_path, - const std::string &api_path) - -> api_error override; + [[nodiscard]] auto create_directory_clone_source_meta( + const std::string &source_api_path, + const std::string &api_path) -> api_error override; [[nodiscard]] auto create_directory(const std::string &api_path, api_meta_map &meta) -> api_error override; @@ -123,82 +120,76 @@ public: [[nodiscard]] auto create_file(const std::string &api_path, api_meta_map &meta) -> api_error override; - [[nodiscard]] auto get_api_path_from_source(const std::string &source_path, - std::string &api_path) const - -> api_error override; - - [[nodiscard]] auto get_directory_items(const std::string &api_path, - directory_item_list &list) const - -> api_error override; - - [[nodiscard]] auto get_file_size(const std::string &api_path, - std::uint64_t &file_size) const - -> api_error override; - - [[nodiscard]] auto get_filesystem_item(const std::string &api_path, - bool directory, - filesystem_item &fsi) const - -> api_error override; - - [[nodiscard]] auto get_filesystem_item_and_file(const std::string &api_path, - api_file &f, - filesystem_item &fsi) const - -> api_error override; + [[nodiscard]] auto + get_api_path_from_source(const std::string &source_path, + std::string &api_path) const -> api_error override; [[nodiscard]] auto - get_filesystem_item_from_source_path(const std::string &source_path, - filesystem_item &fsi) const - -> api_error override; + get_directory_items(const std::string &api_path, + directory_item_list &list) const -> api_error override; - [[nodiscard]] auto get_item_meta(const std::string &api_path, - api_meta_map &meta) const - -> api_error override; + [[nodiscard]] auto + get_file_size(const std::string &api_path, + std::uint64_t &file_size) const -> api_error override; - [[nodiscard]] auto get_item_meta(const std::string &api_path, - const std::string &key, - std::string &value) const - -> api_error override; + [[nodiscard]] auto + get_filesystem_item(const std::string &api_path, bool directory, + filesystem_item &fsi) const -> api_error override; - [[nodiscard]] auto get_pinned_files() const - -> std::vector override; + [[nodiscard]] auto get_filesystem_item_and_file( + const std::string &api_path, api_file &f, + filesystem_item &fsi) const -> api_error override; + + [[nodiscard]] auto get_filesystem_item_from_source_path( + const std::string &source_path, + filesystem_item &fsi) const -> api_error override; + + [[nodiscard]] auto + get_item_meta(const std::string &api_path, + api_meta_map &meta) const -> api_error override; + + [[nodiscard]] auto + get_item_meta(const std::string &api_path, const std::string &key, + std::string &value) const -> api_error override; + + [[nodiscard]] auto + get_pinned_files() const -> std::vector override; [[nodiscard]] auto get_total_item_count() const -> std::uint64_t override; [[nodiscard]] auto get_used_drive_space() const -> std::uint64_t override; - [[nodiscard]] auto is_file_writeable(const std::string &api_path) const - -> bool override; + [[nodiscard]] auto + is_file_writeable(const std::string &api_path) const -> bool override; [[nodiscard]] auto is_read_only() const -> bool override { return false; } - [[nodiscard]] auto remove_directory(const std::string &api_path) - -> api_error override; + [[nodiscard]] auto + remove_directory(const std::string &api_path) -> api_error override; - [[nodiscard]] auto remove_file(const std::string &api_path) - -> api_error override; + [[nodiscard]] auto + remove_file(const std::string &api_path) -> api_error override; - [[nodiscard]] auto remove_item_meta(const std::string &api_path, - const std::string &key) - -> api_error override; + [[nodiscard]] auto + remove_item_meta(const std::string &api_path, + const std::string &key) -> api_error override; - [[nodiscard]] auto set_item_meta(const std::string &api_path, - const std::string &key, - const std::string &value) - -> api_error override; + [[nodiscard]] auto + set_item_meta(const std::string &api_path, const std::string &key, + const std::string &value) -> api_error override; - [[nodiscard]] auto set_item_meta(const std::string &api_path, - const api_meta_map &meta) - -> api_error override; + [[nodiscard]] auto + set_item_meta(const std::string &api_path, + const api_meta_map &meta) -> api_error override; [[nodiscard]] auto start(api_item_added_callback api_item_added, i_file_manager *mgr) -> bool override; void stop() override; - [[nodiscard]] auto upload_file(const std::string &api_path, - const std::string &source_path, - stop_type &stop_requested) - -> api_error override; + [[nodiscard]] auto + upload_file(const std::string &api_path, const std::string &source_path, + stop_type &stop_requested) -> api_error override; }; } // namespace repertory diff --git a/repertory/librepertory/src/providers/base_provider.cpp b/repertory/librepertory/src/providers/base_provider.cpp index 3bae8761..f28e29e7 100644 --- a/repertory/librepertory/src/providers/base_provider.cpp +++ b/repertory/librepertory/src/providers/base_provider.cpp @@ -33,8 +33,8 @@ namespace repertory { auto base_provider::create_api_file(std::string path, std::string key, - std::uint64_t size, std::uint64_t file_time) - -> api_file { + std::uint64_t size, + std::uint64_t file_time) -> api_file { api_file file{}; file.api_path = utils::path::create_api_path(path); file.api_parent = utils::path::get_parent_api_path(file.api_path); @@ -66,8 +66,8 @@ auto base_provider::create_api_file(std::string path, std::uint64_t size, } auto base_provider::create_directory_clone_source_meta( - const std::string &source_api_path, const std::string &api_path) - -> api_error { + const std::string &source_api_path, + const std::string &api_path) -> api_error { REPERTORY_USES_FUNCTION_NAME(); bool exists{}; @@ -164,8 +164,8 @@ auto base_provider::create_directory(const std::string &api_path, return set_item_meta(api_path, meta); } -auto base_provider::create_file(const std::string &api_path, api_meta_map &meta) - -> api_error { +auto base_provider::create_file(const std::string &api_path, + api_meta_map &meta) -> api_error { REPERTORY_USES_FUNCTION_NAME(); bool exists{}; @@ -222,9 +222,8 @@ auto base_provider::create_file(const std::string &api_path, api_meta_map &meta) return api_error::error; } -auto base_provider::get_api_path_from_source(const std::string &source_path, - std::string &api_path) const - -> api_error { +auto base_provider::get_api_path_from_source( + const std::string &source_path, std::string &api_path) const -> api_error { REPERTORY_USES_FUNCTION_NAME(); if (source_path.empty()) { @@ -237,9 +236,8 @@ auto base_provider::get_api_path_from_source(const std::string &source_path, return db3_->get_api_path(source_path, api_path); } -auto base_provider::get_directory_items(const std::string &api_path, - directory_item_list &list) const - -> api_error { +auto base_provider::get_directory_items( + const std::string &api_path, directory_item_list &list) const -> api_error { REPERTORY_USES_FUNCTION_NAME(); bool exists{}; @@ -303,10 +301,9 @@ auto base_provider::get_file_size(const std::string &api_path, return api_error::success; } -auto base_provider::get_filesystem_item(const std::string &api_path, - bool directory, - filesystem_item &fsi) const - -> api_error { +auto base_provider::get_filesystem_item( + const std::string &api_path, bool directory, + filesystem_item &fsi) const -> api_error { bool exists{}; auto res = is_directory(api_path, exists); if (res != api_error::success) { @@ -339,10 +336,9 @@ auto base_provider::get_filesystem_item(const std::string &api_path, return api_error::success; } -auto base_provider::get_filesystem_item_and_file(const std::string &api_path, - api_file &file, - filesystem_item &fsi) const - -> api_error { +auto base_provider::get_filesystem_item_and_file( + const std::string &api_path, api_file &file, + filesystem_item &fsi) const -> api_error { auto res = get_file(api_path, file); if (res != api_error::success) { return res; @@ -417,7 +413,7 @@ auto base_provider::is_file_writeable(const std::string &api_path) const return not exists; } -void base_provider::remove_deleted_files(bool source_only) { +void base_provider::remove_deleted_files() { REPERTORY_USES_FUNCTION_NAME(); if (not is_read_only()) { @@ -460,10 +456,6 @@ void base_provider::remove_deleted_files(bool source_only) { } } - if (source_only) { - return; - } - struct removed_item final { std::string api_path; bool directory{}; @@ -722,10 +714,8 @@ auto base_provider::start(api_item_added_callback api_item_added, return false; } - remove_deleted_files(true); - polling::instance().set_callback({"check_deleted", polling::frequency::low, - [this]() { remove_deleted_files(false); }}); + [this]() { remove_deleted_files(); }}); return true; } diff --git a/repertory/librepertory/src/providers/s3/s3_provider.cpp b/repertory/librepertory/src/providers/s3/s3_provider.cpp index 1ddb40ec..ab81c22a 100644 --- a/repertory/librepertory/src/providers/s3/s3_provider.cpp +++ b/repertory/librepertory/src/providers/s3/s3_provider.cpp @@ -43,9 +43,8 @@ s3_provider::s3_provider(app_config &config, i_http_comm &comm) get_comm().enable_s3_path_style(config.get_s3_config().use_path_style); } -auto s3_provider::add_if_not_found(api_file &file, - const std::string &object_name) const - -> api_error { +auto s3_provider::add_if_not_found( + api_file &file, const std::string &object_name) const -> api_error { api_meta_map meta{}; if (get_item_meta(file.api_path, meta) == api_error::item_not_found) { auto err = create_path_directories( @@ -71,7 +70,7 @@ auto s3_provider::convert_api_date(std::string_view date) -> std::uint64_t { utils::string::split(date_parts.at(1U), 'Z', true).at(0U)) * 1000000UL; - struct tm tm1{}; + struct tm tm1 {}; #if defined(_WIN32) utils::time::strptime(date_time.c_str(), "%Y-%m-%dT%T", &tm1); return nanos + utils::time::windows_time_t_to_unix_time(_mkgmtime(&tm1)); @@ -162,9 +161,8 @@ auto s3_provider::create_file_extra(const std::string &api_path, return api_error::success; } -auto s3_provider::create_path_directories(const std::string &api_path, - const std::string &key) const - -> api_error { +auto s3_provider::create_path_directories( + const std::string &api_path, const std::string &key) const -> api_error { if (api_path == "/") { return api_error::success; } @@ -297,9 +295,8 @@ auto s3_provider::get_directory_item_count(const std::string &api_path) const return 0U; } -auto s3_provider::get_directory_items_impl(const std::string &api_path, - directory_item_list &list) const - -> api_error { +auto s3_provider::get_directory_items_impl( + const std::string &api_path, directory_item_list &list) const -> api_error { REPERTORY_USES_FUNCTION_NAME(); auto cfg = get_config().get_s3_config(); @@ -444,8 +441,8 @@ auto s3_provider::get_directory_items_impl(const std::string &api_path, return ret; } -auto s3_provider::get_file(const std::string &api_path, api_file &file) const - -> api_error { +auto s3_provider::get_file(const std::string &api_path, + api_file &file) const -> api_error { REPERTORY_USES_FUNCTION_NAME(); try { @@ -462,7 +459,7 @@ auto s3_provider::get_file(const std::string &api_path, api_file &file) const file.api_parent = utils::path::get_parent_api_path(api_path); file.accessed_date = file.changed_date = file.creation_date = file.modified_date = result.last_modified; - file.key = is_encrypted ? utils::path::create_api_path(api_path) : ""; + file.key = is_encrypted ? utils::path::create_api_path(object_name) : ""; std::string size_str; if (get_item_meta(file.api_path, META_SIZE, size_str) == @@ -484,8 +481,8 @@ auto s3_provider::get_file(const std::string &api_path, api_file &file) const return api_error::error; } -auto s3_provider::get_file_list(api_file_list &list, std::string &marker) const - -> api_error { +auto s3_provider::get_file_list(api_file_list &list, + std::string &marker) const -> api_error { REPERTORY_USES_FUNCTION_NAME(); std::string response_data; @@ -494,7 +491,6 @@ auto s3_provider::get_file_list(api_file_list &list, std::string &marker) const std::nullopt, marker)) { return api_error::comm_error; } - marker.clear(); if (response_code != http_error_codes::ok) { utils::error::raise_error(function_name, response_code, @@ -521,8 +517,9 @@ auto s3_provider::get_file_list(api_file_list &list, std::string &marker) const auto node_list = doc.select_nodes("/ListBucketResult/Contents"); for (auto &&node : node_list) { - auto api_path = + auto object_name = std::string{node.node().select_node("Key").node().text().as_string()}; + auto api_path{object_name}; if (utils::string::ends_with(api_path, "/")) { continue; } @@ -549,8 +546,8 @@ auto s3_provider::get_file_list(api_file_list &list, std::string &marker) const ? utils::encryption::encrypting_reader::calculate_decrypted_size( size) : size; - file.key = is_encrypted ? utils::path::create_api_path(api_path) : ""; - auto err = add_if_not_found(file, api_path); + file.key = is_encrypted ? utils::path::create_api_path(object_name) : ""; + auto err = add_if_not_found(file, file.key); if (err != api_error::success) { return err; } @@ -561,9 +558,8 @@ auto s3_provider::get_file_list(api_file_list &list, std::string &marker) const return grab_more ? api_error::more_data : api_error::success; } -auto s3_provider::get_last_modified(bool directory, - const std::string &api_path) const - -> std::uint64_t { +auto s3_provider::get_last_modified( + bool directory, const std::string &api_path) const -> std::uint64_t { bool is_encrypted{}; std::string object_name; head_object_result result{}; @@ -573,10 +569,9 @@ auto s3_provider::get_last_modified(bool directory, : utils::time::get_time_now(); } -auto s3_provider::get_object_info(bool directory, const std::string &api_path, - bool &is_encrypted, std::string &object_name, - head_object_result &result) const - -> api_error { +auto s3_provider::get_object_info( + bool directory, const std::string &api_path, bool &is_encrypted, + std::string &object_name, head_object_result &result) const -> api_error { REPERTORY_USES_FUNCTION_NAME(); try { @@ -625,12 +620,10 @@ auto s3_provider::get_object_info(bool directory, const std::string &api_path, return api_error::error; } -auto s3_provider::get_object_list(std::string &response_data, - long &response_code, - std::optional delimiter, - std::optional prefix, - std::optional token) const - -> bool { +auto s3_provider::get_object_list( + std::string &response_data, long &response_code, + std::optional delimiter, std::optional prefix, + std::optional token) const -> bool { curl::requests::http_get get{}; get.allow_timeout = true; get.aws_service = "aws:amz:" + get_config().get_s3_config().region + ":s3"; @@ -659,8 +652,8 @@ auto s3_provider::get_total_drive_space() const -> std::uint64_t { return std::numeric_limits::max() / std::int64_t(2); } -auto s3_provider::is_directory(const std::string &api_path, bool &exists) const - -> api_error { +auto s3_provider::is_directory(const std::string &api_path, + bool &exists) const -> api_error { REPERTORY_USES_FUNCTION_NAME(); exists = false; @@ -687,8 +680,8 @@ auto s3_provider::is_directory(const std::string &api_path, bool &exists) const return api_error::error; } -auto s3_provider::is_file(const std::string &api_path, bool &exists) const - -> api_error { +auto s3_provider::is_file(const std::string &api_path, + bool &exists) const -> api_error { REPERTORY_USES_FUNCTION_NAME(); exists = false;