diff --git a/repertory/librepertory/include/providers/base_provider.hpp b/repertory/librepertory/include/providers/base_provider.hpp index abd35439..3efa179b 100644 --- a/repertory/librepertory/include/providers/base_provider.hpp +++ b/repertory/librepertory/include/providers/base_provider.hpp @@ -44,23 +44,25 @@ private: api_item_added_callback api_item_added_; std::unique_ptr db3_; i_file_manager *fm_{}; + mutable std::atomic used_space; private: void remove_deleted_files(bool source_only); + void update_used_drive_space() const; + protected: - [[nodiscard]] static auto create_api_file(std::string path, std::string key, - std::uint64_t size, - std::uint64_t file_time) - -> api_file; + [[nodiscard]] static auto + create_api_file(std::string path, std::string key, std::uint64_t size, + std::uint64_t file_time) -> api_file; [[nodiscard]] static auto create_api_file(std::string path, std::uint64_t size, api_meta_map &meta) -> api_file; - [[nodiscard]] virtual auto create_directory_impl(const std::string &api_path, - api_meta_map &meta) - -> api_error = 0; + [[nodiscard]] virtual auto + create_directory_impl(const std::string &api_path, + api_meta_map &meta) -> api_error = 0; [[nodiscard]] virtual auto create_file_extra(const std::string & /* api_path */, @@ -72,8 +74,8 @@ protected: return api_item_added_; } - [[nodiscard]] auto get_api_item_added() const - -> const api_item_added_callback & { + [[nodiscard]] auto + get_api_item_added() const -> const api_item_added_callback & { return api_item_added_; } @@ -97,25 +99,23 @@ protected: return fm_; } - [[nodiscard]] virtual auto get_used_drive_space_impl() const - -> std::uint64_t = 0; + [[nodiscard]] virtual auto + get_used_drive_space_impl() const -> std::uint64_t = 0; - [[nodiscard]] virtual auto remove_directory_impl(const std::string &api_path) - -> api_error = 0; + [[nodiscard]] virtual auto + remove_directory_impl(const std::string &api_path) -> api_error = 0; - [[nodiscard]] virtual auto remove_file_impl(const std::string &api_path) - -> api_error = 0; + [[nodiscard]] virtual auto + remove_file_impl(const std::string &api_path) -> api_error = 0; - [[nodiscard]] virtual auto upload_file_impl(const std::string &api_path, - const std::string &source_path, - stop_type &stop_requested) - -> api_error = 0; + [[nodiscard]] virtual auto + upload_file_impl(const std::string &api_path, const std::string &source_path, + stop_type &stop_requested) -> api_error = 0; public: - [[nodiscard]] auto - create_directory_clone_source_meta(const std::string &source_api_path, - const std::string &api_path) - -> api_error override; + [[nodiscard]] auto create_directory_clone_source_meta( + const std::string &source_api_path, + const std::string &api_path) -> api_error override; [[nodiscard]] auto create_directory(const std::string &api_path, api_meta_map &meta) -> api_error override; @@ -123,82 +123,76 @@ public: [[nodiscard]] auto create_file(const std::string &api_path, api_meta_map &meta) -> api_error override; - [[nodiscard]] auto get_api_path_from_source(const std::string &source_path, - std::string &api_path) const - -> api_error override; - - [[nodiscard]] auto get_directory_items(const std::string &api_path, - directory_item_list &list) const - -> api_error override; - - [[nodiscard]] auto get_file_size(const std::string &api_path, - std::uint64_t &file_size) const - -> api_error override; - - [[nodiscard]] auto get_filesystem_item(const std::string &api_path, - bool directory, - filesystem_item &fsi) const - -> api_error override; - - [[nodiscard]] auto get_filesystem_item_and_file(const std::string &api_path, - api_file &f, - filesystem_item &fsi) const - -> api_error override; + [[nodiscard]] auto + get_api_path_from_source(const std::string &source_path, + std::string &api_path) const -> api_error override; [[nodiscard]] auto - get_filesystem_item_from_source_path(const std::string &source_path, - filesystem_item &fsi) const - -> api_error override; + get_directory_items(const std::string &api_path, + directory_item_list &list) const -> api_error override; - [[nodiscard]] auto get_item_meta(const std::string &api_path, - api_meta_map &meta) const - -> api_error override; + [[nodiscard]] auto + get_file_size(const std::string &api_path, + std::uint64_t &file_size) const -> api_error override; - [[nodiscard]] auto get_item_meta(const std::string &api_path, - const std::string &key, - std::string &value) const - -> api_error override; + [[nodiscard]] auto + get_filesystem_item(const std::string &api_path, bool directory, + filesystem_item &fsi) const -> api_error override; - [[nodiscard]] auto get_pinned_files() const - -> std::vector override; + [[nodiscard]] auto get_filesystem_item_and_file( + const std::string &api_path, api_file &f, + filesystem_item &fsi) const -> api_error override; + + [[nodiscard]] auto get_filesystem_item_from_source_path( + const std::string &source_path, + filesystem_item &fsi) const -> api_error override; + + [[nodiscard]] auto + get_item_meta(const std::string &api_path, + api_meta_map &meta) const -> api_error override; + + [[nodiscard]] auto + get_item_meta(const std::string &api_path, const std::string &key, + std::string &value) const -> api_error override; + + [[nodiscard]] auto + get_pinned_files() const -> std::vector override; [[nodiscard]] auto get_total_item_count() const -> std::uint64_t override; [[nodiscard]] auto get_used_drive_space() const -> std::uint64_t override; - [[nodiscard]] auto is_file_writeable(const std::string &api_path) const - -> bool override; + [[nodiscard]] auto + is_file_writeable(const std::string &api_path) const -> bool override; [[nodiscard]] auto is_read_only() const -> bool override { return false; } - [[nodiscard]] auto remove_directory(const std::string &api_path) - -> api_error override; + [[nodiscard]] auto + remove_directory(const std::string &api_path) -> api_error override; - [[nodiscard]] auto remove_file(const std::string &api_path) - -> api_error override; + [[nodiscard]] auto + remove_file(const std::string &api_path) -> api_error override; - [[nodiscard]] auto remove_item_meta(const std::string &api_path, - const std::string &key) - -> api_error override; + [[nodiscard]] auto + remove_item_meta(const std::string &api_path, + const std::string &key) -> api_error override; - [[nodiscard]] auto set_item_meta(const std::string &api_path, - const std::string &key, - const std::string &value) - -> api_error override; + [[nodiscard]] auto + set_item_meta(const std::string &api_path, const std::string &key, + const std::string &value) -> api_error override; - [[nodiscard]] auto set_item_meta(const std::string &api_path, - const api_meta_map &meta) - -> api_error override; + [[nodiscard]] auto + set_item_meta(const std::string &api_path, + const api_meta_map &meta) -> api_error override; [[nodiscard]] auto start(api_item_added_callback api_item_added, i_file_manager *mgr) -> bool override; void stop() override; - [[nodiscard]] auto upload_file(const std::string &api_path, - const std::string &source_path, - stop_type &stop_requested) - -> api_error override; + [[nodiscard]] auto + upload_file(const std::string &api_path, const std::string &source_path, + stop_type &stop_requested) -> api_error override; }; } // namespace repertory diff --git a/repertory/librepertory/include/providers/s3/s3_provider.hpp b/repertory/librepertory/include/providers/s3/s3_provider.hpp index 147b6d59..23b191e2 100644 --- a/repertory/librepertory/include/providers/s3/s3_provider.hpp +++ b/repertory/librepertory/include/providers/s3/s3_provider.hpp @@ -47,71 +47,70 @@ public: auto operator=(s3_provider &&) -> s3_provider & = delete; private: - [[nodiscard]] auto add_if_not_found(api_file &file, - const std::string &object_name) const - -> api_error; + [[nodiscard]] auto + add_if_not_found(api_file &file, + const std::string &object_name) const -> api_error; - [[nodiscard]] auto create_file_extra(const std::string &api_path, - api_meta_map &meta) - -> api_error override; + [[nodiscard]] auto + create_file_extra(const std::string &api_path, + api_meta_map &meta) -> api_error override; - [[nodiscard]] auto create_path_directories(const std::string &api_path, - const std::string &key) const - -> api_error; + [[nodiscard]] auto + create_path_directories(const std::string &api_path, + const std::string &key) const -> api_error; - [[nodiscard]] auto decrypt_object_name(std::string &object_name) const - -> api_error; + [[nodiscard]] auto + decrypt_object_name(std::string &object_name) const -> api_error; - [[nodiscard]] auto get_last_modified(bool directory, - const std::string &api_path) const - -> std::uint64_t; + [[nodiscard]] auto + get_last_modified(bool directory, + const std::string &api_path) const -> std::uint64_t; [[nodiscard]] auto get_object_info(bool directory, const std::string &api_path, bool &is_encrypted, std::string &object_name, head_object_result &result) const -> api_error; - [[nodiscard]] auto - get_object_list(std::string &response_data, long &response_code, - std::optional delimiter = std::nullopt, - std::optional prefix = std::nullopt) const - -> bool; + [[nodiscard]] auto get_object_list( + std::string &response_data, long &response_code, + std::optional delimiter = std::nullopt, + std::optional prefix = std::nullopt, + std::optional token = std::nullopt) const -> bool; protected: - [[nodiscard]] auto create_directory_impl(const std::string &api_path, - api_meta_map &meta) - -> api_error override; + [[nodiscard]] auto + create_directory_impl(const std::string &api_path, + api_meta_map &meta) -> api_error override; [[nodiscard]] auto get_directory_items_impl(const std::string &api_path, directory_item_list &list) const -> api_error override; - [[nodiscard]] auto get_used_drive_space_impl() const - -> std::uint64_t override; + [[nodiscard]] auto + get_used_drive_space_impl() const -> std::uint64_t override; - [[nodiscard]] auto remove_directory_impl(const std::string &api_path) - -> api_error override; + [[nodiscard]] auto + remove_directory_impl(const std::string &api_path) -> api_error override; - [[nodiscard]] auto remove_file_impl(const std::string &api_path) - -> api_error override; + [[nodiscard]] auto + remove_file_impl(const std::string &api_path) -> api_error override; - [[nodiscard]] auto upload_file_impl(const std::string &api_path, - const std::string &source_path, - stop_type &stop_requested) - -> api_error override; + [[nodiscard]] auto + upload_file_impl(const std::string &api_path, const std::string &source_path, + stop_type &stop_requested) -> api_error override; public: - [[nodiscard]] static auto convert_api_date(std::string_view date) - -> std::uint64_t; + [[nodiscard]] static auto + convert_api_date(std::string_view date) -> std::uint64_t; [[nodiscard]] auto get_directory_item_count(const std::string &api_path) const -> std::uint64_t override; - [[nodiscard]] auto get_file(const std::string &api_path, api_file &file) const - -> api_error override; + [[nodiscard]] auto get_file(const std::string &api_path, + api_file &file) const -> api_error override; - [[nodiscard]] auto get_file_list(api_file_list &list) const - -> api_error override; + [[nodiscard]] auto + get_file_list(api_file_list &list) const -> api_error override; [[nodiscard]] auto get_total_drive_space() const -> std::uint64_t override; @@ -122,8 +121,8 @@ public: [[nodiscard]] auto is_directory(const std::string &api_path, bool &exists) const -> api_error override; - [[nodiscard]] auto is_file(const std::string &api_path, bool &exists) const - -> api_error override; + [[nodiscard]] auto is_file(const std::string &api_path, + bool &exists) const -> api_error override; [[nodiscard]] auto is_online() const -> bool override; @@ -131,15 +130,14 @@ public: return false; }; - [[nodiscard]] auto read_file_bytes(const std::string &api_path, - std::size_t size, std::uint64_t offset, - data_buffer &data, - stop_type &stop_requested) - -> api_error override; + [[nodiscard]] auto + read_file_bytes(const std::string &api_path, std::size_t size, + std::uint64_t offset, data_buffer &data, + stop_type &stop_requested) -> api_error override; - [[nodiscard]] auto rename_file(const std::string &from_api_path, - const std::string &to_api_path) - -> api_error override; + [[nodiscard]] auto + rename_file(const std::string &from_api_path, + const std::string &to_api_path) -> api_error override; [[nodiscard]] auto start(api_item_added_callback api_item_added, i_file_manager *mgr) -> bool override; diff --git a/repertory/librepertory/src/providers/base_provider.cpp b/repertory/librepertory/src/providers/base_provider.cpp index 0e959e91..d7161a0e 100644 --- a/repertory/librepertory/src/providers/base_provider.cpp +++ b/repertory/librepertory/src/providers/base_provider.cpp @@ -33,8 +33,8 @@ namespace repertory { auto base_provider::create_api_file(std::string path, std::string key, - std::uint64_t size, std::uint64_t file_time) - -> api_file { + std::uint64_t size, + std::uint64_t file_time) -> api_file { api_file file{}; file.api_path = utils::path::create_api_path(path); file.api_parent = utils::path::get_parent_api_path(file.api_path); @@ -66,8 +66,8 @@ auto base_provider::create_api_file(std::string path, std::uint64_t size, } auto base_provider::create_directory_clone_source_meta( - const std::string &source_api_path, const std::string &api_path) - -> api_error { + const std::string &source_api_path, + const std::string &api_path) -> api_error { REPERTORY_USES_FUNCTION_NAME(); bool exists{}; @@ -164,8 +164,8 @@ auto base_provider::create_directory(const std::string &api_path, return set_item_meta(api_path, meta); } -auto base_provider::create_file(const std::string &api_path, api_meta_map &meta) - -> api_error { +auto base_provider::create_file(const std::string &api_path, + api_meta_map &meta) -> api_error { REPERTORY_USES_FUNCTION_NAME(); bool exists{}; @@ -222,9 +222,8 @@ auto base_provider::create_file(const std::string &api_path, api_meta_map &meta) return api_error::error; } -auto base_provider::get_api_path_from_source(const std::string &source_path, - std::string &api_path) const - -> api_error { +auto base_provider::get_api_path_from_source( + const std::string &source_path, std::string &api_path) const -> api_error { REPERTORY_USES_FUNCTION_NAME(); if (source_path.empty()) { @@ -237,9 +236,8 @@ auto base_provider::get_api_path_from_source(const std::string &source_path, return db3_->get_api_path(source_path, api_path); } -auto base_provider::get_directory_items(const std::string &api_path, - directory_item_list &list) const - -> api_error { +auto base_provider::get_directory_items( + const std::string &api_path, directory_item_list &list) const -> api_error { REPERTORY_USES_FUNCTION_NAME(); bool exists{}; @@ -303,10 +301,9 @@ auto base_provider::get_file_size(const std::string &api_path, return api_error::success; } -auto base_provider::get_filesystem_item(const std::string &api_path, - bool directory, - filesystem_item &fsi) const - -> api_error { +auto base_provider::get_filesystem_item( + const std::string &api_path, bool directory, + filesystem_item &fsi) const -> api_error { bool exists{}; auto res = is_directory(api_path, exists); if (res != api_error::success) { @@ -339,10 +336,9 @@ auto base_provider::get_filesystem_item(const std::string &api_path, return api_error::success; } -auto base_provider::get_filesystem_item_and_file(const std::string &api_path, - api_file &file, - filesystem_item &fsi) const - -> api_error { +auto base_provider::get_filesystem_item_and_file( + const std::string &api_path, api_file &file, + filesystem_item &fsi) const -> api_error { auto res = get_file(api_path, file); if (res != api_error::success) { return res; @@ -403,18 +399,11 @@ auto base_provider::get_total_item_count() const -> std::uint64_t { } auto base_provider::get_used_drive_space() const -> std::uint64_t { - REPERTORY_USES_FUNCTION_NAME(); - - try { - auto used_space = get_used_drive_space_impl(); - get_file_mgr()->update_used_space(used_space); - return used_space; - } catch (const std::exception &ex) { - utils::error::raise_error(function_name, ex, - "failed to get used drive space"); + if (used_space == 0U) { + update_used_drive_space(); } - return 0U; + return used_space; } auto base_provider::is_file_writeable(const std::string &api_path) const @@ -471,10 +460,10 @@ void base_provider::remove_deleted_files(bool source_only) { return; } - struct removed_item { - std::string api_path{}; + struct removed_item final { + std::string api_path; bool directory{}; - std::string source_path{}; + std::string source_path; }; api_file_list list{}; @@ -713,6 +702,8 @@ auto base_provider::start(api_item_added_callback api_item_added, polling::instance().set_callback({"check_deleted", polling::frequency::low, [this]() { remove_deleted_files(false); }}); + polling::instance().set_callback({"drive_space", polling::frequency::high, + [this]() { update_used_drive_space(); }}); return true; } @@ -721,6 +712,19 @@ void base_provider::stop() { db3_.reset(); } +void base_provider::update_used_drive_space() const { + REPERTORY_USES_FUNCTION_NAME(); + + try { + auto next_used_space = get_used_drive_space_impl(); + get_file_mgr()->update_used_space(next_used_space); + used_space = next_used_space; + } catch (const std::exception &ex) { + utils::error::raise_error(function_name, ex, + "failed to get used drive space"); + } +} + auto base_provider::upload_file(const std::string &api_path, const std::string &source_path, stop_type &stop_requested) -> api_error { diff --git a/repertory/librepertory/src/providers/s3/s3_provider.cpp b/repertory/librepertory/src/providers/s3/s3_provider.cpp index 61e9a965..840cdc42 100644 --- a/repertory/librepertory/src/providers/s3/s3_provider.cpp +++ b/repertory/librepertory/src/providers/s3/s3_provider.cpp @@ -44,9 +44,8 @@ s3_provider::s3_provider(app_config &config, i_http_comm &comm) get_comm().enable_s3_path_style(config.get_s3_config().use_path_style); } -auto s3_provider::add_if_not_found(api_file &file, - const std::string &object_name) const - -> api_error { +auto s3_provider::add_if_not_found( + api_file &file, const std::string &object_name) const -> api_error { api_meta_map meta{}; if (get_item_meta(file.api_path, meta) == api_error::item_not_found) { auto err = create_path_directories( @@ -72,7 +71,7 @@ auto s3_provider::convert_api_date(std::string_view date) -> std::uint64_t { utils::string::split(date_parts.at(1U), 'Z', true).at(0U)) * 1000000UL; - struct tm tm1{}; + struct tm tm1 {}; #if defined(_WIN32) utils::time::strptime(date_time.c_str(), "%Y-%m-%dT%T", &tm1); return nanos + utils::time::windows_time_t_to_unix_time(_mkgmtime(&tm1)); @@ -163,9 +162,8 @@ auto s3_provider::create_file_extra(const std::string &api_path, return api_error::success; } -auto s3_provider::create_path_directories(const std::string &api_path, - const std::string &key) const - -> api_error { +auto s3_provider::create_path_directories( + const std::string &api_path, const std::string &key) const -> api_error { if (api_path == "/") { return api_error::success; } @@ -281,9 +279,8 @@ auto s3_provider::get_directory_item_count(const std::string &api_path) const return 0U; } -auto s3_provider::get_directory_items_impl(const std::string &api_path, - directory_item_list &list) const - -> api_error { +auto s3_provider::get_directory_items_impl( + const std::string &api_path, directory_item_list &list) const -> api_error { REPERTORY_USES_FUNCTION_NAME(); const auto cfg = get_config().get_s3_config(); @@ -402,8 +399,8 @@ auto s3_provider::get_directory_items_impl(const std::string &api_path, return ret; } -auto s3_provider::get_file(const std::string &api_path, api_file &file) const - -> api_error { +auto s3_provider::get_file(const std::string &api_path, + api_file &file) const -> api_error { REPERTORY_USES_FUNCTION_NAME(); try { @@ -500,9 +497,8 @@ auto s3_provider::get_file_list(api_file_list &list) const -> api_error { return api_error::success; } -auto s3_provider::get_last_modified(bool directory, - const std::string &api_path) const - -> std::uint64_t { +auto s3_provider::get_last_modified( + bool directory, const std::string &api_path) const -> std::uint64_t { bool is_encrypted{}; std::string object_name; head_object_result result{}; @@ -512,10 +508,9 @@ auto s3_provider::get_last_modified(bool directory, : utils::time::get_time_now(); } -auto s3_provider::get_object_info(bool directory, const std::string &api_path, - bool &is_encrypted, std::string &object_name, - head_object_result &result) const - -> api_error { +auto s3_provider::get_object_info( + bool directory, const std::string &api_path, bool &is_encrypted, + std::string &object_name, head_object_result &result) const -> api_error { REPERTORY_USES_FUNCTION_NAME(); try { @@ -564,11 +559,10 @@ auto s3_provider::get_object_info(bool directory, const std::string &api_path, return api_error::error; } -auto s3_provider::get_object_list(std::string &response_data, - long &response_code, - std::optional delimiter, - std::optional prefix) const - -> bool { +auto s3_provider::get_object_list( + std::string &response_data, long &response_code, + std::optional delimiter, std::optional prefix, + std::optional token) const -> bool { curl::requests::http_get get{}; get.allow_timeout = true; get.aws_service = "aws:amz:" + get_config().get_s3_config().region + ":s3"; @@ -581,6 +575,9 @@ auto s3_provider::get_object_list(std::string &response_data, get.query["prefix"] = prefix.value(); utils::string::left_trim(get.query["prefix"], '/'); } + if (token.has_value() && not token.value().empty()) { + get.query["continuation-token"] = token.value(); + } get.response_handler = [&response_data](const data_buffer &data, long /*response_code*/) { response_data = std::string(data.begin(), data.end()); @@ -595,38 +592,57 @@ auto s3_provider::get_total_drive_space() const -> std::uint64_t { } auto s3_provider::get_used_drive_space_impl() const -> std::uint64_t { - std::string response_data; - long response_code{}; - if (not get_object_list(response_data, response_code)) { - return 0U; + auto grab_more{true}; + std::string token{}; + std::uint64_t total_size{}; + while (grab_more) { + std::string response_data; + long response_code{}; + if (not get_object_list(response_data, response_code, std::nullopt, + std::nullopt, token)) { + return total_size; + } + + if (response_code != http_error_codes::ok) { + return total_size; + } + + pugi::xml_document doc; + auto res = doc.load_string(response_data.c_str()); + if (res.status != pugi::xml_parse_status::status_ok) { + return total_size; + } + + const auto cfg = get_config().get_s3_config(); + const auto is_encrypted = not cfg.encryption_token.empty(); + + auto node_list = doc.select_nodes("/ListBucketResult/Contents"); + grab_more = doc.select_node("/ListBucketResult/IsTruncated") + .node() + .text() + .as_bool(); + if (grab_more) { + token = doc.select_node("/ListBucketResult/NextContinuationToken") + .node() + .text() + .as_string(); + } + + total_size = std::accumulate( + node_list.begin(), node_list.end(), total_size, + [&is_encrypted](std::uint64_t total, auto node) -> std::uint64_t { + auto size = node.node().select_node("Size").node().text().as_ullong(); + return total + (is_encrypted ? utils::encryption::encrypting_reader:: + calculate_decrypted_size(size) + : size); + }); } - if (response_code != http_error_codes::ok) { - return 0U; - } - - pugi::xml_document doc; - auto res = doc.load_string(response_data.c_str()); - if (res.status != pugi::xml_parse_status::status_ok) { - return 0U; - } - - const auto cfg = get_config().get_s3_config(); - const auto is_encrypted = not cfg.encryption_token.empty(); - - auto node_list = doc.select_nodes("/ListBucketResult/Contents"); - return std::accumulate( - node_list.begin(), node_list.end(), std::uint64_t(0U), - [&is_encrypted](std::uint64_t total, auto node) -> std::uint64_t { - auto size = node.node().select_node("Size").node().text().as_ullong(); - return total + (is_encrypted ? utils::encryption::encrypting_reader:: - calculate_decrypted_size(size) - : size); - }); + return total_size; } -auto s3_provider::is_directory(const std::string &api_path, bool &exists) const - -> api_error { +auto s3_provider::is_directory(const std::string &api_path, + bool &exists) const -> api_error { REPERTORY_USES_FUNCTION_NAME(); exists = false; @@ -653,8 +669,8 @@ auto s3_provider::is_directory(const std::string &api_path, bool &exists) const return api_error::error; } -auto s3_provider::is_file(const std::string &api_path, bool &exists) const - -> api_error { +auto s3_provider::is_file(const std::string &api_path, + bool &exists) const -> api_error { REPERTORY_USES_FUNCTION_NAME(); exists = false;