diff --git a/repertory/librepertory/src/providers/s3/s3_provider.cpp b/repertory/librepertory/src/providers/s3/s3_provider.cpp index 69cf2e4b..0ad73b2a 100644 --- a/repertory/librepertory/src/providers/s3/s3_provider.cpp +++ b/repertory/librepertory/src/providers/s3/s3_provider.cpp @@ -43,20 +43,22 @@ namespace repertory { s3_provider::s3_provider(app_config &config, i_http_comm &comm) : base_provider(config, comm) {} -auto s3_provider::add_if_not_found( - api_file &file, const std::string &object_name) const -> api_error { +auto s3_provider::add_if_not_found(api_file &file, + const std::string &object_name) const + -> api_error { api_meta_map meta{}; - if (get_item_meta(file.api_path, meta) == api_error::item_not_found) { - auto err = create_path_directories( + auto res = get_item_meta(file.api_path, meta); + if (res == api_error::item_not_found) { + res = create_path_directories( file.api_parent, utils::path::get_parent_api_path(object_name)); - if (err != api_error::success) { - return err; + if (res != api_error::success) { + return res; } get_api_item_added()(false, file); } - return api_error::success; + return res; } auto s3_provider::convert_api_date(std::string_view date) -> std::uint64_t { @@ -70,7 +72,7 @@ auto s3_provider::convert_api_date(std::string_view date) -> std::uint64_t { utils::string::split(date_parts.at(1U), 'Z', true).at(0U)) * 1000000UL; - struct tm tm1 {}; + struct tm tm1{}; #if defined(_WIN32) utils::time::strptime(date_time.c_str(), "%Y-%m-%dT%T", &tm1); return nanos + utils::time::windows_time_t_to_unix_time(_mkgmtime(&tm1)); @@ -168,8 +170,9 @@ auto s3_provider::create_file_extra(const std::string &api_path, return api_error::success; } -auto s3_provider::create_path_directories( - const std::string &api_path, const std::string &key) const -> api_error { +auto s3_provider::create_path_directories(const std::string &api_path, + const std::string &key) const + -> api_error { REPERTORY_USES_FUNCTION_NAME(); if (api_path == "/") { @@ -347,34 +350,99 @@ auto s3_provider::get_directory_item_count(const std::string &api_path) const return 0U; } -auto s3_provider::get_directory_items_impl( - const std::string &api_path, directory_item_list &list) const -> api_error { +auto s3_provider::get_directory_items_impl(const std::string &api_path, + directory_item_list &list) const + -> api_error { REPERTORY_USES_FUNCTION_NAME(); - const auto &cfg = get_s3_config(); - auto is_encrypted = not cfg.encryption_token.empty(); + const auto &cfg{get_s3_config()}; + auto is_encrypted{not cfg.encryption_token.empty()}; + + const auto add_diretory_item = [this, &cfg, &is_encrypted, &list]( + const std::string &child_object_name, + bool directory, auto node) -> api_error { + auto res{api_error::success}; + + directory_item dir_item{}; + dir_item.api_path = child_object_name; + dir_item.directory = directory; + if (is_encrypted) { + if (not utils::encryption::decrypt_file_path(cfg.encryption_token, + dir_item.api_path)) { + return api_error::decryption_error; + } + } + dir_item.api_path = utils::path::create_api_path(dir_item.api_path); + dir_item.api_parent = utils::path::get_parent_api_path(dir_item.api_path); + + if (directory) { + res = get_item_meta(dir_item.api_path, dir_item.meta); + if (res == api_error::item_not_found) { + res = create_path_directories(dir_item.api_path, child_object_name); + } + } else { + std::string size_str; + if (get_item_meta(dir_item.api_path, META_SIZE, size_str) == + api_error::success) { + dir_item.size = utils::string::to_uint64(size_str); + } else { + auto size = node.select_node("Size").node().text().as_ullong(); + + dir_item.size = is_encrypted ? utils::encryption::encrypting_reader:: + calculate_decrypted_size(size) + : size; + } + + res = get_item_meta(dir_item.api_path, dir_item.meta); + if (res == api_error::item_not_found) { + auto last_modified = convert_api_date( + node.select_node("LastModified").node().text().as_string()); + + api_file file{}; + file.api_path = dir_item.api_path; + file.api_parent = dir_item.api_parent; + file.accessed_date = file.changed_date = file.creation_date = + file.modified_date = last_modified; + file.file_size = dir_item.size; + if (is_encrypted) { + file.key = child_object_name; + } + + res = add_if_not_found(file, child_object_name); + } + } + + if (res != api_error::success) { + return res; + } + + list.push_back(std::move(dir_item)); + return api_error::success; + }; - auto ret = api_error::success; std::string key; if (is_encrypted) { - ret = get_item_meta(api_path, META_KEY, key); - if (ret != api_error::success) { - return ret; + auto res = get_item_meta(api_path, META_KEY, key); + if (res != api_error::success) { + return res; } } - auto object_name = + auto object_name{ api_path == "/" ? "" - : utils::path::create_api_path(is_encrypted ? key : api_path); + : utils::path::create_api_path(is_encrypted ? key : api_path), + }; - std::string response_data{}; - long response_code{}; - auto prefix = object_name.empty() ? object_name : object_name + "/"; + auto prefix{ + object_name.empty() ? object_name : object_name + "/", + }; auto grab_more{true}; std::string token{}; while (grab_more) { + std::string response_data{}; + long response_code{}; if (not get_object_list(response_data, response_code, "/", prefix, token)) { return api_error::comm_error; } @@ -411,34 +479,10 @@ auto s3_provider::get_directory_items_impl( for (const auto &node : node_list) { auto child_object_name = utils::path::create_api_path( utils::path::combine("/", {node.node().text().as_string()})); - directory_item dir_item{}; - dir_item.api_path = child_object_name; - if (is_encrypted) { - if (not utils::encryption::decrypt_file_path(cfg.encryption_token, - dir_item.api_path)) { - return api_error::decryption_error; - } - } - dir_item.api_path = utils::path::create_api_path(dir_item.api_path); - dir_item.api_parent = utils::path::get_parent_api_path(dir_item.api_path); - dir_item.directory = true; - dir_item.size = 0U; - - auto res = get_item_meta(dir_item.api_path, dir_item.meta); - if (res == api_error::item_not_found) { - res = create_path_directories(dir_item.api_path, child_object_name); - if (res != api_error::success) { - return res; - } - - res = get_item_meta(dir_item.api_path, dir_item.meta); - } - + auto res = add_diretory_item(child_object_name, true, node.node()); if (res != api_error::success) { return res; } - - list.push_back(std::move(dir_item)); } node_list = doc.select_nodes("/ListBucketResult/Contents"); @@ -449,66 +493,18 @@ auto s3_provider::get_directory_items_impl( continue; } - directory_item dir_item{}; - dir_item.api_path = child_object_name; - if (is_encrypted) { - if (not utils::encryption::decrypt_file_path(cfg.encryption_token, - dir_item.api_path)) { - return api_error::decryption_error; - } - } - dir_item.api_path = utils::path::create_api_path(dir_item.api_path); - dir_item.api_parent = utils::path::get_parent_api_path(dir_item.api_path); - dir_item.directory = false; - - std::string size_str; - if (get_item_meta(dir_item.api_path, META_SIZE, size_str) == - api_error::success) { - dir_item.size = utils::string::to_uint64(size_str); - } else { - auto size = node.node().select_node("Size").node().text().as_ullong(); - - dir_item.size = is_encrypted ? utils::encryption::encrypting_reader:: - calculate_decrypted_size(size) - : size; - } - - auto res = get_item_meta(dir_item.api_path, dir_item.meta); - if (res == api_error::item_not_found) { - auto last_modified = convert_api_date( - node.node().select_node("LastModified").node().text().as_string()); - - api_file file{}; - file.api_path = dir_item.api_path; - file.api_parent = dir_item.api_parent; - file.accessed_date = file.changed_date = file.creation_date = - file.modified_date = last_modified; - file.file_size = dir_item.size; - if (is_encrypted) { - file.key = child_object_name; - } - - res = add_if_not_found(file, child_object_name); - if (res != api_error::success) { - return res; - } - - res = get_item_meta(dir_item.api_path, dir_item.meta); - } - + auto res = add_diretory_item(child_object_name, false, node.node()); if (res != api_error::success) { return res; } - - list.push_back(std::move(dir_item)); } } - return ret; + return api_error::success; } -auto s3_provider::get_file(const std::string &api_path, - api_file &file) const -> api_error { +auto s3_provider::get_file(const std::string &api_path, api_file &file) const + -> api_error { REPERTORY_USES_FUNCTION_NAME(); try { @@ -547,8 +543,8 @@ auto s3_provider::get_file(const std::string &api_path, return api_error::error; } -auto s3_provider::get_file_list(api_file_list &list, - std::string &marker) const -> api_error { +auto s3_provider::get_file_list(api_file_list &list, std::string &marker) const + -> api_error { REPERTORY_USES_FUNCTION_NAME(); std::string response_data; @@ -623,8 +619,9 @@ auto s3_provider::get_file_list(api_file_list &list, return grab_more ? api_error::more_data : api_error::success; } -auto s3_provider::get_last_modified( - bool directory, const std::string &api_path) const -> std::uint64_t { +auto s3_provider::get_last_modified(bool directory, + const std::string &api_path) const + -> std::uint64_t { bool is_encrypted{}; std::string object_name; head_object_result result{}; @@ -634,9 +631,10 @@ auto s3_provider::get_last_modified( : utils::time::get_time_now(); } -auto s3_provider::get_object_info( - bool directory, const std::string &api_path, bool &is_encrypted, - std::string &object_name, head_object_result &result) const -> api_error { +auto s3_provider::get_object_info(bool directory, const std::string &api_path, + bool &is_encrypted, std::string &object_name, + head_object_result &result) const + -> api_error { REPERTORY_USES_FUNCTION_NAME(); try { @@ -691,10 +689,12 @@ auto s3_provider::get_object_info( return api_error::error; } -auto s3_provider::get_object_list( - std::string &response_data, long &response_code, - std::optional delimiter, std::optional prefix, - std::optional token) const -> bool { +auto s3_provider::get_object_list(std::string &response_data, + long &response_code, + std::optional delimiter, + std::optional prefix, + std::optional token) const + -> bool { curl::requests::http_get get{}; get.allow_timeout = true; get.aws_service = "aws:amz:" + get_s3_config().region + ":s3"; @@ -722,8 +722,8 @@ auto s3_provider::get_total_drive_space() const -> std::uint64_t { return std::numeric_limits::max() / std::int64_t(2); } -auto s3_provider::is_directory(const std::string &api_path, - bool &exists) const -> api_error { +auto s3_provider::is_directory(const std::string &api_path, bool &exists) const + -> api_error { REPERTORY_USES_FUNCTION_NAME(); exists = false; @@ -750,8 +750,8 @@ auto s3_provider::is_directory(const std::string &api_path, return api_error::error; } -auto s3_provider::is_file(const std::string &api_path, - bool &exists) const -> api_error { +auto s3_provider::is_file(const std::string &api_path, bool &exists) const + -> api_error { REPERTORY_USES_FUNCTION_NAME(); exists = false;