From 329ba1d5e4aababb093c6e90dceea43865d2d4fc Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Sat, 30 Nov 2024 17:38:24 -0600 Subject: [PATCH] s3 fixes --- .../src/providers/s3/s3_provider.cpp | 173 ++++++++++-------- 1 file changed, 95 insertions(+), 78 deletions(-) diff --git a/repertory/librepertory/src/providers/s3/s3_provider.cpp b/repertory/librepertory/src/providers/s3/s3_provider.cpp index d5e3f98f..6bfe24b2 100644 --- a/repertory/librepertory/src/providers/s3/s3_provider.cpp +++ b/repertory/librepertory/src/providers/s3/s3_provider.cpp @@ -320,95 +320,112 @@ auto s3_provider::get_directory_items_impl( long response_code{}; auto prefix = object_name.empty() ? object_name : object_name + "/"; - if (not get_object_list(response_data, response_code, "/", prefix)) { - return api_error::comm_error; - } - - if (response_code == http_error_codes::not_found) { - return api_error::directory_not_found; - } - - if (response_code != http_error_codes::ok) { - utils::error::raise_api_path_error(function_name, api_path, response_code, - "failed to get directory items"); - return api_error::comm_error; - } - - pugi::xml_document doc; - auto parse_res = doc.load_string(response_data.c_str()); - if (parse_res.status != pugi::xml_parse_status::status_ok) { - return api_error::error; - } - - const auto add_directory_item = - [&](bool directory, const std::string &name, std::uint64_t last_modified, - std::function get_size) - -> api_error { - auto child_api_path = - utils::path::create_api_path(utils::path::combine("/", {name})); - std::string child_object_name; - if (is_encrypted) { - child_object_name = child_api_path; - if (not utils::encryption::decrypt_file_path(cfg.encryption_token, - child_api_path)) { - return api_error::decryption_error; - } + auto grab_more{true}; + std::string token{}; + while (grab_more) { + if (not get_object_list(response_data, response_code, "/", prefix, token)) { + return api_error::comm_error; } - directory_item dir_item{}; - dir_item.api_path = child_api_path; - dir_item.api_parent = utils::path::get_parent_api_path(dir_item.api_path); - dir_item.directory = directory; - dir_item.size = get_size(dir_item); - ret = get_item_meta(child_api_path, dir_item.meta); - if (ret == api_error::item_not_found) { - if (directory) { - ret = create_path_directories(child_api_path, child_object_name); - if (ret != api_error::success) { - return ret; - } - } else { - auto file = create_api_file(child_api_path, child_object_name, - dir_item.size, last_modified); - ret = add_if_not_found(file, child_object_name); - if (ret != api_error::success) { - return ret; + if (response_code == http_error_codes::not_found) { + return api_error::directory_not_found; + } + + if (response_code != http_error_codes::ok) { + utils::error::raise_api_path_error(function_name, api_path, response_code, + "failed to get directory items"); + return api_error::comm_error; + } + + pugi::xml_document doc; + auto parse_res = doc.load_string(response_data.c_str()); + if (parse_res.status != pugi::xml_parse_status::status_ok) { + return api_error::error; + } + + grab_more = doc.select_node("/ListBucketResult/IsTruncated") + .node() + .text() + .as_bool(); + if (grab_more) { + token = doc.select_node("/ListBucketResult/NextContinuationToken") + .node() + .text() + .as_string(); + } + + const auto add_directory_item = + [&](bool directory, const std::string &name, + std::uint64_t last_modified, + std::function get_size) + -> api_error { + auto child_api_path = + utils::path::create_api_path(utils::path::combine("/", {name})); + std::string child_object_name; + if (is_encrypted) { + child_object_name = child_api_path; + if (not utils::encryption::decrypt_file_path(cfg.encryption_token, + child_api_path)) { + return api_error::decryption_error; } } + directory_item dir_item{}; + dir_item.api_path = child_api_path; + dir_item.api_parent = utils::path::get_parent_api_path(dir_item.api_path); + dir_item.directory = directory; + dir_item.size = get_size(dir_item); ret = get_item_meta(child_api_path, dir_item.meta); - } + if (ret == api_error::item_not_found) { + if (directory) { + ret = create_path_directories(child_api_path, child_object_name); + if (ret != api_error::success) { + return ret; + } + } else { + auto file = create_api_file(child_api_path, child_object_name, + dir_item.size, last_modified); + ret = add_if_not_found(file, child_object_name); + if (ret != api_error::success) { + return ret; + } + } - if (ret != api_error::success) { - return ret; - } + ret = get_item_meta(child_api_path, dir_item.meta); + } - list.push_back(std::move(dir_item)); - return api_error::success; - }; + if (ret != api_error::success) { + return ret; + } - auto node_list = doc.select_nodes("/ListBucketResult/CommonPrefixes/Prefix"); - for (auto &&node : node_list) { - add_directory_item( - true, node.node().text().as_string(), 0U, - [](const directory_item &) -> std::uint64_t { return 0U; }); - } + list.push_back(std::move(dir_item)); + return api_error::success; + }; - node_list = doc.select_nodes("/ListBucketResult/Contents"); - for (auto &&node : node_list) { - auto child_object_name = utils::path::create_api_path( - node.node().select_node("Key").node().text().as_string()); - if (child_object_name != utils::path::create_api_path(prefix)) { - auto size = node.node().select_node("Size").node().text().as_ullong(); - auto last_modified = convert_api_date( - node.node().select_node("LastModified").node().text().as_string()); + auto node_list = + doc.select_nodes("/ListBucketResult/CommonPrefixes/Prefix"); + for (auto &&node : node_list) { add_directory_item( - false, child_object_name, last_modified, - [&is_encrypted, &size](const directory_item &) -> std::uint64_t { - return is_encrypted ? utils::encryption::encrypting_reader:: - calculate_decrypted_size(size) - : size; - }); + true, node.node().text().as_string(), 0U, + [](const directory_item &) -> std::uint64_t { return 0U; }); + } + + node_list = doc.select_nodes("/ListBucketResult/Contents"); + for (auto &&node : node_list) { + auto child_object_name = utils::path::create_api_path( + node.node().select_node("Key").node().text().as_string()); + if (child_object_name != utils::path::create_api_path(prefix)) { + auto size = node.node().select_node("Size").node().text().as_ullong(); + auto last_modified = convert_api_date( + node.node().select_node("LastModified").node().text().as_string()); + add_directory_item( + false, child_object_name, last_modified, + [&is_encrypted, &size](const directory_item &) -> std::uint64_t { + return is_encrypted ? utils::encryption::encrypting_reader:: + calculate_decrypted_size(size) + : size; + }); + } } }