From 7faf648919258b9d7e9c9a0f5cc623b8253b7253 Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Sat, 30 Nov 2024 17:21:41 -0600 Subject: [PATCH] s3 fixes --- .../src/providers/s3/s3_provider.cpp | 176 ++++++++++-------- 1 file changed, 98 insertions(+), 78 deletions(-) diff --git a/repertory/librepertory/src/providers/s3/s3_provider.cpp b/repertory/librepertory/src/providers/s3/s3_provider.cpp index 840cdc42..d5e3f98f 100644 --- a/repertory/librepertory/src/providers/s3/s3_provider.cpp +++ b/repertory/librepertory/src/providers/s3/s3_provider.cpp @@ -22,7 +22,6 @@ #include "providers/s3/s3_provider.hpp" #include "app_config.hpp" -#include "comm/curl/curl_comm.hpp" #include "comm/i_http_comm.hpp" #include "file_manager/i_file_manager.hpp" #include "types/repertory.hpp" @@ -243,35 +242,52 @@ auto s3_provider::get_directory_item_count(const std::string &api_path) const long response_code{}; auto prefix = object_name.empty() ? object_name : object_name + "/"; - if (not get_object_list(response_data, response_code, "/", prefix)) { - return 0U; + auto grab_more{true}; + std::string token{}; + std::uint64_t total_count{}; + while (grab_more) { + if (not get_object_list(response_data, response_code, "/", prefix, + token)) { + return total_count; + } + + if (response_code == http_error_codes::not_found) { + return total_count; + } + + if (response_code != http_error_codes::ok) { + return total_count; + } + + pugi::xml_document doc; + auto res = doc.load_string(response_data.c_str()); + if (res.status != pugi::xml_parse_status::status_ok) { + return total_count; + } + + grab_more = doc.select_node("/ListBucketResult/IsTruncated") + .node() + .text() + .as_bool(); + if (grab_more) { + token = doc.select_node("/ListBucketResult/NextContinuationToken") + .node() + .text() + .as_string(); + } + + auto node_list = + doc.select_nodes("/ListBucketResult/CommonPrefixes/Prefix"); + total_count += node_list.size(); + + node_list = doc.select_nodes("/ListBucketResult/Contents"); + total_count += node_list.size(); + if (not prefix.empty()) { + --total_count; + } } - if (response_code == http_error_codes::not_found) { - return 0U; - } - - if (response_code != http_error_codes::ok) { - return 0U; - } - - pugi::xml_document doc; - auto res = doc.load_string(response_data.c_str()); - if (res.status != pugi::xml_parse_status::status_ok) { - return 0U; - } - - auto node_list = - doc.select_nodes("/ListBucketResult/CommonPrefixes/Prefix"); - std::uint64_t ret = node_list.size(); - - node_list = doc.select_nodes("/ListBucketResult/Contents"); - ret += node_list.size(); - if (not prefix.empty()) { - --ret; - } - - return ret; + return total_count; } catch (const std::exception &e) { utils::error::raise_error(function_name, e, "exception occurred"); } @@ -434,63 +450,67 @@ auto s3_provider::get_file(const std::string &api_path, auto s3_provider::get_file_list(api_file_list &list) const -> api_error { REPERTORY_USES_FUNCTION_NAME(); - std::string response_data; - long response_code{}; - if (not get_object_list(response_data, response_code)) { - return api_error::comm_error; - } + auto grab_more{true}; + std::string token{}; + while (grab_more) { + std::string response_data; + long response_code{}; + if (not get_object_list(response_data, response_code, std::nullopt, + std::nullopt, token)) { + return api_error::comm_error; + } - if (response_code != http_error_codes::ok) { - utils::error::raise_error(function_name, response_code, - "failed to get file list"); - return api_error::comm_error; - } + if (response_code != http_error_codes::ok) { + utils::error::raise_error(function_name, response_code, + "failed to get file list"); + return api_error::comm_error; + } - pugi::xml_document doc; - auto res = doc.load_string(response_data.c_str()); - if (res.status != pugi::xml_parse_status::status_ok) { - utils::error::raise_error(function_name, res.status, - "failed to parse xml document"); - return api_error::comm_error; - } + pugi::xml_document doc; + auto res = doc.load_string(response_data.c_str()); + if (res.status != pugi::xml_parse_status::status_ok) { + utils::error::raise_error(function_name, res.status, + "failed to parse xml document"); + return api_error::comm_error; + } - auto node_list = doc.select_nodes("/ListBucketResult/Contents"); - for (auto &&node : node_list) { - auto api_path = - std::string{node.node().select_node("Key").node().text().as_string()}; - if (not utils::string::ends_with(api_path, "/")) { - auto is_encrypted = - not get_config().get_s3_config().encryption_token.empty(); - if (is_encrypted) { - auto err = decrypt_object_name(api_path); + auto node_list = doc.select_nodes("/ListBucketResult/Contents"); + for (auto &&node : node_list) { + auto api_path = + std::string{node.node().select_node("Key").node().text().as_string()}; + if (not utils::string::ends_with(api_path, "/")) { + auto is_encrypted = + not get_config().get_s3_config().encryption_token.empty(); + if (is_encrypted) { + auto err = decrypt_object_name(api_path); + if (err != api_error::success) { + return err; + } + } + + auto size = node.node().select_node("Size").node().text().as_ullong(); + + api_file file{}; + file.api_path = utils::path::create_api_path(api_path); + file.api_parent = utils::path::get_parent_api_path(file.api_path); + file.accessed_date = file.changed_date = file.creation_date = + file.modified_date = + convert_api_date(node.node() + .select_node("LastModified") + .node() + .text() + .as_string()); + file.file_size = is_encrypted ? utils::encryption::encrypting_reader:: + calculate_decrypted_size(size) + : size; + file.key = is_encrypted ? utils::path::create_api_path(api_path) : ""; + auto err = add_if_not_found(file, api_path); if (err != api_error::success) { return err; } + + list.push_back(std::move(file)); } - - auto size = node.node().select_node("Size").node().text().as_ullong(); - - api_file file{}; - file.api_path = utils::path::create_api_path(api_path); - file.api_parent = utils::path::get_parent_api_path(file.api_path); - file.accessed_date = file.changed_date = file.creation_date = - file.modified_date = convert_api_date(node.node() - .select_node("LastModified") - .node() - .text() - .as_string()); - file.file_size = - is_encrypted - ? utils::encryption::encrypting_reader::calculate_decrypted_size( - size) - : size; - file.key = is_encrypted ? utils::path::create_api_path(api_path) : ""; - auto err = add_if_not_found(file, api_path); - if (err != api_error::success) { - return err; - } - - list.push_back(std::move(file)); } }