BlockStorage/repertory
1
0
Fork 0
Code Issues 8 Pull Requests Releases 6 Wiki Activity

s3 fixes

Browse Source
This commit is contained in:
Scott E. Graves 2024-11-30 17:38:24 -06:00
parent 7faf648919
commit 329ba1d5e4
1 changed files with 95 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

173
repertory/librepertory/src/providers/s3/s3_provider.cpp
View File

@ -320,95 +320,112 @@ auto s3_provider::get_directory_items_impl(
long response_code{}; long response_code{};
auto prefix = object_name.empty() ? object_name : object_name + "/"; auto prefix = object_name.empty() ? object_name : object_name + "/";
if (not get_object_list(response_data, response_code, "/", prefix)) { auto grab_more{true};
return api_error::comm_error; std::string token{};
} while (grab_more) {
if (not get_object_list(response_data, response_code, "/", prefix, token)) {
if (response_code == http_error_codes::not_found) { return api_error::comm_error;
return api_error::directory_not_found;
}
if (response_code != http_error_codes::ok) {
utils::error::raise_api_path_error(function_name, api_path, response_code,
"failed to get directory items");
return api_error::comm_error;
}
pugi::xml_document doc;
auto parse_res = doc.load_string(response_data.c_str());
if (parse_res.status != pugi::xml_parse_status::status_ok) {
return api_error::error;
}
const auto add_directory_item =
[&](bool directory, const std::string &name, std::uint64_t last_modified,
std::function<std::uint64_t(const directory_item &)> get_size)
-> api_error {
auto child_api_path =
utils::path::create_api_path(utils::path::combine("/", {name}));
std::string child_object_name;
if (is_encrypted) {
child_object_name = child_api_path;
if (not utils::encryption::decrypt_file_path(cfg.encryption_token,
child_api_path)) {
return api_error::decryption_error;
}
} }
directory_item dir_item{}; if (response_code == http_error_codes::not_found) {
dir_item.api_path = child_api_path; return api_error::directory_not_found;
dir_item.api_parent = utils::path::get_parent_api_path(dir_item.api_path); }
dir_item.directory = directory;
dir_item.size = get_size(dir_item); if (response_code != http_error_codes::ok) {
ret = get_item_meta(child_api_path, dir_item.meta); utils::error::raise_api_path_error(function_name, api_path, response_code,
if (ret == api_error::item_not_found) { "failed to get directory items");
if (directory) { return api_error::comm_error;
ret = create_path_directories(child_api_path, child_object_name); }
if (ret != api_error::success) {
return ret; pugi::xml_document doc;
} auto parse_res = doc.load_string(response_data.c_str());
} else { if (parse_res.status != pugi::xml_parse_status::status_ok) {
auto file = create_api_file(child_api_path, child_object_name, return api_error::error;
dir_item.size, last_modified); }
ret = add_if_not_found(file, child_object_name);
if (ret != api_error::success) { grab_more = doc.select_node("/ListBucketResult/IsTruncated")
return ret; .node()
.text()
.as_bool();
if (grab_more) {
token = doc.select_node("/ListBucketResult/NextContinuationToken")
.node()
.text()
.as_string();
}
const auto add_directory_item =
[&](bool directory, const std::string &name,
std::uint64_t last_modified,
std::function<std::uint64_t(const directory_item &)> get_size)
-> api_error {
auto child_api_path =
utils::path::create_api_path(utils::path::combine("/", {name}));
std::string child_object_name;
if (is_encrypted) {
child_object_name = child_api_path;
if (not utils::encryption::decrypt_file_path(cfg.encryption_token,
child_api_path)) {
return api_error::decryption_error;
} }
} }
directory_item dir_item{};
dir_item.api_path = child_api_path;
dir_item.api_parent = utils::path::get_parent_api_path(dir_item.api_path);
dir_item.directory = directory;
dir_item.size = get_size(dir_item);
ret = get_item_meta(child_api_path, dir_item.meta); ret = get_item_meta(child_api_path, dir_item.meta);
} if (ret == api_error::item_not_found) {
if (directory) {
ret = create_path_directories(child_api_path, child_object_name);
if (ret != api_error::success) {
return ret;
}
} else {
auto file = create_api_file(child_api_path, child_object_name,
dir_item.size, last_modified);
ret = add_if_not_found(file, child_object_name);
if (ret != api_error::success) {
return ret;
}
}
if (ret != api_error::success) { ret = get_item_meta(child_api_path, dir_item.meta);
return ret; }
}
list.push_back(std::move(dir_item)); if (ret != api_error::success) {
return api_error::success; return ret;
}; }
auto node_list = doc.select_nodes("/ListBucketResult/CommonPrefixes/Prefix"); list.push_back(std::move(dir_item));
for (auto &&node : node_list) { return api_error::success;
add_directory_item( };
true, node.node().text().as_string(), 0U,
[](const directory_item &) -> std::uint64_t { return 0U; });
}
node_list = doc.select_nodes("/ListBucketResult/Contents"); auto node_list =
for (auto &&node : node_list) { doc.select_nodes("/ListBucketResult/CommonPrefixes/Prefix");
auto child_object_name = utils::path::create_api_path( for (auto &&node : node_list) {
node.node().select_node("Key").node().text().as_string());
if (child_object_name != utils::path::create_api_path(prefix)) {
auto size = node.node().select_node("Size").node().text().as_ullong();
auto last_modified = convert_api_date(
node.node().select_node("LastModified").node().text().as_string());
add_directory_item( add_directory_item(
false, child_object_name, last_modified, true, node.node().text().as_string(), 0U,
[&is_encrypted, &size](const directory_item &) -> std::uint64_t { [](const directory_item &) -> std::uint64_t { return 0U; });
return is_encrypted ? utils::encryption::encrypting_reader:: }
calculate_decrypted_size(size)
: size; node_list = doc.select_nodes("/ListBucketResult/Contents");
}); for (auto &&node : node_list) {
auto child_object_name = utils::path::create_api_path(
node.node().select_node("Key").node().text().as_string());
if (child_object_name != utils::path::create_api_path(prefix)) {
auto size = node.node().select_node("Size").node().text().as_ullong();
auto last_modified = convert_api_date(
node.node().select_node("LastModified").node().text().as_string());
add_directory_item(
false, child_object_name, last_modified,
[&is_encrypted, &size](const directory_item &) -> std::uint64_t {
return is_encrypted ? utils::encryption::encrypting_reader::
calculate_decrypted_size(size)
: size;
});
}
} }
} }