BlockStorage/repertory
1
0
Fork 0
Code Issues 8 Pull Requests Releases 6 Wiki Activity

refactor

Browse Source
This commit is contained in:
Scott E. Graves
2025-01-02 07:45:08 -06:00
parent f44cc99a82
commit 1ae8b8b793
2 changed files with 157 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
repertory/librepertory/include/providers/s3/s3_provider.hpp
View File

@ -89,6 +89,9 @@ private:
return s3_config_;
}
[[nodiscard]] auto set_meta_key(const std::string &api_path,
api_meta_map &meta) -> api_error;
protected:
[[nodiscard]] auto create_directory_impl(const std::string &api_path,
api_meta_map &meta)

276
repertory/librepertory/src/providers/s3/s3_provider.cpp
View File

@ -60,7 +60,7 @@ auto s3_provider::add_if_not_found(api_file &file,
const std::string &object_name) const
-> api_error {
api_meta_map meta{};
auto res = get_item_meta(file.api_path, meta);
auto res{get_item_meta(file.api_path, meta)};
if (res == api_error::item_not_found) {
res = create_directory_paths(file.api_parent,
utils::path::get_parent_api_path(object_name));
@ -76,14 +76,15 @@ auto s3_provider::add_if_not_found(api_file &file,
auto s3_provider::convert_api_date(std::string_view date) -> std::uint64_t {
// 2009-10-12T17:50:30.000Z
auto date_parts = utils::string::split(date, '.', true);
auto date_time = date_parts.at(0U);
auto nanos =
auto date_parts{utils::string::split(date, '.', true)};
auto date_time{date_parts.at(0U)};
auto nanos{
(date_parts.size() <= 1U)
? 0U
: utils::string::to_uint64(
utils::string::split(date_parts.at(1U), 'Z', true).at(0U)) *
1000000UL;
1000000UL,
};
struct tm tm1{};
#if defined(_WIN32)
@ -101,7 +102,7 @@ auto s3_provider::create_directory_object(const std::string &api_path,
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
const auto &cfg = get_s3_config();
const auto &cfg{get_s3_config()};
std::string response_data;
curl::requests::http_put_file put_dir{};
@ -113,7 +114,7 @@ auto s3_provider::create_directory_object(const std::string &api_path,
response_data = std::string(data.begin(), data.end());
};
auto res = set_request_path(put_dir, object_name + '/');
auto res{set_request_path(put_dir, object_name + '/')};
if (res != api_error::success) {
return res;
}
@ -141,30 +142,13 @@ auto s3_provider::create_directory_impl(const std::string &api_path,
api_meta_map &meta) -> api_error {
REPERTORY_USES_FUNCTION_NAME();
const auto &cfg = get_s3_config();
auto is_encrypted = not cfg.encryption_token.empty();
if (is_encrypted) {
std::string encrypted_file_path;
auto res = get_item_meta(utils::path::get_parent_api_path(api_path),
META_KEY, encrypted_file_path);
auto res{set_meta_key(api_path, meta)};
if (res != api_error::success) {
utils::error::raise_api_path_error(function_name, api_path, res,
"failed to create file");
return res;
}
data_buffer result;
utils::encryption::encrypt_data(
cfg.encryption_token,
*(utils::string::split(api_path, '/', false).end() - 1U), result);
meta[META_KEY] = utils::path::create_api_path(
utils::path::combine(utils::path::create_api_path(encrypted_file_path),
{
utils::collection::to_hex_string(result),
}));
}
const auto &cfg{get_s3_config()};
auto is_encrypted{not cfg.encryption_token.empty()};
return create_directory_object(
api_path,
@ -180,12 +164,12 @@ auto s3_provider::create_directory_paths(const std::string &api_path,
return api_error::success;
}
const auto &cfg = get_s3_config();
auto encryption_token = cfg.encryption_token;
auto is_encrypted = not encryption_token.empty();
const auto &cfg{get_s3_config()};
auto encryption_token{cfg.encryption_token};
auto is_encrypted{not encryption_token.empty()};
auto path_parts = utils::string::split(api_path, '/', false);
auto key_parts = utils::string::split(key, '/', false);
auto path_parts{utils::string::split(api_path, '/', false)};
auto key_parts{utils::string::split(key, '/', false)};
if (is_encrypted && key_parts.size() != path_parts.size()) {
return api_error::error;
@ -202,7 +186,7 @@ auto s3_provider::create_directory_paths(const std::string &api_path,
utils::path::combine(cur_path, {path_parts.at(idx)}));
std::string value;
auto res = get_item_meta(cur_path, META_DIRECTORY, value);
auto res{get_item_meta(cur_path, META_DIRECTORY, value)};
if (res == api_error::success) {
if (not utils::string::to_bool(value)) {
return api_error::item_exists;
@ -225,9 +209,11 @@ auto s3_provider::create_directory_paths(const std::string &api_path,
}
}
auto dir = create_api_file(cur_path, cur_key, 0U,
auto dir{
create_api_file(cur_path, cur_key, 0U,
exists ? get_last_modified(true, cur_path)
: utils::time::get_time_now());
: utils::time::get_time_now()),
};
get_api_item_added()(true, dir);
}
@ -241,30 +227,7 @@ auto s3_provider::create_directory_paths(const std::string &api_path,
auto s3_provider::create_file_extra(const std::string &api_path,
api_meta_map &meta) -> api_error {
REPERTORY_USES_FUNCTION_NAME();
const auto &cfg = get_s3_config();
if (not cfg.encryption_token.empty()) {
std::string encrypted_file_path;
auto res = get_item_meta(utils::path::get_parent_api_path(api_path),
META_KEY, encrypted_file_path);
if (res != api_error::success) {
utils::error::raise_api_path_error(function_name, api_path, res,
"failed to create file");
return res;
}
data_buffer result;
utils::encryption::encrypt_data(
cfg.encryption_token,
*(utils::string::split(api_path, '/', false).end() - 1U), result);
meta[META_KEY] = utils::path::create_api_path(
utils::path::combine(utils::path::create_api_path(encrypted_file_path),
{utils::collection::to_hex_string(result)}));
}
return api_error::success;
return set_meta_key(api_path, meta);
}
auto s3_provider::decrypt_object_name(std::string &object_name) const
@ -282,24 +245,26 @@ auto s3_provider::get_directory_item_count(const std::string &api_path) const
REPERTORY_USES_FUNCTION_NAME();
try {
const auto &cfg = get_s3_config();
auto is_encrypted = not cfg.encryption_token.empty();
const auto &cfg{get_s3_config()};
auto is_encrypted{not cfg.encryption_token.empty()};
std::string key;
if (is_encrypted) {
auto res = get_item_meta(api_path, META_KEY, key);
auto res{get_item_meta(api_path, META_KEY, key)};
if (res != api_error::success) {
return 0U;
}
}
auto object_name =
auto object_name{
api_path == "/"
? ""
: utils::path::create_api_path(is_encrypted ? key : api_path);
: utils::path::create_api_path(is_encrypted ? key : api_path),
};
std::string response_data{};
long response_code{};
auto prefix = object_name.empty() ? object_name : object_name + "/";
auto prefix{object_name.empty() ? object_name : object_name + "/"};
auto grab_more{true};
std::string token{};
@ -319,7 +284,7 @@ auto s3_provider::get_directory_item_count(const std::string &api_path) const
}
pugi::xml_document doc;
auto res = doc.load_string(response_data.c_str());
auto res{doc.load_string(response_data.c_str())};
if (res.status != pugi::xml_parse_status::status_ok) {
return total_count;
}
@ -335,8 +300,9 @@ auto s3_provider::get_directory_item_count(const std::string &api_path) const
.as_string();
}
auto node_list =
doc.select_nodes("/ListBucketResult/CommonPrefixes/Prefix");
auto node_list{
doc.select_nodes("/ListBucketResult/CommonPrefixes/Prefix"),
};
total_count += node_list.size();
node_list = doc.select_nodes("/ListBucketResult/Contents");
@ -390,7 +356,7 @@ auto s3_provider::get_directory_items_impl(const std::string &api_path,
api_error::success) {
dir_item.size = utils::string::to_uint64(size_str);
} else {
auto size = node.select_node("Size").node().text().as_ullong();
auto size{node.select_node("Size").node().text().as_ullong()};
dir_item.size = is_encrypted ? utils::encryption::encrypting_reader::
calculate_decrypted_size(size)
@ -399,8 +365,10 @@ auto s3_provider::get_directory_items_impl(const std::string &api_path,
res = get_item_meta(dir_item.api_path, dir_item.meta);
if (res == api_error::item_not_found) {
auto last_modified = convert_api_date(
node.select_node("LastModified").node().text().as_string());
auto last_modified{
convert_api_date(
node.select_node("LastModified").node().text().as_string()),
};
api_file file{};
file.api_path = dir_item.api_path;
@ -426,7 +394,7 @@ auto s3_provider::get_directory_items_impl(const std::string &api_path,
std::string key;
if (is_encrypted) {
auto res = get_item_meta(api_path, META_KEY, key);
auto res{get_item_meta(api_path, META_KEY, key)};
if (res != api_error::success) {
return res;
}
@ -462,7 +430,7 @@ auto s3_provider::get_directory_items_impl(const std::string &api_path,
}
pugi::xml_document doc;
auto parse_res = doc.load_string(response_data.c_str());
auto parse_res{doc.load_string(response_data.c_str())};
if (parse_res.status != pugi::xml_parse_status::status_ok) {
return api_error::error;
}
@ -478,12 +446,15 @@ auto s3_provider::get_directory_items_impl(const std::string &api_path,
.as_string();
}
auto node_list =
doc.select_nodes("/ListBucketResult/CommonPrefixes/Prefix");
auto node_list{
doc.select_nodes("/ListBucketResult/CommonPrefixes/Prefix"),
};
for (const auto &node : node_list) {
auto child_object_name = utils::path::create_api_path(
utils::path::combine("/", {node.node().text().as_string()}));
auto res = add_diretory_item(child_object_name, true, node.node());
auto child_object_name{
utils::path::create_api_path(
utils::path::combine("/", {node.node().text().as_string()})),
};
auto res{add_diretory_item(child_object_name, true, node.node())};
if (res != api_error::success) {
return res;
}
@ -491,13 +462,15 @@ auto s3_provider::get_directory_items_impl(const std::string &api_path,
node_list = doc.select_nodes("/ListBucketResult/Contents");
for (const auto &node : node_list) {
auto child_object_name = utils::path::create_api_path(
node.node().select_node("Key").node().text().as_string());
auto child_object_name{
utils::path::create_api_path(
node.node().select_node("Key").node().text().as_string()),
};
if (child_object_name == utils::path::create_api_path(prefix)) {
continue;
}
auto res = add_diretory_item(child_object_name, false, node.node());
auto res{add_diretory_item(child_object_name, false, node.node())};
if (res != api_error::success) {
return res;
}
@ -515,8 +488,9 @@ auto s3_provider::get_file(const std::string &api_path, api_file &file) const
bool is_encrypted{};
std::string object_name;
head_object_result result{};
auto res =
get_object_info(false, api_path, is_encrypted, object_name, result);
auto res{
get_object_info(false, api_path, is_encrypted, object_name, result),
};
if (res != api_error::success) {
return res;
}
@ -566,17 +540,19 @@ auto s3_provider::get_file_list(api_file_list &list, std::string &marker) const
}
pugi::xml_document doc;
auto result = doc.load_string(response_data.c_str());
auto result{doc.load_string(response_data.c_str())};
if (result.status != pugi::xml_parse_status::status_ok) {
utils::error::raise_error(function_name, result.status,
"failed to parse xml document");
return api_error::comm_error;
}
auto grab_more = doc.select_node("/ListBucketResult/IsTruncated")
auto grab_more{
doc.select_node("/ListBucketResult/IsTruncated")
.node()
.text()
.as_bool();
.as_bool(),
};
if (grab_more) {
marker = doc.select_node("/ListBucketResult/NextContinuationToken")
.node()
@ -584,24 +560,26 @@ auto s3_provider::get_file_list(api_file_list &list, std::string &marker) const
.as_string();
}
auto node_list = doc.select_nodes("/ListBucketResult/Contents");
auto node_list{doc.select_nodes("/ListBucketResult/Contents")};
for (const auto &node : node_list) {
auto object_name =
std::string{node.node().select_node("Key").node().text().as_string()};
auto object_name{
std::string(node.node().select_node("Key").node().text().as_string()),
};
if (utils::string::ends_with(object_name, "/")) {
continue;
}
auto api_path{object_name};
auto is_encrypted = not get_s3_config().encryption_token.empty();
auto is_encrypted{not get_s3_config().encryption_token.empty()};
if (is_encrypted) {
auto res = decrypt_object_name(api_path);
auto res{decrypt_object_name(api_path)};
if (res != api_error::success) {
return res;
}
}
auto size = node.node().select_node("Size").node().text().as_ullong();
auto size{node.node().select_node("Size").node().text().as_ullong()};
api_file file{};
file.api_path = utils::path::create_api_path(api_path);
@ -618,7 +596,7 @@ auto s3_provider::get_file_list(api_file_list &list, std::string &marker) const
size)
: size;
file.key = is_encrypted ? utils::path::create_api_path(object_name) : "";
auto res = add_if_not_found(file, file.key);
auto res{add_if_not_found(file, file.key)};
if (res != api_error::success) {
return res;
}
@ -653,12 +631,12 @@ auto s3_provider::get_object_info(bool directory, const std::string &api_path,
REPERTORY_USES_FUNCTION_NAME();
try {
const auto &cfg = get_s3_config();
const auto &cfg{get_s3_config()};
is_encrypted = not cfg.encryption_token.empty();
std::string key;
if (is_encrypted) {
auto res = get_item_meta(api_path, META_KEY, key);
auto res{get_item_meta(api_path, META_KEY, key)};
if (res != api_error::success) {
return res;
}
@ -675,8 +653,9 @@ auto s3_provider::get_object_info(bool directory, const std::string &api_path,
long /*response_code*/) {
response_data = std::string(data.begin(), data.end());
};
auto res =
set_request_path(head, directory ? object_name + '/' : object_name);
auto res{
set_request_path(head, directory ? object_name + '/' : object_name),
};
if (res != api_error::success) {
return res;
}
@ -755,8 +734,9 @@ auto s3_provider::is_directory(const std::string &api_path, bool &exists) const
bool is_encrypted{};
std::string object_name;
head_object_result result{};
auto res =
get_object_info(true, api_path, is_encrypted, object_name, result);
auto res{
get_object_info(true, api_path, is_encrypted, object_name, result),
};
if (res != api_error::item_not_found && res != api_error::success) {
return res;
}
@ -782,8 +762,9 @@ auto s3_provider::is_file(const std::string &api_path, bool &exists) const
bool is_encrypted{};
std::string object_name;
head_object_result result{};
auto res =
get_object_info(false, api_path, is_encrypted, object_name, result);
auto res{
get_object_info(false, api_path, is_encrypted, object_name, result),
};
if (res != api_error::item_not_found && res != api_error::success) {
return res;
}
@ -807,24 +788,26 @@ auto s3_provider::read_file_bytes(const std::string &api_path, std::size_t size,
REPERTORY_USES_FUNCTION_NAME();
try {
const auto &cfg = get_s3_config();
auto is_encrypted = not cfg.encryption_token.empty();
const auto &cfg{get_s3_config()};
auto is_encrypted{not cfg.encryption_token.empty()};
std::string key;
if (is_encrypted) {
auto res = get_item_meta(api_path, META_KEY, key);
auto res{get_item_meta(api_path, META_KEY, key)};
if (res != api_error::success) {
return res;
}
}
auto object_name =
utils::path::create_api_path(is_encrypted ? key : api_path);
auto object_name{
utils::path::create_api_path(is_encrypted ? key : api_path),
};
const auto read_bytes =
[this, &api_path, &cfg, &object_name,
&stop_requested](std::size_t read_size, std::size_t read_offset,
data_buffer &read_buffer) -> api_error {
auto res = api_error::error;
auto res{api_error::error};
for (std::uint32_t idx = 0U;
not stop_requested && res != api_error::success &&
idx < get_config().get_retry_read_count() + 1U;
@ -885,12 +868,12 @@ auto s3_provider::read_file_bytes(const std::string &api_path, std::size_t size,
}
std::string temp;
auto res = get_item_meta(api_path, META_SIZE, temp);
auto res{get_item_meta(api_path, META_SIZE, temp)};
if (res != api_error::success) {
return res;
}
auto total_size = utils::string::to_uint64(temp);
auto total_size{utils::string::to_uint64(temp)};
return utils::encryption::read_encrypted_range(
{offset, offset + size - 1U},
utils::encryption::generate_key<utils::encryption::hash_256_t>(
@ -916,19 +899,20 @@ auto s3_provider::remove_directory_impl(const std::string &api_path)
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
const auto &cfg = get_s3_config();
auto is_encrypted = not cfg.encryption_token.empty();
const auto &cfg{get_s3_config()};
auto is_encrypted{not cfg.encryption_token.empty()};
std::string key;
if (is_encrypted) {
auto res = get_item_meta(api_path, META_KEY, key);
auto res{get_item_meta(api_path, META_KEY, key)};
if (res != api_error::success) {
return res;
}
}
auto object_name =
utils::path::create_api_path(is_encrypted ? key : api_path);
auto object_name{
utils::path::create_api_path(is_encrypted ? key : api_path),
};
std::string response_data;
curl::requests::http_delete del_dir{};
@ -939,7 +923,7 @@ auto s3_provider::remove_directory_impl(const std::string &api_path)
response_data = std::string(data.begin(), data.end());
};
auto res = set_request_path(del_dir, object_name + '/');
auto res{set_request_path(del_dir, object_name + '/')};
if (res != api_error::success) {
return res;
}
@ -968,19 +952,20 @@ auto s3_provider::remove_directory_impl(const std::string &api_path)
auto s3_provider::remove_file_impl(const std::string &api_path) -> api_error {
REPERTORY_USES_FUNCTION_NAME();
const auto &cfg = get_s3_config();
auto is_encrypted = not cfg.encryption_token.empty();
const auto &cfg{get_s3_config()};
auto is_encrypted{not cfg.encryption_token.empty()};
std::string key;
if (is_encrypted) {
auto res = get_item_meta(api_path, META_KEY, key);
auto res{get_item_meta(api_path, META_KEY, key)};
if (res != api_error::success) {
return res;
}
}
auto object_name =
utils::path::create_api_path(is_encrypted ? key : api_path);
auto object_name{
utils::path::create_api_path(is_encrypted ? key : api_path),
};
std::string response_data;
curl::requests::http_delete del_file{};
@ -990,7 +975,7 @@ auto s3_provider::remove_file_impl(const std::string &api_path) -> api_error {
long /*response_code*/) {
response_data = std::string(data.begin(), data.end());
};
auto res = set_request_path(del_file, object_name);
auto res{set_request_path(del_file, object_name)};
if (res != api_error::success) {
return res;
}
@ -1022,6 +1007,40 @@ auto s3_provider::rename_file(const std::string & /* from_api_path */,
return api_error::not_implemented;
}
auto s3_provider::set_meta_key(const std::string &api_path, api_meta_map &meta)
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
const auto &cfg{get_s3_config()};
auto is_encrypted{not cfg.encryption_token.empty()};
if (not is_encrypted) {
return api_error::success;
}
std::string encrypted_parent_path;
auto res{
get_item_meta(utils::path::get_parent_api_path(api_path), META_KEY,
encrypted_parent_path),
};
if (res != api_error::success) {
utils::error::raise_api_path_error(function_name, api_path, res,
"failed to create file");
return res;
}
data_buffer result;
utils::encryption::encrypt_data(
cfg.encryption_token,
*(utils::string::split(api_path, '/', false).end() - 1U), result);
meta[META_KEY] = utils::path::create_api_path(
utils::path::combine(utils::path::create_api_path(encrypted_parent_path),
{
utils::collection::to_hex_string(result),
}));
return api_error::success;
}
auto s3_provider::start(api_item_added_callback api_item_added,
i_file_manager *mgr) -> bool {
event_system::instance().raise<service_started>("s3_provider");
@ -1043,26 +1062,27 @@ auto s3_provider::upload_file_impl(const std::string &api_path,
std::uint64_t file_size{};
if (utils::file::file{source_path}.exists()) {
auto opt_size = utils::file::file{source_path}.size();
auto opt_size{utils::file::file{source_path}.size()};
if (not opt_size.has_value()) {
return api_error::comm_error;
}
file_size = opt_size.value();
}
const auto &cfg = get_s3_config();
auto is_encrypted = not cfg.encryption_token.empty();
const auto &cfg{get_s3_config()};
auto is_encrypted{not cfg.encryption_token.empty()};
std::string key;
if (is_encrypted) {
auto res = get_item_meta(api_path, META_KEY, key);
auto res{get_item_meta(api_path, META_KEY, key)};
if (res != api_error::success) {
return res;
}
}
auto object_name =
utils::path::create_api_path(is_encrypted ? key : api_path);
auto object_name{
utils::path::create_api_path(is_encrypted ? key : api_path),
};
std::string response_data;
curl::requests::http_put_file put_file{};
@ -1073,7 +1093,7 @@ auto s3_provider::upload_file_impl(const std::string &api_path,
};
put_file.source_path = source_path;
auto res = set_request_path(put_file, object_name);
auto res{set_request_path(put_file, object_name)};
if (res != api_error::success) {
return res;
}