diff --git a/repertory/librepertory/include/providers/s3/s3_provider.hpp b/repertory/librepertory/include/providers/s3/s3_provider.hpp index 0fc3f27e..20804914 100644 --- a/repertory/librepertory/include/providers/s3/s3_provider.hpp +++ b/repertory/librepertory/include/providers/s3/s3_provider.hpp @@ -89,6 +89,9 @@ private: return s3_config_; } + [[nodiscard]] auto set_meta_key(const std::string &api_path, + api_meta_map &meta) -> api_error; + protected: [[nodiscard]] auto create_directory_impl(const std::string &api_path, api_meta_map &meta) diff --git a/repertory/librepertory/src/providers/s3/s3_provider.cpp b/repertory/librepertory/src/providers/s3/s3_provider.cpp index 35b4a3d2..de7ddebf 100644 --- a/repertory/librepertory/src/providers/s3/s3_provider.cpp +++ b/repertory/librepertory/src/providers/s3/s3_provider.cpp @@ -60,7 +60,7 @@ auto s3_provider::add_if_not_found(api_file &file, const std::string &object_name) const -> api_error { api_meta_map meta{}; - auto res = get_item_meta(file.api_path, meta); + auto res{get_item_meta(file.api_path, meta)}; if (res == api_error::item_not_found) { res = create_directory_paths(file.api_parent, utils::path::get_parent_api_path(object_name)); @@ -76,14 +76,15 @@ auto s3_provider::add_if_not_found(api_file &file, auto s3_provider::convert_api_date(std::string_view date) -> std::uint64_t { // 2009-10-12T17:50:30.000Z - auto date_parts = utils::string::split(date, '.', true); - auto date_time = date_parts.at(0U); - auto nanos = + auto date_parts{utils::string::split(date, '.', true)}; + auto date_time{date_parts.at(0U)}; + auto nanos{ (date_parts.size() <= 1U) ? 0U : utils::string::to_uint64( utils::string::split(date_parts.at(1U), 'Z', true).at(0U)) * - 1000000UL; + 1000000UL, + }; struct tm tm1{}; #if defined(_WIN32) @@ -101,7 +102,7 @@ auto s3_provider::create_directory_object(const std::string &api_path, -> api_error { REPERTORY_USES_FUNCTION_NAME(); - const auto &cfg = get_s3_config(); + const auto &cfg{get_s3_config()}; std::string response_data; curl::requests::http_put_file put_dir{}; @@ -113,7 +114,7 @@ auto s3_provider::create_directory_object(const std::string &api_path, response_data = std::string(data.begin(), data.end()); }; - auto res = set_request_path(put_dir, object_name + '/'); + auto res{set_request_path(put_dir, object_name + '/')}; if (res != api_error::success) { return res; } @@ -141,31 +142,14 @@ auto s3_provider::create_directory_impl(const std::string &api_path, api_meta_map &meta) -> api_error { REPERTORY_USES_FUNCTION_NAME(); - const auto &cfg = get_s3_config(); - auto is_encrypted = not cfg.encryption_token.empty(); - - if (is_encrypted) { - std::string encrypted_file_path; - auto res = get_item_meta(utils::path::get_parent_api_path(api_path), - META_KEY, encrypted_file_path); - if (res != api_error::success) { - utils::error::raise_api_path_error(function_name, api_path, res, - "failed to create file"); - return res; - } - - data_buffer result; - utils::encryption::encrypt_data( - cfg.encryption_token, - *(utils::string::split(api_path, '/', false).end() - 1U), result); - - meta[META_KEY] = utils::path::create_api_path( - utils::path::combine(utils::path::create_api_path(encrypted_file_path), - { - utils::collection::to_hex_string(result), - })); + auto res{set_meta_key(api_path, meta)}; + if (res != api_error::success) { + return res; } + const auto &cfg{get_s3_config()}; + auto is_encrypted{not cfg.encryption_token.empty()}; + return create_directory_object( api_path, utils::path::create_api_path(is_encrypted ? meta[META_KEY] : api_path)); @@ -180,12 +164,12 @@ auto s3_provider::create_directory_paths(const std::string &api_path, return api_error::success; } - const auto &cfg = get_s3_config(); - auto encryption_token = cfg.encryption_token; - auto is_encrypted = not encryption_token.empty(); + const auto &cfg{get_s3_config()}; + auto encryption_token{cfg.encryption_token}; + auto is_encrypted{not encryption_token.empty()}; - auto path_parts = utils::string::split(api_path, '/', false); - auto key_parts = utils::string::split(key, '/', false); + auto path_parts{utils::string::split(api_path, '/', false)}; + auto key_parts{utils::string::split(key, '/', false)}; if (is_encrypted && key_parts.size() != path_parts.size()) { return api_error::error; @@ -202,7 +186,7 @@ auto s3_provider::create_directory_paths(const std::string &api_path, utils::path::combine(cur_path, {path_parts.at(idx)})); std::string value; - auto res = get_item_meta(cur_path, META_DIRECTORY, value); + auto res{get_item_meta(cur_path, META_DIRECTORY, value)}; if (res == api_error::success) { if (not utils::string::to_bool(value)) { return api_error::item_exists; @@ -225,9 +209,11 @@ auto s3_provider::create_directory_paths(const std::string &api_path, } } - auto dir = create_api_file(cur_path, cur_key, 0U, - exists ? get_last_modified(true, cur_path) - : utils::time::get_time_now()); + auto dir{ + create_api_file(cur_path, cur_key, 0U, + exists ? get_last_modified(true, cur_path) + : utils::time::get_time_now()), + }; get_api_item_added()(true, dir); } @@ -241,30 +227,7 @@ auto s3_provider::create_directory_paths(const std::string &api_path, auto s3_provider::create_file_extra(const std::string &api_path, api_meta_map &meta) -> api_error { - REPERTORY_USES_FUNCTION_NAME(); - - const auto &cfg = get_s3_config(); - if (not cfg.encryption_token.empty()) { - std::string encrypted_file_path; - auto res = get_item_meta(utils::path::get_parent_api_path(api_path), - META_KEY, encrypted_file_path); - if (res != api_error::success) { - utils::error::raise_api_path_error(function_name, api_path, res, - "failed to create file"); - return res; - } - - data_buffer result; - utils::encryption::encrypt_data( - cfg.encryption_token, - *(utils::string::split(api_path, '/', false).end() - 1U), result); - - meta[META_KEY] = utils::path::create_api_path( - utils::path::combine(utils::path::create_api_path(encrypted_file_path), - {utils::collection::to_hex_string(result)})); - } - - return api_error::success; + return set_meta_key(api_path, meta); } auto s3_provider::decrypt_object_name(std::string &object_name) const @@ -282,24 +245,26 @@ auto s3_provider::get_directory_item_count(const std::string &api_path) const REPERTORY_USES_FUNCTION_NAME(); try { - const auto &cfg = get_s3_config(); - auto is_encrypted = not cfg.encryption_token.empty(); + const auto &cfg{get_s3_config()}; + auto is_encrypted{not cfg.encryption_token.empty()}; + std::string key; if (is_encrypted) { - auto res = get_item_meta(api_path, META_KEY, key); + auto res{get_item_meta(api_path, META_KEY, key)}; if (res != api_error::success) { return 0U; } } - auto object_name = + auto object_name{ api_path == "/" ? "" - : utils::path::create_api_path(is_encrypted ? key : api_path); + : utils::path::create_api_path(is_encrypted ? key : api_path), + }; std::string response_data{}; long response_code{}; - auto prefix = object_name.empty() ? object_name : object_name + "/"; + auto prefix{object_name.empty() ? object_name : object_name + "/"}; auto grab_more{true}; std::string token{}; @@ -319,7 +284,7 @@ auto s3_provider::get_directory_item_count(const std::string &api_path) const } pugi::xml_document doc; - auto res = doc.load_string(response_data.c_str()); + auto res{doc.load_string(response_data.c_str())}; if (res.status != pugi::xml_parse_status::status_ok) { return total_count; } @@ -335,8 +300,9 @@ auto s3_provider::get_directory_item_count(const std::string &api_path) const .as_string(); } - auto node_list = - doc.select_nodes("/ListBucketResult/CommonPrefixes/Prefix"); + auto node_list{ + doc.select_nodes("/ListBucketResult/CommonPrefixes/Prefix"), + }; total_count += node_list.size(); node_list = doc.select_nodes("/ListBucketResult/Contents"); @@ -390,7 +356,7 @@ auto s3_provider::get_directory_items_impl(const std::string &api_path, api_error::success) { dir_item.size = utils::string::to_uint64(size_str); } else { - auto size = node.select_node("Size").node().text().as_ullong(); + auto size{node.select_node("Size").node().text().as_ullong()}; dir_item.size = is_encrypted ? utils::encryption::encrypting_reader:: calculate_decrypted_size(size) @@ -399,8 +365,10 @@ auto s3_provider::get_directory_items_impl(const std::string &api_path, res = get_item_meta(dir_item.api_path, dir_item.meta); if (res == api_error::item_not_found) { - auto last_modified = convert_api_date( - node.select_node("LastModified").node().text().as_string()); + auto last_modified{ + convert_api_date( + node.select_node("LastModified").node().text().as_string()), + }; api_file file{}; file.api_path = dir_item.api_path; @@ -426,7 +394,7 @@ auto s3_provider::get_directory_items_impl(const std::string &api_path, std::string key; if (is_encrypted) { - auto res = get_item_meta(api_path, META_KEY, key); + auto res{get_item_meta(api_path, META_KEY, key)}; if (res != api_error::success) { return res; } @@ -462,7 +430,7 @@ auto s3_provider::get_directory_items_impl(const std::string &api_path, } pugi::xml_document doc; - auto parse_res = doc.load_string(response_data.c_str()); + auto parse_res{doc.load_string(response_data.c_str())}; if (parse_res.status != pugi::xml_parse_status::status_ok) { return api_error::error; } @@ -478,12 +446,15 @@ auto s3_provider::get_directory_items_impl(const std::string &api_path, .as_string(); } - auto node_list = - doc.select_nodes("/ListBucketResult/CommonPrefixes/Prefix"); + auto node_list{ + doc.select_nodes("/ListBucketResult/CommonPrefixes/Prefix"), + }; for (const auto &node : node_list) { - auto child_object_name = utils::path::create_api_path( - utils::path::combine("/", {node.node().text().as_string()})); - auto res = add_diretory_item(child_object_name, true, node.node()); + auto child_object_name{ + utils::path::create_api_path( + utils::path::combine("/", {node.node().text().as_string()})), + }; + auto res{add_diretory_item(child_object_name, true, node.node())}; if (res != api_error::success) { return res; } @@ -491,13 +462,15 @@ auto s3_provider::get_directory_items_impl(const std::string &api_path, node_list = doc.select_nodes("/ListBucketResult/Contents"); for (const auto &node : node_list) { - auto child_object_name = utils::path::create_api_path( - node.node().select_node("Key").node().text().as_string()); + auto child_object_name{ + utils::path::create_api_path( + node.node().select_node("Key").node().text().as_string()), + }; if (child_object_name == utils::path::create_api_path(prefix)) { continue; } - auto res = add_diretory_item(child_object_name, false, node.node()); + auto res{add_diretory_item(child_object_name, false, node.node())}; if (res != api_error::success) { return res; } @@ -515,8 +488,9 @@ auto s3_provider::get_file(const std::string &api_path, api_file &file) const bool is_encrypted{}; std::string object_name; head_object_result result{}; - auto res = - get_object_info(false, api_path, is_encrypted, object_name, result); + auto res{ + get_object_info(false, api_path, is_encrypted, object_name, result), + }; if (res != api_error::success) { return res; } @@ -566,17 +540,19 @@ auto s3_provider::get_file_list(api_file_list &list, std::string &marker) const } pugi::xml_document doc; - auto result = doc.load_string(response_data.c_str()); + auto result{doc.load_string(response_data.c_str())}; if (result.status != pugi::xml_parse_status::status_ok) { utils::error::raise_error(function_name, result.status, "failed to parse xml document"); return api_error::comm_error; } - auto grab_more = doc.select_node("/ListBucketResult/IsTruncated") - .node() - .text() - .as_bool(); + auto grab_more{ + doc.select_node("/ListBucketResult/IsTruncated") + .node() + .text() + .as_bool(), + }; if (grab_more) { marker = doc.select_node("/ListBucketResult/NextContinuationToken") .node() @@ -584,24 +560,26 @@ auto s3_provider::get_file_list(api_file_list &list, std::string &marker) const .as_string(); } - auto node_list = doc.select_nodes("/ListBucketResult/Contents"); + auto node_list{doc.select_nodes("/ListBucketResult/Contents")}; for (const auto &node : node_list) { - auto object_name = - std::string{node.node().select_node("Key").node().text().as_string()}; + auto object_name{ + std::string(node.node().select_node("Key").node().text().as_string()), + }; if (utils::string::ends_with(object_name, "/")) { continue; } auto api_path{object_name}; - auto is_encrypted = not get_s3_config().encryption_token.empty(); + auto is_encrypted{not get_s3_config().encryption_token.empty()}; + if (is_encrypted) { - auto res = decrypt_object_name(api_path); + auto res{decrypt_object_name(api_path)}; if (res != api_error::success) { return res; } } - auto size = node.node().select_node("Size").node().text().as_ullong(); + auto size{node.node().select_node("Size").node().text().as_ullong()}; api_file file{}; file.api_path = utils::path::create_api_path(api_path); @@ -618,7 +596,7 @@ auto s3_provider::get_file_list(api_file_list &list, std::string &marker) const size) : size; file.key = is_encrypted ? utils::path::create_api_path(object_name) : ""; - auto res = add_if_not_found(file, file.key); + auto res{add_if_not_found(file, file.key)}; if (res != api_error::success) { return res; } @@ -653,12 +631,12 @@ auto s3_provider::get_object_info(bool directory, const std::string &api_path, REPERTORY_USES_FUNCTION_NAME(); try { - const auto &cfg = get_s3_config(); + const auto &cfg{get_s3_config()}; is_encrypted = not cfg.encryption_token.empty(); std::string key; if (is_encrypted) { - auto res = get_item_meta(api_path, META_KEY, key); + auto res{get_item_meta(api_path, META_KEY, key)}; if (res != api_error::success) { return res; } @@ -675,8 +653,9 @@ auto s3_provider::get_object_info(bool directory, const std::string &api_path, long /*response_code*/) { response_data = std::string(data.begin(), data.end()); }; - auto res = - set_request_path(head, directory ? object_name + '/' : object_name); + auto res{ + set_request_path(head, directory ? object_name + '/' : object_name), + }; if (res != api_error::success) { return res; } @@ -755,8 +734,9 @@ auto s3_provider::is_directory(const std::string &api_path, bool &exists) const bool is_encrypted{}; std::string object_name; head_object_result result{}; - auto res = - get_object_info(true, api_path, is_encrypted, object_name, result); + auto res{ + get_object_info(true, api_path, is_encrypted, object_name, result), + }; if (res != api_error::item_not_found && res != api_error::success) { return res; } @@ -782,8 +762,9 @@ auto s3_provider::is_file(const std::string &api_path, bool &exists) const bool is_encrypted{}; std::string object_name; head_object_result result{}; - auto res = - get_object_info(false, api_path, is_encrypted, object_name, result); + auto res{ + get_object_info(false, api_path, is_encrypted, object_name, result), + }; if (res != api_error::item_not_found && res != api_error::success) { return res; } @@ -807,24 +788,26 @@ auto s3_provider::read_file_bytes(const std::string &api_path, std::size_t size, REPERTORY_USES_FUNCTION_NAME(); try { - const auto &cfg = get_s3_config(); - auto is_encrypted = not cfg.encryption_token.empty(); + const auto &cfg{get_s3_config()}; + auto is_encrypted{not cfg.encryption_token.empty()}; + std::string key; if (is_encrypted) { - auto res = get_item_meta(api_path, META_KEY, key); + auto res{get_item_meta(api_path, META_KEY, key)}; if (res != api_error::success) { return res; } } - auto object_name = - utils::path::create_api_path(is_encrypted ? key : api_path); + auto object_name{ + utils::path::create_api_path(is_encrypted ? key : api_path), + }; const auto read_bytes = [this, &api_path, &cfg, &object_name, &stop_requested](std::size_t read_size, std::size_t read_offset, data_buffer &read_buffer) -> api_error { - auto res = api_error::error; + auto res{api_error::error}; for (std::uint32_t idx = 0U; not stop_requested && res != api_error::success && idx < get_config().get_retry_read_count() + 1U; @@ -885,12 +868,12 @@ auto s3_provider::read_file_bytes(const std::string &api_path, std::size_t size, } std::string temp; - auto res = get_item_meta(api_path, META_SIZE, temp); + auto res{get_item_meta(api_path, META_SIZE, temp)}; if (res != api_error::success) { return res; } - auto total_size = utils::string::to_uint64(temp); + auto total_size{utils::string::to_uint64(temp)}; return utils::encryption::read_encrypted_range( {offset, offset + size - 1U}, utils::encryption::generate_key( @@ -916,19 +899,20 @@ auto s3_provider::remove_directory_impl(const std::string &api_path) -> api_error { REPERTORY_USES_FUNCTION_NAME(); - const auto &cfg = get_s3_config(); - auto is_encrypted = not cfg.encryption_token.empty(); + const auto &cfg{get_s3_config()}; + auto is_encrypted{not cfg.encryption_token.empty()}; std::string key; if (is_encrypted) { - auto res = get_item_meta(api_path, META_KEY, key); + auto res{get_item_meta(api_path, META_KEY, key)}; if (res != api_error::success) { return res; } } - auto object_name = - utils::path::create_api_path(is_encrypted ? key : api_path); + auto object_name{ + utils::path::create_api_path(is_encrypted ? key : api_path), + }; std::string response_data; curl::requests::http_delete del_dir{}; @@ -939,7 +923,7 @@ auto s3_provider::remove_directory_impl(const std::string &api_path) response_data = std::string(data.begin(), data.end()); }; - auto res = set_request_path(del_dir, object_name + '/'); + auto res{set_request_path(del_dir, object_name + '/')}; if (res != api_error::success) { return res; } @@ -968,19 +952,20 @@ auto s3_provider::remove_directory_impl(const std::string &api_path) auto s3_provider::remove_file_impl(const std::string &api_path) -> api_error { REPERTORY_USES_FUNCTION_NAME(); - const auto &cfg = get_s3_config(); - auto is_encrypted = not cfg.encryption_token.empty(); + const auto &cfg{get_s3_config()}; + auto is_encrypted{not cfg.encryption_token.empty()}; std::string key; if (is_encrypted) { - auto res = get_item_meta(api_path, META_KEY, key); + auto res{get_item_meta(api_path, META_KEY, key)}; if (res != api_error::success) { return res; } } - auto object_name = - utils::path::create_api_path(is_encrypted ? key : api_path); + auto object_name{ + utils::path::create_api_path(is_encrypted ? key : api_path), + }; std::string response_data; curl::requests::http_delete del_file{}; @@ -990,7 +975,7 @@ auto s3_provider::remove_file_impl(const std::string &api_path) -> api_error { long /*response_code*/) { response_data = std::string(data.begin(), data.end()); }; - auto res = set_request_path(del_file, object_name); + auto res{set_request_path(del_file, object_name)}; if (res != api_error::success) { return res; } @@ -1022,6 +1007,40 @@ auto s3_provider::rename_file(const std::string & /* from_api_path */, return api_error::not_implemented; } +auto s3_provider::set_meta_key(const std::string &api_path, api_meta_map &meta) + -> api_error { + REPERTORY_USES_FUNCTION_NAME(); + + const auto &cfg{get_s3_config()}; + auto is_encrypted{not cfg.encryption_token.empty()}; + if (not is_encrypted) { + return api_error::success; + } + + std::string encrypted_parent_path; + auto res{ + get_item_meta(utils::path::get_parent_api_path(api_path), META_KEY, + encrypted_parent_path), + }; + if (res != api_error::success) { + utils::error::raise_api_path_error(function_name, api_path, res, + "failed to create file"); + return res; + } + + data_buffer result; + utils::encryption::encrypt_data( + cfg.encryption_token, + *(utils::string::split(api_path, '/', false).end() - 1U), result); + + meta[META_KEY] = utils::path::create_api_path( + utils::path::combine(utils::path::create_api_path(encrypted_parent_path), + { + utils::collection::to_hex_string(result), + })); + return api_error::success; +} + auto s3_provider::start(api_item_added_callback api_item_added, i_file_manager *mgr) -> bool { event_system::instance().raise("s3_provider"); @@ -1043,26 +1062,27 @@ auto s3_provider::upload_file_impl(const std::string &api_path, std::uint64_t file_size{}; if (utils::file::file{source_path}.exists()) { - auto opt_size = utils::file::file{source_path}.size(); + auto opt_size{utils::file::file{source_path}.size()}; if (not opt_size.has_value()) { return api_error::comm_error; } file_size = opt_size.value(); } - const auto &cfg = get_s3_config(); - auto is_encrypted = not cfg.encryption_token.empty(); + const auto &cfg{get_s3_config()}; + auto is_encrypted{not cfg.encryption_token.empty()}; std::string key; if (is_encrypted) { - auto res = get_item_meta(api_path, META_KEY, key); + auto res{get_item_meta(api_path, META_KEY, key)}; if (res != api_error::success) { return res; } } - auto object_name = - utils::path::create_api_path(is_encrypted ? key : api_path); + auto object_name{ + utils::path::create_api_path(is_encrypted ? key : api_path), + }; std::string response_data; curl::requests::http_put_file put_file{}; @@ -1073,7 +1093,7 @@ auto s3_provider::upload_file_impl(const std::string &api_path, }; put_file.source_path = source_path; - auto res = set_request_path(put_file, object_name); + auto res{set_request_path(put_file, object_name)}; if (res != api_error::success) { return res; }