From 57213c007f846b43e2aca2e51539675f16afda1e Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Fri, 29 Aug 2025 09:03:16 -0500 Subject: [PATCH] prepare for argon2id --- .../include/providers/s3/s3_provider.hpp | 5 + .../librepertory/include/types/repertory.hpp | 11 +- .../src/providers/s3/s3_provider.cpp | 278 ++++++++++-------- 3 files changed, 171 insertions(+), 123 deletions(-) diff --git a/repertory/librepertory/include/providers/s3/s3_provider.hpp b/repertory/librepertory/include/providers/s3/s3_provider.hpp index 0bde9a7f..b937abda 100644 --- a/repertory/librepertory/include/providers/s3/s3_provider.hpp +++ b/repertory/librepertory/include/providers/s3/s3_provider.hpp @@ -90,6 +90,11 @@ private: return s3_config_; } + [[nodiscard]] auto read_file_bytes(const std::string &api_path, + std::size_t size, std::uint64_t offset, + data_buffer &data, bool encrypted, + stop_type &stop_requested) -> api_error; + [[nodiscard]] auto set_meta_key(const std::string &api_path, api_meta_map &meta) -> api_error; diff --git a/repertory/librepertory/include/types/repertory.hpp b/repertory/librepertory/include/types/repertory.hpp index 84a860df..20d0153f 100644 --- a/repertory/librepertory/include/types/repertory.hpp +++ b/repertory/librepertory/include/types/repertory.hpp @@ -68,6 +68,7 @@ inline constexpr std::string META_CHANGED{"changed"}; inline constexpr std::string META_CREATION{"creation"}; inline constexpr std::string META_DIRECTORY{"directory"}; inline constexpr std::string META_GID{"gid"}; +inline constexpr std::string META_KDF{"kdf"}; inline constexpr std::string META_KEY{"key"}; inline constexpr std::string META_MODE{"mode"}; inline constexpr std::string META_MODIFIED{"modified"}; @@ -78,11 +79,11 @@ inline constexpr std::string META_SOURCE{"source"}; inline constexpr std::string META_UID{"uid"}; inline constexpr std::string META_WRITTEN{"written"}; -inline constexpr std::array META_USED_NAMES = { - META_ACCESSED, META_ATTRIBUTES, META_BACKUP, META_CHANGED, - META_CREATION, META_DIRECTORY, META_GID, META_KEY, - META_MODE, META_MODIFIED, META_OSXFLAGS, META_PINNED, - META_SIZE, META_SOURCE, META_UID, META_WRITTEN, +inline constexpr std::array META_USED_NAMES = { + META_ACCESSED, META_ATTRIBUTES, META_BACKUP, META_CHANGED, META_CREATION, + META_DIRECTORY, META_GID, META_KDF, META_KEY, META_MODE, + META_MODIFIED, META_OSXFLAGS, META_PINNED, META_SIZE, META_SOURCE, + META_UID, META_WRITTEN, }; using api_meta_map = std::map; diff --git a/repertory/librepertory/src/providers/s3/s3_provider.cpp b/repertory/librepertory/src/providers/s3/s3_provider.cpp index e93279a9..bf6928ba 100644 --- a/repertory/librepertory/src/providers/s3/s3_provider.cpp +++ b/repertory/librepertory/src/providers/s3/s3_provider.cpp @@ -815,124 +815,6 @@ auto s3_provider::is_online() const -> bool { return false; } -auto s3_provider::read_file_bytes(const std::string &api_path, std::size_t size, - std::uint64_t offset, data_buffer &data, - stop_type &stop_requested) -> api_error { - REPERTORY_USES_FUNCTION_NAME(); - - try { - const auto &cfg{get_s3_config()}; - auto is_encrypted{not cfg.encryption_token.empty()}; - - std::string key; - if (is_encrypted) { - auto res{get_item_meta(api_path, META_KEY, key)}; - if (res != api_error::success) { - return res; - } - } - - auto object_name{ - utils::path::create_api_path(is_encrypted ? key : api_path), - }; - - const auto read_bytes = - [this, &api_path, &cfg, &object_name, - &stop_requested](std::size_t read_size, std::size_t read_offset, - data_buffer &read_buffer) -> api_error { - auto res{api_error::error}; - for (std::uint32_t idx{0U}; - not(stop_requested || app_config::get_stop_requested()) && - res != api_error::success && - idx < get_config().get_retry_read_count() + 1U; - ++idx) { - if (idx > 0U) { - read_buffer.clear(); - - std::this_thread::sleep_for(1s); - } - - curl::requests::http_get get{}; - get.aws_service = "aws:amz:" + cfg.region + ":s3"; - get.headers["response-content-type"] = "binary/octet-stream"; - get.range = {{ - .begin = read_offset, - .end = read_offset + read_size - 1U, - }}; - get.response_handler = [&read_buffer](auto &&response_data, - long /*response_code*/) { - read_buffer = response_data; - }; - - res = set_request_path(get, object_name); - if (res != api_error::success) { - return res; - } - - const auto notify_retry = [=](long response_code) { - auto msg = - fmt::format("read file bytes failed|offset|{}|size|{}|retry|{}", - std::to_string(read_offset), - std::to_string(read_size), std::to_string(idx + 1U)); - if (response_code == 0) { - utils::error::raise_api_path_error(function_name, api_path, - api_error::comm_error, msg); - } else { - utils::error::raise_api_path_error(function_name, api_path, - response_code, msg); - } - }; - - long response_code{}; - if (not get_comm().make_request(get, response_code, stop_requested)) { - notify_retry(response_code); - continue; - } - - if (response_code < http_error_codes::ok || - response_code >= http_error_codes::multiple_choices) { - notify_retry(response_code); - continue; - } - - res = api_error::success; - } - - return res; - }; - - if (not is_encrypted) { - return read_bytes(size, offset, data); - } - - std::string temp; - auto res{get_item_meta(api_path, META_SIZE, temp)}; - if (res != api_error::success) { - return res; - } - - auto total_size{utils::string::to_uint64(temp)}; - return utils::encryption::read_encrypted_range( - {.begin = offset, .end = offset + size - 1U}, - utils::encryption::generate_key( - cfg.encryption_token), - [&](data_buffer &ct_buffer, std::uint64_t start_offset, - std::uint64_t end_offset) -> bool { - return read_bytes((end_offset - start_offset + 1U), - start_offset, - ct_buffer) == api_error::success; - }, - total_size, data) - ? api_error::success - : api_error::decryption_error; - - } catch (const std::exception &e) { - utils::error::raise_error(function_name, e, "exception occurred"); - } - - return api_error::error; -} - auto s3_provider::remove_directory_impl(const std::string &api_path) -> api_error { REPERTORY_USES_FUNCTION_NAME(); @@ -1085,7 +967,40 @@ auto s3_provider::start(api_item_added_callback api_item_added, event_system::instance().raise(function_name, "s3_provider"); + const auto &cfg{get_s3_config()}; + auto ret = base_provider::start(api_item_added, mgr); + if (ret && not cfg.encryption_token.empty()) { + std::string kdf_str; + auto res = get_item_meta("/", META_KDF, kdf_str); + ret == res == api_error::success; + if (ret) { + if (kdf_str.empty()) { + if (not search_keys_for_kdf()) { + kdf_.seal(); + res = set_item_meta("/", META_KDF, nlohmann::json(kdf_).dump()); + ret == res == api_error::success; + if (not ret) { + utils::error::raise_api_path_error(function_name, "/", res, + "set item meta failed"); + } + } + } else { + kdf_ = nlohmann::json::parse(kdf_str).get(); + } + } else { + utils::error::raise_api_path_error(function_name, "/", res, + "get item meta failed"); + } + + if (ret) { + key_ = utils::encryption::recreate_key( + cfg.encryption_token, kdf); + } else { + base_provider::stop(); + } + } + event_system::instance().raise(function_name, "s3_provider"); return ret; @@ -1165,4 +1080,131 @@ auto s3_provider::upload_file_impl(const std::string &api_path, return api_error::success; } + +auto s3_provider::read_file_bytes(const std::string &api_path, std::size_t size, + std::uint64_t offset, data_buffer &data, + stop_type &stop_requested) -> api_error { + return read_file_bytes(api_path, size, offset, data, + not get_s3_config().encryption_token.empty(), + stop_requested); +} + +auto s3_provider::read_file_bytes(const std::string &api_path, std::size_t size, + std::uint64_t offset, data_buffer &data, + bool encrypted, stop_type &stop_requested) + -> api_error { + REPERTORY_USES_FUNCTION_NAME(); + + try { + const auto &cfg{get_s3_config()}; + + std::string key; + if (encrypted) { + auto res{get_item_meta(api_path, META_KEY, key)}; + if (res != api_error::success) { + return res; + } + } + + auto object_name{ + utils::path::create_api_path(encrypted ? key : api_path), + }; + + const auto read_bytes = + [this, &api_path, &cfg, &object_name, + &stop_requested](std::size_t read_size, std::size_t read_offset, + data_buffer &read_buffer) -> api_error { + auto res{api_error::error}; + for (std::uint32_t idx{0U}; + not(stop_requested || app_config::get_stop_requested()) && + res != api_error::success && + idx < get_config().get_retry_read_count() + 1U; + ++idx) { + if (idx > 0U) { + read_buffer.clear(); + + std::this_thread::sleep_for(1s); + } + + curl::requests::http_get get{}; + get.aws_service = "aws:amz:" + cfg.region + ":s3"; + get.headers["response-content-type"] = "binary/octet-stream"; + get.range = {{ + .begin = read_offset, + .end = read_offset + read_size - 1U, + }}; + get.response_handler = [&read_buffer](auto &&response_data, + long /*response_code*/) { + read_buffer = response_data; + }; + + res = set_request_path(get, object_name); + if (res != api_error::success) { + return res; + } + + const auto notify_retry = [=](long response_code) { + auto msg = + fmt::format("read file bytes failed|offset|{}|size|{}|retry|{}", + std::to_string(read_offset), + std::to_string(read_size), std::to_string(idx + 1U)); + if (response_code == 0) { + utils::error::raise_api_path_error(function_name, api_path, + api_error::comm_error, msg); + } else { + utils::error::raise_api_path_error(function_name, api_path, + response_code, msg); + } + }; + + long response_code{}; + if (not get_comm().make_request(get, response_code, stop_requested)) { + notify_retry(response_code); + continue; + } + + if (response_code < http_error_codes::ok || + response_code >= http_error_codes::multiple_choices) { + notify_retry(response_code); + continue; + } + + res = api_error::success; + } + + return res; + }; + + if (not encrypted) { + return read_bytes(size, offset, data); + } + + std::string temp; + auto res{get_item_meta(api_path, META_SIZE, temp)}; + if (res != api_error::success) { + return res; + } + + auto total_size{utils::string::to_uint64(temp)}; + return utils::encryption::read_encrypted_range( + {.begin = offset, .end = offset + size - 1U}, + utils::encryption::generate_key( + cfg.encryption_token), + [&](data_buffer &ct_buffer, std::uint64_t start_offset, + std::uint64_t end_offset) -> bool { + return read_bytes((end_offset - start_offset + 1U), + start_offset, + ct_buffer) == api_error::success; + }, + total_size, data) + ? api_error::success + : api_error::decryption_error; + + } catch (const std::exception &e) { + utils::error::raise_error(function_name, e, "exception occurred"); + } + + return api_error::error; +} + } // namespace repertory