From 62194271c08e2f223300404e8e1cd4b0ba3c2614 Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Fri, 29 Aug 2025 16:43:50 -0500 Subject: [PATCH] broken build --- .../include/providers/s3/s3_provider.hpp | 12 +- .../src/providers/s3/s3_provider.cpp | 194 +++++++++++++++--- 2 files changed, 180 insertions(+), 26 deletions(-) diff --git a/repertory/librepertory/include/providers/s3/s3_provider.hpp b/repertory/librepertory/include/providers/s3/s3_provider.hpp index b937abda..2d09c7f2 100644 --- a/repertory/librepertory/include/providers/s3/s3_provider.hpp +++ b/repertory/librepertory/include/providers/s3/s3_provider.hpp @@ -24,6 +24,8 @@ #include "providers/base_provider.hpp" #include "types/repertory.hpp" +#include "utils/encryption.hpp" +#include "utils/hash.hpp" namespace repertory { class app_config; @@ -48,6 +50,9 @@ public: private: s3_config s3_config_; + bool legacy_bucket_{true}; + utils::encryption::kdf_config master_kdf_cfg_; + utils::hash::hash_256_t master_key_; private: [[nodiscard]] auto add_if_not_found(api_file &file, @@ -66,8 +71,8 @@ private: api_meta_map &meta) -> api_error override; - [[nodiscard]] auto decrypt_object_name(std::string &object_name) const - -> api_error; + [[nodiscard]] auto decrypt_object_name(std::string &object_name, + bool &uses_kdf) const -> api_error; [[nodiscard]] auto get_last_modified(bool directory, const std::string &api_path, @@ -95,6 +100,9 @@ private: data_buffer &data, bool encrypted, stop_type &stop_requested) -> api_error; + [[nodiscard]] auto search_keys_for_kdf(std::string_view encryption_token) + -> bool; + [[nodiscard]] auto set_meta_key(const std::string &api_path, api_meta_map &meta) -> api_error; diff --git a/repertory/librepertory/src/providers/s3/s3_provider.cpp b/repertory/librepertory/src/providers/s3/s3_provider.cpp index 101517a5..a9eeb0a8 100644 --- a/repertory/librepertory/src/providers/s3/s3_provider.cpp +++ b/repertory/librepertory/src/providers/s3/s3_provider.cpp @@ -927,6 +927,92 @@ auto s3_provider::rename_file(const std::string & /* from_api_path */, return api_error::not_implemented; } +auto s3_provider::search_keys_for_kdf(std::string_view encryption_token) + -> bool { + REPERTORY_USES_FUNCTION_NAME(); + + auto grab_more{true}; + std::string token{}; + while (grab_more) { + std::string response_data{}; + long response_code{}; + if (not get_object_list(response_data, response_code, "/", prefix, token)) { + throw utils::error::create_exception(function_name, + {"failed to get object list"}); + } + + if (response_code == http_error_codes::not_found) { + throw utils::error::create_exception(function_name, + {"failed to get object list"}); + } + + if (response_code != http_error_codes::ok) { + throw utils::error::create_exception(function_name, + {"failed to get object list"}); + } + + pugi::xml_document doc; + auto parse_res{doc.load_string(response_data.c_str())}; + if (parse_res.status != pugi::xml_parse_status::status_ok) { + throw utils::error::create_exception(function_name, + {"failed to get object list"}); + } + + grab_more = doc.select_node("/ListBucketResult/IsTruncated") + .node() + .text() + .as_bool(); + if (grab_more) { + token = doc.select_node("/ListBucketResult/NextContinuationToken") + .node() + .text() + .as_string(); + } + + node_list = doc.select_nodes("/ListBucketResult/Contents"); + for (const auto &node : node_list) { + auto object_name{ + node.node().select_node("Key").node().text().as_string()), + }; + if (object_name == "/")) { + continue; + } + + data_buffer buffer; + if (not utils::collection::from_hex_string(object_name, buffer)) { + continue; + } + + kdf_config cfg; + if (not kdf_config::from_header(buffer, cfg)) { + continue; + } + + cfg.unique_id = 0U; + cfg.seal(); + + master_kdf_cfg_ = cfg; + if (not utils::encryption::recreate_key_argon2id( + encryption_token, master_kdf_cfg_, master_key_)) { + throw utils::error::create_exception( + function_name, {"failed to recreate master key from kdf"}); + } + + auto res = + set_item_meta("/", META_KDF, nlohmann::json(master_kdf_cfg_).dump()); + if (res == api_error::success) { + legacy_bucket_ = false; + return true; + } + + throw utils::error::create_exception(function_name, + {"failed to set meta kdf"}); + } + } + + return false; +} + auto s3_provider::set_meta_key(const std::string &api_path, api_meta_map &meta) -> api_error { REPERTORY_USES_FUNCTION_NAME(); @@ -949,9 +1035,20 @@ auto s3_provider::set_meta_key(const std::string &api_path, api_meta_map &meta) } data_buffer result; - utils::encryption::encrypt_data( - cfg.encryption_token, - *(utils::string::split(api_path, '/', false).end() - 1U), result); + if (legacy_bucket_) { + utils::encryption::encrypt_data( + cfg.encryption_token, + *(utils::string::split(api_path, '/', false).end() - 1U), result); + } else { + auto [key, cfg] = master_kdf_cfg_.create_subkey( + utils::encryption::kdf_context::path, id, master_key_); + + utils::encryption::encrypt_data( + key, *(utils::string::split(api_path, '/', false).end() - 1U), result); + + auto hdr = cfg.to_header(); + result.insert(result.begin(), hdr.begin(), hdr.end()); + } meta[META_KEY] = utils::path::create_api_path( utils::path::combine(utils::path::create_api_path(encrypted_parent_path), @@ -976,27 +1073,44 @@ auto s3_provider::start(api_item_added_callback api_item_added, ret == res == api_error::success; if (ret) { if (kdf_str.empty()) { - if (not search_keys_for_kdf()) { - kdf_.seal(); - res = set_item_meta("/", META_KDF, nlohmann::json(kdf_).dump()); - ret == res == api_error::success; - if (not ret) { - utils::error::raise_api_path_error(function_name, "/", res, - "set item meta failed"); + try { + if (not search_keys_for_kdf(cfg.encryption_token)) { + if (get_directory_item_count("/") == 0U) { + legacy_bucket_ = false; + master_kdf_cfg_.seal(); + master_key_ = + utils::encryption::generate_key( + cfg.encryption_token, master_kdf_cfg_); + + auto res = set_item_meta("/", META_KDF, + nlohmann::json(master_kdf_cfg_).dump()); + ret == res == api_error::success; + if (not ret) { + utils::error::raise_api_path_error(function_name, "/", res, + "set kdf in meta failed"); + } + } } + } catch (const std::exception &e) { + utils::error::raise_error(function_name, e, "exception occurred"); + ret = false; } } else { - kdf_ = nlohmann::json::parse(kdf_str).get(); + master_kdf_cfg_ = + nlohmann::json::parse(kdf_str).get(); + if (not utils::encryption::recreate_key_argon2id( + cfg.encryption_token, master_kdf_cfg_, master_key_)) { + ret = false; + utils::error::raise_error(function_name, + "failed to recreate master key from kdf"); + } } } else { utils::error::raise_api_path_error(function_name, "/", res, - "get item meta failed"); + "get kdf from meta failed"); } - if (ret) { - key_ = utils::encryption::recreate_key( - cfg.encryption_token, kdf); - } else { + if (not ret) { base_provider::stop(); } } @@ -1060,10 +1174,24 @@ auto s3_provider::upload_file_impl(const std::string &api_path, } if (is_encrypted && file_size > 0U) { - put_file.reader = std::make_shared( - object_name, source_path, - []() -> bool { return app_config::get_stop_requested(); }, - cfg.encryption_token, std::nullopt, -1); + if (legacy_bucket_) { + put_file.reader = std::make_shared( + object_name, source_path, + []() -> bool { return app_config::get_stop_requested(); }, + cfg.encryption_token, std::nullopt, -1); + } else { + put_file.reader = std::make_shared( + object_name, source_path, + []() -> bool { return app_config::get_stop_requested(); }, + master_key_, master_kdf_cfg_, std::nullopt, -1); + + res = set_item_meta( + "/", META_KDF, + nlohmann::json(*put_file.reader->get_kdf_config_for_data()).dump()); + if (res == api_error::success) { + return res; + } + } } long response_code{}; @@ -1185,11 +1313,30 @@ auto s3_provider::read_file_bytes(const std::string &api_path, std::size_t size, return res; } + utils::hash::hash_256_t key; + if (legacy_bucket_) { + key = utils::encryption::generate_key( + cfg.encryption_token); + } else { + res = get_item_meta(api_path, META_KDF, temp); + if (res != api_error::success) { + return res; + } + + if (not utils::encryption::recreate_key_argon2id( + cfg.encryption_token, + nlohmann::json::parse(temp).get(), + key)) { + throw utils::error::create_exception( + function_name, {"failed to recreate data key from kdf"}); + return api_error::error; + } + } + auto total_size{utils::string::to_uint64(temp)}; return utils::encryption::read_encrypted_range( - {.begin = offset, .end = offset + size - 1U}, - utils::encryption::generate_key( - cfg.encryption_token), + {.begin = offset, .end = offset + size - 1U}, key, + not legacy_bucket_, [&](data_buffer &ct_buffer, std::uint64_t start_offset, std::uint64_t end_offset) -> bool { return read_bytes((end_offset - start_offset + 1U), @@ -1206,5 +1353,4 @@ auto s3_provider::read_file_bytes(const std::string &api_path, std::size_t size, return api_error::error; } - } // namespace repertory