From 72db4e12cdb9ab180768b99876dc1bc35c1fc42a Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Fri, 29 Aug 2025 19:39:36 -0500 Subject: [PATCH] Implement secure key via KDF for transparent data encryption/decryption #60 --- .cspell/words.txt | 2 + .../include/providers/s3/s3_provider.hpp | 15 ++-- .../src/providers/s3/s3_provider.cpp | 76 +++++++++---------- support/include/utils/encryption.hpp | 42 ++++++---- support/src/utils/encryption.cpp | 41 ++++++++++ 5 files changed, 115 insertions(+), 61 deletions(-) diff --git a/.cspell/words.txt b/.cspell/words.txt index 0ec5a6a0..fdfaa4ee 100644 --- a/.cspell/words.txt +++ b/.cspell/words.txt @@ -27,6 +27,7 @@ cpptrace cppvsdbg create_notraverse crypto_aead_xchacha20poly1305_ietf_npubbytes +cspan cstdint curl_zstd curle_couldnt_resolve_host @@ -197,6 +198,7 @@ pugi pugixml_project puint32 pvoid +pwhash pwstr rdrw remote_winfsp diff --git a/repertory/librepertory/include/providers/s3/s3_provider.hpp b/repertory/librepertory/include/providers/s3/s3_provider.hpp index 2d09c7f2..b239b730 100644 --- a/repertory/librepertory/include/providers/s3/s3_provider.hpp +++ b/repertory/librepertory/include/providers/s3/s3_provider.hpp @@ -50,9 +50,11 @@ public: private: s3_config s3_config_; + +private: bool legacy_bucket_{true}; - utils::encryption::kdf_config master_kdf_cfg_; - utils::hash::hash_256_t master_key_; + utils::encryption::kdf_config master_kdf_cfg_{}; + utils::hash::hash_256_t master_key_{}; private: [[nodiscard]] auto add_if_not_found(api_file &file, @@ -71,8 +73,8 @@ private: api_meta_map &meta) -> api_error override; - [[nodiscard]] auto decrypt_object_name(std::string &object_name, - bool &uses_kdf) const -> api_error; + [[nodiscard]] auto decrypt_object_name(std::string &object_name) const + -> api_error; [[nodiscard]] auto get_last_modified(bool directory, const std::string &api_path, @@ -95,11 +97,6 @@ private: return s3_config_; } - [[nodiscard]] auto read_file_bytes(const std::string &api_path, - std::size_t size, std::uint64_t offset, - data_buffer &data, bool encrypted, - stop_type &stop_requested) -> api_error; - [[nodiscard]] auto search_keys_for_kdf(std::string_view encryption_token) -> bool; diff --git a/repertory/librepertory/src/providers/s3/s3_provider.cpp b/repertory/librepertory/src/providers/s3/s3_provider.cpp index a9eeb0a8..d734c849 100644 --- a/repertory/librepertory/src/providers/s3/s3_provider.cpp +++ b/repertory/librepertory/src/providers/s3/s3_provider.cpp @@ -32,6 +32,7 @@ #include "types/repertory.hpp" #include "types/s3.hpp" #include "utils/collection.hpp" +#include "utils/common.hpp" #include "utils/config.hpp" #include "utils/encrypting_reader.hpp" #include "utils/encryption.hpp" @@ -240,8 +241,16 @@ auto s3_provider::create_file_extra(const std::string &api_path, auto s3_provider::decrypt_object_name(std::string &object_name) const -> api_error { - if (utils::encryption::decrypt_file_path(get_s3_config().encryption_token, - object_name)) { + if (legacy_bucket_) { + if (utils::encryption::decrypt_file_path(get_s3_config().encryption_token, + object_name)) { + return api_error::success; + } + + return api_error::decryption_error; + } + + if (utils::encryption::decrypt_file_path(master_key_, object_name)) { return api_error::success; } @@ -278,8 +287,7 @@ auto s3_provider::get_directory_item_count(const std::string &api_path) const std::string token{}; std::uint64_t total_count{}; while (grab_more) { - if (not get_object_list(response_data, response_code, "/", prefix, - token)) { + if (not get_object_list(response_data, response_code, "/", "", token)) { return total_count; } @@ -936,7 +944,7 @@ auto s3_provider::search_keys_for_kdf(std::string_view encryption_token) while (grab_more) { std::string response_data{}; long response_code{}; - if (not get_object_list(response_data, response_code, "/", prefix, token)) { + if (not get_object_list(response_data, response_code, "/", "", token)) { throw utils::error::create_exception(function_name, {"failed to get object list"}); } @@ -969,12 +977,12 @@ auto s3_provider::search_keys_for_kdf(std::string_view encryption_token) .as_string(); } - node_list = doc.select_nodes("/ListBucketResult/Contents"); + auto node_list = doc.select_nodes("/ListBucketResult/Contents"); for (const auto &node : node_list) { - auto object_name{ - node.node().select_node("Key").node().text().as_string()), + std::string object_name{ + node.node().select_node("Key").node().text().as_string(), }; - if (object_name == "/")) { + if (object_name == "/") { continue; } @@ -983,8 +991,8 @@ auto s3_provider::search_keys_for_kdf(std::string_view encryption_token) continue; } - kdf_config cfg; - if (not kdf_config::from_header(buffer, cfg)) { + utils::encryption::kdf_config cfg; + if (not utils::encryption::kdf_config::from_header(buffer, cfg)) { continue; } @@ -1040,13 +1048,15 @@ auto s3_provider::set_meta_key(const std::string &api_path, api_meta_map &meta) cfg.encryption_token, *(utils::string::split(api_path, '/', false).end() - 1U), result); } else { - auto [key, cfg] = master_kdf_cfg_.create_subkey( - utils::encryption::kdf_context::path, id, master_key_); + auto [path_key, path_cfg] = master_kdf_cfg_.create_subkey( + utils::encryption::kdf_context::path, + utils::generate_secure_random(), master_key_); utils::encryption::encrypt_data( - key, *(utils::string::split(api_path, '/', false).end() - 1U), result); + path_key, *(utils::string::split(api_path, '/', false).end() - 1U), + result); - auto hdr = cfg.to_header(); + auto hdr = path_cfg.to_header(); result.insert(result.begin(), hdr.begin(), hdr.end()); } @@ -1070,7 +1080,7 @@ auto s3_provider::start(api_item_added_callback api_item_added, if (ret && not cfg.encryption_token.empty()) { std::string kdf_str; auto res = get_item_meta("/", META_KDF, kdf_str); - ret == res == api_error::success; + ret = res == api_error::success; if (ret) { if (kdf_str.empty()) { try { @@ -1082,9 +1092,9 @@ auto s3_provider::start(api_item_added_callback api_item_added, utils::encryption::generate_key( cfg.encryption_token, master_kdf_cfg_); - auto res = set_item_meta("/", META_KDF, - nlohmann::json(master_kdf_cfg_).dump()); - ret == res == api_error::success; + res = set_item_meta("/", META_KDF, + nlohmann::json(master_kdf_cfg_).dump()); + ret = res == api_error::success; if (not ret) { utils::error::raise_api_path_error(function_name, "/", res, "set kdf in meta failed"); @@ -1212,19 +1222,11 @@ auto s3_provider::upload_file_impl(const std::string &api_path, auto s3_provider::read_file_bytes(const std::string &api_path, std::size_t size, std::uint64_t offset, data_buffer &data, stop_type &stop_requested) -> api_error { - return read_file_bytes(api_path, size, offset, data, - not get_s3_config().encryption_token.empty(), - stop_requested); -} - -auto s3_provider::read_file_bytes(const std::string &api_path, std::size_t size, - std::uint64_t offset, data_buffer &data, - bool encrypted, stop_type &stop_requested) - -> api_error { REPERTORY_USES_FUNCTION_NAME(); try { const auto &cfg{get_s3_config()}; + bool encrypted{not cfg.encryption_token.empty()}; std::string key; if (encrypted) { @@ -1313,9 +1315,9 @@ auto s3_provider::read_file_bytes(const std::string &api_path, std::size_t size, return res; } - utils::hash::hash_256_t key; + utils::hash::hash_256_t data_key; if (legacy_bucket_) { - key = utils::encryption::generate_key( + data_key = utils::encryption::generate_key( cfg.encryption_token); } else { res = get_item_meta(api_path, META_KDF, temp); @@ -1323,19 +1325,15 @@ auto s3_provider::read_file_bytes(const std::string &api_path, std::size_t size, return res; } - if (not utils::encryption::recreate_key_argon2id( - cfg.encryption_token, - nlohmann::json::parse(temp).get(), - key)) { - throw utils::error::create_exception( - function_name, {"failed to recreate data key from kdf"}); - return api_error::error; - } + auto data_cfg = + nlohmann::json::parse(temp).get(); + data_key = data_cfg.recreate_subkey(utils::encryption::kdf_context::data, + master_key_); } auto total_size{utils::string::to_uint64(temp)}; return utils::encryption::read_encrypted_range( - {.begin = offset, .end = offset + size - 1U}, key, + {.begin = offset, .end = offset + size - 1U}, data_key, not legacy_bucket_, [&](data_buffer &ct_buffer, std::uint64_t start_offset, std::uint64_t end_offset) -> bool { diff --git a/support/include/utils/encryption.hpp b/support/include/utils/encryption.hpp index dbc65ee1..dd80e5a7 100644 --- a/support/include/utils/encryption.hpp +++ b/support/include/utils/encryption.hpp @@ -143,6 +143,16 @@ struct kdf_config final { return {sub_key, cfg}; } + template + [[nodiscard]] auto recreate_subkey(kdf_context ctx, + const hash_t &master_key) const -> hash_t { + hash_t sub_key; + crypto_kdf_derive_from_key(sub_key.data(), sub_key.size(), unique_id, + get_kdf_context_name(ctx).data(), + master_key.data()); + return sub_key; + } + [[nodiscard]] static auto from_header(data_cspan data, kdf_config &cfg) -> bool; @@ -201,19 +211,19 @@ template utils::hash::hash_256_t &key) -> bool; template -[[nodiscard]] inline bool +[[nodiscard]] inline auto detect_and_recreate_key(string_t password, data_cspan header, hash_t &key, - std::optional &cfg); + std::optional &cfg) -> bool; template -[[nodiscard]] inline bool +[[nodiscard]] inline auto detect_and_recreate_key(std::string_view password, data_cspan header, - hash_t &key, std::optional &cfg); + hash_t &key, std::optional &cfg) -> bool; template -[[nodiscard]] inline bool +[[nodiscard]] inline auto detect_and_recreate_key(std::wstring_view password, data_cspan header, - hash_t &key, std::optional &cfg); + hash_t &key, std::optional &cfg) -> bool; [[nodiscard]] auto decrypt_file_name(std::string_view encryption_token, std::string &file_name) -> bool; @@ -229,6 +239,12 @@ detect_and_recreate_key(std::wstring_view password, data_cspan header, const kdf_config &cfg, std::string &file_path) -> bool; +[[nodiscard]] auto decrypt_file_name(const utils::hash::hash_256_t &master_key, + std::string &file_name) -> bool; + +[[nodiscard]] auto decrypt_file_path(const utils::hash::hash_256_t &master_key, + std::string &file_path) -> bool; + template [[nodiscard]] inline auto decrypt_data(const std::array &key, const unsigned char *buffer, @@ -570,9 +586,9 @@ inline auto recreate_key(std::wstring_view password, const kdf_config &cfg) } template -inline bool detect_and_recreate_key(string_t password, data_cspan header, - hash_t &key, - std::optional &cfg) { +inline auto detect_and_recreate_key(string_t password, data_cspan header, + hash_t &key, std::optional &cfg) + -> bool { if (header.size() >= kdf_config::size()) { kdf_config tmp{}; if (kdf_config::from_header(header.first(kdf_config::size()), tmp)) { @@ -587,17 +603,17 @@ inline bool detect_and_recreate_key(string_t password, data_cspan header, } template -inline bool detect_and_recreate_key(std::string_view password, +inline auto detect_and_recreate_key(std::string_view password, data_cspan header, hash_t &key, - std::optional &cfg) { + std::optional &cfg) -> bool { return detect_and_recreate_key(password, header, key, cfg); } template -inline bool detect_and_recreate_key(std::wstring_view password, +inline auto detect_and_recreate_key(std::wstring_view password, data_cspan header, hash_t &key, - std::optional &cfg) { + std::optional &cfg) -> bool { return detect_and_recreate_key(password, header, key, cfg); } diff --git a/support/src/utils/encryption.cpp b/support/src/utils/encryption.cpp index d8efd0a8..f600ae18 100644 --- a/support/src/utils/encryption.cpp +++ b/support/src/utils/encryption.cpp @@ -126,6 +126,27 @@ auto decrypt_file_path(std::string_view encryption_token, const kdf_config &cfg, return true; } +auto decrypt_file_path(const utils::hash::hash_256_t &master_key, + std::string &file_path) -> bool { + std::vector decrypted_parts; + for (const auto &part : std::filesystem::path(file_path)) { + auto file_name = part.string(); + if (file_name == "/") { + continue; + } + + if (not decrypt_file_name(master_key, file_name)) { + return false; + } + + decrypted_parts.push_back(file_name); + } + + file_path = + utils::path::create_api_path(utils::string::join(decrypted_parts, '/')); + return true; +} + auto decrypt_file_name(std::string_view encryption_token, std::string &file_name) -> bool { data_buffer buffer; @@ -149,6 +170,26 @@ auto decrypt_file_name(std::string_view encryption_token, const kdf_config &cfg, file_name); } +auto decrypt_file_name(const utils::hash::hash_256_t &master_key, + std::string &file_name) -> bool { + data_buffer buffer; + if (not utils::collection::from_hex_string(file_name, buffer)) { + return false; + } + + utils::encryption::kdf_config path_cfg; + if (not utils::encryption::kdf_config::from_header(buffer, path_cfg)) { + return false; + } + + auto path_key = path_cfg.recreate_subkey(utils::encryption::kdf_context::path, + master_key); + + file_name.clear(); + return utils::encryption::decrypt_data( + path_key, &buffer[utils::encryption::kdf_config::size()], file_name); +} + template [[nodiscard]] auto read_encrypted_range(http_range range, const utils::hash::hash_256_t &key,