BlockStorage/repertory
1
0
Fork 0
Code Issues 4 Pull Requests Releases 8 Wiki Activity

Implement secure key via KDF for transparent data encryption/decryption #60

Browse Source
This commit is contained in:
Scott E. Graves
2025-08-29 19:39:36 -05:00
parent 62194271c0
commit 72db4e12cd
5 changed files with 115 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.cspell/words.txt
View File

@@ -27,6 +27,7 @@ cpptrace
cppvsdbg
create_notraverse
crypto_aead_xchacha20poly1305_ietf_npubbytes
cspan
cstdint
curl_zstd
curle_couldnt_resolve_host
@@ -197,6 +198,7 @@ pugi
pugixml_project
puint32
pvoid
pwhash
pwstr
rdrw
remote_winfsp

15
repertory/librepertory/include/providers/s3/s3_provider.hpp
View File

@@ -50,9 +50,11 @@ public:
private:
s3_config s3_config_;
private:
bool legacy_bucket_{true};
utils::encryption::kdf_config master_kdf_cfg_;
utils::hash::hash_256_t master_key_;
utils::encryption::kdf_config master_kdf_cfg_{};
utils::hash::hash_256_t master_key_{};
private:
[[nodiscard]] auto add_if_not_found(api_file &file,
@@ -71,8 +73,8 @@ private:
api_meta_map &meta)
-> api_error override;
[[nodiscard]] auto decrypt_object_name(std::string &object_name,
bool &uses_kdf) const -> api_error;
[[nodiscard]] auto decrypt_object_name(std::string &object_name) const
-> api_error;
[[nodiscard]] auto get_last_modified(bool directory,
const std::string &api_path,
@@ -95,11 +97,6 @@ private:
return s3_config_;
}
[[nodiscard]] auto read_file_bytes(const std::string &api_path,
std::size_t size, std::uint64_t offset,
data_buffer &data, bool encrypted,
stop_type &stop_requested) -> api_error;
[[nodiscard]] auto search_keys_for_kdf(std::string_view encryption_token)
-> bool;

70
repertory/librepertory/src/providers/s3/s3_provider.cpp
View File

@@ -32,6 +32,7 @@
#include "types/repertory.hpp"
#include "types/s3.hpp"
#include "utils/collection.hpp"
#include "utils/common.hpp"
#include "utils/config.hpp"
#include "utils/encrypting_reader.hpp"
#include "utils/encryption.hpp"
@@ -240,6 +241,7 @@ auto s3_provider::create_file_extra(const std::string &api_path,
auto s3_provider::decrypt_object_name(std::string &object_name) const
-> api_error {
if (legacy_bucket_) {
if (utils::encryption::decrypt_file_path(get_s3_config().encryption_token,
object_name)) {
return api_error::success;
@@ -248,6 +250,13 @@ auto s3_provider::decrypt_object_name(std::string &object_name) const
return api_error::decryption_error;
}
if (utils::encryption::decrypt_file_path(master_key_, object_name)) {
return api_error::success;
}
return api_error::decryption_error;
}
auto s3_provider::get_directory_item_count(const std::string &api_path) const
-> std::uint64_t {
REPERTORY_USES_FUNCTION_NAME();
@@ -278,8 +287,7 @@ auto s3_provider::get_directory_item_count(const std::string &api_path) const
std::string token{};
std::uint64_t total_count{};
while (grab_more) {
if (not get_object_list(response_data, response_code, "/", prefix,
token)) {
if (not get_object_list(response_data, response_code, "/", "", token)) {
return total_count;
}
@@ -936,7 +944,7 @@ auto s3_provider::search_keys_for_kdf(std::string_view encryption_token)
while (grab_more) {
std::string response_data{};
long response_code{};
if (not get_object_list(response_data, response_code, "/", prefix, token)) {
if (not get_object_list(response_data, response_code, "/", "", token)) {
throw utils::error::create_exception(function_name,
{"failed to get object list"});
}
@@ -969,12 +977,12 @@ auto s3_provider::search_keys_for_kdf(std::string_view encryption_token)
.as_string();
}
node_list = doc.select_nodes("/ListBucketResult/Contents");
auto node_list = doc.select_nodes("/ListBucketResult/Contents");
for (const auto &node : node_list) {
auto object_name{
node.node().select_node("Key").node().text().as_string()),
std::string object_name{
node.node().select_node("Key").node().text().as_string(),
};
if (object_name == "/")) {
if (object_name == "/") {
continue;
}
@@ -983,8 +991,8 @@ auto s3_provider::search_keys_for_kdf(std::string_view encryption_token)
continue;
}
kdf_config cfg;
if (not kdf_config::from_header(buffer, cfg)) {
utils::encryption::kdf_config cfg;
if (not utils::encryption::kdf_config::from_header(buffer, cfg)) {
continue;
}
@@ -1040,13 +1048,15 @@ auto s3_provider::set_meta_key(const std::string &api_path, api_meta_map &meta)
cfg.encryption_token,
*(utils::string::split(api_path, '/', false).end() - 1U), result);
} else {
auto [key, cfg] = master_kdf_cfg_.create_subkey(
utils::encryption::kdf_context::path, id, master_key_);
auto [path_key, path_cfg] = master_kdf_cfg_.create_subkey(
utils::encryption::kdf_context::path,
utils::generate_secure_random<std::uint64_t>(), master_key_);
utils::encryption::encrypt_data(
key, *(utils::string::split(api_path, '/', false).end() - 1U), result);
path_key, *(utils::string::split(api_path, '/', false).end() - 1U),
result);
auto hdr = cfg.to_header();
auto hdr = path_cfg.to_header();
result.insert(result.begin(), hdr.begin(), hdr.end());
}
@@ -1070,7 +1080,7 @@ auto s3_provider::start(api_item_added_callback api_item_added,
if (ret && not cfg.encryption_token.empty()) {
std::string kdf_str;
auto res = get_item_meta("/", META_KDF, kdf_str);
ret == res == api_error::success;
ret = res == api_error::success;
if (ret) {
if (kdf_str.empty()) {
try {
@@ -1082,9 +1092,9 @@ auto s3_provider::start(api_item_added_callback api_item_added,
utils::encryption::generate_key<utils::hash::hash_256_t>(
cfg.encryption_token, master_kdf_cfg_);
auto res = set_item_meta("/", META_KDF,
res = set_item_meta("/", META_KDF,
nlohmann::json(master_kdf_cfg_).dump());
ret == res == api_error::success;
ret = res == api_error::success;
if (not ret) {
utils::error::raise_api_path_error(function_name, "/", res,
"set kdf in meta failed");
@@ -1212,19 +1222,11 @@ auto s3_provider::upload_file_impl(const std::string &api_path,
auto s3_provider::read_file_bytes(const std::string &api_path, std::size_t size,
std::uint64_t offset, data_buffer &data,
stop_type &stop_requested) -> api_error {
return read_file_bytes(api_path, size, offset, data,
not get_s3_config().encryption_token.empty(),
stop_requested);
}
auto s3_provider::read_file_bytes(const std::string &api_path, std::size_t size,
std::uint64_t offset, data_buffer &data,
bool encrypted, stop_type &stop_requested)
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
try {
const auto &cfg{get_s3_config()};
bool encrypted{not cfg.encryption_token.empty()};
std::string key;
if (encrypted) {
@@ -1313,9 +1315,9 @@ auto s3_provider::read_file_bytes(const std::string &api_path, std::size_t size,
return res;
}
utils::hash::hash_256_t key;
utils::hash::hash_256_t data_key;
if (legacy_bucket_) {
key = utils::encryption::generate_key<utils::hash::hash_256_t>(
data_key = utils::encryption::generate_key<utils::hash::hash_256_t>(
cfg.encryption_token);
} else {
res = get_item_meta(api_path, META_KDF, temp);
@@ -1323,19 +1325,15 @@ auto s3_provider::read_file_bytes(const std::string &api_path, std::size_t size,
return res;
}
if (not utils::encryption::recreate_key_argon2id(
cfg.encryption_token,
nlohmann::json::parse(temp).get<utils::encryption::kdf_config>(),
key)) {
throw utils::error::create_exception(
function_name, {"failed to recreate data key from kdf"});
return api_error::error;
}
auto data_cfg =
nlohmann::json::parse(temp).get<utils::encryption::kdf_config>();
data_key = data_cfg.recreate_subkey(utils::encryption::kdf_context::data,
master_key_);
}
auto total_size{utils::string::to_uint64(temp)};
return utils::encryption::read_encrypted_range(
{.begin = offset, .end = offset + size - 1U}, key,
{.begin = offset, .end = offset + size - 1U}, data_key,
not legacy_bucket_,
[&](data_buffer &ct_buffer, std::uint64_t start_offset,
std::uint64_t end_offset) -> bool {

42
support/include/utils/encryption.hpp
View File

@@ -143,6 +143,16 @@ struct kdf_config final {
return {sub_key, cfg};
}
template <typename hash_t>
[[nodiscard]] auto recreate_subkey(kdf_context ctx,
const hash_t &master_key) const -> hash_t {
hash_t sub_key;
crypto_kdf_derive_from_key(sub_key.data(), sub_key.size(), unique_id,
get_kdf_context_name(ctx).data(),
master_key.data());
return sub_key;
}
[[nodiscard]] static auto from_header(data_cspan data, kdf_config &cfg)
-> bool;
@@ -201,19 +211,19 @@ template <typename string_t>
utils::hash::hash_256_t &key) -> bool;
template <typename hash_t, typename string_t>
[[nodiscard]] inline bool
[[nodiscard]] inline auto
detect_and_recreate_key(string_t password, data_cspan header, hash_t &key,
std::optional<kdf_config> &cfg);
std::optional<kdf_config> &cfg) -> bool;
template <typename hash_t>
[[nodiscard]] inline bool
[[nodiscard]] inline auto
detect_and_recreate_key(std::string_view password, data_cspan header,
hash_t &key, std::optional<kdf_config> &cfg);
hash_t &key, std::optional<kdf_config> &cfg) -> bool;
template <typename hash_t>
[[nodiscard]] inline bool
[[nodiscard]] inline auto
detect_and_recreate_key(std::wstring_view password, data_cspan header,
hash_t &key, std::optional<kdf_config> &cfg);
hash_t &key, std::optional<kdf_config> &cfg) -> bool;
[[nodiscard]] auto decrypt_file_name(std::string_view encryption_token,
std::string &file_name) -> bool;
@@ -229,6 +239,12 @@ detect_and_recreate_key(std::wstring_view password, data_cspan header,
const kdf_config &cfg,
std::string &file_path) -> bool;
[[nodiscard]] auto decrypt_file_name(const utils::hash::hash_256_t &master_key,
std::string &file_name) -> bool;
[[nodiscard]] auto decrypt_file_path(const utils::hash::hash_256_t &master_key,
std::string &file_path) -> bool;
template <typename result_t, typename arr_t, std::size_t arr_size>
[[nodiscard]] inline auto decrypt_data(const std::array<arr_t, arr_size> &key,
const unsigned char *buffer,
@@ -570,9 +586,9 @@ inline auto recreate_key(std::wstring_view password, const kdf_config &cfg)
}
template <typename hash_t, typename string_t>
inline bool detect_and_recreate_key(string_t password, data_cspan header,
hash_t &key,
std::optional<kdf_config> &cfg) {
inline auto detect_and_recreate_key(string_t password, data_cspan header,
hash_t &key, std::optional<kdf_config> &cfg)
-> bool {
if (header.size() >= kdf_config::size()) {
kdf_config tmp{};
if (kdf_config::from_header(header.first(kdf_config::size()), tmp)) {
@@ -587,17 +603,17 @@ inline bool detect_and_recreate_key(string_t password, data_cspan header,
}
template <typename hash_t>
inline bool detect_and_recreate_key(std::string_view password,
inline auto detect_and_recreate_key(std::string_view password,
data_cspan header, hash_t &key,
std::optional<kdf_config> &cfg) {
std::optional<kdf_config> &cfg) -> bool {
return detect_and_recreate_key<hash_t, std::string_view>(password, header,
key, cfg);
}
template <typename hash_t>
inline bool detect_and_recreate_key(std::wstring_view password,
inline auto detect_and_recreate_key(std::wstring_view password,
data_cspan header, hash_t &key,
std::optional<kdf_config> &cfg) {
std::optional<kdf_config> &cfg) -> bool {
return detect_and_recreate_key<hash_t, std::wstring_view>(password, header,
key, cfg);
}

41
support/src/utils/encryption.cpp
View File

@@ -126,6 +126,27 @@ auto decrypt_file_path(std::string_view encryption_token, const kdf_config &cfg,
return true;
}
auto decrypt_file_path(const utils::hash::hash_256_t &master_key,
std::string &file_path) -> bool {
std::vector<std::string> decrypted_parts;
for (const auto &part : std::filesystem::path(file_path)) {
auto file_name = part.string();
if (file_name == "/") {
continue;
}
if (not decrypt_file_name(master_key, file_name)) {
return false;
}
decrypted_parts.push_back(file_name);
}
file_path =
utils::path::create_api_path(utils::string::join(decrypted_parts, '/'));
return true;
}
auto decrypt_file_name(std::string_view encryption_token,
std::string &file_name) -> bool {
data_buffer buffer;
@@ -149,6 +170,26 @@ auto decrypt_file_name(std::string_view encryption_token, const kdf_config &cfg,
file_name);
}
auto decrypt_file_name(const utils::hash::hash_256_t &master_key,
std::string &file_name) -> bool {
data_buffer buffer;
if (not utils::collection::from_hex_string(file_name, buffer)) {
return false;
}
utils::encryption::kdf_config path_cfg;
if (not utils::encryption::kdf_config::from_header(buffer, path_cfg)) {
return false;
}
auto path_key = path_cfg.recreate_subkey(utils::encryption::kdf_context::path,
master_key);
file_name.clear();
return utils::encryption::decrypt_data(
path_key, &buffer[utils::encryption::kdf_config::size()], file_name);
}
template <typename data_t>
[[nodiscard]] auto
read_encrypted_range(http_range range, const utils::hash::hash_256_t &key,