BlockStorage/repertory
1
0
Fork 0
Code Issues 3 Pull Requests Releases 9 Wiki Activity

updated build system

Browse Source
This commit is contained in:
sgraves
2025-08-28 18:34:21 -05:00
parent c9362b8802
commit fa3419c7cc
12 changed files with 996 additions and 194 deletions
Download Patch File Download Diff File Expand all files Collapse all files
support/src/utils/encrypting_reader.cpp
+206 -84
View File
@@ -181,7 +181,8 @@ encrypting_reader::encrypting_reader(
std::string_view file_name, std::string_view source_path,
stop_type_callback stop_requested_cb, std::string_view token,
std::optional<std::string> relative_parent_path, std::size_t error_return)
: key_(utils::encryption::generate_key<utils::hash::hash_256_t>(token)),
: keys_(utils::encryption::generate_key<utils::hash::hash_256_t>(token),
utils::encryption::generate_key<utils::hash::hash_256_t>(token)),
stop_requested_cb_(std::move(stop_requested_cb)),
error_return_(error_return),
source_file_(utils::file::file::open_or_create_file(source_path, true)) {
@@ -189,12 +190,13 @@ encrypting_reader::encrypting_reader(
create_encrypted_paths(file_name, relative_parent_path);
}
encrypting_reader::encrypting_reader(std::string_view encrypted_file_path,
encrypting_reader::encrypting_reader(stop_type_callback stop_requested_cb,
std::string_view encrypted_file_path,
std::string_view source_path,
stop_type_callback stop_requested_cb,
std::string_view token,
std::size_t error_return)
: key_(utils::encryption::generate_key<utils::hash::hash_256_t>(token)),
: keys_(utils::encryption::generate_key<utils::hash::hash_256_t>(token),
utils::encryption::generate_key<utils::hash::hash_256_t>(token)),
stop_requested_cb_(std::move(stop_requested_cb)),
error_return_(error_return),
source_file_(utils::file::file::open_or_create_file(source_path, true)),
@@ -205,13 +207,14 @@ encrypting_reader::encrypting_reader(std::string_view encrypted_file_path,
}
encrypting_reader::encrypting_reader(
std::string_view encrypted_file_path, std::string_view source_path,
stop_type_callback stop_requested_cb, std::string_view token,
stop_type_callback stop_requested_cb, std::string_view encrypted_file_path,
std::string_view source_path, std::string_view token,
std::vector<
std::array<unsigned char, crypto_aead_xchacha20poly1305_IETF_NPUBBYTES>>
iv_list,
std::size_t error_return)
: key_(utils::encryption::generate_key<utils::hash::hash_256_t>(token)),
: keys_(utils::encryption::generate_key<utils::hash::hash_256_t>(token),
utils::encryption::generate_key<utils::hash::hash_256_t>(token)),
stop_requested_cb_(std::move(stop_requested_cb)),
error_return_(error_return),
source_file_(utils::file::file::open_or_create_file(source_path, true)),
@@ -227,48 +230,50 @@ encrypting_reader::encrypting_reader(
stop_type_callback stop_requested_cb, std::string_view token,
kdf_config cfg, std::optional<std::string> relative_parent_path,
std::size_t error_return)
: key_(
utils::encryption::generate_key<utils::hash::hash_256_t>(token, cfg)),
: keys_(
utils::encryption::generate_key<utils::hash::hash_256_t>(token, cfg),
utils::encryption::generate_key<utils::hash::hash_256_t>(token)),
stop_requested_cb_(std::move(stop_requested_cb)),
error_return_(error_return),
source_file_(utils::file::file::open_or_create_file(source_path, true)) {
kdf_header_ = cfg.to_header();
kdf_headers_ = {cfg.to_header(), cfg.to_header()};
data_buffer result;
utils::encryption::encrypt_data(
key_, reinterpret_cast<const unsigned char *>(file_name.data()),
keys_.second, reinterpret_cast<const unsigned char *>(file_name.data()),
file_name.size(), result);
common_initialize(true);
create_encrypted_paths(file_name, relative_parent_path);
}
encrypting_reader::encrypting_reader(std::string_view encrypted_file_path,
encrypting_reader::encrypting_reader(stop_type_callback stop_requested_cb,
std::string_view encrypted_file_path,
std::string_view source_path,
stop_type_callback stop_requested_cb,
std::string_view token, kdf_config cfg,
std::size_t error_return)
: key_(
utils::encryption::generate_key<utils::hash::hash_256_t>(token, cfg)),
: keys_(
utils::encryption::generate_key<utils::hash::hash_256_t>(token, cfg),
utils::encryption::generate_key<utils::hash::hash_256_t>(token)),
stop_requested_cb_(std::move(stop_requested_cb)),
error_return_(error_return),
source_file_(utils::file::file::open_or_create_file(source_path, true)),
encrypted_file_name_(
utils::path::strip_to_file_name(std::string{encrypted_file_path})),
encrypted_file_path_(encrypted_file_path) {
kdf_header_ = cfg.to_header();
kdf_headers_ = {cfg.to_header(), cfg.to_header()};
common_initialize(true);
}
encrypting_reader::encrypting_reader(
std::string_view encrypted_file_path, std::string_view source_path,
stop_type_callback stop_requested_cb, std::string_view token,
kdf_config cfg,
stop_type_callback stop_requested_cb, std::string_view encrypted_file_path,
std::string_view source_path, std::string_view token, kdf_config cfg,
std::vector<
std::array<unsigned char, crypto_aead_xchacha20poly1305_IETF_NPUBBYTES>>
iv_list,
std::size_t error_return)
: key_(
utils::encryption::generate_key<utils::hash::hash_256_t>(token, cfg)),
: keys_(
utils::encryption::generate_key<utils::hash::hash_256_t>(token, cfg),
utils::encryption::generate_key<utils::hash::hash_256_t>(token)),
stop_requested_cb_(std::move(stop_requested_cb)),
error_return_(error_return),
source_file_(utils::file::file::open_or_create_file(source_path, true)),
@@ -276,12 +281,74 @@ encrypting_reader::encrypting_reader(
utils::path::strip_to_file_name(std::string{encrypted_file_path})),
encrypted_file_path_(encrypted_file_path),
iv_list_(std::move(iv_list)) {
kdf_header_ = cfg.to_header();
kdf_headers_ = {cfg.to_header(), cfg.to_header()};
common_initialize(false);
}
encrypting_reader::encrypting_reader(
std::string_view file_name, std::string_view source_path,
stop_type_callback stop_requested_cb,
const utils::hash::hash_256_t &master_key, const kdf_config &cfg,
std::optional<std::string> relative_parent_path, std::size_t error_return)
: stop_requested_cb_(std::move(stop_requested_cb)),
error_return_(error_return),
source_file_(utils::file::file::open_or_create_file(source_path, true)) {
common_initialize_kdf_data(cfg, master_key);
kdf_config path_cfg;
std::tie(keys_.second, path_cfg) = cfg.create_subkey(
kdf_context::path, utils::generate_secure_random<std::uint64_t>(),
master_key);
kdf_headers_->second = path_cfg.to_header();
data_buffer result;
utils::encryption::encrypt_data(
keys_.second, reinterpret_cast<const unsigned char *>(file_name.data()),
file_name.size(), result);
common_initialize(true);
create_encrypted_paths(file_name, relative_parent_path);
}
encrypting_reader::encrypting_reader(stop_type_callback stop_requested_cb,
std::string_view encrypted_file_path,
std::string_view source_path,
const utils::hash::hash_256_t &master_key,
const kdf_config &cfg,
std::size_t error_return)
: keys_(),
stop_requested_cb_(std::move(stop_requested_cb)),
error_return_(error_return),
source_file_(utils::file::file::open_or_create_file(source_path, true)),
encrypted_file_name_(
utils::path::strip_to_file_name(std::string{encrypted_file_path})),
encrypted_file_path_(encrypted_file_path) {
common_initialize_kdf_data(cfg, master_key);
common_initialize_kdf_path(master_key);
common_initialize(true);
}
encrypting_reader::encrypting_reader(
stop_type_callback stop_requested_cb, std::string_view encrypted_file_path,
std::string_view source_path, const utils::hash::hash_256_t &master_key,
const kdf_config &cfg,
std::vector<
std::array<unsigned char, crypto_aead_xchacha20poly1305_IETF_NPUBBYTES>>
iv_list,
std::size_t error_return)
: stop_requested_cb_(std::move(stop_requested_cb)),
error_return_(error_return),
source_file_(utils::file::file::open_or_create_file(source_path, true)),
encrypted_file_name_(
utils::path::strip_to_file_name(std::string{encrypted_file_path})),
encrypted_file_path_(encrypted_file_path),
iv_list_(std::move(iv_list)) {
common_initialize_kdf_data(cfg, master_key);
common_initialize_kdf_path(master_key);
common_initialize(false);
}
encrypting_reader::encrypting_reader(const encrypting_reader &reader)
: key_(reader.key_),
: keys_(reader.keys_),
stop_requested_cb_(reader.stop_requested_cb_),
error_return_(reader.error_return_),
source_file_(
@@ -290,7 +357,7 @@ encrypting_reader::encrypting_reader(const encrypting_reader &reader)
encrypted_file_path_(reader.encrypted_file_path_),
iv_list_(reader.iv_list_),
chunk_buffers_(reader.chunk_buffers_),
kdf_header_(reader.kdf_header_),
kdf_headers_(reader.kdf_headers_),
last_data_chunk_(reader.last_data_chunk_),
last_data_chunk_size_(reader.last_data_chunk_size_),
read_offset_(reader.read_offset_),
@@ -307,46 +374,6 @@ encrypting_reader::encrypting_reader(const encrypting_reader &reader)
}
}
void encrypting_reader::common_initialize(bool procces_iv_list) {
REPERTORY_USES_FUNCTION_NAME();
if (not *source_file_) {
throw utils::error::create_exception(function_name,
{
"file open failed",
source_file_->get_path(),
});
}
auto opt_size = source_file_->size();
if (not opt_size.has_value()) {
throw utils::error::create_exception(function_name,
{
"failed to get file size",
source_file_->get_path(),
});
}
auto file_size = opt_size.value();
auto total_chunks = utils::divide_with_ceiling(
file_size, static_cast<std::uint64_t>(data_chunk_size_));
total_size_ = file_size + (total_chunks * encryption_header_size) +
(kdf_header_.has_value() ? kdf_header_->size() : 0U);
last_data_chunk_ = total_chunks - 1U;
last_data_chunk_size_ = (file_size <= data_chunk_size_) ? file_size
: (file_size % data_chunk_size_) == 0U
? data_chunk_size_
: file_size % data_chunk_size_;
if (not procces_iv_list) {
return;
}
iv_list_.resize(total_chunks);
for (auto &data : iv_list_) {
randombytes_buf(data.data(), data.size());
}
}
auto encrypting_reader::calculate_decrypted_size(std::uint64_t total_size,
bool uses_kdf)
-> std::uint64_t {
@@ -382,15 +409,88 @@ auto encrypting_reader::calculate_encrypted_size(std::string_view source_path,
(uses_kdf ? kdf_config::size() : 0U);
}
void encrypting_reader::common_initialize(bool procces_iv_list) {
REPERTORY_USES_FUNCTION_NAME();
if (not *source_file_) {
throw utils::error::create_exception(function_name,
{
"file open failed",
source_file_->get_path(),
});
}
auto opt_size = source_file_->size();
if (not opt_size.has_value()) {
throw utils::error::create_exception(function_name,
{
"failed to get file size",
source_file_->get_path(),
});
}
auto file_size = opt_size.value();
auto total_chunks = utils::divide_with_ceiling(
file_size, static_cast<std::uint64_t>(data_chunk_size_));
total_size_ = file_size + (total_chunks * encryption_header_size) +
(kdf_headers_.has_value() ? kdf_headers_->first.size() : 0U);
last_data_chunk_ = total_chunks - 1U;
last_data_chunk_size_ = (file_size <= data_chunk_size_) ? file_size
: (file_size % data_chunk_size_) == 0U
? data_chunk_size_
: file_size % data_chunk_size_;
if (not procces_iv_list) {
return;
}
iv_list_.resize(total_chunks);
for (auto &data : iv_list_) {
randombytes_buf(data.data(), data.size());
}
}
void encrypting_reader::common_initialize_kdf_data(
const kdf_config &cfg, const utils::hash::hash_256_t &master_key) {
auto [data_key, data_cfg] = cfg.create_subkey(
kdf_context::data, utils::generate_secure_random<std::uint64_t>(),
master_key);
keys_.first = std::move(data_key);
kdf_headers_ = {data_cfg.to_header(), {}};
}
void encrypting_reader::common_initialize_kdf_path(
const utils::hash::hash_256_t &master_key) {
REPERTORY_USES_FUNCTION_NAME();
data_buffer buffer;
if (not utils::collection::from_hex_string(encrypted_file_path_, buffer)) {
throw utils::error::create_exception(
function_name, {"failed to convert encrypted path from hex to bytes"});
}
kdf_config path_cfg;
if (not kdf_config::from_header(buffer, path_cfg)) {
throw utils::error::create_exception(
function_name, {"failed to create path kdf config from header"});
}
utils::hash::hash_256_t path_key;
std::tie(path_key, std::ignore) =
path_cfg.create_subkey(kdf_context::path, path_cfg.unique_id, master_key);
kdf_headers_->second = path_cfg.to_header();
}
void encrypting_reader::create_encrypted_paths(
std::string_view file_name,
std::optional<std::string> relative_parent_path) {
data_buffer result;
utils::encryption::encrypt_data(
key_, reinterpret_cast<const unsigned char *>(file_name.data()),
keys_.second, reinterpret_cast<const unsigned char *>(file_name.data()),
file_name.size(), result);
if (kdf_header_.has_value()) {
result.insert(result.begin(), kdf_header_->begin(), kdf_header_->end());
if (kdf_headers_.has_value()) {
result.insert(result.begin(), kdf_headers_->second.begin(),
kdf_headers_->second.end());
}
encrypted_file_name_ = utils::collection::to_hex_string(result);
@@ -403,10 +503,11 @@ void encrypting_reader::create_encrypted_paths(
utils::string::split(relative_parent_path.value(),
utils::path::directory_seperator, false)) {
utils::encryption::encrypt_data(
key_, reinterpret_cast<const unsigned char *>(part.c_str()),
keys_.second, reinterpret_cast<const unsigned char *>(part.c_str()),
strnlen(part.c_str(), part.size()), result);
if (kdf_header_.has_value()) {
result.insert(result.begin(), kdf_header_->begin(), kdf_header_->end());
if (kdf_headers_.has_value()) {
result.insert(result.begin(), kdf_headers_->second.begin(),
kdf_headers_->second.end());
}
encrypted_file_path_ += '/' + utils::collection::to_hex_string(result);
@@ -421,15 +522,35 @@ auto encrypting_reader::create_iostream() const
std::make_unique<encrypting_streambuf>(*this));
}
auto encrypting_reader::get_kdf_config() const -> std::optional<kdf_config> {
auto encrypting_reader::get_kdf_config_for_data() const
-> std::optional<kdf_config> {
REPERTORY_USES_FUNCTION_NAME();
if (not kdf_header_.has_value()) {
if (not kdf_headers_.has_value()) {
return std::nullopt;
}
kdf_config cfg;
if (not kdf_config::from_header(kdf_header_.value(), cfg)) {
if (not kdf_config::from_header(kdf_headers_->first, cfg)) {
throw utils::error::create_exception(function_name,
{
"invalid kdf header",
});
}
return cfg;
}
auto encrypting_reader::get_kdf_config_for_path() const
-> std::optional<kdf_config> {
REPERTORY_USES_FUNCTION_NAME();
if (not kdf_headers_.has_value()) {
return std::nullopt;
}
kdf_config cfg;
if (not kdf_config::from_header(kdf_headers_->second, cfg)) {
throw utils::error::create_exception(function_name,
{
"invalid kdf header",
@@ -443,6 +564,8 @@ auto encrypting_reader::reader_function(char *buffer, size_t size,
size_t nitems) -> size_t {
REPERTORY_USES_FUNCTION_NAME();
std::span<char> dest(buffer, size);
auto read_size =
static_cast<std::uint64_t>(size) * static_cast<std::uint64_t>(nitems);
if (read_size == 0U) {
@@ -453,17 +576,17 @@ auto encrypting_reader::reader_function(char *buffer, size_t size,
std::size_t total_read{};
auto total_size{total_size_};
if (kdf_header_.has_value()) {
total_size -= kdf_header_->size();
if (kdf_headers_.has_value()) {
auto &hdr = kdf_headers_->first;
total_size -= hdr.size();
if (read_offset < kdf_header_->size()) {
if (read_offset < hdr.size()) {
auto to_read{
utils::calculate_read_size(kdf_header_->size(), read_size,
read_offset),
utils::calculate_read_size(hdr.size(), read_size, read_offset),
};
read_offset_ += to_read;
std::memcpy(buffer, &kdf_header_->data()[read_offset], to_read);
std::memcpy(&dest[total_read], &hdr.at(read_offset), to_read);
if (read_size - to_read == 0) {
return to_read;
}
@@ -471,9 +594,8 @@ auto encrypting_reader::reader_function(char *buffer, size_t size,
read_offset = 0U;
read_size -= to_read;
total_read += to_read;
buffer += to_read;
} else {
read_offset -= kdf_header_->size();
read_offset -= hdr.size();
}
}
@@ -501,8 +623,8 @@ auto encrypting_reader::reader_function(char *buffer, size_t size,
static_cast<std::uint64_t>(data_chunk_size_),
&bytes_read);
if (ret) {
utils::encryption::encrypt_data(iv_list_.at(chunk), key_, file_data,
chunk_buffer);
utils::encryption::encrypt_data(iv_list_.at(chunk), keys_.first,
file_data, chunk_buffer);
}
} else if (chunk != 0U) {
chunk_buffers_.erase(chunk - 1U);
@@ -510,7 +632,7 @@ auto encrypting_reader::reader_function(char *buffer, size_t size,
auto &chunk_buffer = chunk_buffers_[chunk];
auto to_read = std::min(chunk_buffer.size() - chunk_offset, remain);
std::memcpy(buffer + total_read, &chunk_buffer[chunk_offset], to_read);
std::memcpy(&dest[total_read], &chunk_buffer[chunk_offset], to_read);
total_read += to_read;
remain -= to_read;
chunk_offset = 0U;