BlockStorage/repertory
1
0
Fork 0
Code Issues 3 Pull Requests Releases 9 Wiki Activity

updated build system

Browse Source
This commit is contained in:
sgraves
2025-08-12 18:28:15 -05:00
parent 02d59cb047
commit 3effb4c80a
50 changed files with 1915 additions and 536 deletions
Download Patch File Download Diff File Expand all files Collapse all files
support/src/utils/encrypting_reader.cpp
+227 -143
View File
@@ -25,6 +25,7 @@
#include "utils/collection.hpp"
#include "utils/common.hpp"
#include "utils/config.hpp"
#include "utils/encryption.hpp"
#include "utils/error.hpp"
#include "utils/file.hpp"
@@ -119,7 +120,7 @@ protected:
reader_.set_read_position(reinterpret_cast<std::uintptr_t>(gptr()));
char c{};
const auto res = encrypting_reader::reader_function(&c, 1U, 1U, &reader_);
auto res = encrypting_reader::reader_function(&c, 1U, 1U, &reader_);
if (res != 1) {
return traits_type::eof();
}
@@ -180,60 +181,12 @@ encrypting_reader::encrypting_reader(
std::string_view file_name, std::string_view source_path,
stop_type_callback stop_requested_cb, std::string_view token,
std::optional<std::string> relative_parent_path, std::size_t error_return)
: key_(utils::encryption::generate_key<utils::encryption::hash_256_t>(
token)),
: key_(utils::encryption::generate_key<utils::hash::hash_256_t>(token)),
stop_requested_cb_(std::move(stop_requested_cb)),
error_return_(error_return),
source_file_(utils::file::file::open_or_create_file(source_path, true)) {
REPERTORY_USES_FUNCTION_NAME();
if (not*source_file_) {
throw utils::error::create_exception(function_name, {
"file open failed",
source_path,
});
}
data_buffer result;
utils::encryption::encrypt_data(
key_, reinterpret_cast<const unsigned char *>(file_name.data()),
file_name.size(), result);
encrypted_file_name_ = utils::collection::to_hex_string(result);
if (relative_parent_path.has_value()) {
for (auto &&part :
utils::string::split(relative_parent_path.value(),
utils::path::directory_seperator, false)) {
utils::encryption::encrypt_data(
key_, reinterpret_cast<const unsigned char *>(part.c_str()),
strnlen(part.c_str(), part.size()), result);
encrypted_file_path_ += '/' + utils::collection::to_hex_string(result);
}
encrypted_file_path_ += '/' + encrypted_file_name_;
}
auto opt_size = source_file_->size();
if (not opt_size.has_value()) {
throw utils::error::create_exception(function_name,
{
"failed to get file size",
source_file_->get_path(),
});
}
auto file_size = opt_size.value();
const auto total_chunks = utils::divide_with_ceiling(
file_size, static_cast<std::uint64_t>(data_chunk_size_));
total_size_ = file_size + (total_chunks * encryption_header_size);
last_data_chunk_ = total_chunks - 1U;
last_data_chunk_size_ = (file_size <= data_chunk_size_) ? file_size
: (file_size % data_chunk_size_) == 0U
? data_chunk_size_
: file_size % data_chunk_size_;
iv_list_.resize(total_chunks);
for (auto &iv : iv_list_) {
randombytes_buf(iv.data(), iv.size());
}
common_initialize(true);
create_encrypted_paths(file_name, relative_parent_path);
}
encrypting_reader::encrypting_reader(std::string_view encrypted_file_path,
@@ -241,45 +194,14 @@ encrypting_reader::encrypting_reader(std::string_view encrypted_file_path,
stop_type_callback stop_requested_cb,
std::string_view token,
std::size_t error_return)
: key_(utils::encryption::generate_key<utils::encryption::hash_256_t>(
token)),
: key_(utils::encryption::generate_key<utils::hash::hash_256_t>(token)),
stop_requested_cb_(std::move(stop_requested_cb)),
error_return_(error_return),
source_file_(utils::file::file::open_or_create_file(source_path, true)) {
REPERTORY_USES_FUNCTION_NAME();
if (not*source_file_) {
throw utils::error::create_exception(function_name, {
"file open failed",
source_path,
});
}
encrypted_file_path_ = encrypted_file_path;
encrypted_file_name_ = utils::path::strip_to_file_name(encrypted_file_path_);
auto opt_size = source_file_->size();
if (not opt_size.has_value()) {
throw utils::error::create_exception(function_name,
{
"failed to get file size",
source_file_->get_path(),
});
}
auto file_size = opt_size.value();
const auto total_chunks = utils::divide_with_ceiling(
file_size, static_cast<std::uint64_t>(data_chunk_size_));
total_size_ = file_size + (total_chunks * encryption_header_size);
last_data_chunk_ = total_chunks - 1U;
last_data_chunk_size_ = (file_size <= data_chunk_size_) ? file_size
: (file_size % data_chunk_size_) == 0U
? data_chunk_size_
: file_size % data_chunk_size_;
iv_list_.resize(total_chunks);
for (auto &iv : iv_list_) {
randombytes_buf(iv.data(), iv.size());
}
source_file_(utils::file::file::open_or_create_file(source_path, true)),
encrypted_file_name_(
utils::path::strip_to_file_name(std::string{encrypted_file_path})),
encrypted_file_path_(encrypted_file_path) {
common_initialize(true);
}
encrypting_reader::encrypting_reader(
@@ -289,43 +211,73 @@ encrypting_reader::encrypting_reader(
std::array<unsigned char, crypto_aead_xchacha20poly1305_IETF_NPUBBYTES>>
iv_list,
std::size_t error_return)
: key_(utils::encryption::generate_key<utils::encryption::hash_256_t>(
token)),
: key_(utils::encryption::generate_key<utils::hash::hash_256_t>(token)),
stop_requested_cb_(std::move(stop_requested_cb)),
error_return_(error_return),
source_file_(utils::file::file::open_or_create_file(source_path, true)),
encrypted_file_name_(
utils::path::strip_to_file_name(std::string{encrypted_file_path})),
encrypted_file_path_(encrypted_file_path),
iv_list_(std::move(iv_list)) {
common_initialize(false);
}
encrypting_reader::encrypting_reader(
std::string_view file_name, std::string_view source_path,
stop_type_callback stop_requested_cb, std::string_view token,
kdf_config cfg, std::optional<std::string> relative_parent_path,
std::size_t error_return)
: key_(
utils::encryption::generate_key<utils::hash::hash_256_t>(token, cfg)),
stop_requested_cb_(std::move(stop_requested_cb)),
error_return_(error_return),
source_file_(utils::file::file::open_or_create_file(source_path, true)) {
REPERTORY_USES_FUNCTION_NAME();
kdf_header_ = cfg.to_header();
if (not*source_file_) {
throw utils::error::create_exception(function_name, {
"file open failed",
source_path,
});
}
data_buffer result;
utils::encryption::encrypt_data(
key_, reinterpret_cast<const unsigned char *>(file_name.data()),
file_name.size(), result);
common_initialize(true);
create_encrypted_paths(file_name, relative_parent_path);
}
encrypted_file_path_ = encrypted_file_path;
encrypted_file_name_ = utils::path::strip_to_file_name(encrypted_file_path_);
encrypting_reader::encrypting_reader(std::string_view encrypted_file_path,
std::string_view source_path,
stop_type_callback stop_requested_cb,
std::string_view token, kdf_config cfg,
std::size_t error_return)
: key_(
utils::encryption::generate_key<utils::hash::hash_256_t>(token, cfg)),
stop_requested_cb_(std::move(stop_requested_cb)),
error_return_(error_return),
source_file_(utils::file::file::open_or_create_file(source_path, true)),
encrypted_file_name_(
utils::path::strip_to_file_name(std::string{encrypted_file_path})),
encrypted_file_path_(encrypted_file_path) {
kdf_header_ = cfg.to_header();
common_initialize(true);
}
auto opt_size = source_file_->size();
if (not opt_size.has_value()) {
throw utils::error::create_exception(
function_name, {
"get file size failed",
std::to_string(utils::get_last_error_code()),
source_file_->get_path(),
});
}
auto file_size{opt_size.value()};
const auto total_chunks = utils::divide_with_ceiling(
file_size, static_cast<std::uint64_t>(data_chunk_size_));
total_size_ = file_size + (total_chunks * encryption_header_size);
last_data_chunk_ = total_chunks - 1U;
last_data_chunk_size_ = (file_size <= data_chunk_size_) ? file_size
: (file_size % data_chunk_size_) == 0U
? data_chunk_size_
: file_size % data_chunk_size_;
iv_list_ = std::move(iv_list);
encrypting_reader::encrypting_reader(
std::string_view encrypted_file_path, std::string_view source_path,
stop_type_callback stop_requested_cb, std::string_view token,
kdf_config cfg,
std::vector<
std::array<unsigned char, crypto_aead_xchacha20poly1305_IETF_NPUBBYTES>>
iv_list,
std::size_t error_return)
: key_(
utils::encryption::generate_key<utils::hash::hash_256_t>(token, cfg)),
stop_requested_cb_(std::move(stop_requested_cb)),
error_return_(error_return),
source_file_(utils::file::file::open_or_create_file(source_path, true)),
encrypted_file_name_(
utils::path::strip_to_file_name(std::string{encrypted_file_path})),
encrypted_file_path_(encrypted_file_path),
iv_list_(std::move(iv_list)) {
kdf_header_ = cfg.to_header();
common_initialize(false);
}
encrypting_reader::encrypting_reader(const encrypting_reader &reader)
@@ -334,17 +286,18 @@ encrypting_reader::encrypting_reader(const encrypting_reader &reader)
error_return_(reader.error_return_),
source_file_(
utils::file::file::open_file(reader.source_file_->get_path(), true)),
chunk_buffers_(reader.chunk_buffers_),
encrypted_file_name_(reader.encrypted_file_name_),
encrypted_file_path_(reader.encrypted_file_path_),
iv_list_(reader.iv_list_),
chunk_buffers_(reader.chunk_buffers_),
kdf_header_(reader.kdf_header_),
last_data_chunk_(reader.last_data_chunk_),
last_data_chunk_size_(reader.last_data_chunk_size_),
read_offset_(reader.read_offset_),
total_size_(reader.total_size_) {
REPERTORY_USES_FUNCTION_NAME();
if (not*source_file_) {
if (not *source_file_) {
throw utils::error::create_exception(
function_name, {
"file open failed",
@@ -354,15 +307,61 @@ encrypting_reader::encrypting_reader(const encrypting_reader &reader)
}
}
auto encrypting_reader::calculate_decrypted_size(std::uint64_t total_size)
void encrypting_reader::common_initialize(bool procces_iv_list) {
REPERTORY_USES_FUNCTION_NAME();
if (not *source_file_) {
throw utils::error::create_exception(function_name,
{
"file open failed",
source_file_->get_path(),
});
}
auto opt_size = source_file_->size();
if (not opt_size.has_value()) {
throw utils::error::create_exception(function_name,
{
"failed to get file size",
source_file_->get_path(),
});
}
auto file_size = opt_size.value();
auto total_chunks = utils::divide_with_ceiling(
file_size, static_cast<std::uint64_t>(data_chunk_size_));
total_size_ = file_size + (total_chunks * encryption_header_size) +
(kdf_header_.has_value() ? kdf_header_->size() : 0U);
last_data_chunk_ = total_chunks - 1U;
last_data_chunk_size_ = (file_size <= data_chunk_size_) ? file_size
: (file_size % data_chunk_size_) == 0U
? data_chunk_size_
: file_size % data_chunk_size_;
if (not procces_iv_list) {
return;
}
iv_list_.resize(total_chunks);
for (auto &data : iv_list_) {
randombytes_buf(data.data(), data.size());
}
}
auto encrypting_reader::calculate_decrypted_size(std::uint64_t total_size,
bool uses_kdf)
-> std::uint64_t {
if (uses_kdf) {
total_size -= kdf_config::size();
}
return total_size - (utils::divide_with_ceiling(
total_size, static_cast<std::uint64_t>(
get_encrypted_chunk_size())) *
encryption_header_size);
}
auto encrypting_reader::calculate_encrypted_size(std::string_view source_path)
auto encrypting_reader::calculate_encrypted_size(std::string_view source_path,
bool uses_kdf)
-> std::uint64_t {
REPERTORY_USES_FUNCTION_NAME();
@@ -377,9 +376,43 @@ auto encrypting_reader::calculate_encrypted_size(std::string_view source_path)
}
auto file_size{opt_size.value()};
const auto total_chunks = utils::divide_with_ceiling(
auto total_chunks = utils::divide_with_ceiling(
file_size, static_cast<std::uint64_t>(data_chunk_size_));
return file_size + (total_chunks * encryption_header_size);
return file_size + (total_chunks * encryption_header_size) +
(uses_kdf ? kdf_config::size() : 0U);
}
void encrypting_reader::create_encrypted_paths(
std::string_view file_name,
std::optional<std::string> relative_parent_path) {
data_buffer result;
utils::encryption::encrypt_data(
key_, reinterpret_cast<const unsigned char *>(file_name.data()),
file_name.size(), result);
if (kdf_header_.has_value()) {
result.insert(result.begin(), kdf_header_->begin(), kdf_header_->end());
}
encrypted_file_name_ = utils::collection::to_hex_string(result);
if (not relative_parent_path.has_value()) {
return;
}
for (const auto &part :
utils::string::split(relative_parent_path.value(),
utils::path::directory_seperator, false)) {
utils::encryption::encrypt_data(
key_, reinterpret_cast<const unsigned char *>(part.c_str()),
strnlen(part.c_str(), part.size()), result);
if (kdf_header_.has_value()) {
result.insert(result.begin(), kdf_header_->begin(), kdf_header_->end());
}
encrypted_file_path_ += '/' + utils::collection::to_hex_string(result);
}
encrypted_file_path_ += '/' + encrypted_file_name_;
}
auto encrypting_reader::create_iostream() const
@@ -388,24 +421,73 @@ auto encrypting_reader::create_iostream() const
std::make_unique<encrypting_streambuf>(*this));
}
auto encrypting_reader::get_kdf_config() const -> std::optional<kdf_config> {
REPERTORY_USES_FUNCTION_NAME();
if (not kdf_header_.has_value()) {
return std::nullopt;
}
kdf_config cfg;
if (not kdf_config::from_header(kdf_header_.value(), cfg)) {
throw utils::error::create_exception(function_name,
{
"invalid kdf header",
});
}
return cfg;
}
auto encrypting_reader::reader_function(char *buffer, size_t size,
size_t nitems) -> size_t {
REPERTORY_USES_FUNCTION_NAME();
const auto read_size = static_cast<std::size_t>(std::min(
static_cast<std::uint64_t>(size * nitems), total_size_ - read_offset_));
auto read_size =
static_cast<std::uint64_t>(size) * static_cast<std::uint64_t>(nitems);
if (read_size == 0U) {
return 0U;
}
auto chunk = read_offset_ / encrypted_chunk_size_;
auto chunk_offset = read_offset_ % encrypted_chunk_size_;
auto read_offset{read_offset_};
std::size_t total_read{};
auto total_size{total_size_};
auto ret = false;
if (read_offset_ < total_size_) {
if (kdf_header_.has_value()) {
total_size -= kdf_header_->size();
if (read_offset < kdf_header_->size()) {
auto to_read{
utils::calculate_read_size(kdf_header_->size(), read_size,
read_offset),
};
read_offset_ += to_read;
std::memcpy(buffer, &kdf_header_->data()[read_offset], to_read);
if (read_size - to_read == 0) {
return to_read;
}
read_offset = 0U;
read_size -= to_read;
total_read += to_read;
buffer += to_read;
} else {
read_offset -= kdf_header_->size();
}
}
auto chunk = static_cast<std::size_t>(read_offset / encrypted_chunk_size_);
auto chunk_offset =
static_cast<std::size_t>(read_offset % encrypted_chunk_size_);
auto remain = utils::calculate_read_size(total_size, read_size, read_offset);
auto ret{false};
if (read_offset < total_size) {
try {
ret = true;
auto remain = read_size;
while (not get_stop_requested() && ret && (remain != 0U)) {
if (chunk_buffers_.find(chunk) == chunk_buffers_.end()) {
if (not chunk_buffers_.contains(chunk)) {
auto &chunk_buffer = chunk_buffers_[chunk];
data_buffer file_data(chunk == last_data_chunk_
? last_data_chunk_size_
@@ -413,24 +495,26 @@ auto encrypting_reader::reader_function(char *buffer, size_t size,
chunk_buffer.resize(file_data.size() + encryption_header_size);
std::size_t bytes_read{};
if ((ret = source_file_->read(file_data, chunk * data_chunk_size_,
&bytes_read))) {
ret = source_file_->read(
file_data,
static_cast<std::uint64_t>(chunk) *
static_cast<std::uint64_t>(data_chunk_size_),
&bytes_read);
if (ret) {
utils::encryption::encrypt_data(iv_list_.at(chunk), key_, file_data,
chunk_buffer);
}
} else if (chunk) {
chunk_buffers_.erase(chunk - 1u);
} else if (chunk != 0U) {
chunk_buffers_.erase(chunk - 1U);
}
auto &chunk_buffer = chunk_buffers_[chunk];
const auto to_read = std::min(
static_cast<std::size_t>(chunk_buffer.size() - chunk_offset),
remain);
auto to_read = std::min(chunk_buffer.size() - chunk_offset, remain);
std::memcpy(buffer + total_read, &chunk_buffer[chunk_offset], to_read);
total_read += to_read;
remain -= to_read;
chunk_offset = 0u;
chunk++;
chunk_offset = 0U;
++chunk;
read_offset_ += to_read;
}
} catch (const std::exception &e) {