BlockStorage/repertory
1
0
Fork 0
Code Issues 11 Pull Requests Releases 7 Wiki Activity

refactor s3 provider

Browse Source
This commit is contained in:
Scott E. Graves
2023-11-11 11:32:14 -06:00
parent f2c1f64f02
commit 68476cbc00
55 changed files with 1504 additions and 5050 deletions
Download Patch File Download Diff File Expand all files Collapse all files

120
src/utils/encrypting_reader.cpp
View File

@@ -31,6 +31,12 @@
namespace repertory::utils::encryption {
class encrypting_streambuf final : public encrypting_reader::streambuf {
public:
encrypting_streambuf(const encrypting_streambuf &) = default;
encrypting_streambuf(encrypting_streambuf &&) = delete;
auto operator=(const encrypting_streambuf &)
-> encrypting_streambuf & = delete;
auto operator=(encrypting_streambuf &&) -> encrypting_streambuf & = delete;
explicit encrypting_streambuf(const encrypting_reader &reader)
: reader_(reader) {
setg(reinterpret_cast<char *>(0), reinterpret_cast<char *>(0),
@@ -102,7 +108,7 @@ protected:
static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(gptr())));
char c{};
const auto res = reader_.reader_function(&c, 1u, 1u, &reader_);
const auto res = encrypting_reader::reader_function(&c, 1U, 1U, &reader_);
if (res != 1) {
return traits_type::eof();
}
@@ -118,7 +124,8 @@ protected:
reader_.set_read_position(
static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(gptr())));
const auto res = reader_.reader_function(ptr, 1u, count, &reader_);
const auto res = encrypting_reader::reader_function(
ptr, 1U, static_cast<std::size_t>(count), &reader_);
if ((res == reader_.get_error_return()) ||
(reader_.get_stop_requested() && (res == CURL_READFUNC_ABORT))) {
return traits_type::eof();
@@ -126,12 +133,20 @@ protected:
setg(eback(), gptr() + res,
reinterpret_cast<char *>(reader_.get_total_size()));
return res;
return static_cast<std::streamsize>(res);
}
};
class encrypting_reader_iostream final : public encrypting_reader::iostream {
public:
encrypting_reader_iostream(const encrypting_reader_iostream &) = delete;
encrypting_reader_iostream(encrypting_reader_iostream &&) = delete;
auto operator=(const encrypting_reader_iostream &)
-> encrypting_reader_iostream & = delete;
auto operator=(encrypting_reader_iostream &&)
-> encrypting_reader_iostream & = delete;
explicit encrypting_reader_iostream(
std::unique_ptr<encrypting_streambuf> buffer)
: encrypting_reader::iostream(buffer.get()), buffer_(std::move(buffer)) {}
@@ -146,14 +161,14 @@ const std::size_t encrypting_reader::header_size_ = ([]() {
return crypto_aead_xchacha20poly1305_IETF_NPUBBYTES +
crypto_aead_xchacha20poly1305_IETF_ABYTES;
})();
const std::size_t encrypting_reader::data_chunk_size_ = (8u * 1024u * 1024u);
const std::size_t encrypting_reader::data_chunk_size_ = (8UL * 1024UL * 1024UL);
const std::size_t encrypting_reader::encrypted_chunk_size_ =
data_chunk_size_ + header_size_;
encrypting_reader::encrypting_reader(
const std::string &file_name, const std::string &source_path,
stop_type &stop_requested, const std::string &token,
std::optional<std::string> relative_parent_path, const size_t error_return)
std::optional<std::string> relative_parent_path, std::size_t error_return)
: key_(utils::encryption::generate_key(token)),
stop_requested_(stop_requested),
error_return_(error_return) {
@@ -191,24 +206,22 @@ encrypting_reader::encrypting_reader(
file_size, static_cast<std::uint64_t>(data_chunk_size_)));
total_size_ =
file_size + (total_chunks * encrypting_reader::get_header_size());
last_data_chunk_ = total_chunks - 1u;
last_data_chunk_ = total_chunks - 1U;
last_data_chunk_size_ = static_cast<std::size_t>(
(file_size <= data_chunk_size_) ? file_size
: (file_size % data_chunk_size_) ? file_size % data_chunk_size_
: data_chunk_size_);
(file_size <= data_chunk_size_) ? file_size
: (file_size % data_chunk_size_) == 0U ? data_chunk_size_
: file_size % data_chunk_size_);
iv_list_.resize(total_chunks);
for (auto &iv : iv_list_) {
randombytes_buf(iv.data(), iv.size());
}
}
encrypting_reader::encrypting_reader(
const std::string &encrypted_file_path, const std::string &source_path,
stop_type &stop_requested, const std::string &token,
std::vector<
std::array<unsigned char, crypto_aead_xchacha20poly1305_IETF_NPUBBYTES>>
iv_list,
const size_t error_return)
encrypting_reader::encrypting_reader(const std::string &encrypted_file_path,
const std::string &source_path,
stop_type &stop_requested,
const std::string &token,
std::size_t error_return)
: key_(utils::encryption::generate_key(token)),
stop_requested_(stop_requested),
error_return_(error_return) {
@@ -232,27 +245,68 @@ encrypting_reader::encrypting_reader(
file_size, static_cast<std::uint64_t>(data_chunk_size_)));
total_size_ =
file_size + (total_chunks * encrypting_reader::get_header_size());
last_data_chunk_ = total_chunks - 1u;
last_data_chunk_ = total_chunks - 1U;
last_data_chunk_size_ = static_cast<std::size_t>(
(file_size <= data_chunk_size_) ? file_size
: (file_size % data_chunk_size_) ? file_size % data_chunk_size_
: data_chunk_size_);
(file_size <= data_chunk_size_) ? file_size
: (file_size % data_chunk_size_) == 0U ? data_chunk_size_
: file_size % data_chunk_size_);
iv_list_.resize(total_chunks);
for (auto &iv : iv_list_) {
randombytes_buf(iv.data(), iv.size());
}
}
encrypting_reader::encrypting_reader(
const std::string &encrypted_file_path, const std::string &source_path,
stop_type &stop_requested, const std::string &token,
std::vector<
std::array<unsigned char, crypto_aead_xchacha20poly1305_IETF_NPUBBYTES>>
iv_list,
std::size_t error_return)
: key_(utils::encryption::generate_key(token)),
stop_requested_(stop_requested),
error_return_(error_return) {
const auto res = native_file::open(source_path, false, source_file_);
if (res != api_error::success) {
throw std::runtime_error("file open failed|src|" + source_path + '|' +
api_error_to_string(res));
}
encrypted_file_path_ = encrypted_file_path;
encrypted_file_name_ =
std::filesystem::path(encrypted_file_path_).filename().string();
std::uint64_t file_size{};
if (not utils::file::get_file_size(source_path, file_size)) {
throw std::runtime_error("get file size failed|src|" + source_path + '|' +
std::to_string(utils::get_last_error_code()));
}
const auto total_chunks = static_cast<std::size_t>(utils::divide_with_ceiling(
file_size, static_cast<std::uint64_t>(data_chunk_size_)));
total_size_ =
file_size + (total_chunks * encrypting_reader::get_header_size());
last_data_chunk_ = total_chunks - 1U;
last_data_chunk_size_ = static_cast<std::size_t>(
(file_size <= data_chunk_size_) ? file_size
: (file_size % data_chunk_size_) == 0U ? data_chunk_size_
: file_size % data_chunk_size_);
iv_list_ = std::move(iv_list);
}
encrypting_reader::encrypting_reader(const encrypting_reader &r)
: key_(r.key_),
stop_requested_(r.stop_requested_),
error_return_(r.error_return_),
chunk_buffers_(r.chunk_buffers_),
encrypted_file_name_(r.encrypted_file_name_),
encrypted_file_path_(r.encrypted_file_path_),
iv_list_(r.iv_list_),
last_data_chunk_(r.last_data_chunk_),
last_data_chunk_size_(r.last_data_chunk_size_),
read_offset_(r.read_offset_),
source_file_(native_file::clone(r.source_file_)),
total_size_(r.total_size_) {}
encrypting_reader::encrypting_reader(const encrypting_reader &reader)
: key_(reader.key_),
stop_requested_(reader.stop_requested_),
error_return_(reader.error_return_),
chunk_buffers_(reader.chunk_buffers_),
encrypted_file_name_(reader.encrypted_file_name_),
encrypted_file_path_(reader.encrypted_file_path_),
iv_list_(reader.iv_list_),
last_data_chunk_(reader.last_data_chunk_),
last_data_chunk_size_(reader.last_data_chunk_size_),
read_offset_(reader.read_offset_),
source_file_(native_file::clone(reader.source_file_)),
total_size_(reader.total_size_) {}
encrypting_reader::~encrypting_reader() {
if (source_file_) {

6
src/utils/encryption.cpp
View File

@@ -104,7 +104,7 @@ auto read_encrypted_range(
const auto start_chunk =
static_cast<std::size_t>(range.begin / data_chunk_size);
const auto end_chunk = static_cast<std::size_t>(range.end / data_chunk_size);
auto remain = range.end - range.begin + 1u;
auto remain = range.end - range.begin + 1U;
auto source_offset = static_cast<std::size_t>(range.begin % data_chunk_size);
for (std::size_t chunk = start_chunk; chunk <= end_chunk; chunk++) {
@@ -112,8 +112,8 @@ auto read_encrypted_range(
const auto start_offset = chunk * encrypted_chunk_size;
const auto end_offset = std::min(
start_offset + (total_size - (chunk * data_chunk_size)) + header_size -
1u,
static_cast<std::uint64_t>(start_offset + encrypted_chunk_size - 1u));
1U,
static_cast<std::uint64_t>(start_offset + encrypted_chunk_size - 1U));
const auto result = reader(ct, start_offset, end_offset);
if (result != api_error::success) {