BlockStorage/repertory
1
0
Fork 0
Code Issues 3 Pull Requests Releases 9 Wiki Activity

prepare for argon2id

Browse Source
This commit is contained in:
sgraves76
2025-08-29 09:03:16 -05:00
parent c9362b8802
commit 57213c007f
3 changed files with 171 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
repertory/librepertory/include/providers/s3/s3_provider.hpp
View File

@@ -90,6 +90,11 @@ private:
return s3_config_;
}
[[nodiscard]] auto read_file_bytes(const std::string &api_path,
std::size_t size, std::uint64_t offset,
data_buffer &data, bool encrypted,
stop_type &stop_requested) -> api_error;
[[nodiscard]] auto set_meta_key(const std::string &api_path,
api_meta_map &meta) -> api_error;

11
repertory/librepertory/include/types/repertory.hpp
View File

@@ -68,6 +68,7 @@ inline constexpr std::string META_CHANGED{"changed"};
inline constexpr std::string META_CREATION{"creation"};
inline constexpr std::string META_DIRECTORY{"directory"};
inline constexpr std::string META_GID{"gid"};
inline constexpr std::string META_KDF{"kdf"};
inline constexpr std::string META_KEY{"key"};
inline constexpr std::string META_MODE{"mode"};
inline constexpr std::string META_MODIFIED{"modified"};
@@ -78,11 +79,11 @@ inline constexpr std::string META_SOURCE{"source"};
inline constexpr std::string META_UID{"uid"};
inline constexpr std::string META_WRITTEN{"written"};
inline constexpr std::array<std::string, 16U> META_USED_NAMES = {
META_ACCESSED, META_ATTRIBUTES, META_BACKUP, META_CHANGED,
META_CREATION, META_DIRECTORY, META_GID, META_KEY,
META_MODE, META_MODIFIED, META_OSXFLAGS, META_PINNED,
META_SIZE, META_SOURCE, META_UID, META_WRITTEN,
inline constexpr std::array<std::string, 17U> META_USED_NAMES = {
META_ACCESSED, META_ATTRIBUTES, META_BACKUP, META_CHANGED, META_CREATION,
META_DIRECTORY, META_GID, META_KDF, META_KEY, META_MODE,
META_MODIFIED, META_OSXFLAGS, META_PINNED, META_SIZE, META_SOURCE,
META_UID, META_WRITTEN,
};
using api_meta_map = std::map<std::string, std::string>;

278
repertory/librepertory/src/providers/s3/s3_provider.cpp
View File

@@ -815,124 +815,6 @@ auto s3_provider::is_online() const -> bool {
return false;
}
auto s3_provider::read_file_bytes(const std::string &api_path, std::size_t size,
std::uint64_t offset, data_buffer &data,
stop_type &stop_requested) -> api_error {
REPERTORY_USES_FUNCTION_NAME();
try {
const auto &cfg{get_s3_config()};
auto is_encrypted{not cfg.encryption_token.empty()};
std::string key;
if (is_encrypted) {
auto res{get_item_meta(api_path, META_KEY, key)};
if (res != api_error::success) {
return res;
}
}
auto object_name{
utils::path::create_api_path(is_encrypted ? key : api_path),
};
const auto read_bytes =
[this, &api_path, &cfg, &object_name,
&stop_requested](std::size_t read_size, std::size_t read_offset,
data_buffer &read_buffer) -> api_error {
auto res{api_error::error};
for (std::uint32_t idx{0U};
not(stop_requested || app_config::get_stop_requested()) &&
res != api_error::success &&
idx < get_config().get_retry_read_count() + 1U;
++idx) {
if (idx > 0U) {
read_buffer.clear();
std::this_thread::sleep_for(1s);
}
curl::requests::http_get get{};
get.aws_service = "aws:amz:" + cfg.region + ":s3";
get.headers["response-content-type"] = "binary/octet-stream";
get.range = {{
.begin = read_offset,
.end = read_offset + read_size - 1U,
}};
get.response_handler = [&read_buffer](auto &&response_data,
long /*response_code*/) {
read_buffer = response_data;
};
res = set_request_path(get, object_name);
if (res != api_error::success) {
return res;
}
const auto notify_retry = [=](long response_code) {
auto msg =
fmt::format("read file bytes failed|offset|{}|size|{}|retry|{}",
std::to_string(read_offset),
std::to_string(read_size), std::to_string(idx + 1U));
if (response_code == 0) {
utils::error::raise_api_path_error(function_name, api_path,
api_error::comm_error, msg);
} else {
utils::error::raise_api_path_error(function_name, api_path,
response_code, msg);
}
};
long response_code{};
if (not get_comm().make_request(get, response_code, stop_requested)) {
notify_retry(response_code);
continue;
}
if (response_code < http_error_codes::ok ||
response_code >= http_error_codes::multiple_choices) {
notify_retry(response_code);
continue;
}
res = api_error::success;
}
return res;
};
if (not is_encrypted) {
return read_bytes(size, offset, data);
}
std::string temp;
auto res{get_item_meta(api_path, META_SIZE, temp)};
if (res != api_error::success) {
return res;
}
auto total_size{utils::string::to_uint64(temp)};
return utils::encryption::read_encrypted_range(
{.begin = offset, .end = offset + size - 1U},
utils::encryption::generate_key<utils::hash::hash_256_t>(
cfg.encryption_token),
[&](data_buffer &ct_buffer, std::uint64_t start_offset,
std::uint64_t end_offset) -> bool {
return read_bytes((end_offset - start_offset + 1U),
start_offset,
ct_buffer) == api_error::success;
},
total_size, data)
? api_error::success
: api_error::decryption_error;
} catch (const std::exception &e) {
utils::error::raise_error(function_name, e, "exception occurred");
}
return api_error::error;
}
auto s3_provider::remove_directory_impl(const std::string &api_path)
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
@@ -1085,7 +967,40 @@ auto s3_provider::start(api_item_added_callback api_item_added,
event_system::instance().raise<service_start_begin>(function_name,
"s3_provider");
const auto &cfg{get_s3_config()};
auto ret = base_provider::start(api_item_added, mgr);
if (ret && not cfg.encryption_token.empty()) {
std::string kdf_str;
auto res = get_item_meta("/", META_KDF, kdf_str);
ret == res == api_error::success;
if (ret) {
if (kdf_str.empty()) {
if (not search_keys_for_kdf()) {
kdf_.seal();
res = set_item_meta("/", META_KDF, nlohmann::json(kdf_).dump());
ret == res == api_error::success;
if (not ret) {
utils::error::raise_api_path_error(function_name, "/", res,
"set item meta failed");
}
}
} else {
kdf_ = nlohmann::json::parse(kdf_str).get<kdf_config>();
}
} else {
utils::error::raise_api_path_error(function_name, "/", res,
"get item meta failed");
}
if (ret) {
key_ = utils::encryption::recreate_key<utils::hash::hash_256_t>(
cfg.encryption_token, kdf);
} else {
base_provider::stop();
}
}
event_system::instance().raise<service_start_end>(function_name,
"s3_provider");
return ret;
@@ -1165,4 +1080,131 @@ auto s3_provider::upload_file_impl(const std::string &api_path,
return api_error::success;
}
auto s3_provider::read_file_bytes(const std::string &api_path, std::size_t size,
std::uint64_t offset, data_buffer &data,
stop_type &stop_requested) -> api_error {
return read_file_bytes(api_path, size, offset, data,
not get_s3_config().encryption_token.empty(),
stop_requested);
}
auto s3_provider::read_file_bytes(const std::string &api_path, std::size_t size,
std::uint64_t offset, data_buffer &data,
bool encrypted, stop_type &stop_requested)
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
try {
const auto &cfg{get_s3_config()};
std::string key;
if (encrypted) {
auto res{get_item_meta(api_path, META_KEY, key)};
if (res != api_error::success) {
return res;
}
}
auto object_name{
utils::path::create_api_path(encrypted ? key : api_path),
};
const auto read_bytes =
[this, &api_path, &cfg, &object_name,
&stop_requested](std::size_t read_size, std::size_t read_offset,
data_buffer &read_buffer) -> api_error {
auto res{api_error::error};
for (std::uint32_t idx{0U};
not(stop_requested || app_config::get_stop_requested()) &&
res != api_error::success &&
idx < get_config().get_retry_read_count() + 1U;
++idx) {
if (idx > 0U) {
read_buffer.clear();
std::this_thread::sleep_for(1s);
}
curl::requests::http_get get{};
get.aws_service = "aws:amz:" + cfg.region + ":s3";
get.headers["response-content-type"] = "binary/octet-stream";
get.range = {{
.begin = read_offset,
.end = read_offset + read_size - 1U,
}};
get.response_handler = [&read_buffer](auto &&response_data,
long /*response_code*/) {
read_buffer = response_data;
};
res = set_request_path(get, object_name);
if (res != api_error::success) {
return res;
}
const auto notify_retry = [=](long response_code) {
auto msg =
fmt::format("read file bytes failed|offset|{}|size|{}|retry|{}",
std::to_string(read_offset),
std::to_string(read_size), std::to_string(idx + 1U));
if (response_code == 0) {
utils::error::raise_api_path_error(function_name, api_path,
api_error::comm_error, msg);
} else {
utils::error::raise_api_path_error(function_name, api_path,
response_code, msg);
}
};
long response_code{};
if (not get_comm().make_request(get, response_code, stop_requested)) {
notify_retry(response_code);
continue;
}
if (response_code < http_error_codes::ok ||
response_code >= http_error_codes::multiple_choices) {
notify_retry(response_code);
continue;
}
res = api_error::success;
}
return res;
};
if (not encrypted) {
return read_bytes(size, offset, data);
}
std::string temp;
auto res{get_item_meta(api_path, META_SIZE, temp)};
if (res != api_error::success) {
return res;
}
auto total_size{utils::string::to_uint64(temp)};
return utils::encryption::read_encrypted_range(
{.begin = offset, .end = offset + size - 1U},
utils::encryption::generate_key<utils::hash::hash_256_t>(
cfg.encryption_token),
[&](data_buffer &ct_buffer, std::uint64_t start_offset,
std::uint64_t end_offset) -> bool {
return read_bytes((end_offset - start_offset + 1U),
start_offset,
ct_buffer) == api_error::success;
},
total_size, data)
? api_error::success
: api_error::decryption_error;
} catch (const std::exception &e) {
utils::error::raise_error(function_name, e, "exception occurred");
}
return api_error::error;
}
} // namespace repertory