BlockStorage/repertory
1
0
Fork 0
Code Issues 7 Pull Requests Releases 6 Wiki Activity

refactor s3 provider

Browse Source
This commit is contained in:
Scott E. Graves 2023-11-11 15:35:35 -06:00
parent f88239a13e
commit b87e1df140
4 changed files with 221 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/types/repertory.hpp
View File

@ -169,7 +169,7 @@ struct directory_item {
std::string api_parent;
bool directory{false};
std::uint64_t size{};
api_meta_map meta;
api_meta_map meta{};
bool resolved{false};
[[nodiscard]] static auto from_json(const json &item) -> directory_item {

64
src/comm/curl/requests/http_put_file.cpp
View File

@ -27,40 +27,42 @@ namespace repertory::curl::requests {
auto http_put_file::set_method(CURL *curl, stop_type &stop_requested) const
-> bool {
curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "PUT");
curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L);
if (not source_path.empty()) {
curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L);
if (reader) {
curl_easy_setopt(curl, CURLOPT_READDATA, reader.get());
curl_easy_setopt(
curl, CURLOPT_READFUNCTION,
static_cast<curl_read_callback>(
utils::encryption::encrypting_reader::reader_function));
curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE,
reader->get_total_size());
return true;
}
read_info = std::make_shared<read_file_info>(read_file_info{
stop_requested,
});
if (native_file::open(source_path, read_info->nf) != api_error::success) {
return false;
}
read_info->nf->set_auto_close(true);
std::uint64_t file_size{};
if (not read_info->nf->get_file_size(file_size)) {
return false;
}
curl_easy_setopt(curl, CURLOPT_READDATA, read_info.get());
curl_easy_setopt(curl, CURLOPT_READFUNCTION, read_file_data);
curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, file_size);
if (source_path.empty()) {
curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, 0L);
return true;
}
if (reader) {
curl_easy_setopt(curl, CURLOPT_READDATA, reader.get());
curl_easy_setopt(
curl, CURLOPT_READFUNCTION,
static_cast<curl_read_callback>(
utils::encryption::encrypting_reader::reader_function));
curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, reader->get_total_size());
return true;
}
read_info = std::make_shared<read_file_info>(read_file_info{
stop_requested,
});
if (native_file::open(source_path, read_info->nf) != api_error::success) {
return false;
}
read_info->nf->set_auto_close(true);
std::uint64_t file_size{};
if (not read_info->nf->get_file_size(file_size)) {
return false;
}
curl_easy_setopt(curl, CURLOPT_READDATA, read_info.get());
curl_easy_setopt(curl, CURLOPT_READFUNCTION, read_file_data);
curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, file_size);
return true;
}
} // namespace repertory::curl::requests

193
src/providers/s3/s3_provider.cpp
View File

@ -132,24 +132,36 @@ auto s3_provider::create_directory(const std::string &api_path,
try {
const auto cfg = config_.get_s3_config();
const auto is_encrypted = not cfg.encryption_token.empty();
stop_type stop_requested{false};
std::string key;
if (is_encrypted) {
res = get_item_meta(api_path, META_KEY, key);
std::string encrypted_file_path;
res = get_item_meta(utils::path::get_parent_api_path(api_path), META_KEY,
encrypted_file_path);
if (res != api_error::success) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, res,
"failed to create file");
return res;
}
data_buffer result;
utils::encryption::encrypt_data(
cfg.encryption_token,
*(utils::string::split(api_path, '/', false).end() - 1U), result);
meta[META_KEY] = utils::path::create_api_path(utils::path::combine(
utils::path::create_api_path(encrypted_file_path),
{utils::to_hex_string(result)}));
}
const auto object_name =
utils::path::create_api_path(is_encrypted ? key : api_path);
utils::path::create_api_path(is_encrypted ? meta[META_KEY] : api_path);
curl::requests::http_put_file put_file{};
put_file.allow_timeout = true;
put_file.aws_service = "aws:amz:" + cfg.region + ":s3";
put_file.path = object_name + '/';
stop_type stop_requested{false};
long response_code{};
if (not comm_.make_request(put_file, response_code, stop_requested)) {
utils::error::raise_api_path_error(__FUNCTION__, api_path,
@ -249,8 +261,29 @@ auto s3_provider::create_file(const std::string &api_path, api_meta_map &meta)
return api_error::item_exists;
}
stop_type stop_requested{false};
try {
meta[META_DIRECTORY] = utils::string::from_bool(false);
if (not config_.get_s3_config().encryption_token.empty()) {
std::string encrypted_file_path;
res = get_item_meta(utils::path::get_parent_api_path(api_path), META_KEY,
encrypted_file_path);
if (res != api_error::success) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, res,
"failed to create file");
return res;
}
data_buffer result;
utils::encryption::encrypt_data(
config_.get_s3_config().encryption_token,
*(utils::string::split(api_path, '/', false).end() - 1U), result);
meta[META_KEY] = utils::path::create_api_path(utils::path::combine(
utils::path::create_api_path(encrypted_file_path),
{utils::to_hex_string(result)}));
}
res = set_item_meta(api_path, meta);
if (res != api_error::success) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, res,
@ -258,8 +291,12 @@ auto s3_provider::create_file(const std::string &api_path, api_meta_map &meta)
return res;
}
stop_type stop_requested{false};
return upload_file(api_path, meta[META_SOURCE], stop_requested);
res = upload_file(api_path, meta[META_SOURCE], stop_requested);
if (res != api_error::success) {
db_->Delete(rocksdb::WriteOptions(), api_path);
}
return res;
} catch (const std::exception &e) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, e,
"failed to create file");
@ -359,11 +396,135 @@ auto s3_provider::get_api_path_from_source(const std::string &source_path,
}
auto s3_provider::get_directory_item_count(const std::string &api_path) const
-> std::uint64_t {}
-> std::uint64_t {
// TODO implement this
return 0U;
}
auto s3_provider::get_directory_items(const std::string &api_path,
directory_item_list &list) const
-> api_error {}
-> api_error {
try {
const auto cfg = config_.get_s3_config();
const auto is_encrypted = not cfg.encryption_token.empty();
std::string key;
if (is_encrypted) {
auto res = get_item_meta(api_path, META_KEY, key);
if (res != api_error::success) {
return res;
}
}
const auto object_name =
api_path == "/"
? ""
: utils::path::create_api_path(is_encrypted ? key : api_path);
std::string response_data{};
long response_code{};
auto prefix = object_name.empty() ? object_name : object_name + "/";
if (not get_object_list(comm_, config_.get_s3_config(), response_data,
response_code, "/", prefix)) {
return api_error::comm_error;
}
if (response_code == http_error_codes::not_found) {
return api_error::directory_not_found;
}
if (response_code != http_error_codes::ok) {
return api_error::comm_error;
}
pugi::xml_document doc;
auto res = doc.load_string(response_data.c_str());
if (res.status != pugi::xml_parse_status::status_ok) {
return api_error::error;
}
const auto add_directory_item =
[&](bool directory, const std::string &name,
std::function<std::uint64_t(const directory_item &)> get_size)
-> api_error {
auto child_api_path =
utils::path::create_api_path(utils::path::combine("/", {name}));
std::string child_object_name;
if (is_encrypted) {
child_object_name = child_api_path;
auto res = utils::encryption::decrypt_file_path(cfg.encryption_token,
child_api_path);
if (res != api_error::success) {
return res;
}
}
directory_item dir_item{};
dir_item.api_path = child_api_path;
dir_item.api_parent = utils::path::get_parent_api_path(dir_item.api_path);
dir_item.directory = directory;
dir_item.size = get_size(dir_item);
auto res = get_item_meta(child_api_path, dir_item.meta);
if (not directory && res == api_error::item_not_found) {
api_file file{};
res = get_file(child_api_path, file);
}
if (res != api_error::success) {
return res;
}
list.push_back(std::move(dir_item));
return api_error::success;
};
auto node_list =
doc.select_nodes("/ListBucketResult/CommonPrefixes/Prefix");
for (const auto &node : node_list) {
add_directory_item(
true, node.node().text().as_string(),
[](const directory_item &) -> std::uint64_t { return 0U; });
}
node_list = doc.select_nodes("/ListBucketResult/Contents");
for (const auto &node : node_list) {
auto child_object_name =
std::string{node.node().select_node("Key").node().text().as_string()};
if (child_object_name != prefix) {
auto size = node.node().select_node("Size").node().text().as_ullong();
add_directory_item(
false, child_object_name,
[&is_encrypted, &size](const directory_item &) -> std::uint64_t {
return is_encrypted ? utils::encryption::encrypting_reader::
calculate_decrypted_size(size)
: size;
});
}
}
std::sort(list.begin(), list.end(),
[](const auto &item1, const auto &item2) -> bool {
return (item1.directory && not item2.directory) ||
(not(item2.directory && not item1.directory) &&
(item1.api_path.compare(item2.api_path) < 0));
});
list.insert(list.begin(), directory_item{
"..",
"",
true,
});
list.insert(list.begin(), directory_item{
".",
"",
true,
});
return api_error::success;
} catch (const std::exception &e) {
utils::error::raise_error(__FUNCTION__, e, "exception occurred");
}
return api_error::error;
}
auto s3_provider::get_file(const std::string &api_path, api_file &file) const
-> api_error {
@ -906,9 +1067,13 @@ void s3_provider::remove_deleted_files() {
}
}
auto s3_provider::remove_directory(const std::string &api_path) -> api_error {}
auto s3_provider::remove_directory(const std::string &api_path) -> api_error {
return api_error::not_implemented;
}
auto s3_provider::remove_file(const std::string &api_path) -> api_error {}
auto s3_provider::remove_file(const std::string &api_path) -> api_error {
return api_error::not_implemented;
}
auto s3_provider::remove_item_meta(const std::string &api_path,
const std::string &key) -> api_error {
@ -1055,6 +1220,12 @@ auto s3_provider::upload_file(const std::string &api_path,
return error;
};
try {
std::uint64_t file_size{};
if (utils::file::is_file(source_path) &&
not utils::file::get_file_size(source_path, file_size)) {
return notify_end(api_error::comm_error);
}
const auto cfg = config_.get_s3_config();
const auto is_encrypted = not cfg.encryption_token.empty();
@ -1074,7 +1245,7 @@ auto s3_provider::upload_file(const std::string &api_path,
put_file.path = object_name;
put_file.source_path = source_path;
if (is_encrypted) {
if (is_encrypted && file_size > 0U) {
static stop_type no_stop{false};
put_file.reader = std::make_shared<utils::encryption::encrypting_reader>(

8
src/utils/encrypting_reader.cpp
View File

@ -172,7 +172,7 @@ encrypting_reader::encrypting_reader(
: key_(utils::encryption::generate_key(token)),
stop_requested_(stop_requested),
error_return_(error_return) {
const auto res = native_file::open(
const auto res = native_file::create_or_open(
source_path, not relative_parent_path.has_value(), source_file_);
if (res != api_error::success) {
throw std::runtime_error("file open failed|src|" + source_path + '|' +
@ -225,7 +225,8 @@ encrypting_reader::encrypting_reader(const std::string &encrypted_file_path,
: key_(utils::encryption::generate_key(token)),
stop_requested_(stop_requested),
error_return_(error_return) {
const auto res = native_file::open(source_path, false, source_file_);
const auto res =
native_file::create_or_open(source_path, false, source_file_);
if (res != api_error::success) {
throw std::runtime_error("file open failed|src|" + source_path + '|' +
api_error_to_string(res));
@ -266,7 +267,8 @@ encrypting_reader::encrypting_reader(
: key_(utils::encryption::generate_key(token)),
stop_requested_(stop_requested),
error_return_(error_return) {
const auto res = native_file::open(source_path, false, source_file_);
const auto res =
native_file::create_or_open(source_path, false, source_file_);
if (res != api_error::success) {
throw std::runtime_error("file open failed|src|" + source_path + '|' +
api_error_to_string(res));