From b87e1df140f5a8cd046bf5ebba67903a1d968f1a Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Sat, 11 Nov 2023 15:35:35 -0600 Subject: [PATCH] refactor s3 provider --- include/types/repertory.hpp | 2 +- src/comm/curl/requests/http_put_file.cpp | 64 ++++---- src/providers/s3/s3_provider.cpp | 193 +++++++++++++++++++++-- src/utils/encrypting_reader.cpp | 8 +- 4 files changed, 221 insertions(+), 46 deletions(-) diff --git a/include/types/repertory.hpp b/include/types/repertory.hpp index 6aeb7d7e..9c086f04 100644 --- a/include/types/repertory.hpp +++ b/include/types/repertory.hpp @@ -169,7 +169,7 @@ struct directory_item { std::string api_parent; bool directory{false}; std::uint64_t size{}; - api_meta_map meta; + api_meta_map meta{}; bool resolved{false}; [[nodiscard]] static auto from_json(const json &item) -> directory_item { diff --git a/src/comm/curl/requests/http_put_file.cpp b/src/comm/curl/requests/http_put_file.cpp index ed6f7e9b..2149e821 100644 --- a/src/comm/curl/requests/http_put_file.cpp +++ b/src/comm/curl/requests/http_put_file.cpp @@ -27,40 +27,42 @@ namespace repertory::curl::requests { auto http_put_file::set_method(CURL *curl, stop_type &stop_requested) const -> bool { curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "PUT"); + curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L); - if (not source_path.empty()) { - curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L); - if (reader) { - curl_easy_setopt(curl, CURLOPT_READDATA, reader.get()); - curl_easy_setopt( - curl, CURLOPT_READFUNCTION, - static_cast( - utils::encryption::encrypting_reader::reader_function)); - curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, - reader->get_total_size()); - return true; - } - - read_info = std::make_shared(read_file_info{ - stop_requested, - }); - - if (native_file::open(source_path, read_info->nf) != api_error::success) { - return false; - } - - read_info->nf->set_auto_close(true); - - std::uint64_t file_size{}; - if (not read_info->nf->get_file_size(file_size)) { - return false; - } - - curl_easy_setopt(curl, CURLOPT_READDATA, read_info.get()); - curl_easy_setopt(curl, CURLOPT_READFUNCTION, read_file_data); - curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, file_size); + if (source_path.empty()) { + curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, 0L); + return true; } + if (reader) { + curl_easy_setopt(curl, CURLOPT_READDATA, reader.get()); + curl_easy_setopt( + curl, CURLOPT_READFUNCTION, + static_cast( + utils::encryption::encrypting_reader::reader_function)); + curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, reader->get_total_size()); + return true; + } + + read_info = std::make_shared(read_file_info{ + stop_requested, + }); + + if (native_file::open(source_path, read_info->nf) != api_error::success) { + return false; + } + + read_info->nf->set_auto_close(true); + + std::uint64_t file_size{}; + if (not read_info->nf->get_file_size(file_size)) { + return false; + } + + curl_easy_setopt(curl, CURLOPT_READDATA, read_info.get()); + curl_easy_setopt(curl, CURLOPT_READFUNCTION, read_file_data); + curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, file_size); + return true; } } // namespace repertory::curl::requests diff --git a/src/providers/s3/s3_provider.cpp b/src/providers/s3/s3_provider.cpp index 62e6e4a6..291e9d8d 100644 --- a/src/providers/s3/s3_provider.cpp +++ b/src/providers/s3/s3_provider.cpp @@ -132,24 +132,36 @@ auto s3_provider::create_directory(const std::string &api_path, try { const auto cfg = config_.get_s3_config(); const auto is_encrypted = not cfg.encryption_token.empty(); + stop_type stop_requested{false}; - std::string key; if (is_encrypted) { - res = get_item_meta(api_path, META_KEY, key); + std::string encrypted_file_path; + res = get_item_meta(utils::path::get_parent_api_path(api_path), META_KEY, + encrypted_file_path); if (res != api_error::success) { + utils::error::raise_api_path_error(__FUNCTION__, api_path, res, + "failed to create file"); return res; } + + data_buffer result; + utils::encryption::encrypt_data( + cfg.encryption_token, + *(utils::string::split(api_path, '/', false).end() - 1U), result); + + meta[META_KEY] = utils::path::create_api_path(utils::path::combine( + utils::path::create_api_path(encrypted_file_path), + {utils::to_hex_string(result)})); } const auto object_name = - utils::path::create_api_path(is_encrypted ? key : api_path); + utils::path::create_api_path(is_encrypted ? meta[META_KEY] : api_path); curl::requests::http_put_file put_file{}; put_file.allow_timeout = true; put_file.aws_service = "aws:amz:" + cfg.region + ":s3"; put_file.path = object_name + '/'; - stop_type stop_requested{false}; long response_code{}; if (not comm_.make_request(put_file, response_code, stop_requested)) { utils::error::raise_api_path_error(__FUNCTION__, api_path, @@ -249,8 +261,29 @@ auto s3_provider::create_file(const std::string &api_path, api_meta_map &meta) return api_error::item_exists; } + stop_type stop_requested{false}; try { meta[META_DIRECTORY] = utils::string::from_bool(false); + if (not config_.get_s3_config().encryption_token.empty()) { + std::string encrypted_file_path; + res = get_item_meta(utils::path::get_parent_api_path(api_path), META_KEY, + encrypted_file_path); + if (res != api_error::success) { + utils::error::raise_api_path_error(__FUNCTION__, api_path, res, + "failed to create file"); + return res; + } + + data_buffer result; + utils::encryption::encrypt_data( + config_.get_s3_config().encryption_token, + *(utils::string::split(api_path, '/', false).end() - 1U), result); + + meta[META_KEY] = utils::path::create_api_path(utils::path::combine( + utils::path::create_api_path(encrypted_file_path), + {utils::to_hex_string(result)})); + } + res = set_item_meta(api_path, meta); if (res != api_error::success) { utils::error::raise_api_path_error(__FUNCTION__, api_path, res, @@ -258,8 +291,12 @@ auto s3_provider::create_file(const std::string &api_path, api_meta_map &meta) return res; } - stop_type stop_requested{false}; - return upload_file(api_path, meta[META_SOURCE], stop_requested); + res = upload_file(api_path, meta[META_SOURCE], stop_requested); + if (res != api_error::success) { + db_->Delete(rocksdb::WriteOptions(), api_path); + } + + return res; } catch (const std::exception &e) { utils::error::raise_api_path_error(__FUNCTION__, api_path, e, "failed to create file"); @@ -359,11 +396,135 @@ auto s3_provider::get_api_path_from_source(const std::string &source_path, } auto s3_provider::get_directory_item_count(const std::string &api_path) const - -> std::uint64_t {} + -> std::uint64_t { + // TODO implement this + return 0U; +} auto s3_provider::get_directory_items(const std::string &api_path, directory_item_list &list) const - -> api_error {} + -> api_error { + try { + const auto cfg = config_.get_s3_config(); + const auto is_encrypted = not cfg.encryption_token.empty(); + + std::string key; + if (is_encrypted) { + auto res = get_item_meta(api_path, META_KEY, key); + if (res != api_error::success) { + return res; + } + } + + const auto object_name = + api_path == "/" + ? "" + : utils::path::create_api_path(is_encrypted ? key : api_path); + + std::string response_data{}; + long response_code{}; + auto prefix = object_name.empty() ? object_name : object_name + "/"; + + if (not get_object_list(comm_, config_.get_s3_config(), response_data, + response_code, "/", prefix)) { + return api_error::comm_error; + } + + if (response_code == http_error_codes::not_found) { + return api_error::directory_not_found; + } + + if (response_code != http_error_codes::ok) { + return api_error::comm_error; + } + + pugi::xml_document doc; + auto res = doc.load_string(response_data.c_str()); + if (res.status != pugi::xml_parse_status::status_ok) { + return api_error::error; + } + + const auto add_directory_item = + [&](bool directory, const std::string &name, + std::function get_size) + -> api_error { + auto child_api_path = + utils::path::create_api_path(utils::path::combine("/", {name})); + std::string child_object_name; + if (is_encrypted) { + child_object_name = child_api_path; + auto res = utils::encryption::decrypt_file_path(cfg.encryption_token, + child_api_path); + if (res != api_error::success) { + return res; + } + } + + directory_item dir_item{}; + dir_item.api_path = child_api_path; + dir_item.api_parent = utils::path::get_parent_api_path(dir_item.api_path); + dir_item.directory = directory; + dir_item.size = get_size(dir_item); + auto res = get_item_meta(child_api_path, dir_item.meta); + if (not directory && res == api_error::item_not_found) { + api_file file{}; + res = get_file(child_api_path, file); + } + if (res != api_error::success) { + return res; + } + list.push_back(std::move(dir_item)); + return api_error::success; + }; + + auto node_list = + doc.select_nodes("/ListBucketResult/CommonPrefixes/Prefix"); + for (const auto &node : node_list) { + add_directory_item( + true, node.node().text().as_string(), + [](const directory_item &) -> std::uint64_t { return 0U; }); + } + + node_list = doc.select_nodes("/ListBucketResult/Contents"); + for (const auto &node : node_list) { + auto child_object_name = + std::string{node.node().select_node("Key").node().text().as_string()}; + if (child_object_name != prefix) { + auto size = node.node().select_node("Size").node().text().as_ullong(); + add_directory_item( + false, child_object_name, + [&is_encrypted, &size](const directory_item &) -> std::uint64_t { + return is_encrypted ? utils::encryption::encrypting_reader:: + calculate_decrypted_size(size) + : size; + }); + } + } + + std::sort(list.begin(), list.end(), + [](const auto &item1, const auto &item2) -> bool { + return (item1.directory && not item2.directory) || + (not(item2.directory && not item1.directory) && + (item1.api_path.compare(item2.api_path) < 0)); + }); + + list.insert(list.begin(), directory_item{ + "..", + "", + true, + }); + list.insert(list.begin(), directory_item{ + ".", + "", + true, + }); + return api_error::success; + } catch (const std::exception &e) { + utils::error::raise_error(__FUNCTION__, e, "exception occurred"); + } + + return api_error::error; +} auto s3_provider::get_file(const std::string &api_path, api_file &file) const -> api_error { @@ -906,9 +1067,13 @@ void s3_provider::remove_deleted_files() { } } -auto s3_provider::remove_directory(const std::string &api_path) -> api_error {} +auto s3_provider::remove_directory(const std::string &api_path) -> api_error { + return api_error::not_implemented; +} -auto s3_provider::remove_file(const std::string &api_path) -> api_error {} +auto s3_provider::remove_file(const std::string &api_path) -> api_error { + return api_error::not_implemented; +} auto s3_provider::remove_item_meta(const std::string &api_path, const std::string &key) -> api_error { @@ -1055,6 +1220,12 @@ auto s3_provider::upload_file(const std::string &api_path, return error; }; try { + std::uint64_t file_size{}; + if (utils::file::is_file(source_path) && + not utils::file::get_file_size(source_path, file_size)) { + return notify_end(api_error::comm_error); + } + const auto cfg = config_.get_s3_config(); const auto is_encrypted = not cfg.encryption_token.empty(); @@ -1074,7 +1245,7 @@ auto s3_provider::upload_file(const std::string &api_path, put_file.path = object_name; put_file.source_path = source_path; - if (is_encrypted) { + if (is_encrypted && file_size > 0U) { static stop_type no_stop{false}; put_file.reader = std::make_shared( diff --git a/src/utils/encrypting_reader.cpp b/src/utils/encrypting_reader.cpp index c7000eb6..353a177a 100644 --- a/src/utils/encrypting_reader.cpp +++ b/src/utils/encrypting_reader.cpp @@ -172,7 +172,7 @@ encrypting_reader::encrypting_reader( : key_(utils::encryption::generate_key(token)), stop_requested_(stop_requested), error_return_(error_return) { - const auto res = native_file::open( + const auto res = native_file::create_or_open( source_path, not relative_parent_path.has_value(), source_file_); if (res != api_error::success) { throw std::runtime_error("file open failed|src|" + source_path + '|' + @@ -225,7 +225,8 @@ encrypting_reader::encrypting_reader(const std::string &encrypted_file_path, : key_(utils::encryption::generate_key(token)), stop_requested_(stop_requested), error_return_(error_return) { - const auto res = native_file::open(source_path, false, source_file_); + const auto res = + native_file::create_or_open(source_path, false, source_file_); if (res != api_error::success) { throw std::runtime_error("file open failed|src|" + source_path + '|' + api_error_to_string(res)); @@ -266,7 +267,8 @@ encrypting_reader::encrypting_reader( : key_(utils::encryption::generate_key(token)), stop_requested_(stop_requested), error_return_(error_return) { - const auto res = native_file::open(source_path, false, source_file_); + const auto res = + native_file::create_or_open(source_path, false, source_file_); if (res != api_error::success) { throw std::runtime_error("file open failed|src|" + source_path + '|' + api_error_to_string(res));