diff --git a/include/comm/s3/s3_requests.hpp b/include/comm/s3/s3_requests.hpp deleted file mode 100644 index b0cb37b9..00000000 --- a/include/comm/s3/s3_requests.hpp +++ /dev/null @@ -1,117 +0,0 @@ -/* - Copyright <2018-2023> - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. -*/ -#ifndef INCLUDE_COMM_S3_S3_REQUESTS_HPP_ -#define INCLUDE_COMM_S3_S3_REQUESTS_HPP_ -#if defined(REPERTORY_ENABLE_S3) - -#include "comm/s3/s3_requests_curl.hpp" - -#include "types/repertory.hpp" -#include "types/s3.hpp" -#include "utils/string_utils.hpp" - -namespace repertory { -template -[[nodiscard]] inline auto -create_directory_object_request(client_type &client, const s3_config &config, - const std::string &object_name, - long &response_code) -> bool { - return create_directory_object_request_impl(client, config, object_name, - response_code); -} - -template -[[nodiscard]] inline auto delete_object_request(client_type &client, - const s3_config &config, - const std::string &object_name, - long &response_code) -> bool { - return delete_object_request_impl(client, config, object_name, response_code); -} - -template -[[nodiscard]] inline auto -head_object_request(client_type &client, const s3_config &config, - const std::string &object_name, head_object_result &result, - long &response_code) -> bool { - return head_object_request_impl(client, config, object_name, result, - response_code); -} - -template -[[nodiscard]] inline auto -list_directories_request(client_type &client, const s3_config &config, - list_directories_result &result, long &response_code) - -> bool { - return list_directories_request_impl(client, config, result, response_code); -} - -template -[[nodiscard]] inline auto -list_files_request(client_type &client, const s3_config &config, - const get_api_file_token_callback &get_api_file_token, - const get_name_callback &get_name, list_files_result &result, - long &response_code) -> bool { - return list_files_request_impl(client, config, get_api_file_token, get_name, - result, response_code); -} - -template -[[nodiscard]] inline auto list_objects_in_directory_request( - client_type &client, const s3_config &config, - const std::string &object_name, meta_provider_callback meta_provider, - list_objects_result &result, long &response_code) -> bool { - return list_objects_in_directory_request_impl( - client, config, object_name, meta_provider, result, response_code); -} - -template -[[nodiscard]] inline auto -list_objects_request(client_type &client, const s3_config &config, - list_objects_result &result, long &response_code) -> bool { - return list_objects_request_impl(client, config, result, response_code); -} - -template -[[nodiscard]] inline auto -put_object_request(client_type &client, const s3_config &config, - std::string object_name, const std::string &source_path, - const std::string &encryption_token, - get_key_callback get_key, set_key_callback set_key, - long &response_code, stop_type &stop_requested) -> bool { - return put_object_request_impl(client, config, object_name, source_path, - encryption_token, get_key, set_key, - response_code, stop_requested); -} - -template -[[nodiscard]] inline auto -read_object_request(client_type &client, const s3_config &config, - const std::string &object_name, std::size_t size, - std::uint64_t offset, data_buffer &data, - long &response_code, stop_type &stop_requested) -> bool { - return read_object_request_impl(client, config, object_name, size, offset, - data, response_code, stop_requested); -} -} // namespace repertory - -#endif -#endif // INCLUDE_COMM_S3_S3_REQUESTS_HPP_ diff --git a/include/comm/s3/s3_requests_curl.hpp b/include/comm/s3/s3_requests_curl.hpp deleted file mode 100644 index fcc6eaf7..00000000 --- a/include/comm/s3/s3_requests_curl.hpp +++ /dev/null @@ -1,83 +0,0 @@ -/* - Copyright <2018-2023> - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. -*/ -#ifndef INCLUDE_COMM_S3_S3_REQUESTS_CURL_HPP_ -#define INCLUDE_COMM_S3_S3_REQUESTS_CURL_HPP_ -#if defined(REPERTORY_ENABLE_S3) - -#include "comm/i_http_comm.hpp" -#include "types/repertory.hpp" -#include "types/s3.hpp" -#include "utils/string_utils.hpp" -#include "utils/utils.hpp" - -namespace repertory { -[[nodiscard]] auto create_directory_object_request_impl( - i_http_comm &client, const s3_config &config, - const std::string &object_name, long &response_code) -> bool; - -[[nodiscard]] auto delete_object_request_impl(i_http_comm &client, - const s3_config &config, - const std::string &object_name, - long &response_code) -> bool; - -[[nodiscard]] auto head_object_request_impl(i_http_comm &client, - const s3_config &config, - const std::string &object_name, - head_object_result &result, - long &response_code) -> bool; - -[[nodiscard]] auto -list_directories_request_impl(i_http_comm &client, const s3_config &config, - list_directories_result &result, - long &response_code) -> bool; - -[[nodiscard]] auto -list_files_request_impl(i_http_comm &client, const s3_config &config, - const get_api_file_token_callback &get_api_file_token, - const get_name_callback &get_name, - list_files_result &result, long &response_code) -> bool; - -[[nodiscard]] auto list_objects_in_directory_request_impl( - i_http_comm &client, const s3_config &config, - const std::string &object_name, meta_provider_callback meta_provider, - list_objects_result &result, long &response_code) -> bool; - -[[nodiscard]] auto list_objects_request_impl(i_http_comm &client, - const s3_config &config, - list_objects_result &result, - long &response_code) -> bool; - -[[nodiscard]] auto -put_object_request_impl(i_http_comm &client, const s3_config &config, - std::string object_name, const std::string &source_path, - const std::string &encryption_token, - get_key_callback get_key, set_key_callback set_key, - long &response_code, stop_type &stop_requested) -> bool; - -[[nodiscard]] auto read_object_request_impl( - i_http_comm &client, const s3_config &config, - const std::string &object_name, std::size_t size, std::uint64_t offset, - data_buffer &data, long &response_code, stop_type &stop_requested) -> bool; -} // namespace repertory - -#endif // REPERTORY_ENABLE_S3 -#endif // INCLUDE_COMM_S3_S3_REQUESTS_CURL_HPP_ diff --git a/src/comm/s3/s3_requests_curl.cpp b/src/comm/s3/s3_requests_curl.cpp deleted file mode 100644 index 74c9f72e..00000000 --- a/src/comm/s3/s3_requests_curl.cpp +++ /dev/null @@ -1,399 +0,0 @@ -/* - Copyright <2018-2023> - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. -*/ -#if defined(REPERTORY_ENABLE_S3) - -#include "comm/s3/s3_requests.hpp" - -#include "comm/curl/curl_comm.hpp" -#include "comm/curl/requests/http_get.hpp" -#include "utils/encryption.hpp" -#include "utils/error_utils.hpp" -#include "utils/path_utils.hpp" - -namespace repertory { -namespace { -[[nodiscard]] auto -get_object_list(i_http_comm &client, const s3_config &config, - std::string &response_data, long &response_code, - std::optional delimiter = std::nullopt, - std::optional prefix = std::nullopt) -> bool { - curl::requests::http_get get{}; - get.allow_timeout = true; - get.aws_service = "aws:amz:" + config.region + ":s3"; - get.path = '/'; - get.query["list-type"] = "2"; - if (delimiter.has_value() && not delimiter.value().empty()) { - get.query["delimiter"] = delimiter.value(); - } - if (prefix.has_value() && not prefix.value().empty()) { - get.query["prefix"] = prefix.value(); - } - get.response_handler = [&response_data](const data_buffer &data, - long /*response_code*/) { - response_data = std::string(data.begin(), data.end()); - }; - - stop_type stop_requested{}; - return client.make_request(get, response_code, stop_requested); -} -} // namespace - -auto create_directory_object_request_impl(i_http_comm &client, - const s3_config &config, - const std::string &object_name, - long &response_code) -> bool { - try { - curl::requests::http_put_file put_file{}; - put_file.allow_timeout = true; - put_file.aws_service = "aws:amz:" + config.region + ":s3"; - put_file.path = '/' + object_name; - - stop_type stop_requested{false}; - return client.make_request(put_file, response_code, stop_requested); - } catch (const std::exception &e) { - utils::error::raise_error(__FUNCTION__, e, "exception occurred"); - } - - return false; -} - -auto delete_object_request_impl(i_http_comm &client, const s3_config &config, - const std::string &object_name, - long &response_code) -> bool { - try { - head_object_result result{}; - if (not head_object_request_impl(client, config, object_name, result, - response_code)) { - return false; - } - - if (response_code == http_error_codes::not_found) { - return true; - } - - curl::requests::http_delete del{}; - del.allow_timeout = true; - del.aws_service = "aws:amz:" + config.region + ":s3"; - del.path = '/' + object_name; - - stop_type stop_requested{false}; - return client.make_request(del, response_code, stop_requested); - } catch (const std::exception &e) { - utils::error::raise_error(__FUNCTION__, e, "exception occurred"); - } - - return false; -} - -auto head_object_request_impl(i_http_comm &client, const s3_config &config, - const std::string &object_name, - head_object_result &result, long &response_code) - -> bool { - try { - curl::requests::http_head head{}; - head.allow_timeout = true; - head.aws_service = "aws:amz:" + config.region + ":s3"; - head.path = '/' + object_name; - head.response_headers = http_headers{}; - - stop_type stop_requested{false}; - if (not client.make_request(head, response_code, stop_requested)) { - return false; - } - - if (response_code == http_error_codes::ok) { - result.from_headers(head.response_headers.value()); - } - - return true; - } catch (const std::exception &e) { - utils::error::raise_error(__FUNCTION__, e, "exception occurred"); - } - - return false; -} - -auto list_directories_request_impl(i_http_comm &client, const s3_config &config, - list_directories_result &result, - long &response_code) -> bool { - try { - std::string response_data{}; - if (not get_object_list(client, config, response_data, response_code)) { - return false; - } - - if (response_code != http_error_codes::ok) { - return false; - } - - pugi::xml_document doc; - auto res = doc.load_string(response_data.c_str()); - if (res.status != pugi::xml_parse_status::status_ok) { - return false; - } - - auto node_list = doc.select_nodes("/ListBucketResult/Contents"); - for (const auto &node : node_list) { - const auto *object_name = - node.node().select_node("Key").node().text().as_string(); - if (utils::string::ends_with(object_name, "/")) { - api_file directory{}; - directory.api_path = utils::path::create_api_path(object_name); - directory.api_parent = - utils::path::get_parent_api_path(directory.api_path); - directory.accessed_date = utils::get_file_time_now(); - directory.changed_date = utils::convert_api_date( - node.node().select_node("LastModified").node().text().as_string()); - directory.creation_date = directory.changed_date; - directory.modified_date = directory.changed_date; - result.emplace_back(std::move(directory)); - } - } - - return true; - } catch (const std::exception &e) { - utils::error::raise_error(__FUNCTION__, e, "exception occurred"); - } - - return false; -} - -auto list_files_request_impl( - i_http_comm &client, const s3_config &config, - const get_api_file_token_callback &get_api_file_token, - const get_name_callback &get_name, list_files_result &result, - long &response_code) -> bool { - try { - std::string response_data{}; - if (not get_object_list(client, config, response_data, response_code)) { - return false; - } - - if (response_code != http_error_codes::ok) { - return false; - } - - pugi::xml_document doc; - auto res = doc.load_string(response_data.c_str()); - if (res.status != pugi::xml_parse_status::status_ok) { - return false; - } - - auto node_list = doc.select_nodes("/ListBucketResult/Contents"); - for (const auto &node : node_list) { - std::string object_name = - node.node().select_node("Key").node().text().as_string(); - if (not utils::string::ends_with(object_name, "/")) { - api_file file{}; - object_name = get_name( - *(utils::string::split(object_name, '/', false).end() - 1U), - object_name); - file.api_path = utils::path::create_api_path(object_name); - file.api_parent = utils::path::get_parent_api_path(file.api_path); - file.accessed_date = utils::get_file_time_now(); - // file.encryption_token = get_api_file_token(file.api_path); - auto size = node.node().select_node("Size").node().text().as_ullong(); - // file.file_size = file.encryption_token.empty() - // ? size - // : utils::encryption::encrypting_reader:: - // calculate_decrypted_size(size); - file.changed_date = utils::convert_api_date( - node.node().select_node("LastModified").node().text().as_string()); - file.creation_date = file.changed_date; - file.modified_date = file.changed_date; - result.emplace_back(std::move(file)); - } - } - - return true; - } catch (const std::exception &e) { - utils::error::raise_error(__FUNCTION__, e, "exception occurred"); - } - - return false; -} - -auto list_objects_in_directory_request_impl( - i_http_comm &client, const s3_config &config, - const std::string &object_name, meta_provider_callback meta_provider, - list_objects_result &result, long &response_code) -> bool { - try { - std::string response_data{}; - auto prefix = object_name.empty() ? object_name : object_name + "/"; - if (not get_object_list(client, config, response_data, response_code, "/", - prefix)) { - return false; - } - - if (response_code != http_error_codes::ok) { - return false; - } - - pugi::xml_document doc; - auto res = doc.load_string(response_data.c_str()); - if (res.status != pugi::xml_parse_status::status_ok) { - return false; - } - - const auto add_directory_item = - [&](bool directory, const std::string &name, - std::function get_size) { - directory_item dir_item{}; - dir_item.api_path = - utils::path::create_api_path(utils::path::combine("/", {name})); - dir_item.api_parent = - utils::path::get_parent_api_path(dir_item.api_path); - dir_item.directory = directory; - dir_item.size = get_size(dir_item); - meta_provider(dir_item); - result.emplace_back(std::move(dir_item)); - }; - - auto node_list = - doc.select_nodes("/ListBucketResult/CommonPrefixes/Prefix"); - for (const auto &node : node_list) { - add_directory_item( - true, node.node().text().as_string(), - [](const directory_item &) -> std::uint64_t { return 0U; }); - } - - node_list = doc.select_nodes("/ListBucketResult/Contents"); - for (const auto &node : node_list) { - const auto *child_object_name = - node.node().select_node("Key").node().text().as_string(); - if (child_object_name != prefix) { - auto size = node.node().select_node("Size").node().text().as_ullong(); - add_directory_item( - false, child_object_name, - [&size](const directory_item &) -> std::uint64_t { return size; }); - } - } - - return true; - } catch (const std::exception &e) { - utils::error::raise_error(__FUNCTION__, e, "exception occurred"); - } - - return false; -} - -auto list_objects_request_impl(i_http_comm &client, const s3_config &config, - list_objects_result &result, long &response_code) - -> bool { - try { - std::string response_data{}; - if (not get_object_list(client, config, response_data, response_code)) { - return false; - } - - if (response_code != http_error_codes::ok) { - return false; - } - - pugi::xml_document doc; - auto res = doc.load_string(response_data.c_str()); - if (res.status != pugi::xml_parse_status::status_ok) { - return false; - } - - auto node_list = doc.select_nodes("/ListBucketResult/Contents"); - for (const auto &node : node_list) { - const auto *object_name = - node.node().select_node("Key").node().text().as_string(); - auto size = node.node().select_node("Size").node().text().as_ullong(); - directory_item dir_item{}; - dir_item.api_path = utils::path::create_api_path(object_name); - dir_item.api_parent = utils::path::get_parent_api_path(dir_item.api_path); - dir_item.directory = utils::string::ends_with(object_name, "/"); - dir_item.size = dir_item.directory ? 0U : size; - dir_item.resolved = false; - result.emplace_back(std::move(dir_item)); - } - - return true; - } catch (const std::exception &e) { - utils::error::raise_error(__FUNCTION__, e, "exception occurred"); - } - - return false; -} - -auto put_object_request_impl(i_http_comm &client, const s3_config &config, - std::string object_name, - const std::string &source_path, - const std::string &encryption_token, - get_key_callback get_key, set_key_callback set_key, - long &response_code, stop_type &stop_requested) - -> bool { - try { - curl::requests::http_put_file put_file{}; - put_file.aws_service = "aws:amz:" + config.region + ":s3"; - put_file.path = '/' + object_name; - put_file.source_path = source_path; - - if (not encryption_token.empty()) { - static stop_type no_stop{false}; - - put_file.reader = std::make_shared( - *(utils::string::split(object_name, '/', false).end() - 1U), - source_path, no_stop, encryption_token, std::nullopt, -1); - auto key = get_key(); - if (key.empty()) { - key = put_file.reader->get_encrypted_file_name(); - set_key(key); - } - } - - return client.make_request(put_file, response_code, stop_requested); - } catch (const std::exception &e) { - utils::error::raise_error(__FUNCTION__, e, "exception occurred"); - } - - return false; -} - -auto read_object_request_impl(i_http_comm &client, const s3_config &config, - const std::string &object_name, std::size_t size, - std::uint64_t offset, data_buffer &data, - long &response_code, stop_type &stop_requested) - -> bool { - try { - curl::requests::http_get get{}; - get.aws_service = "aws:amz:" + config.region + ":s3"; - get.headers["response-content-type"] = "binary/octet-stream"; - get.path = '/' + object_name; - get.range = {{offset, offset + size - 1}}; - get.response_handler = [&data](const data_buffer &response_data, - long /*response_code*/) { - data = response_data; - }; - - return client.make_request(get, response_code, stop_requested); - } catch (const std::exception &e) { - utils::error::raise_error(__FUNCTION__, e, "exception occurred"); - } - - return false; -} -} // namespace repertory - -#endif