Compare commits
2 Commits
57ca2c7c6d
...
87f83b6f30
| Author | SHA1 | Date | |
|---|---|---|---|
| 87f83b6f30 | |||
| ace81d797e |
@ -1,117 +0,0 @@
|
|||||||
/*
|
|
||||||
Copyright <2018-2023> <scott.e.graves@protonmail.com>
|
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
||||||
of this software and associated documentation files (the "Software"), to deal
|
|
||||||
in the Software without restriction, including without limitation the rights
|
|
||||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
||||||
copies of the Software, and to permit persons to whom the Software is
|
|
||||||
furnished to do so, subject to the following conditions:
|
|
||||||
|
|
||||||
The above copyright notice and this permission notice shall be included in all
|
|
||||||
copies or substantial portions of the Software.
|
|
||||||
|
|
||||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
||||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
||||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
||||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
||||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
||||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
||||||
SOFTWARE.
|
|
||||||
*/
|
|
||||||
#ifndef INCLUDE_COMM_S3_S3_REQUESTS_HPP_
|
|
||||||
#define INCLUDE_COMM_S3_S3_REQUESTS_HPP_
|
|
||||||
#if defined(REPERTORY_ENABLE_S3)
|
|
||||||
|
|
||||||
#include "comm/s3/s3_requests_curl.hpp"
|
|
||||||
|
|
||||||
#include "types/repertory.hpp"
|
|
||||||
#include "types/s3.hpp"
|
|
||||||
#include "utils/string_utils.hpp"
|
|
||||||
|
|
||||||
namespace repertory {
|
|
||||||
template <typename client_type>
|
|
||||||
[[nodiscard]] inline auto
|
|
||||||
create_directory_object_request(client_type &client, const s3_config &config,
|
|
||||||
const std::string &object_name,
|
|
||||||
long &response_code) -> bool {
|
|
||||||
return create_directory_object_request_impl(client, config, object_name,
|
|
||||||
response_code);
|
|
||||||
}
|
|
||||||
|
|
||||||
template <typename client_type>
|
|
||||||
[[nodiscard]] inline auto delete_object_request(client_type &client,
|
|
||||||
const s3_config &config,
|
|
||||||
const std::string &object_name,
|
|
||||||
long &response_code) -> bool {
|
|
||||||
return delete_object_request_impl(client, config, object_name, response_code);
|
|
||||||
}
|
|
||||||
|
|
||||||
template <typename client_type>
|
|
||||||
[[nodiscard]] inline auto
|
|
||||||
head_object_request(client_type &client, const s3_config &config,
|
|
||||||
const std::string &object_name, head_object_result &result,
|
|
||||||
long &response_code) -> bool {
|
|
||||||
return head_object_request_impl(client, config, object_name, result,
|
|
||||||
response_code);
|
|
||||||
}
|
|
||||||
|
|
||||||
template <typename client_type>
|
|
||||||
[[nodiscard]] inline auto
|
|
||||||
list_directories_request(client_type &client, const s3_config &config,
|
|
||||||
list_directories_result &result, long &response_code)
|
|
||||||
-> bool {
|
|
||||||
return list_directories_request_impl(client, config, result, response_code);
|
|
||||||
}
|
|
||||||
|
|
||||||
template <typename client_type>
|
|
||||||
[[nodiscard]] inline auto
|
|
||||||
list_files_request(client_type &client, const s3_config &config,
|
|
||||||
const get_api_file_token_callback &get_api_file_token,
|
|
||||||
const get_name_callback &get_name, list_files_result &result,
|
|
||||||
long &response_code) -> bool {
|
|
||||||
return list_files_request_impl(client, config, get_api_file_token, get_name,
|
|
||||||
result, response_code);
|
|
||||||
}
|
|
||||||
|
|
||||||
template <typename client_type>
|
|
||||||
[[nodiscard]] inline auto list_objects_in_directory_request(
|
|
||||||
client_type &client, const s3_config &config,
|
|
||||||
const std::string &object_name, meta_provider_callback meta_provider,
|
|
||||||
list_objects_result &result, long &response_code) -> bool {
|
|
||||||
return list_objects_in_directory_request_impl(
|
|
||||||
client, config, object_name, meta_provider, result, response_code);
|
|
||||||
}
|
|
||||||
|
|
||||||
template <typename client_type>
|
|
||||||
[[nodiscard]] inline auto
|
|
||||||
list_objects_request(client_type &client, const s3_config &config,
|
|
||||||
list_objects_result &result, long &response_code) -> bool {
|
|
||||||
return list_objects_request_impl(client, config, result, response_code);
|
|
||||||
}
|
|
||||||
|
|
||||||
template <typename client_type>
|
|
||||||
[[nodiscard]] inline auto
|
|
||||||
put_object_request(client_type &client, const s3_config &config,
|
|
||||||
std::string object_name, const std::string &source_path,
|
|
||||||
const std::string &encryption_token,
|
|
||||||
get_key_callback get_key, set_key_callback set_key,
|
|
||||||
long &response_code, stop_type &stop_requested) -> bool {
|
|
||||||
return put_object_request_impl(client, config, object_name, source_path,
|
|
||||||
encryption_token, get_key, set_key,
|
|
||||||
response_code, stop_requested);
|
|
||||||
}
|
|
||||||
|
|
||||||
template <typename client_type>
|
|
||||||
[[nodiscard]] inline auto
|
|
||||||
read_object_request(client_type &client, const s3_config &config,
|
|
||||||
const std::string &object_name, std::size_t size,
|
|
||||||
std::uint64_t offset, data_buffer &data,
|
|
||||||
long &response_code, stop_type &stop_requested) -> bool {
|
|
||||||
return read_object_request_impl(client, config, object_name, size, offset,
|
|
||||||
data, response_code, stop_requested);
|
|
||||||
}
|
|
||||||
} // namespace repertory
|
|
||||||
|
|
||||||
#endif
|
|
||||||
#endif // INCLUDE_COMM_S3_S3_REQUESTS_HPP_
|
|
||||||
@ -1,83 +0,0 @@
|
|||||||
/*
|
|
||||||
Copyright <2018-2023> <scott.e.graves@protonmail.com>
|
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
||||||
of this software and associated documentation files (the "Software"), to deal
|
|
||||||
in the Software without restriction, including without limitation the rights
|
|
||||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
||||||
copies of the Software, and to permit persons to whom the Software is
|
|
||||||
furnished to do so, subject to the following conditions:
|
|
||||||
|
|
||||||
The above copyright notice and this permission notice shall be included in all
|
|
||||||
copies or substantial portions of the Software.
|
|
||||||
|
|
||||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
||||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
||||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
||||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
||||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
||||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
||||||
SOFTWARE.
|
|
||||||
*/
|
|
||||||
#ifndef INCLUDE_COMM_S3_S3_REQUESTS_CURL_HPP_
|
|
||||||
#define INCLUDE_COMM_S3_S3_REQUESTS_CURL_HPP_
|
|
||||||
#if defined(REPERTORY_ENABLE_S3)
|
|
||||||
|
|
||||||
#include "comm/i_http_comm.hpp"
|
|
||||||
#include "types/repertory.hpp"
|
|
||||||
#include "types/s3.hpp"
|
|
||||||
#include "utils/string_utils.hpp"
|
|
||||||
#include "utils/utils.hpp"
|
|
||||||
|
|
||||||
namespace repertory {
|
|
||||||
[[nodiscard]] auto create_directory_object_request_impl(
|
|
||||||
i_http_comm &client, const s3_config &config,
|
|
||||||
const std::string &object_name, long &response_code) -> bool;
|
|
||||||
|
|
||||||
[[nodiscard]] auto delete_object_request_impl(i_http_comm &client,
|
|
||||||
const s3_config &config,
|
|
||||||
const std::string &object_name,
|
|
||||||
long &response_code) -> bool;
|
|
||||||
|
|
||||||
[[nodiscard]] auto head_object_request_impl(i_http_comm &client,
|
|
||||||
const s3_config &config,
|
|
||||||
const std::string &object_name,
|
|
||||||
head_object_result &result,
|
|
||||||
long &response_code) -> bool;
|
|
||||||
|
|
||||||
[[nodiscard]] auto
|
|
||||||
list_directories_request_impl(i_http_comm &client, const s3_config &config,
|
|
||||||
list_directories_result &result,
|
|
||||||
long &response_code) -> bool;
|
|
||||||
|
|
||||||
[[nodiscard]] auto
|
|
||||||
list_files_request_impl(i_http_comm &client, const s3_config &config,
|
|
||||||
const get_api_file_token_callback &get_api_file_token,
|
|
||||||
const get_name_callback &get_name,
|
|
||||||
list_files_result &result, long &response_code) -> bool;
|
|
||||||
|
|
||||||
[[nodiscard]] auto list_objects_in_directory_request_impl(
|
|
||||||
i_http_comm &client, const s3_config &config,
|
|
||||||
const std::string &object_name, meta_provider_callback meta_provider,
|
|
||||||
list_objects_result &result, long &response_code) -> bool;
|
|
||||||
|
|
||||||
[[nodiscard]] auto list_objects_request_impl(i_http_comm &client,
|
|
||||||
const s3_config &config,
|
|
||||||
list_objects_result &result,
|
|
||||||
long &response_code) -> bool;
|
|
||||||
|
|
||||||
[[nodiscard]] auto
|
|
||||||
put_object_request_impl(i_http_comm &client, const s3_config &config,
|
|
||||||
std::string object_name, const std::string &source_path,
|
|
||||||
const std::string &encryption_token,
|
|
||||||
get_key_callback get_key, set_key_callback set_key,
|
|
||||||
long &response_code, stop_type &stop_requested) -> bool;
|
|
||||||
|
|
||||||
[[nodiscard]] auto read_object_request_impl(
|
|
||||||
i_http_comm &client, const s3_config &config,
|
|
||||||
const std::string &object_name, std::size_t size, std::uint64_t offset,
|
|
||||||
data_buffer &data, long &response_code, stop_type &stop_requested) -> bool;
|
|
||||||
} // namespace repertory
|
|
||||||
|
|
||||||
#endif // REPERTORY_ENABLE_S3
|
|
||||||
#endif // INCLUDE_COMM_S3_S3_REQUESTS_CURL_HPP_
|
|
||||||
@ -1,399 +0,0 @@
|
|||||||
/*
|
|
||||||
Copyright <2018-2023> <scott.e.graves@protonmail.com>
|
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
||||||
of this software and associated documentation files (the "Software"), to deal
|
|
||||||
in the Software without restriction, including without limitation the rights
|
|
||||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
||||||
copies of the Software, and to permit persons to whom the Software is
|
|
||||||
furnished to do so, subject to the following conditions:
|
|
||||||
|
|
||||||
The above copyright notice and this permission notice shall be included in all
|
|
||||||
copies or substantial portions of the Software.
|
|
||||||
|
|
||||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
||||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
||||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
||||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
||||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
||||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
||||||
SOFTWARE.
|
|
||||||
*/
|
|
||||||
#if defined(REPERTORY_ENABLE_S3)
|
|
||||||
|
|
||||||
#include "comm/s3/s3_requests.hpp"
|
|
||||||
|
|
||||||
#include "comm/curl/curl_comm.hpp"
|
|
||||||
#include "comm/curl/requests/http_get.hpp"
|
|
||||||
#include "utils/encryption.hpp"
|
|
||||||
#include "utils/error_utils.hpp"
|
|
||||||
#include "utils/path_utils.hpp"
|
|
||||||
|
|
||||||
namespace repertory {
|
|
||||||
namespace {
|
|
||||||
[[nodiscard]] auto
|
|
||||||
get_object_list(i_http_comm &client, const s3_config &config,
|
|
||||||
std::string &response_data, long &response_code,
|
|
||||||
std::optional<std::string> delimiter = std::nullopt,
|
|
||||||
std::optional<std::string> prefix = std::nullopt) -> bool {
|
|
||||||
curl::requests::http_get get{};
|
|
||||||
get.allow_timeout = true;
|
|
||||||
get.aws_service = "aws:amz:" + config.region + ":s3";
|
|
||||||
get.path = '/';
|
|
||||||
get.query["list-type"] = "2";
|
|
||||||
if (delimiter.has_value() && not delimiter.value().empty()) {
|
|
||||||
get.query["delimiter"] = delimiter.value();
|
|
||||||
}
|
|
||||||
if (prefix.has_value() && not prefix.value().empty()) {
|
|
||||||
get.query["prefix"] = prefix.value();
|
|
||||||
}
|
|
||||||
get.response_handler = [&response_data](const data_buffer &data,
|
|
||||||
long /*response_code*/) {
|
|
||||||
response_data = std::string(data.begin(), data.end());
|
|
||||||
};
|
|
||||||
|
|
||||||
stop_type stop_requested{};
|
|
||||||
return client.make_request(get, response_code, stop_requested);
|
|
||||||
}
|
|
||||||
} // namespace
|
|
||||||
|
|
||||||
auto create_directory_object_request_impl(i_http_comm &client,
|
|
||||||
const s3_config &config,
|
|
||||||
const std::string &object_name,
|
|
||||||
long &response_code) -> bool {
|
|
||||||
try {
|
|
||||||
curl::requests::http_put_file put_file{};
|
|
||||||
put_file.allow_timeout = true;
|
|
||||||
put_file.aws_service = "aws:amz:" + config.region + ":s3";
|
|
||||||
put_file.path = '/' + object_name;
|
|
||||||
|
|
||||||
stop_type stop_requested{false};
|
|
||||||
return client.make_request(put_file, response_code, stop_requested);
|
|
||||||
} catch (const std::exception &e) {
|
|
||||||
utils::error::raise_error(__FUNCTION__, e, "exception occurred");
|
|
||||||
}
|
|
||||||
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
auto delete_object_request_impl(i_http_comm &client, const s3_config &config,
|
|
||||||
const std::string &object_name,
|
|
||||||
long &response_code) -> bool {
|
|
||||||
try {
|
|
||||||
head_object_result result{};
|
|
||||||
if (not head_object_request_impl(client, config, object_name, result,
|
|
||||||
response_code)) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (response_code == http_error_codes::not_found) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
curl::requests::http_delete del{};
|
|
||||||
del.allow_timeout = true;
|
|
||||||
del.aws_service = "aws:amz:" + config.region + ":s3";
|
|
||||||
del.path = '/' + object_name;
|
|
||||||
|
|
||||||
stop_type stop_requested{false};
|
|
||||||
return client.make_request(del, response_code, stop_requested);
|
|
||||||
} catch (const std::exception &e) {
|
|
||||||
utils::error::raise_error(__FUNCTION__, e, "exception occurred");
|
|
||||||
}
|
|
||||||
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
auto head_object_request_impl(i_http_comm &client, const s3_config &config,
|
|
||||||
const std::string &object_name,
|
|
||||||
head_object_result &result, long &response_code)
|
|
||||||
-> bool {
|
|
||||||
try {
|
|
||||||
curl::requests::http_head head{};
|
|
||||||
head.allow_timeout = true;
|
|
||||||
head.aws_service = "aws:amz:" + config.region + ":s3";
|
|
||||||
head.path = '/' + object_name;
|
|
||||||
head.response_headers = http_headers{};
|
|
||||||
|
|
||||||
stop_type stop_requested{false};
|
|
||||||
if (not client.make_request(head, response_code, stop_requested)) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (response_code == http_error_codes::ok) {
|
|
||||||
result.from_headers(head.response_headers.value());
|
|
||||||
}
|
|
||||||
|
|
||||||
return true;
|
|
||||||
} catch (const std::exception &e) {
|
|
||||||
utils::error::raise_error(__FUNCTION__, e, "exception occurred");
|
|
||||||
}
|
|
||||||
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
auto list_directories_request_impl(i_http_comm &client, const s3_config &config,
|
|
||||||
list_directories_result &result,
|
|
||||||
long &response_code) -> bool {
|
|
||||||
try {
|
|
||||||
std::string response_data{};
|
|
||||||
if (not get_object_list(client, config, response_data, response_code)) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (response_code != http_error_codes::ok) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
pugi::xml_document doc;
|
|
||||||
auto res = doc.load_string(response_data.c_str());
|
|
||||||
if (res.status != pugi::xml_parse_status::status_ok) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
auto node_list = doc.select_nodes("/ListBucketResult/Contents");
|
|
||||||
for (const auto &node : node_list) {
|
|
||||||
const auto *object_name =
|
|
||||||
node.node().select_node("Key").node().text().as_string();
|
|
||||||
if (utils::string::ends_with(object_name, "/")) {
|
|
||||||
api_file directory{};
|
|
||||||
directory.api_path = utils::path::create_api_path(object_name);
|
|
||||||
directory.api_parent =
|
|
||||||
utils::path::get_parent_api_path(directory.api_path);
|
|
||||||
directory.accessed_date = utils::get_file_time_now();
|
|
||||||
directory.changed_date = utils::convert_api_date(
|
|
||||||
node.node().select_node("LastModified").node().text().as_string());
|
|
||||||
directory.creation_date = directory.changed_date;
|
|
||||||
directory.modified_date = directory.changed_date;
|
|
||||||
result.emplace_back(std::move(directory));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return true;
|
|
||||||
} catch (const std::exception &e) {
|
|
||||||
utils::error::raise_error(__FUNCTION__, e, "exception occurred");
|
|
||||||
}
|
|
||||||
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
auto list_files_request_impl(
|
|
||||||
i_http_comm &client, const s3_config &config,
|
|
||||||
const get_api_file_token_callback &get_api_file_token,
|
|
||||||
const get_name_callback &get_name, list_files_result &result,
|
|
||||||
long &response_code) -> bool {
|
|
||||||
try {
|
|
||||||
std::string response_data{};
|
|
||||||
if (not get_object_list(client, config, response_data, response_code)) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (response_code != http_error_codes::ok) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
pugi::xml_document doc;
|
|
||||||
auto res = doc.load_string(response_data.c_str());
|
|
||||||
if (res.status != pugi::xml_parse_status::status_ok) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
auto node_list = doc.select_nodes("/ListBucketResult/Contents");
|
|
||||||
for (const auto &node : node_list) {
|
|
||||||
std::string object_name =
|
|
||||||
node.node().select_node("Key").node().text().as_string();
|
|
||||||
if (not utils::string::ends_with(object_name, "/")) {
|
|
||||||
api_file file{};
|
|
||||||
object_name = get_name(
|
|
||||||
*(utils::string::split(object_name, '/', false).end() - 1U),
|
|
||||||
object_name);
|
|
||||||
file.api_path = utils::path::create_api_path(object_name);
|
|
||||||
file.api_parent = utils::path::get_parent_api_path(file.api_path);
|
|
||||||
file.accessed_date = utils::get_file_time_now();
|
|
||||||
// file.encryption_token = get_api_file_token(file.api_path);
|
|
||||||
auto size = node.node().select_node("Size").node().text().as_ullong();
|
|
||||||
// file.file_size = file.encryption_token.empty()
|
|
||||||
// ? size
|
|
||||||
// : utils::encryption::encrypting_reader::
|
|
||||||
// calculate_decrypted_size(size);
|
|
||||||
file.changed_date = utils::convert_api_date(
|
|
||||||
node.node().select_node("LastModified").node().text().as_string());
|
|
||||||
file.creation_date = file.changed_date;
|
|
||||||
file.modified_date = file.changed_date;
|
|
||||||
result.emplace_back(std::move(file));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return true;
|
|
||||||
} catch (const std::exception &e) {
|
|
||||||
utils::error::raise_error(__FUNCTION__, e, "exception occurred");
|
|
||||||
}
|
|
||||||
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
auto list_objects_in_directory_request_impl(
|
|
||||||
i_http_comm &client, const s3_config &config,
|
|
||||||
const std::string &object_name, meta_provider_callback meta_provider,
|
|
||||||
list_objects_result &result, long &response_code) -> bool {
|
|
||||||
try {
|
|
||||||
std::string response_data{};
|
|
||||||
auto prefix = object_name.empty() ? object_name : object_name + "/";
|
|
||||||
if (not get_object_list(client, config, response_data, response_code, "/",
|
|
||||||
prefix)) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (response_code != http_error_codes::ok) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
pugi::xml_document doc;
|
|
||||||
auto res = doc.load_string(response_data.c_str());
|
|
||||||
if (res.status != pugi::xml_parse_status::status_ok) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
const auto add_directory_item =
|
|
||||||
[&](bool directory, const std::string &name,
|
|
||||||
std::function<std::uint64_t(const directory_item &)> get_size) {
|
|
||||||
directory_item dir_item{};
|
|
||||||
dir_item.api_path =
|
|
||||||
utils::path::create_api_path(utils::path::combine("/", {name}));
|
|
||||||
dir_item.api_parent =
|
|
||||||
utils::path::get_parent_api_path(dir_item.api_path);
|
|
||||||
dir_item.directory = directory;
|
|
||||||
dir_item.size = get_size(dir_item);
|
|
||||||
meta_provider(dir_item);
|
|
||||||
result.emplace_back(std::move(dir_item));
|
|
||||||
};
|
|
||||||
|
|
||||||
auto node_list =
|
|
||||||
doc.select_nodes("/ListBucketResult/CommonPrefixes/Prefix");
|
|
||||||
for (const auto &node : node_list) {
|
|
||||||
add_directory_item(
|
|
||||||
true, node.node().text().as_string(),
|
|
||||||
[](const directory_item &) -> std::uint64_t { return 0U; });
|
|
||||||
}
|
|
||||||
|
|
||||||
node_list = doc.select_nodes("/ListBucketResult/Contents");
|
|
||||||
for (const auto &node : node_list) {
|
|
||||||
const auto *child_object_name =
|
|
||||||
node.node().select_node("Key").node().text().as_string();
|
|
||||||
if (child_object_name != prefix) {
|
|
||||||
auto size = node.node().select_node("Size").node().text().as_ullong();
|
|
||||||
add_directory_item(
|
|
||||||
false, child_object_name,
|
|
||||||
[&size](const directory_item &) -> std::uint64_t { return size; });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return true;
|
|
||||||
} catch (const std::exception &e) {
|
|
||||||
utils::error::raise_error(__FUNCTION__, e, "exception occurred");
|
|
||||||
}
|
|
||||||
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
auto list_objects_request_impl(i_http_comm &client, const s3_config &config,
|
|
||||||
list_objects_result &result, long &response_code)
|
|
||||||
-> bool {
|
|
||||||
try {
|
|
||||||
std::string response_data{};
|
|
||||||
if (not get_object_list(client, config, response_data, response_code)) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (response_code != http_error_codes::ok) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
pugi::xml_document doc;
|
|
||||||
auto res = doc.load_string(response_data.c_str());
|
|
||||||
if (res.status != pugi::xml_parse_status::status_ok) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
auto node_list = doc.select_nodes("/ListBucketResult/Contents");
|
|
||||||
for (const auto &node : node_list) {
|
|
||||||
const auto *object_name =
|
|
||||||
node.node().select_node("Key").node().text().as_string();
|
|
||||||
auto size = node.node().select_node("Size").node().text().as_ullong();
|
|
||||||
directory_item dir_item{};
|
|
||||||
dir_item.api_path = utils::path::create_api_path(object_name);
|
|
||||||
dir_item.api_parent = utils::path::get_parent_api_path(dir_item.api_path);
|
|
||||||
dir_item.directory = utils::string::ends_with(object_name, "/");
|
|
||||||
dir_item.size = dir_item.directory ? 0U : size;
|
|
||||||
dir_item.resolved = false;
|
|
||||||
result.emplace_back(std::move(dir_item));
|
|
||||||
}
|
|
||||||
|
|
||||||
return true;
|
|
||||||
} catch (const std::exception &e) {
|
|
||||||
utils::error::raise_error(__FUNCTION__, e, "exception occurred");
|
|
||||||
}
|
|
||||||
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
auto put_object_request_impl(i_http_comm &client, const s3_config &config,
|
|
||||||
std::string object_name,
|
|
||||||
const std::string &source_path,
|
|
||||||
const std::string &encryption_token,
|
|
||||||
get_key_callback get_key, set_key_callback set_key,
|
|
||||||
long &response_code, stop_type &stop_requested)
|
|
||||||
-> bool {
|
|
||||||
try {
|
|
||||||
curl::requests::http_put_file put_file{};
|
|
||||||
put_file.aws_service = "aws:amz:" + config.region + ":s3";
|
|
||||||
put_file.path = '/' + object_name;
|
|
||||||
put_file.source_path = source_path;
|
|
||||||
|
|
||||||
if (not encryption_token.empty()) {
|
|
||||||
static stop_type no_stop{false};
|
|
||||||
|
|
||||||
put_file.reader = std::make_shared<utils::encryption::encrypting_reader>(
|
|
||||||
*(utils::string::split(object_name, '/', false).end() - 1U),
|
|
||||||
source_path, no_stop, encryption_token, std::nullopt, -1);
|
|
||||||
auto key = get_key();
|
|
||||||
if (key.empty()) {
|
|
||||||
key = put_file.reader->get_encrypted_file_name();
|
|
||||||
set_key(key);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return client.make_request(put_file, response_code, stop_requested);
|
|
||||||
} catch (const std::exception &e) {
|
|
||||||
utils::error::raise_error(__FUNCTION__, e, "exception occurred");
|
|
||||||
}
|
|
||||||
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
auto read_object_request_impl(i_http_comm &client, const s3_config &config,
|
|
||||||
const std::string &object_name, std::size_t size,
|
|
||||||
std::uint64_t offset, data_buffer &data,
|
|
||||||
long &response_code, stop_type &stop_requested)
|
|
||||||
-> bool {
|
|
||||||
try {
|
|
||||||
curl::requests::http_get get{};
|
|
||||||
get.aws_service = "aws:amz:" + config.region + ":s3";
|
|
||||||
get.headers["response-content-type"] = "binary/octet-stream";
|
|
||||||
get.path = '/' + object_name;
|
|
||||||
get.range = {{offset, offset + size - 1}};
|
|
||||||
get.response_handler = [&data](const data_buffer &response_data,
|
|
||||||
long /*response_code*/) {
|
|
||||||
data = response_data;
|
|
||||||
};
|
|
||||||
|
|
||||||
return client.make_request(get, response_code, stop_requested);
|
|
||||||
} catch (const std::exception &e) {
|
|
||||||
utils::error::raise_error(__FUNCTION__, e, "exception occurred");
|
|
||||||
}
|
|
||||||
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
} // namespace repertory
|
|
||||||
|
|
||||||
#endif
|
|
||||||
@ -133,6 +133,7 @@ const auto DEFAULT_S3_CONFIG = "{\n"
|
|||||||
" \"SecretKey\": \"\",\n"
|
" \"SecretKey\": \"\",\n"
|
||||||
" \"TimeoutMs\": 60000,\n"
|
" \"TimeoutMs\": 60000,\n"
|
||||||
" \"URL\": \"\",\n"
|
" \"URL\": \"\",\n"
|
||||||
|
" \"UsePathStyle\": false,\n"
|
||||||
" \"UseRegionInURL\": false\n"
|
" \"UseRegionInURL\": false\n"
|
||||||
" },\n"
|
" },\n"
|
||||||
" \"Version\": " +
|
" \"Version\": " +
|
||||||
|
|||||||
149
tests/fixtures/s3_provider_file_list_fixture.hpp
vendored
149
tests/fixtures/s3_provider_file_list_fixture.hpp
vendored
@ -1,149 +0,0 @@
|
|||||||
/*
|
|
||||||
Copyright <2018-2023> <scott.e.graves@protonmail.com>
|
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
||||||
of this software and associated documentation files (the "Software"), to deal
|
|
||||||
in the Software without restriction, including without limitation the rights
|
|
||||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
||||||
copies of the Software, and to permit persons to whom the Software is
|
|
||||||
furnished to do so, subject to the following conditions:
|
|
||||||
|
|
||||||
The above copyright notice and this permission notice shall be included in all
|
|
||||||
copies or substantial portions of the Software.
|
|
||||||
|
|
||||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
||||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
||||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
||||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
||||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
||||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
||||||
SOFTWARE.
|
|
||||||
*/
|
|
||||||
#ifndef REPERTORY_S3_PROVIDER_FILE_LIST_FIXTURE_H
|
|
||||||
#define REPERTORY_S3_PROVIDER_FILE_LIST_FIXTURE_H
|
|
||||||
|
|
||||||
#if defined(REPERTORY_ENABLE_S3) && defined(REPERTORY_ENABLE_S3_TESTING)
|
|
||||||
#if 0
|
|
||||||
#include "test_common.hpp"
|
|
||||||
|
|
||||||
#include "mocks/mock_s3_comm.hpp"
|
|
||||||
#include "providers/s3/s3_provider.hpp"
|
|
||||||
#include "types/s3.hpp"
|
|
||||||
#include "utils/path_utils.hpp"
|
|
||||||
|
|
||||||
namespace repertory {
|
|
||||||
class s3_provider_file_list_test : public ::testing::Test {
|
|
||||||
public:
|
|
||||||
console_consumer c;
|
|
||||||
api_file_list list;
|
|
||||||
std::unique_ptr<app_config> cfg;
|
|
||||||
std::unique_ptr<mock_s3_comm> comm;
|
|
||||||
std::unique_ptr<s3_provider> provider;
|
|
||||||
std::array<std::uint64_t, 10> times = {
|
|
||||||
utils::get_file_time_now(), utils::get_file_time_now(),
|
|
||||||
utils::get_file_time_now(), utils::get_file_time_now(),
|
|
||||||
utils::get_file_time_now(), utils::get_file_time_now(),
|
|
||||||
utils::get_file_time_now(), utils::get_file_time_now(),
|
|
||||||
utils::get_file_time_now(), utils::get_file_time_now(),
|
|
||||||
};
|
|
||||||
|
|
||||||
public:
|
|
||||||
void SetUp() override {
|
|
||||||
ASSERT_TRUE(utils::file::delete_directory_recursively(
|
|
||||||
utils::path::absolute("./data")));
|
|
||||||
|
|
||||||
event_system::instance().start();
|
|
||||||
|
|
||||||
cfg = std::make_unique<app_config>(provider_type::s3,
|
|
||||||
utils::path::absolute("./data"));
|
|
||||||
EXPECT_FALSE(cfg->set_value_by_name("S3Config.Bucket", "bucket").empty());
|
|
||||||
EXPECT_FALSE(
|
|
||||||
cfg->set_value_by_name("S3Config.URL", "https://url.com").empty());
|
|
||||||
comm = std::make_unique<mock_s3_comm>(cfg->get_s3_config());
|
|
||||||
provider = std::make_unique<s3_provider>(*cfg, *comm);
|
|
||||||
|
|
||||||
const auto create_file = [this](std::size_t idx,
|
|
||||||
bool create_sub_directory = false) {
|
|
||||||
api_file file{};
|
|
||||||
file.accessed_date = times[idx];
|
|
||||||
if (create_sub_directory) {
|
|
||||||
file.api_path = utils::path::create_api_path(
|
|
||||||
"/dir/file_" + std::to_string(idx) + ".txt");
|
|
||||||
} else {
|
|
||||||
file.api_path = utils::path::create_api_path(
|
|
||||||
"/file_" + std::to_string(idx) + ".txt");
|
|
||||||
}
|
|
||||||
file.api_parent = utils::path::get_parent_api_path(file.api_path);
|
|
||||||
file.changed_date = times[idx] + 1u;
|
|
||||||
file.creation_date = times[idx] + 2u;
|
|
||||||
file.file_size = 100u + idx;
|
|
||||||
file.modified_date = times[idx] + 3u;
|
|
||||||
this->list.emplace_back(std::move(file));
|
|
||||||
};
|
|
||||||
|
|
||||||
for (std::size_t idx = 0u; idx < times.size(); idx++) {
|
|
||||||
create_file(idx);
|
|
||||||
}
|
|
||||||
|
|
||||||
for (std::size_t idx = 0u; idx < times.size(); idx++) {
|
|
||||||
create_file(idx, true);
|
|
||||||
}
|
|
||||||
|
|
||||||
EXPECT_CALL(*comm, get_file_list)
|
|
||||||
.WillRepeatedly(
|
|
||||||
[this](const get_api_file_token_callback &get_api_file_token,
|
|
||||||
const get_name_callback &get_name, api_file_list &l) {
|
|
||||||
for (auto i : list) {
|
|
||||||
auto object_name = i.api_path;
|
|
||||||
object_name = get_name(
|
|
||||||
*(utils::string::split(i.api_path, '/', false).end() - 1u),
|
|
||||||
object_name);
|
|
||||||
i.api_path = object_name;
|
|
||||||
i.api_parent = utils::path::get_parent_api_path(i.api_path);
|
|
||||||
i.encryption_token = get_api_file_token(i.api_path);
|
|
||||||
l.emplace_back(i);
|
|
||||||
}
|
|
||||||
return api_error::success;
|
|
||||||
});
|
|
||||||
|
|
||||||
EXPECT_CALL(*comm, get_file)
|
|
||||||
.WillRepeatedly(
|
|
||||||
[this](const std::string &api_path, const get_key_callback &get_key,
|
|
||||||
const get_name_callback &get_name,
|
|
||||||
const get_token_callback &get_token, api_file &file) {
|
|
||||||
auto f = std::find_if(list.begin(), list.end(),
|
|
||||||
[&api_path](const auto &cur_file) -> bool {
|
|
||||||
return cur_file.api_path == api_path;
|
|
||||||
});
|
|
||||||
if (f == list.end()) {
|
|
||||||
return api_error::item_not_found;
|
|
||||||
}
|
|
||||||
|
|
||||||
file = *f;
|
|
||||||
|
|
||||||
auto object_name = api_path;
|
|
||||||
const auto key = get_key();
|
|
||||||
object_name = get_name(key, object_name);
|
|
||||||
|
|
||||||
file.api_path = utils::path::create_api_path(object_name);
|
|
||||||
file.api_parent = utils::path::get_parent_api_path(file.api_path);
|
|
||||||
file.encryption_token = get_token();
|
|
||||||
return api_error::success;
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
void TearDown() override {
|
|
||||||
provider->stop();
|
|
||||||
comm.reset();
|
|
||||||
provider.reset();
|
|
||||||
cfg.reset();
|
|
||||||
event_system::instance().stop();
|
|
||||||
EXPECT_TRUE(utils::file::delete_directory_recursively(
|
|
||||||
utils::path::absolute("./data")));
|
|
||||||
}
|
|
||||||
};
|
|
||||||
} // namespace repertory
|
|
||||||
|
|
||||||
#endif
|
|
||||||
#endif // REPERTORY_ENABLE_S3_TESTING
|
|
||||||
#endif // REPERTORY_S3_PROVIDER_FILE_LIST_FIXTURE_H
|
|
||||||
@ -1,79 +0,0 @@
|
|||||||
/*
|
|
||||||
Copyright <2018-2023> <scott.e.graves@protonmail.com>
|
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
||||||
of this software and associated documentation files (the "Software"), to deal
|
|
||||||
in the Software without restriction, including without limitation the rights
|
|
||||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
||||||
copies of the Software, and to permit persons to whom the Software is
|
|
||||||
furnished to do so, subject to the following conditions:
|
|
||||||
|
|
||||||
The above copyright notice and this permission notice shall be included in all
|
|
||||||
copies or substantial portions of the Software.
|
|
||||||
|
|
||||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
||||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
||||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
||||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
||||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
||||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
||||||
SOFTWARE.
|
|
||||||
*/
|
|
||||||
#if defined(REPERTORY_ENABLE_S3) && defined(REPERTORY_ENABLE_S3_TESTING)
|
|
||||||
|
|
||||||
#include "test_common.hpp"
|
|
||||||
|
|
||||||
#include "app_config.hpp"
|
|
||||||
#include "comm/curl/curl_comm.hpp"
|
|
||||||
#include "events/consumers/console_consumer.hpp"
|
|
||||||
#include "events/event_system.hpp"
|
|
||||||
#include "file_manager/file_manager.hpp"
|
|
||||||
#include "platform/platform.hpp"
|
|
||||||
#include "providers/s3/s3_provider.hpp"
|
|
||||||
#include "types/s3.hpp"
|
|
||||||
#include "types/startup_exception.hpp"
|
|
||||||
#include "utils/path_utils.hpp"
|
|
||||||
#include "utils/utils.hpp"
|
|
||||||
|
|
||||||
namespace repertory {
|
|
||||||
TEST(s3_provider, get_file_list) {
|
|
||||||
{
|
|
||||||
const auto config_path = utils::path::absolute("./s3_provider_test");
|
|
||||||
ASSERT_TRUE(utils::file::delete_directory_recursively(config_path));
|
|
||||||
|
|
||||||
console_consumer con{};
|
|
||||||
event_system::instance().start();
|
|
||||||
{
|
|
||||||
app_config cfg(provider_type::s3, config_path);
|
|
||||||
{
|
|
||||||
app_config src_cfg(provider_type::s3,
|
|
||||||
utils::path::combine(get_test_dir(), {"storj"}));
|
|
||||||
cfg.set_s3_config(src_cfg.get_s3_config());
|
|
||||||
}
|
|
||||||
|
|
||||||
curl_comm comm{cfg.get_s3_config()};
|
|
||||||
s3_provider provider{cfg, comm};
|
|
||||||
file_manager mgr(cfg, provider);
|
|
||||||
mgr.start();
|
|
||||||
|
|
||||||
EXPECT_TRUE(provider.start(
|
|
||||||
[&provider](bool directory, api_file &file) -> api_error {
|
|
||||||
return provider_meta_handler(provider, directory, file);
|
|
||||||
},
|
|
||||||
&mgr));
|
|
||||||
|
|
||||||
api_file_list list{};
|
|
||||||
EXPECT_EQ(api_error::success, provider.get_file_list(list));
|
|
||||||
provider.stop();
|
|
||||||
mgr.stop();
|
|
||||||
}
|
|
||||||
event_system::instance().stop();
|
|
||||||
|
|
||||||
ASSERT_TRUE(utils::file::delete_directory_recursively(config_path));
|
|
||||||
}
|
|
||||||
|
|
||||||
EXPECT_TRUE(utils::file::delete_directory_recursively("./s3_provider_test"));
|
|
||||||
}
|
|
||||||
} // namespace repertory
|
|
||||||
|
|
||||||
#endif // REPERTORY_ENABLE_S3_TESTING
|
|
||||||
Reference in New Issue
Block a user