BlockStorage/repertory
1
0
Fork 0
Code Issues 10 Pull Requests Releases 7 Wiki Activity

2.0.0-rc (#9)
Some checks failed
BlockStorage/repertory_osx/pipeline/head This commit looks good
Details
BlockStorage/repertory_windows/pipeline/head This commit looks good
Details
BlockStorage/repertory/pipeline/head There was a failure building this commit
Details
BlockStorage/repertory_linux_builds/pipeline/head This commit looks good
Details
BlockStorage/repertory_osx_builds/pipeline/head There was a failure building this commit
Details

Browse Source 
### Issues

* \#1 \[bug\] Unable to mount S3 due to 'item_not_found' exception
* \#2 Require bucket name for S3 mounts
* \#3 \[bug\] File size is not being updated in S3 mount
* \#4 Upgrade to libfuse-3.x.x
* \#5 Switch to renterd for Sia support
* \#6 Switch to cpp-httplib to further reduce dependencies
* \#7 Remove global_data and calculate used disk space per provider
* \#8 Switch to libcurl for S3 mount support

### Changes from v1.x.x

* Added read-only encrypt provider
  * Pass-through mount point that transparently encrypts source data using `XChaCha20-Poly1305`
* Added S3 encryption support via `XChaCha20-Poly1305`
* Added replay protection to remote mounts
* Added support base64 writes in remote FUSE
* Created static linked Linux binaries for `amd64` and `aarch64` using `musl-libc`
* Removed legacy Sia renter support
* Removed Skynet support
* Fixed multiple remote mount WinFSP API issues on \*NIX servers
* Implemented chunked read and write
  * Writes for non-cached files are performed in chunks of 8Mib
* Removed `repertory-ui` support
* Removed `FreeBSD` support
* Switched to `libsodium` over `CryptoPP`
* Switched to `XChaCha20-Poly1305` for remote mounts
* Updated `GoogleTest` to v1.14.0
* Updated `JSON for Modern C++` to v3.11.2
* Updated `OpenSSL` to v1.1.1w
* Updated `RocksDB` to v8.5.3
* Updated `WinFSP` to 2023
* Updated `boost` to v1.78.0
* Updated `cURL` to v8.3.0
* Updated `zlib` to v1.3
* Use `upload_manager` for all providers
  * Adds a delay to uploads to prevent excessive API calls
  * Supports re-upload after mount restart for incomplete uploads
  * NOTE: Uploads for all providers are full file (no resume support)
    * Multipart upload support is planned for S3

Reviewed-on: #9
This commit is contained in:
Scott E. Graves
2023-10-29 06:55:59 +00:00
parent 3ff46723b8
commit f43c41f88a
839 changed files with 98214 additions and 92959 deletions
Download Patch File Download Diff File Expand all files Collapse all files

677
src/comm/aws_s3/aws_s3_comm.cpp
View File

@@ -1,677 +0,0 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#if defined(REPERTORY_ENABLE_S3)
#include "comm/aws_s3/aws_s3_comm.hpp"
#include "app_config.hpp"
#include "events/events.hpp"
#include "events/event_system.hpp"
#include "providers/i_provider.hpp"
#include "types/repertory.hpp"
#include "utils/encryption.hpp"
#include "utils/path_utils.hpp"
#include "utils/polling.hpp"
#include "utils/string_utils.hpp"
namespace repertory {
static const i_s3_comm::get_key_callback empty_key = []() { return ""; };
aws_s3_comm::aws_s3_comm(const app_config &config)
: config_(config), s3_config_(config.get_s3_config()) {
s3_config_.bucket = utils::string::trim(s3_config_.bucket);
Aws::InitAPI(sdk_options_);
if (config_.get_event_level() >= event_level::debug) {
Aws::Utils::Logging::InitializeAWSLogging(
Aws::MakeShared<Aws::Utils::Logging::DefaultLogSystem>(
"Repertory", Aws::Utils::Logging::LogLevel::Trace,
utils::path::combine(config_.get_log_directory(), {"aws_sdk_"})));
}
Aws::Auth::AWSCredentials credentials;
credentials.SetAWSAccessKeyId(s3_config_.access_key);
credentials.SetAWSSecretKey(s3_config_.secret_key);
Aws::Client::ClientConfiguration configuration;
configuration.endpointOverride = s3_config_.url;
configuration.httpRequestTimeoutMs = s3_config_.timeout_ms;
configuration.region = s3_config_.region;
configuration.requestTimeoutMs = s3_config_.timeout_ms;
configuration.connectTimeoutMs = s3_config_.timeout_ms;
// TODO make configurable
const auto enable_path_style = utils::string::begins_with(s3_config_.url, "http://localhost") ||
utils::string::begins_with(s3_config_.url, "https://localhost") ||
utils::string::begins_with(s3_config_.url, "http://127.0.0.1") ||
utils::string::begins_with(s3_config_.url, "https://127.0.0.1");
s3_client_ = std::make_unique<Aws::S3::S3Client>(
credentials, configuration, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
not enable_path_style);
polling::instance().set_callback(
{"s3_directory_cache", false, [this]() { this->clear_expired_directories(); }});
}
aws_s3_comm::~aws_s3_comm() {
polling::instance().remove_callback("s3_directory_cache");
Aws::ShutdownAPI(sdk_options_);
Aws::Utils::Logging::ShutdownAWSLogging();
s3_client_.reset();
}
void aws_s3_comm::clear_expired_directories() {
recur_mutex_lock l(cached_directories_mutex_);
std::vector<std::string> expired_list;
for (const auto &kv : cached_directories_) {
if (kv.second.expiration <= std::chrono::system_clock::now()) {
expired_list.emplace_back(kv.first);
}
}
for (const auto &expired : expired_list) {
event_system::instance().raise<debug_log>(__FUNCTION__, expired, "expired");
cached_directories_.erase(expired);
}
}
api_error aws_s3_comm::create_bucket(const std::string &api_path) {
std::string bucket_name, object_name;
get_bucket_name_and_object_name(api_path, empty_key, bucket_name, object_name);
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(__FUNCTION__, bucket_name, "begin");
}
auto ret = api_error::access_denied;
if (s3_config_.bucket.empty()) {
Aws::S3::Model::CreateBucketRequest request{};
request.SetBucket(bucket_name);
const auto outcome = s3_client_->CreateBucket(request);
if (outcome.IsSuccess()) {
remove_cached_directory(utils::path::get_parent_api_path(api_path));
ret = api_error::success;
} else {
const auto &error = outcome.GetError();
event_system::instance().raise<repertory_exception>(
__FUNCTION__, error.GetExceptionName() + "|" + error.GetMessage());
ret = api_error::comm_error;
}
}
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(__FUNCTION__, bucket_name,
"end|" + std::to_string(static_cast<int>(ret)));
}
return ret;
}
bool aws_s3_comm::exists(const std::string &api_path, const get_key_callback &get_key) const {
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(__FUNCTION__, api_path, "begin");
}
if (api_path == "/") {
return false;
}
auto ret = false;
if (not get_cached_file_exists(api_path, ret)) {
std::string bucket_name, object_name;
get_bucket_name_and_object_name(api_path, get_key, bucket_name, object_name);
Aws::S3::Model::HeadObjectRequest request{};
request.SetBucket(bucket_name);
request.SetKey(object_name);
ret = s3_client_->HeadObject(request).IsSuccess();
}
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(__FUNCTION__, api_path, "end|" + std::to_string(ret));
}
return ret;
}
void aws_s3_comm::get_bucket_name_and_object_name(const std::string &api_path,
const get_key_callback &get_key,
std::string &bucket_name,
std::string &object_name) const {
bucket_name = s3_config_.bucket;
object_name = api_path.substr(1);
if (bucket_name.empty()) {
bucket_name = utils::string::split(api_path, '/')[1];
object_name = object_name.substr(bucket_name.size());
if (not object_name.empty() && (object_name[0] == '/')) {
object_name = object_name.substr(1);
}
}
const auto key = get_key();
if (not key.empty()) {
auto parts = utils::string::split(object_name, '/', false);
parts[parts.size() - 1u] = key;
object_name = utils::string::join(parts, '/');
}
}
bool aws_s3_comm::get_cached_directory_item_count(const std::string &api_path,
std::size_t &count) const {
recur_mutex_lock l(cached_directories_mutex_);
if (cached_directories_.find(api_path) != cached_directories_.end()) {
count = cached_directories_.at(api_path).items.size();
return true;
}
return false;
}
bool aws_s3_comm::get_cached_directory_items(const std::string &api_path,
const meta_provider_callback &meta_provider,
directory_item_list &list) const {
unique_recur_mutex_lock l(cached_directories_mutex_);
if (cached_directories_.find(api_path) != cached_directories_.end()) {
auto &cachedEntry = cached_directories_.at(api_path);
list = cachedEntry.items;
cached_directories_[api_path].reset_timeout(
std::chrono::seconds(config_.get_s3_config().cache_timeout_secs));
l.unlock();
for (auto &item : list) {
meta_provider(item, false);
}
return true;
}
return false;
}
bool aws_s3_comm::get_cached_file_exists(const std::string &api_path, bool &exists) const {
exists = false;
unique_recur_mutex_lock l(cached_directories_mutex_);
const auto parent_api_path = utils::path::get_parent_api_path(api_path);
if (cached_directories_.find(parent_api_path) != cached_directories_.end()) {
auto &entry = cached_directories_.at(parent_api_path);
exists =
std::find_if(entry.items.begin(), entry.items.end(), [&api_path](const auto &item) -> bool {
return not item.directory && (api_path == item.api_path);
}) != entry.items.end();
if (exists) {
cached_directories_[api_path].reset_timeout(
std::chrono::seconds(config_.get_s3_config().cache_timeout_secs));
return true;
}
}
return false;
}
std::size_t
aws_s3_comm::get_directory_item_count(const std::string &api_path,
const meta_provider_callback &meta_provider) const {
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(__FUNCTION__, api_path, "begin");
}
std::size_t ret = 0u;
if (not(s3_config_.bucket.empty() && (api_path == "/"))) {
if (not get_cached_directory_item_count(api_path, ret)) {
directory_item_list list;
grab_directory_items(api_path, meta_provider, list);
return list.size();
}
}
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(__FUNCTION__, api_path, "end|" + std::to_string(ret));
}
return ret;
}
api_error aws_s3_comm::get_directory_items(const std::string &api_path,
const meta_provider_callback &meta_provider,
directory_item_list &list) const {
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(__FUNCTION__, api_path, "begin");
}
auto ret = api_error::success;
if (not get_cached_directory_items(api_path, meta_provider, list)) {
ret = grab_directory_items(api_path, meta_provider, list);
}
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(__FUNCTION__, api_path,
"end|" + std::to_string(std::uint8_t(ret)));
}
return ret;
}
api_error aws_s3_comm::get_file(const std::string &api_path, const get_key_callback &get_key,
const get_name_callback &get_name,
const get_token_callback &get_token, api_file &file) const {
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(__FUNCTION__, api_path, "begin");
}
auto ret = api_error::success;
std::string bucket_name, object_name;
get_bucket_name_and_object_name(api_path, get_key, bucket_name, object_name);
Aws::S3::Model::HeadObjectRequest request{};
request.SetBucket(bucket_name);
request.SetKey(object_name);
const auto outcome = s3_client_->HeadObject(request);
if (outcome.IsSuccess()) {
const auto key = get_key();
auto object = outcome.GetResult();
object_name = get_name(key, object_name);
file.accessed_date = utils::get_file_time_now();
file.api_path = utils::path::create_api_path(utils::path::combine(bucket_name, {object_name}));
file.api_parent = utils::path::get_parent_api_path(file.api_path);
file.changed_date = object.GetLastModified().Millis() * 1000u * 1000u;
file.created_date = object.GetLastModified().Millis() * 1000u * 1000u;
file.encryption_token = get_token();
if (file.encryption_token.empty()) {
file.file_size = object.GetContentLength();
} else {
file.file_size =
utils::encryption::encrypting_reader::calculate_decrypted_size(object.GetContentLength());
}
file.modified_date = object.GetLastModified().Millis() * 1000u * 1000u;
file.recoverable = true;
file.redundancy = 3.0;
} else {
const auto &error = outcome.GetError();
event_system::instance().raise<repertory_exception>(__FUNCTION__, error.GetExceptionName() +
"|" + error.GetMessage());
ret = api_error::comm_error;
}
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(__FUNCTION__, api_path,
"end|" + std::to_string(std::uint8_t(ret)));
}
return ret;
}
api_error aws_s3_comm::get_file_list(const get_api_file_token_callback &get_api_file_token,
const get_name_callback &get_name, api_file_list &list) const {
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(__FUNCTION__, "/", "begin");
}
list.clear();
auto ret = api_error::success;
const auto bucket_name = s3_config_.bucket;
if (bucket_name.empty()) {
const auto outcome = s3_client_->ListBuckets();
if (outcome.IsSuccess()) {
const auto &bucket_list = outcome.GetResult().GetBuckets();
for (const auto &bucket : bucket_list) {
get_file_list(bucket.GetName(), get_api_file_token, get_name, list);
}
} else {
const auto &error = outcome.GetError();
event_system::instance().raise<repertory_exception>(
__FUNCTION__, error.GetExceptionName() + "|" + error.GetMessage());
ret = api_error::comm_error;
}
} else {
ret = get_file_list("", get_api_file_token, get_name, list);
}
return ret;
}
api_error aws_s3_comm::get_file_list(const std::string &bucket_name,
const get_api_file_token_callback &get_api_file_token,
const get_name_callback &get_name, api_file_list &list) const {
auto ret = api_error::success;
Aws::S3::Model::ListObjectsRequest request{};
request.SetBucket(bucket_name.empty() ? s3_config_.bucket : bucket_name);
const auto outcome = s3_client_->ListObjects(request);
if (outcome.IsSuccess()) {
const auto &object_list = outcome.GetResult().GetContents();
for (auto const &object : object_list) {
api_file file{};
file.accessed_date = utils::get_file_time_now();
std::string object_name = object.GetKey();
object_name =
get_name(*(utils::string::split(object_name, '/', false).end() - 1u), object_name);
file.api_path = utils::path::create_api_path(utils::path::combine(
bucket_name.empty() ? s3_config_.bucket : bucket_name, {object_name}));
file.api_parent = utils::path::get_parent_api_path(file.api_path);
file.changed_date = object.GetLastModified().Millis() * 1000u * 1000u;
file.created_date = object.GetLastModified().Millis() * 1000u * 1000u;
file.encryption_token = get_api_file_token(file.api_path);
if (file.encryption_token.empty()) {
file.file_size = object.GetSize();
} else {
file.file_size =
object.GetSize() -
(utils::divide_with_ceiling(
static_cast<std::uint64_t>(object.GetSize()),
static_cast<std::uint64_t>(
utils::encryption::encrypting_reader::get_encrypted_chunk_size())) *
utils::encryption::encrypting_reader::get_header_size());
}
file.modified_date = object.GetLastModified().Millis() * 1000u * 1000u;
file.recoverable = true;
file.redundancy = 3.0;
list.emplace_back(std::move(file));
}
} else {
const auto &error = outcome.GetError();
event_system::instance().raise<repertory_exception>(__FUNCTION__, error.GetExceptionName() +
"|" + error.GetMessage());
ret = api_error::comm_error;
}
return ret;
}
api_error aws_s3_comm::grab_directory_items(const std::string &api_path,
const meta_provider_callback &meta_provider,
directory_item_list &list) const {
auto ret = api_error::success;
if (s3_config_.bucket.empty() && (api_path == "/")) {
const auto outcome = s3_client_->ListBuckets();
if (outcome.IsSuccess()) {
const auto &bucket_list = outcome.GetResult().GetBuckets();
for (const auto &bucket : bucket_list) {
directory_item di{};
di.api_path = utils::path::create_api_path(utils::path::combine("/", {bucket.GetName()}));
di.api_parent = utils::path::get_parent_api_path(di.api_path);
di.directory = true;
di.size = get_directory_item_count(di.api_path, meta_provider);
meta_provider(di, true);
list.emplace_back(std::move(di));
}
set_cached_directory_items(api_path, list);
} else {
const auto &error = outcome.GetError();
event_system::instance().raise<repertory_exception>(
__FUNCTION__, error.GetExceptionName() + "|" + error.GetMessage());
ret = api_error::comm_error;
}
} else {
std::string bucket_name, object_name;
get_bucket_name_and_object_name(api_path, empty_key, bucket_name, object_name);
Aws::S3::Model::ListObjectsRequest request{};
request.SetBucket(bucket_name);
request.SetDelimiter("/");
request.SetPrefix(object_name.empty() ? object_name : object_name + "/");
const auto outcome = s3_client_->ListObjects(request);
if (outcome.IsSuccess()) {
const auto &object_list = outcome.GetResult().GetContents();
for (auto const &object : object_list) {
directory_item item{};
item.api_path =
utils::path::create_api_path(utils::path::combine(bucket_name, {object.GetKey()}));
item.api_parent = utils::path::get_parent_api_path(item.api_path);
item.directory = false;
item.size = object.GetSize();
meta_provider(item, true);
list.emplace_back(std::move(item));
}
set_cached_directory_items(api_path, list);
} else {
const auto &error = outcome.GetError();
event_system::instance().raise<repertory_exception>(
__FUNCTION__, error.GetExceptionName() + "|" + error.GetMessage());
ret = api_error::comm_error;
}
}
return ret;
}
api_error aws_s3_comm::read_file_bytes(const std::string &api_path, const std::size_t &size,
const std::uint64_t &offset, std::vector<char> &data,
const get_key_callback &get_key,
const get_size_callback &get_size,
const get_token_callback &get_token,
const bool &stop_requested) const {
auto ret = api_error::success;
data.clear();
std::string bucket_name, object_name;
get_bucket_name_and_object_name(api_path, get_key, bucket_name, object_name);
const auto encryption_token = get_token();
const auto data_size = get_size();
if (encryption_token.empty()) {
Aws::S3::Model::GetObjectRequest request{};
request.SetBucket(bucket_name);
request.SetKey(object_name);
request.SetResponseContentType("application/octet-stream");
request.SetRange("bytes=" + utils::string::from_uint64(offset) + "-" +
utils::string::from_uint64(offset + size - 1u));
request.SetContinueRequestHandler(
[&stop_requested](const Aws::Http::HttpRequest *) { return not stop_requested; });
auto outcome = s3_client_->GetObject(request);
if (outcome.IsSuccess()) {
auto result = outcome.GetResultWithOwnership();
const auto len = result.GetContentLength();
data.resize(len);
result.GetBody().read(&data[0], len);
} else {
const auto &error = outcome.GetError();
event_system::instance().raise<repertory_exception>(
__FUNCTION__, error.GetExceptionName() + "|" + error.GetMessage());
ret = api_error::comm_error;
}
} else {
const auto key = utils::encryption::generate_key(encryption_token);
ret = utils::encryption::read_encrypted_range(
{offset, offset + size - 1}, key,
[&](std::vector<char> &ct, const std::uint64_t &start_offset,
const std::uint64_t &end_offset) -> api_error {
return read_file_bytes(
api_path, (end_offset - start_offset + 1u), start_offset, ct, get_key, get_size,
[]() -> std::string { return ""; }, stop_requested);
},
data_size, data);
}
return ret;
}
api_error aws_s3_comm::remove_bucket(const std::string &api_path) {
std::string bucket_name, object_name;
get_bucket_name_and_object_name(api_path, empty_key, bucket_name, object_name);
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(__FUNCTION__, bucket_name, "begin");
}
auto ret = api_error::access_denied;
if (s3_config_.bucket.empty()) {
Aws::S3::Model::DeleteBucketRequest request{};
request.SetBucket(bucket_name);
const auto outcome = s3_client_->DeleteBucket(request);
if (outcome.IsSuccess()) {
remove_cached_directory(api_path);
ret = api_error::success;
} else {
const auto &error = outcome.GetError();
event_system::instance().raise<repertory_exception>(
__FUNCTION__, error.GetExceptionName() + "|" + error.GetMessage());
ret = api_error::comm_error;
}
}
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(__FUNCTION__, bucket_name,
"end|" + std::to_string(static_cast<int>(ret)));
}
return ret;
}
void aws_s3_comm::remove_cached_directory(const std::string &api_path) {
recur_mutex_lock l(cached_directories_mutex_);
cached_directories_.erase(api_path);
}
api_error aws_s3_comm::remove_file(const std::string &api_path, const get_key_callback &get_key) {
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(__FUNCTION__, api_path, "begin");
}
auto ret = api_error::success;
std::string bucket_name, object_name;
get_bucket_name_and_object_name(api_path, get_key, bucket_name, object_name);
Aws::S3::Model::DeleteObjectRequest request{};
request.SetBucket(bucket_name);
request.SetKey(object_name);
const auto outcome = s3_client_->DeleteObject(request);
if (outcome.IsSuccess()) {
remove_cached_directory(utils::path::get_parent_api_path(api_path));
} else {
const auto &error = outcome.GetError();
event_system::instance().raise<repertory_exception>(__FUNCTION__, error.GetExceptionName() +
"|" + error.GetMessage());
ret = api_error::comm_error;
}
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(__FUNCTION__, api_path,
"end|" + std::to_string(std::uint8_t(ret)));
}
return ret;
}
api_error aws_s3_comm::rename_file(const std::string & /*api_path*/,
const std::string & /*new_api_path*/) {
return api_error::not_implemented;
/* if (config_.get_event_level() >= event_level::debug) { */
/* event_system::instance().raise<debug_log>(__FUNCTION__, api_path, "begin"); */
/* } */
/* auto ret = api_error::success; */
/* */
/* std::string bucket_name, object_name; */
/* get_bucket_name_and_object_name(api_path, bucket_name, object_name); */
/* */
/* std::string new_object_name; */
/* get_bucket_name_and_object_name(new_api_path, bucket_name, new_object_name); */
/* */
/* Aws::S3::Model::CopyObjectRequest request{}; */
/* request.SetBucket(bucket_name); */
/* request.SetCopySource(bucket_name + '/' + object_name); */
/* request.SetKey(new_object_name); */
/* */
/* const auto outcome = s3_client_->CopyObject(request); */
/* if (outcome.IsSuccess()) { */
/* ret = remove_file(api_path); */
/* } else { */
/* const auto &error = outcome.GetError(); */
/* event_system::instance().raise<repertory_exception>(__FUNCTION__, error.GetExceptionName()
* +
* "|" + */
/* error.GetMessage()); */
/* ret = api_error::comm_error; */
/* } */
/* */
/* if (config_.get_event_level() >= event_level::debug) { */
/* event_system::instance().raise<debug_log>(__FUNCTION__, api_path, */
/* "end|" + std::to_string(std::uint8_t(ret))); */
/* } */
/* return ret; */
}
void aws_s3_comm::set_cached_directory_items(const std::string &api_path,
directory_item_list list) const {
recur_mutex_lock l(cached_directories_mutex_);
cached_directories_[api_path].items = std::move(list);
cached_directories_[api_path].reset_timeout(
std::chrono::seconds(config_.get_s3_config().cache_timeout_secs));
}
api_error aws_s3_comm::upload_file(const std::string &api_path, const std::string &source_path,
const std::string &encryption_token,
const get_key_callback &get_key, const set_key_callback &set_key,
const bool &stop_requested) {
static const auto no_stop = false;
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(__FUNCTION__, api_path, "begin");
}
auto ret = api_error::success;
std::string bucket_name, object_name;
get_bucket_name_and_object_name(api_path, get_key, bucket_name, object_name);
std::shared_ptr<Aws::IOStream> file_stream;
if (encryption_token.empty()) {
file_stream = Aws::MakeShared<Aws::FStream>(&source_path[0], &source_path[0],
std::ios_base::in | std::ios_base::binary);
} else {
const auto file_name = ([&api_path]() -> std::string {
return *(utils::string::split(api_path, '/', false).end() - 1u);
})();
const auto reader =
utils::encryption::encrypting_reader(file_name, source_path, no_stop, encryption_token, -1);
auto key = get_key();
if (key.empty()) {
key = reader.get_encrypted_file_name();
set_key(key);
auto parts = utils::string::split(object_name, '/', false);
parts[parts.size() - 1u] = key;
object_name = utils::string::join(parts, '/');
}
file_stream = reader.create_iostream();
}
file_stream->seekg(0);
Aws::S3::Model::PutObjectRequest request{};
request.SetBucket(bucket_name);
request.SetKey(object_name);
request.SetBody(file_stream);
request.SetContinueRequestHandler(
[&stop_requested](const Aws::Http::HttpRequest *) { return not stop_requested; });
const auto outcome = s3_client_->PutObject(request);
if (outcome.IsSuccess()) {
remove_cached_directory(utils::path::get_parent_api_path(api_path));
} else {
const auto &error = outcome.GetError();
event_system::instance().raise<repertory_exception>(__FUNCTION__, error.GetExceptionName() +
"|" + error.GetMessage());
ret = api_error::upload_failed;
}
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(__FUNCTION__, api_path,
"end|" + std::to_string(std::uint8_t(ret)));
}
return ret;
}
} // namespace repertory
#endif // REPERTORY_ENABLE_S3

949
src/comm/curl/curl_comm.cpp
View File

@@ -1,849 +1,176 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "comm/curl/curl_comm.hpp"
#include "comm/curl/curl_resolver.hpp"
#include "curl/curl.h"
#include "types/repertory.hpp"
#include "utils/Base64.hpp"
#include "utils/encryption.hpp"
#include "utils/encrypting_reader.hpp"
#include "utils/file_utils.hpp"
#include "utils/path_utils.hpp"
#include "utils/string_utils.hpp"
namespace repertory {
struct curl_setup {
bool allow_timeout = false;
const host_config &hc;
const http_parameters *parameters = nullptr;
http_headers *headers = nullptr;
bool post = false;
raw_write_data *write_data = nullptr;
std::string *write_string = nullptr;
std::unique_ptr<curl_resolver> resolver;
};
const curl_comm::write_callback curl_comm::write_data =
static_cast<curl_comm::write_callback>([](char *buffer, size_t size,
size_t nitems,
void *outstream) -> size_t {
auto &info = *reinterpret_cast<read_write_info *>(outstream);
std::copy(buffer, buffer + (size * nitems),
std::back_inserter(info.data));
return info.stop_requested ? 0 : size * nitems;
});
struct raw_write_data {
std::vector<char> *buffer;
const bool &stop_requested;
};
struct read_data {
const bool *stop_requested = nullptr;
native_file *nf = nullptr;
std::uint64_t offset = 0u;
};
CURL *curl_comm::common_curl_setup(const std::string &path, curl_setup &setup, std::string &url,
std::string &fields) {
return common_curl_setup(utils::create_curl(), path, setup, url, fields);
}
CURL *curl_comm::common_curl_setup(CURL *curl_handle, const std::string &path, curl_setup &setup,
std::string &url, std::string &fields) {
url = construct_url(curl_handle, path, setup.hc);
if (setup.post) {
curl_easy_setopt(curl_handle, CURLOPT_HTTPGET, 0);
curl_easy_setopt(curl_handle, CURLOPT_POST, 1L);
if (setup.parameters && not setup.parameters->empty()) {
for (const auto &param : *setup.parameters) {
if (not fields.empty()) {
fields += "&";
}
if (param.first ==
"new" + app_config::get_provider_path_name(config_.get_provider_type())) {
fields += (param.first + "=" + url_encode(curl_handle, param.second));
} else {
fields += (param.first + "=" + param.second);
}
const curl_comm::write_callback curl_comm::write_headers =
static_cast<curl_comm::write_callback>([](char *buffer, size_t size,
size_t nitems,
void *outstream) -> size_t {
auto &headers = *reinterpret_cast<http_headers *>(outstream);
const auto header = std::string(buffer, size * nitems);
const auto parts = utils::string::split(header, ':');
if (parts.size() > 1u) {
auto data = header.substr(parts[0u].size() + 1u);
utils::string::left_trim(data);
utils::string::right_trim(data, '\r');
utils::string::right_trim(data, '\n');
utils::string::right_trim(data, '\r');
headers[utils::string::to_lower(parts[0u])] = data;
}
curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, &fields[0]);
} else {
curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, "");
return size * nitems;
});
curl_comm::curl_comm(host_config hc)
: host_config_(std::move(hc)), s3_config_(std::nullopt) {}
curl_comm::curl_comm(s3_config s3)
: host_config_(std::nullopt), s3_config_(std::move(s3)) {}
auto curl_comm::construct_url(CURL *curl, const std::string &relative_path,
const host_config &hc) -> std::string {
auto custom_port =
(((hc.protocol == "http") && (hc.api_port == 80U || hc.api_port == 0U)) ||
((hc.protocol == "https") && (hc.api_port == 443U || hc.api_port == 0U)))
? ""
: ":" + std::to_string(hc.api_port);
auto url = hc.protocol + "://" +
utils::string::trim_copy(hc.host_name_or_ip) + custom_port;
static const auto complete_url = [](const std::string &current_path,
const std::string &parent_path,
std::string &final_url) -> std::string & {
final_url += utils::path::create_api_path(current_path);
if (utils::string::ends_with(parent_path, "/")) {
final_url += '/';
}
} else {
curl_easy_setopt(curl_handle, CURLOPT_POST, 0);
curl_easy_setopt(curl_handle, CURLOPT_HTTPGET, 1L);
if (setup.parameters && not setup.parameters->empty()) {
url += "?";
for (const auto &param : *setup.parameters) {
if (url[url.size() - 1] != '?') {
url += "&";
}
url += (param.first + "=" + url_encode(curl_handle, param.second));
}
}
}
if (not setup.hc.agent_string.empty()) {
curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, &setup.hc.agent_string[0]);
}
if (setup.write_data) {
setup.write_data->buffer->clear();
curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, setup.write_data);
curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, write_data_callback_);
} else if (setup.write_string) {
setup.write_string->clear();
curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, setup.write_string);
curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, write_string_callback_);
}
curl_easy_setopt(curl_handle, CURLOPT_URL, url.c_str());
if (setup.allow_timeout) {
curl_easy_setopt(curl_handle, CURLOPT_TIMEOUT_MS, setup.hc.timeout_ms);
}
if (not setup.hc.api_password.empty()) {
curl_easy_setopt(curl_handle, CURLOPT_USERNAME, "");
curl_easy_setopt(curl_handle, CURLOPT_PASSWORD, &setup.hc.api_password[0]);
}
if (setup.headers) {
curl_easy_setopt(curl_handle, CURLOPT_HEADERDATA, setup.headers);
curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, write_header_callback_);
}
std::vector<std::string> items = {"localhost:" + std::to_string(setup.hc.api_port) +
":127.0.0.1"};
setup.resolver = std::make_unique<curl_resolver>(curl_handle, items);
return curl_handle;
}
std::string curl_comm::construct_url(CURL *curl_handle, const std::string &relative_path,
const host_config &hc) {
auto custom_port = (((hc.protocol == "http") && (hc.api_port == 80u)) ||
((hc.protocol == "https") && (hc.api_port == 443u)))
? ""
: ":" + std::to_string(hc.api_port);
auto ret = hc.protocol + "://" + utils::string::trim_copy(hc.host_name_or_ip) + custom_port;
auto path = utils::path::combine("/", {hc.path});
if (relative_path.empty()) {
ret += utils::path::create_api_path(path);
if (utils::string::ends_with(hc.path, "/")) {
ret += '/';
}
} else {
path = utils::path::combine(path, {url_encode(curl_handle, relative_path, true)});
ret += utils::path::create_api_path(path);
if (utils::string::ends_with(relative_path, "/")) {
ret += '/';
}
}
return ret;
}
bool curl_comm::create_auth_session(CURL *&curl_handle, const app_config &config, host_config hc,
std::string &session) {
auto ret = true;
if (not curl_handle) {
curl_handle = utils::create_curl();
}
if (not hc.auth_url.empty() && not hc.auth_user.empty()) {
event_system::instance().raise<comm_auth_begin>(hc.auth_url, hc.auth_user);
ret = false;
session = utils::create_uuid_string();
const auto cookie_path = utils::path::combine(config.get_data_directory(), {session + ".txt"});
http_headers headers{};
auto url = utils::string::right_trim(utils::string::trim(hc.auth_url), '/') + "/api/login";
if (not hc.agent_string.empty()) {
curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, &hc.agent_string[0]);
}
if (not hc.api_password.empty()) {
curl_easy_setopt(curl_handle, CURLOPT_USERNAME, "");
curl_easy_setopt(curl_handle, CURLOPT_PASSWORD, &hc.api_password[0]);
}
struct curl_slist *hs = nullptr;
hs = curl_slist_append(hs, "Content-Type: application/json");
curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, hs);
const auto payload = json({
{"email", hc.auth_user},
{"password", hc.auth_password},
})
.dump(2);
curl_easy_setopt(curl_handle, CURLOPT_COOKIEFILE, cookie_path.c_str());
curl_easy_setopt(curl_handle, CURLOPT_COOKIEJAR, cookie_path.c_str());
curl_easy_setopt(curl_handle, CURLOPT_HEADERDATA, &headers);
curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, write_header_callback_);
curl_easy_setopt(curl_handle, CURLOPT_HTTPGET, 0L);
curl_easy_setopt(curl_handle, CURLOPT_POST, 1L);
curl_easy_setopt(curl_handle, CURLOPT_TIMEOUT_MS, hc.timeout_ms);
curl_easy_setopt(curl_handle, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, payload.c_str());
long code{};
auto res = curl_easy_perform(curl_handle);
curl_easy_getinfo(curl_handle, CURLINFO_HTTP_CODE, &code);
if ((res == CURLE_OK) && ((code >= 200) && (code < 300))) {
auto cookie = headers["set-cookie"];
if (cookie.empty()) {
code = 400;
} else {
cookie = headers["set-cookie"];
if (cookie.empty() || not utils::string::contains(cookie, "skynet-jwt=")) {
code = 401;
} else {
ret = true;
}
}
}
if (ret) {
curl_easy_setopt(curl_handle, CURLOPT_COOKIELIST, "FLUSH");
update_auth_session(utils::reset_curl(curl_handle), config, session);
} else {
curl_easy_cleanup(curl_handle);
release_auth_session(config, hc, session);
}
curl_slist_free_all(hs);
event_system::instance().raise<comm_auth_end>(hc.auth_url, hc.auth_user, res, code);
}
return ret;
}
api_error curl_comm::get_or_post(const host_config &hc, const bool &post, const std::string &path,
const http_parameters &parameters, json &data, json &error,
http_headers *headers, std::function<void(CURL *curl_handle)> cb) {
std::string result;
auto setup = curl_setup{not post, hc, &parameters, headers, post, nullptr, &result, nullptr};
std::string fields;
std::string url;
auto *curl_handle = common_curl_setup(path, setup, url, fields);
if (cb) {
cb(curl_handle);
}
if (config_.get_event_level() >= event_level::verbose) {
if (post) {
event_system::instance().raise<comm_post_begin>(url, fields);
} else {
event_system::instance().raise<comm_get_begin>(url);
}
}
std::unique_ptr<std::chrono::system_clock::time_point> tp(nullptr);
if (config_.get_enable_comm_duration_events()) {
tp = std::make_unique<std::chrono::system_clock::time_point>(std::chrono::system_clock::now());
}
const auto curl_code = curl_easy_perform(curl_handle);
if (curl_code == CURLE_OPERATION_TIMEDOUT) {
event_system::instance().raise<repertory_exception>(__FUNCTION__, "CURL timeout: " + path);
}
long http_code = -1;
curl_easy_getinfo(curl_handle, CURLINFO_RESPONSE_CODE, &http_code);
if (tp != nullptr) {
const auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::system_clock::now() - *tp);
event_system::instance().raise<comm_duration>(url, std::to_string(duration.count()));
}
const auto ret = process_json_response(url, curl_code, http_code, result, data, error);
if (config_.get_event_level() >= event_level::verbose) {
if (post) {
event_system::instance().raise<comm_post_end>(
url, curl_code, http_code, ((ret == api_error::success) ? data.dump(2) : error.dump(2)));
} else {
event_system::instance().raise<comm_get_end>(
url, curl_code, http_code, ((ret == api_error::success) ? data.dump(2) : error.dump(2)));
}
}
curl_easy_cleanup(curl_handle);
return ret;
}
api_error curl_comm::get_range(const host_config &hc, const std::string &path,
const std::uint64_t &data_size, const http_parameters &parameters,
const std::string &encryption_token, std::vector<char> &data,
const http_ranges &ranges, json &error, http_headers *headers,
const bool &stop_requested) {
if (encryption_token.empty()) {
return get_range_unencrypted(hc, path, parameters, data, ranges, error, headers,
stop_requested);
}
if (ranges.empty()) {
return api_error::error;
}
const auto key = utils::encryption::generate_key(encryption_token);
for (const auto &range : ranges) {
const auto result = utils::encryption::read_encrypted_range(
range, key,
[&](std::vector<char> &ct, const std::uint64_t &start_offset,
const std::uint64_t &end_offset) -> api_error {
const auto ret =
get_range_unencrypted(hc, path, parameters, ct, {{start_offset, end_offset}}, error,
headers, stop_requested);
headers = nullptr;
return ret;
},
data_size, data);
if (result != api_error::success) {
return result;
}
}
return api_error::success;
}
api_error curl_comm::get_range_unencrypted(const host_config &hc, const std::string &path,
const http_parameters &parameters,
std::vector<char> &data, const http_ranges &ranges,
json &error, http_headers *headers,
const bool &stop_requested) {
raw_write_data wd = {&data, stop_requested};
auto setup = curl_setup{false, hc, &parameters, headers, false, &wd, nullptr, nullptr};
std::string fields;
std::string url;
auto *curl_handle = common_curl_setup(path, setup, url, fields);
std::string range_list;
if (not ranges.empty()) {
range_list = std::accumulate(
std::next(ranges.begin()), ranges.end(), http_range_to_string(ranges[0]),
[](const auto &l, const auto &r) { return l + ',' + http_range_to_string(r); });
curl_easy_setopt(curl_handle, CURLOPT_RANGE, &range_list[0]);
}
return execute_binary_operation<comm_get_range_begin, comm_get_range_end>(curl_handle, url, data,
error, stop_requested);
}
api_error curl_comm::get_raw(const host_config &hc, const std::string &path,
const http_parameters &parameters, std::vector<char> &data,
json &error, const bool &stop_requested) {
raw_write_data wd = {&data, stop_requested};
auto setup = curl_setup{false, hc, &parameters, nullptr, false, &wd, nullptr, nullptr};
std::string fields;
std::string url;
return execute_binary_operation<comm_get_range_begin, comm_get_range_end>(
common_curl_setup(path, setup, url, fields), url, data, error, stop_requested);
}
std::string curl_comm::http_range_to_string(const http_range &range) {
return std::to_string(range.begin) + '-' + std::to_string(range.end);
}
api_error curl_comm::post_file(const host_config &hc, const std::string &path,
const std::string &source_path, const http_parameters &parameters,
json &data, json &error, const bool &stop_requested) {
auto ret = api_error::os_error;
std::uint64_t file_size{};
if (utils::file::get_file_size(source_path, file_size)) {
std::string result;
auto setup = curl_setup{false, hc, &parameters, nullptr, true, nullptr, &result, nullptr};
std::string fields;
std::string url;
auto *curl_handle = common_curl_setup(path, setup, url, fields);
curl_easy_setopt(curl_handle, CURLOPT_UPLOAD, 1L);
curl_easy_setopt(curl_handle, CURLOPT_INFILESIZE_LARGE, file_size);
native_file::native_file_ptr nf;
native_file::create_or_open(source_path, nf);
if (nf) {
read_data rd = {&stop_requested};
rd.nf = nf.get();
curl_easy_setopt(curl_handle, CURLOPT_READDATA, &rd);
curl_easy_setopt(curl_handle, CURLOPT_READFUNCTION, read_data_callback_);
ret = execute_json_operation<comm_post_file_begin, comm_post_file_end>(
curl_handle, url, result, data, error, stop_requested);
nf->close();
}
}
return ret;
}
api_error curl_comm::post_multipart_file(const host_config &hc, const std::string &path,
const std::string &file_name,
const std::string &source_path,
const std::string &encryption_token, json &data,
json &error, const bool &stop_requested) {
std::string result;
std::uint64_t file_size{};
std::unique_ptr<utils::encryption::encrypting_reader> reader;
if (encryption_token.empty()) {
if (not utils::file::get_file_size(source_path, file_size)) {
return api_error::os_error;
}
} else {
try {
reader = std::make_unique<utils::encryption::encrypting_reader>(
file_name, source_path, stop_requested, encryption_token);
file_size = reader->get_total_size();
} catch (const std::exception &e) {
event_system::instance().raise<repertory_exception>(__FUNCTION__, e.what());
return api_error::error;
}
}
std::string session;
CURL *curl_handle = nullptr;
if (not session_manager_.create_auth_session(curl_handle, config_, hc, session)) {
return api_error::access_denied;
}
if (config_.get_provider_type() == provider_type::skynet) {
curl_easy_cleanup(curl_handle);
const auto fn = reader ? reader->get_encrypted_file_name() : file_name;
const auto fs = reader ? reader->get_total_size() : file_size;
std::string location;
if (tus_upload_create(hc, fn, fs, location)) {
std::string skylink;
if (tus_upload(hc, source_path, fn, fs, location, skylink, stop_requested, reader.get())) {
data["skylink"] = skylink;
return api_error::success;
}
}
return api_error::comm_error;
}
auto setup = curl_setup{false, hc, nullptr, nullptr, true, nullptr, &result, nullptr};
std::string fields;
std::string url;
common_curl_setup(curl_handle, path, setup, url, fields);
auto *curl_mime = curl_mime_init(curl_handle);
auto *mime_part = curl_mime_addpart(curl_mime);
curl_mime_name(mime_part, "file");
auto ret = api_error::success;
if (encryption_token.empty()) {
curl_mime_filename(mime_part, &file_name[0]);
curl_mime_filedata(mime_part, &source_path[0]);
} else {
try {
curl_mime_filename(mime_part, reader->get_encrypted_file_name().c_str());
curl_mime_data_cb(
mime_part, reader->get_total_size(),
static_cast<curl_read_callback>(utils::encryption::encrypting_reader::reader_function),
nullptr, nullptr, reader.get());
} catch (const std::exception &e) {
event_system::instance().raise<repertory_exception>(__FUNCTION__, e.what());
ret = api_error::error;
}
}
if (ret == api_error::success) {
curl_easy_setopt(curl_handle, CURLOPT_MIMEPOST, curl_mime);
ret = execute_json_operation<comm_post_multi_part_file_begin, comm_post_multi_part_file_end>(
curl_handle, url, result, data, error, stop_requested,
(ret == api_error::success) ? CURLE_OK : CURLE_SEND_ERROR);
}
curl_mime_free(curl_mime);
curl_easy_cleanup(curl_handle);
/* if (not session.empty() && (ret == api_error::success)) { */
/* auto pin_hc = hc; */
/* */
/* const auto skylink = data["skylink"].get<std::string>(); */
/* utils::string::replace(pin_hc.path, "/skyfile", "/pin/" + skylink); */
/* */
/* ret = api_error::comm_error; */
/* for (std::uint8_t i = 0u; not stop_requested && (i < 30u) && (ret != api_error::success);
* i++) { */
/* if (i) { */
/* event_system::instance().raise<repertory_exception>( */
/* __FUNCTION__, "RETRY [" + std::to_string(i) + "] Pin failed for file: " +
* file_name); */
/* std::this_thread::sleep_for(1s); */
/* } */
/* */
/* json response; */
/* http_headers headers{}; */
/* ret = get_or_post(pin_hc, true, "", {}, response, error, &headers, [&](CURL *curl_handle)
* { */
/* session_manager_.update_auth_session(curl_handle, config_, hc); */
/* }); */
/* } */
/* } */
session_manager_.release_auth_session(config_, hc, session);
return ret;
}
api_error curl_comm::process_binary_response(const std::string &url, const CURLcode &res,
const long &http_code, std::vector<char> data,
json &error) {
const auto ret = process_response(
url, res, http_code, data.size(),
[&]() -> std::string { return (data.empty() ? "" : std::string(&data[0], data.size())); },
nullptr, error);
if (ret != api_error::success) {
data.clear();
}
return ret;
}
api_error curl_comm::process_json_response(const std::string &url, const CURLcode &res,
const long &http_code, const std::string &result,
json &data, json &error) {
const auto ret = process_response(
url, res, http_code, result.size(), [&]() -> std::string { return result; },
[&]() {
if (result.length()) {
data = json::parse(result.c_str());
}
},
error);
if (ret != api_error::success) {
data.clear();
}
return ret;
}
api_error curl_comm::process_response(const std::string &url, const CURLcode &res,
const long &http_code, const std::size_t &data_size,
const std::function<std::string()> &to_string_convertor,
const std::function<void()> &success_handler,
json &error) const {
auto ret = api_error::success;
auto construct_error = [&]() {
ret = api_error::comm_error;
const auto *curl_string = curl_easy_strerror(res);
std::string error_string(curl_string ? curl_string : "");
error["message"] = error_string + ":" + std::to_string(http_code);
error["url"] = url;
return final_url;
};
if ((res == CURLE_OK) && ((http_code >= 200) && (http_code < 300))) {
if (success_handler) {
success_handler();
}
} else if (data_size == 0u) {
construct_error();
} else {
try {
const auto tmp = json::parse(to_string_convertor().c_str());
if (tmp.find("message") != tmp.end()) {
ret = api_error::comm_error;
error = tmp;
error["url"] = url;
} else {
construct_error();
}
} catch (...) {
construct_error();
}
}
return ret;
auto path = utils::path::combine("/", {hc.path});
return relative_path.empty()
? complete_url(path, hc.path, url)
: complete_url(utils::path::combine(
path, {url_encode(curl, relative_path, true)}),
relative_path, url);
}
void curl_comm::release_auth_session(const app_config &config, host_config hc,
const std::string &session) {
if (not hc.auth_url.empty() && not hc.auth_user.empty()) {
event_system::instance().raise<comm_auth_logout_begin>(hc.auth_url, hc.auth_user);
const auto cookie_path = utils::path::combine(config.get_data_directory(), {session + ".txt"});
const auto url =
utils::string::right_trim(utils::string::trim(hc.auth_url), '/') + "/api/logout";
auto curl_comm::create_host_config(const s3_config &config,
bool use_s3_path_style) -> host_config {
host_config hc{};
hc.api_password = config.secret_key;
hc.api_user = config.access_key;
auto *curl_handle = utils::create_curl();
if (not hc.agent_string.empty()) {
curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, &hc.agent_string[0]);
auto pos = config.url.find(':');
hc.host_name_or_ip = config.url.substr(pos + 3U);
if (config.use_region_in_url && not config.region.empty()) {
auto parts = utils::string::split(hc.host_name_or_ip, '.', false);
if (parts.size() > 1U) {
parts.insert(parts.begin() + 1U, config.region);
hc.host_name_or_ip = utils::string::join(parts, '.');
}
if (not hc.api_password.empty()) {
curl_easy_setopt(curl_handle, CURLOPT_USERNAME, "");
curl_easy_setopt(curl_handle, CURLOPT_PASSWORD, &hc.api_password[0]);
}
curl_easy_setopt(curl_handle, CURLOPT_COOKIEFILE, cookie_path.c_str());
curl_easy_setopt(curl_handle, CURLOPT_COOKIEJAR, cookie_path.c_str());
curl_easy_setopt(curl_handle, CURLOPT_HTTPGET, 0L);
curl_easy_setopt(curl_handle, CURLOPT_POST, 1L);
curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, "");
curl_easy_setopt(curl_handle, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl_handle, CURLOPT_TIMEOUT_MS, hc.timeout_ms);
curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, write_null_callback_);
const auto curl_code = curl_easy_perform(curl_handle);
long http_code = -1;
curl_easy_getinfo(curl_handle, CURLINFO_HTTP_CODE, &http_code);
curl_easy_cleanup(curl_handle);
event_system::instance().raise<comm_auth_logout_end>(hc.auth_url, hc.auth_user, curl_code,
http_code);
utils::file::delete_file(cookie_path);
}
if (not use_s3_path_style) {
hc.host_name_or_ip = config.bucket + '.' + hc.host_name_or_ip;
}
hc.protocol = config.url.substr(0U, pos);
if (use_s3_path_style) {
hc.path = '/' + config.bucket;
}
return hc;
}
bool curl_comm::tus_upload(host_config hc, const std::string &source_path,
const std::string &file_name, std::uint64_t file_size,
const std::string &location, std::string &skylink,
const bool &stop_requested,
utils::encryption::encrypting_reader *reader) {
static const constexpr std::uint64_t max_skynet_file_size = (1ull << 22ull) * 10ull;
auto tus_hc = hc;
utils::string::replace(tus_hc.path, "/skyfile", "/tus");
auto ret = true;
std::uint64_t offset = 0u;
native_file::native_file_ptr nf;
while (ret && file_size) {
const auto chunk_size = std::min(max_skynet_file_size, file_size);
auto *curl_handle = utils::create_curl();
curl_easy_setopt(curl_handle, CURLOPT_CUSTOMREQUEST, "PATCH");
curl_easy_setopt(curl_handle, CURLOPT_HTTPGET, 0);
curl_easy_setopt(curl_handle, CURLOPT_NOBODY, 0);
const auto upload_offset = "Upload-Offset: " + std::to_string(offset);
const auto content_length = "Content-Length: " + std::to_string(chunk_size);
struct curl_slist *hs = nullptr;
hs = curl_slist_append(hs, "tus-resumable: 1.0.0");
hs = curl_slist_append(hs, "Content-Type: application/offset+octet-stream");
hs = curl_slist_append(hs, content_length.c_str());
hs = curl_slist_append(hs, upload_offset.c_str());
curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, hs);
curl_easy_setopt(curl_handle, CURLOPT_INFILESIZE_LARGE, chunk_size);
curl_easy_setopt(curl_handle, CURLOPT_UPLOAD, 1L);
session_manager_.update_auth_session(curl_handle, config_, hc);
read_data rd = {&stop_requested};
if (reader) {
curl_easy_setopt(curl_handle, CURLOPT_READDATA, reader);
curl_easy_setopt(
curl_handle, CURLOPT_READFUNCTION,
static_cast<curl_read_callback>(utils::encryption::encrypting_reader::reader_function));
} else {
if (not nf) {
native_file::create_or_open(source_path, nf);
}
if (nf) {
rd.nf = nf.get();
rd.offset = offset;
curl_easy_setopt(curl_handle, CURLOPT_READDATA, &rd);
curl_easy_setopt(curl_handle, CURLOPT_READFUNCTION, read_data_callback_);
} else {
ret = false;
}
}
if (ret) {
event_system::instance().raise<comm_tus_upload_begin>(file_name, location, file_size, offset);
curl_easy_setopt(curl_handle, CURLOPT_URL, location.c_str());
const auto curl_code = curl_easy_perform(curl_handle);
long http_code = -1;
curl_easy_getinfo(curl_handle, CURLINFO_HTTP_CODE, &http_code);
event_system::instance().raise<comm_tus_upload_end>(file_name, location, file_size, offset,
curl_code, http_code);
if ((ret = ((curl_code == CURLE_OK) && (http_code >= 200) && (http_code < 300)))) {
file_size -= chunk_size;
offset += chunk_size;
}
}
curl_easy_cleanup(curl_handle);
curl_slist_free_all(hs);
}
if (nf) {
nf->close();
}
if (ret) {
auto *curl_handle = utils::create_curl();
http_headers headers{};
curl_easy_setopt(curl_handle, CURLOPT_CUSTOMREQUEST, "HEAD");
curl_easy_setopt(curl_handle, CURLOPT_NOBODY, 1L);
curl_easy_setopt(curl_handle, CURLOPT_HEADERDATA, &headers);
curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, write_header_callback_);
session_manager_.update_auth_session(curl_handle, config_, hc);
struct curl_slist *hs = nullptr;
hs = curl_slist_append(hs, "tus-resumable: 1.0.0");
curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, hs);
curl_easy_setopt(curl_handle, CURLOPT_URL, location.c_str());
const auto res = curl_easy_perform(curl_handle);
long http_code = -1;
curl_easy_getinfo(curl_handle, CURLINFO_HTTP_CODE, &http_code);
ret = ((res == CURLE_OK) && (http_code >= 200) && (http_code < 300));
skylink = headers["skynet-skylink"];
curl_easy_cleanup(curl_handle);
curl_slist_free_all(hs);
}
return ret;
void curl_comm::enable_s3_path_style(bool enable) {
use_s3_path_style_ = enable;
}
bool curl_comm::tus_upload_create(host_config hc, const std::string &fileName,
const std::uint64_t &fileSize, std::string &location) {
auto tus_hc = hc;
utils::string::replace(tus_hc.path, "/skyfile", "/tus");
auto *curl_handle = utils::create_curl();
const auto url = construct_url(curl_handle, "", tus_hc);
event_system::instance().raise<comm_tus_upload_create_begin>(fileName, url);
http_headers headers{};
curl_easy_setopt(curl_handle, CURLOPT_HTTPGET, 0);
curl_easy_setopt(curl_handle, CURLOPT_POST, 1L);
curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, "");
curl_easy_setopt(curl_handle, CURLOPT_HEADERDATA, &headers);
curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, write_header_callback_);
const auto upload_length = "upload-length: " + utils::string::from_uint64(fileSize);
const auto upload_metadata = "upload-metadata: filename " + macaron::Base64::Encode(fileName) +
",filetype " + macaron::Base64::Encode("application/octet-stream");
struct curl_slist *hs = nullptr;
hs = curl_slist_append(hs, "tus-resumable: 1.0.0");
hs = curl_slist_append(hs, "Content-Length: 0");
hs = curl_slist_append(hs, upload_length.c_str());
hs = curl_slist_append(hs, upload_metadata.c_str());
curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, hs);
session_manager_.update_auth_session(curl_handle, config_, hc);
curl_easy_setopt(curl_handle, CURLOPT_URL, url.c_str());
const auto res = curl_easy_perform(curl_handle);
long http_code = -1;
curl_easy_getinfo(curl_handle, CURLINFO_HTTP_CODE, &http_code);
const auto ret = (res == CURLE_OK && http_code == 201);
if (ret) {
location = headers["location"];
}
curl_easy_cleanup(curl_handle);
curl_slist_free_all(hs);
event_system::instance().raise<comm_tus_upload_create_end>(fileName, url, res, http_code);
return ret;
auto curl_comm::make_request(const curl::requests::http_delete &del,
long &response_code,
stop_type &stop_requested) const -> bool {
return make_request(
s3_config_.has_value()
? create_host_config(s3_config_.value(), use_s3_path_style_)
: host_config_.value(),
del, response_code, stop_requested);
}
void curl_comm::update_auth_session(CURL *curl_handle, const app_config &config,
const std::string &session) {
const auto cookie_path = utils::path::combine(config.get_data_directory(), {session + ".txt"});
curl_easy_setopt(curl_handle, CURLOPT_COOKIEFILE, cookie_path.c_str());
curl_easy_setopt(curl_handle, CURLOPT_COOKIEJAR, cookie_path.c_str());
auto curl_comm::make_request(const curl::requests::http_get &get,
long &response_code,
stop_type &stop_requested) const -> bool {
return make_request(
s3_config_.has_value()
? create_host_config(s3_config_.value(), use_s3_path_style_)
: host_config_.value(),
get, response_code, stop_requested);
}
std::string curl_comm::url_encode(CURL *curl_handle, const std::string &data,
const bool &allow_slash) {
auto *value = curl_easy_escape(curl_handle, data.c_str(), 0);
auto curl_comm::make_request(const curl::requests::http_head &head,
long &response_code,
stop_type &stop_requested) const -> bool {
return make_request(
s3_config_.has_value()
? create_host_config(s3_config_.value(), use_s3_path_style_)
: host_config_.value(),
head, response_code, stop_requested);
}
auto curl_comm::make_request(const curl::requests::http_put_file &put_file,
long &response_code,
stop_type &stop_requested) const -> bool {
return make_request(
s3_config_.has_value()
? create_host_config(s3_config_.value(), use_s3_path_style_)
: host_config_.value(),
put_file, response_code, stop_requested);
}
auto curl_comm::url_encode(CURL *curl, const std::string &data,
bool allow_slash) -> std::string {
auto *value =
curl_easy_escape(curl, data.c_str(), static_cast<int>(data.size()));
std::string ret = value;
curl_free(value);
if (allow_slash) {
utils::string::replace(ret, "%2F", "/");
}
return ret;
}
curl_comm::curl_read_callback curl_comm::read_data_callback_ =
static_cast<curl_comm::curl_read_callback>(
[](char *buffer, size_t size, size_t nitems, void *instream) -> size_t {
auto *rd = reinterpret_cast<read_data *>(instream);
std::size_t bytes_read{};
const auto ret = rd->nf->read_bytes(buffer, size * nitems, rd->offset, bytes_read);
if (ret) {
rd->offset += bytes_read;
}
return ret && not *rd->stop_requested ? bytes_read : CURL_READFUNC_ABORT;
});
curl_comm::curl_write_callback curl_comm::write_data_callback_ =
static_cast<curl_comm::curl_write_callback>(
[](char *buffer, size_t size, size_t nitems, void *outstream) -> size_t {
auto *wd = reinterpret_cast<raw_write_data *>(outstream);
std::copy(buffer, buffer + (size * nitems), std::back_inserter(*wd->buffer));
return wd->stop_requested ? 0 : size * nitems;
});
curl_comm::curl_write_callback curl_comm::write_header_callback_ =
static_cast<curl_comm::curl_write_callback>(
[](char *buffer, size_t size, size_t nitems, void *outstream) -> size_t {
auto &headers = *reinterpret_cast<http_headers *>(outstream);
const auto header = std::string(buffer, size * nitems);
const auto parts = utils::string::split(header, ':');
if (parts.size() > 1u) {
auto data = header.substr(parts[0u].size() + 1u);
utils::string::left_trim(data);
utils::string::right_trim(data, '\r');
utils::string::right_trim(data, '\n');
utils::string::right_trim(data, '\r');
headers[utils::string::to_lower(parts[0u])] = data;
}
return size * nitems;
});
curl_comm::curl_write_callback curl_comm::write_null_callback_ =
static_cast<curl_comm::curl_write_callback>(
[](char *, size_t size, size_t nitems, void *) -> size_t { return size * nitems; });
curl_comm::curl_write_callback curl_comm::write_string_callback_ =
static_cast<curl_comm::curl_write_callback>(
[](char *buffer, size_t size, size_t nitems, void *outstream) -> size_t {
(*reinterpret_cast<std::string *>(outstream)) += std::string(buffer, size * nitems);
return size * nitems;
});
} // namespace repertory

43
src/comm/curl/curl_resolver.cpp
View File

@@ -1,43 +0,0 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#include "comm/curl/curl_resolver.hpp"
namespace repertory {
curl_resolver::curl_resolver(CURL *handle, std::vector<std::string> items, const bool &ignore_root)
: items_(std::move(items)) {
#ifndef _WIN32
if (ignore_root && (getuid() == 0)) {
items_.clear();
}
#endif
for (const auto &item : items_) {
host_list_ = curl_slist_append(host_list_, &item[0u]);
}
if (host_list_) {
curl_easy_setopt(handle, CURLOPT_RESOLVE, host_list_);
}
}
curl_resolver::~curl_resolver() {
if (host_list_) {
curl_slist_free_all(host_list_);
}
}
} // namespace repertory

40
src/comm/curl/multi_request.cpp
View File

@@ -1,27 +1,33 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "comm/curl/multi_request.hpp"
#include "utils/utils.hpp"
namespace repertory {
multi_request::multi_request(CURL *curl_handle, const bool &stop_requested)
: curl_handle_(curl_handle), stop_requested_(stop_requested), multi_handle_(curl_multi_init()) {
multi_request::multi_request(CURL *curl_handle, stop_type &stop_requested)
: curl_handle_(curl_handle),
stop_requested_(stop_requested),
multi_handle_(curl_multi_init()) {
curl_multi_add_handle(multi_handle_, curl_handle);
}
@@ -48,9 +54,11 @@ void multi_request::get_result(CURLcode &curl_code, long &http_code) {
if (not stop_requested_) {
int remaining_messages = 0;
auto *multi_result = curl_multi_info_read(multi_handle_, &remaining_messages);
auto *multi_result =
curl_multi_info_read(multi_handle_, &remaining_messages);
if (multi_result && (multi_result->msg == CURLMSG_DONE)) {
curl_easy_getinfo(multi_result->easy_handle, CURLINFO_RESPONSE_CODE, &http_code);
curl_easy_getinfo(multi_result->easy_handle, CURLINFO_RESPONSE_CODE,
&http_code);
curl_code = multi_result->data.result;
}
}

80
src/comm/curl/requests/http_put_file.cpp Normal file
View File

@@ -0,0 +1,80 @@
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "comm/curl/requests/http_put_file.hpp"
#include "utils/string_utils.hpp"
namespace repertory::curl::requests {
auto http_put_file::get_path() const -> std::string {
if (reader) {
auto updated_path = path;
return utils::string::replace(updated_path, file_name,
reader->get_encrypted_file_name());
}
return http_request_base::get_path();
}
auto http_put_file::set_method(CURL *curl, stop_type &stop_requested) const
-> bool {
curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "PUT");
if (not source_path.empty()) {
curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L);
if (not encryption_token.value_or("").empty()) {
if (not reader) {
reader = std::make_shared<utils::encryption::encrypting_reader>(
file_name, source_path, stop_requested, encryption_token.value());
}
curl_easy_setopt(curl, CURLOPT_READDATA, reader.get());
curl_easy_setopt(
curl, CURLOPT_READFUNCTION,
static_cast<curl_read_callback>(
utils::encryption::encrypting_reader::reader_function));
curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE,
reader->get_total_size());
return true;
}
read_info = std::make_shared<read_file_info>(read_file_info{
stop_requested,
});
if (native_file::open(source_path, read_info->nf) != api_error::success) {
return false;
}
read_info->nf->set_auto_close(true);
std::uint64_t file_size{};
if (not read_info->nf->get_file_size(file_size)) {
return false;
}
curl_easy_setopt(curl, CURLOPT_READDATA, read_info.get());
curl_easy_setopt(curl, CURLOPT_READFUNCTION, read_file_data);
curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, file_size);
}
return true;
}
} // namespace repertory::curl::requests

73
src/comm/curl/session_manager.cpp
View File

@@ -1,73 +0,0 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#include "comm/curl/session_manager.hpp"
#include "app_config.hpp"
#include "comm/curl/curl_comm.hpp"
#include "utils/utils.hpp"
namespace repertory {
bool session_manager::create_auth_session(CURL *&curl_handle, const app_config &config,
host_config hc, std::string &session) {
auto ret = true;
if (not curl_handle) {
curl_handle = utils::create_curl();
}
if (not hc.auth_url.empty() && not hc.auth_user.empty()) {
mutex_lock l(session_mutex_);
if (session_.empty()) {
if ((ret = curl_comm::create_auth_session(curl_handle, config, hc, session))) {
session_ = session;
}
} else {
session = session_;
curl_comm::update_auth_session(curl_handle, config, session_);
}
if (ret) {
session_count_++;
}
}
return ret;
}
void session_manager::release_auth_session(const app_config &config, host_config hc,
const std::string &session) {
if (not hc.auth_url.empty() && not hc.auth_user.empty()) {
mutex_lock l(session_mutex_);
if (not session_.empty() && (session == session_) && session_count_) {
if (not --session_count_) {
curl_comm::release_auth_session(config, hc, session_);
session_.clear();
}
}
}
}
void session_manager::update_auth_session(CURL *curl_handle, const app_config &config,
const host_config &hc) {
mutex_lock l(session_mutex_);
if (hc.auth_url.empty() || hc.auth_user.empty() || session_.empty()) {
return;
}
curl_comm::update_auth_session(curl_handle, config, session_);
}
} // namespace repertory

78
src/comm/packet/client_pool.cpp
View File

@@ -1,29 +1,35 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "comm/packet/client_pool.hpp"
#include "events/events.hpp"
#include "events/event_system.hpp"
#include "events/events.hpp"
#include "utils/error_utils.hpp"
namespace repertory {
void client_pool::pool::execute(const std::uint64_t &thread_id, const worker_callback &worker,
const worker_complete_callback &worker_complete) {
const auto index = static_cast<std::size_t>(thread_id % pool_queues_.size());
void client_pool::pool::execute(
std::uint64_t thread_id, const worker_callback &worker,
const worker_complete_callback &worker_complete) {
const auto index = thread_id % pool_queues_.size();
auto wi = std::make_shared<work_item>(worker, worker_complete);
auto &pool_queue = pool_queues_[index];
@@ -33,14 +39,15 @@ void client_pool::pool::execute(const std::uint64_t &thread_id, const worker_cal
queue_lock.unlock();
}
client_pool::pool::pool(const std::uint8_t &pool_size) {
client_pool::pool::pool(std::uint8_t pool_size) {
event_system::instance().raise<service_started>("client_pool");
thread_index_ = 0u;
for (std::uint8_t i = 0u; i < pool_size; i++) {
pool_queues_.emplace_back(std::make_unique<work_queue>());
}
for (std::size_t i = 0u; i < pool_queues_.size(); i++) {
pool_threads_.emplace_back(std::thread([this]() {
pool_threads_.emplace_back([this]() {
const auto thread_index = thread_index_++;
auto &pool_queue = pool_queues_[thread_index];
@@ -58,18 +65,18 @@ client_pool::pool::pool(const std::uint8_t &pool_size) {
}
while (not queue.empty()) {
auto workItem = queue.front();
auto item = queue.front();
queue.pop_front();
queue_notify.notify_all();
queue_lock.unlock();
try {
const auto result = workItem->work();
workItem->work_complete(result);
const auto result = item->work();
item->work_complete(result);
} catch (const std::exception &e) {
workItem->work_complete(utils::translate_api_error(api_error::error));
event_system::instance().raise<repertory_exception>(__FUNCTION__,
e.what() ? e.what() : "unknown");
item->work_complete(utils::from_api_error(api_error::error));
utils::error::raise_error(__FUNCTION__, e,
"exception occurred in work item");
}
queue_lock.lock();
@@ -86,14 +93,14 @@ client_pool::pool::pool(const std::uint8_t &pool_size) {
queue_notify.notify_all();
queue_lock.unlock();
wi->work_complete(utils::translate_api_error(api_error::download_stopped));
wi->work_complete(utils::from_api_error(api_error::download_stopped));
queue_lock.lock();
}
queue_notify.notify_all();
queue_lock.unlock();
}));
});
}
}
@@ -113,7 +120,7 @@ void client_pool::pool::shutdown() {
pool_threads_.clear();
}
void client_pool::execute(const std::string &client_id, const std::uint64_t &thread_id,
void client_pool::execute(const std::string &client_id, std::uint64_t thread_id,
const worker_callback &worker,
const worker_complete_callback &worker_complete) {
unique_mutex_lock pool_lock(pool_mutex_);
@@ -136,15 +143,18 @@ void client_pool::remove_client(const std::string &client_id) {
}
void client_pool::shutdown() {
unique_mutex_lock pool_lock(pool_mutex_);
if (not shutdown_) {
shutdown_ = true;
event_system::instance().raise<service_shutdown>("client_pool");
for (auto &kv : pool_lookup_) {
kv.second->shutdown();
event_system::instance().raise<service_shutdown_begin>("client_pool");
unique_mutex_lock pool_lock(pool_mutex_);
if (not shutdown_) {
shutdown_ = true;
for (auto &kv : pool_lookup_) {
kv.second->shutdown();
}
pool_lookup_.clear();
}
pool_lookup_.clear();
pool_lock.unlock();
event_system::instance().raise<service_shutdown_end>("client_pool");
}
pool_lock.unlock();
}
} // namespace repertory

143
src/comm/packet/packet.cpp
View File

@@ -1,27 +1,32 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "comm/packet/packet.hpp"
#include "events/events.hpp"
#include "events/event_system.hpp"
#include "events/events.hpp"
#include "types/remote.hpp"
#include "types/repertory.hpp"
#include "utils/encryption.hpp"
#include "utils/error_utils.hpp"
#include "utils/utils.hpp"
namespace repertory {
@@ -30,46 +35,47 @@ void packet::clear() {
decode_offset_ = 0u;
}
packet::error_type packet::decode(std::string &data) {
auto packet::decode(std::string &data) -> packet::error_type {
const auto *str = &buffer_[decode_offset_];
const auto length = strnlen(str, buffer_.size() - decode_offset_);
data = std::string(str, length);
decode_offset_ += (length + 1);
return utils::translate_api_error(api_error::success);
return utils::from_api_error(api_error::success);
}
packet::error_type packet::decode(std::wstring &data) {
auto packet::decode(std::wstring &data) -> packet::error_type {
std::string utf8_string;
const auto ret = decode(utf8_string);
if (ret == 0) {
data = utils::string::from_utf8(utf8_string);
}
return utils::translate_api_error(api_error::success);
return utils::from_api_error(api_error::success);
}
packet::error_type packet::decode(void *&ptr) {
auto packet::decode(void *&ptr) -> packet::error_type {
return decode(reinterpret_cast<std::uint64_t &>(ptr));
}
packet::error_type packet::decode(void *buffer, const size_t &size) {
auto packet::decode(void *buffer, std::size_t size) -> packet::error_type {
if (size) {
const auto read_size = utils::calculate_read_size(buffer_.size(), size, decode_offset_);
const auto read_size =
utils::calculate_read_size(buffer_.size(), size, decode_offset_);
if (read_size == size) {
memcpy(buffer, &buffer_[decode_offset_], size);
decode_offset_ += size;
return utils::translate_api_error(api_error::success);
return utils::from_api_error(api_error::success);
}
return ((decode_offset_ + size) > buffer_.size())
? utils::translate_api_error(api_error::buffer_overflow)
: utils::translate_api_error(api_error::buffer_too_small);
? utils::from_api_error(api_error::buffer_overflow)
: utils::from_api_error(api_error::buffer_too_small);
}
return utils::translate_api_error(api_error::success);
return utils::from_api_error(api_error::success);
}
packet::error_type packet::decode(std::int8_t &i) {
auto packet::decode(std::int8_t &i) -> packet::error_type {
const auto ret = decode(&i, sizeof(i));
if (ret == 0) {
boost::endian::big_to_native_inplace(i);
@@ -77,7 +83,7 @@ packet::error_type packet::decode(std::int8_t &i) {
return ret;
}
packet::error_type packet::decode(std::uint8_t &i) {
auto packet::decode(std::uint8_t &i) -> packet::error_type {
const auto ret = decode(&i, sizeof(i));
if (ret == 0) {
boost::endian::big_to_native_inplace(i);
@@ -85,7 +91,7 @@ packet::error_type packet::decode(std::uint8_t &i) {
return ret;
}
packet::error_type packet::decode(std::int16_t &i) {
auto packet::decode(std::int16_t &i) -> packet::error_type {
const auto ret = decode(&i, sizeof(i));
if (ret == 0) {
boost::endian::big_to_native_inplace(i);
@@ -93,7 +99,7 @@ packet::error_type packet::decode(std::int16_t &i) {
return ret;
}
packet::error_type packet::decode(std::uint16_t &i) {
auto packet::decode(std::uint16_t &i) -> packet::error_type {
const auto ret = decode(&i, sizeof(i));
if (ret == 0) {
boost::endian::big_to_native_inplace(i);
@@ -101,7 +107,7 @@ packet::error_type packet::decode(std::uint16_t &i) {
return ret;
}
packet::error_type packet::decode(std::int32_t &i) {
auto packet::decode(std::int32_t &i) -> packet::error_type {
const auto ret = decode(&i, sizeof(i));
if (ret == 0) {
boost::endian::big_to_native_inplace(i);
@@ -109,7 +115,7 @@ packet::error_type packet::decode(std::int32_t &i) {
return ret;
}
packet::error_type packet::decode(std::uint32_t &i) {
auto packet::decode(std::uint32_t &i) -> packet::error_type {
const auto ret = decode(&i, sizeof(i));
if (ret == 0) {
boost::endian::big_to_native_inplace(i);
@@ -117,7 +123,7 @@ packet::error_type packet::decode(std::uint32_t &i) {
return ret;
}
packet::error_type packet::decode(std::int64_t &i) {
auto packet::decode(std::int64_t &i) -> packet::error_type {
const auto ret = decode(&i, sizeof(i));
if (ret == 0) {
boost::endian::big_to_native_inplace(i);
@@ -125,7 +131,7 @@ packet::error_type packet::decode(std::int64_t &i) {
return ret;
}
packet::error_type packet::decode(std::uint64_t &i) {
auto packet::decode(std::uint64_t &i) -> packet::error_type {
const auto ret = decode(&i, sizeof(i));
if (ret == 0) {
boost::endian::big_to_native_inplace(i);
@@ -133,7 +139,7 @@ packet::error_type packet::decode(std::uint64_t &i) {
return ret;
}
packet::error_type packet::decode(remote::setattr_x &i) {
auto packet::decode(remote::setattr_x &i) -> packet::error_type {
const auto ret = decode(&i, sizeof(i));
if (ret == 0) {
boost::endian::big_to_native_inplace(i.acctime);
@@ -152,7 +158,7 @@ packet::error_type packet::decode(remote::setattr_x &i) {
return ret;
}
packet::error_type packet::decode(remote::stat &i) {
auto packet::decode(remote::stat &i) -> packet::error_type {
const auto ret = decode(&i, sizeof(i));
if (ret == 0) {
boost::endian::big_to_native_inplace(i.st_mode);
@@ -172,7 +178,7 @@ packet::error_type packet::decode(remote::stat &i) {
return ret;
}
packet::error_type packet::decode(remote::statfs &i) {
auto packet::decode(remote::statfs &i) -> packet::error_type {
const auto ret = decode(&i, sizeof(i));
if (ret == 0) {
boost::endian::big_to_native_inplace(i.f_bavail);
@@ -185,7 +191,7 @@ packet::error_type packet::decode(remote::statfs &i) {
return ret;
}
packet::error_type packet::decode(remote::statfs_x &i) {
auto packet::decode(remote::statfs_x &i) -> packet::error_type {
auto ret = decode(*dynamic_cast<remote::statfs *>(&i));
if (ret == 0) {
ret = decode(&i.f_mntfromname[0], 1024);
@@ -193,7 +199,7 @@ packet::error_type packet::decode(remote::statfs_x &i) {
return ret;
}
packet::error_type packet::decode(remote::file_info &i) {
auto packet::decode(remote::file_info &i) -> packet::error_type {
const auto ret = decode(&i, sizeof(i));
if (ret == 0) {
boost::endian::big_to_native_inplace(i.AllocationSize);
@@ -212,15 +218,14 @@ packet::error_type packet::decode(remote::file_info &i) {
return ret;
}
int packet::decode_json(packet &response, json &json_data) {
auto packet::decode_json(packet &response, json &json_data) -> int {
int ret = 0;
std::string data;
if ((ret = response.decode(data)) == 0) {
try {
json_data = json::parse(data);
} catch (const std::exception &e) {
event_system::instance().raise<repertory_exception>(
__FUNCTION__, e.what() ? e.what() : "Failed to parse JSON string");
utils::error::raise_error(__FUNCTION__, e, "failed to parse json string");
ret = -EIO;
}
}
@@ -228,25 +233,26 @@ int packet::decode_json(packet &response, json &json_data) {
return ret;
}
packet::error_type packet::decrypt(const std::string &token) {
auto ret = utils::translate_api_error(api_error::success);
auto packet::decrypt(const std::string &token) -> packet::error_type {
auto ret = utils::from_api_error(api_error::success);
try {
std::vector<char> result;
data_buffer result;
if (not utils::encryption::decrypt_data(token, &buffer_[decode_offset_],
buffer_.size() - decode_offset_, result)) {
throw std::runtime_error("Decryption failed");
buffer_.size() - decode_offset_,
result)) {
throw std::runtime_error("decryption failed");
}
buffer_ = std::move(result);
decode_offset_ = 0;
} catch (const std::exception &e) {
event_system::instance().raise<repertory_exception>(__FUNCTION__, e.what());
ret = utils::translate_api_error(api_error::error);
utils::error::raise_error(__FUNCTION__, e, "exception occurred");
ret = utils::from_api_error(api_error::error);
}
return ret;
}
void packet::encode(const void *buffer, const std::size_t &size, bool should_reserve) {
void packet::encode(const void *buffer, std::size_t size, bool should_reserve) {
if (size) {
if (should_reserve) {
buffer_.reserve(buffer_.size() + size);
@@ -263,11 +269,17 @@ void packet::encode(const std::string &str) {
buffer_.emplace_back(0);
}
void packet::encode(wchar_t *str) { encode(utils::string::to_utf8(str ? str : L"")); }
void packet::encode(wchar_t *str) {
encode(utils::string::to_utf8(str ? str : L""));
}
void packet::encode(const wchar_t *str) { encode(utils::string::to_utf8(str ? str : L"")); }
void packet::encode(const wchar_t *str) {
encode(utils::string::to_utf8(str ? str : L""));
}
void packet::encode(const std::wstring &str) { encode(utils::string::to_utf8(str)); }
void packet::encode(const std::wstring &str) {
encode(utils::string::to_utf8(str));
}
void packet::encode(std::int8_t i) {
boost::endian::native_to_big_inplace(i);
@@ -375,7 +387,8 @@ void packet::encode(remote::file_info i) {
encode(&i, sizeof(i), true);
}
void packet::encode_top(const void *buffer, const std::size_t &size, bool should_reserve) {
void packet::encode_top(const void *buffer, std::size_t size,
bool should_reserve) {
if (size) {
if (should_reserve) {
buffer_.reserve(buffer_.size() + size);
@@ -386,13 +399,15 @@ void packet::encode_top(const void *buffer, const std::size_t &size, bool should
}
void packet::encode_top(const std::string &str) {
const auto len = strnlen(&str[0], str.size());
buffer_.reserve(len + 1 + buffer_.size());
const auto len = strnlen(str.c_str(), str.size());
buffer_.reserve(len + 1U + buffer_.size());
encode_top(&str[0], len, false);
buffer_.insert(buffer_.begin() + len, 0);
buffer_.insert(buffer_.begin() + static_cast<std::int32_t>(len), 0);
}
void packet::encode_top(const std::wstring &str) { encode_top(utils::string::to_utf8(str)); }
void packet::encode_top(const std::wstring &str) {
encode_top(utils::string::to_utf8(str));
}
void packet::encode_top(std::int8_t i) {
boost::endian::native_to_big_inplace(i);
@@ -502,22 +517,22 @@ void packet::encode_top(remote::file_info i) {
void packet::encrypt(const std::string &token) {
try {
std::vector<char> result;
data_buffer result;
utils::encryption::encrypt_data(token, buffer_, result);
buffer_ = std::move(result);
encode_top(static_cast<std::uint32_t>(buffer_.size()));
} catch (const std::exception &e) {
event_system::instance().raise<repertory_exception>(__FUNCTION__, e.what());
utils::error::raise_error(__FUNCTION__, e, "exception occurred");
}
}
void packet::transfer_into(std::vector<char> &buffer) {
void packet::transfer_into(data_buffer &buffer) {
buffer = std::move(buffer_);
buffer_ = std::vector<char>();
buffer_ = data_buffer();
decode_offset_ = 0;
}
packet &packet::operator=(const std::vector<char> &buffer) noexcept {
auto packet::operator=(const data_buffer &buffer) noexcept -> packet & {
if (&buffer_ != &buffer) {
buffer_ = buffer;
decode_offset_ = 0;
@@ -526,7 +541,7 @@ packet &packet::operator=(const std::vector<char> &buffer) noexcept {
return *this;
}
packet &packet::operator=(std::vector<char> &&buffer) noexcept {
auto packet::operator=(data_buffer &&buffer) noexcept -> packet & {
if (&buffer_ != &buffer) {
buffer_ = std::move(buffer);
decode_offset_ = 0;
@@ -535,7 +550,7 @@ packet &packet::operator=(std::vector<char> &&buffer) noexcept {
return *this;
}
packet &packet::operator=(const packet &p) noexcept {
auto packet::operator=(const packet &p) noexcept -> packet & {
if (this != &p) {
buffer_ = p.buffer_;
decode_offset_ = p.decode_offset_;
@@ -544,7 +559,7 @@ packet &packet::operator=(const packet &p) noexcept {
return *this;
}
packet &packet::operator=(packet &&p) noexcept {
auto packet::operator=(packet &&p) noexcept -> packet & {
if (this != &p) {
buffer_ = std::move(p.buffer_);
decode_offset_ = p.decode_offset_;

126
src/comm/packet/packet_client.cpp
View File

@@ -1,37 +1,44 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "comm/packet/packet_client.hpp"
#include "events/events.hpp"
#include "types/repertory.hpp"
#include "utils/error_utils.hpp"
#include "utils/timeout.hpp"
namespace repertory {
// clang-format off
E_SIMPLE2(packet_client_timeout, error, true,
std::string, event_name, en, E_STRING,
E_SIMPLE2(packet_client_timeout, error, true,
std::string, event_name, en, E_STRING,
std::string, message, msg, E_STRING
);
// clang-format on
packet_client::packet_client(std::string host_name_or_ip, const std::uint8_t &max_connections,
const std::uint16_t &port, const std::uint16_t &receive_timeout,
const std::uint16_t &send_timeout, std::string encryption_token)
packet_client::packet_client(std::string host_name_or_ip,
std::uint8_t max_connections, std::uint16_t port,
std::uint16_t receive_timeout,
std::uint16_t send_timeout,
std::string encryption_token)
: io_context_(),
host_name_or_ip_(std::move(host_name_or_ip)),
max_connections_(max_connections ? max_connections : 20u),
@@ -47,10 +54,10 @@ packet_client::~packet_client() {
io_context_.stop();
}
void packet_client::close(client &client) const {
void packet_client::close(client &cli) const {
try {
boost::system::error_code ec;
client.socket.close(ec);
cli.socket.close(ec);
} catch (...) {
}
}
@@ -65,8 +72,7 @@ void packet_client::close_all() {
unique_id_ = utils::create_uuid_string();
}
bool packet_client::connect(client &c) {
auto ret = false;
void packet_client::connect(client &c) {
try {
resolve();
boost::asio::connect(c.socket, resolve_results_);
@@ -74,16 +80,16 @@ bool packet_client::connect(client &c) {
c.socket.set_option(boost::asio::socket_base::linger(false, 0));
packet response;
read_packet(c, response);
ret = true;
const auto res = read_packet(c, response);
if (res != 0) {
throw std::runtime_error(std::to_string(res));
}
} catch (const std::exception &e) {
event_system::instance().raise<repertory_exception>(__FUNCTION__, e.what());
utils::error::raise_error(__FUNCTION__, e, "connection handshake failed");
}
return ret;
}
std::shared_ptr<packet_client::client> packet_client::get_client() {
auto packet_client::get_client() -> std::shared_ptr<packet_client::client> {
std::shared_ptr<client> ret;
unique_mutex_lock clients_lock(clients_mutex_);
@@ -109,22 +115,25 @@ void packet_client::put_client(std::shared_ptr<client> &c) {
}
}
packet::error_type packet_client::read_packet(client &c, packet &response) {
std::vector<char> buffer(sizeof(std::uint32_t));
auto packet_client::read_packet(client &c, packet &response)
-> packet::error_type {
data_buffer buffer(sizeof(std::uint32_t));
const auto read_buffer = [&]() {
std::uint32_t offset = 0u;
while (offset < buffer.size()) {
const auto bytes_read =
boost::asio::read(c.socket, boost::asio::buffer(&buffer[offset], buffer.size() - offset));
const auto bytes_read = boost::asio::read(
c.socket,
boost::asio::buffer(&buffer[offset], buffer.size() - offset));
if (bytes_read <= 0) {
throw std::runtime_error("Read failed: " + std::to_string(bytes_read));
throw std::runtime_error("read failed|" + std::to_string(bytes_read));
}
offset += static_cast<std::uint32_t>(bytes_read);
}
};
read_buffer();
const auto size = boost::endian::big_to_native(*reinterpret_cast<std::uint32_t *>(&buffer[0u]));
const auto size = boost::endian::big_to_native(
*reinterpret_cast<std::uint32_t *>(&buffer[0u]));
buffer.resize(size);
read_buffer();
@@ -140,34 +149,37 @@ packet::error_type packet_client::read_packet(client &c, packet &response) {
void packet_client::resolve() {
if (resolve_results_.empty()) {
resolve_results_ =
tcp::resolver(io_context_).resolve({host_name_or_ip_, std::to_string(port_)});
resolve_results_ = tcp::resolver(io_context_)
.resolve({host_name_or_ip_, std::to_string(port_)});
}
}
packet::error_type packet_client::send(const std::string &method, std::uint32_t &service_flags) {
auto packet_client::send(const std::string &method,
std::uint32_t &service_flags) -> packet::error_type {
packet request;
return send(method, request, service_flags);
}
packet::error_type packet_client::send(const std::string &method, packet &request,
std::uint32_t &service_flags) {
auto packet_client::send(const std::string &method, packet &request,
std::uint32_t &service_flags) -> packet::error_type {
packet response;
return send(method, request, response, service_flags);
}
packet::error_type packet_client::send(const std::string &method, packet &request, packet &response,
std::uint32_t &service_flags) {
auto packet_client::send(const std::string &method, packet &request,
packet &response, std::uint32_t &service_flags)
-> packet::error_type {
auto success = false;
packet::error_type ret = utils::translate_api_error(api_error::error);
packet::error_type ret = utils::from_api_error(api_error::error);
request.encode_top(method);
request.encode_top(utils::get_thread_id());
request.encode_top(unique_id_);
request.encode_top(PACKET_SERVICE_FLAGS);
request.encode_top(get_repertory_version());
static const auto max_attempts = 2;
for (auto i = 1; allow_connections_ && not success && (i <= max_attempts); i++) {
static const std::uint8_t max_attempts = 5u;
for (std::uint8_t i = 1u;
allow_connections_ && not success && (i <= max_attempts); i++) {
auto c = get_client();
if (c) {
try {
@@ -176,7 +188,8 @@ packet::error_type packet_client::send(const std::string &method, packet &reques
timeout request_timeout(
[this, method, c]() {
event_system::instance().raise<packet_client_timeout>("Request", method);
event_system::instance().raise<packet_client_timeout>("request",
method);
close(*c.get());
},
std::chrono::seconds(send_timeout_));
@@ -184,9 +197,11 @@ packet::error_type packet_client::send(const std::string &method, packet &reques
std::uint32_t offset = 0u;
while (offset < request.get_size()) {
const auto bytes_written = boost::asio::write(
c->socket, boost::asio::buffer(&request[offset], request.get_size() - offset));
c->socket, boost::asio::buffer(&request[offset],
request.get_size() - offset));
if (bytes_written <= 0) {
throw std::runtime_error("Write failed: " + std::to_string(bytes_written));
throw std::runtime_error("write failed|" +
std::to_string(bytes_written));
}
offset += static_cast<std::uint32_t>(bytes_written);
}
@@ -194,7 +209,8 @@ packet::error_type packet_client::send(const std::string &method, packet &reques
timeout response_timeout(
[this, method, c]() {
event_system::instance().raise<packet_client_timeout>("Response", method);
event_system::instance().raise<packet_client_timeout>("response",
method);
close(*c.get());
},
std::chrono::seconds(receive_timeout_));
@@ -202,13 +218,17 @@ packet::error_type packet_client::send(const std::string &method, packet &reques
ret = read_packet(*c, response);
response_timeout.disable();
if (ret == 0) {
response.decode(service_flags);
response.decode(ret);
success = true;
put_client(c);
if ((ret = response.decode(service_flags)) == 0) {
packet::error_type res{};
if ((ret = response.decode(res)) == 0) {
ret = res;
success = true;
put_client(c);
}
}
}
} catch (const std::exception &e) {
event_system::instance().raise<repertory_exception>(__FUNCTION__, e.what());
utils::error::raise_error(__FUNCTION__, e, "send failed");
close_all();
if (allow_connections_ && (i < max_attempts)) {
std::this_thread::sleep_for(1s);
@@ -217,7 +237,7 @@ packet::error_type packet_client::send(const std::string &method, packet &reques
}
if (not allow_connections_) {
ret = utils::translate_api_error(api_error::error);
ret = utils::from_api_error(api_error::error);
success = true;
}
}

139
src/comm/packet/packet_server.cpp
View File

@@ -1,37 +1,49 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "comm/packet/packet_server.hpp"
#include "events/events.hpp"
#include "events/event_system.hpp"
#include "comm/packet/packet.hpp"
#include "events/event_system.hpp"
#include "events/events.hpp"
#include "types/repertory.hpp"
#include "utils/error_utils.hpp"
#include "utils/string_utils.hpp"
#include "utils/utils.hpp"
namespace repertory {
packet_server::packet_server(const std::uint16_t &port, std::string token, std::uint8_t pool_size,
closed_callback closed, message_handler_callback message_handler)
: encryption_token_(std::move(token)), closed_(closed), message_handler_(message_handler) {
using std::thread;
packet_server::packet_server(std::uint16_t port, std::string token,
std::uint8_t pool_size, closed_callback closed,
message_handler_callback message_handler)
: encryption_token_(std::move(token)),
closed_(std::move(closed)),
message_handler_(std::move(message_handler)) {
initialize(port, pool_size);
event_system::instance().raise<service_started>("packet_server");
}
packet_server::~packet_server() {
event_system::instance().raise<service_shutdown>("packet_server");
event_system::instance().raise<service_shutdown_begin>("packet_server");
std::thread([this]() {
for (std::size_t i = 0u; i < service_threads_.size(); i++) {
io_context_.stop();
@@ -40,6 +52,7 @@ packet_server::~packet_server() {
server_thread_->join();
server_thread_.reset();
event_system::instance().raise<service_shutdown_end>("packet_server");
}
void packet_server::add_client(connection &c, const std::string &client_id) {
@@ -64,12 +77,13 @@ void packet_server::initialize(const uint16_t &port, uint8_t pool_size) {
acceptor.bind(endpoint);
acceptor.listen();
} catch (const std::exception &e) {
event_system::instance().raise<repertory_exception>(__FUNCTION__, e.what());
repertory::utils::error::raise_error(__FUNCTION__, e,
"exception occurred");
}
listen_for_connection(acceptor);
for (std::uint8_t i = 0u; i < pool_size; i++) {
service_threads_.emplace_back(std::thread([this]() { io_context_.run(); }));
service_threads_.emplace_back([this]() { io_context_.run(); });
}
for (auto &th : service_threads_) {
@@ -80,14 +94,16 @@ void packet_server::initialize(const uint16_t &port, uint8_t pool_size) {
void packet_server::listen_for_connection(tcp::acceptor &acceptor) {
auto c = std::make_shared<packet_server::connection>(io_context_, acceptor);
acceptor.async_accept(
c->socket, boost::bind(&packet_server::on_accept, this, c, boost::asio::placeholders::error));
acceptor.async_accept(c->socket,
boost::bind(&packet_server::on_accept, this, c,
boost::asio::placeholders::error));
}
void packet_server::on_accept(std::shared_ptr<connection> c, boost::system::error_code ec) {
void packet_server::on_accept(std::shared_ptr<connection> c,
boost::system::error_code ec) {
listen_for_connection(c->acceptor);
if (ec) {
event_system::instance().raise<repertory_exception>(__FUNCTION__, ec.message());
utils::error::raise_error(__FUNCTION__, ec.message());
std::this_thread::sleep_for(1s);
} else {
c->socket.set_option(boost::asio::ip::tcp::no_delay(true));
@@ -101,30 +117,34 @@ void packet_server::on_accept(std::shared_ptr<connection> c, boost::system::erro
}
void packet_server::read_header(std::shared_ptr<connection> c) {
static const std::string function_name = __FUNCTION__;
c->buffer.resize(sizeof(std::uint32_t));
boost::asio::async_read(c->socket, boost::asio::buffer(&c->buffer[0u], c->buffer.size()),
[this, c](boost::system::error_code ec, std::size_t) {
if (ec) {
remove_client(*c);
event_system::instance().raise<repertory_exception>(__FUNCTION__,
ec.message());
} else {
auto to_read = *reinterpret_cast<std::uint32_t *>(&c->buffer[0u]);
boost::endian::big_to_native_inplace(to_read);
read_packet(c, to_read);
}
});
boost::asio::async_read(
c->socket, boost::asio::buffer(&c->buffer[0u], c->buffer.size()),
[this, c](boost::system::error_code ec, std::size_t) {
if (ec) {
remove_client(*c);
repertory::utils::error::raise_error(function_name, ec.message());
} else {
auto to_read = *reinterpret_cast<std::uint32_t *>(&c->buffer[0u]);
boost::endian::big_to_native_inplace(to_read);
read_packet(c, to_read);
}
});
}
void packet_server::read_packet(std::shared_ptr<connection> c, const std::uint32_t &data_size) {
void packet_server::read_packet(std::shared_ptr<connection> c,
std::uint32_t data_size) {
try {
const auto read_buffer = [&]() {
std::uint32_t offset = 0u;
while (offset < c->buffer.size()) {
const auto bytes_read = boost::asio::read(
c->socket, boost::asio::buffer(&c->buffer[offset], c->buffer.size() - offset));
c->socket,
boost::asio::buffer(&c->buffer[offset], c->buffer.size() - offset));
if (bytes_read <= 0) {
throw std::runtime_error("read failed: " + std::to_string(bytes_read));
throw std::runtime_error("read failed|" + std::to_string(bytes_read));
}
offset += static_cast<std::uint32_t>(bytes_read);
}
@@ -147,7 +167,8 @@ void packet_server::read_packet(std::shared_ptr<connection> c, const std::uint32
std::string version;
if ((ret = request->decode(version)) == 0) {
if (utils::compare_version_strings(version, MIN_REMOTE_VERSION) >= 0) {
if (utils::compare_version_strings(
version, REPERTORY_MIN_REMOTE_VERSION) >= 0) {
std::uint32_t service_flags = 0u;
DECODE_OR_IGNORE(request, service_flags);
@@ -166,17 +187,18 @@ void packet_server::read_packet(std::shared_ptr<connection> c, const std::uint32
}
should_send_response = false;
message_handler_(service_flags, client_id, thread_id, method, request.get(),
*response,
[this, c, request, response](const packet::error_type &result) {
message_handler_(service_flags, client_id, thread_id, method,
request.get(), *response,
[this, c, request,
response](const packet::error_type &result) {
this->send_response(c, result, *response);
});
}
} else {
ret = utils::translate_api_error(api_error::incompatible_version);
ret = utils::from_api_error(api_error::incompatible_version);
}
} else {
ret = utils::translate_api_error(api_error::invalid_version);
ret = utils::from_api_error(api_error::invalid_version);
}
} else {
throw std::runtime_error("invalid nonce");
@@ -189,7 +211,7 @@ void packet_server::read_packet(std::shared_ptr<connection> c, const std::uint32
}
} catch (const std::exception &e) {
remove_client(*c);
event_system::instance().raise<repertory_exception>(__FUNCTION__, e.what());
utils::error::raise_error(__FUNCTION__, e, "exception occurred");
}
}
@@ -203,23 +225,26 @@ void packet_server::remove_client(connection &c) {
}
}
void packet_server::send_response(std::shared_ptr<connection> c, const packet::error_type &result,
void packet_server::send_response(std::shared_ptr<connection> c,
const packet::error_type &result,
packet &response) {
static const std::string function_name = __FUNCTION__;
response.encode_top(result);
response.encode_top(PACKET_SERVICE_FLAGS);
response.encode_top(c->nonce);
response.encrypt(encryption_token_);
response.transfer_into(c->buffer);
boost::asio::async_write(c->socket, boost::asio::buffer(c->buffer),
[this, c](boost::system::error_code ec, std::size_t /*length*/) {
if (ec) {
remove_client(*c);
event_system::instance().raise<repertory_exception>(__FUNCTION__,
ec.message());
} else {
read_header(c);
}
});
boost::asio::async_write(
c->socket, boost::asio::buffer(c->buffer),
[this, c](boost::system::error_code ec, std::size_t /*length*/) {
if (ec) {
remove_client(*c);
utils::error::raise_error(function_name, ec.message());
} else {
read_header(c);
}
});
}
} // namespace repertory

575
src/comm/s3/s3_comm.cpp Normal file
View File

@@ -0,0 +1,575 @@
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#if defined(REPERTORY_ENABLE_S3)
#include "comm/s3/s3_comm.hpp"
#include "app_config.hpp"
#include "comm/curl/curl_comm.hpp"
#include "comm/s3/s3_requests.hpp"
#include "events/event_system.hpp"
#include "events/events.hpp"
#include "providers/i_provider.hpp"
#include "types/repertory.hpp"
#include "types/s3.hpp"
#include "utils/encryption.hpp"
#include "utils/error_utils.hpp"
#include "utils/path_utils.hpp"
#include "utils/polling.hpp"
#include "utils/string_utils.hpp"
namespace repertory {
static const get_key_callback empty_key = []() { return ""; };
s3_comm::s3_comm(const app_config &config)
: config_(config), s3_config_(config.get_s3_config()) {
s3_config_.bucket = utils::string::trim(s3_config_.bucket);
// TODO make configurable
const auto enable_path_style =
utils::string::begins_with(s3_config_.url, "http://localhost") ||
utils::string::begins_with(s3_config_.url, "https://localhost") ||
utils::string::begins_with(s3_config_.url, "http://127.0.0.1") ||
utils::string::begins_with(s3_config_.url, "https://127.0.0.1");
s3_client_ = std::make_unique<curl_comm>(s3_config_);
s3_client_->enable_s3_path_style(enable_path_style);
polling::instance().set_callback(
{"s3_directory_cache", polling::frequency::high,
[this]() { this->clear_expired_directories(); }});
}
s3_comm::s3_comm(s3_comm &&comm)
: config_(std::move(comm.config_)),
s3_config_(std::move(comm.s3_config_)),
s3_client_(std::move(comm.s3_client_)) {
comm.active_ = false;
polling::instance().set_callback(
{"s3_directory_cache", polling::frequency::high,
[this]() { this->clear_expired_directories(); }});
}
s3_comm::~s3_comm() {
if (active_) {
polling::instance().remove_callback("s3_directory_cache");
}
}
void s3_comm::clear_expired_directories() {
recur_mutex_lock l(cached_directories_mutex_);
std::vector<std::string> expired_list;
for (const auto &kv : cached_directories_) {
if (kv.second.expiration <= std::chrono::system_clock::now()) {
expired_list.emplace_back(kv.first);
}
}
for (const auto &expired : expired_list) {
event_system::instance().raise<debug_log>(__FUNCTION__, expired, "expired");
cached_directories_.erase(expired);
}
}
auto s3_comm::create_directory(const std::string &api_path) -> api_error {
raise_begin(__FUNCTION__, api_path);
long response_code{};
auto object_name = get_object_name(api_path, empty_key) + '/';
if (not create_directory_object_request(*s3_client_, s3_config_, object_name,
response_code)) {
return raise_end(__FUNCTION__, api_path, api_error::comm_error,
response_code);
}
return raise_end(__FUNCTION__, api_path,
response_code == 200 ? api_error::success
: api_error::comm_error,
response_code);
}
auto s3_comm::directory_exists(const std::string &api_path) const -> api_error {
raise_begin(__FUNCTION__, api_path);
auto object_name = get_object_name(api_path, empty_key) + "/";
head_object_result result{};
long response_code{};
if (head_object_request(*s3_client_, s3_config_, object_name, result,
response_code)) {
if (response_code == 404) {
return raise_end(__FUNCTION__, api_path, api_error::directory_not_found,
response_code);
}
return raise_end(__FUNCTION__, api_path, api_error::directory_exists,
response_code);
}
return raise_end(__FUNCTION__, api_path, api_error::comm_error,
response_code);
}
auto s3_comm::file_exists(const std::string &api_path,
const get_key_callback &get_key) const -> api_error {
raise_begin(__FUNCTION__, api_path);
if (get_cached_file_exists(api_path)) {
return raise_end(__FUNCTION__, api_path, api_error::item_exists, 200);
}
auto object_name = get_object_name(api_path, get_key);
head_object_result result{};
long response_code{};
if (head_object_request(*s3_client_, s3_config_, object_name, result,
response_code)) {
if (response_code == 404) {
return raise_end(__FUNCTION__, api_path, api_error::item_not_found,
response_code);
}
return raise_end(__FUNCTION__, api_path, api_error::item_exists,
response_code);
}
return raise_end(__FUNCTION__, api_path, api_error::directory_exists,
response_code);
}
auto s3_comm::get_object_list(std::vector<directory_item> &list) const
-> api_error {
raise_begin(__FUNCTION__, "/");
long response_code{};
auto success =
list_objects_request(*s3_client_, s3_config_, list, response_code);
return raise_end(__FUNCTION__, "/",
success ? api_error::success : api_error::comm_error,
response_code);
}
auto s3_comm::get_object_name(const std::string &api_path,
const get_key_callback &get_key) const
-> std::string {
auto object_name = utils::path::create_api_path(api_path).substr(1);
const auto key = get_key();
if (not key.empty()) {
auto parts = utils::string::split(object_name, '/', false);
parts[parts.size() - 1u] = key;
object_name = utils::string::join(parts, '/');
}
return object_name;
}
auto s3_comm::get_cached_directory_item_count(const std::string &api_path,
std::size_t &count) const
-> bool {
recur_mutex_lock l(cached_directories_mutex_);
if (cached_directories_.find(api_path) != cached_directories_.end()) {
count = cached_directories_.at(api_path).items.size();
return true;
}
return false;
}
auto s3_comm::get_cached_directory_items(const std::string &api_path,
meta_provider_callback meta_provider,
directory_item_list &list) const
-> bool {
unique_recur_mutex_lock l(cached_directories_mutex_);
if (cached_directories_.find(api_path) != cached_directories_.end()) {
auto &cachedEntry = cached_directories_.at(api_path);
list = cachedEntry.items;
cached_directories_[api_path].reset_timeout(
std::chrono::seconds(config_.get_s3_config().cache_timeout_secs));
l.unlock();
for (auto &item : list) {
meta_provider(item);
}
return true;
}
return false;
}
auto s3_comm::get_cached_file_exists(const std::string &api_path) const
-> bool {
unique_recur_mutex_lock l(cached_directories_mutex_);
const auto parent_api_path = utils::path::get_parent_api_path(api_path);
if (cached_directories_.find(parent_api_path) != cached_directories_.end()) {
auto &entry = cached_directories_.at(parent_api_path);
if (std::find_if(entry.items.begin(), entry.items.end(),
[&api_path](const auto &item) -> bool {
return not item.directory && (api_path == item.api_path);
}) != entry.items.end()) {
cached_directories_[api_path].reset_timeout(
std::chrono::seconds(config_.get_s3_config().cache_timeout_secs));
return true;
}
}
return false;
}
auto s3_comm::get_directory_item_count(
const std::string &api_path, meta_provider_callback meta_provider) const
-> std::size_t {
raise_begin(__FUNCTION__, api_path);
std::size_t ret = 0u;
if (not get_cached_directory_item_count(api_path, ret)) {
directory_item_list list;
const auto res = grab_directory_items(api_path, meta_provider, list);
if (res != api_error::success) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, res,
"failed to grab directory items");
}
return list.size();
}
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(__FUNCTION__, api_path,
"end|" + std::to_string(ret));
}
return ret;
}
auto s3_comm::get_directory_items(const std::string &api_path,
meta_provider_callback meta_provider,
directory_item_list &list) const
-> api_error {
raise_begin(__FUNCTION__, api_path);
auto ret = api_error::success;
if (not get_cached_directory_items(api_path, meta_provider, list)) {
ret = grab_directory_items(api_path, meta_provider, list);
}
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(
__FUNCTION__, api_path, "end|" + api_error_to_string(ret));
}
return ret;
}
auto s3_comm::get_directory_list(api_file_list &list) const -> api_error {
raise_begin(__FUNCTION__, "/");
long response_code{};
auto success =
list_directories_request(*s3_client_, s3_config_, list, response_code);
return raise_end(__FUNCTION__, "/",
success ? api_error::success : api_error::comm_error,
response_code);
}
auto s3_comm::get_file(const std::string &api_path,
const get_key_callback &get_key,
const get_name_callback &get_name,
const get_token_callback &get_token,
api_file &file) const -> api_error {
raise_begin(__FUNCTION__, api_path);
auto ret = api_error::success;
auto object_name = get_object_name(api_path, get_key);
head_object_result result{};
long response_code{};
if (head_object_request(*s3_client_, s3_config_, object_name, result,
response_code)) {
const auto key = get_key();
object_name = get_name(key, object_name);
file.accessed_date = utils::get_file_time_now();
file.api_path = utils::path::create_api_path(object_name);
file.api_parent = utils::path::get_parent_api_path(file.api_path);
file.changed_date = utils::aws::format_time(result.last_modified);
file.creation_date = utils::aws::format_time(result.last_modified);
file.encryption_token = get_token();
file.file_size =
file.encryption_token.empty()
? result.content_length
: utils::encryption::encrypting_reader::calculate_decrypted_size(
result.content_length);
file.modified_date = utils::aws::format_time(result.last_modified);
} else {
utils::error::raise_api_path_error(__FUNCTION__, api_path, response_code,
"head object request failed");
ret = api_error::comm_error;
}
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(
__FUNCTION__, api_path, "end|" + api_error_to_string(ret));
}
return ret;
}
auto s3_comm::get_file_list(
const get_api_file_token_callback &get_api_file_token,
const get_name_callback &get_name, api_file_list &list) const -> api_error {
raise_begin(__FUNCTION__, "/");
long response_code{};
auto success = list_files_request(*s3_client_, s3_config_, get_api_file_token,
get_name, list, response_code);
return raise_end(__FUNCTION__, "/",
success ? api_error::success : api_error::comm_error,
response_code);
}
auto s3_comm::grab_directory_items(const std::string &api_path,
meta_provider_callback meta_provider,
directory_item_list &list) const
-> api_error {
auto object_name = get_object_name(api_path, empty_key);
long response_code{};
if (list_objects_in_directory_request(*s3_client_, s3_config_, object_name,
meta_provider, list, response_code)) {
if (response_code == 404) {
return api_error::directory_not_found;
}
if (response_code != 200) {
return api_error::comm_error;
}
set_cached_directory_items(api_path, list);
return api_error::success;
}
return api_error::comm_error;
}
void s3_comm::raise_begin(const std::string &function_name,
const std::string &api_path) const {
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(function_name, api_path,
"begin|");
}
}
auto s3_comm::raise_end(const std::string &function_name,
const std::string &api_path, const api_error &error,
long code) const -> api_error {
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(
function_name, api_path,
"end|" + api_error_to_string(error) + '|' + std::to_string(code));
}
return error;
}
auto s3_comm::read_file_bytes(const std::string &api_path, std::size_t size,
std::uint64_t offset, data_buffer &data,
const get_key_callback &get_key,
const get_size_callback &get_size,
const get_token_callback &get_token,
stop_type &stop_requested) const -> api_error {
data.clear();
auto object_name = get_object_name(api_path, get_key);
const auto encryption_token = get_token();
const auto data_size = get_size();
if (encryption_token.empty()) {
long response_code{};
if (not read_object_request(*s3_client_, s3_config_, object_name, size,
offset, data, response_code, stop_requested)) {
auto res =
stop_requested ? api_error::download_stopped : api_error::comm_error;
utils::error::raise_api_path_error(__FUNCTION__, api_path, res,
"failed to read file bytes");
return res;
}
if (response_code < 200 || response_code >= 300) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, response_code,
"failed to read file bytes");
return api_error::comm_error;
}
return api_error::success;
}
const auto key = utils::encryption::generate_key(encryption_token);
return utils::encryption::read_encrypted_range(
{offset, offset + size - 1}, key,
[&](data_buffer &ct, std::uint64_t start_offset,
std::uint64_t end_offset) -> api_error {
return read_file_bytes(
api_path, (end_offset - start_offset + 1u), start_offset, ct,
get_key, get_size, []() -> std::string { return ""; },
stop_requested);
},
data_size, data);
}
void s3_comm::remove_cached_directory(const std::string &api_path) {
recur_mutex_lock l(cached_directories_mutex_);
cached_directories_.erase(api_path);
}
auto s3_comm::remove_directory(const std::string &api_path) -> api_error {
raise_begin(__FUNCTION__, api_path);
auto object_name = get_object_name(api_path, empty_key) + "/";
long response_code{};
if (delete_object_request(*s3_client_, s3_config_, object_name,
response_code)) {
if (response_code == 404) {
return raise_end(__FUNCTION__, api_path, api_error::directory_not_found,
response_code);
}
if (response_code != 204) {
return raise_end(__FUNCTION__, api_path, api_error::comm_error,
response_code);
}
remove_cached_directory(utils::path::get_parent_api_path(api_path));
remove_cached_directory(api_path);
return raise_end(__FUNCTION__, api_path, api_error::success, response_code);
}
return raise_end(__FUNCTION__, api_path, api_error::comm_error,
response_code);
}
auto s3_comm::remove_file(const std::string &api_path,
const get_key_callback &get_key) -> api_error {
raise_begin(__FUNCTION__, api_path);
auto object_name = get_object_name(api_path, get_key);
long response_code{};
if (delete_object_request(*s3_client_, s3_config_, object_name,
response_code)) {
if (response_code == 404) {
return raise_end(__FUNCTION__, api_path, api_error::item_not_found,
response_code);
}
if (response_code != 204) {
return raise_end(__FUNCTION__, api_path, api_error::comm_error,
response_code);
}
remove_cached_directory(utils::path::get_parent_api_path(api_path));
return raise_end(__FUNCTION__, api_path, api_error::success, response_code);
}
return raise_end(__FUNCTION__, api_path, api_error::comm_error,
response_code);
}
auto s3_comm::rename_file(const std::string & /*api_path*/,
const std::string & /*new_api_path*/) -> api_error {
return api_error::not_implemented;
/* if (config_.get_event_level() >= event_level::debug) { */
/* event_system::instance().raise<debug_log>(__FUNCTION__, api_path,
* "begin"); */
/* } */
/* auto ret = api_error::success; */
/* */
/* std::string bucket_name, object_name; */
/* get_object_name(api_path, bucket_name, object_name); */
/* */
/* std::string new_object_name; */
/* get_object_name(new_api_path, bucket_name, new_object_name); */
/* */
/* Aws::S3::Model::CopyObjectRequest request{}; */
/* request.SetBucket(bucket_name); */
/* request.SetCopySource(bucket_name + '/' + object_name); */
/* request.SetKey(new_object_name); */
/* */
/* const auto outcome = s3_client_->CopyObject(request); */
/* if (outcome.IsSuccess()) { */
/* ret = remove_file(api_path); */
/* } else { */
/* const auto &error = outcome.GetError(); */
/* event_system::instance().raise<repertory_exception>(__FUNCTION__,
* error.GetExceptionName()
* +
* "|" + */
/* error.GetMessage());
*/
/* ret = api_error::comm_error; */
/* } */
/* */
/* if (config_.get_event_level() >= event_level::debug) { */
/* event_system::instance().raise<debug_log>(__FUNCTION__, api_path, */
/* "end|" +
* std::to_string(std::uint8_t(ret))); */
/* } */
/* return ret; */
}
void s3_comm::set_cached_directory_items(const std::string &api_path,
directory_item_list list) const {
recur_mutex_lock l(cached_directories_mutex_);
cached_directories_[api_path].items = std::move(list);
cached_directories_[api_path].reset_timeout(
std::chrono::seconds(config_.get_s3_config().cache_timeout_secs));
}
auto s3_comm::upload_file(const std::string &api_path,
const std::string &source_path,
const std::string &encryption_token,
const get_key_callback &get_key,
const set_key_callback &set_key,
stop_type &stop_requested) -> api_error {
raise_begin(__FUNCTION__, api_path);
auto object_name = get_object_name(api_path, get_key);
long response_code{};
if (not put_object_request(*s3_client_, s3_config_, object_name, source_path,
encryption_token, get_key, set_key, response_code,
stop_requested)) {
return raise_end(__FUNCTION__, api_path,
stop_requested ? api_error::upload_stopped
: api_error::upload_failed,
response_code);
}
if (response_code != 200) {
return raise_end(__FUNCTION__, api_path, api_error::comm_error,
response_code);
}
remove_cached_directory(utils::path::get_parent_api_path(api_path));
return raise_end(__FUNCTION__, api_path, api_error::success, response_code);
}
} // namespace repertory
#endif // REPERTORY_ENABLE_S3

399
src/comm/s3/s3_requests_curl.cpp Normal file
View File

@@ -0,0 +1,399 @@
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#if defined(REPERTORY_ENABLE_S3)
#include "comm/s3/s3_requests.hpp"
#include "comm/curl/curl_comm.hpp"
#include "comm/curl/requests/http_get.hpp"
#include "utils/encryption.hpp"
#include "utils/error_utils.hpp"
#include "utils/path_utils.hpp"
namespace repertory {
namespace {
[[nodiscard]] auto
get_object_list(i_http_comm &client, const s3_config &config,
std::string &response_data, long &response_code,
std::optional<std::string> delimiter = std::nullopt,
std::optional<std::string> prefix = std::nullopt) -> bool {
curl::requests::http_get get{};
get.aws_service = "aws:amz:" + config.region + ":s3";
get.path = '/';
get.query["list-type"] = "2";
if (delimiter.has_value() && not delimiter.value().empty()) {
get.query["delimiter"] = delimiter.value();
}
if (prefix.has_value() && not prefix.value().empty()) {
get.query["prefix"] = prefix.value();
}
get.response_handler = [&response_data](const data_buffer &data,
long /*response_code*/) {
response_data = std::string(data.begin(), data.end());
};
stop_type stop_requested{};
return client.make_request(get, response_code, stop_requested);
}
} // namespace
auto create_directory_object_request_impl(i_http_comm &client,
const s3_config &config,
const std::string &object_name,
long &response_code) -> bool {
try {
curl::requests::http_put_file put_file{};
put_file.aws_service = "aws:amz:" + config.region + ":s3";
put_file.file_name =
*(utils::string::split(object_name, '/', false).end() - 1U);
put_file.path = '/' + object_name;
stop_type stop_requested{false};
return client.make_request(put_file, response_code, stop_requested);
} catch (const std::exception &e) {
utils::error::raise_error(__FUNCTION__, e, "exception occurred");
}
return false;
}
auto delete_object_request_impl(i_http_comm &client, const s3_config &config,
const std::string &object_name,
long &response_code) -> bool {
try {
head_object_result result{};
if (not head_object_request_impl(client, config, object_name, result,
response_code)) {
return false;
}
if (response_code == 404) {
return true;
}
curl::requests::http_delete del{};
del.aws_service = "aws:amz:" + config.region + ":s3";
del.path = '/' + object_name;
stop_type stop_requested{false};
return client.make_request(del, response_code, stop_requested);
} catch (const std::exception &e) {
utils::error::raise_error(__FUNCTION__, e, "exception occurred");
}
return false;
}
auto head_object_request_impl(i_http_comm &client, const s3_config &config,
const std::string &object_name,
head_object_result &result, long &response_code)
-> bool {
try {
curl::requests::http_head head{};
head.aws_service = "aws:amz:" + config.region + ":s3";
head.path = '/' + object_name;
head.response_headers = http_headers{};
stop_type stop_requested{false};
if (not client.make_request(head, response_code, stop_requested)) {
return false;
}
if (response_code == 200) {
result.from_headers(head.response_headers.value());
}
return true;
} catch (const std::exception &e) {
utils::error::raise_error(__FUNCTION__, e, "exception occurred");
}
return false;
}
auto list_directories_request_impl(i_http_comm &client, const s3_config &config,
list_directories_result &result,
long &response_code) -> bool {
try {
std::string response_data{};
if (not get_object_list(client, config, response_data, response_code)) {
return false;
}
if (response_code != 200) {
return false;
}
pugi::xml_document doc;
auto res = doc.load_string(response_data.c_str());
if (res.status != pugi::xml_parse_status::status_ok) {
return false;
}
auto node_list = doc.select_nodes("/ListBucketResult/Contents");
for (const auto &node : node_list) {
auto object_name =
node.node().select_node("Key").node().text().as_string();
if (utils::string::ends_with(object_name, "/")) {
api_file directory{};
directory.api_path = utils::path::create_api_path(object_name);
directory.api_parent =
utils::path::get_parent_api_path(directory.api_path);
directory.accessed_date = utils::get_file_time_now();
directory.changed_date = utils::convert_api_date(
node.node().select_node("LastModified").node().text().as_string());
directory.creation_date = directory.changed_date;
directory.modified_date = directory.changed_date;
result.emplace_back(std::move(directory));
}
}
return true;
} catch (const std::exception &e) {
utils::error::raise_error(__FUNCTION__, e, "exception occurred");
}
return false;
}
auto list_files_request_impl(
i_http_comm &client, const s3_config &config,
const get_api_file_token_callback &get_api_file_token,
const get_name_callback &get_name, list_files_result &result,
long &response_code) -> bool {
try {
std::string response_data{};
if (not get_object_list(client, config, response_data, response_code)) {
return false;
}
if (response_code != 200) {
return false;
}
pugi::xml_document doc;
auto res = doc.load_string(response_data.c_str());
if (res.status != pugi::xml_parse_status::status_ok) {
return false;
}
auto node_list = doc.select_nodes("/ListBucketResult/Contents");
for (const auto &node : node_list) {
std::string object_name =
node.node().select_node("Key").node().text().as_string();
if (not utils::string::ends_with(object_name, "/")) {
api_file file{};
object_name = get_name(
*(utils::string::split(object_name, '/', false).end() - 1u),
object_name);
file.api_path = utils::path::create_api_path(object_name);
file.api_parent = utils::path::get_parent_api_path(file.api_path);
file.accessed_date = utils::get_file_time_now();
file.encryption_token = get_api_file_token(file.api_path);
auto size = node.node().select_node("Size").node().text().as_ullong();
file.file_size = file.encryption_token.empty()
? size
: utils::encryption::encrypting_reader::
calculate_decrypted_size(size);
file.changed_date = utils::convert_api_date(
node.node().select_node("LastModified").node().text().as_string());
file.creation_date = file.changed_date;
file.modified_date = file.changed_date;
result.emplace_back(std::move(file));
}
}
return true;
} catch (const std::exception &e) {
utils::error::raise_error(__FUNCTION__, e, "exception occurred");
}
return false;
}
auto list_objects_in_directory_request_impl(
i_http_comm &client, const s3_config &config,
const std::string &object_name, meta_provider_callback meta_provider,
list_objects_result &result, long &response_code) -> bool {
try {
std::string response_data{};
auto prefix = object_name.empty() ? object_name : object_name + "/";
if (not get_object_list(client, config, response_data, response_code, "/",
prefix)) {
return false;
}
if (response_code != 200) {
return false;
}
pugi::xml_document doc;
auto res = doc.load_string(response_data.c_str());
if (res.status != pugi::xml_parse_status::status_ok) {
return false;
}
const auto add_directory_item =
[&](bool directory, const std::string &name,
std::function<std::uint64_t(const directory_item &)> get_size) {
directory_item di{};
di.api_path =
utils::path::create_api_path(utils::path::combine("/", {name}));
di.api_parent = utils::path::get_parent_api_path(di.api_path);
di.directory = directory;
di.size = get_size(di);
meta_provider(di);
result.emplace_back(std::move(di));
};
auto node_list =
doc.select_nodes("/ListBucketResult/CommonPrefixes/Prefix");
for (const auto &node : node_list) {
add_directory_item(
true, node.node().text().as_string(),
[](const directory_item &) -> std::uint64_t { return 0U; });
}
node_list = doc.select_nodes("/ListBucketResult/Contents");
for (const auto &node : node_list) {
auto child_object_name =
node.node().select_node("Key").node().text().as_string();
if (child_object_name != prefix) {
auto size = node.node().select_node("Size").node().text().as_ullong();
add_directory_item(
false, child_object_name,
[&size](const directory_item &) -> std::uint64_t { return size; });
}
}
return true;
} catch (const std::exception &e) {
utils::error::raise_error(__FUNCTION__, e, "exception occurred");
}
return false;
}
auto list_objects_request_impl(i_http_comm &client, const s3_config &config,
list_objects_result &result, long &response_code)
-> bool {
try {
std::string response_data{};
if (not get_object_list(client, config, response_data, response_code)) {
return false;
}
if (response_code != 200) {
return false;
}
pugi::xml_document doc;
auto res = doc.load_string(response_data.c_str());
if (res.status != pugi::xml_parse_status::status_ok) {
return false;
}
auto node_list = doc.select_nodes("/ListBucketResult/Contents");
for (const auto &node : node_list) {
auto object_name =
node.node().select_node("Key").node().text().as_string();
auto size = node.node().select_node("Size").node().text().as_ullong();
directory_item di{};
di.api_path = utils::path::create_api_path(object_name);
di.api_parent = utils::path::get_parent_api_path(di.api_path);
di.directory = utils::string::ends_with(object_name, "/");
di.size = di.directory ? 0U : size;
di.resolved = false;
result.emplace_back(std::move(di));
}
return true;
} catch (const std::exception &e) {
utils::error::raise_error(__FUNCTION__, e, "exception occurred");
}
return false;
}
auto put_object_request_impl(i_http_comm &client, const s3_config &config,
std::string object_name,
const std::string &source_path,
const std::string &encryption_token,
get_key_callback get_key, set_key_callback set_key,
long &response_code, stop_type &stop_requested)
-> bool {
try {
curl::requests::http_put_file put_file{};
put_file.aws_service = "aws:amz:" + config.region + ":s3";
put_file.encryption_token = encryption_token;
put_file.file_name =
*(utils::string::split(object_name, '/', false).end() - 1U);
put_file.path = '/' + object_name;
put_file.source_path = source_path;
if (not encryption_token.empty()) {
static stop_type no_stop{false};
put_file.reader = std::make_shared<utils::encryption::encrypting_reader>(
put_file.file_name, source_path, no_stop, encryption_token,
std::nullopt, -1);
auto key = get_key();
if (key.empty()) {
key = put_file.reader->get_encrypted_file_name();
set_key(key);
}
}
return client.make_request(put_file, response_code, stop_requested);
} catch (const std::exception &e) {
utils::error::raise_error(__FUNCTION__, e, "exception occurred");
}
return false;
}
auto read_object_request_impl(i_http_comm &client, const s3_config &config,
const std::string &object_name, std::size_t size,
std::uint64_t offset, data_buffer &data,
long &response_code, stop_type &stop_requested)
-> bool {
try {
curl::requests::http_get get{};
get.aws_service = "aws:amz:" + config.region + ":s3";
get.headers["response-content-type"] = "binary/octet-stream";
get.path = '/' + object_name;
get.range = {{offset, offset + size - 1}};
get.response_handler = [&data](const data_buffer &response_data,
long /*response_code*/) {
data = response_data;
};
return client.make_request(get, response_code, stop_requested);
} catch (const std::exception &e) {
utils::error::raise_error(__FUNCTION__, e, "exception occurred");
}
return false;
}
} // namespace repertory
#endif