BlockStorage/repertory
1
0
Fork 0
Code Issues 8 Pull Requests Releases 6 Wiki Activity

2.0.0-rc (#9)
Some checks failed
BlockStorage/repertory_osx/pipeline/head This commit looks good
Details
BlockStorage/repertory_windows/pipeline/head This commit looks good
Details
BlockStorage/repertory/pipeline/head There was a failure building this commit
Details
BlockStorage/repertory_linux_builds/pipeline/head This commit looks good
Details
BlockStorage/repertory_osx_builds/pipeline/head There was a failure building this commit
Details

Browse Source 
### Issues

* \#1 \[bug\] Unable to mount S3 due to 'item_not_found' exception
* \#2 Require bucket name for S3 mounts
* \#3 \[bug\] File size is not being updated in S3 mount
* \#4 Upgrade to libfuse-3.x.x
* \#5 Switch to renterd for Sia support
* \#6 Switch to cpp-httplib to further reduce dependencies
* \#7 Remove global_data and calculate used disk space per provider
* \#8 Switch to libcurl for S3 mount support

### Changes from v1.x.x

* Added read-only encrypt provider
  * Pass-through mount point that transparently encrypts source data using `XChaCha20-Poly1305`
* Added S3 encryption support via `XChaCha20-Poly1305`
* Added replay protection to remote mounts
* Added support base64 writes in remote FUSE
* Created static linked Linux binaries for `amd64` and `aarch64` using `musl-libc`
* Removed legacy Sia renter support
* Removed Skynet support
* Fixed multiple remote mount WinFSP API issues on \*NIX servers
* Implemented chunked read and write
  * Writes for non-cached files are performed in chunks of 8Mib
* Removed `repertory-ui` support
* Removed `FreeBSD` support
* Switched to `libsodium` over `CryptoPP`
* Switched to `XChaCha20-Poly1305` for remote mounts
* Updated `GoogleTest` to v1.14.0
* Updated `JSON for Modern C++` to v3.11.2
* Updated `OpenSSL` to v1.1.1w
* Updated `RocksDB` to v8.5.3
* Updated `WinFSP` to 2023
* Updated `boost` to v1.78.0
* Updated `cURL` to v8.3.0
* Updated `zlib` to v1.3
* Use `upload_manager` for all providers
  * Adds a delay to uploads to prevent excessive API calls
  * Supports re-upload after mount restart for incomplete uploads
  * NOTE: Uploads for all providers are full file (no resume support)
    * Multipart upload support is planned for S3

Reviewed-on: #9
This commit is contained in:
Scott E. Graves
2023-10-29 06:55:59 +00:00
parent 3ff46723b8
commit f43c41f88a
839 changed files with 98214 additions and 92959 deletions
Download Patch File Download Diff File Expand all files Collapse all files

124
include/comm/aws_s3/aws_s3_comm.hpp
View File

@ -1,124 +0,0 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#ifndef INCLUDE_COMM_AWS_S3_AWS_S3_COMM_HPP_
#define INCLUDE_COMM_AWS_S3_AWS_S3_COMM_HPP_
#if defined(REPERTORY_ENABLE_S3)
#include "common.hpp"
#include "comm/i_s3_comm.hpp"
namespace repertory {
class app_config;
class aws_s3_comm final : public virtual i_s3_comm {
public:
explicit aws_s3_comm(const app_config &config);
~aws_s3_comm() override;
private:
struct cache_entry final {
std::chrono::system_clock::time_point expiration;
directory_item_list items;
void reset_timeout(std::chrono::seconds timeout) {
timeout = std::max(std::chrono::seconds(5u), timeout);
expiration = std::chrono::system_clock::now() + timeout;
}
};
private:
const app_config &config_;
s3_config s3_config_;
Aws::SDKOptions sdk_options_;
std::unique_ptr<Aws::S3::S3Client> s3_client_;
mutable std::recursive_mutex cached_directories_mutex_;
mutable std::unordered_map<std::string, cache_entry> cached_directories_;
private:
void clear_expired_directories();
bool get_cached_directory_item_count(const std::string &api_path, std::size_t &count) const;
bool get_cached_directory_items(const std::string &api_path,
const meta_provider_callback &meta_provider,
directory_item_list &list) const;
bool get_cached_file_exists(const std::string &api_path, bool &exists) const;
api_error grab_directory_items(const std::string &api_path,
const meta_provider_callback &meta_provider,
directory_item_list &list) const;
api_error get_file_list(const std::string &bucket_name,
const get_api_file_token_callback &get_api_file_token,
const get_name_callback &get_name, api_file_list &list) const;
void remove_cached_directory(const std::string &api_path);
void set_cached_directory_items(const std::string &api_path, directory_item_list list) const;
public:
api_error create_bucket(const std::string &api_path) override;
bool exists(const std::string &api_path, const get_key_callback &get_key) const override;
void get_bucket_name_and_object_name(const std::string &api_path, const get_key_callback &getKey,
std::string &bucketName,
std::string &objectName) const override;
std::size_t get_directory_item_count(const std::string &api_path,
const meta_provider_callback &meta_provider) const override;
api_error get_directory_items(const std::string &api_path,
const meta_provider_callback &meta_provider,
directory_item_list &list) const override;
api_error get_file(const std::string &api_path, const get_key_callback &get_key,
const get_name_callback &get_name, const get_token_callback &get_token,
api_file &file) const override;
api_error get_file_list(const get_api_file_token_callback &get_api_file_token,
const get_name_callback &get_name, api_file_list &list) const override;
s3_config get_s3_config() override { return s3_config_; }
s3_config get_s3_config() const override { return s3_config_; }
bool is_online() const override { return s3_client_->ListBuckets().IsSuccess(); }
api_error read_file_bytes(const std::string &api_path, const std::size_t &size,
const std::uint64_t &offset, std::vector<char> &data,
const get_key_callback &get_key, const get_size_callback &get_size,
const get_token_callback &get_token,
const bool &stop_requested) const override;
api_error remove_bucket(const std::string &api_path) override;
api_error remove_file(const std::string &api_path, const get_key_callback &get_key) override;
api_error rename_file(const std::string &api_path, const std::string &new_api_path) override;
api_error upload_file(const std::string &api_path, const std::string &source_path,
const std::string &encryption_token, const get_key_callback &get_key,
const set_key_callback &set_key, const bool &stop_requested) override;
};
} // namespace repertory
#endif // REPERTORY_ENABLE_S3
#endif // INCLUDE_COMM_AWS_S3_AWS_S3_COMM_HPP_

430
include/comm/curl/curl_comm.hpp
View File

@ -1,269 +1,231 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#ifndef INCLUDE_COMM_CURL_CURL_COMM_HPP_
#define INCLUDE_COMM_CURL_CURL_COMM_HPP_
#include "common.hpp"
#include "comm/curl/multi_request.hpp"
#include "comm/curl/session_manager.hpp"
#include "comm/i_comm.hpp"
#include "app_config.hpp"
#include "comm/i_http_comm.hpp"
#include "utils/encryption.hpp"
#include "utils/utils.hpp"
namespace repertory {
class curl_resolver;
namespace utils::encryption {
class encrypting_reader;
}
struct curl_setup;
struct raw_write_data;
class curl_comm : public virtual i_comm {
class curl_comm final : public i_http_comm {
public:
typedef size_t (*curl_read_callback)(char *, size_t, size_t, void *);
typedef size_t (*curl_write_callback)(char *, size_t, size_t, void *);
curl_comm() = delete;
static curl_read_callback read_data_callback_;
static curl_write_callback write_data_callback_;
static curl_write_callback write_header_callback_;
static curl_write_callback write_null_callback_;
static curl_write_callback write_string_callback_;
explicit curl_comm(host_config hc);
public:
explicit curl_comm(const app_config &config) : config_(config) {}
~curl_comm() override = default;
explicit curl_comm(s3_config s3);
private:
const app_config &config_;
session_manager session_manager_;
using write_callback = size_t (*)(char *, size_t, size_t, void *);
public:
static std::string construct_url(CURL *curl_handle, const std::string &relative_path,
const host_config &hc);
struct read_write_info final {
repertory::data_buffer data{};
repertory::stop_type &stop_requested;
};
static bool create_auth_session(CURL *&curl_handle, const app_config &config, host_config hc,
std::string &session);
static std::string http_range_to_string(const http_range &range);
static void release_auth_session(const app_config &config, host_config hc,
const std::string &session);
static void update_auth_session(CURL *curl_handle, const app_config &config,
const std::string &session);
static const write_callback write_data;
static const write_callback write_headers;
private:
CURL *common_curl_setup(const std::string &path, curl_setup &setup, std::string &url,
std::string &fields);
std::optional<host_config> host_config_;
std::optional<s3_config> s3_config_;
CURL *common_curl_setup(CURL *curl_handle, const std::string &path, curl_setup &setup,
std::string &url, std::string &fields);
template <typename begin, typename end>
api_error execute_binary_operation(CURL *curl_handle, const std::string &url,
std::vector<char> &data, json &error,
const bool &stop_requested,
const CURLcode &default_code = CURLE_OK) {
auto curl_code = default_code;
long http_code = 400;
execute_operation<begin>(curl_handle, url, curl_code, http_code, stop_requested);
const auto ret = process_binary_response(url, curl_code, http_code, data, error);
if (config_.get_event_level() >= end::level) {
event_system::instance().raise<end>(url, curl_code, http_code,
((ret == api_error::success) ? "" : error.dump(2)));
}
return ret;
}
template <typename begin, typename end>
api_error execute_json_operation(CURL *curl_handle, const std::string &url,
const std::string &result, json &data, json &error,
const bool &stop_requested,
const CURLcode &default_code = CURLE_OK) {
auto curl_code = default_code;
long http_code = 400;
execute_operation<begin>(curl_handle, url, curl_code, http_code, stop_requested);
const auto ret = process_json_response(url, curl_code, http_code, result, data, error);
if (config_.get_event_level() >= end::level) {
event_system::instance().raise<end>(url, curl_code, http_code,
((ret == api_error::success) ? "" : error.dump(2)));
}
return ret;
}
template <typename begin>
void execute_operation(CURL *curl_handle, const std::string &url, CURLcode &curl_code,
long &http_code, const bool &stop_requested) {
if (config_.get_event_level() >= begin::level) {
event_system::instance().raise<begin>(url);
}
if (curl_code == CURLE_OK) {
multi_request request(curl_handle, stop_requested);
request.get_result(curl_code, http_code);
}
}
api_error get_or_post(const host_config &hc, const bool &post, const std::string &path,
const http_parameters &parameters, json &data, json &error,
http_headers *headers = nullptr,
std::function<void(CURL *curl_handle)> cb = nullptr);
api_error get_range(const host_config &hc, const std::string &path,
const std::uint64_t &data_size, const http_parameters &parameters,
const std::string &encryption_token, std::vector<char> &data,
const http_ranges &ranges, json &error, http_headers *headers,
const bool &stop_requested);
api_error get_range_unencrypted(const host_config &hc, const std::string &path,
const http_parameters &parameters, std::vector<char> &data,
const http_ranges &ranges, json &error, http_headers *headers,
const bool &stop_requested);
api_error process_binary_response(const std::string &url, const CURLcode &res,
const long &http_code, std::vector<char> data, json &error);
api_error process_json_response(const std::string &url, const CURLcode &res,
const long &http_code, const std::string &result, json &data,
json &error);
api_error process_response(const std::string &url, const CURLcode &res, const long &http_code,
const std::size_t &data_size,
const std::function<std::string()> &to_string_converter,
const std::function<void()> &success_handler, json &error) const;
static std::string url_encode(CURL *curl_handle, const std::string &data,
const bool &allow_slash = false);
private:
bool use_s3_path_style_{false};
public:
api_error get(const std::string &path, json &data, json &error) override {
return get_or_post(config_.get_host_config(), false, path, {}, data, error);
[[nodiscard]] static auto construct_url(CURL *curl,
const std::string &relative_path,
const host_config &hc) -> std::string;
[[nodiscard]] static auto create_host_config(const s3_config &config,
bool use_s3_path_style)
-> host_config;
[[nodiscard]] static auto url_encode(CURL *curl, const std::string &data,
bool allow_slash) -> std::string;
template <typename request_type>
[[nodiscard]] static auto
make_encrypted_request(const host_config &hc, const request_type &request,
long &response_code, stop_type &stop_requested)
-> bool {
response_code = 0;
if (not request.decryption_token.has_value() ||
request.decryption_token.value().empty()) {
return false;
}
if (not request.range.has_value()) {
return false;
}
if (not request.total_size.has_value()) {
return false;
}
data_buffer data{};
const auto key =
utils::encryption::generate_key(request.decryption_token.value());
const auto result = utils::encryption::read_encrypted_range(
request.range.value(), key,
[&](std::vector<char> &ct, std::uint64_t start_offset,
std::uint64_t end_offset) -> api_error {
auto encrypted_request = request;
encrypted_request.decryption_token = std::nullopt;
encrypted_request.range = {{start_offset, end_offset}};
encrypted_request.response_handler = [&ct](const auto &encrypted_data,
long /*response_code*/) {
ct = encrypted_data;
};
encrypted_request.total_size = std::nullopt;
if (not make_request(hc, encrypted_request, response_code,
stop_requested)) {
return api_error::comm_error;
}
if (response_code != 200) {
return api_error::comm_error;
}
return api_error::success;
},
request.total_size.value(), data);
if (result != api_error::success) {
return false;
}
if (request.response_handler.has_value()) {
request.response_handler.value()(data, response_code);
}
return true;
}
api_error get(const host_config &hc, const std::string &path, json &data, json &error) override {
return get_or_post(hc, false, path, {}, data, error);
template <typename request_type>
[[nodiscard]] static auto
make_request(const host_config &hc, const request_type &request,
long &response_code, stop_type &stop_requested) -> bool {
if (request.decryption_token.has_value() &&
not request.decryption_token.value().empty()) {
return make_encrypted_request(hc, request, response_code, stop_requested);
}
response_code = 0;
auto *curl = utils::create_curl();
if (not request.set_method(curl, stop_requested)) {
return false;
}
if (not hc.agent_string.empty()) {
curl_easy_setopt(curl, CURLOPT_USERAGENT, hc.agent_string.c_str());
}
if (request.allow_timeout && hc.timeout_ms) {
curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, hc.timeout_ms);
}
std::string range_list{};
if (request.range.has_value()) {
range_list = std::to_string(request.range.value().begin) + '-' +
std::to_string(request.range.value().end);
curl_easy_setopt(curl, CURLOPT_RANGE, range_list.c_str());
}
if (request.response_headers.has_value()) {
curl_easy_setopt(curl, CURLOPT_HEADERDATA,
&request.response_headers.value());
curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, write_headers);
}
read_write_info write_info{{}, stop_requested};
if (request.response_handler.has_value()) {
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &write_info);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
}
std::string parameters{};
for (const auto &kv : request.query) {
parameters += (parameters.empty() ? '?' : '&') + kv.first + '=' +
url_encode(curl, kv.second, false);
}
if (not hc.api_password.empty()) {
curl_easy_setopt(curl, CURLOPT_USERNAME, hc.api_user.c_str());
curl_easy_setopt(curl, CURLOPT_PASSWORD, hc.api_password.c_str());
} else if (not hc.api_user.empty()) {
curl_easy_setopt(curl, CURLOPT_USERNAME, hc.api_user.c_str());
}
if (request.aws_service.has_value()) {
curl_easy_setopt(curl, CURLOPT_AWS_SIGV4,
request.aws_service.value().c_str());
}
auto url = construct_url(curl, request.get_path(), hc) + parameters;
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
multi_request curl_request(curl, stop_requested);
CURLcode curl_code{};
curl_request.get_result(curl_code, response_code);
if (curl_code != CURLE_OK) {
return false;
}
if (request.response_handler.has_value()) {
request.response_handler.value()(write_info.data, response_code);
}
return true;
}
api_error get(const std::string &path, const http_parameters &parameters, json &data,
json &error) override {
return get_or_post(config_.get_host_config(), false, path, parameters, data, error);
}
public:
void enable_s3_path_style(bool enable) override;
api_error get(const host_config &hc, const std::string &path, const http_parameters &parameters,
json &data, json &error) override {
return get_or_post(hc, false, path, parameters, data, error);
}
[[nodiscard]] auto make_request(const curl::requests::http_delete &del,
long &response_code,
stop_type &stop_requested) const
-> bool override;
api_error get_range(const std::string &path, const std::uint64_t &data_size,
const http_parameters &parameters, const std::string &encryption_token,
std::vector<char> &data, const http_ranges &ranges, json &error,
const bool &stop_requested) override {
return get_range(config_.get_host_config(), path, data_size, parameters, encryption_token, data,
ranges, error, nullptr, stop_requested);
}
[[nodiscard]] auto make_request(const curl::requests::http_get &get,
long &response_code,
stop_type &stop_requested) const
-> bool override;
api_error get_range(const host_config &hc, const std::string &path,
const std::uint64_t &data_size, const http_parameters &parameters,
const std::string &encryption_token, std::vector<char> &data,
const http_ranges &ranges, json &error, const bool &stop_requested) override {
return get_range(hc, path, data_size, parameters, encryption_token, data, ranges, error,
nullptr, stop_requested);
}
[[nodiscard]] auto make_request(const curl::requests::http_head &head,
long &response_code,
stop_type &stop_requested) const
-> bool override;
api_error get_range_and_headers(const std::string &path, const std::uint64_t &dataSize,
const http_parameters &parameters,
const std::string &encryption_token, std::vector<char> &data,
const http_ranges &ranges, json &error, http_headers &headers,
const bool &stop_requested) override {
return get_range(config_.get_host_config(), path, dataSize, parameters, encryption_token, data,
ranges, error, &headers, stop_requested);
}
api_error get_range_and_headers(const host_config &hc, const std::string &path,
const std::uint64_t &dataSize, const http_parameters &parameters,
const std::string &encryption_token, std::vector<char> &data,
const http_ranges &ranges, json &error, http_headers &headers,
const bool &stop_requested) override {
return get_range(hc, path, dataSize, parameters, encryption_token, data, ranges, error,
&headers, stop_requested);
}
api_error get_raw(const std::string &path, const http_parameters &parameters,
std::vector<char> &data, json &error, const bool &stop_requested) override {
return get_raw(config_.get_host_config(), path, parameters, data, error, stop_requested);
}
api_error get_raw(const host_config &hc, const std::string &path,
const http_parameters &parameters, std::vector<char> &data, json &error,
const bool &stop_requested) override;
api_error post(const std::string &path, json &data, json &error) override {
return get_or_post(config_.get_host_config(), true, path, {}, data, error);
}
api_error post(const host_config &hc, const std::string &path, json &data, json &error) override {
return get_or_post(hc, true, path, {}, data, error);
}
api_error post(const std::string &path, const http_parameters &parameters, json &data,
json &error) override {
return get_or_post(config_.get_host_config(), true, path, parameters, data, error);
}
api_error post(const host_config &hc, const std::string &path, const http_parameters &parameters,
json &data, json &error) override {
return get_or_post(hc, true, path, parameters, data, error);
}
api_error post_file(const std::string &path, const std::string &source_path,
const http_parameters &parameters, json &data, json &error,
const bool &stop_requested) override {
return post_file(config_.get_host_config(), path, source_path, parameters, data, error,
stop_requested);
}
api_error post_file(const host_config &hc, const std::string &path,
const std::string &source_path, const http_parameters &parameters, json &data,
json &error, const bool &stop_requested) override;
api_error post_multipart_file(const std::string &path, const std::string &file_name,
const std::string &source_path, const std::string &encryption_token,
json &data, json &error, const bool &stop_requested) override {
return post_multipart_file(config_.get_host_config(), path, file_name, source_path,
encryption_token, data, error, stop_requested);
}
api_error post_multipart_file(const host_config &hc, const std::string &path,
const std::string &file_name, const std::string &source_path,
const std::string &encryption_token, json &data, json &error,
const bool &stop_requested) override;
bool tus_upload(host_config hc, const std::string &source_path, const std::string &file_name,
std::uint64_t file_size, const std::string &location, std::string &skylink,
const bool &stop_requested, utils::encryption::encrypting_reader *reader);
bool tus_upload_create(host_config hc, const std::string &file_name,
const std::uint64_t &file_size, std::string &location);
[[nodiscard]] auto make_request(const curl::requests::http_put_file &put_file,
long &response_code,
stop_type &stop_requested) const
-> bool override;
};
} // namespace repertory

45
include/comm/curl/curl_resolver.hpp
View File

@ -1,45 +0,0 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#ifndef INCLUDE_COMM_CURL_CURL_RESOLVER_HPP_
#define INCLUDE_COMM_CURL_CURL_RESOLVER_HPP_
#include "common.hpp"
namespace repertory {
class curl_resolver final {
public:
curl_resolver() = delete;
curl_resolver(const curl_resolver &) noexcept = delete;
curl_resolver(curl_resolver &&) noexcept = delete;
curl_resolver &operator=(const curl_resolver &) noexcept = delete;
curl_resolver &operator=(curl_resolver &&) noexcept = delete;
public:
curl_resolver(CURL *handle, std::vector<std::string> items, const bool &ignore_root = false);
~curl_resolver();
private:
std::vector<std::string> items_;
struct curl_slist *host_list_ = nullptr;
};
} // namespace repertory
#endif // INCLUDE_COMM_CURL_CURL_RESOLVER_HPP_

33
include/comm/curl/multi_request.hpp
View File

@ -1,36 +1,39 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#ifndef INCLUDE_COMM_CURL_MULTI_REQUEST_HPP_
#define INCLUDE_COMM_CURL_MULTI_REQUEST_HPP_
#include "common.hpp"
#include "types/repertory.hpp"
namespace repertory {
class multi_request final {
public:
multi_request(CURL *curl_handle, const bool &stop_requested);
multi_request(CURL *curl_handle, stop_type &stop_requested);
~multi_request();
private:
CURL *curl_handle_;
const bool &stop_requested_;
stop_type &stop_requested_;
CURLM *multi_handle_;
public:

40
include/comm/curl/requests/http_delete.hpp Normal file
View File

@ -0,0 +1,40 @@
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#ifndef INCLUDE_COMM_CURL_CURL_REQUESTS_HTTP_DELETE_HPP_
#define INCLUDE_COMM_CURL_CURL_REQUESTS_HTTP_DELETE_HPP_
#include "comm/curl/requests/http_request_base.hpp"
namespace repertory::curl::requests {
struct http_delete final : http_request_base {
~http_delete() override = default;
[[nodiscard]] auto set_method(CURL *curl,
stop_type & /* stop_requested */) const
-> bool override {
curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE");
return true;
}
};
} // namespace repertory::curl::requests
#endif // INCLUDE_COMM_CURL_CURL_REQUESTS_HTTP_DELETE_HPP_

40
include/comm/curl/requests/http_get.hpp Normal file
View File

@ -0,0 +1,40 @@
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#ifndef INCLUDE_COMM_CURL_CURL_REQUESTS_HTTP_GET_HPP_
#define INCLUDE_COMM_CURL_CURL_REQUESTS_HTTP_GET_HPP_
#include "comm/curl/requests/http_request_base.hpp"
namespace repertory::curl::requests {
struct http_get final : http_request_base {
~http_get() override = default;
[[nodiscard]] auto set_method(CURL *curl,
stop_type & /*stop_requested*/) const
-> bool override {
curl_easy_setopt(curl, CURLOPT_HTTPGET, 1L);
return true;
}
};
} // namespace repertory::curl::requests
#endif // INCLUDE_COMM_CURL_CURL_REQUESTS_HTTP_GET_HPP_

41
include/comm/curl/requests/http_head.hpp Normal file
View File

@ -0,0 +1,41 @@
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#ifndef INCLUDE_COMM_CURL_CURL_REQUESTS_HTTP_HEAD_HPP_
#define INCLUDE_COMM_CURL_CURL_REQUESTS_HTTP_HEAD_HPP_
#include "comm/curl/requests/http_request_base.hpp"
namespace repertory::curl::requests {
struct http_head final : http_request_base {
~http_head() override = default;
[[nodiscard]] auto set_method(CURL *curl,
stop_type & /* stop_requested */) const
-> bool override {
curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "HEAD");
curl_easy_setopt(curl, CURLOPT_NOBODY, 1L);
return true;
}
};
} // namespace repertory::curl::requests
#endif // INCLUDE_COMM_CURL_CURL_REQUESTS_HTTP_HEAD_HPP_

47
include/comm/curl/requests/http_put_file.hpp Normal file
View File

@ -0,0 +1,47 @@
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#ifndef INCLUDE_COMM_CURL_CURL_REQUESTS_HTTP_PUT_FILE_HPP_
#define INCLUDE_COMM_CURL_CURL_REQUESTS_HTTP_PUT_FILE_HPP_
#include "comm/curl/requests/http_request_base.hpp"
#include "utils/encrypting_reader.hpp"
namespace repertory::curl::requests {
struct http_put_file final : http_request_base {
~http_put_file() override = default;
std::optional<std::string> encryption_token{};
std::string file_name{};
mutable std::shared_ptr<utils::encryption::encrypting_reader> reader{};
std::string source_path{};
[[nodiscard]] auto get_path() const -> std::string override;
[[nodiscard]] auto set_method(CURL *curl, stop_type &stop_requested) const
-> bool override;
private:
mutable std::shared_ptr<read_file_info> read_info{};
};
} // namespace repertory::curl::requests
#endif // INCLUDE_COMM_CURL_CURL_REQUESTS_HTTP_PUT_FILE_HPP_

74
include/comm/curl/requests/http_request_base.hpp Normal file
View File

@ -0,0 +1,74 @@
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#ifndef INCLUDE_COMM_CURL_CURL_REQUESTS_HTTP_REQUEST_BASE_HPP_
#define INCLUDE_COMM_CURL_CURL_REQUESTS_HTTP_REQUEST_BASE_HPP_
#include "types/repertory.hpp"
#include "utils/native_file.hpp"
namespace repertory::curl::requests {
using read_callback = size_t (*)(char *, size_t, size_t, void *);
using response_callback =
std::function<void(const data_buffer &data, long response_code)>;
struct read_file_info final {
stop_type &stop_requested;
native_file::native_file_ptr nf{};
std::uint64_t offset{};
};
inline const auto read_file_data = static_cast<read_callback>(
[](char *buffer, size_t size, size_t nitems, void *instream) -> size_t {
auto *rd = reinterpret_cast<read_file_info *>(instream);
std::size_t bytes_read{};
auto ret =
rd->nf->read_bytes(buffer, size * nitems, rd->offset, bytes_read);
if (ret) {
rd->offset += bytes_read;
}
return ret && not rd->stop_requested ? bytes_read : CURL_READFUNC_ABORT;
});
struct http_request_base {
virtual ~http_request_base() = default;
bool allow_timeout{};
std::optional<std::string> aws_service;
std::optional<std::string> decryption_token{};
http_headers headers{};
std::string path{};
query_parameters query{};
std::optional<http_range> range{};
std::optional<response_callback> response_handler;
std::optional<http_headers> response_headers;
std::optional<std::uint64_t> total_size{};
[[nodiscard]] virtual auto get_path() const -> std::string { return path; }
[[nodiscard]] virtual auto set_method(CURL *curl,
stop_type &stop_requested) const
-> bool = 0;
};
} // namespace repertory::curl::requests
#endif // INCLUDE_COMM_CURL_CURL_REQUESTS_HTTP_REQUEST_BASE_HPP_

43
include/comm/curl/session_manager.hpp
View File

@ -1,43 +0,0 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#ifndef INCLUDE_COMM_CURL_SESSION_MANAGER_HPP_
#define INCLUDE_COMM_CURL_SESSION_MANAGER_HPP_
#include "common.hpp"
#include "types/repertory.hpp"
namespace repertory {
class app_config;
class session_manager final {
private:
std::string session_;
std::uint64_t session_count_ = 0u;
std::mutex session_mutex_;
public:
bool create_auth_session(CURL *&curl_handle, const app_config &config, host_config hc,
std::string &session);
void release_auth_session(const app_config &config, host_config hc, const std::string &session);
void update_auth_session(CURL *curl_handle, const app_config &config, const host_config &hc);
};
} // namespace repertory
#endif // INCLUDE_COMM_CURL_SESSION_MANAGER_HPP_

107
include/comm/i_comm.hpp
View File

@ -1,107 +0,0 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#ifndef INCLUDE_COMM_I_COMM_HPP_
#define INCLUDE_COMM_I_COMM_HPP_
#include "common.hpp"
#include "types/repertory.hpp"
namespace repertory {
class i_comm {
INTERFACE_SETUP(i_comm);
public:
virtual api_error get(const std::string &path, json &data, json &error) = 0;
virtual api_error get(const host_config &hc, const std::string &path, json &data,
json &error) = 0;
virtual api_error get(const std::string &path, const http_parameters &parameters, json &data,
json &error) = 0;
virtual api_error get(const host_config &hc, const std::string &path,
const http_parameters &parameters, json &data, json &error) = 0;
virtual api_error get_range(const std::string &path, const std::uint64_t &data_size,
const http_parameters &parameters,
const std::string &encryption_token, std::vector<char> &data,
const http_ranges &ranges, json &error,
const bool &stop_requested) = 0;
virtual api_error get_range(const host_config &hc, const std::string &path,
const std::uint64_t &data_size, const http_parameters &parameters,
const std::string &encryption_token, std::vector<char> &data,
const http_ranges &ranges, json &error,
const bool &stop_requested) = 0;
virtual api_error get_range_and_headers(const std::string &path, const std::uint64_t &data_size,
const http_parameters &parameters,
const std::string &encryption_token,
std::vector<char> &data, const http_ranges &ranges,
json &error, http_headers &headers,
const bool &stop_requested) = 0;
virtual api_error get_range_and_headers(const host_config &hc, const std::string &path,
const std::uint64_t &data_size,
const http_parameters &parameters,
const std::string &encryption_token,
std::vector<char> &data, const http_ranges &ranges,
json &error, http_headers &headers,
const bool &stop_requested) = 0;
virtual api_error get_raw(const std::string &path, const http_parameters &parameters,
std::vector<char> &data, json &error, const bool &stop_requested) = 0;
virtual api_error get_raw(const host_config &hc, const std::string &path,
const http_parameters &parameters, std::vector<char> &data, json &error,
const bool &stop_requested) = 0;
virtual api_error post(const std::string &path, json &data, json &error) = 0;
virtual api_error post(const host_config &hc, const std::string &path, json &data,
json &error) = 0;
virtual api_error post(const std::string &path, const http_parameters &parameters, json &data,
json &error) = 0;
virtual api_error post(const host_config &hc, const std::string &path,
const http_parameters &parameters, json &data, json &error) = 0;
virtual api_error post_file(const std::string &path, const std::string &sourcePath,
const http_parameters &parameters, json &data, json &error,
const bool &stop_requested) = 0;
virtual api_error post_file(const host_config &hc, const std::string &path,
const std::string &sourcePath, const http_parameters &parameters,
json &data, json &error, const bool &stop_requested) = 0;
virtual api_error post_multipart_file(const std::string &path, const std::string &file_name,
const std::string &source_path,
const std::string &encryption_token, json &data,
json &error, const bool &stop_requested) = 0;
virtual api_error post_multipart_file(const host_config &hc, const std::string &path,
const std::string &file_name,
const std::string &source_path,
const std::string &encryption_token, json &data,
json &error, const bool &stop_requested) = 0;
};
} // namespace repertory
#endif // INCLUDE_COMM_I_COMM_HPP_

59
include/comm/i_http_comm.hpp Normal file
View File

@ -0,0 +1,59 @@
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#ifndef INCLUDE_COMM_I_HTTP_COMM_HPP_
#define INCLUDE_COMM_I_HTTP_COMM_HPP_
#include "comm/curl/requests/http_delete.hpp"
#include "comm/curl/requests/http_get.hpp"
#include "comm/curl/requests/http_head.hpp"
#include "comm/curl/requests/http_put_file.hpp"
#include "types/repertory.hpp"
namespace repertory {
struct i_http_comm {
INTERFACE_SETUP(i_http_comm);
public:
virtual void enable_s3_path_style(bool enable) = 0;
[[nodiscard]] virtual auto
make_request(const curl::requests::http_delete &del, long &response_code,
stop_type &stop_requested) const -> bool = 0;
[[nodiscard]] virtual auto make_request(const curl::requests::http_get &get,
long &response_code,
stop_type &stop_requested) const
-> bool = 0;
[[nodiscard]] virtual auto make_request(const curl::requests::http_head &head,
long &response_code,
stop_type &stop_requested) const
-> bool = 0;
[[nodiscard]] virtual auto
make_request(const curl::requests::http_put_file &put_file,
long &response_code, stop_type &stop_requested) const
-> bool = 0;
};
} // namespace repertory
#endif // INCLUDE_COMM_I_HTTP_COMM_HPP_

125
include/comm/i_s3_comm.hpp
View File

@ -1,90 +1,107 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#ifndef INCLUDE_COMM_I_S3_COMM_HPP_
#define INCLUDE_COMM_I_S3_COMM_HPP_
#if defined(REPERTORY_ENABLE_S3)
#include "common.hpp"
#include "types/repertory.hpp"
#include "types/s3.hpp"
namespace repertory {
class i_provider;
class i_s3_comm {
INTERFACE_SETUP(i_s3_comm);
public:
typedef std::function<std::string(const std::string &api_path)> get_api_file_token_callback;
typedef std::function<std::string()> get_key_callback;
typedef std::function<std::string(const std::string &key, const std::string &object_name)>
get_name_callback;
typedef std::function<std::uint64_t()> get_size_callback;
typedef std::function<std::string()> get_token_callback;
typedef std::function<api_error(const std::string &key)> set_key_callback;
[[nodiscard]] virtual auto create_directory(const std::string &api_path)
-> api_error = 0;
public:
virtual api_error create_bucket(const std::string &api_path) = 0;
[[nodiscard]] virtual auto directory_exists(const std::string &api_path) const
-> api_error = 0;
virtual bool exists(const std::string &api_path, const get_key_callback &get_key) const = 0;
[[nodiscard]] virtual auto file_exists(const std::string &api_path,
const get_key_callback &get_key) const
-> api_error = 0;
virtual void get_bucket_name_and_object_name(const std::string &api_path,
const get_key_callback &get_key,
std::string &bucket_name,
std::string &object_name) const = 0;
virtual std::size_t
[[nodiscard]] virtual auto
get_directory_item_count(const std::string &api_path,
const meta_provider_callback &meta_provider) const = 0;
meta_provider_callback meta_provider) const
-> std::size_t = 0;
virtual api_error get_directory_items(const std::string &api_path,
const meta_provider_callback &meta_provider,
directory_item_list &list) const = 0;
[[nodiscard]] virtual auto
get_directory_items(const std::string &api_path,
meta_provider_callback meta_provider,
directory_item_list &list) const -> api_error = 0;
virtual s3_config get_s3_config() = 0;
[[nodiscard]] virtual auto get_directory_list(api_file_list &list) const
-> api_error = 0;
virtual s3_config get_s3_config() const = 0;
[[nodiscard]] virtual auto get_file(const std::string &api_path,
const get_key_callback &get_key,
const get_name_callback &get_name,
const get_token_callback &get_token,
api_file &file) const -> api_error = 0;
virtual api_error get_file(const std::string &api_path, const get_key_callback &get_key,
const get_name_callback &get_name, const get_token_callback &get_token,
api_file &file) const = 0;
[[nodiscard]] virtual auto
get_file_list(const get_api_file_token_callback &get_api_file_token,
const get_name_callback &get_name, api_file_list &list) const
-> api_error = 0;
virtual api_error get_file_list(const get_api_file_token_callback &get_api_file_token,
const get_name_callback &get_name, api_file_list &list) const = 0;
[[nodiscard]] virtual auto
get_object_list(std::vector<directory_item> &list) const -> api_error = 0;
virtual bool is_online() const = 0;
[[nodiscard]] virtual auto
get_object_name(const std::string &api_path,
const get_key_callback &get_key) const -> std::string = 0;
virtual api_error read_file_bytes(const std::string &api_path, const std::size_t &size,
const std::uint64_t &offset, std::vector<char> &data,
const get_key_callback &get_key,
const get_size_callback &get_size,
const get_token_callback &get_token,
const bool &stop_requested) const = 0;
[[nodiscard]] virtual auto get_s3_config() -> s3_config = 0;
virtual api_error remove_file(const std::string &api_path, const get_key_callback &get_key) = 0;
[[nodiscard]] virtual auto get_s3_config() const -> s3_config = 0;
virtual api_error remove_bucket(const std::string &api_path) = 0;
[[nodiscard]] virtual auto is_online() const -> bool = 0;
virtual api_error rename_file(const std::string &api_path, const std::string &new_api_path) = 0;
[[nodiscard]] virtual auto read_file_bytes(
const std::string &api_path, std::size_t size, std::uint64_t offset,
data_buffer &data, const get_key_callback &get_key,
const get_size_callback &get_size, const get_token_callback &get_token,
stop_type &stop_requested) const -> api_error = 0;
virtual api_error upload_file(const std::string &api_path, const std::string &source_path,
const std::string &encryption_token,
const get_key_callback &get_key, const set_key_callback &set_key,
const bool &stop_requested) = 0;
[[nodiscard]] virtual auto remove_directory(const std::string &api_path)
-> api_error = 0;
[[nodiscard]] virtual auto remove_file(const std::string &api_path,
const get_key_callback &get_key)
-> api_error = 0;
[[nodiscard]] virtual auto rename_file(const std::string &api_path,
const std::string &new_api_path)
-> api_error = 0;
[[nodiscard]] virtual auto
upload_file(const std::string &api_path, const std::string &source_path,
const std::string &encryption_token,
const get_key_callback &get_key, const set_key_callback &set_key,
stop_type &stop_requested) -> api_error = 0;
};
} // namespace repertory

50
include/comm/packet/client_pool.hpp
View File

@ -1,40 +1,45 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#ifndef INCLUDE_COMM_PACKET_CLIENT_POOL_HPP_
#define INCLUDE_COMM_PACKET_CLIENT_POOL_HPP_
#include "common.hpp"
#include "comm/packet/packet.hpp"
#include "types/repertory.hpp"
namespace repertory {
class client_pool final {
public:
typedef std::function<packet::error_type()> worker_callback;
typedef std::function<void(const packet::error_type &result)> worker_complete_callback;
using worker_callback = std::function<packet::error_type()>;
using worker_complete_callback =
std::function<void(const packet::error_type &)>;
private:
class pool final {
private:
struct work_item final {
work_item(const worker_callback &worker, const worker_complete_callback &worker_complete)
: work(worker), work_complete(worker_complete) {}
work_item(worker_callback worker,
worker_complete_callback worker_complete)
: work(std::move(worker)),
work_complete(std::move(worker_complete)) {}
worker_callback work;
worker_complete_callback work_complete;
@ -47,7 +52,7 @@ private:
};
public:
explicit pool(const std::uint8_t &pool_size);
explicit pool(std::uint8_t pool_size);
~pool() { shutdown(); }
@ -58,14 +63,14 @@ private:
std::atomic<std::uint8_t> thread_index_;
public:
void execute(const std::uint64_t &thread_id, const worker_callback &worker,
void execute(std::uint64_t thread_id, const worker_callback &worker,
const worker_complete_callback &worker_complete);
void shutdown();
};
public:
explicit client_pool(const std::uint8_t &pool_size = 10u)
explicit client_pool(std::uint8_t pool_size = 10u)
: pool_size_(pool_size ? pool_size : 10u) {}
~client_pool() { shutdown(); }
@ -77,8 +82,9 @@ private:
bool shutdown_ = false;
public:
void execute(const std::string &client_id, const std::uint64_t &thread_id,
const worker_callback &worker, const worker_complete_callback &worker_complete);
void execute(const std::string &client_id, std::uint64_t thread_id,
const worker_callback &worker,
const worker_complete_callback &worker_complete);
void remove_client(const std::string &client_id);

137
include/comm/packet/packet.hpp
View File

@ -1,110 +1,116 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#ifndef INCLUDE_COMM_PACKET_PACKET_HPP_
#define INCLUDE_COMM_PACKET_PACKET_HPP_
#include "common.hpp"
#include "types/remote.hpp"
#include "types/repertory.hpp"
namespace repertory {
#define DECODE_OR_RETURN(p, value) \
if ((ret = (p)->decode(value)) != 0) \
#define DECODE_OR_RETURN(p, value) \
if ((ret = (p)->decode(value)) != 0) \
return ret
#define DECODE_OR_IGNORE(p, value) \
if (ret == 0) \
#define DECODE_OR_IGNORE(p, value) \
if (ret == 0) \
ret = (p)->decode(value)
class packet final {
public:
typedef std::int32_t error_type;
using error_type = std::int32_t;
public:
packet() = default;
explicit packet(std::vector<char> buffer) : buffer_(std::move(buffer)) {}
explicit packet(data_buffer buffer) : buffer_(std::move(buffer)) {}
explicit packet(std::vector<char> &&buffer) : buffer_(std::move(buffer)) {}
explicit packet(data_buffer &&buffer) : buffer_(std::move(buffer)) {}
packet(const packet &p) noexcept : buffer_(p.buffer_), decode_offset_(p.decode_offset_) {}
packet(const packet &p) noexcept = default;
packet(packet &&p) noexcept : buffer_(std::move(p.buffer_)), decode_offset_(p.decode_offset_) {}
packet(packet &&p) noexcept
: buffer_(std::move(p.buffer_)), decode_offset_(p.decode_offset_) {}
private:
std::vector<char> buffer_;
data_buffer buffer_;
std::size_t decode_offset_ = 0u;
public:
static int decode_json(packet &response, json &json_data);
[[nodiscard]] static auto decode_json(packet &response, json &json_data)
-> int;
public:
void clear();
char *current_pointer() {
return (decode_offset_ < buffer_.size()) ? &buffer_[decode_offset_] : nullptr;
[[nodiscard]] auto current_pointer() -> char * {
return (decode_offset_ < buffer_.size()) ? &buffer_[decode_offset_]
: nullptr;
}
const char *current_pointer() const {
return (decode_offset_ < buffer_.size()) ? &buffer_[decode_offset_] : nullptr;
[[nodiscard]] auto current_pointer() const -> const char * {
return (decode_offset_ < buffer_.size()) ? &buffer_[decode_offset_]
: nullptr;
}
error_type decode(std::string &data);
[[nodiscard]] auto decode(std::string &data) -> error_type;
error_type decode(std::wstring &data);
[[nodiscard]] auto decode(std::wstring &data) -> error_type;
error_type decode(void *buffer, const size_t &size);
[[nodiscard]] auto decode(void *buffer, std::size_t size) -> error_type;
error_type decode(void *&ptr);
[[nodiscard]] auto decode(void *&ptr) -> error_type;
error_type decode(std::int8_t &i);
[[nodiscard]] auto decode(std::int8_t &i) -> error_type;
error_type decode(std::uint8_t &i);
[[nodiscard]] auto decode(std::uint8_t &i) -> error_type;
error_type decode(std::int16_t &i);
[[nodiscard]] auto decode(std::int16_t &i) -> error_type;
error_type decode(std::uint16_t &i);
[[nodiscard]] auto decode(std::uint16_t &i) -> error_type;
error_type decode(std::int32_t &i);
[[nodiscard]] auto decode(std::int32_t &i) -> error_type;
error_type decode(std::uint32_t &i);
[[nodiscard]] auto decode(std::uint32_t &i) -> error_type;
error_type decode(std::int64_t &i);
[[nodiscard]] auto decode(std::int64_t &i) -> error_type;
error_type decode(std::uint64_t &i);
[[nodiscard]] auto decode(std::uint64_t &i) -> error_type;
error_type decode(remote::open_flags &i) {
[[nodiscard]] auto decode(remote::open_flags &i) -> error_type {
return decode(reinterpret_cast<std::uint32_t &>(i));
}
error_type decode(remote::setattr_x &i);
[[nodiscard]] auto decode(remote::setattr_x &i) -> error_type;
error_type decode(remote::stat &i);
[[nodiscard]] auto decode(remote::stat &i) -> error_type;
error_type decode(remote::statfs &i);
[[nodiscard]] auto decode(remote::statfs &i) -> error_type;
error_type decode(remote::statfs_x &i);
[[nodiscard]] auto decode(remote::statfs_x &i) -> error_type;
error_type decode(remote::file_info &i);
[[nodiscard]] auto decode(remote::file_info &i) -> error_type;
error_type decrypt(const std::string &token);
[[nodiscard]] auto decrypt(const std::string &token) -> error_type;
void encode(const void *buffer, const std::size_t &size, bool should_reserve = true);
void encode(const void *buffer, std::size_t size, bool should_reserve = true);
void encode(char *str) { encode(std::string(str ? str : "")); }
@ -150,14 +156,16 @@ public:
void encode(remote::file_info i);
void encode_top(const void *buffer, const std::size_t &size, bool should_reserve = true);
void encode_top(const void *buffer, std::size_t size,
bool should_reserve = true);
void encode_top(const std::string &str);
void encode_top(const std::wstring &str);
void encode_top(void *ptr) {
encode_top(static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(ptr)));
encode_top(
static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(ptr)));
}
void encode_top(std::int8_t i);
@ -176,7 +184,9 @@ public:
void encode_top(std::uint64_t i);
void encode_top(remote::open_flags i) { encode_top(static_cast<std::uint32_t>(i)); }
void encode_top(remote::open_flags i) {
encode_top(static_cast<std::uint32_t>(i));
}
void encode_top(remote::setattr_x i);
@ -190,24 +200,31 @@ public:
void encrypt(const std::string &token);
std::uint32_t get_size() const { return static_cast<std::uint32_t>(buffer_.size()); }
[[nodiscard]] auto get_size() const -> std::uint32_t {
return static_cast<std::uint32_t>(buffer_.size());
}
void transfer_into(std::vector<char> &buffer);
void transfer_into(data_buffer &buffer);
public:
packet &operator=(const std::vector<char> &buffer) noexcept;
auto operator=(const data_buffer &buffer) noexcept -> packet &;
packet &operator=(std::vector<char> &&buffer) noexcept;
auto operator=(data_buffer &&buffer) noexcept -> packet &;
packet &operator=(const packet &p) noexcept;
auto operator=(const packet &p) noexcept -> packet &;
packet &operator=(packet &&p) noexcept;
auto operator=(packet &&p) noexcept -> packet &;
char &operator[](const size_t &index) { return buffer_[index]; }
[[nodiscard]] auto operator[](std::size_t index) -> char & {
return buffer_[index];
}
const char &operator[](const size_t &index) const { return buffer_.at(index); }
[[nodiscard]] auto operator[](std::size_t index) const -> const char & {
return buffer_.at(index);
}
};
typedef packet packet;
using packet = packet;
} // namespace repertory
#endif // INCLUDE_COMM_PACKET_PACKET_HPP_

59
include/comm/packet/packet_client.hpp
View File

@ -1,25 +1,27 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#ifndef INCLUDE_COMM_PACKET_PACKET_CLIENT_HPP_
#define INCLUDE_COMM_PACKET_PACKET_CLIENT_HPP_
#include "common.hpp"
#include "comm/packet/packet.hpp"
using boost::asio::ip::tcp;
@ -27,17 +29,17 @@ using boost::asio::ip::tcp;
namespace repertory {
class packet_client final {
private:
struct client {
client(boost::asio::io_context &ctx) : nonce(""), socket(ctx) {}
struct client final {
explicit client(boost::asio::io_context &ctx) : nonce(""), socket(ctx) {}
std::string nonce;
tcp::socket socket;
};
public:
packet_client(std::string host_name_or_ip, const std::uint8_t &max_connections,
const std::uint16_t &port, const std::uint16_t &receive_timeout,
const std::uint16_t &send_timeout, std::string encryption_token);
packet_client(std::string host_name_or_ip, std::uint8_t max_connections,
std::uint16_t port, std::uint16_t receive_timeout,
std::uint16_t send_timeout, std::string encryption_token);
~packet_client();
@ -52,7 +54,8 @@ private:
std::string unique_id_;
bool allow_connections_ = true;
boost::asio::ip::basic_resolver<boost::asio::ip::tcp>::results_type resolve_results_;
boost::asio::ip::basic_resolver<boost::asio::ip::tcp>::results_type
resolve_results_;
std::mutex clients_mutex_;
std::vector<std::shared_ptr<client>> clients_;
@ -61,23 +64,27 @@ private:
void close_all();
bool connect(client &c);
void connect(client &c);
std::shared_ptr<client> get_client();
[[nodiscard]] auto get_client() -> std::shared_ptr<client>;
void put_client(std::shared_ptr<client> &c);
packet::error_type read_packet(client &c, packet &response);
[[nodiscard]] auto read_packet(client &c, packet &response)
-> packet::error_type;
void resolve();
public:
packet::error_type send(const std::string &method, std::uint32_t &service_flags);
[[nodiscard]] auto send(const std::string &method,
std::uint32_t &service_flags) -> packet::error_type;
packet::error_type send(const std::string &method, packet &request, std::uint32_t &service_flags);
[[nodiscard]] auto send(const std::string &method, packet &request,
std::uint32_t &service_flags) -> packet::error_type;
packet::error_type send(const std::string &method, packet &request, packet &response,
std::uint32_t &service_flags);
[[nodiscard]] auto send(const std::string &method, packet &request,
packet &response, std::uint32_t &service_flags)
-> packet::error_type;
};
} // namespace repertory

58
include/comm/packet/packet_server.hpp
View File

@ -1,26 +1,29 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#ifndef INCLUDE_COMM_PACKET_PACKET_SERVER_HPP_
#define INCLUDE_COMM_PACKET_PACKET_SERVER_HPP_
#include "common.hpp"
#include "comm/packet/client_pool.hpp"
#include "types/repertory.hpp"
#include "utils/utils.hpp"
using namespace boost::asio;
@ -29,28 +32,27 @@ using boost::asio::ip::tcp;
namespace repertory {
class packet_server final {
public:
typedef std::function<void(const std::string &clientId)> closed_callback;
typedef client_pool::worker_complete_callback message_complete_callback;
typedef std::function<void(const std::uint32_t &service_flags, const std::string &client_id,
const std::uint64_t &thread_id, const std::string &method,
packet *request, packet &response,
message_complete_callback message_complete)>
message_handler_callback;
using closed_callback = std::function<void(const std::string &)>;
using message_complete_callback = client_pool::worker_complete_callback;
using message_handler_callback = std::function<void(
std::uint32_t, const std::string &, std::uint64_t, const std::string &,
packet *, packet &, message_complete_callback)>;
public:
packet_server(const std::uint16_t &port, std::string token, std::uint8_t pool_size,
closed_callback closed, message_handler_callback message_handler);
packet_server(std::uint16_t port, std::string token, std::uint8_t pool_size,
closed_callback closed,
message_handler_callback message_handler);
~packet_server();
private:
struct connection {
connection(boost::asio::io_service &io_service, tcp::acceptor &acceptor)
: socket(io_service), acceptor(acceptor) {}
connection(boost::asio::io_service &io_service, tcp::acceptor &acceptor_)
: socket(io_service), acceptor(acceptor_) {}
tcp::socket socket;
tcp::acceptor &acceptor;
std::vector<char> buffer;
data_buffer buffer;
std::string client_id;
std::string nonce;
@ -78,12 +80,12 @@ private:
void read_header(std::shared_ptr<connection> c);
void read_packet(std::shared_ptr<connection> c, const std::uint32_t &data_size);
void read_packet(std::shared_ptr<connection> c, std::uint32_t data_size);
void remove_client(connection &c);
void send_response(std::shared_ptr<connection> c, const packet::error_type &result,
packet &response);
void send_response(std::shared_ptr<connection> c,
const packet::error_type &result, packet &response);
};
} // namespace repertory

177
include/comm/s3/s3_comm.hpp Normal file
View File

@ -0,0 +1,177 @@
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#ifndef INCLUDE_COMM_S3_S3_COMM_HPP_
#define INCLUDE_COMM_S3_S3_COMM_HPP_
#if defined(REPERTORY_ENABLE_S3)
#include "comm/i_http_comm.hpp"
#include "comm/i_s3_comm.hpp"
#include "types/repertory.hpp"
namespace repertory {
class app_config;
class s3_comm final : public i_s3_comm {
public:
explicit s3_comm(const app_config &config);
s3_comm(s3_comm &&comm);
~s3_comm() override;
private:
struct cache_entry final {
std::chrono::system_clock::time_point expiration;
directory_item_list items;
void reset_timeout(std::chrono::seconds timeout) {
timeout = std::max(std::chrono::seconds(5u), timeout);
expiration = std::chrono::system_clock::now() + timeout;
}
};
private:
const app_config &config_;
s3_config s3_config_;
std::unique_ptr<i_http_comm> s3_client_;
private:
mutable std::recursive_mutex cached_directories_mutex_;
mutable std::unordered_map<std::string, cache_entry> cached_directories_;
protected:
bool active_ = true;
private:
void clear_expired_directories();
[[nodiscard]] auto
get_cached_directory_item_count(const std::string &api_path,
std::size_t &count) const -> bool;
[[nodiscard]] auto
get_cached_directory_items(const std::string &api_path,
meta_provider_callback meta_provider,
directory_item_list &list) const -> bool;
[[nodiscard]] auto get_cached_file_exists(const std::string &api_path) const
-> bool;
[[nodiscard]] auto grab_directory_items(const std::string &api_path,
meta_provider_callback meta_provider,
directory_item_list &list) const
-> api_error;
void raise_begin(const std::string &function_name,
const std::string &api_path) const;
[[nodiscard]] auto raise_end(const std::string &function_name,
const std::string &api_path,
const api_error &error, long code) const
-> api_error;
void remove_cached_directory(const std::string &api_path);
void set_cached_directory_items(const std::string &api_path,
directory_item_list list) const;
public:
[[nodiscard]] auto create_directory(const std::string &api_path)
-> api_error override;
[[nodiscard]] auto directory_exists(const std::string &api_path) const
-> api_error override;
[[nodiscard]] auto file_exists(const std::string &api_path,
const get_key_callback &get_key) const
-> api_error override;
[[nodiscard]] auto
get_directory_item_count(const std::string &api_path,
meta_provider_callback meta_provider) const
-> std::size_t override;
[[nodiscard]] auto get_directory_items(const std::string &api_path,
meta_provider_callback meta_provider,
directory_item_list &list) const
-> api_error override;
[[nodiscard]] auto get_directory_list(api_file_list &list) const
-> api_error override;
[[nodiscard]] auto get_file(const std::string &api_path,
const get_key_callback &get_key,
const get_name_callback &get_name,
const get_token_callback &get_token,
api_file &file) const -> api_error override;
[[nodiscard]] auto
get_file_list(const get_api_file_token_callback &get_api_file_token,
const get_name_callback &get_name, api_file_list &list) const
-> api_error override;
[[nodiscard]] auto get_object_list(std::vector<directory_item> &list) const
-> api_error override;
[[nodiscard]] auto get_object_name(const std::string &api_path,
const get_key_callback &getKey) const
-> std::string override;
[[nodiscard]] auto get_s3_config() -> s3_config override {
return s3_config_;
}
[[nodiscard]] auto get_s3_config() const -> s3_config override {
return s3_config_;
}
[[nodiscard]] auto is_online() const -> bool override {
// TODO implement this
return true;
}
[[nodiscard]] auto read_file_bytes(
const std::string &api_path, std::size_t size, std::uint64_t offset,
data_buffer &data, const get_key_callback &get_key,
const get_size_callback &get_size, const get_token_callback &get_token,
stop_type &stop_requested) const -> api_error override;
[[nodiscard]] auto remove_directory(const std::string &api_path)
-> api_error override;
[[nodiscard]] auto remove_file(const std::string &api_path,
const get_key_callback &get_key)
-> api_error override;
[[nodiscard]] auto rename_file(const std::string &api_path,
const std::string &new_api_path)
-> api_error override;
[[nodiscard]] auto
upload_file(const std::string &api_path, const std::string &source_path,
const std::string &encryption_token,
const get_key_callback &get_key, const set_key_callback &set_key,
stop_type &stop_requested) -> api_error override;
};
} // namespace repertory
#endif // REPERTORY_ENABLE_S3
#endif // INCLUDE_COMM_S3_S3_COMM_HPP_

117
include/comm/s3/s3_requests.hpp Normal file
View File

@ -0,0 +1,117 @@
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#ifndef INCLUDE_COMM_S3_S3_REQUESTS_HPP_
#define INCLUDE_COMM_S3_S3_REQUESTS_HPP_
#if defined(REPERTORY_ENABLE_S3)
#include "comm/s3/s3_requests_curl.hpp"
#include "types/repertory.hpp"
#include "types/s3.hpp"
#include "utils/string_utils.hpp"
namespace repertory {
template <typename client_type>
[[nodiscard]] inline auto
create_directory_object_request(client_type &client, const s3_config &config,
const std::string &object_name,
long &response_code) -> bool {
return create_directory_object_request_impl(client, config, object_name,
response_code);
}
template <typename client_type>
[[nodiscard]] inline auto delete_object_request(client_type &client,
const s3_config &config,
const std::string &object_name,
long &response_code) -> bool {
return delete_object_request_impl(client, config, object_name, response_code);
}
template <typename client_type>
[[nodiscard]] inline auto
head_object_request(client_type &client, const s3_config &config,
const std::string &object_name, head_object_result &result,
long &response_code) -> bool {
return head_object_request_impl(client, config, object_name, result,
response_code);
}
template <typename client_type>
[[nodiscard]] inline auto
list_directories_request(client_type &client, const s3_config &config,
list_directories_result &result, long &response_code)
-> bool {
return list_directories_request_impl(client, config, result, response_code);
}
template <typename client_type>
[[nodiscard]] inline auto
list_files_request(client_type &client, const s3_config &config,
const get_api_file_token_callback &get_api_file_token,
const get_name_callback &get_name, list_files_result &result,
long &response_code) -> bool {
return list_files_request_impl(client, config, get_api_file_token, get_name,
result, response_code);
}
template <typename client_type>
[[nodiscard]] inline auto list_objects_in_directory_request(
client_type &client, const s3_config &config,
const std::string &object_name, meta_provider_callback meta_provider,
list_objects_result &result, long &response_code) -> bool {
return list_objects_in_directory_request_impl(
client, config, object_name, meta_provider, result, response_code);
}
template <typename client_type>
[[nodiscard]] inline auto
list_objects_request(client_type &client, const s3_config &config,
list_objects_result &result, long &response_code) -> bool {
return list_objects_request_impl(client, config, result, response_code);
}
template <typename client_type>
[[nodiscard]] inline auto
put_object_request(client_type &client, const s3_config &config,
std::string object_name, const std::string &source_path,
const std::string &encryption_token,
get_key_callback get_key, set_key_callback set_key,
long &response_code, stop_type &stop_requested) -> bool {
return put_object_request_impl(client, config, object_name, source_path,
encryption_token, get_key, set_key,
response_code, stop_requested);
}
template <typename client_type>
[[nodiscard]] inline auto
read_object_request(client_type &client, const s3_config &config,
const std::string &object_name, std::size_t size,
std::uint64_t offset, data_buffer &data,
long &response_code, stop_type &stop_requested) -> bool {
return read_object_request_impl(client, config, object_name, size, offset,
data, response_code, stop_requested);
}
} // namespace repertory
#endif
#endif // INCLUDE_COMM_S3_S3_REQUESTS_HPP_

83
include/comm/s3/s3_requests_curl.hpp Normal file
View File

@ -0,0 +1,83 @@
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#ifndef INCLUDE_COMM_S3_S3_REQUESTS_CURL_HPP_
#define INCLUDE_COMM_S3_S3_REQUESTS_CURL_HPP_
#if defined(REPERTORY_ENABLE_S3)
#include "comm/i_http_comm.hpp"
#include "types/repertory.hpp"
#include "types/s3.hpp"
#include "utils/string_utils.hpp"
#include "utils/utils.hpp"
namespace repertory {
[[nodiscard]] auto create_directory_object_request_impl(
i_http_comm &client, const s3_config &config,
const std::string &object_name, long &response_code) -> bool;
[[nodiscard]] auto delete_object_request_impl(i_http_comm &client,
const s3_config &config,
const std::string &object_name,
long &response_code) -> bool;
[[nodiscard]] auto head_object_request_impl(i_http_comm &client,
const s3_config &config,
const std::string &object_name,
head_object_result &result,
long &response_code) -> bool;
[[nodiscard]] auto
list_directories_request_impl(i_http_comm &client, const s3_config &config,
list_directories_result &result,
long &response_code) -> bool;
[[nodiscard]] auto
list_files_request_impl(i_http_comm &client, const s3_config &config,
const get_api_file_token_callback &get_api_file_token,
const get_name_callback &get_name,
list_files_result &result, long &response_code) -> bool;
[[nodiscard]] auto list_objects_in_directory_request_impl(
i_http_comm &client, const s3_config &config,
const std::string &object_name, meta_provider_callback meta_provider,
list_objects_result &result, long &response_code) -> bool;
[[nodiscard]] auto list_objects_request_impl(i_http_comm &client,
const s3_config &config,
list_objects_result &result,
long &response_code) -> bool;
[[nodiscard]] auto
put_object_request_impl(i_http_comm &client, const s3_config &config,
std::string object_name, const std::string &source_path,
const std::string &encryption_token,
get_key_callback get_key, set_key_callback set_key,
long &response_code, stop_type &stop_requested) -> bool;
[[nodiscard]] auto read_object_request_impl(
i_http_comm &client, const s3_config &config,
const std::string &object_name, std::size_t size, std::uint64_t offset,
data_buffer &data, long &response_code, stop_type &stop_requested) -> bool;
} // namespace repertory
#endif // REPERTORY_ENABLE_S3
#endif // INCLUDE_COMM_S3_S3_REQUESTS_CURL_HPP_