BlockStorage/repertory
1
0
Fork 0
Code Issues 8 Pull Requests Releases 8 Wiki Activity

v2.0.6-release (#50)
All checks were successful
BlockStorage/repertory/pipeline/head This commit looks good
Details

Browse Source 
Reviewed-on: #50
This commit is contained in:
Scott E. Graves
2025-07-25 07:51:03 -05:00
parent 62555e6125
commit 87d336141c
311 changed files with 13495 additions and 12079 deletions
Download Patch File Download Diff File Expand all files Collapse all files

217
repertory/librepertory/include/comm/curl/curl_comm.hpp
View File

@@ -23,6 +23,7 @@
#define REPERTORY_INCLUDE_COMM_CURL_CURL_COMM_HPP_
#include "app_config.hpp"
#include "comm/curl/curl_shared.hpp"
#include "comm/curl/multi_request.hpp"
#include "comm/i_http_comm.hpp"
#include "events/event_system.hpp"
@@ -48,14 +49,12 @@ private:
static const write_callback write_data;
static const write_callback write_headers;
static constexpr std::uint8_t retry_request_count{5U};
private:
std::optional<host_config> host_config_;
std::optional<s3_config> s3_config_;
private:
bool use_s3_path_style_{false};
public:
[[nodiscard]] static auto create_curl() -> CURL *;
@@ -67,8 +66,7 @@ public:
const host_config &cfg)
-> std::string;
[[nodiscard]] static auto create_host_config(const s3_config &cfg,
bool use_s3_path_style)
[[nodiscard]] static auto create_host_config(const s3_config &cfg)
-> host_config;
[[nodiscard]] static auto url_encode(CURL *curl, const std::string &data,
@@ -139,107 +137,128 @@ public:
long &response_code, stop_type &stop_requested) -> bool {
REPERTORY_USES_FUNCTION_NAME();
if (request.decryption_token.has_value() &&
not request.decryption_token.value().empty()) {
return make_encrypted_request(cfg, request, response_code,
stop_requested);
}
response_code = 0;
auto *curl = create_curl();
if (not request.set_method(curl, stop_requested)) {
return false;
}
if (not cfg.agent_string.empty()) {
curl_easy_setopt(curl, CURLOPT_USERAGENT, cfg.agent_string.c_str());
}
if (request.allow_timeout && cfg.timeout_ms) {
curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, cfg.timeout_ms);
}
std::string range_list{};
if (request.range.has_value()) {
range_list = std::to_string(request.range.value().begin) + '-' +
std::to_string(request.range.value().end);
curl_easy_setopt(curl, CURLOPT_RANGE, range_list.c_str());
}
if (request.response_headers.has_value()) {
curl_easy_setopt(curl, CURLOPT_HEADERDATA,
&request.response_headers.value());
curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, write_headers);
}
read_write_info write_info{
{},
[&stop_requested]() -> bool {
return stop_requested || app_config::get_stop_requested();
},
};
if (request.response_handler.has_value()) {
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &write_info);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
}
std::string parameters{};
for (const auto &param : request.query) {
parameters += (parameters.empty() ? '?' : '&') + param.first + '=' +
url_encode(curl, param.second, false);
}
if (not cfg.api_password.empty()) {
curl_easy_setopt(curl, CURLOPT_USERNAME, cfg.api_user.c_str());
curl_easy_setopt(curl, CURLOPT_PASSWORD, cfg.api_password.c_str());
} else if (not cfg.api_user.empty()) {
curl_easy_setopt(curl, CURLOPT_USERNAME, cfg.api_user.c_str());
}
if (request.aws_service.has_value()) {
curl_easy_setopt(curl, CURLOPT_AWS_SIGV4,
request.aws_service.value().c_str());
}
curl_slist *header_list{nullptr};
if (not request.headers.empty()) {
for (const auto &header : request.headers) {
header_list = curl_slist_append(
header_list,
fmt::format("{}: {}", header.first, header.second).c_str());
}
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, header_list);
}
auto url = construct_url(curl, request.get_path(), cfg) + parameters;
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
multi_request curl_request(curl, stop_requested);
CURLcode curl_code{};
curl_request.get_result(curl_code, response_code);
const auto do_request = [&]() -> bool {
if (request.decryption_token.has_value() &&
not request.decryption_token.value().empty()) {
return make_encrypted_request(cfg, request, response_code,
stop_requested);
}
if (header_list != nullptr) {
curl_slist_free_all(header_list);
response_code = 0;
auto *curl = create_curl();
if (not request.set_method(curl, stop_requested)) {
return false;
}
if (not cfg.agent_string.empty()) {
curl_easy_setopt(curl, CURLOPT_USERAGENT, cfg.agent_string.c_str());
}
if (request.allow_timeout && cfg.timeout_ms) {
curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, cfg.timeout_ms);
}
std::string range_list{};
if (request.range.has_value()) {
range_list = std::to_string(request.range.value().begin) + '-' +
std::to_string(request.range.value().end);
curl_easy_setopt(curl, CURLOPT_RANGE, range_list.c_str());
}
if (request.response_headers.has_value()) {
curl_easy_setopt(curl, CURLOPT_HEADERDATA,
&request.response_headers.value());
curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, write_headers);
}
read_write_info write_info{
{},
[&stop_requested]() -> bool {
return stop_requested || app_config::get_stop_requested();
},
};
if (request.response_handler.has_value()) {
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &write_info);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
}
std::string parameters{};
for (const auto &param : request.query) {
parameters += (parameters.empty() ? '?' : '&') + param.first + '=' +
url_encode(curl, param.second, false);
}
if (not cfg.api_password.empty()) {
curl_easy_setopt(curl, CURLOPT_USERNAME, cfg.api_user.c_str());
curl_easy_setopt(curl, CURLOPT_PASSWORD, cfg.api_password.c_str());
} else if (not cfg.api_user.empty()) {
curl_easy_setopt(curl, CURLOPT_USERNAME, cfg.api_user.c_str());
}
if (request.aws_service.has_value()) {
curl_easy_setopt(curl, CURLOPT_AWS_SIGV4,
request.aws_service.value().c_str());
}
curl_slist *header_list{nullptr};
if (not request.headers.empty()) {
for (const auto &header : request.headers) {
header_list = curl_slist_append(
header_list,
fmt::format("{}: {}", header.first, header.second).c_str());
}
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, header_list);
}
curl_shared::set_share(curl);
auto url = construct_url(curl, request.get_path(), cfg) + parameters;
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
multi_request curl_request(curl, stop_requested);
curl_code = CURLE_OK;
curl_request.get_result(curl_code, response_code);
if (header_list != nullptr) {
curl_slist_free_all(header_list);
}
if (curl_code != CURLE_OK) {
event_system::instance().raise<curl_error>(curl_code, function_name,
request.get_type(), url);
return false;
}
if (request.response_handler.has_value()) {
request.response_handler.value()(write_info.data, response_code);
}
return true;
};
bool ret{false};
for (std::uint8_t retry = 0U; !ret && retry < retry_request_count;
++retry) {
ret = do_request();
if (ret) {
break;
}
if (curl_code == CURLE_COULDNT_RESOLVE_HOST) {
std::this_thread::sleep_for(1s);
continue;
}
break;
}
if (curl_code != CURLE_OK) {
event_system::instance().raise<curl_error>(curl_code, function_name,
url);
return false;
}
if (request.response_handler.has_value()) {
request.response_handler.value()(write_info.data, response_code);
}
return true;
return ret;
}
public:
void enable_s3_path_style(bool enable) override;
[[nodiscard]] auto make_request(const curl::requests::http_delete &del,
long &response_code,
stop_type &stop_requested) const

67
repertory/librepertory/include/comm/curl/curl_shared.hpp Normal file
View File

@@ -0,0 +1,67 @@
/*
Copyright <2018-2025> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#ifndef REPERTORY_INCLUDE_COMM_CURL_CURL_SHARED_HPP_
#define REPERTORY_INCLUDE_COMM_CURL_CURL_SHARED_HPP_
namespace repertory {
class curl_shared final {
private:
struct curl_sh_deleter final {
void operator()(CURLSH *ptr) {
if (ptr != nullptr) {
curl_share_cleanup(ptr);
}
}
};
using curl_sh_t = std::unique_ptr<CURLSH, curl_sh_deleter>;
public:
curl_shared() = delete;
curl_shared(const curl_shared &) = delete;
curl_shared(curl_shared &&) = delete;
~curl_shared() = delete;
auto operator=(const curl_shared &) -> curl_shared & = delete;
auto operator=(curl_shared &&) -> curl_shared & = delete;
private:
static curl_sh_t cache_;
static std::recursive_mutex mtx_;
private:
static void lock_callback(CURL * /* curl */, curl_lock_data /* data */,
curl_lock_access /* access */, void * /* ptr */);
static void unlock_callback(CURL * /* curl */, curl_lock_data /* data */,
curl_lock_access /* access */, void * /* ptr */);
public:
static void cleanup();
[[nodiscard]] static auto init() -> bool;
static void set_share(CURL *curl);
};
} // namespace repertory
#endif // REPERTORY_INCLUDE_COMM_CURL_DNS_CACHE_HPP_

10
repertory/librepertory/include/comm/curl/requests/http_delete.hpp
View File

@@ -26,11 +26,13 @@
namespace repertory::curl::requests {
struct http_delete final : http_request_base {
~http_delete() override = default;
[[nodiscard]] auto get_type() const -> std::string override {
return "delete";
}
[[nodiscard]] auto
set_method(CURL *curl,
stop_type & /* stop_requested */) const -> bool override {
[[nodiscard]] auto set_method(CURL *curl,
stop_type & /* stop_requested */) const
-> bool override {
curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE");
return true;
}

8
repertory/librepertory/include/comm/curl/requests/http_get.hpp
View File

@@ -33,9 +33,11 @@ struct http_get final : http_request_base {
auto operator=(http_get &&) -> http_get & = default;
~http_get() override = default;
[[nodiscard]] auto
set_method(CURL *curl,
stop_type & /*stop_requested*/) const -> bool override {
[[nodiscard]] auto get_type() const -> std::string override { return "get"; }
[[nodiscard]] auto set_method(CURL *curl,
stop_type & /*stop_requested*/) const
-> bool override {
curl_easy_setopt(curl, CURLOPT_HTTPGET, 1L);
return true;
}

8
repertory/librepertory/include/comm/curl/requests/http_head.hpp
View File

@@ -26,11 +26,11 @@
namespace repertory::curl::requests {
struct http_head final : http_request_base {
~http_head() override = default;
[[nodiscard]] auto get_type() const -> std::string override { return "head"; }
[[nodiscard]] auto
set_method(CURL *curl,
stop_type & /* stop_requested */) const -> bool override {
[[nodiscard]] auto set_method(CURL *curl,
stop_type & /* stop_requested */) const
-> bool override {
curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "HEAD");
curl_easy_setopt(curl, CURLOPT_NOBODY, 1L);
return true;

10
repertory/librepertory/include/comm/curl/requests/http_post.hpp
View File

@@ -26,16 +26,10 @@
namespace repertory::curl::requests {
struct http_post final : http_request_base {
http_post() = default;
http_post(const http_post &) = default;
http_post(http_post &&) = default;
auto operator=(const http_post &) -> http_post & = default;
auto operator=(http_post &&) -> http_post & = default;
~http_post() override = default;
std::optional<nlohmann::json> json;
[[nodiscard]] auto get_type() const -> std::string override { return "post"; }
[[nodiscard]] auto set_method(CURL *curl,
stop_type & /*stop_requested*/) const
-> bool override;

11
repertory/librepertory/include/comm/curl/requests/http_put_file.hpp
View File

@@ -27,18 +27,11 @@
namespace repertory::curl::requests {
struct http_put_file final : http_request_base {
http_put_file() = default;
http_put_file(const http_put_file &) = default;
http_put_file(http_put_file &&) = default;
auto operator=(const http_put_file &) -> http_put_file & = default;
auto operator=(http_put_file &&) -> http_put_file & = default;
~http_put_file() override = default;
std::shared_ptr<utils::encryption::encrypting_reader> reader;
std::string source_path;
[[nodiscard]] auto get_type() const -> std::string override { return "put"; }
[[nodiscard]] auto set_method(CURL *curl, stop_type &stop_requested) const
-> bool override;

2
repertory/librepertory/include/comm/curl/requests/http_request_base.hpp
View File

@@ -61,6 +61,8 @@ struct http_request_base {
[[nodiscard]] virtual auto get_path() const -> std::string { return path; }
[[nodiscard]] virtual auto get_type() const -> std::string = 0;
[[nodiscard]] virtual auto set_method(CURL *curl,
stop_type &stop_requested) const
-> bool = 0;

27
repertory/librepertory/include/comm/i_http_comm.hpp
View File

@@ -34,28 +34,29 @@ struct i_http_comm {
INTERFACE_SETUP(i_http_comm);
public:
virtual void enable_s3_path_style(bool enable) = 0;
[[nodiscard]] virtual auto
make_request(const curl::requests::http_delete &del, long &response_code,
stop_type &stop_requested) const -> bool = 0;
[[nodiscard]] virtual auto
make_request(const curl::requests::http_get &get, long &response_code,
stop_type &stop_requested) const -> bool = 0;
[[nodiscard]] virtual auto make_request(const curl::requests::http_get &get,
long &response_code,
stop_type &stop_requested) const
-> bool = 0;
[[nodiscard]] virtual auto
make_request(const curl::requests::http_head &head, long &response_code,
stop_type &stop_requested) const -> bool = 0;
[[nodiscard]] virtual auto make_request(const curl::requests::http_head &head,
long &response_code,
stop_type &stop_requested) const
-> bool = 0;
[[nodiscard]] virtual auto
make_request(const curl::requests::http_post &post, long &response_code,
stop_type &stop_requested) const -> bool = 0;
[[nodiscard]] virtual auto make_request(const curl::requests::http_post &post,
long &response_code,
stop_type &stop_requested) const
-> bool = 0;
[[nodiscard]] virtual auto
make_request(const curl::requests::http_put_file &put_file,
long &response_code,
stop_type &stop_requested) const -> bool = 0;
long &response_code, stop_type &stop_requested) const
-> bool = 0;
};
} // namespace repertory

2
repertory/librepertory/include/comm/packet/client_pool.hpp
View File

@@ -94,7 +94,7 @@ private:
bool shutdown_ = false;
private:
static constexpr const auto min_pool_size = 10U;
static constexpr auto min_pool_size = 10U;
public:
void execute(const std::string &client_id, std::uint64_t thread_id,