BlockStorage/repertory
1
0
Fork 0
Code Issues 7 Pull Requests Releases 6 Wiki Activity

Add support for bucket name in Sia provider #16
All checks were successful
BlockStorage/repertory/pipeline/head This commit looks good
Details

Browse Source
This commit is contained in:
Scott E. Graves 2024-09-28 09:52:57 -05:00
parent 692c92b0df
commit aa92f3d401
5 changed files with 44 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
repertory/librepertory/include/app_config.hpp
View File

@ -98,6 +98,7 @@ private:
std::string cache_directory_; std::string cache_directory_;
host_config hc_; host_config hc_;
s3_config s3_config_; s3_config s3_config_;
sia_config sia_config_{"default"};
std::uint64_t version_{REPERTORY_CONFIG_VERSION}; std::uint64_t version_{REPERTORY_CONFIG_VERSION};
std::string log_directory_; std::string log_directory_;
mutable std::recursive_mutex read_write_mutex_; mutable std::recursive_mutex read_write_mutex_;
@ -295,6 +296,10 @@ public:
[[nodiscard]] auto get_s3_config() const -> s3_config { return s3_config_; } [[nodiscard]] auto get_s3_config() const -> s3_config { return s3_config_; }
[[nodiscard]] auto get_sia_config() const -> sia_config {
return sia_config_;
}
[[nodiscard]] auto get_value_by_name(const std::string &name) -> std::string; [[nodiscard]] auto get_value_by_name(const std::string &name) -> std::string;
[[nodiscard]] auto get_version() const -> std::uint64_t { return version_; } [[nodiscard]] auto get_version() const -> std::uint64_t { return version_; }

5
repertory/librepertory/include/types/repertory.hpp
View File

@ -270,7 +270,6 @@ from_json(const json &j, host_config &hc) {
struct s3_config final { struct s3_config final {
std::string access_key{}; std::string access_key{};
std::string bucket{}; std::string bucket{};
std::uint16_t cache_timeout_secs{60U};
std::string encryption_token{}; std::string encryption_token{};
std::string region{"any"}; std::string region{"any"};
std::string secret_key{}; std::string secret_key{};
@ -280,6 +279,10 @@ struct s3_config final {
bool use_region_in_url{false}; bool use_region_in_url{false};
}; };
struct sia_config final {
std::string bucket{};
};
using api_file_list = std::vector<api_file>; using api_file_list = std::vector<api_file>;
using api_file_provider_callback = std::function<void(api_file &)>; using api_file_provider_callback = std::function<void(api_file &)>;
using api_item_added_callback = std::function<api_error(bool, api_file &)>; using api_item_added_callback = std::function<api_error(bool, api_file &)>;

32
repertory/librepertory/src/app_config.cpp
View File

@ -259,7 +259,6 @@ auto app_config::get_json() const -> json {
{ {
{"AccessKey", s3_config_.access_key}, {"AccessKey", s3_config_.access_key},
{"Bucket", s3_config_.bucket}, {"Bucket", s3_config_.bucket},
{"CacheTimeoutSeconds", s3_config_.cache_timeout_secs},
{"EncryptionToken", s3_config_.encryption_token}, {"EncryptionToken", s3_config_.encryption_token},
{"Region", s3_config_.region}, {"Region", s3_config_.region},
{"SecretKey", s3_config_.secret_key}, {"SecretKey", s3_config_.secret_key},
@ -268,6 +267,10 @@ auto app_config::get_json() const -> json {
{"UsePathStyle", s3_config_.use_path_style}, {"UsePathStyle", s3_config_.use_path_style},
{"UseRegionInURL", s3_config_.use_region_in_url}, {"UseRegionInURL", s3_config_.use_region_in_url},
}}, }},
{"SiaConfig",
{
{"Bucket", sia_config_.bucket},
}},
{"Version", version_}}; {"Version", version_}};
if (prov_ == provider_type::encrypt) { if (prov_ == provider_type::encrypt) {
@ -482,9 +485,6 @@ auto app_config::get_value_by_name(const std::string &name) -> std::string {
if (name == "S3Config.EncryptionToken") { if (name == "S3Config.EncryptionToken") {
return s3_config_.encryption_token; return s3_config_.encryption_token;
} }
if (name == "S3Config.CacheTimeoutSeconds") {
return std::to_string(s3_config_.cache_timeout_secs);
}
if (name == "S3Config.Region") { if (name == "S3Config.Region") {
return s3_config_.region; return s3_config_.region;
} }
@ -503,6 +503,9 @@ auto app_config::get_value_by_name(const std::string &name) -> std::string {
if (name == "S3Config.TimeoutMs") { if (name == "S3Config.TimeoutMs") {
return std::to_string(s3_config_.timeout_ms); return std::to_string(s3_config_.timeout_ms);
} }
if (name == "SiaConfig.Bucket") {
return sia_config_.bucket;
}
} catch (const std::exception &e) { } catch (const std::exception &e) {
utils::error::raise_error(function_name, e, "exception occurred"); utils::error::raise_error(function_name, e, "exception occurred");
} }
@ -582,8 +585,6 @@ auto app_config::load() -> bool {
auto s3_cfg = s3_config_; auto s3_cfg = s3_config_;
get_value(s3_config_json, "AccessKey", s3_cfg.access_key, ret); get_value(s3_config_json, "AccessKey", s3_cfg.access_key, ret);
get_value(s3_config_json, "Bucket", s3_cfg.bucket, ret); get_value(s3_config_json, "Bucket", s3_cfg.bucket, ret);
get_value(s3_config_json, "CacheTimeoutSeconds",
s3_cfg.cache_timeout_secs, ret);
get_value(s3_config_json, "EncryptionToken", get_value(s3_config_json, "EncryptionToken",
s3_cfg.encryption_token, ret); s3_cfg.encryption_token, ret);
get_value(s3_config_json, "Region", s3_cfg.region, ret); get_value(s3_config_json, "Region", s3_cfg.region, ret);
@ -599,6 +600,15 @@ auto app_config::load() -> bool {
ret = false; ret = false;
} }
if (json_document.find("SiaConfig") != json_document.end()) {
auto sia_config_json = json_document["SiaConfig"];
auto sia_cfg = sia_config_;
get_value(sia_config_json, "Bucket", sia_cfg.bucket, ret);
sia_config_ = sia_cfg;
} else {
ret = false;
}
get_value(json_document, "ReadAheadCount", read_ahead_count_, ret); get_value(json_document, "ReadAheadCount", read_ahead_count_, ret);
get_value(json_document, "RingBufferFileSize", ring_buffer_file_size_, get_value(json_document, "RingBufferFileSize", ring_buffer_file_size_,
ret); ret);
@ -888,12 +898,6 @@ auto app_config::set_value_by_name(const std::string &name,
set_value(s3_config_.bucket, value); set_value(s3_config_.bucket, value);
return s3_config_.bucket; return s3_config_.bucket;
} }
if (name == "S3Config.CacheTimeoutSeconds") {
const auto timeout =
std::max(std::uint16_t(5U), utils::string::to_uint16(value));
set_value(s3_config_.cache_timeout_secs, timeout);
return std::to_string(s3_config_.cache_timeout_secs);
}
if (name == "S3Config.Region") { if (name == "S3Config.Region") {
set_value(s3_config_.region, value); set_value(s3_config_.region, value);
return s3_config_.region; return s3_config_.region;
@ -922,6 +926,10 @@ auto app_config::set_value_by_name(const std::string &name,
set_value(s3_config_.encryption_token, value); set_value(s3_config_.encryption_token, value);
return s3_config_.encryption_token; return s3_config_.encryption_token;
} }
if (name == "SiaConfig.Bucket") {
set_value(sia_config_.bucket, value);
return sia_config_.bucket;
}
} catch (const std::exception &e) { } catch (const std::exception &e) {
utils::error::raise_error(function_name, e, "exception occurred"); utils::error::raise_error(function_name, e, "exception occurred");
} }

13
repertory/librepertory/src/providers/sia/sia_provider.cpp
View File

@ -61,6 +61,7 @@ auto sia_provider::create_directory_impl(
curl::requests::http_put_file put_file{}; curl::requests::http_put_file put_file{};
put_file.allow_timeout = true; put_file.allow_timeout = true;
put_file.path = "/api/worker/objects" + api_path + "/"; put_file.path = "/api/worker/objects" + api_path + "/";
put_file.query["bucket"] = get_config().get_sia_config().bucket;
long response_code{}; long response_code{};
stop_type stop_requested{}; stop_type stop_requested{};
@ -286,6 +287,7 @@ auto sia_provider::get_object_info(const std::string &api_path,
curl::requests::http_get get{}; curl::requests::http_get get{};
get.allow_timeout = true; get.allow_timeout = true;
get.path = "/api/bus/objects" + api_path; get.path = "/api/bus/objects" + api_path;
get.query["bucket"] = get_config().get_sia_config().bucket;
get.response_handler = [&object_info](const data_buffer &data, get.response_handler = [&object_info](const data_buffer &data,
long response_code) { long response_code) {
@ -328,6 +330,7 @@ auto sia_provider::get_object_list(const std::string &api_path,
curl::requests::http_get get{}; curl::requests::http_get get{};
get.allow_timeout = true; get.allow_timeout = true;
get.path = "/api/bus/objects" + api_path + "/"; get.path = "/api/bus/objects" + api_path + "/";
get.query["bucket"] = get_config().get_sia_config().bucket;
get.response_handler = [&object_list](const data_buffer &data, get.response_handler = [&object_list](const data_buffer &data,
long response_code) { long response_code) {
@ -363,6 +366,7 @@ auto sia_provider::get_total_drive_space() const -> std::uint64_t {
curl::requests::http_get get{}; curl::requests::http_get get{};
get.allow_timeout = true; get.allow_timeout = true;
get.path = "/api/autopilot/config"; get.path = "/api/autopilot/config";
get.query["bucket"] = get_config().get_sia_config().bucket;
json config_data{}; json config_data{};
get.response_handler = [&config_data](const data_buffer &data, get.response_handler = [&config_data](const data_buffer &data,
@ -401,6 +405,7 @@ auto sia_provider::get_used_drive_space_impl() const -> std::uint64_t {
curl::requests::http_get get{}; curl::requests::http_get get{};
get.allow_timeout = true; get.allow_timeout = true;
get.path = "/api/bus/stats/objects"; get.path = "/api/bus/stats/objects";
get.query["bucket"] = get_config().get_sia_config().bucket;
json object_data{}; json object_data{};
get.response_handler = [&object_data](const data_buffer &data, get.response_handler = [&object_data](const data_buffer &data,
@ -501,6 +506,7 @@ auto sia_provider::is_online() const -> bool {
curl::requests::http_get get{}; curl::requests::http_get get{};
get.allow_timeout = true; get.allow_timeout = true;
get.path = "/api/bus/consensus/state"; get.path = "/api/bus/consensus/state";
get.query["bucket"] = get_config().get_sia_config().bucket;
json state_data{}; json state_data{};
get.response_handler = [&state_data](const data_buffer &data, get.response_handler = [&state_data](const data_buffer &data,
@ -543,6 +549,7 @@ auto sia_provider::read_file_bytes(const std::string &api_path,
curl::requests::http_get get{}; curl::requests::http_get get{};
get.path = "/api/worker/objects" + api_path; get.path = "/api/worker/objects" + api_path;
get.query["bucket"] = get_config().get_sia_config().bucket;
get.range = {{ get.range = {{
offset, offset,
offset + size - 1U, offset + size - 1U,
@ -599,6 +606,7 @@ auto sia_provider::remove_directory_impl(const std::string &api_path)
curl::requests::http_delete del{}; curl::requests::http_delete del{};
del.allow_timeout = true; del.allow_timeout = true;
del.path = "/api/bus/objects" + api_path + "/"; del.path = "/api/bus/objects" + api_path + "/";
del.query["bucket"] = get_config().get_sia_config().bucket;
long response_code{}; long response_code{};
stop_type stop_requested{}; stop_type stop_requested{};
@ -626,6 +634,7 @@ auto sia_provider::remove_file_impl(const std::string &api_path) -> api_error {
curl::requests::http_delete del{}; curl::requests::http_delete del{};
del.allow_timeout = true; del.allow_timeout = true;
del.path = "/api/bus/objects" + api_path; del.path = "/api/bus/objects" + api_path;
del.query["bucket"] = get_config().get_sia_config().bucket;
long response_code{}; long response_code{};
stop_type stop_requested{}; stop_type stop_requested{};
@ -653,12 +662,13 @@ auto sia_provider::rename_file(const std::string &from_api_path,
}; };
curl::requests::http_post post{}; curl::requests::http_post post{};
post.path = "/api/bus/objects/rename";
post.json = nlohmann::json({ post.json = nlohmann::json({
{"from", from_api_path}, {"from", from_api_path},
{"to", to_api_path}, {"to", to_api_path},
{"mode", "single"}, {"mode", "single"},
}); });
post.path = "/api/bus/objects/rename";
post.query["bucket"] = get_config().get_sia_config().bucket;
long response_code{}; long response_code{};
stop_type stop_requested{}; stop_type stop_requested{};
@ -701,6 +711,7 @@ auto sia_provider::upload_file_impl(const std::string &api_path,
curl::requests::http_put_file put_file{}; curl::requests::http_put_file put_file{};
put_file.path = "/api/worker/objects" + api_path; put_file.path = "/api/worker/objects" + api_path;
put_file.query["bucket"] = get_config().get_sia_config().bucket;
put_file.source_path = source_path; put_file.source_path = source_path;
long response_code{}; long response_code{};

12
repertory/repertory_test/src/config_test.cpp
View File

@ -98,6 +98,9 @@ const auto DEFAULT_SIA_CONFIG = "{\n"
" },\n" " },\n"
" \"RetryReadCount\": 6,\n" " \"RetryReadCount\": 6,\n"
" \"RingBufferFileSize\": 512,\n" " \"RingBufferFileSize\": 512,\n"
" \"SiaConfig\": {\n"
" \"Bucket\": \"default\"\n"
" },\n"
" \"Version\": " + " \"Version\": " +
std::to_string(REPERTORY_CONFIG_VERSION) + std::to_string(REPERTORY_CONFIG_VERSION) +
"\n" "\n"
@ -815,13 +818,4 @@ TEST_F(config_test, retry_read_count_minimum_value) {
EXPECT_EQ(2, config.get_retry_read_count()); EXPECT_EQ(2, config.get_retry_read_count());
} }
} }
TEST_F(config_test, cache_timeout_seconds_minimum_value) {
{
app_config config(provider_type::s3, s3_directory);
EXPECT_FALSE(
config.set_value_by_name("S3Config.CacheTimeoutSeconds", "1").empty());
EXPECT_EQ(std::uint16_t(5U), config.get_s3_config().cache_timeout_secs);
}
}
} // namespace repertory } // namespace repertory