BlockStorage/repertory
1
0
Fork 0
Code Issues 8 Pull Requests Releases 6 Wiki Activity

refactor app config

Browse Source
This commit is contained in:
Scott E. Graves 2024-12-20 13:12:18 -06:00
parent 2e858bdd5a
commit 1c2759f2d7
7 changed files with 954 additions and 753 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CHANGELOG.md
View File

@ -2,6 +2,10 @@
## v2.0.2-rc ## v2.0.2-rc
### BREAKING CHANGES
* Refactored `config.json` - will need to verify configuration settings prior to mounting
### Issues ### Issues
* \#12 \[Unit Test\] Complete all providers unit tests * \#12 \[Unit Test\] Complete all providers unit tests

73
repertory/librepertory/include/app_config.hpp
View File

@ -71,7 +71,6 @@ private:
#if defined(_WIN32) #if defined(_WIN32)
std::atomic<bool> enable_mount_manager_; std::atomic<bool> enable_mount_manager_;
#endif // defined(_WIN32) #endif // defined(_WIN32)
std::atomic<bool> enable_remote_mount_;
std::atomic<event_level> event_level_; std::atomic<event_level> event_level_;
std::atomic<std::uint32_t> eviction_delay_mins_; std::atomic<std::uint32_t> eviction_delay_mins_;
std::atomic<bool> eviction_uses_accessed_time_; std::atomic<bool> eviction_uses_accessed_time_;
@ -84,9 +83,6 @@ private:
std::atomic<std::uint16_t> online_check_retry_secs_; std::atomic<std::uint16_t> online_check_retry_secs_;
std::atomic<std::uint16_t> orphaned_file_retention_days_; std::atomic<std::uint16_t> orphaned_file_retention_days_;
atomic<download_type> preferred_download_type_; atomic<download_type> preferred_download_type_;
std::atomic<std::uint8_t> remote_client_pool_size_;
std::atomic<std::uint16_t> remote_api_port_;
atomic<std::string> remote_encryption_token_;
std::atomic<std::uint16_t> retry_read_count_; std::atomic<std::uint16_t> retry_read_count_;
std::atomic<std::uint16_t> ring_buffer_file_size_; std::atomic<std::uint16_t> ring_buffer_file_size_;
std::atomic<std::uint16_t> task_wait_ms_; std::atomic<std::uint16_t> task_wait_ms_;
@ -99,47 +95,25 @@ private:
std::string log_directory_; std::string log_directory_;
mutable std::recursive_mutex read_write_mutex_; mutable std::recursive_mutex read_write_mutex_;
atomic<remote::remote_config> remote_config_; atomic<remote::remote_config> remote_config_;
atomic<remote::remote_mount> remote_mount_;
atomic<s3_config> s3_config_; atomic<s3_config> s3_config_;
atomic<sia_config> sia_config_; atomic<sia_config> sia_config_;
std::unordered_map<std::string, std::function<std::string()>>
value_get_lookup_;
std::unordered_map<std::string,
std::function<std::string(const std::string &)>>
value_set_lookup_;
std::uint64_t version_{REPERTORY_CONFIG_VERSION}; std::uint64_t version_{REPERTORY_CONFIG_VERSION};
private: private:
template <typename dest> template <typename dest>
auto get_value(const json &data, const std::string &name, dest &dst, auto get_value(const json &data, const std::string &name, dest &dst,
bool &success) -> bool { bool &success) -> bool;
REPERTORY_USES_FUNCTION_NAME();
auto ret{false};
try {
if (data.find(name) != data.end()) {
data.at(name).get_to(dst);
ret = true;
} else {
success = false;
}
} catch (const std::exception &ex) {
utils::error::raise_error(function_name, ex, "exception occurred");
success = false;
ret = false;
}
return ret;
}
[[nodiscard]] auto load() -> bool; [[nodiscard]] auto load() -> bool;
template <typename dest, typename source> template <typename dest, typename source>
auto set_value(dest &dst, const source &src) -> bool { auto set_value(dest &dst, const source &src) -> bool;
auto ret{false};
if (dst != src) {
dst = src;
config_changed_ = true;
save();
ret = true;
}
return ret;
}
public: public:
[[nodiscard]] auto get_api_auth() const -> std::string { return api_auth_; } [[nodiscard]] auto get_api_auth() const -> std::string { return api_auth_; }
@ -189,10 +163,6 @@ public:
} }
#endif // defined(_WIN32) #endif // defined(_WIN32)
[[nodiscard]] auto get_enable_remote_mount() const -> bool {
return enable_remote_mount_;
}
[[nodiscard]] auto get_event_level() const -> event_level { [[nodiscard]] auto get_event_level() const -> event_level {
return event_level_; return event_level_;
} }
@ -254,21 +224,12 @@ public:
return prov_; return prov_;
} }
[[nodiscard]] auto get_remote_client_pool_size() const -> std::uint8_t {
return std::max(static_cast<std::uint8_t>(5U),
remote_client_pool_size_.load());
}
[[nodiscard]] auto get_remote_api_port() const -> std::uint16_t {
return remote_api_port_;
}
[[nodiscard]] auto get_remote_config() const -> remote::remote_config { [[nodiscard]] auto get_remote_config() const -> remote::remote_config {
return remote_config_; return remote_config_;
} }
[[nodiscard]] auto get_remote_encryption_token() const -> std::string { [[nodiscard]] auto get_remote_mount() const -> remote::remote_mount {
return remote_encryption_token_; return remote_mount_;
} }
[[nodiscard]] auto get_retry_read_count() const -> std::uint16_t { [[nodiscard]] auto get_retry_read_count() const -> std::uint16_t {
@ -337,8 +298,6 @@ public:
} }
#endif // defined(_WIN32) #endif // defined(_WIN32)
void set_enable_remote_mount(bool enable_remote_mount);
void set_event_level(const event_level &level) { void set_event_level(const event_level &level) {
if (set_value(event_level_, level)) { if (set_value(event_level_, level)) {
event_system::instance().raise<event_level_changed>( event_system::instance().raise<event_level_changed>(
@ -394,23 +353,13 @@ public:
set_value(preferred_download_type_, type); set_value(preferred_download_type_, type);
} }
void set_remote_client_pool_size(std::uint8_t remote_client_pool_size) {
set_value(remote_client_pool_size_, remote_client_pool_size);
}
void set_ring_buffer_file_size(std::uint16_t ring_buffer_file_size) { void set_ring_buffer_file_size(std::uint16_t ring_buffer_file_size) {
set_value(ring_buffer_file_size_, ring_buffer_file_size); set_value(ring_buffer_file_size_, ring_buffer_file_size);
} }
void set_remote_api_port(std::uint16_t remote_api_port) {
set_value(remote_api_port_, remote_api_port);
}
void set_remote_config(remote::remote_config cfg); void set_remote_config(remote::remote_config cfg);
void set_remote_encryption_token(const std::string &remote_encryption_token) { void set_remote_mount(remote::remote_mount cfg);
set_value(remote_encryption_token_, remote_encryption_token);
}
void set_retry_read_count(std::uint16_t retry_read_count) { void set_retry_read_count(std::uint16_t retry_read_count) {
set_value(retry_read_count_, retry_read_count); set_value(retry_read_count_, retry_read_count);

43
repertory/librepertory/include/types/remote.hpp
View File

@ -64,6 +64,31 @@ struct remote_config final {
} }
}; };
struct remote_mount final {
std::uint16_t api_port{};
std::uint8_t client_pool_size{20U};
bool enable{false};
std::string encryption_token;
auto operator==(const remote_mount &cfg) const noexcept -> bool {
if (&cfg == this) {
return true;
}
return api_port == cfg.api_port &&
client_pool_size == cfg.client_pool_size && enable == cfg.enable &&
encryption_token == cfg.encryption_token;
}
auto operator!=(const remote_mount &cfg) const noexcept -> bool {
if (&cfg == this) {
return false;
}
return not(cfg == *this);
}
};
using block_count = std::uint64_t; using block_count = std::uint64_t;
using block_size = std::uint32_t; using block_size = std::uint32_t;
using file_handle = std::uint64_t; using file_handle = std::uint64_t;
@ -214,6 +239,24 @@ template <> struct adl_serializer<repertory::remote::remote_config> {
data.at(repertory::JSON_SEND_TIMEOUT_MS).get_to(value.send_timeout_ms); data.at(repertory::JSON_SEND_TIMEOUT_MS).get_to(value.send_timeout_ms);
} }
}; };
template <> struct adl_serializer<repertory::remote::remote_mount> {
static void to_json(json &data,
const repertory::remote::remote_mount &value) {
data[repertory::JSON_API_PORT] = value.api_port;
data[repertory::JSON_CLIENT_POOL_SIZE] = value.client_pool_size;
data[repertory::JSON_ENABLE_REMOTE_MOUNT] = value.enable;
data[repertory::JSON_ENCRYPTION_TOKEN] = value.encryption_token;
}
static void from_json(const json &data,
repertory::remote::remote_mount &value) {
data.at(repertory::JSON_API_PORT).get_to(value.api_port);
data.at(repertory::JSON_CLIENT_POOL_SIZE).get_to(value.client_pool_size);
data.at(repertory::JSON_ENABLE_REMOTE_MOUNT).get_to(value.enable);
data.at(repertory::JSON_ENCRYPTION_TOKEN).get_to(value.encryption_token);
}
};
NLOHMANN_JSON_NAMESPACE_END NLOHMANN_JSON_NAMESPACE_END
#endif // REPERTORY_INCLUDE_TYPES_REMOTE_HPP_ #endif // REPERTORY_INCLUDE_TYPES_REMOTE_HPP_

33
repertory/librepertory/include/types/repertory.hpp
View File

@ -407,19 +407,51 @@ inline constexpr const auto JSON_API_PASSWORD{"ApiPassword"};
inline constexpr const auto JSON_API_PATH{"ApiPath"}; inline constexpr const auto JSON_API_PATH{"ApiPath"};
inline constexpr const auto JSON_API_PORT{"ApiPort"}; inline constexpr const auto JSON_API_PORT{"ApiPort"};
inline constexpr const auto JSON_API_USER{"ApiUser"}; inline constexpr const auto JSON_API_USER{"ApiUser"};
inline constexpr const auto JSON_BACKGROUND_DOWNLOAD_TIMEOUT_SECS{
"ChunkDownloaderTimeoutSeconds"};
inline constexpr const auto JSON_BUCKET{"Bucket"}; inline constexpr const auto JSON_BUCKET{"Bucket"};
inline constexpr const auto JSON_CLIENT_POOL_SIZE{"ClientPoolSize"};
inline constexpr const auto JSON_DATABASE_TYPE{"DatabaseType"};
inline constexpr const auto JSON_DIRECTORY{"Directory"}; inline constexpr const auto JSON_DIRECTORY{"Directory"};
inline constexpr const auto JSON_ENABLE_CHUNK_DOWNLOADER_TIMEOUT{
"EnableChunkDownloaderTimeout"};
inline constexpr const auto JSON_ENABLE_COMM_DURATION_EVENTS{
"EnableCommDurationEvents"};
inline constexpr const auto JSON_ENABLE_DRIVE_EVENTS{"EnableDriveEvents"};
inline constexpr const auto JSON_ENABLE_MOUNT_MANAGER{"EnableMountManager"};
inline constexpr const auto JSON_ENABLE_REMOTE_MOUNT{"Enable"};
inline constexpr const auto JSON_ENCRYPTION_TOKEN{"EncryptionToken"}; inline constexpr const auto JSON_ENCRYPTION_TOKEN{"EncryptionToken"};
inline constexpr const auto JSON_ENCRYPT_CONFIG{"EncryptConfig"}; inline constexpr const auto JSON_ENCRYPT_CONFIG{"EncryptConfig"};
inline constexpr const auto JSON_EVENT_LEVEL{"EventLevel"};
inline constexpr const auto JSON_EVICTION_DELAY_MINS{"EvictionDelayMinutes"};
inline constexpr const auto JSON_EVICTION_USE_ACCESS_TIME{
"EvictionUseAccessedTime"};
inline constexpr const auto JSON_HIGH_FREQ_INTERVAL_SECS{
"HighFreqIntervalSeconds"};
inline constexpr const auto JSON_HOST_CONFIG{"HostConfig"}; inline constexpr const auto JSON_HOST_CONFIG{"HostConfig"};
inline constexpr const auto JSON_HOST_NAME_OR_IP{"HostNameOrIp"}; inline constexpr const auto JSON_HOST_NAME_OR_IP{"HostNameOrIp"};
inline constexpr const auto JSON_LOW_FREQ_INTERVAL_SECS{
"LowFreqIntervalSeconds"};
inline constexpr const auto JSON_MAX_CACHE_SIZE_BYTES{"MaxCacheSizeBytes"};
inline constexpr const auto JSON_MAX_CONNECTIONS{"MaxConnections"}; inline constexpr const auto JSON_MAX_CONNECTIONS{"MaxConnections"};
inline constexpr const auto JSON_MAX_UPLOAD_COUNT{"MaxUploadCount"};
inline constexpr const auto JSON_MED_FREQ_INTERVAL_SECS{
"MedFreqIntervalSeconds"};
inline constexpr const auto JSON_META{"Meta"}; inline constexpr const auto JSON_META{"Meta"};
inline constexpr const auto JSON_ONLINE_CHECK_RETRY_SECS{
"OnlineCheckRetrySeconds"};
inline constexpr const auto JSON_ORPHANED_FILE_RETENTION_DAYS{
"OrphanedFileRetentionDays"};
inline constexpr const auto JSON_PATH{"Path"}; inline constexpr const auto JSON_PATH{"Path"};
inline constexpr const auto JSON_PREFERRED_DOWNLOAD_TYPE{
"PreferredDownloadType"};
inline constexpr const auto JSON_PROTOCOL{"Protocol"}; inline constexpr const auto JSON_PROTOCOL{"Protocol"};
inline constexpr const auto JSON_RECV_TIMEOUT_MS{"ReceiveTimeoutMs"}; inline constexpr const auto JSON_RECV_TIMEOUT_MS{"ReceiveTimeoutMs"};
inline constexpr const auto JSON_REGION{"Region"}; inline constexpr const auto JSON_REGION{"Region"};
inline constexpr const auto JSON_REMOTE_CONFIG{"RemoteConfig"}; inline constexpr const auto JSON_REMOTE_CONFIG{"RemoteConfig"};
inline constexpr const auto JSON_REMOTE_MOUNT{"RemoteMount"};
inline constexpr const auto JSON_RETRY_READ_COUNT{"RetryReadCount"};
inline constexpr const auto JSON_RING_BUFFER_FILE_SIZE{"RingBufferFileSize"};
inline constexpr const auto JSON_S3_CONFIG{"S3Config"}; inline constexpr const auto JSON_S3_CONFIG{"S3Config"};
inline constexpr const auto JSON_SECRET_KEY{"SecretKey"}; inline constexpr const auto JSON_SECRET_KEY{"SecretKey"};
inline constexpr const auto JSON_SEND_TIMEOUT_MS{"SendTimeoutMs"}; inline constexpr const auto JSON_SEND_TIMEOUT_MS{"SendTimeoutMs"};
@ -430,6 +462,7 @@ inline constexpr const auto JSON_TIMEOUT_MS{"TimeoutMs"};
inline constexpr const auto JSON_URL{"URL"}; inline constexpr const auto JSON_URL{"URL"};
inline constexpr const auto JSON_USE_PATH_STYLE{"UsePathStyle"}; inline constexpr const auto JSON_USE_PATH_STYLE{"UsePathStyle"};
inline constexpr const auto JSON_USE_REGION_IN_URL{"UseRegionInURL"}; inline constexpr const auto JSON_USE_REGION_IN_URL{"UseRegionInURL"};
inline constexpr const auto JSON_VERSION{"Version"};
} // namespace repertory } // namespace repertory
NLOHMANN_JSON_NAMESPACE_BEGIN NLOHMANN_JSON_NAMESPACE_BEGIN

1391
repertory/librepertory/src/app_config.cpp
View File

File diff suppressed because it is too large Load Diff

136
repertory/repertory_test/src/config_test.cpp
View File

@ -31,30 +31,34 @@ namespace repertory {
class config_test : public ::testing::Test { class config_test : public ::testing::Test {
public: public:
static console_consumer cs; static console_consumer cs;
static std::atomic<std::uint64_t> idx;
std::string s3_directory{ std::string s3_directory;
utils::path::combine(test::get_test_output_dir(), {"config_test", "s3"})}; std::string sia_directory;
std::string sia_directory{utils::path::combine(test::get_test_output_dir(),
{"config_test", "sia"})};
void SetUp() override { void SetUp() override {
s3_directory = utils::path::combine(test::get_test_output_dir(),
{
"config_test",
"s3",
std::to_string(++idx),
});
sia_directory = utils::path::combine(test::get_test_output_dir(),
{
"config_test",
"sia",
std::to_string(++idx),
});
event_system::instance().start(); event_system::instance().start();
ASSERT_TRUE(
utils::file::directory(
utils::path::combine(test::get_test_output_dir(), {"config_test"}))
.remove_recursively());
} }
void TearDown() override { void TearDown() override { event_system::instance().stop(); }
ASSERT_TRUE(
utils::file::directory(
utils::path::combine(test::get_test_output_dir(), {"config_test"}))
.remove_recursively());
event_system::instance().stop();
}
}; };
console_consumer config_test::cs;
std::atomic<std::uint64_t> config_test::idx{0U};
TEST_F(config_test, api_path) { TEST_F(config_test, api_path) {
std::string original_value; std::string original_value;
{ {
@ -538,19 +542,19 @@ TEST_F(config_test, enable_remote_mount) {
// } // }
// } // }
TEST_F(config_test, remote_api_port) { // TEST_F(config_test, remote_api_port) {
std::uint16_t original_value{}; // std::uint16_t original_value{};
{ // {
app_config config(provider_type::sia, sia_directory); // app_config config(provider_type::sia, sia_directory);
original_value = config.get_remote_api_port(); // original_value = config.get_remote_api_port();
config.set_remote_api_port(original_value + 5); // config.set_remote_api_port(original_value + 5);
EXPECT_EQ(original_value + 5, config.get_remote_api_port()); // EXPECT_EQ(original_value + 5, config.get_remote_api_port());
} // }
{ // {
app_config config(provider_type::sia, sia_directory); // app_config config(provider_type::sia, sia_directory);
EXPECT_EQ(original_value + 5, config.get_remote_api_port()); // EXPECT_EQ(original_value + 5, config.get_remote_api_port());
} // }
} // }
// TEST_F(config_test, remote_receive_timeout_secs) { // TEST_F(config_test, remote_receive_timeout_secs) {
// std::uint16_t original_value{}; // std::uint16_t original_value{};
@ -580,43 +584,43 @@ TEST_F(config_test, remote_api_port) {
// } // }
// } // }
TEST_F(config_test, remote_encryption_token) { // TEST_F(config_test, remote_encryption_token) {
{ // {
app_config config(provider_type::sia, sia_directory); // app_config config(provider_type::sia, sia_directory);
config.set_remote_encryption_token("myToken"); // config.set_remote_encryption_token("myToken");
EXPECT_STREQ("myToken", config.get_remote_encryption_token().c_str()); // EXPECT_STREQ("myToken", config.get_remote_encryption_token().c_str());
} // }
{ // {
app_config config(provider_type::sia, sia_directory); // app_config config(provider_type::sia, sia_directory);
EXPECT_STREQ("myToken", config.get_remote_encryption_token().c_str()); // EXPECT_STREQ("myToken", config.get_remote_encryption_token().c_str());
} // }
} // }
//
TEST_F(config_test, remote_client_pool_size) { // TEST_F(config_test, remote_client_pool_size) {
std::uint8_t original_value{}; // std::uint8_t original_value{};
{ // {
app_config config(provider_type::sia, sia_directory); // app_config config(provider_type::sia, sia_directory);
original_value = config.get_remote_client_pool_size(); // original_value = config.get_remote_client_pool_size();
config.set_remote_client_pool_size(original_value + 5); // config.set_remote_client_pool_size(original_value + 5);
EXPECT_EQ(original_value + 5, config.get_remote_client_pool_size()); // EXPECT_EQ(original_value + 5, config.get_remote_client_pool_size());
} // }
{ // {
app_config config(provider_type::sia, sia_directory); // app_config config(provider_type::sia, sia_directory);
EXPECT_EQ(original_value + 5, config.get_remote_client_pool_size()); // EXPECT_EQ(original_value + 5, config.get_remote_client_pool_size());
} // }
} // }
//
TEST_F(config_test, remote_client_pool_size_minimum_value) { // TEST_F(config_test, remote_client_pool_size_minimum_value) {
{ // {
app_config config(provider_type::sia, sia_directory); // app_config config(provider_type::sia, sia_directory);
config.set_remote_client_pool_size(0); // config.set_remote_client_pool_size(0);
EXPECT_EQ(5, config.get_remote_client_pool_size()); // EXPECT_EQ(5, config.get_remote_client_pool_size());
} // }
{ // {
app_config config(provider_type::sia, sia_directory); // app_config config(provider_type::sia, sia_directory);
EXPECT_EQ(5, config.get_remote_client_pool_size()); // EXPECT_EQ(5, config.get_remote_client_pool_size());
} // }
} // }
// TEST_F(config_test, remote_max_connections) { // TEST_F(config_test, remote_max_connections) {
// std::uint8_t original_value{}; // std::uint8_t original_value{};

27
repertory/repertory_test/src/json_serialize_test.cpp
View File

@ -27,16 +27,15 @@
namespace repertory { namespace repertory {
TEST(json_serialize, can_handle_directory_item) { TEST(json_serialize, can_handle_directory_item) {
directory_item cfg{ directory_item cfg{
"api", "parent", true, 2U, {{META_DIRECTORY, "true"}}, false, "api", "parent", true, 2U, {{META_DIRECTORY, "true"}},
}; };
json data(cfg); json data(cfg);
EXPECT_STREQ("api", data.at(JSON_API_PATH).get<std::string>().c_str()); EXPECT_STREQ("api", data.at(JSON_API_PATH).get<std::string>().c_str());
EXPECT_STREQ("parent", data.at(JSON_API_PARENT).get<std::string>().c_str()); EXPECT_STREQ("parent", data.at(JSON_API_PARENT).get<std::string>().c_str());
EXPECT_TRUE(data.at(JSON_DIRECTORY).get<bool>()); EXPECT_TRUE(data.at(JSON_DIRECTORY).get<bool>());
EXPECT_STREQ("true", EXPECT_STREQ(
data.at("meta").at(META_DIRECTORY).get<std::string>().c_str()); "true", data.at(JSON_META).at(META_DIRECTORY).get<std::string>().c_str());
EXPECT_FALSE(data.at("Resolved").get<bool>());
{ {
auto cfg2 = data.get<directory_item>(); auto cfg2 = data.get<directory_item>();
@ -45,7 +44,6 @@ TEST(json_serialize, can_handle_directory_item) {
EXPECT_EQ(cfg2.directory, cfg.directory); EXPECT_EQ(cfg2.directory, cfg.directory);
EXPECT_STREQ(cfg2.meta.at(META_DIRECTORY).c_str(), EXPECT_STREQ(cfg2.meta.at(META_DIRECTORY).c_str(),
cfg.meta.at(META_DIRECTORY).c_str()); cfg.meta.at(META_DIRECTORY).c_str());
EXPECT_EQ(cfg2.resolved, cfg.resolved);
} }
} }
@ -122,6 +120,25 @@ TEST(json_serialize, can_handle_remote_config) {
} }
} }
TEST(json_serialize, can_handle_remote_mount) {
remote::remote_mount cfg{1024U, 21U, true, "token"};
json data(cfg);
EXPECT_EQ(1024U, data.at(JSON_API_PORT).get<std::uint16_t>());
EXPECT_EQ(21U, data.at(JSON_CLIENT_POOL_SIZE).get<std::uint16_t>());
EXPECT_TRUE(data.at(JSON_ENABLE_REMOTE_MOUNT).get<bool>());
EXPECT_STREQ("token",
data.at(JSON_ENCRYPTION_TOKEN).get<std::string>().c_str());
{
auto cfg2 = data.get<remote::remote_mount>();
EXPECT_EQ(cfg2.api_port, cfg.api_port);
EXPECT_EQ(cfg2.client_pool_size, cfg.client_pool_size);
EXPECT_EQ(cfg2.enable, cfg.enable);
EXPECT_STREQ(cfg2.encryption_token.c_str(), cfg.encryption_token.c_str());
}
}
TEST(json_serialize, can_handle_s3_config) { TEST(json_serialize, can_handle_s3_config) {
s3_config cfg{ s3_config cfg{
"access", "bucket", "token", "region", "secret", 31U, "url", true, false, "access", "bucket", "token", "region", "secret", 31U, "url", true, false,