diff --git a/repertory/librepertory/include/app_config.hpp b/repertory/librepertory/include/app_config.hpp index a3c3f8ec..ebaa96f4 100644 --- a/repertory/librepertory/include/app_config.hpp +++ b/repertory/librepertory/include/app_config.hpp @@ -28,86 +28,28 @@ #include "utils/error_utils.hpp" namespace repertory { -template class atomic final { -public: - atomic() : mtx_(std::make_shared()) {} - atomic(const atomic &) = default; - atomic(data_t data) - : data_(std::move(data)), mtx_(std::make_shared()) {} - atomic(data_t data, std::unique_ptr mtx) - : data_(std::move(data)), mtx_(std::move(mtx)) {} - atomic(atomic &&) = default; - - ~atomic() = default; - -private: - data_t data_; - std::shared_ptr mtx_; - -public: - auto operator=(const atomic &) -> atomic & = default; - - auto operator=(atomic &&) -> atomic & = default; - - auto operator==(const atomic &at_data) const -> bool { - mutex_lock lock(*mtx_); - return static_cast(at_data) == data_; - } - - auto operator==(const data_t &data) const -> bool { - mutex_lock lock(*mtx_); - return data == data_; - } - - auto operator!=(const atomic &at_data) const -> bool { - mutex_lock lock(*mtx_); - return static_cast(at_data) != data_; - } - - auto operator!=(const data_t &data) const -> bool { - mutex_lock lock(*mtx_); - return data != data_; - } - - auto operator=(const data_t &data) -> atomic & { - mutex_lock lock(*mtx_); - if (&data != &data_) { - data_ = data; - } - - return *this; - } - - operator data_t() const { - mutex_lock lock(*mtx_); - return data_; - } -}; - -using atomic_string = atomic; - class app_config final { public: - [[nodiscard]] static auto - default_agent_name(const provider_type &prov) -> std::string; + [[nodiscard]] static auto default_agent_name(const provider_type &prov) + -> std::string; - [[nodiscard]] static auto - default_api_port(const provider_type &prov) -> std::uint16_t; + [[nodiscard]] static auto default_api_port(const provider_type &prov) + -> std::uint16_t; - [[nodiscard]] static auto - default_data_directory(const provider_type &prov) -> std::string; + [[nodiscard]] static auto default_data_directory(const provider_type &prov) + -> std::string; - [[nodiscard]] static auto - default_remote_port(const provider_type &prov) -> std::uint16_t; + [[nodiscard]] static auto default_remote_port(const provider_type &prov) + -> std::uint16_t; - [[nodiscard]] static auto - default_rpc_port(const provider_type &prov) -> std::uint16_t; + [[nodiscard]] static auto default_rpc_port(const provider_type &prov) + -> std::uint16_t; - [[nodiscard]] static auto - get_provider_display_name(const provider_type &prov) -> std::string; + [[nodiscard]] static auto get_provider_display_name(const provider_type &prov) + -> std::string; - [[nodiscard]] static auto - get_provider_name(const provider_type &prov) -> std::string; + [[nodiscard]] static auto get_provider_name(const provider_type &prov) + -> std::string; public: app_config(const provider_type &prov, std::string_view data_directory = ""); @@ -116,11 +58,11 @@ public: private: provider_type prov_; - atomic_string api_auth_; + atomic api_auth_; std::atomic api_port_; - atomic_string api_user_; + atomic api_user_; std::atomic config_changed_; - database_type db_type_{database_type::rocksdb}; + std::atomic db_type_{database_type::rocksdb}; std::atomic download_timeout_secs_; std::atomic enable_chunk_downloader_timeout_; std::atomic enable_comm_duration_events_; @@ -131,62 +73,58 @@ private: #endif // defined(_WIN32) std::atomic enable_remote_mount_; std::atomic event_level_; - std::uint32_t eviction_delay_mins_; + std::atomic eviction_delay_mins_; std::atomic eviction_uses_accessed_time_; - std::uint16_t high_freq_interval_secs_; + std::atomic high_freq_interval_secs_; std::atomic is_remote_mount_; - std::uint16_t low_freq_interval_secs_; - std::uint64_t max_cache_size_bytes_; + std::atomic low_freq_interval_secs_; + std::atomic max_cache_size_bytes_; std::atomic max_upload_count_; - std::uint16_t med_freq_interval_secs_; + std::atomic med_freq_interval_secs_; std::atomic min_download_timeout_secs_; - std::uint16_t online_check_retry_secs_; - std::uint16_t orphaned_file_retention_days_; - atomic_string preferred_download_type_; + std::atomic online_check_retry_secs_; + std::atomic orphaned_file_retention_days_; + atomic preferred_download_type_; std::atomic read_ahead_count_; std::atomic remote_client_pool_size_; - atomic_string remote_host_name_or_ip_; + atomic remote_host_name_or_ip_; std::atomic remote_max_connections_; - std::uint16_t remote_port_; - std::uint16_t remote_receive_timeout_secs_; - std::uint16_t remote_send_timeout_secs_; - atomic_string remote_token_; - std::uint16_t retry_read_count_; - std::uint16_t ring_buffer_file_size_; - std::uint16_t task_wait_ms_; + std::atomic remote_port_; + std::atomic remote_receive_timeout_secs_; + std::atomic remote_send_timeout_secs_; + atomic remote_token_; + std::atomic retry_read_count_; + std::atomic ring_buffer_file_size_; + std::atomic task_wait_ms_; private: std::string cache_directory_; std::string data_directory_; - encrypt_config encrypt_config_; - host_config hc_{}; + atomic encrypt_config_; + atomic hc_; std::string log_directory_; mutable std::recursive_mutex read_write_mutex_; - s3_config s3_config_{}; - sia_config sia_config_{}; + atomic s3_config_; + atomic sia_config_; std::uint64_t version_{REPERTORY_CONFIG_VERSION}; - mutable std::recursive_mutex remote_mount_mutex_; private: - auto get_database_value(const json &json_document, const std::string &name, - database_type &dst, bool &success_flag) -> bool; - template - auto get_value(const json &json_document, const std::string &name, dest &dst, - bool &success_flag) -> bool { + auto get_value(const json &data, const std::string &name, dest &dst, + bool &success) -> bool { REPERTORY_USES_FUNCTION_NAME(); auto ret{false}; try { - if (json_document.find(name) != json_document.end()) { - json_document.at(name).get_to(dst); + if (data.find(name) != data.end()) { + data.at(name).get_to(dst); ret = true; } else { - success_flag = false; + success = false; } } catch (const json::exception &ex) { utils::error::raise_error(function_name, ex, "exception occurred"); - success_flag = false; + success = false; ret = false; } @@ -198,7 +136,6 @@ private: template auto set_value(dest &dst, const source &src) -> bool { auto ret{false}; - recur_mutex_lock lock(read_write_mutex_); if (dst != src) { dst = src; config_changed_ = true; @@ -221,7 +158,8 @@ public: } [[nodiscard]] auto get_chunk_downloader_timeout_secs() const -> std::uint8_t { - return std::max(min_download_timeout_secs_, download_timeout_secs_); + return std::max(static_cast(5U), + min_download_timeout_secs_.load()); } [[nodiscard]] auto get_config_file_path() const -> std::string; @@ -277,7 +215,8 @@ public: } [[nodiscard]] auto get_high_frequency_interval_secs() const -> std::uint16_t { - return std::max(static_cast(1U), high_freq_interval_secs_); + return std::max(static_cast(1U), + high_freq_interval_secs_.load()); } [[nodiscard]] auto get_host_config() const -> host_config { return hc_; } @@ -293,7 +232,8 @@ public: } [[nodiscard]] auto get_low_frequency_interval_secs() const -> std::uint16_t { - return std::max(static_cast(1U), low_freq_interval_secs_); + return std::max(static_cast(1U), + low_freq_interval_secs_.load()); } [[nodiscard]] auto get_max_cache_size_bytes() const -> std::uint64_t; @@ -303,17 +243,18 @@ public: } [[nodiscard]] auto get_med_frequency_interval_secs() const -> std::uint16_t { - return std::max(static_cast(1U), med_freq_interval_secs_); + return std::max(static_cast(1U), + med_freq_interval_secs_.load()); } [[nodiscard]] auto get_online_check_retry_secs() const -> std::uint16_t { - return std::max(std::uint16_t(15U), online_check_retry_secs_); + return std::max(std::uint16_t(15U), online_check_retry_secs_.load()); } [[nodiscard]] auto get_orphaned_file_retention_days() const -> std::uint16_t { return std::min(static_cast(31U), std::max(static_cast(1U), - orphaned_file_retention_days_)); + orphaned_file_retention_days_.load())); } [[nodiscard]] auto get_preferred_download_type() const -> download_type { @@ -360,13 +301,13 @@ public: } [[nodiscard]] auto get_retry_read_count() const -> std::uint16_t { - return std::max(std::uint16_t(2), retry_read_count_); + return std::max(std::uint16_t(2), retry_read_count_.load()); } [[nodiscard]] auto get_ring_buffer_file_size() const -> std::uint16_t { - return std::max( - static_cast(64U), - std::min(static_cast(1024U), ring_buffer_file_size_)); + return std::max(static_cast(64U), + std::min(static_cast(1024U), + ring_buffer_file_size_.load())); } [[nodiscard]] auto get_s3_config() const -> s3_config { return s3_config_; } @@ -376,10 +317,11 @@ public: } [[nodiscard]] auto get_task_wait_ms() const -> std::uint16_t { - return std::max(static_cast(50U), task_wait_ms_); + return std::max(static_cast(50U), task_wait_ms_.load()); } - [[nodiscard]] auto get_value_by_name(const std::string &name) -> std::string; + [[nodiscard]] auto get_value_by_name(const std::string &name) const + -> std::string; [[nodiscard]] auto get_version() const -> std::uint64_t { return version_; } @@ -437,6 +379,8 @@ public: } } + void set_encrypt_config(encrypt_config cfg); + void set_eviction_delay_mins(std::uint32_t eviction_delay_mins) { set_value(eviction_delay_mins_, eviction_delay_mins); } @@ -450,25 +394,7 @@ public: set_value(high_freq_interval_secs_, high_frequency_interval_secs); } -#if defined(PROJECT_TESTING) - void set_host_config(host_config hc) { - config_changed_ = true; - hc_ = std::move(hc); - save(); - } - - void set_s3_config(s3_config s3) { - config_changed_ = true; - s3_config_ = std::move(s3); - save(); - } - - void set_sia_config(sia_config sia) { - config_changed_ = true; - sia_config_ = std::move(sia); - save(); - } -#endif // defined(PROJECT_TESTING) + void set_host_config(host_config cfg); void set_is_remote_mount(bool is_remote_mount); @@ -544,6 +470,10 @@ public: set_value(retry_read_count_, retry_read_count); } + void set_s3_config(s3_config cfg); + + void set_sia_config(sia_config cfg); + void set_task_wait_ms(std::uint16_t task_wait_ms) { set_value(task_wait_ms_, task_wait_ms); } @@ -553,39 +483,4 @@ public: }; } // namespace repertory -NLOHMANN_JSON_NAMESPACE_BEGIN -template <> struct adl_serializer { - static void to_json(json &data, const repertory::atomic_string &value) { - data = static_cast(value); - } - - static void from_json(const json &data, repertory::atomic_string &value) { - value = data.get(); - } -}; - -template -struct adl_serializer> { - static void to_json(json &data, const std::atomic &value) { - data = value.load(); - } - - static void from_json(const json &data, std::atomic &value) { - value.store(data.get()); - } -}; - -template <> struct adl_serializer> { - static void to_json(json &data, - const std::atomic &value) { - data = repertory::event_level_to_string(value.load()); - } - - static void from_json(const json &data, - std::atomic &value) { - value.store(repertory::event_level_from_string(data.get())); - } -}; -NLOHMANN_JSON_NAMESPACE_END - #endif // REPERTORY_INCLUDE_APP_CONFIG_HPP_ diff --git a/repertory/librepertory/include/events/event.hpp b/repertory/librepertory/include/events/event.hpp index 0d5147cc..a179ddaf 100644 --- a/repertory/librepertory/include/events/event.hpp +++ b/repertory/librepertory/include/events/event.hpp @@ -32,9 +32,9 @@ enum class event_level { trace, }; -auto event_level_from_string(std::string level) -> event_level; +[[nodiscard]] auto event_level_from_string(std::string level) -> event_level; -auto event_level_to_string(event_level level) -> std::string; +[[nodiscard]] auto event_level_to_string(event_level level) -> std::string; class event { protected: @@ -72,4 +72,18 @@ public: }; } // namespace repertory +NLOHMANN_JSON_NAMESPACE_BEGIN +template <> struct adl_serializer> { + static void to_json(json &data, + const std::atomic &value) { + data = repertory::event_level_to_string(value.load()); + } + + static void from_json(const json &data, + std::atomic &value) { + value.store(repertory::event_level_from_string(data.get())); + } +}; +NLOHMANN_JSON_NAMESPACE_END + #endif // REPERTORY_INCLUDE_EVENTS_EVENT_HPP_ diff --git a/repertory/librepertory/include/types/repertory.hpp b/repertory/librepertory/include/types/repertory.hpp index eee347c5..c0517f4c 100644 --- a/repertory/librepertory/include/types/repertory.hpp +++ b/repertory/librepertory/include/types/repertory.hpp @@ -23,6 +23,62 @@ #define REPERTORY_INCLUDE_TYPES_REPERTORY_HPP_ namespace repertory { +template class atomic final { +public: + atomic() : mtx_(std::make_shared()) {} + atomic(const atomic &) = default; + atomic(data_t data) + : data_(std::move(data)), mtx_(std::make_shared()) {} + atomic(data_t data, std::unique_ptr mtx) + : data_(std::move(data)), mtx_(std::move(mtx)) {} + atomic(atomic &&) = default; + + ~atomic() = default; + +private: + data_t data_; + std::shared_ptr mtx_; + +public: + auto operator=(const atomic &) -> atomic & = default; + + auto operator=(atomic &&) -> atomic & = default; + + auto operator=(const data_t &data) -> atomic & { + mutex_lock lock(*mtx_); + if (&data != &data_) { + data_ = data; + } + + return *this; + } + + [[nodiscard]] auto operator==(const atomic &at_data) const -> bool { + mutex_lock lock(*mtx_); + return static_cast(at_data) == data_; + } + + [[nodiscard]] auto operator==(const data_t &data) const -> bool { + mutex_lock lock(*mtx_); + return data == data_; + } + + [[nodiscard]] auto operator!=(const atomic &at_data) const -> bool { + mutex_lock lock(*mtx_); + return static_cast(at_data) != data_; + } + + [[nodiscard]] auto operator!=(const data_t &data) const -> bool { + mutex_lock lock(*mtx_); + return data != data_; + } + + [[nodiscard]] operator data_t() const { + mutex_lock lock(*mtx_); + return data_; + } +}; + inline constexpr const auto max_time{ std::numeric_limits::max(), }; @@ -105,8 +161,9 @@ enum class database_type { rocksdb, sqlite, }; -[[nodiscard]] auto database_type_from_string(std::string type, - const database_type &default_type) +[[nodiscard]] auto +database_type_from_string(std::string type, + database_type default_type = database_type::rocksdb) -> database_type; [[nodiscard]] auto database_type_to_string(const database_type &type) @@ -117,8 +174,9 @@ enum class download_type { fallback, ring_buffer, }; -[[nodiscard]] auto download_type_from_string(std::string type, - const download_type &default_type) +[[nodiscard]] auto +download_type_from_string(std::string type, + download_type default_type = download_type::fallback) -> download_type; [[nodiscard]] auto download_type_to_string(const download_type &type) @@ -215,6 +273,22 @@ struct directory_item final { struct encrypt_config final { std::string encryption_token; std::string path; + + auto operator==(const encrypt_config &cfg) const noexcept -> bool { + if (&cfg != this) { + return encryption_token == cfg.encryption_token && path == cfg.path; + } + + return true; + } + + auto operator!=(const encrypt_config &cfg) const noexcept -> bool { + if (&cfg != this) { + return not(cfg == *this); + } + + return false; + } }; struct filesystem_item final { @@ -243,6 +317,7 @@ struct host_config final { host_name_or_ip == cfg.host_name_or_ip && path == cfg.path && protocol == cfg.protocol && timeout_ms == cfg.timeout_ms; } + return true; } @@ -250,42 +325,11 @@ struct host_config final { if (&cfg != this) { return not(cfg == *this); } + return false; } }; -#if defined(__GNUG__) -__attribute__((unused)) -#endif -static void -to_json(json &data, const host_config &cfg) { - data = json{ - {"AgentString", cfg.agent_string}, - {"ApiPassword", cfg.api_password}, - {"ApiPort", cfg.api_port}, - {"ApiUser", cfg.api_user}, - {"HostNameOrIp", cfg.host_name_or_ip}, - {"Path", cfg.path}, - {"Protocol", cfg.protocol}, - {"TimeoutMs", cfg.timeout_ms}, - }; -} - -#if defined(__GNUG__) -__attribute__((unused)) -#endif -static void -from_json(const json &data, host_config &cfg) { - data.at("AgentString").get_to(cfg.agent_string); - data.at("ApiPassword").get_to(cfg.api_password); - data.at("ApiPort").get_to(cfg.api_port); - data.at("AuthUser").get_to(cfg.api_user); - data.at("HostNameOrIp").get_to(cfg.host_name_or_ip); - data.at("Path").get_to(cfg.path); - data.at("Protocol").get_to(cfg.protocol); - data.at("TimeoutMs").get_to(cfg.timeout_ms); -} - struct s3_config final { std::string access_key; std::string bucket; @@ -296,10 +340,46 @@ struct s3_config final { std::string url; bool use_path_style{false}; bool use_region_in_url{false}; + + auto operator==(const s3_config &cfg) const noexcept -> bool { + if (&cfg != this) { + return access_key == cfg.access_key && bucket == cfg.bucket && + encryption_token == cfg.encryption_token && region == cfg.region && + secret_key == cfg.secret_key && timeout_ms == cfg.timeout_ms && + url == cfg.url && use_path_style == cfg.use_path_style && + use_region_in_url == cfg.use_region_in_url; + } + + return true; + } + + auto operator!=(const s3_config &cfg) const noexcept -> bool { + if (&cfg != this) { + return not(cfg == *this); + } + + return false; + } }; struct sia_config final { std::string bucket; + + auto operator==(const sia_config &cfg) const noexcept -> bool { + if (&cfg != this) { + return bucket == cfg.bucket; + } + + return true; + } + + auto operator!=(const sia_config &cfg) const noexcept -> bool { + if (&cfg != this) { + return not(cfg == *this); + } + + return false; + } }; using api_file_list = std::vector; @@ -309,4 +389,119 @@ using directory_item_list = std::vector; using meta_provider_callback = std::function; } // namespace repertory +NLOHMANN_JSON_NAMESPACE_BEGIN +template <> struct adl_serializer { + static void to_json(json &data, const repertory::encrypt_config &value) { + data = json{ + {"EncryptionToken", value.encryption_token}, + {"Path", value.path}, + }; + } + + static void from_json(const json &data, repertory::encrypt_config &value) { + data.at("EncryptionToken").get_to(value.encryption_token); + data.at("Path").get_to(value.path); + } +}; + +template <> struct adl_serializer { + static void to_json(json &data, const repertory::host_config &value) { + data = json{ + {"AgentString", value.agent_string}, + {"ApiPassword", value.api_password}, + {"ApiPort", value.api_port}, + {"ApiUser", value.api_user}, + {"HostNameOrIp", value.host_name_or_ip}, + {"Path", value.path}, + {"Protocol", value.protocol}, + {"TimeoutMs", value.timeout_ms}, + }; + } + + static void from_json(const json &data, repertory::host_config &value) { + data.at("AgentString").get_to(value.agent_string); + data.at("ApiPassword").get_to(value.api_password); + data.at("ApiPort").get_to(value.api_port); + data.at("AuthUser").get_to(value.api_user); + data.at("HostNameOrIp").get_to(value.host_name_or_ip); + data.at("Path").get_to(value.path); + data.at("Protocol").get_to(value.protocol); + data.at("TimeoutMs").get_to(value.timeout_ms); + } +}; + +template <> struct adl_serializer { + static void to_json(json &data, const repertory::s3_config &value) { + data = json{ + {"AccessKey", value.access_key}, + {"Bucket", value.bucket}, + {"EncryptionToken", value.encryption_token}, + {"Region", value.region}, + {"SecretKey", value.secret_key}, + {"TimeoutMs", value.timeout_ms}, + {"URL", value.url}, + {"UsePathStyle", value.use_path_style}, + {"UseRegionInURL", value.use_region_in_url}, + }; + } + + static void from_json(const json &data, repertory::s3_config &value) { + data.at("AccessKey").get_to(value.access_key); + data.at("Bucket").get_to(value.bucket); + data.at("EncryptionToken").get_to(value.encryption_token); + data.at("Region").get_to(value.region); + data.at("SecretKey").get_to(value.secret_key); + data.at("TimeoutMs").get_to(value.timeout_ms); + data.at("URL").get_to(value.url); + data.at("UsePathStyle").get_to(value.use_path_style); + data.at("UseRegionInURL").get_to(value.use_region_in_url); + } +}; + +template <> struct adl_serializer { + static void to_json(json &data, const repertory::sia_config &value) { + data = json{ + {"Bucket", value.bucket}, + }; + } + + static void from_json(const json &data, repertory::sia_config &value) { + data.at("Bucket").get_to(value.bucket); + } +}; + +template struct adl_serializer> { + static void to_json(json &data, const repertory::atomic &value) { + data = static_cast(value); + } + + static void from_json(const json &data, repertory::atomic &value) { + value = data.get(); + } +}; + +template +struct adl_serializer> { + static void to_json(json &data, const std::atomic &value) { + data = value.load(); + } + + static void from_json(const json &data, std::atomic &value) { + value.store(data.get()); + } +}; + +template <> struct adl_serializer> { + static void to_json(json &data, + const std::atomic &value) { + data = repertory::database_type_to_string(value.load()); + } + + static void from_json(const json &data, + std::atomic &value) { + value.store(repertory::database_type_from_string(data.get())); + } +}; +NLOHMANN_JSON_NAMESPACE_END + #endif // REPERTORY_INCLUDE_TYPES_REPERTORY_HPP_ diff --git a/repertory/librepertory/src/app_config.cpp b/repertory/librepertory/src/app_config.cpp index 2847ec60..513cda6b 100644 --- a/repertory/librepertory/src/app_config.cpp +++ b/repertory/librepertory/src/app_config.cpp @@ -30,29 +30,31 @@ #include "utils/utils.hpp" namespace { -constexpr const auto default_api_auth_size = 48U; -constexpr const auto default_download_timeout_ces = 30U; -constexpr const auto default_eviction_delay_mins = 10U; -constexpr const auto default_high_freq_interval_secs = 30U; -constexpr const auto default_low_freq_interval_secs = 60U * 60U; -constexpr const auto default_max_cache_size_bytes = - 20ULL * 1024ULL * 1024ULL * 1024ULL; -constexpr const auto default_max_upload_count = 5U; -constexpr const auto default_med_freq_interval_secs = 2U * 60U; -constexpr const auto default_min_download_timeout_secs = 5U; -constexpr const auto default_online_check_retry_secs = 60U; -constexpr const auto default_orphaned_file_retention_days = 15U; -constexpr const auto default_read_ahead_count = 4U; -constexpr const auto default_remote_client_pool_size = 10U; -constexpr const auto default_remote_host_name_or_ip = ""; -constexpr const auto default_remote_max_connections = 20U; -constexpr const auto default_remote_receive_timeout_secs = 120U; -constexpr const auto default_remote_send_timeout_secs = 30U; -constexpr const auto default_remote_token = ""; -constexpr const auto default_retry_read_count = 6U; -constexpr const auto default_ring_buffer_file_size = 512U; -constexpr const auto default_task_wait_ms = 100U; -constexpr const auto retry_save_count = 5U; +constexpr const auto default_api_auth_size{48U}; +constexpr const auto default_download_timeout_ces{30U}; +constexpr const auto default_eviction_delay_mins{10U}; +constexpr const auto default_high_freq_interval_secs{30U}; +constexpr const auto default_low_freq_interval_secs{0U * 60U}; +constexpr const auto default_max_cache_size_bytes{ + 20ULL * 1024ULL * 1024ULL * 1024ULL, +}; +constexpr const auto default_max_upload_count{5U}; +constexpr const auto default_med_freq_interval_secs{2U * 60U}; +constexpr const auto default_min_download_timeout_secs{5U}; +constexpr const auto default_online_check_retry_secs{60U}; +constexpr const auto default_orphaned_file_retention_days{15U}; +constexpr const auto default_read_ahead_count{4U}; +constexpr const auto default_remote_client_pool_size{10U}; +constexpr const auto default_remote_host_name_or_ip{""}; +constexpr const auto default_remote_max_connections{20U}; +constexpr const auto default_remote_receive_timeout_secs{120U}; +constexpr const auto default_remote_send_timeout_secs{30U}; +constexpr const auto default_remote_token{""}; +constexpr const auto default_retry_read_count{6U}; +constexpr const auto default_ring_buffer_file_size{512U}; +constexpr const auto default_task_wait_ms{100U}; +constexpr const auto min_cache_size_bytes{100ULL * 1024ULL * 1024ULL}; +constexpr const auto retry_save_count{5U}; } // namespace namespace repertory { @@ -103,8 +105,10 @@ app_config::app_config(const provider_type &prov, cache_directory_ = utils::path::combine(data_directory_, {"cache"}); log_directory_ = utils::path::combine(data_directory_, {"logs"}); - hc_.agent_string = default_agent_name(prov_); - hc_.api_port = default_api_port(prov_); + auto cfg = get_host_config(); + cfg.agent_string = default_agent_name(prov_); + cfg.api_port = default_api_port(prov_); + set_host_config(cfg); if (not utils::file::directory(data_directory_).create_directory()) { throw startup_exception("unable to create: " + data_directory_); @@ -123,33 +127,6 @@ app_config::app_config(const provider_type &prov, } } -auto app_config::get_config_file_path() const -> std::string { - return utils::path::combine(data_directory_, {"config.json"}); -} - -auto app_config::get_database_value(const json &json_document, - const std::string &name, database_type &dst, - bool &success_flag) -> bool { - REPERTORY_USES_FUNCTION_NAME(); - - auto ret{false}; - try { - if (json_document.find(name) != json_document.end()) { - dst = database_type_from_string(json_document[name].get(), - database_type::rocksdb); - ret = true; - } else { - success_flag = false; - } - } catch (const json::exception &ex) { - utils::error::raise_error(function_name, ex, "exception occurred"); - success_flag = false; - ret = false; - } - - return ret; -} - auto app_config::default_agent_name(const provider_type &prov) -> std::string { static const std::array(provider_type::unknown)> @@ -229,6 +206,10 @@ auto app_config::default_rpc_port(const provider_type &prov) -> std::uint16_t { return PROVIDER_RPC_PORTS.at(static_cast(prov)); } +auto app_config::get_config_file_path() const -> std::string { + return utils::path::combine(data_directory_, {"config.json"}); +} + auto app_config::get_json() const -> json { json ret = { {"ApiAuth", api_auth_}, @@ -243,23 +224,12 @@ auto app_config::get_json() const -> json { {"EnableMountManager", enable_mount_manager_}, #endif // defined(_WIN32) {"EnableMaxCacheSize", enable_max_cache_size_}, - {"EncryptConfig", - { - {"EncryptionToken", encrypt_config_.encryption_token}, - {"Path", encrypt_config_.path}, - }}, + {"EncryptConfig", encrypt_config_}, {"EventLevel", event_level_to_string(event_level_)}, {"EvictionDelayMinutes", eviction_delay_mins_}, {"EvictionUsesAccessedTime", eviction_uses_accessed_time_}, {"HighFreqIntervalSeconds", high_freq_interval_secs_}, - {"HostConfig", - { - {"AgentString", hc_.agent_string}, - {"ApiPassword", hc_.api_password}, - {"ApiPort", hc_.api_port}, - {"HostNameOrIp", hc_.host_name_or_ip}, - {"TimeoutMs", hc_.timeout_ms}, - }}, + {"HostConfig", hc_}, {"LowFreqIntervalSeconds", low_freq_interval_secs_}, {"MaxCacheSizeBytes", max_cache_size_bytes_}, {"MaxUploadCount", max_upload_count_}, @@ -284,22 +254,8 @@ auto app_config::get_json() const -> json { }, {"RetryReadCount", retry_read_count_}, {"RingBufferFileSize", ring_buffer_file_size_}, - {"S3Config", - { - {"AccessKey", s3_config_.access_key}, - {"Bucket", s3_config_.bucket}, - {"EncryptionToken", s3_config_.encryption_token}, - {"Region", s3_config_.region}, - {"SecretKey", s3_config_.secret_key}, - {"TimeoutMs", s3_config_.timeout_ms}, - {"URL", s3_config_.url}, - {"UsePathStyle", s3_config_.use_path_style}, - {"UseRegionInURL", s3_config_.use_region_in_url}, - }}, - {"SiaConfig", - { - {"Bucket", sia_config_.bucket}, - }}, + {"S3Config", s3_config_}, + {"SiaConfig", sia_config_}, {"TaskWaitMillis", task_wait_ms_}, {"Version", version_}}; @@ -355,9 +311,7 @@ auto app_config::get_json() const -> json { } auto app_config::get_max_cache_size_bytes() const -> std::uint64_t { - auto max_space = - std::max(static_cast(100ULL * 1024ULL * 1024ULL), - max_cache_size_bytes_); + auto max_space = std::max(min_cache_size_bytes, max_cache_size_bytes_.load()); auto free_space = utils::file::get_free_drive_space(get_cache_directory()); return free_space.has_value() ? std::min(free_space.value(), max_space) : max_space; @@ -388,7 +342,8 @@ auto app_config::get_provider_name(const provider_type &prov) -> std::string { return PROVIDER_NAMES.at(static_cast(prov)); } -auto app_config::get_value_by_name(const std::string &name) -> std::string { +auto app_config::get_value_by_name(const std::string &name) const + -> std::string { REPERTORY_USES_FUNCTION_NAME(); try { @@ -421,14 +376,14 @@ auto app_config::get_value_by_name(const std::string &name) -> std::string { #if defined(_WIN32) } if (name == "EnableMountManager") { - return std::to_string(get_enable_mount_manager()); + return utils::string::from_bool(get_enable_mount_manager()); #endif // defined(_WIN32) } if (name == "EncryptConfig.Path") { - return utils::path::absolute(encrypt_config_.path); + return utils::path::absolute(get_encrypt_config().path); } if (name == "EncryptConfig.EncryptionToken") { - return encrypt_config_.encryption_token; + return get_encrypt_config().encryption_token; } if (name == "EventLevel") { return event_level_to_string(get_event_level()); @@ -443,19 +398,19 @@ auto app_config::get_value_by_name(const std::string &name) -> std::string { return std::to_string(get_high_frequency_interval_secs()); } if (name == "HostConfig.AgentString") { - return hc_.agent_string; + return get_host_config().agent_string; } if (name == "HostConfig.ApiPassword") { - return hc_.api_password; + return get_host_config().api_password; } if (name == "HostConfig.ApiPort") { - return std::to_string(hc_.api_port); + return std::to_string(get_host_config().api_port); } if (name == "HostConfig.HostNameOrIp") { - return hc_.host_name_or_ip; + return get_host_config().host_name_or_ip; } if (name == "HostConfig.TimeoutMs") { - return std::to_string(hc_.timeout_ms); + return std::to_string(get_host_config().timeout_ms); } if (name == "LowFreqIntervalSeconds") { return std::to_string(get_low_frequency_interval_secs()); @@ -516,34 +471,34 @@ auto app_config::get_value_by_name(const std::string &name) -> std::string { return std::to_string(get_ring_buffer_file_size()); } if (name == "S3Config.AccessKey") { - return s3_config_.access_key; + return get_s3_config().access_key; } if (name == "S3Config.Bucket") { - return s3_config_.bucket; + return get_s3_config().bucket; } if (name == "S3Config.EncryptionToken") { - return s3_config_.encryption_token; + return get_s3_config().encryption_token; } if (name == "S3Config.Region") { - return s3_config_.region; + return get_s3_config().region; } if (name == "S3Config.SecretKey") { - return s3_config_.secret_key; + return get_s3_config().secret_key; } if (name == "S3Config.URL") { - return s3_config_.url; + return get_s3_config().url; } if (name == "S3Config.UsePathStyle") { - return utils::string::from_bool(s3_config_.use_path_style); + return utils::string::from_bool(get_s3_config().use_path_style); } if (name == "S3Config.UseRegionInURL") { - return utils::string::from_bool(s3_config_.use_region_in_url); + return utils::string::from_bool(get_s3_config().use_region_in_url); } if (name == "S3Config.TimeoutMs") { - return std::to_string(s3_config_.timeout_ms); + return std::to_string(get_s3_config().timeout_ms); } if (name == "SiaConfig.Bucket") { - return sia_config_.bucket; + return get_sia_config().bucket; } if (name == "TaskWaitMillis") { return std::to_string(get_task_wait_ms()); @@ -559,7 +514,7 @@ auto app_config::load() -> bool { auto ret{false}; - const auto config_file_path = get_config_file_path(); + auto config_file_path = get_config_file_path(); recur_mutex_lock lock(read_write_mutex_); if (utils::file::file(config_file_path).exists()) { try { @@ -567,18 +522,18 @@ auto app_config::load() -> bool { if (config_file.is_open()) { std::stringstream stream; stream << config_file.rdbuf(); - const auto json_text = stream.str(); + auto json_text = stream.str(); config_file.close(); ret = not json_text.empty(); if (ret) { - const auto json_document = json::parse(json_text); + auto json_document = json::parse(json_text); get_value(json_document, "ApiAuth", api_auth_, ret); get_value(json_document, "ApiPort", api_port_, ret); get_value(json_document, "ApiUser", api_user_, ret); get_value(json_document, "ChunkDownloaderTimeoutSeconds", download_timeout_secs_, ret); - get_database_value(json_document, "DatabaseType", db_type_, ret); + get_value(json_document, "DatabaseType", db_type_, ret); get_value(json_document, "EvictionDelayMinutes", eviction_delay_mins_, ret); get_value(json_document, "EvictionUsesAccessedTime", @@ -591,13 +546,8 @@ auto app_config::load() -> bool { ret); if (json_document.find("EncryptConfig") != json_document.end()) { - auto encrypt_config_json = json_document["EncryptConfig"]; - auto encrypt = encrypt_config_; - get_value(encrypt_config_json, "Path", encrypt.path, ret); - encrypt_config_.path = utils::path::absolute(encrypt_config_.path); - get_value(encrypt_config_json, "EncryptionToken", - encrypt.encryption_token, ret); - encrypt_config_ = encrypt; + json_document.at("EncryptConfig") + .get_to>(encrypt_config_); } else { ret = false; } @@ -608,44 +558,20 @@ auto app_config::load() -> bool { } if (json_document.find("HostConfig") != json_document.end()) { - auto host_config_json = json_document["HostConfig"]; - auto cfg = hc_; - get_value(host_config_json, "AgentString", cfg.agent_string, ret); - get_value(host_config_json, "ApiPassword", cfg.api_password, ret); - get_value(host_config_json, "ApiPort", cfg.api_port, ret); - get_value(host_config_json, "HostNameOrIp", cfg.host_name_or_ip, - ret); - get_value(host_config_json, "TimeoutMs", cfg.timeout_ms, ret); - hc_ = cfg; + json_document.at("HostConfig").get_to>(hc_); } else { ret = false; } if (json_document.find("S3Config") != json_document.end()) { - auto s3_config_json = json_document["S3Config"]; - auto s3_cfg = s3_config_; - get_value(s3_config_json, "AccessKey", s3_cfg.access_key, ret); - get_value(s3_config_json, "Bucket", s3_cfg.bucket, ret); - get_value(s3_config_json, "EncryptionToken", - s3_cfg.encryption_token, ret); - get_value(s3_config_json, "Region", s3_cfg.region, ret); - get_value(s3_config_json, "SecretKey", s3_cfg.secret_key, ret); - get_value(s3_config_json, "TimeoutMs", s3_cfg.timeout_ms, ret); - get_value(s3_config_json, "URL", s3_cfg.url, ret); - get_value(s3_config_json, "UsePathStyle", s3_cfg.use_path_style, - ret); - get_value(s3_config_json, "UseRegionInURL", - s3_cfg.use_region_in_url, ret); - s3_config_ = s3_cfg; + json_document.at("S3Config").get_to>(s3_config_); } else { ret = false; } if (json_document.find("SiaConfig") != json_document.end()) { - auto sia_config_json = json_document["SiaConfig"]; - auto sia_cfg = sia_config_; - get_value(sia_config_json, "Bucket", sia_cfg.bucket, ret); - sia_config_ = sia_cfg; + json_document.at("SiaConfig") + .get_to>(sia_config_); } else { ret = false; } @@ -727,17 +653,17 @@ auto app_config::load() -> bool { void app_config::save() { REPERTORY_USES_FUNCTION_NAME(); - const auto file_path = get_config_file_path(); recur_mutex_lock lock(read_write_mutex_); + + auto file_path = get_config_file_path(); if (config_changed_ || not utils::file::file(file_path).exists()) { - if (not utils::file::directory(data_directory_).exists()) { - if (not utils::file::directory{data_directory_}.create_directory()) { - utils::error::raise_error( - function_name, "failed to create directory|sp|" + data_directory_ + - "|err|" + - std::to_string(utils::get_last_error_code())); - } + if (not utils::file::directory{data_directory_}.create_directory()) { + utils::error::raise_error( + function_name, "failed to create directory|sp|" + data_directory_ + + "|err|" + + std::to_string(utils::get_last_error_code())); } + config_changed_ = false; json data = get_json(); auto success = false; @@ -751,17 +677,25 @@ void app_config::save() { } void app_config::set_enable_remote_mount(bool enable_remote_mount) { - recur_mutex_lock remote_lock(remote_mount_mutex_); if (get_is_remote_mount()) { set_value(enable_remote_mount_, false); - } else { - set_value(enable_remote_mount_, enable_remote_mount); + return; } + + set_value(enable_remote_mount_, enable_remote_mount); } -void app_config::set_is_remote_mount(bool is_remote_mount) { - recur_mutex_lock remote_lock(remote_mount_mutex_); +void app_config::set_encrypt_config(encrypt_config cfg) { + set_value(encrypt_config_, cfg); +} +void app_config::set_host_config(host_config cfg) { set_value(hc_, cfg); } + +void app_config::set_s3_config(s3_config cfg) { set_value(s3_config_, cfg); } + +void app_config::set_sia_config(sia_config cfg) { set_value(sia_config_, cfg); } + +void app_config::set_is_remote_mount(bool is_remote_mount) { if (get_enable_remote_mount()) { set_value(is_remote_mount_, false); return; @@ -819,12 +753,16 @@ auto app_config::set_value_by_name(const std::string &name, #endif // defined(_WIN32) } if (name == "EncryptConfig.EncryptionToken") { - set_value(encrypt_config_.encryption_token, value); - return encrypt_config_.encryption_token; + auto cfg = get_encrypt_config(); + cfg.encryption_token = value; + set_encrypt_config(cfg); + return get_encrypt_config().encryption_token; } if (name == "EncryptConfig.Path") { - set_value(encrypt_config_.path, utils::path::absolute(value)); - return encrypt_config_.path; + auto cfg = get_encrypt_config(); + cfg.path = value; + set_encrypt_config(cfg); + return get_encrypt_config().path; } if (name == "EventLevel") { set_event_level(event_level_from_string(value)); @@ -843,24 +781,34 @@ auto app_config::set_value_by_name(const std::string &name, return std::to_string(get_high_frequency_interval_secs()); } if (name == "HostConfig.AgentString") { - set_value(hc_.agent_string, value); - return hc_.agent_string; + auto cfg = get_host_config(); + cfg.agent_string = value; + set_host_config(cfg); + return get_host_config().agent_string; } if (name == "HostConfig.ApiPassword") { - set_value(hc_.api_password, value); - return hc_.api_password; + auto cfg = get_host_config(); + cfg.api_password = value; + set_host_config(cfg); + return get_host_config().api_password; } if (name == "HostConfig.ApiPort") { - set_value(hc_.api_port, utils::string::to_uint16(value)); - return std::to_string(hc_.api_port); + auto cfg = get_host_config(); + cfg.api_port = utils::string::to_uint16(value); + set_host_config(cfg); + return std::to_string(get_host_config().api_port); } if (name == "HostConfig.HostNameOrIp") { - set_value(hc_.host_name_or_ip, value); - return hc_.host_name_or_ip; + auto cfg = get_host_config(); + cfg.host_name_or_ip = value; + set_host_config(cfg); + return get_host_config().host_name_or_ip; } if (name == "HostConfig.TimeoutMs") { - set_value(hc_.timeout_ms, utils::string::to_uint32(value)); - return std::to_string(hc_.timeout_ms); + auto cfg = get_host_config(); + cfg.timeout_ms = utils::string::to_uint32(value); + set_host_config(cfg); + return std::to_string(get_host_config().timeout_ms); } if (name == "LowFreqIntervalSeconds") { set_low_frequency_interval_secs(utils::string::to_uint16(value)); @@ -940,44 +888,64 @@ auto app_config::set_value_by_name(const std::string &name, return std::to_string(get_ring_buffer_file_size()); } if (name == "S3Config.AccessKey") { - set_value(s3_config_.access_key, value); - return s3_config_.access_key; + auto cfg = get_s3_config(); + cfg.access_key = value; + set_s3_config(cfg); + return get_s3_config().access_key; } if (name == "S3Config.Bucket") { - set_value(s3_config_.bucket, value); - return s3_config_.bucket; - } - if (name == "S3Config.Region") { - set_value(s3_config_.region, value); - return s3_config_.region; - } - if (name == "S3Config.SecretKey") { - set_value(s3_config_.secret_key, value); - return s3_config_.secret_key; - } - if (name == "S3Config.URL") { - set_value(s3_config_.url, value); - return s3_config_.url; - } - if (name == "S3Config.UsePathStyle") { - set_value(s3_config_.use_path_style, utils::string::to_bool(value)); - return utils::string::from_bool(s3_config_.use_path_style); - } - if (name == "S3Config.UseRegionInURL") { - set_value(s3_config_.use_region_in_url, utils::string::to_bool(value)); - return utils::string::from_bool(s3_config_.use_region_in_url); - } - if (name == "S3Config.TimeoutMs") { - set_value(s3_config_.timeout_ms, utils::string::to_uint32(value)); - return std::to_string(s3_config_.timeout_ms); + auto cfg = get_s3_config(); + cfg.bucket = value; + set_s3_config(cfg); + return get_s3_config().bucket; } if (name == "S3Config.EncryptionToken") { - set_value(s3_config_.encryption_token, value); - return s3_config_.encryption_token; + auto cfg = get_s3_config(); + cfg.encryption_token = value; + set_s3_config(cfg); + return get_s3_config().encryption_token; + } + if (name == "S3Config.Region") { + auto cfg = get_s3_config(); + cfg.region = value; + set_s3_config(cfg); + return get_s3_config().region; + } + if (name == "S3Config.SecretKey") { + auto cfg = get_s3_config(); + cfg.secret_key = value; + set_s3_config(cfg); + return get_s3_config().secret_key; + } + if (name == "S3Config.TimeoutMs") { + auto cfg = get_s3_config(); + cfg.timeout_ms = utils::string::to_uint32(value); + set_s3_config(cfg); + return std::to_string(get_s3_config().timeout_ms); + } + if (name == "S3Config.URL") { + auto cfg = get_s3_config(); + cfg.url = value; + set_s3_config(cfg); + return get_s3_config().url; + } + if (name == "S3Config.UsePathStyle") { + auto cfg = get_s3_config(); + cfg.use_path_style = utils::string::to_bool(value); + set_s3_config(cfg); + return utils::string::from_bool(get_s3_config().use_path_style); + } + if (name == "S3Config.UseRegionInURL") { + auto cfg = get_s3_config(); + cfg.use_region_in_url = utils::string::to_bool(value); + set_s3_config(cfg); + return utils::string::from_bool(get_s3_config().use_region_in_url); } if (name == "SiaConfig.Bucket") { - set_value(sia_config_.bucket, value); - return sia_config_.bucket; + auto cfg = get_sia_config(); + cfg.bucket = value; + set_sia_config(cfg); + return get_sia_config().bucket; } if (name == "TaskWaitMillis") { set_task_wait_ms(utils::string::to_uint16(value)); diff --git a/repertory/librepertory/src/types/repertory.cpp b/repertory/librepertory/src/types/repertory.cpp index 8094fc37..bd89bdf0 100644 --- a/repertory/librepertory/src/types/repertory.cpp +++ b/repertory/librepertory/src/types/repertory.cpp @@ -25,8 +25,7 @@ #include "utils/string.hpp" namespace repertory { -auto database_type_from_string(std::string type, - const database_type &default_type) +auto database_type_from_string(std::string type, database_type default_type) -> database_type { type = utils::string::to_lower(utils::string::trim(type)); if (type == "rocksdb") { @@ -51,8 +50,7 @@ auto database_type_to_string(const database_type &type) -> std::string { } } -auto download_type_from_string(std::string type, - const download_type &default_type) +auto download_type_from_string(std::string type, download_type default_type) -> download_type { type = utils::string::to_lower(utils::string::trim(type)); if (type == "direct") {