From 7bd31b1c0a576e6ed791067338d426c33971b25a Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Sat, 7 Dec 2024 13:40:11 -0600 Subject: [PATCH] added medium frequency --- repertory/librepertory/include/app_config.hpp | 30 +++++++++++------ repertory/librepertory/src/app_config.cpp | 15 ++++++++- .../src/file_manager/file_manager.cpp | 24 ++++++++++---- repertory/repertory_test/src/config_test.cpp | 32 ++++++++++++++----- 4 files changed, 76 insertions(+), 25 deletions(-) diff --git a/repertory/librepertory/include/app_config.hpp b/repertory/librepertory/include/app_config.hpp index b3de2a64..d6c09016 100644 --- a/repertory/librepertory/include/app_config.hpp +++ b/repertory/librepertory/include/app_config.hpp @@ -76,11 +76,12 @@ private: event_level event_level_; std::uint32_t eviction_delay_mins_; bool eviction_uses_accessed_time_; - std::uint8_t high_freq_interval_secs_; + std::uint16_t high_freq_interval_secs_; bool is_remote_mount_; - std::uint32_t low_freq_interval_secs_; + std::uint16_t low_freq_interval_secs_; std::uint64_t max_cache_size_bytes_; std::uint8_t max_upload_count_; + std::uint16_t med_freq_interval_secs_; std::uint8_t min_download_timeout_secs_; std::uint16_t online_check_retry_secs_; std::uint16_t orphaned_file_retention_days_; @@ -187,7 +188,7 @@ public: [[nodiscard]] auto get_enable_mount_manager() const -> bool { return enable_mount_manager_; } -#endif +#endif // defined(_WIN32) [[nodiscard]] auto get_enable_max_cache_size() const -> bool { return enable_max_cache_size_; @@ -209,8 +210,8 @@ public: return eviction_uses_accessed_time_; } - [[nodiscard]] auto get_high_frequency_interval_secs() const -> std::uint8_t { - return std::max(static_cast(1U), high_freq_interval_secs_); + [[nodiscard]] auto get_high_frequency_interval_secs() const -> std::uint16_t { + return std::max(static_cast(1U), high_freq_interval_secs_); } [[nodiscard]] auto get_host_config() const -> host_config { return hc_; } @@ -225,8 +226,8 @@ public: return log_directory_; } - [[nodiscard]] auto get_low_frequency_interval_secs() const -> std::uint32_t { - return std::max(1U, low_freq_interval_secs_); + [[nodiscard]] auto get_low_frequency_interval_secs() const -> std::uint16_t { + return std::max(static_cast(1U), low_freq_interval_secs_); } [[nodiscard]] auto get_max_cache_size_bytes() const -> std::uint64_t; @@ -235,6 +236,10 @@ public: return std::max(std::uint8_t(1U), max_upload_count_); } + [[nodiscard]] auto get_med_frequency_interval_secs() const -> std::uint16_t { + return std::max(static_cast(1U), med_freq_interval_secs_); + } + [[nodiscard]] auto get_online_check_retry_secs() const -> std::uint16_t { return std::max(std::uint16_t(15U), online_check_retry_secs_); } @@ -349,7 +354,7 @@ public: void set_enable_mount_manager(bool enable_mount_manager) { set_value(enable_mount_manager_, enable_mount_manager); } -#endif +#endif // defined(_WIN32) void set_enable_remote_mount(bool enable_remote_mount); @@ -369,7 +374,7 @@ public: } void - set_high_frequency_interval_secs(std::uint8_t high_frequency_interval_secs) { + set_high_frequency_interval_secs(std::uint16_t high_frequency_interval_secs) { set_value(high_freq_interval_secs_, high_frequency_interval_secs); } @@ -396,7 +401,7 @@ public: void set_is_remote_mount(bool is_remote_mount); void - set_low_frequency_interval_secs(std::uint32_t low_frequency_interval_secs) { + set_low_frequency_interval_secs(std::uint16_t low_frequency_interval_secs) { set_value(low_freq_interval_secs_, low_frequency_interval_secs); } @@ -408,6 +413,11 @@ public: set_value(max_upload_count_, max_upload_count); } + void + set_med_frequency_interval_secs(std::uint16_t med_frequency_interval_secs) { + set_value(med_freq_interval_secs_, med_frequency_interval_secs); + } + void set_online_check_retry_secs(std::uint16_t online_check_retry_secs) { set_value(online_check_retry_secs_, online_check_retry_secs); } diff --git a/repertory/librepertory/src/app_config.cpp b/repertory/librepertory/src/app_config.cpp index 067a1067..b5f3714d 100644 --- a/repertory/librepertory/src/app_config.cpp +++ b/repertory/librepertory/src/app_config.cpp @@ -34,6 +34,7 @@ constexpr const auto default_api_auth_size = 48U; constexpr const auto default_download_timeout_ces = 30U; constexpr const auto default_eviction_delay_mins = 10U; constexpr const auto default_high_freq_interval_secs = 30U; +constexpr const auto default_med_freq_interval_secs = 5U * 60U; constexpr const auto default_low_freq_interval_secs = 60U * 60U; constexpr const auto default_max_cache_size_bytes = 20ULL * 1024ULL * 1024ULL * 1024ULL; @@ -82,6 +83,7 @@ app_config::app_config(const provider_type &prov, low_freq_interval_secs_(default_low_freq_interval_secs), max_cache_size_bytes_(default_max_cache_size_bytes), max_upload_count_(default_max_upload_count), + med_freq_interval_secs_(default_med_freq_interval_secs), min_download_timeout_secs_(default_min_download_timeout_secs), online_check_retry_secs_(default_online_check_retry_secs), orphaned_file_retention_days_(default_orphaned_file_retention_days), @@ -237,6 +239,7 @@ auto app_config::get_json() const -> json { {"LowFreqIntervalSeconds", low_freq_interval_secs_}, {"MaxCacheSizeBytes", max_cache_size_bytes_}, {"MaxUploadCount", max_upload_count_}, + {"MedFreqIntervalSeconds", med_freq_interval_secs_}, {"OnlineCheckRetrySeconds", online_check_retry_secs_}, {"OrphanedFileRetentionDays", orphaned_file_retention_days_}, {"PreferredDownloadType", preferred_download_type_}, @@ -312,6 +315,7 @@ auto app_config::get_json() const -> json { ret.erase("LowFreqIntervalSeconds"); ret.erase("MaxCacheSizeBytes"); ret.erase("MaxUploadCount"); + ret.erase("MedFreqIntervalSeconds"); ret.erase("OnlineCheckRetrySeconds"); ret.erase("OrphanedFileRetentionDays"); ret.erase("PreferredDownloadType"); @@ -428,6 +432,9 @@ auto app_config::get_value_by_name(const std::string &name) -> std::string { if (name == "LowFreqIntervalSeconds") { return std::to_string(get_low_frequency_interval_secs()); } + if (name == "MedFreqIntervalSeconds") { + return std::to_string(get_med_frequency_interval_secs()); + } if (name == "MaxCacheSizeBytes") { return std::to_string(get_max_cache_size_bytes()); } @@ -633,6 +640,8 @@ auto app_config::load() -> bool { high_freq_interval_secs_, ret); get_value(json_document, "LowFreqIntervalSeconds", low_freq_interval_secs_, ret); + get_value(json_document, "MedFreqIntervalSeconds", + med_freq_interval_secs_, ret); get_value(json_document, "OrphanedFileRetentionDays", orphaned_file_retention_days_, ret); get_value(json_document, "PreferredDownloadType", @@ -820,9 +829,13 @@ auto app_config::set_value_by_name(const std::string &name, return std::to_string(hc_.timeout_ms); } if (name == "LowFreqIntervalSeconds") { - set_low_frequency_interval_secs(utils::string::to_uint32(value)); + set_low_frequency_interval_secs(utils::string::to_uint16(value)); return std::to_string(get_low_frequency_interval_secs()); } + if (name == "MedFreqIntervalSeconds") { + set_med_frequency_interval_secs(utils::string::to_uint16(value)); + return std::to_string(get_med_frequency_interval_secs()); + } if (name == "MaxCacheSizeBytes") { set_max_cache_size_bytes(utils::string::to_uint64(value)); return std::to_string(get_max_cache_size_bytes()); diff --git a/repertory/librepertory/src/file_manager/file_manager.cpp b/repertory/librepertory/src/file_manager/file_manager.cpp index a6de367a..dcfaa366 100644 --- a/repertory/librepertory/src/file_manager/file_manager.cpp +++ b/repertory/librepertory/src/file_manager/file_manager.cpp @@ -779,11 +779,20 @@ void file_manager::start() { stop_requested_ = false; - polling::instance().set_callback({"timed_out_close", - polling::frequency::second, - [this](auto && /* stop_requested */) { - this->close_timed_out_files(); - }}); + polling::instance().set_callback({ + "db_cleanup", + polling::frequency::high, + [this](auto && /* stop_requested */) { + mutex_lock lock(upload_mtx_); + sqlite3_db_release_memory(db_.get()); + }, + }); + + polling::instance().set_callback({ + "timed_out_close", + polling::frequency::second, + [this](auto && /* stop_requested */) { this->close_timed_out_files(); }, + }); if (provider_.is_read_only()) { stop_requested_ = false; @@ -897,9 +906,12 @@ void file_manager::stop() { } event_system::instance().raise("file_manager"); - polling::instance().remove_callback("timed_out_close"); + stop_requested_ = true; + polling::instance().remove_callback("db_cleanup"); + polling::instance().remove_callback("timed_out_close"); + unique_mutex_lock upload_lock(upload_mtx_); upload_notify_.notify_all(); upload_lock.unlock(); diff --git a/repertory/repertory_test/src/config_test.cpp b/repertory/repertory_test/src/config_test.cpp index 1c0b4fc1..93359f1e 100644 --- a/repertory/repertory_test/src/config_test.cpp +++ b/repertory/repertory_test/src/config_test.cpp @@ -81,6 +81,7 @@ const auto DEFAULT_SIA_CONFIG = "{\n" " \"LowFreqIntervalSeconds\": 3600,\n" " \"MaxCacheSizeBytes\": 21474836480,\n" " \"MaxUploadCount\": 5,\n" + " \"MedFreqIntervalSeconds\": 300,\n" " \"OnlineCheckRetrySeconds\": 60,\n" " \"OrphanedFileRetentionDays\": 15,\n" " \"PreferredDownloadType\": \"fallback\",\n" @@ -126,6 +127,7 @@ const auto DEFAULT_S3_CONFIG = "{\n" " \"LowFreqIntervalSeconds\": 3600,\n" " \"MaxCacheSizeBytes\": 21474836480,\n" " \"MaxUploadCount\": 5,\n" + " \"MedFreqIntervalSeconds\": 300,\n" " \"OnlineCheckRetrySeconds\": 60,\n" " \"OrphanedFileRetentionDays\": 15,\n" " \"PreferredDownloadType\": \"fallback\",\n" @@ -378,30 +380,44 @@ TEST_F(config_test, eviction_uses_accessed_time) { } TEST_F(config_test, high_frequency_interval_secs) { - std::uint8_t original_value{}; + std::uint16_t original_value{}; { app_config config(provider_type::sia, sia_directory); original_value = config.get_high_frequency_interval_secs(); - config.set_high_frequency_interval_secs(original_value + 5); - EXPECT_EQ(original_value + 5, config.get_high_frequency_interval_secs()); + config.set_high_frequency_interval_secs(original_value + 5U); + EXPECT_EQ(original_value + 5U, config.get_high_frequency_interval_secs()); } { app_config config(provider_type::sia, sia_directory); - EXPECT_EQ(original_value + 5, config.get_high_frequency_interval_secs()); + EXPECT_EQ(original_value + 5U, config.get_high_frequency_interval_secs()); } } TEST_F(config_test, low_frequency_interval_secs) { - std::uint32_t original_value{}; + std::uint16_t original_value{}; { app_config config(provider_type::sia, sia_directory); original_value = config.get_low_frequency_interval_secs(); - config.set_low_frequency_interval_secs(original_value + 5); - EXPECT_EQ(original_value + 5, config.get_low_frequency_interval_secs()); + config.set_low_frequency_interval_secs(original_value + 5U); + EXPECT_EQ(original_value + 5U, config.get_low_frequency_interval_secs()); } { app_config config(provider_type::sia, sia_directory); - EXPECT_EQ(original_value + 5, config.get_low_frequency_interval_secs()); + EXPECT_EQ(original_value + 5U, config.get_low_frequency_interval_secs()); + } +} + +TEST_F(config_test, med_frequency_interval_secs) { + std::uint16_t original_value{}; + { + app_config config(provider_type::sia, sia_directory); + original_value = config.get_med_frequency_interval_secs(); + config.set_med_frequency_interval_secs(original_value + 5U); + EXPECT_EQ(original_value + 5U, config.get_med_frequency_interval_secs()); + } + { + app_config config(provider_type::sia, sia_directory); + EXPECT_EQ(original_value + 5U, config.get_med_frequency_interval_secs()); } }