removed legacy option

This commit is contained in:
Scott E. Graves 2024-12-19 12:58:06 -06:00
parent 274471a066
commit 2ca277ddf7
6 changed files with 53 additions and 104 deletions

View File

@ -68,7 +68,6 @@ private:
std::atomic<bool> enable_chunk_downloader_timeout_; std::atomic<bool> enable_chunk_downloader_timeout_;
std::atomic<bool> enable_comm_duration_events_; std::atomic<bool> enable_comm_duration_events_;
std::atomic<bool> enable_drive_events_; std::atomic<bool> enable_drive_events_;
std::atomic<bool> enable_max_cache_size_;
#if defined(_WIN32) #if defined(_WIN32)
std::atomic<bool> enable_mount_manager_; std::atomic<bool> enable_mount_manager_;
#endif // defined(_WIN32) #endif // defined(_WIN32)
@ -190,10 +189,6 @@ public:
} }
#endif // defined(_WIN32) #endif // defined(_WIN32)
[[nodiscard]] auto get_enable_max_cache_size() const -> bool {
return enable_max_cache_size_;
}
[[nodiscard]] auto get_enable_remote_mount() const -> bool { [[nodiscard]] auto get_enable_remote_mount() const -> bool {
return enable_remote_mount_; return enable_remote_mount_;
} }
@ -335,10 +330,6 @@ public:
set_value(enable_drive_events_, enable_drive_events); set_value(enable_drive_events_, enable_drive_events);
} }
void set_enable_max_cache_size(bool enable_max_cache_size) {
set_value(enable_max_cache_size_, enable_max_cache_size);
}
#if defined(_WIN32) #if defined(_WIN32)
void set_enable_mount_manager(bool enable_mount_manager) { void set_enable_mount_manager(bool enable_mount_manager) {
set_value(enable_mount_manager_, enable_mount_manager); set_value(enable_mount_manager_, enable_mount_manager);

View File

@ -119,7 +119,7 @@ public:
open_file_data ofd, std::uint64_t &handle, open_file_data ofd, std::uint64_t &handle,
std::shared_ptr<i_open_file> &file) -> api_error; std::shared_ptr<i_open_file> &file) -> api_error;
[[nodiscard]] auto evict_file(const std::string &api_path) -> bool override; auto evict_file(const std::string &api_path) -> bool override;
[[nodiscard]] auto get_directory_items(const std::string &api_path) const [[nodiscard]] auto get_directory_items(const std::string &api_path) const
-> directory_item_list override; -> directory_item_list override;

View File

@ -31,8 +31,7 @@ class i_file_manager {
INTERFACE_SETUP(i_file_manager); INTERFACE_SETUP(i_file_manager);
public: public:
[[nodiscard]] virtual auto evict_file(const std::string &api_path) virtual auto evict_file(const std::string &api_path) -> bool = 0;
-> bool = 0;
[[nodiscard]] virtual auto [[nodiscard]] virtual auto
get_directory_items(const std::string &api_path) const get_directory_items(const std::string &api_path) const

View File

@ -68,7 +68,6 @@ app_config::app_config(const provider_type &prov,
enable_chunk_downloader_timeout_(true), enable_chunk_downloader_timeout_(true),
enable_comm_duration_events_(false), enable_comm_duration_events_(false),
enable_drive_events_(false), enable_drive_events_(false),
enable_max_cache_size_(false),
#if defined(_WIN32) #if defined(_WIN32)
enable_mount_manager_(false), enable_mount_manager_(false),
#endif // defined(_WIN32) #endif // defined(_WIN32)
@ -216,7 +215,6 @@ auto app_config::get_json() const -> json {
#if defined(_WIN32) #if defined(_WIN32)
{"EnableMountManager", enable_mount_manager_}, {"EnableMountManager", enable_mount_manager_},
#endif // defined(_WIN32) #endif // defined(_WIN32)
{"EnableMaxCacheSize", enable_max_cache_size_},
{"EncryptConfig", encrypt_config_}, {"EncryptConfig", encrypt_config_},
{"EventLevel", event_level_}, {"EventLevel", event_level_},
{"EvictionDelayMinutes", eviction_delay_mins_}, {"EvictionDelayMinutes", eviction_delay_mins_},
@ -250,7 +248,6 @@ auto app_config::get_json() const -> json {
if (prov_ == provider_type::encrypt) { if (prov_ == provider_type::encrypt) {
ret.erase("ChunkDownloaderTimeoutSeconds"); ret.erase("ChunkDownloaderTimeoutSeconds");
ret.erase("EnableChunkDownloaderTimeout"); ret.erase("EnableChunkDownloaderTimeout");
ret.erase("EnableMaxCacheSize");
ret.erase("EvictionDelayMinutes"); ret.erase("EvictionDelayMinutes");
ret.erase("EvictionUsesAccessedTime"); ret.erase("EvictionUsesAccessedTime");
ret.erase("HostConfig"); ret.erase("HostConfig");
@ -277,7 +274,6 @@ auto app_config::get_json() const -> json {
ret.erase("ChunkDownloaderTimeoutSeconds"); ret.erase("ChunkDownloaderTimeoutSeconds");
ret.erase("DatabaseType"); ret.erase("DatabaseType");
ret.erase("EnableChunkDownloaderTimeout"); ret.erase("EnableChunkDownloaderTimeout");
ret.erase("EnableMaxCacheSize");
ret.erase("EncryptConfig"); ret.erase("EncryptConfig");
ret.erase("EvictionDelayMinutes"); ret.erase("EvictionDelayMinutes");
ret.erase("HighFreqIntervalSeconds"); ret.erase("HighFreqIntervalSeconds");
@ -359,9 +355,6 @@ auto app_config::get_value_by_name(const std::string &name) const
} }
if (name == "EnableDriveEvents") { if (name == "EnableDriveEvents") {
return utils::string::from_bool(get_enable_drive_events()); return utils::string::from_bool(get_enable_drive_events());
}
if (name == "EnableMaxCacheSize") {
return utils::string::from_bool(get_enable_max_cache_size());
#if defined(_WIN32) #if defined(_WIN32)
} }
if (name == "EnableMountManager") { if (name == "EnableMountManager") {
@ -565,8 +558,6 @@ auto app_config::load() -> bool {
get_value(json_document, "RingBufferFileSize", ring_buffer_file_size_, get_value(json_document, "RingBufferFileSize", ring_buffer_file_size_,
ret); ret);
get_value(json_document, "TaskWaitMillis", task_wait_ms_, ret); get_value(json_document, "TaskWaitMillis", task_wait_ms_, ret);
get_value(json_document, "EnableMaxCacheSize", enable_max_cache_size_,
ret);
#if defined(_WIN32) #if defined(_WIN32)
get_value(json_document, "EnableMountManager", enable_mount_manager_, get_value(json_document, "EnableMountManager", enable_mount_manager_,
ret); ret);
@ -708,10 +699,6 @@ auto app_config::set_value_by_name(const std::string &name,
if (name == "EnableDriveEvents") { if (name == "EnableDriveEvents") {
set_enable_drive_events(utils::string::to_bool(value)); set_enable_drive_events(utils::string::to_bool(value));
return utils::string::from_bool(get_enable_drive_events()); return utils::string::from_bool(get_enable_drive_events());
}
if (name == "EnableMaxCacheSize") {
set_enable_max_cache_size(utils::string::to_bool(value));
return utils::string::from_bool(get_enable_max_cache_size());
#if defined(_WIN32) #if defined(_WIN32)
} }
if (name == "EnableMountManager") { if (name == "EnableMountManager") {

View File

@ -36,12 +36,10 @@ auto eviction::check_minimum_requirements(const std::string &file_path)
-> bool { -> bool {
REPERTORY_USES_FUNCTION_NAME(); REPERTORY_USES_FUNCTION_NAME();
auto check_file = utils::file::file{file_path}; auto file = utils::file::file{file_path};
auto reference_time = file.get_time(config_.get_eviction_uses_accessed_time()
auto reference_time = ? utils::file::time_type::accessed
check_file.get_time(config_.get_eviction_uses_accessed_time() : utils::file::time_type::modified);
? utils::file::time_type::accessed
: utils::file::time_type::modified);
if (not reference_time.has_value()) { if (not reference_time.has_value()) {
utils::error::raise_error(function_name, utils::get_last_error_code(), utils::error::raise_error(function_name, utils::get_last_error_code(),
@ -49,18 +47,18 @@ auto eviction::check_minimum_requirements(const std::string &file_path)
return false; return false;
} }
auto delay = (config_.get_eviction_delay_mins() * 60UL) * auto delay =
utils::time::NANOS_PER_SECOND; static_cast<std::uint64_t>(config_.get_eviction_delay_mins() * 60U) *
utils::time::NANOS_PER_SECOND;
return ((reference_time.value() + static_cast<std::uint64_t>(delay)) <= return (reference_time.value() + delay) <= utils::time::get_time_now();
utils::time::get_time_now());
} }
auto eviction::get_filtered_cached_files() -> std::deque<std::string> { auto eviction::get_filtered_cached_files() -> std::deque<std::string> {
auto list = auto list =
utils::file::get_directory_files(config_.get_cache_directory(), true); utils::file::get_directory_files(config_.get_cache_directory(), true);
list.erase(std::remove_if(list.begin(), list.end(), list.erase(std::remove_if(list.begin(), list.end(),
[this](const std::string &path) -> bool { [this](auto &&path) -> bool {
return not this->check_minimum_requirements(path); return not this->check_minimum_requirements(path);
}), }),
list.end()); list.end());
@ -70,65 +68,54 @@ auto eviction::get_filtered_cached_files() -> std::deque<std::string> {
void eviction::service_function() { void eviction::service_function() {
REPERTORY_USES_FUNCTION_NAME(); REPERTORY_USES_FUNCTION_NAME();
auto should_evict = true; auto cached_files_list = get_filtered_cached_files();
while (not get_stop_requested() && not cached_files_list.empty()) {
auto file_path = cached_files_list.front();
cached_files_list.pop_front();
// Handle maximum cache size eviction try {
auto used_bytes = std::string api_path;
utils::file::directory{config_.get_cache_directory()}.size(); if (provider_.get_api_path_from_source(file_path, api_path) !=
if (config_.get_enable_max_cache_size()) { api_error::success) {
should_evict = (used_bytes > config_.get_max_cache_size_bytes()); continue;
}
if (should_evict) {
// Remove cached source files that don't meet minimum requirements
auto cached_files_list = get_filtered_cached_files();
while (not get_stop_requested() && should_evict &&
not cached_files_list.empty()) {
try {
std::string api_path;
if (provider_.get_api_path_from_source(
cached_files_list.front(), api_path) == api_error::success) {
api_file file{};
filesystem_item fsi{};
if (provider_.get_filesystem_item_and_file(api_path, file, fsi) ==
api_error::success) {
// Only evict files that match expected size
auto opt_size = utils::file::file{cached_files_list.front()}.size();
if (opt_size.has_value()) {
auto file_size{opt_size.value()};
if (file_size == fsi.size) {
// Try to evict file
if (fm_.evict_file(fsi.api_path) &&
config_.get_enable_max_cache_size()) {
// Restrict number of items evicted if maximum cache size is
// enabled
used_bytes -= file_size;
should_evict =
(used_bytes > config_.get_max_cache_size_bytes());
}
}
} else {
utils::error::raise_api_path_error(
function_name, file.api_path, file.source_path,
utils::get_last_error_code(), "failed to get file size");
}
}
}
} catch (const std::exception &ex) {
utils::error::raise_error(function_name, ex,
"failed to process cached file|sp|" +
cached_files_list.front());
} }
cached_files_list.pop_front(); api_file file{};
filesystem_item fsi{};
if (provider_.get_filesystem_item_and_file(api_path, file, fsi) !=
api_error::success) {
continue;
}
auto opt_size = utils::file::file{file_path}.size();
if (not opt_size.has_value()) {
utils::error::raise_api_path_error(
function_name, file.api_path, file.source_path,
utils::get_last_error_code(), "failed to get file size");
continue;
}
if (opt_size.value() != fsi.size) {
continue;
}
fm_.evict_file(fsi.api_path);
} catch (const std::exception &ex) {
utils::error::raise_error(
function_name, ex,
fmt::format("failed to process cached file|sp|{}", file_path));
} }
} }
if (not get_stop_requested()) { if (get_stop_requested()) {
unique_mutex_lock lock(get_mutex()); return;
if (not get_stop_requested()) {
get_notify().wait_for(lock, 30s);
}
} }
unique_mutex_lock lock(get_mutex());
if (get_stop_requested()) {
return;
}
get_notify().wait_for(lock, 30s);
} }
} // namespace repertory } // namespace repertory

View File

@ -64,7 +64,6 @@ const auto DEFAULT_SIA_CONFIG = "{\n"
" \"EnableChunkDownloaderTimeout\": true,\n" " \"EnableChunkDownloaderTimeout\": true,\n"
" \"EnableCommDurationEvents\": false,\n" " \"EnableCommDurationEvents\": false,\n"
" \"EnableDriveEvents\": false,\n" " \"EnableDriveEvents\": false,\n"
" \"EnableMaxCacheSize\": false,\n"
#if defined(_WIN32) #if defined(_WIN32)
" \"EnableMountManager\": false,\n" " \"EnableMountManager\": false,\n"
#endif #endif
@ -112,7 +111,6 @@ const auto DEFAULT_S3_CONFIG = "{\n"
" \"EnableChunkDownloaderTimeout\": true,\n" " \"EnableChunkDownloaderTimeout\": true,\n"
" \"EnableCommDurationEvents\": false,\n" " \"EnableCommDurationEvents\": false,\n"
" \"EnableDriveEvents\": false,\n" " \"EnableDriveEvents\": false,\n"
" \"EnableMaxCacheSize\": false,\n"
#if defined(_WIN32) #if defined(_WIN32)
" \"EnableMountManager\": false,\n" " \"EnableMountManager\": false,\n"
#endif #endif
@ -298,19 +296,6 @@ TEST_F(config_test, enable_drive_events) {
} }
} }
TEST_F(config_test, enable_max_cache_size) {
bool original_value{};
{
app_config config(provider_type::sia, sia_directory);
original_value = config.get_enable_max_cache_size();
config.set_enable_max_cache_size(not original_value);
EXPECT_EQ(not original_value, config.get_enable_max_cache_size());
}
{
app_config config(provider_type::sia, sia_directory);
EXPECT_EQ(not original_value, config.get_enable_max_cache_size());
}
}
#if defined(_WIN32) #if defined(_WIN32)
TEST_F(config_test, enable_mount_manager) { TEST_F(config_test, enable_mount_manager) {
bool original_value; bool original_value;