removed legacy option
This commit is contained in:
parent
274471a066
commit
2ca277ddf7
@ -68,7 +68,6 @@ private:
|
|||||||
std::atomic<bool> enable_chunk_downloader_timeout_;
|
std::atomic<bool> enable_chunk_downloader_timeout_;
|
||||||
std::atomic<bool> enable_comm_duration_events_;
|
std::atomic<bool> enable_comm_duration_events_;
|
||||||
std::atomic<bool> enable_drive_events_;
|
std::atomic<bool> enable_drive_events_;
|
||||||
std::atomic<bool> enable_max_cache_size_;
|
|
||||||
#if defined(_WIN32)
|
#if defined(_WIN32)
|
||||||
std::atomic<bool> enable_mount_manager_;
|
std::atomic<bool> enable_mount_manager_;
|
||||||
#endif // defined(_WIN32)
|
#endif // defined(_WIN32)
|
||||||
@ -190,10 +189,6 @@ public:
|
|||||||
}
|
}
|
||||||
#endif // defined(_WIN32)
|
#endif // defined(_WIN32)
|
||||||
|
|
||||||
[[nodiscard]] auto get_enable_max_cache_size() const -> bool {
|
|
||||||
return enable_max_cache_size_;
|
|
||||||
}
|
|
||||||
|
|
||||||
[[nodiscard]] auto get_enable_remote_mount() const -> bool {
|
[[nodiscard]] auto get_enable_remote_mount() const -> bool {
|
||||||
return enable_remote_mount_;
|
return enable_remote_mount_;
|
||||||
}
|
}
|
||||||
@ -335,10 +330,6 @@ public:
|
|||||||
set_value(enable_drive_events_, enable_drive_events);
|
set_value(enable_drive_events_, enable_drive_events);
|
||||||
}
|
}
|
||||||
|
|
||||||
void set_enable_max_cache_size(bool enable_max_cache_size) {
|
|
||||||
set_value(enable_max_cache_size_, enable_max_cache_size);
|
|
||||||
}
|
|
||||||
|
|
||||||
#if defined(_WIN32)
|
#if defined(_WIN32)
|
||||||
void set_enable_mount_manager(bool enable_mount_manager) {
|
void set_enable_mount_manager(bool enable_mount_manager) {
|
||||||
set_value(enable_mount_manager_, enable_mount_manager);
|
set_value(enable_mount_manager_, enable_mount_manager);
|
||||||
|
@ -119,7 +119,7 @@ public:
|
|||||||
open_file_data ofd, std::uint64_t &handle,
|
open_file_data ofd, std::uint64_t &handle,
|
||||||
std::shared_ptr<i_open_file> &file) -> api_error;
|
std::shared_ptr<i_open_file> &file) -> api_error;
|
||||||
|
|
||||||
[[nodiscard]] auto evict_file(const std::string &api_path) -> bool override;
|
auto evict_file(const std::string &api_path) -> bool override;
|
||||||
|
|
||||||
[[nodiscard]] auto get_directory_items(const std::string &api_path) const
|
[[nodiscard]] auto get_directory_items(const std::string &api_path) const
|
||||||
-> directory_item_list override;
|
-> directory_item_list override;
|
||||||
|
@ -31,8 +31,7 @@ class i_file_manager {
|
|||||||
INTERFACE_SETUP(i_file_manager);
|
INTERFACE_SETUP(i_file_manager);
|
||||||
|
|
||||||
public:
|
public:
|
||||||
[[nodiscard]] virtual auto evict_file(const std::string &api_path)
|
virtual auto evict_file(const std::string &api_path) -> bool = 0;
|
||||||
-> bool = 0;
|
|
||||||
|
|
||||||
[[nodiscard]] virtual auto
|
[[nodiscard]] virtual auto
|
||||||
get_directory_items(const std::string &api_path) const
|
get_directory_items(const std::string &api_path) const
|
||||||
|
@ -68,7 +68,6 @@ app_config::app_config(const provider_type &prov,
|
|||||||
enable_chunk_downloader_timeout_(true),
|
enable_chunk_downloader_timeout_(true),
|
||||||
enable_comm_duration_events_(false),
|
enable_comm_duration_events_(false),
|
||||||
enable_drive_events_(false),
|
enable_drive_events_(false),
|
||||||
enable_max_cache_size_(false),
|
|
||||||
#if defined(_WIN32)
|
#if defined(_WIN32)
|
||||||
enable_mount_manager_(false),
|
enable_mount_manager_(false),
|
||||||
#endif // defined(_WIN32)
|
#endif // defined(_WIN32)
|
||||||
@ -216,7 +215,6 @@ auto app_config::get_json() const -> json {
|
|||||||
#if defined(_WIN32)
|
#if defined(_WIN32)
|
||||||
{"EnableMountManager", enable_mount_manager_},
|
{"EnableMountManager", enable_mount_manager_},
|
||||||
#endif // defined(_WIN32)
|
#endif // defined(_WIN32)
|
||||||
{"EnableMaxCacheSize", enable_max_cache_size_},
|
|
||||||
{"EncryptConfig", encrypt_config_},
|
{"EncryptConfig", encrypt_config_},
|
||||||
{"EventLevel", event_level_},
|
{"EventLevel", event_level_},
|
||||||
{"EvictionDelayMinutes", eviction_delay_mins_},
|
{"EvictionDelayMinutes", eviction_delay_mins_},
|
||||||
@ -250,7 +248,6 @@ auto app_config::get_json() const -> json {
|
|||||||
if (prov_ == provider_type::encrypt) {
|
if (prov_ == provider_type::encrypt) {
|
||||||
ret.erase("ChunkDownloaderTimeoutSeconds");
|
ret.erase("ChunkDownloaderTimeoutSeconds");
|
||||||
ret.erase("EnableChunkDownloaderTimeout");
|
ret.erase("EnableChunkDownloaderTimeout");
|
||||||
ret.erase("EnableMaxCacheSize");
|
|
||||||
ret.erase("EvictionDelayMinutes");
|
ret.erase("EvictionDelayMinutes");
|
||||||
ret.erase("EvictionUsesAccessedTime");
|
ret.erase("EvictionUsesAccessedTime");
|
||||||
ret.erase("HostConfig");
|
ret.erase("HostConfig");
|
||||||
@ -277,7 +274,6 @@ auto app_config::get_json() const -> json {
|
|||||||
ret.erase("ChunkDownloaderTimeoutSeconds");
|
ret.erase("ChunkDownloaderTimeoutSeconds");
|
||||||
ret.erase("DatabaseType");
|
ret.erase("DatabaseType");
|
||||||
ret.erase("EnableChunkDownloaderTimeout");
|
ret.erase("EnableChunkDownloaderTimeout");
|
||||||
ret.erase("EnableMaxCacheSize");
|
|
||||||
ret.erase("EncryptConfig");
|
ret.erase("EncryptConfig");
|
||||||
ret.erase("EvictionDelayMinutes");
|
ret.erase("EvictionDelayMinutes");
|
||||||
ret.erase("HighFreqIntervalSeconds");
|
ret.erase("HighFreqIntervalSeconds");
|
||||||
@ -359,9 +355,6 @@ auto app_config::get_value_by_name(const std::string &name) const
|
|||||||
}
|
}
|
||||||
if (name == "EnableDriveEvents") {
|
if (name == "EnableDriveEvents") {
|
||||||
return utils::string::from_bool(get_enable_drive_events());
|
return utils::string::from_bool(get_enable_drive_events());
|
||||||
}
|
|
||||||
if (name == "EnableMaxCacheSize") {
|
|
||||||
return utils::string::from_bool(get_enable_max_cache_size());
|
|
||||||
#if defined(_WIN32)
|
#if defined(_WIN32)
|
||||||
}
|
}
|
||||||
if (name == "EnableMountManager") {
|
if (name == "EnableMountManager") {
|
||||||
@ -565,8 +558,6 @@ auto app_config::load() -> bool {
|
|||||||
get_value(json_document, "RingBufferFileSize", ring_buffer_file_size_,
|
get_value(json_document, "RingBufferFileSize", ring_buffer_file_size_,
|
||||||
ret);
|
ret);
|
||||||
get_value(json_document, "TaskWaitMillis", task_wait_ms_, ret);
|
get_value(json_document, "TaskWaitMillis", task_wait_ms_, ret);
|
||||||
get_value(json_document, "EnableMaxCacheSize", enable_max_cache_size_,
|
|
||||||
ret);
|
|
||||||
#if defined(_WIN32)
|
#if defined(_WIN32)
|
||||||
get_value(json_document, "EnableMountManager", enable_mount_manager_,
|
get_value(json_document, "EnableMountManager", enable_mount_manager_,
|
||||||
ret);
|
ret);
|
||||||
@ -708,10 +699,6 @@ auto app_config::set_value_by_name(const std::string &name,
|
|||||||
if (name == "EnableDriveEvents") {
|
if (name == "EnableDriveEvents") {
|
||||||
set_enable_drive_events(utils::string::to_bool(value));
|
set_enable_drive_events(utils::string::to_bool(value));
|
||||||
return utils::string::from_bool(get_enable_drive_events());
|
return utils::string::from_bool(get_enable_drive_events());
|
||||||
}
|
|
||||||
if (name == "EnableMaxCacheSize") {
|
|
||||||
set_enable_max_cache_size(utils::string::to_bool(value));
|
|
||||||
return utils::string::from_bool(get_enable_max_cache_size());
|
|
||||||
#if defined(_WIN32)
|
#if defined(_WIN32)
|
||||||
}
|
}
|
||||||
if (name == "EnableMountManager") {
|
if (name == "EnableMountManager") {
|
||||||
|
@ -36,12 +36,10 @@ auto eviction::check_minimum_requirements(const std::string &file_path)
|
|||||||
-> bool {
|
-> bool {
|
||||||
REPERTORY_USES_FUNCTION_NAME();
|
REPERTORY_USES_FUNCTION_NAME();
|
||||||
|
|
||||||
auto check_file = utils::file::file{file_path};
|
auto file = utils::file::file{file_path};
|
||||||
|
auto reference_time = file.get_time(config_.get_eviction_uses_accessed_time()
|
||||||
auto reference_time =
|
? utils::file::time_type::accessed
|
||||||
check_file.get_time(config_.get_eviction_uses_accessed_time()
|
: utils::file::time_type::modified);
|
||||||
? utils::file::time_type::accessed
|
|
||||||
: utils::file::time_type::modified);
|
|
||||||
|
|
||||||
if (not reference_time.has_value()) {
|
if (not reference_time.has_value()) {
|
||||||
utils::error::raise_error(function_name, utils::get_last_error_code(),
|
utils::error::raise_error(function_name, utils::get_last_error_code(),
|
||||||
@ -49,18 +47,18 @@ auto eviction::check_minimum_requirements(const std::string &file_path)
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
auto delay = (config_.get_eviction_delay_mins() * 60UL) *
|
auto delay =
|
||||||
utils::time::NANOS_PER_SECOND;
|
static_cast<std::uint64_t>(config_.get_eviction_delay_mins() * 60U) *
|
||||||
|
utils::time::NANOS_PER_SECOND;
|
||||||
|
|
||||||
return ((reference_time.value() + static_cast<std::uint64_t>(delay)) <=
|
return (reference_time.value() + delay) <= utils::time::get_time_now();
|
||||||
utils::time::get_time_now());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
auto eviction::get_filtered_cached_files() -> std::deque<std::string> {
|
auto eviction::get_filtered_cached_files() -> std::deque<std::string> {
|
||||||
auto list =
|
auto list =
|
||||||
utils::file::get_directory_files(config_.get_cache_directory(), true);
|
utils::file::get_directory_files(config_.get_cache_directory(), true);
|
||||||
list.erase(std::remove_if(list.begin(), list.end(),
|
list.erase(std::remove_if(list.begin(), list.end(),
|
||||||
[this](const std::string &path) -> bool {
|
[this](auto &&path) -> bool {
|
||||||
return not this->check_minimum_requirements(path);
|
return not this->check_minimum_requirements(path);
|
||||||
}),
|
}),
|
||||||
list.end());
|
list.end());
|
||||||
@ -70,65 +68,54 @@ auto eviction::get_filtered_cached_files() -> std::deque<std::string> {
|
|||||||
void eviction::service_function() {
|
void eviction::service_function() {
|
||||||
REPERTORY_USES_FUNCTION_NAME();
|
REPERTORY_USES_FUNCTION_NAME();
|
||||||
|
|
||||||
auto should_evict = true;
|
auto cached_files_list = get_filtered_cached_files();
|
||||||
|
while (not get_stop_requested() && not cached_files_list.empty()) {
|
||||||
|
auto file_path = cached_files_list.front();
|
||||||
|
cached_files_list.pop_front();
|
||||||
|
|
||||||
// Handle maximum cache size eviction
|
try {
|
||||||
auto used_bytes =
|
std::string api_path;
|
||||||
utils::file::directory{config_.get_cache_directory()}.size();
|
if (provider_.get_api_path_from_source(file_path, api_path) !=
|
||||||
if (config_.get_enable_max_cache_size()) {
|
api_error::success) {
|
||||||
should_evict = (used_bytes > config_.get_max_cache_size_bytes());
|
continue;
|
||||||
}
|
|
||||||
|
|
||||||
if (should_evict) {
|
|
||||||
// Remove cached source files that don't meet minimum requirements
|
|
||||||
auto cached_files_list = get_filtered_cached_files();
|
|
||||||
while (not get_stop_requested() && should_evict &&
|
|
||||||
not cached_files_list.empty()) {
|
|
||||||
try {
|
|
||||||
std::string api_path;
|
|
||||||
if (provider_.get_api_path_from_source(
|
|
||||||
cached_files_list.front(), api_path) == api_error::success) {
|
|
||||||
api_file file{};
|
|
||||||
filesystem_item fsi{};
|
|
||||||
if (provider_.get_filesystem_item_and_file(api_path, file, fsi) ==
|
|
||||||
api_error::success) {
|
|
||||||
// Only evict files that match expected size
|
|
||||||
auto opt_size = utils::file::file{cached_files_list.front()}.size();
|
|
||||||
if (opt_size.has_value()) {
|
|
||||||
auto file_size{opt_size.value()};
|
|
||||||
if (file_size == fsi.size) {
|
|
||||||
// Try to evict file
|
|
||||||
if (fm_.evict_file(fsi.api_path) &&
|
|
||||||
config_.get_enable_max_cache_size()) {
|
|
||||||
// Restrict number of items evicted if maximum cache size is
|
|
||||||
// enabled
|
|
||||||
used_bytes -= file_size;
|
|
||||||
should_evict =
|
|
||||||
(used_bytes > config_.get_max_cache_size_bytes());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
utils::error::raise_api_path_error(
|
|
||||||
function_name, file.api_path, file.source_path,
|
|
||||||
utils::get_last_error_code(), "failed to get file size");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch (const std::exception &ex) {
|
|
||||||
utils::error::raise_error(function_name, ex,
|
|
||||||
"failed to process cached file|sp|" +
|
|
||||||
cached_files_list.front());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
cached_files_list.pop_front();
|
api_file file{};
|
||||||
|
filesystem_item fsi{};
|
||||||
|
if (provider_.get_filesystem_item_and_file(api_path, file, fsi) !=
|
||||||
|
api_error::success) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
auto opt_size = utils::file::file{file_path}.size();
|
||||||
|
if (not opt_size.has_value()) {
|
||||||
|
utils::error::raise_api_path_error(
|
||||||
|
function_name, file.api_path, file.source_path,
|
||||||
|
utils::get_last_error_code(), "failed to get file size");
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (opt_size.value() != fsi.size) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
fm_.evict_file(fsi.api_path);
|
||||||
|
} catch (const std::exception &ex) {
|
||||||
|
utils::error::raise_error(
|
||||||
|
function_name, ex,
|
||||||
|
fmt::format("failed to process cached file|sp|{}", file_path));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (not get_stop_requested()) {
|
if (get_stop_requested()) {
|
||||||
unique_mutex_lock lock(get_mutex());
|
return;
|
||||||
if (not get_stop_requested()) {
|
|
||||||
get_notify().wait_for(lock, 30s);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
unique_mutex_lock lock(get_mutex());
|
||||||
|
if (get_stop_requested()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
get_notify().wait_for(lock, 30s);
|
||||||
}
|
}
|
||||||
} // namespace repertory
|
} // namespace repertory
|
||||||
|
@ -64,7 +64,6 @@ const auto DEFAULT_SIA_CONFIG = "{\n"
|
|||||||
" \"EnableChunkDownloaderTimeout\": true,\n"
|
" \"EnableChunkDownloaderTimeout\": true,\n"
|
||||||
" \"EnableCommDurationEvents\": false,\n"
|
" \"EnableCommDurationEvents\": false,\n"
|
||||||
" \"EnableDriveEvents\": false,\n"
|
" \"EnableDriveEvents\": false,\n"
|
||||||
" \"EnableMaxCacheSize\": false,\n"
|
|
||||||
#if defined(_WIN32)
|
#if defined(_WIN32)
|
||||||
" \"EnableMountManager\": false,\n"
|
" \"EnableMountManager\": false,\n"
|
||||||
#endif
|
#endif
|
||||||
@ -112,7 +111,6 @@ const auto DEFAULT_S3_CONFIG = "{\n"
|
|||||||
" \"EnableChunkDownloaderTimeout\": true,\n"
|
" \"EnableChunkDownloaderTimeout\": true,\n"
|
||||||
" \"EnableCommDurationEvents\": false,\n"
|
" \"EnableCommDurationEvents\": false,\n"
|
||||||
" \"EnableDriveEvents\": false,\n"
|
" \"EnableDriveEvents\": false,\n"
|
||||||
" \"EnableMaxCacheSize\": false,\n"
|
|
||||||
#if defined(_WIN32)
|
#if defined(_WIN32)
|
||||||
" \"EnableMountManager\": false,\n"
|
" \"EnableMountManager\": false,\n"
|
||||||
#endif
|
#endif
|
||||||
@ -298,19 +296,6 @@ TEST_F(config_test, enable_drive_events) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST_F(config_test, enable_max_cache_size) {
|
|
||||||
bool original_value{};
|
|
||||||
{
|
|
||||||
app_config config(provider_type::sia, sia_directory);
|
|
||||||
original_value = config.get_enable_max_cache_size();
|
|
||||||
config.set_enable_max_cache_size(not original_value);
|
|
||||||
EXPECT_EQ(not original_value, config.get_enable_max_cache_size());
|
|
||||||
}
|
|
||||||
{
|
|
||||||
app_config config(provider_type::sia, sia_directory);
|
|
||||||
EXPECT_EQ(not original_value, config.get_enable_max_cache_size());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
#if defined(_WIN32)
|
#if defined(_WIN32)
|
||||||
TEST_F(config_test, enable_mount_manager) {
|
TEST_F(config_test, enable_mount_manager) {
|
||||||
bool original_value;
|
bool original_value;
|
||||||
|
Loading…
x
Reference in New Issue
Block a user