From e34f0efc79985c779a2fb4bc55be5e10c889b794 Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Wed, 18 Dec 2024 12:05:51 -0600 Subject: [PATCH] Writes should block when maximum cache size is reached #25 --- .../include/file_manager/cache_size_mgr.hpp | 65 +++++++++++ .../providers/encrypt/encrypt_provider.hpp | 2 +- .../src/file_manager/cache_size_mgr.cpp | 106 ++++++++++++++++++ .../src/file_manager/file_manager.cpp | 28 ++++- .../src/file_manager/open_file.cpp | 17 +++ .../src/providers/base_provider.cpp | 39 ++++--- .../providers/encrypt/encrypt_provider.cpp | 3 + 7 files changed, 238 insertions(+), 22 deletions(-) create mode 100644 repertory/librepertory/include/file_manager/cache_size_mgr.hpp create mode 100644 repertory/librepertory/src/file_manager/cache_size_mgr.cpp diff --git a/repertory/librepertory/include/file_manager/cache_size_mgr.hpp b/repertory/librepertory/include/file_manager/cache_size_mgr.hpp new file mode 100644 index 00000000..022d80a6 --- /dev/null +++ b/repertory/librepertory/include/file_manager/cache_size_mgr.hpp @@ -0,0 +1,65 @@ +/* + Copyright <2018-2024> + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ +#ifndef REPERTORY_INCLUDE_FILE_MANAGER_CACHE_SIZE_MGR_HPP_ +#define REPERTORY_INCLUDE_FILE_MANAGER_CACHE_SIZE_MGR_HPP_ + +#include "types/repertory.hpp" + +namespace repertory { +class app_config; + +class cache_size_mgr final { +public: + cache_size_mgr(const cache_size_mgr &) = delete; + cache_size_mgr(cache_size_mgr &&) = delete; + auto operator=(const cache_size_mgr &) -> cache_size_mgr & = delete; + auto operator=(cache_size_mgr &&) -> cache_size_mgr & = delete; + +protected: + cache_size_mgr() = default; + + ~cache_size_mgr() { stop(); } + +private: + static cache_size_mgr instance_; + +private: + app_config *cfg_{nullptr}; + std::uint64_t cache_size_{0U}; + std::mutex mtx_; + std::condition_variable notify_; + stop_type stop_requested_{false}; + +public: + [[nodiscard]] auto expand(std::uint64_t size) -> api_error; + + void initialize(app_config *cfg); + + [[nodiscard]] static auto instance() -> cache_size_mgr & { return instance_; } + + [[nodiscard]] auto shrink(std::uint64_t size) -> api_error; + + void stop(); +}; +} // namespace repertory + +#endif // REPERTORY_INCLUDE_FILE_MANAGER_CACHE_SIZE_MGR_HPP_ diff --git a/repertory/librepertory/include/providers/encrypt/encrypt_provider.hpp b/repertory/librepertory/include/providers/encrypt/encrypt_provider.hpp index ea4847a9..6588b35b 100644 --- a/repertory/librepertory/include/providers/encrypt/encrypt_provider.hpp +++ b/repertory/librepertory/include/providers/encrypt/encrypt_provider.hpp @@ -54,9 +54,9 @@ private: private: app_config &config_; - std::unique_ptr db_; private: + std::unique_ptr db_{nullptr}; i_file_manager *fm_{nullptr}; std::unordered_map> reader_lookup_; std::recursive_mutex reader_lookup_mtx_; diff --git a/repertory/librepertory/src/file_manager/cache_size_mgr.cpp b/repertory/librepertory/src/file_manager/cache_size_mgr.cpp new file mode 100644 index 00000000..138e1647 --- /dev/null +++ b/repertory/librepertory/src/file_manager/cache_size_mgr.cpp @@ -0,0 +1,106 @@ +/* + Copyright <2018-2024> + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ +#include "file_manager/cache_size_mgr.hpp" + +#include "app_config.hpp" +#include "events/event.hpp" +#include "events/event_system.hpp" +#include "types/startup_exception.hpp" +#include "utils/file_utils.hpp" + +namespace repertory { +// clang-format off +E_SIMPLE2(max_cache_size_reached, warn, true, + std::uint64_t, cache_size, sz, E_FROM_UINT64, + std::uint64_t, max_cache_size, max, E_FROM_UINT64 +); +// clang-format on + +cache_size_mgr cache_size_mgr::instance_{}; + +auto cache_size_mgr::expand(std::uint64_t size) -> api_error { + unique_mutex_lock lock(mtx_); + if (cfg_ == nullptr) { + return api_error::error; + } + + cache_size_ += size; + + auto max_cache_size = cfg_->get_max_cache_size_bytes(); + + while (not stop_requested_ && cache_size_ > max_cache_size) { + event_system::instance().raise(cache_size_, + max_cache_size); + notify_.notify_all(); + notify_.wait(lock); + } + + notify_.notify_all(); + + return stop_requested_ ? api_error::error : api_error::success; +} + +void cache_size_mgr::initialize(app_config *cfg) { + if (cfg == nullptr) { + throw startup_exception("app_config must not be null"); + } + + mutex_lock lock(mtx_); + cfg_ = cfg; + + stop_requested_ = false; + + auto cache_dir = utils::file::directory{cfg_->get_cache_directory()}; + if (not cache_dir.create_directory()) { + throw startup_exception(fmt::format("failed to create cache directory|{}", + cache_dir.get_path())); + } + + cache_size_ = cache_dir.size(false); + + notify_.notify_all(); +} + +auto cache_size_mgr::shrink(std::uint64_t size) -> api_error { + mutex_lock lock(mtx_); + if (size >= cache_size_) { + cache_size_ -= size; + } else { + cache_size_ = 0U; + } + + notify_.notify_all(); + + return stop_requested_ ? api_error::error : api_error::success; +} + +void cache_size_mgr::stop() { + if (stop_requested_) { + return; + } + + stop_requested_ = true; + + mutex_lock lock(mtx_); + notify_.notify_all(); +} +} // namespace repertory diff --git a/repertory/librepertory/src/file_manager/file_manager.cpp b/repertory/librepertory/src/file_manager/file_manager.cpp index e9827383..f641d25b 100644 --- a/repertory/librepertory/src/file_manager/file_manager.cpp +++ b/repertory/librepertory/src/file_manager/file_manager.cpp @@ -23,6 +23,7 @@ #include "app_config.hpp" #include "db/file_mgr_db.hpp" +#include "file_manager/cache_size_mgr.hpp" #include "file_manager/events.hpp" #include "file_manager/open_file.hpp" #include "file_manager/open_file_base.hpp" @@ -173,8 +174,16 @@ auto file_manager::evict_file(const std::string &api_path) -> bool { open_file_lookup_.erase(api_path); - auto removed = utils::file::file{source_path}.remove(); + auto file = utils::file::file{source_path}; + auto file_size = file.size().value_or(0U); + auto removed = file.remove(); if (removed) { + res = cache_size_mgr::instance().shrink(file_size); + if (res != api_error::success) { + utils::error::raise_api_path_error(function_name, api_path, res, + "failed to shrink cache"); + } + event_system::instance().raise(api_path, source_path); } @@ -464,12 +473,21 @@ auto file_manager::remove_file(const std::string &api_path) -> api_error { return res; } - if (not utils::file::file{fsi.source_path}.remove()) { - utils::error::raise_api_path_error( - function_name, fsi.api_path, fsi.source_path, - utils::get_last_error_code(), "failed to delete source"); + auto file = utils::file::file{fsi.source_path}; + auto file_size = file.size().value_or(0U); + if (file.remove()) { + res = cache_size_mgr::instance().shrink(file_size); + if (res != api_error::success) { + utils::error::raise_api_path_error(function_name, api_path, res, + "failed to shrink cache"); + } + return api_error::success; } + utils::error::raise_api_path_error( + function_name, fsi.api_path, fsi.source_path, + utils::get_last_error_code(), "failed to delete source"); + return api_error::success; } diff --git a/repertory/librepertory/src/file_manager/open_file.cpp b/repertory/librepertory/src/file_manager/open_file.cpp index e019cbef..b39ee174 100644 --- a/repertory/librepertory/src/file_manager/open_file.cpp +++ b/repertory/librepertory/src/file_manager/open_file.cpp @@ -21,6 +21,7 @@ */ #include "file_manager/open_file.hpp" +#include "file_manager/cache_size_mgr.hpp" #include "file_manager/events.hpp" #include "file_manager/i_upload_manager.hpp" #include "platform/platform.hpp" @@ -419,6 +420,22 @@ auto open_file::resize(std::uint64_t new_file_size) -> api_error { return api_error::invalid_operation; } + if (new_file_size == fsi_.size) { + return api_error::success; + } + + if (new_file_size > fsi_.size) { + auto res = cache_size_mgr::instance().expand(new_file_size - fsi_.size); + if (res != api_error::success) { + return res; + } + } else { + auto res = cache_size_mgr::instance().shrink(fsi_.size - new_file_size); + if (res != api_error::success) { + return res; + } + } + return native_operation( new_file_size, [this, &new_file_size](native_handle) -> api_error { return nf_->truncate(new_file_size) ? api_error::success diff --git a/repertory/librepertory/src/providers/base_provider.cpp b/repertory/librepertory/src/providers/base_provider.cpp index 3d029b21..59e6087b 100644 --- a/repertory/librepertory/src/providers/base_provider.cpp +++ b/repertory/librepertory/src/providers/base_provider.cpp @@ -25,6 +25,7 @@ #include "db/meta_db.hpp" #include "events/event_system.hpp" #include "events/events.hpp" +#include "file_manager/cache_size_mgr.hpp" #include "file_manager/i_file_manager.hpp" #include "platform/platform.hpp" #include "utils/file_utils.hpp" @@ -49,8 +50,8 @@ void base_provider::add_all_items(const stop_type &stop_requested) { } auto base_provider::create_api_file(std::string path, std::string key, - std::uint64_t size, - std::uint64_t file_time) -> api_file { + std::uint64_t size, std::uint64_t file_time) + -> api_file { api_file file{}; file.api_path = utils::path::create_api_path(path); file.api_parent = utils::path::get_parent_api_path(file.api_path); @@ -82,8 +83,8 @@ auto base_provider::create_api_file(std::string path, std::uint64_t size, } auto base_provider::create_directory_clone_source_meta( - const std::string &source_api_path, - const std::string &api_path) -> api_error { + const std::string &source_api_path, const std::string &api_path) + -> api_error { REPERTORY_USES_FUNCTION_NAME(); bool exists{}; @@ -180,8 +181,8 @@ auto base_provider::create_directory(const std::string &api_path, return set_item_meta(api_path, meta); } -auto base_provider::create_file(const std::string &api_path, - api_meta_map &meta) -> api_error { +auto base_provider::create_file(const std::string &api_path, api_meta_map &meta) + -> api_error { REPERTORY_USES_FUNCTION_NAME(); bool exists{}; @@ -238,8 +239,9 @@ auto base_provider::create_file(const std::string &api_path, return api_error::error; } -auto base_provider::get_api_path_from_source( - const std::string &source_path, std::string &api_path) const -> api_error { +auto base_provider::get_api_path_from_source(const std::string &source_path, + std::string &api_path) const + -> api_error { REPERTORY_USES_FUNCTION_NAME(); if (source_path.empty()) { @@ -252,8 +254,9 @@ auto base_provider::get_api_path_from_source( return db3_->get_api_path(source_path, api_path); } -auto base_provider::get_directory_items( - const std::string &api_path, directory_item_list &list) const -> api_error { +auto base_provider::get_directory_items(const std::string &api_path, + directory_item_list &list) const + -> api_error { REPERTORY_USES_FUNCTION_NAME(); bool exists{}; @@ -317,9 +320,10 @@ auto base_provider::get_file_size(const std::string &api_path, return api_error::success; } -auto base_provider::get_filesystem_item( - const std::string &api_path, bool directory, - filesystem_item &fsi) const -> api_error { +auto base_provider::get_filesystem_item(const std::string &api_path, + bool directory, + filesystem_item &fsi) const + -> api_error { bool exists{}; auto res = is_directory(api_path, exists); if (res != api_error::success) { @@ -352,9 +356,10 @@ auto base_provider::get_filesystem_item( return api_error::success; } -auto base_provider::get_filesystem_item_and_file( - const std::string &api_path, api_file &file, - filesystem_item &fsi) const -> api_error { +auto base_provider::get_filesystem_item_and_file(const std::string &api_path, + api_file &file, + filesystem_item &fsi) const + -> api_error { auto res = get_file(api_path, file); if (res != api_error::success) { return res; @@ -751,6 +756,8 @@ auto base_provider::start(api_item_added_callback api_item_added, return false; } + cache_size_mgr::instance().initialize(&config_); + polling::instance().set_callback({ "check_deleted", polling::frequency::low, diff --git a/repertory/librepertory/src/providers/encrypt/encrypt_provider.cpp b/repertory/librepertory/src/providers/encrypt/encrypt_provider.cpp index 005f721d..c73c77f2 100644 --- a/repertory/librepertory/src/providers/encrypt/encrypt_provider.cpp +++ b/repertory/librepertory/src/providers/encrypt/encrypt_provider.cpp @@ -24,6 +24,7 @@ #include "db/file_db.hpp" #include "events/event_system.hpp" #include "events/events.hpp" +#include "file_manager/cache_size_mgr.hpp" #include "types/repertory.hpp" #include "types/startup_exception.hpp" #include "utils/collection.hpp" @@ -868,6 +869,8 @@ auto encrypt_provider::start(api_item_added_callback /*api_item_added*/, } } + cache_size_mgr::instance().initialize(&config_); + polling::instance().set_callback({ "check_deleted", polling::frequency::low,