Writes should block when maximum cache size is reached #25

repertory/librepertory/include/file_manager/cache_size_mgr.hpp

@@ -0,0 +1,65 @@
+/*
+  Copyright <2018-2024> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all
+  copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  SOFTWARE.
+*/
+#ifndef REPERTORY_INCLUDE_FILE_MANAGER_CACHE_SIZE_MGR_HPP_
+#define REPERTORY_INCLUDE_FILE_MANAGER_CACHE_SIZE_MGR_HPP_
+
+#include "types/repertory.hpp"
+
+namespace repertory {
+class app_config;
+
+class cache_size_mgr final {
+public:
+  cache_size_mgr(const cache_size_mgr &) = delete;
+  cache_size_mgr(cache_size_mgr &&) = delete;
+  auto operator=(const cache_size_mgr &) -> cache_size_mgr & = delete;
+  auto operator=(cache_size_mgr &&) -> cache_size_mgr & = delete;
+
+protected:
+  cache_size_mgr() = default;
+
+  ~cache_size_mgr() { stop(); }
+
+private:
+  static cache_size_mgr instance_;
+
+private:
+  app_config *cfg_{nullptr};
+  std::uint64_t cache_size_{0U};
+  std::mutex mtx_;
+  std::condition_variable notify_;
+  stop_type stop_requested_{false};
+
+public:
+  [[nodiscard]] auto expand(std::uint64_t size) -> api_error;
+
+  void initialize(app_config *cfg);
+
+  [[nodiscard]] static auto instance() -> cache_size_mgr & { return instance_; }
+
+  [[nodiscard]] auto shrink(std::uint64_t size) -> api_error;
+
+  void stop();
+};
+} // namespace repertory
+
+#endif // REPERTORY_INCLUDE_FILE_MANAGER_CACHE_SIZE_MGR_HPP_

repertory/librepertory/include/providers/encrypt/encrypt_provider.hpp

@@ -54,9 +54,9 @@ private:
 
 private:
   app_config &config_;
-  std::unique_ptr<i_file_db> db_;
 
 private:
+  std::unique_ptr<i_file_db> db_{nullptr};
   i_file_manager *fm_{nullptr};
   std::unordered_map<std::string, std::shared_ptr<reader_info>> reader_lookup_;
   std::recursive_mutex reader_lookup_mtx_;

repertory/librepertory/src/file_manager/cache_size_mgr.cpp

@@ -0,0 +1,106 @@
+/*
+  Copyright <2018-2024> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all
+  copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  SOFTWARE.
+*/
+#include "file_manager/cache_size_mgr.hpp"
+
+#include "app_config.hpp"
+#include "events/event.hpp"
+#include "events/event_system.hpp"
+#include "types/startup_exception.hpp"
+#include "utils/file_utils.hpp"
+
+namespace repertory {
+// clang-format off
+E_SIMPLE2(max_cache_size_reached, warn, true,
+  std::uint64_t, cache_size, sz, E_FROM_UINT64,
+  std::uint64_t, max_cache_size, max, E_FROM_UINT64 
+);
+// clang-format on
+
+cache_size_mgr cache_size_mgr::instance_{};
+
+auto cache_size_mgr::expand(std::uint64_t size) -> api_error {
+  unique_mutex_lock lock(mtx_);
+  if (cfg_ == nullptr) {
+    return api_error::error;
+  }
+
+  cache_size_ += size;
+
+  auto max_cache_size = cfg_->get_max_cache_size_bytes();
+
+  while (not stop_requested_ && cache_size_ > max_cache_size) {
+    event_system::instance().raise<max_cache_size_reached>(cache_size_,
+                                                           max_cache_size);
+    notify_.notify_all();
+    notify_.wait(lock);
+  }
+
+  notify_.notify_all();
+
+  return stop_requested_ ? api_error::error : api_error::success;
+}
+
+void cache_size_mgr::initialize(app_config *cfg) {
+  if (cfg == nullptr) {
+    throw startup_exception("app_config must not be null");
+  }
+
+  mutex_lock lock(mtx_);
+  cfg_ = cfg;
+
+  stop_requested_ = false;
+
+  auto cache_dir = utils::file::directory{cfg_->get_cache_directory()};
+  if (not cache_dir.create_directory()) {
+    throw startup_exception(fmt::format("failed to create cache directory|{}",
+                                        cache_dir.get_path()));
+  }
+
+  cache_size_ = cache_dir.size(false);
+
+  notify_.notify_all();
+}
+
+auto cache_size_mgr::shrink(std::uint64_t size) -> api_error {
+  mutex_lock lock(mtx_);
+  if (size >= cache_size_) {
+    cache_size_ -= size;
+  } else {
+    cache_size_ = 0U;
+  }
+
+  notify_.notify_all();
+
+  return stop_requested_ ? api_error::error : api_error::success;
+}
+
+void cache_size_mgr::stop() {
+  if (stop_requested_) {
+    return;
+  }
+
+  stop_requested_ = true;
+
+  mutex_lock lock(mtx_);
+  notify_.notify_all();
+}
+} // namespace repertory

repertory/librepertory/src/file_manager/file_manager.cpp

@@ -23,6 +23,7 @@
 
 #include "app_config.hpp"
 #include "db/file_mgr_db.hpp"
+#include "file_manager/cache_size_mgr.hpp"
 #include "file_manager/events.hpp"
 #include "file_manager/open_file.hpp"
 #include "file_manager/open_file_base.hpp"
@@ -173,8 +174,16 @@ auto file_manager::evict_file(const std::string &api_path) -> bool {
 
   open_file_lookup_.erase(api_path);
 
-  auto removed = utils::file::file{source_path}.remove();
+  auto file = utils::file::file{source_path};
+  auto file_size = file.size().value_or(0U);
+  auto removed = file.remove();
   if (removed) {
+    res = cache_size_mgr::instance().shrink(file_size);
+    if (res != api_error::success) {
+      utils::error::raise_api_path_error(function_name, api_path, res,
+                                         "failed to shrink cache");
+    }
+
     event_system::instance().raise<filesystem_item_evicted>(api_path,
                                                             source_path);
   }
@@ -464,11 +473,20 @@ auto file_manager::remove_file(const std::string &api_path) -> api_error {
     return res;
   }
 
-  if (not utils::file::file{fsi.source_path}.remove()) {
+  auto file = utils::file::file{fsi.source_path};
+  auto file_size = file.size().value_or(0U);
+  if (file.remove()) {
+    res = cache_size_mgr::instance().shrink(file_size);
+    if (res != api_error::success) {
+      utils::error::raise_api_path_error(function_name, api_path, res,
+                                         "failed to shrink cache");
+    }
+    return api_error::success;
+  }
+
   utils::error::raise_api_path_error(
       function_name, fsi.api_path, fsi.source_path,
       utils::get_last_error_code(), "failed to delete source");
-  }
 
   return api_error::success;
 }

repertory/librepertory/src/file_manager/open_file.cpp

@@ -21,6 +21,7 @@
 */
 #include "file_manager/open_file.hpp"
 
+#include "file_manager/cache_size_mgr.hpp"
 #include "file_manager/events.hpp"
 #include "file_manager/i_upload_manager.hpp"
 #include "platform/platform.hpp"
@@ -419,6 +420,22 @@ auto open_file::resize(std::uint64_t new_file_size) -> api_error {
     return api_error::invalid_operation;
   }
 
+  if (new_file_size == fsi_.size) {
+    return api_error::success;
+  }
+
+  if (new_file_size > fsi_.size) {
+    auto res = cache_size_mgr::instance().expand(new_file_size - fsi_.size);
+    if (res != api_error::success) {
+      return res;
+    }
+  } else {
+    auto res = cache_size_mgr::instance().shrink(fsi_.size - new_file_size);
+    if (res != api_error::success) {
+      return res;
+    }
+  }
+
   return native_operation(
       new_file_size, [this, &new_file_size](native_handle) -> api_error {
         return nf_->truncate(new_file_size) ? api_error::success

repertory/librepertory/src/providers/base_provider.cpp

@@ -25,6 +25,7 @@
 #include "db/meta_db.hpp"
 #include "events/event_system.hpp"
 #include "events/events.hpp"
+#include "file_manager/cache_size_mgr.hpp"
 #include "file_manager/i_file_manager.hpp"
 #include "platform/platform.hpp"
 #include "utils/file_utils.hpp"
@@ -49,8 +50,8 @@ void base_provider::add_all_items(const stop_type &stop_requested) {
 }
 
 auto base_provider::create_api_file(std::string path, std::string key,
-                                    std::uint64_t size,
+                                    std::uint64_t size, std::uint64_t file_time)
-                                    std::uint64_t file_time) -> api_file {
+    -> api_file {
   api_file file{};
   file.api_path = utils::path::create_api_path(path);
   file.api_parent = utils::path::get_parent_api_path(file.api_path);
@@ -82,8 +83,8 @@ auto base_provider::create_api_file(std::string path, std::uint64_t size,
 }
 
 auto base_provider::create_directory_clone_source_meta(
-    const std::string &source_api_path,
+    const std::string &source_api_path, const std::string &api_path)
-    const std::string &api_path) -> api_error {
+    -> api_error {
   REPERTORY_USES_FUNCTION_NAME();
 
   bool exists{};
@@ -180,8 +181,8 @@ auto base_provider::create_directory(const std::string &api_path,
   return set_item_meta(api_path, meta);
 }
 
-auto base_provider::create_file(const std::string &api_path,
+auto base_provider::create_file(const std::string &api_path, api_meta_map &meta)
-                                api_meta_map &meta) -> api_error {
+    -> api_error {
   REPERTORY_USES_FUNCTION_NAME();
 
   bool exists{};
@@ -238,8 +239,9 @@ auto base_provider::create_file(const std::string &api_path,
   return api_error::error;
 }
 
-auto base_provider::get_api_path_from_source(
+auto base_provider::get_api_path_from_source(const std::string &source_path,
-    const std::string &source_path, std::string &api_path) const -> api_error {
+                                             std::string &api_path) const
+    -> api_error {
   REPERTORY_USES_FUNCTION_NAME();
 
   if (source_path.empty()) {
@@ -252,8 +254,9 @@ auto base_provider::get_api_path_from_source(
   return db3_->get_api_path(source_path, api_path);
 }
 
-auto base_provider::get_directory_items(
+auto base_provider::get_directory_items(const std::string &api_path,
-    const std::string &api_path, directory_item_list &list) const -> api_error {
+                                        directory_item_list &list) const
+    -> api_error {
   REPERTORY_USES_FUNCTION_NAME();
 
   bool exists{};
@@ -317,9 +320,10 @@ auto base_provider::get_file_size(const std::string &api_path,
   return api_error::success;
 }
 
-auto base_provider::get_filesystem_item(
+auto base_provider::get_filesystem_item(const std::string &api_path,
-    const std::string &api_path, bool directory,
+                                        bool directory,
-    filesystem_item &fsi) const -> api_error {
+                                        filesystem_item &fsi) const
+    -> api_error {
   bool exists{};
   auto res = is_directory(api_path, exists);
   if (res != api_error::success) {
@@ -352,9 +356,10 @@ auto base_provider::get_filesystem_item(
   return api_error::success;
 }
 
-auto base_provider::get_filesystem_item_and_file(
+auto base_provider::get_filesystem_item_and_file(const std::string &api_path,
-    const std::string &api_path, api_file &file,
+                                                 api_file &file,
-    filesystem_item &fsi) const -> api_error {
+                                                 filesystem_item &fsi) const
+    -> api_error {
   auto res = get_file(api_path, file);
   if (res != api_error::success) {
     return res;
@@ -751,6 +756,8 @@ auto base_provider::start(api_item_added_callback api_item_added,
     return false;
   }
 
+  cache_size_mgr::instance().initialize(&config_);
+
   polling::instance().set_callback({
       "check_deleted",
       polling::frequency::low,

repertory/librepertory/src/providers/encrypt/encrypt_provider.cpp

@@ -24,6 +24,7 @@
 #include "db/file_db.hpp"
 #include "events/event_system.hpp"
 #include "events/events.hpp"
+#include "file_manager/cache_size_mgr.hpp"
 #include "types/repertory.hpp"
 #include "types/startup_exception.hpp"
 #include "utils/collection.hpp"
@@ -868,6 +869,8 @@ auto encrypt_provider::start(api_item_added_callback /*api_item_added*/,
     }
   }
 
+  cache_size_mgr::instance().initialize(&config_);
+
   polling::instance().set_callback({
       "check_deleted",
       polling::frequency::low,