From 9ebfb3087141e0bf5d480186033162083db1a8c9 Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Tue, 24 Dec 2024 21:21:26 -0600 Subject: [PATCH] fix cache size --- .../include/file_manager/file_manager.hpp | 5 ++ .../include/file_manager/i_open_file.hpp | 38 +++++++------ .../include/file_manager/open_file.hpp | 2 + .../include/file_manager/open_file_base.hpp | 14 +++-- .../src/file_manager/file_manager.cpp | 55 +++++++++++-------- .../src/file_manager/open_file.cpp | 54 ++++++++++-------- 6 files changed, 99 insertions(+), 69 deletions(-) diff --git a/repertory/librepertory/include/file_manager/file_manager.hpp b/repertory/librepertory/include/file_manager/file_manager.hpp index 85a28570..aff84c36 100644 --- a/repertory/librepertory/include/file_manager/file_manager.hpp +++ b/repertory/librepertory/include/file_manager/file_manager.hpp @@ -108,6 +108,11 @@ public: void remove_resume(const std::string &api_path, const std::string &source_path) override; + static auto remove_source_and_shrink_cache(const std::string &api_path, + const std::string &source_path, + std::uint64_t file_size, + bool allocated) -> bool; + void remove_upload(const std::string &api_path) override; void store_resume(const i_open_file &file) override; diff --git a/repertory/librepertory/include/file_manager/i_open_file.hpp b/repertory/librepertory/include/file_manager/i_open_file.hpp index d9aeed62..758d9a17 100644 --- a/repertory/librepertory/include/file_manager/i_open_file.hpp +++ b/repertory/librepertory/include/file_manager/i_open_file.hpp @@ -40,25 +40,25 @@ public: [[nodiscard]] virtual auto get_filesystem_item() const -> filesystem_item = 0; - [[nodiscard]] virtual auto - get_open_data() -> std::map & = 0; + [[nodiscard]] virtual auto get_open_data() + -> std::map & = 0; - [[nodiscard]] virtual auto - get_open_data() const -> const std::map & = 0; + [[nodiscard]] virtual auto get_open_data() const + -> const std::map & = 0; - [[nodiscard]] virtual auto - get_open_data(std::uint64_t handle) -> open_file_data & = 0; + [[nodiscard]] virtual auto get_open_data(std::uint64_t handle) + -> open_file_data & = 0; - [[nodiscard]] virtual auto - get_open_data(std::uint64_t handle) const -> const open_file_data & = 0; + [[nodiscard]] virtual auto get_open_data(std::uint64_t handle) const + -> const open_file_data & = 0; [[nodiscard]] virtual auto get_open_file_count() const -> std::size_t = 0; - [[nodiscard]] virtual auto - get_read_state() const -> boost::dynamic_bitset<> = 0; + [[nodiscard]] virtual auto get_read_state() const + -> boost::dynamic_bitset<> = 0; - [[nodiscard]] virtual auto - get_read_state(std::size_t chunk) const -> bool = 0; + [[nodiscard]] virtual auto get_read_state(std::size_t chunk) const + -> bool = 0; [[nodiscard]] virtual auto get_source_path() const -> std::string = 0; @@ -74,11 +74,11 @@ public: native_operation_callback callback) -> api_error = 0; [[nodiscard]] virtual auto read(std::size_t read_size, - std::uint64_t read_offset, - data_buffer &data) -> api_error = 0; + std::uint64_t read_offset, data_buffer &data) + -> api_error = 0; - [[nodiscard]] virtual auto - resize(std::uint64_t new_file_size) -> api_error = 0; + [[nodiscard]] virtual auto resize(std::uint64_t new_file_size) + -> api_error = 0; virtual void set_api_path(const std::string &api_path) = 0; @@ -93,12 +93,14 @@ class i_closeable_open_file : public i_open_file { public: virtual void add(std::uint64_t handle, open_file_data ofd) = 0; + [[nodiscard]] virtual auto get_allocated() const -> bool = 0; + [[nodiscard]] virtual auto can_close() const -> bool = 0; virtual auto close() -> bool = 0; - [[nodiscard]] virtual auto - get_handles() const -> std::vector = 0; + [[nodiscard]] virtual auto get_handles() const + -> std::vector = 0; [[nodiscard]] virtual auto is_complete() const -> bool = 0; diff --git a/repertory/librepertory/include/file_manager/open_file.hpp b/repertory/librepertory/include/file_manager/open_file.hpp index b317424b..18f0088b 100644 --- a/repertory/librepertory/include/file_manager/open_file.hpp +++ b/repertory/librepertory/include/file_manager/open_file.hpp @@ -98,6 +98,8 @@ protected: public: auto close() -> bool override; + [[nodiscard]] auto get_allocated() const -> bool override; + [[nodiscard]] auto get_read_state() const -> boost::dynamic_bitset<> override; [[nodiscard]] auto get_read_state(std::size_t chunk) const -> bool override; diff --git a/repertory/librepertory/include/file_manager/open_file_base.hpp b/repertory/librepertory/include/file_manager/open_file_base.hpp index 99860e4b..2c5b6550 100644 --- a/repertory/librepertory/include/file_manager/open_file_base.hpp +++ b/repertory/librepertory/include/file_manager/open_file_base.hpp @@ -144,6 +144,8 @@ public: auto close() -> bool override; + [[nodiscard]] auto get_allocated() const -> bool override { return false; } + [[nodiscard]] auto get_api_error() const -> api_error; [[nodiscard]] auto get_api_path() const -> std::string override; @@ -158,17 +160,17 @@ public: [[nodiscard]] auto get_handles() const -> std::vector override; - [[nodiscard]] auto - get_open_data() -> std::map & override; + [[nodiscard]] auto get_open_data() + -> std::map & override; [[nodiscard]] auto get_open_data() const -> const std::map & override; - [[nodiscard]] auto - get_open_data(std::uint64_t handle) -> open_file_data & override; + [[nodiscard]] auto get_open_data(std::uint64_t handle) + -> open_file_data & override; - [[nodiscard]] auto - get_open_data(std::uint64_t handle) const -> const open_file_data & override; + [[nodiscard]] auto get_open_data(std::uint64_t handle) const + -> const open_file_data & override; [[nodiscard]] auto get_open_file_count() const -> std::size_t override; diff --git a/repertory/librepertory/src/file_manager/file_manager.cpp b/repertory/librepertory/src/file_manager/file_manager.cpp index e985eace..35953cd4 100644 --- a/repertory/librepertory/src/file_manager/file_manager.cpp +++ b/repertory/librepertory/src/file_manager/file_manager.cpp @@ -185,19 +185,13 @@ auto file_manager::evict_file(const std::string &api_path) -> bool { closeable_file = open_file_lookup_.at(api_path); } open_file_lookup_.erase(api_path); + auto allocated = closeable_file->get_allocated(); open_lock.unlock(); closeable_file.reset(); - - auto file = utils::file::file{source_path}; - auto removed = file.remove(); + auto removed = remove_source_and_shrink_cache(api_path, source_path, fsi.size, + allocated); if (removed) { - res = cache_size_mgr::instance().shrink(fsi.size); - if (res != api_error::success) { - utils::error::raise_api_path_error(function_name, api_path, res, - "failed to shrink cache"); - } - event_system::instance().raise(api_path, source_path); } @@ -568,20 +562,7 @@ auto file_manager::remove_file(const std::string &api_path) -> api_error { return res; } - auto file = utils::file::file{fsi.source_path}; - if (file.remove()) { - res = cache_size_mgr::instance().shrink(fsi.size); - if (res != api_error::success) { - utils::error::raise_api_path_error(function_name, api_path, res, - "failed to shrink cache"); - } - return api_error::success; - } - - utils::error::raise_api_path_error( - function_name, fsi.api_path, fsi.source_path, - utils::get_last_error_code(), "failed to delete source"); - + remove_source_and_shrink_cache(api_path, fsi.source_path, fsi.size, true); return api_error::success; } @@ -611,6 +592,34 @@ void file_manager::remove_resume(const std::string &api_path, } } +auto file_manager::remove_source_and_shrink_cache( + const std::string &api_path, const std::string &source_path, + std::uint64_t file_size, bool allocated) -> bool { + REPERTORY_USES_FUNCTION_NAME(); + + auto file = utils::file::file{source_path}; + + if (not file.remove()) { + utils::error::raise_api_path_error(function_name, api_path, source_path, + utils::get_last_error_code(), + "failed to delete source"); + return false; + } + + auto source_size = file.size().value_or(0U); + if (not allocated || source_size == 0U) { + return true; + } + + auto res = cache_size_mgr::instance().shrink(file_size); + if (res != api_error::success) { + utils::error::raise_api_path_error(function_name, api_path, source_path, + res, "failed to shrink cache"); + } + + return true; +} + void file_manager::remove_upload(const std::string &api_path) { remove_upload(api_path, false); } diff --git a/repertory/librepertory/src/file_manager/open_file.cpp b/repertory/librepertory/src/file_manager/open_file.cpp index 793e409d..0d0fc373 100644 --- a/repertory/librepertory/src/file_manager/open_file.cpp +++ b/repertory/librepertory/src/file_manager/open_file.cpp @@ -23,6 +23,7 @@ #include "file_manager/cache_size_mgr.hpp" #include "file_manager/events.hpp" +#include "file_manager/file_manager.hpp" #include "file_manager/i_upload_manager.hpp" #include "platform/platform.hpp" #include "providers/i_provider.hpp" @@ -111,26 +112,32 @@ auto open_file::check_allocation() -> api_error { return api_error::success; } - allocated = true; - if (fsi_.size == 0U) { + auto file_size = nf_->size().value_or(0U); + if (file_size == fsi_.size) { + allocated = true; return api_error::success; } - auto file_size = nf_->size().value_or(0U); - if (file_size == fsi_.size) { - return api_error::success; + file_lock.unlock(); + if (file_size > fsi_.size) { + auto res = cache_size_mgr::instance().shrink(file_size - fsi_.size); + if (res != api_error::success) { + return res; + } + } else if (file_size < fsi_.size) { + auto res = cache_size_mgr::instance().expand(fsi_.size - file_size); + if (res != api_error::success) { + return res; + } } + file_lock.lock(); if (not nf_->truncate(fsi_.size)) { return api_error::os_error; } - file_lock.unlock(); - if (file_size > fsi_.size) { - return cache_size_mgr::instance().shrink(file_size - fsi_.size); - } - - return cache_size_mgr::instance().expand(fsi_.size - file_size); + allocated = true; + return api_error::success; } auto open_file::close() -> bool { @@ -190,18 +197,16 @@ auto open_file::close() -> bool { return true; } - auto file = utils::file::file{fsi_.source_path}; - if (file.remove()) { - auto res = cache_size_mgr::instance().shrink(fsi_.size); - if (res != api_error::success) { - utils::error::raise_api_path_error(function_name, fsi_.api_path, - fsi_.source_path, res, - "failed to shrink cache"); - } + if (allocated) { + file_manager::remove_source_and_shrink_cache( + fsi_.api_path, fsi_.source_path, fsi_.size, allocated); } else { - utils::error::raise_api_path_error( - function_name, fsi_.api_path, fsi_.source_path, - utils::get_last_error_code(), "failed to delete source file"); + auto file = utils::file::file{fsi_.source_path}; + if (not file.remove()) { + utils::error::raise_api_path_error( + function_name, fsi_.api_path, fsi_.source_path, + utils::get_last_error_code(), "failed to delete source file"); + } } auto parent = utils::path::get_parent_path(fsi_.source_path); @@ -340,6 +345,11 @@ void open_file::download_range(std::size_t begin_chunk, std::size_t end_chunk, } } +auto open_file::get_allocated() const -> bool { + recur_mutex_lock file_lock(file_mtx_); + return allocated; +} + auto open_file::get_read_state() const -> boost::dynamic_bitset<> { recur_mutex_lock file_lock(file_mtx_); return read_state_;