From eee5587bab221deb1971ff4a26d7a8b4586b2a76 Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Fri, 12 Sep 2025 10:25:33 -0500 Subject: [PATCH] Pinning a file should automatically initiate a download to cache #38 --- CHANGELOG.md | 2 + .../include/file_manager/direct_open_file.hpp | 2 + .../include/file_manager/file_manager.hpp | 3 + .../include/file_manager/i_file_manager.hpp | 3 + .../include/file_manager/i_open_file.hpp | 6 +- .../include/file_manager/open_file.hpp | 2 + .../file_manager/ring_buffer_open_file.hpp | 24 ++++---- .../include/providers/base_provider.hpp | 3 +- .../providers/encrypt/encrypt_provider.hpp | 2 +- .../include/providers/i_provider.hpp | 3 +- .../src/file_manager/file_manager.cpp | 61 ++++++++++++++++++- .../src/file_manager/open_file.cpp | 9 +++ .../src/providers/base_provider.cpp | 31 +++++++++- 13 files changed, 129 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 326baff8..4fae90bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,9 @@ ### Issues +* \#33 Complete initial v2.0 documentation * \#34 Add macOS support +* \#38 Pinning a file should automatically initiate a download to cache * \#51 [ui] UI console window should close after launch * \#52 [ui] Add auto-mount on first launch functionality * \#53 Create Windows installer diff --git a/repertory/librepertory/include/file_manager/direct_open_file.hpp b/repertory/librepertory/include/file_manager/direct_open_file.hpp index ff0eebbd..298bbec1 100644 --- a/repertory/librepertory/include/file_manager/direct_open_file.hpp +++ b/repertory/librepertory/include/file_manager/direct_open_file.hpp @@ -67,6 +67,8 @@ protected: -> api_error override; public: + void download() override {} + [[nodiscard]] auto get_source_path() const -> std::string override { return "direct"; } diff --git a/repertory/librepertory/include/file_manager/file_manager.hpp b/repertory/librepertory/include/file_manager/file_manager.hpp index 26732e49..5322e516 100644 --- a/repertory/librepertory/include/file_manager/file_manager.hpp +++ b/repertory/librepertory/include/file_manager/file_manager.hpp @@ -133,6 +133,9 @@ public: open_file_data ofd, std::uint64_t &handle, std::shared_ptr &file) -> api_error; + [[nodiscard]] auto download_pinned_file(const std::string &api_path) + -> bool override; + [[nodiscard]] auto evict_file(const std::string &api_path) -> bool override; [[nodiscard]] auto get_directory_item(const std::string &api_path, diff --git a/repertory/librepertory/include/file_manager/i_file_manager.hpp b/repertory/librepertory/include/file_manager/i_file_manager.hpp index c8fc81c2..1484a568 100644 --- a/repertory/librepertory/include/file_manager/i_file_manager.hpp +++ b/repertory/librepertory/include/file_manager/i_file_manager.hpp @@ -31,6 +31,9 @@ class i_file_manager { INTERFACE_SETUP(i_file_manager); public: + [[nodiscard]] virtual auto download_pinned_file(const std::string &api_path) + -> bool = 0; + [[nodiscard]] virtual auto evict_file(const std::string &api_path) -> bool = 0; diff --git a/repertory/librepertory/include/file_manager/i_open_file.hpp b/repertory/librepertory/include/file_manager/i_open_file.hpp index 55743e1e..5f980a8d 100644 --- a/repertory/librepertory/include/file_manager/i_open_file.hpp +++ b/repertory/librepertory/include/file_manager/i_open_file.hpp @@ -32,6 +32,8 @@ public: using native_operation_callback = std::function; public: + virtual void download() = 0; + [[nodiscard]] virtual auto get_api_path() const -> std::string = 0; [[nodiscard]] virtual auto get_chunk_size() const -> std::size_t = 0; @@ -97,12 +99,12 @@ class i_closeable_open_file : public i_open_file { public: virtual void add(std::uint64_t handle, open_file_data ofd) = 0; - [[nodiscard]] virtual auto get_allocated() const -> bool = 0; - [[nodiscard]] virtual auto can_close() const -> bool = 0; virtual auto close() -> bool = 0; + [[nodiscard]] virtual auto get_allocated() const -> bool = 0; + [[nodiscard]] virtual auto get_handles() const -> std::vector = 0; diff --git a/repertory/librepertory/include/file_manager/open_file.hpp b/repertory/librepertory/include/file_manager/open_file.hpp index eaf75c4b..e584d90e 100644 --- a/repertory/librepertory/include/file_manager/open_file.hpp +++ b/repertory/librepertory/include/file_manager/open_file.hpp @@ -101,6 +101,8 @@ private: public: auto close() -> bool override; + void download() override; + [[nodiscard]] auto get_allocated() const -> bool override; [[nodiscard]] auto get_read_state() const -> boost::dynamic_bitset<> override; diff --git a/repertory/librepertory/include/file_manager/ring_buffer_open_file.hpp b/repertory/librepertory/include/file_manager/ring_buffer_open_file.hpp index 650e4562..ef865fba 100644 --- a/repertory/librepertory/include/file_manager/ring_buffer_open_file.hpp +++ b/repertory/librepertory/include/file_manager/ring_buffer_open_file.hpp @@ -43,8 +43,8 @@ public: ring_buffer_open_file() = delete; ring_buffer_open_file(const ring_buffer_open_file &) noexcept = delete; ring_buffer_open_file(ring_buffer_open_file &&) noexcept = delete; - auto operator=(ring_buffer_open_file &&) noexcept -> ring_buffer_open_file & = - delete; + auto operator=(ring_buffer_open_file &&) noexcept + -> ring_buffer_open_file & = delete; auto operator=(const ring_buffer_open_file &) noexcept -> ring_buffer_open_file & = delete; @@ -57,14 +57,14 @@ private: protected: [[nodiscard]] auto on_check_start() -> bool override; - [[nodiscard]] auto - on_chunk_downloaded(std::size_t chunk, - const data_buffer &buffer) -> api_error override; + [[nodiscard]] auto on_chunk_downloaded(std::size_t chunk, + const data_buffer &buffer) + -> api_error override; - [[nodiscard]] auto - on_read_chunk(std::size_t chunk, std::size_t read_size, - std::uint64_t read_offset, data_buffer &data, - std::size_t &bytes_read) -> api_error override; + [[nodiscard]] auto on_read_chunk(std::size_t chunk, std::size_t read_size, + std::uint64_t read_offset, data_buffer &data, + std::size_t &bytes_read) + -> api_error override; [[nodiscard]] auto use_buffer(std::size_t chunk, std::function func) @@ -75,8 +75,10 @@ public: std::size_t chunk_size, std::size_t ring_size) -> bool; - [[nodiscard]] auto - native_operation(native_operation_callback callback) -> api_error override; + void download() override {} + + [[nodiscard]] auto native_operation(native_operation_callback callback) + -> api_error override; [[nodiscard]] auto native_operation(std::uint64_t /* new_file_size */, native_operation_callback /* callback */) diff --git a/repertory/librepertory/include/providers/base_provider.hpp b/repertory/librepertory/include/providers/base_provider.hpp index 49e58963..c54fb5d8 100644 --- a/repertory/librepertory/include/providers/base_provider.hpp +++ b/repertory/librepertory/include/providers/base_provider.hpp @@ -211,8 +211,7 @@ public: -> api_error override; [[nodiscard]] auto set_item_meta(const std::string &api_path, - const api_meta_map &meta) - -> api_error override; + api_meta_map meta) -> api_error override; [[nodiscard]] auto start(api_item_added_callback api_item_added, i_file_manager *mgr) -> bool override; diff --git a/repertory/librepertory/include/providers/encrypt/encrypt_provider.hpp b/repertory/librepertory/include/providers/encrypt/encrypt_provider.hpp index e586c488..77ddc722 100644 --- a/repertory/librepertory/include/providers/encrypt/encrypt_provider.hpp +++ b/repertory/librepertory/include/providers/encrypt/encrypt_provider.hpp @@ -230,7 +230,7 @@ public: } [[nodiscard]] auto set_item_meta(const std::string & /*api_path*/, - const api_meta_map & /*meta*/) + api_meta_map /*meta*/) -> api_error override { return api_error::success; } diff --git a/repertory/librepertory/include/providers/i_provider.hpp b/repertory/librepertory/include/providers/i_provider.hpp index d1205f94..bd538c0b 100644 --- a/repertory/librepertory/include/providers/i_provider.hpp +++ b/repertory/librepertory/include/providers/i_provider.hpp @@ -148,8 +148,7 @@ public: -> api_error = 0; [[nodiscard]] virtual auto set_item_meta(const std::string &api_path, - const api_meta_map &meta) - -> api_error = 0; + api_meta_map meta) -> api_error = 0; [[nodiscard]] virtual auto start(api_item_added_callback api_item_added, i_file_manager *mgr) -> bool = 0; diff --git a/repertory/librepertory/src/file_manager/file_manager.cpp b/repertory/librepertory/src/file_manager/file_manager.cpp index 106c7f8a..da2cadef 100644 --- a/repertory/librepertory/src/file_manager/file_manager.cpp +++ b/repertory/librepertory/src/file_manager/file_manager.cpp @@ -151,6 +151,58 @@ auto file_manager::create(const std::string &api_path, api_meta_map &meta, return open(api_path, false, ofd, handle, file); } +auto download_pinned_file(const std::string &api_path) -> bool { + REPERTORY_USES_FUNCTION_NAME(); + + recur_mutex_lock file_lock(open_file_mtx_); + + try { + if (provider_.is_read_only()) { + return false; + } + + auto res = provider_.get_file_meta(api_path, META_PINNED, str_pinned); + if (res != api_error::success) { + return false; + } + if (not utils::string::to_bool(str_pinned)) { + return false; + } + + std::uint64_t handle{REPERTORY_INVALID_HANDLE}; + std::shared_ptr open_file; + if (open_file_lookup_.contains(api_path)) { + auto closeable_file = open_file_lookup_.at(api_path); + if (not get_open_file(closeable_file->get_handles().at(0U), true, + open_file)) { + return false; + } + } else { + res = open(api_path, false, {}, handle); + if (res != api_error::success) { + return false; + } + + if (not get_open_file(handle, true, open_file)) { + close(handle); + return false; + } + } + + open_file.download(); + if (handle != REPERTORY_INVALID_HANDLE) { + close(handle); + } + + return true; + } catch (const std::exception &ex) { + utils::error::raise_api_path_error(function_name, api_path, ex, + "failed to download pinned file"); + } + + return false; +} + auto file_manager::evict_file(const std::string &api_path) -> bool { REPERTORY_USES_FUNCTION_NAME(); @@ -904,7 +956,7 @@ void file_manager::start() { queue_upload(entry.api_path, entry.source_path, false); } - for (const auto &entry : mgr_db_->get_resume_list()) { + for (const auto &entry : get_stored_downloads()) { try { filesystem_item fsi{}; auto res = provider_.get_filesystem_item(entry.api_path, false, fsi); @@ -953,6 +1005,8 @@ void file_manager::start() { : 0U, fsi, provider_, entry.read_state, *this); open_file_lookup_[entry.api_path] = closeable_file; + closeable_file.download(); + event_system::instance().raise( fsi.api_path, fsi.source_path, function_name); } catch (const std::exception &ex) { @@ -960,6 +1014,11 @@ void file_manager::start() { } } + for (const auto &api_path : provider_.get_pinned_files()) { + if (not download_pinned_file(api_path)) { + } + } + upload_thread_ = std::make_unique([this] { upload_handler(); }); event_system::instance().raise(function_name, "file_manager"); diff --git a/repertory/librepertory/src/file_manager/open_file.cpp b/repertory/librepertory/src/file_manager/open_file.cpp index 05ec04f9..0fe96ddc 100644 --- a/repertory/librepertory/src/file_manager/open_file.cpp +++ b/repertory/librepertory/src/file_manager/open_file.cpp @@ -90,6 +90,7 @@ open_file::open_file(std::uint64_t chunk_size, std::uint8_t chunk_timeout, read_state_ = read_state.value(); set_modified(); allocated = true; + update_reader(0U); return; } @@ -272,6 +273,14 @@ auto open_file::close() -> bool { return true; } +void open_file::download() { + unique_recur_mutex_lock rw_lock(rw_mtx_); + auto read_chunk = read_chunk_; + rw_lock.unlock(); + + update_reader(read_chunk); +} + void open_file::download_chunk(std::size_t chunk, bool skip_active, bool should_reset) { REPERTORY_USES_FUNCTION_NAME(); diff --git a/repertory/librepertory/src/providers/base_provider.cpp b/repertory/librepertory/src/providers/base_provider.cpp index 2b1dbaca..c4c51932 100644 --- a/repertory/librepertory/src/providers/base_provider.cpp +++ b/repertory/librepertory/src/providers/base_provider.cpp @@ -831,12 +831,37 @@ void base_provider::remove_unmatched_source_files(stop_type &stop_requested) { auto base_provider::set_item_meta(const std::string &api_path, const std::string &key, const std::string &value) -> api_error { - return meta_db_->set_item_meta(api_path, key, value); + auto ret = meta_db_->set_item_meta(api_path, key, value); + if (ret != api_error::success) { + return ret; + } + + if (key == META_PINNED && utils::string::to_bool(value)) { + if (fm_ == nullptr || not fm_->download_pinned_file(api_path)) { + // TODO handle error + } + } + + return ret; } auto base_provider::set_item_meta(const std::string &api_path, - const api_meta_map &meta) -> api_error { - return meta_db_->set_item_meta(api_path, meta); + api_meta_map meta) -> api_error { + if (meta.contains(META_PINNED)) { + auto res = set_item_meta(api_path, META_PINNED, meta.at(META_PINNED)); + if (res != api_error::success) { + return res; + } + + meta.erase(META_PINNED); + } + + auto ret = meta_db_->set_item_meta(api_path, meta); + if (ret != api_error::success) { + return ret; + } + + return ret; } auto base_provider::start(api_item_added_callback api_item_added,