From cfba3d3082a34e56bdaa0a4ec61d8b049e49e752 Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Mon, 2 Dec 2024 13:44:04 -0600 Subject: [PATCH] refactor used drive space and get_file_list --- .../include/file_manager/file_manager.hpp | 2 - .../include/file_manager/i_file_manager.hpp | 19 +- .../include/providers/base_provider.hpp | 149 ++++++++------- .../providers/encrypt/encrypt_provider.hpp | 3 +- .../include/providers/i_provider.hpp | 3 +- .../include/providers/meta_db.hpp | 8 +- .../include/providers/s3/s3_provider.hpp | 96 +++++----- .../include/providers/sia/sia_provider.hpp | 3 +- .../librepertory/include/types/repertory.hpp | 15 +- .../src/drives/fuse/fuse_drive.cpp | 69 +++---- .../src/drives/winfsp/winfsp_drive.cpp | 2 +- .../src/file_manager/file_manager.cpp | 14 -- .../src/providers/base_provider.cpp | 100 ++++++---- .../providers/encrypt/encrypt_provider.cpp | 4 +- .../librepertory/src/providers/meta_db.cpp | 45 ++++- .../src/providers/s3/s3_provider.cpp | 176 +++++++----------- .../src/providers/sia/sia_provider.cpp | 33 +--- .../librepertory/src/types/repertory.cpp | 6 +- repertory/librepertory/src/utils/polling.cpp | 135 +++++++------- .../include/mocks/mock_provider.hpp | 4 +- .../repertory_test/src/providers_test.cpp | 3 +- 21 files changed, 442 insertions(+), 447 deletions(-) diff --git a/repertory/librepertory/include/file_manager/file_manager.hpp b/repertory/librepertory/include/file_manager/file_manager.hpp index 6a272ea4..412dc981 100644 --- a/repertory/librepertory/include/file_manager/file_manager.hpp +++ b/repertory/librepertory/include/file_manager/file_manager.hpp @@ -161,8 +161,6 @@ public: void start(); void stop(); - - void update_used_space(std::uint64_t &used_space) const override; }; } // namespace repertory diff --git a/repertory/librepertory/include/file_manager/i_file_manager.hpp b/repertory/librepertory/include/file_manager/i_file_manager.hpp index 609d6218..972406ab 100644 --- a/repertory/librepertory/include/file_manager/i_file_manager.hpp +++ b/repertory/librepertory/include/file_manager/i_file_manager.hpp @@ -31,21 +31,20 @@ class i_file_manager { INTERFACE_SETUP(i_file_manager); public: - [[nodiscard]] virtual auto - evict_file(const std::string &api_path) -> bool = 0; - - [[nodiscard]] virtual auto get_directory_items( - const std::string &api_path) const -> directory_item_list = 0; + [[nodiscard]] virtual auto evict_file(const std::string &api_path) + -> bool = 0; [[nodiscard]] virtual auto - get_open_files() const -> std::unordered_map = 0; + get_directory_items(const std::string &api_path) const + -> directory_item_list = 0; + + [[nodiscard]] virtual auto get_open_files() const + -> std::unordered_map = 0; [[nodiscard]] virtual auto has_no_open_file_handles() const -> bool = 0; - [[nodiscard]] virtual auto - is_processing(const std::string &api_path) const -> bool = 0; - - virtual void update_used_space(std::uint64_t &used_space) const = 0; + [[nodiscard]] virtual auto is_processing(const std::string &api_path) const + -> bool = 0; }; } // namespace repertory diff --git a/repertory/librepertory/include/providers/base_provider.hpp b/repertory/librepertory/include/providers/base_provider.hpp index 3efa179b..cdb2637a 100644 --- a/repertory/librepertory/include/providers/base_provider.hpp +++ b/repertory/librepertory/include/providers/base_provider.hpp @@ -44,25 +44,24 @@ private: api_item_added_callback api_item_added_; std::unique_ptr db3_; i_file_manager *fm_{}; - mutable std::atomic used_space; + std::atomic stop_requested_{false}; private: void remove_deleted_files(bool source_only); - void update_used_drive_space() const; - protected: - [[nodiscard]] static auto - create_api_file(std::string path, std::string key, std::uint64_t size, - std::uint64_t file_time) -> api_file; + [[nodiscard]] static auto create_api_file(std::string path, std::string key, + std::uint64_t size, + std::uint64_t file_time) + -> api_file; [[nodiscard]] static auto create_api_file(std::string path, std::uint64_t size, api_meta_map &meta) -> api_file; - [[nodiscard]] virtual auto - create_directory_impl(const std::string &api_path, - api_meta_map &meta) -> api_error = 0; + [[nodiscard]] virtual auto create_directory_impl(const std::string &api_path, + api_meta_map &meta) + -> api_error = 0; [[nodiscard]] virtual auto create_file_extra(const std::string & /* api_path */, @@ -74,8 +73,8 @@ protected: return api_item_added_; } - [[nodiscard]] auto - get_api_item_added() const -> const api_item_added_callback & { + [[nodiscard]] auto get_api_item_added() const + -> const api_item_added_callback & { return api_item_added_; } @@ -89,6 +88,8 @@ protected: [[nodiscard]] auto get_db() -> meta_db & { return *db3_; } + [[nodiscard]] auto get_db() const -> const meta_db & { return *db3_; } + [[nodiscard]] virtual auto get_directory_items_impl(const std::string &api_path, directory_item_list &list) const -> api_error = 0; @@ -99,23 +100,25 @@ protected: return fm_; } - [[nodiscard]] virtual auto - get_used_drive_space_impl() const -> std::uint64_t = 0; + [[nodiscard]] virtual auto get_used_drive_space_impl() const + -> std::uint64_t = 0; - [[nodiscard]] virtual auto - remove_directory_impl(const std::string &api_path) -> api_error = 0; + [[nodiscard]] virtual auto remove_directory_impl(const std::string &api_path) + -> api_error = 0; - [[nodiscard]] virtual auto - remove_file_impl(const std::string &api_path) -> api_error = 0; + [[nodiscard]] virtual auto remove_file_impl(const std::string &api_path) + -> api_error = 0; - [[nodiscard]] virtual auto - upload_file_impl(const std::string &api_path, const std::string &source_path, - stop_type &stop_requested) -> api_error = 0; + [[nodiscard]] virtual auto upload_file_impl(const std::string &api_path, + const std::string &source_path, + stop_type &stop_requested) + -> api_error = 0; public: - [[nodiscard]] auto create_directory_clone_source_meta( - const std::string &source_api_path, - const std::string &api_path) -> api_error override; + [[nodiscard]] auto + create_directory_clone_source_meta(const std::string &source_api_path, + const std::string &api_path) + -> api_error override; [[nodiscard]] auto create_directory(const std::string &api_path, api_meta_map &meta) -> api_error override; @@ -123,76 +126,82 @@ public: [[nodiscard]] auto create_file(const std::string &api_path, api_meta_map &meta) -> api_error override; - [[nodiscard]] auto - get_api_path_from_source(const std::string &source_path, - std::string &api_path) const -> api_error override; + [[nodiscard]] auto get_api_path_from_source(const std::string &source_path, + std::string &api_path) const + -> api_error override; + + [[nodiscard]] auto get_directory_items(const std::string &api_path, + directory_item_list &list) const + -> api_error override; + + [[nodiscard]] auto get_file_size(const std::string &api_path, + std::uint64_t &file_size) const + -> api_error override; + + [[nodiscard]] auto get_filesystem_item(const std::string &api_path, + bool directory, + filesystem_item &fsi) const + -> api_error override; + + [[nodiscard]] auto get_filesystem_item_and_file(const std::string &api_path, + api_file &f, + filesystem_item &fsi) const + -> api_error override; [[nodiscard]] auto - get_directory_items(const std::string &api_path, - directory_item_list &list) const -> api_error override; + get_filesystem_item_from_source_path(const std::string &source_path, + filesystem_item &fsi) const + -> api_error override; - [[nodiscard]] auto - get_file_size(const std::string &api_path, - std::uint64_t &file_size) const -> api_error override; + [[nodiscard]] auto get_item_meta(const std::string &api_path, + api_meta_map &meta) const + -> api_error override; - [[nodiscard]] auto - get_filesystem_item(const std::string &api_path, bool directory, - filesystem_item &fsi) const -> api_error override; + [[nodiscard]] auto get_item_meta(const std::string &api_path, + const std::string &key, + std::string &value) const + -> api_error override; - [[nodiscard]] auto get_filesystem_item_and_file( - const std::string &api_path, api_file &f, - filesystem_item &fsi) const -> api_error override; - - [[nodiscard]] auto get_filesystem_item_from_source_path( - const std::string &source_path, - filesystem_item &fsi) const -> api_error override; - - [[nodiscard]] auto - get_item_meta(const std::string &api_path, - api_meta_map &meta) const -> api_error override; - - [[nodiscard]] auto - get_item_meta(const std::string &api_path, const std::string &key, - std::string &value) const -> api_error override; - - [[nodiscard]] auto - get_pinned_files() const -> std::vector override; + [[nodiscard]] auto get_pinned_files() const + -> std::vector override; [[nodiscard]] auto get_total_item_count() const -> std::uint64_t override; [[nodiscard]] auto get_used_drive_space() const -> std::uint64_t override; - [[nodiscard]] auto - is_file_writeable(const std::string &api_path) const -> bool override; + [[nodiscard]] auto is_file_writeable(const std::string &api_path) const + -> bool override; [[nodiscard]] auto is_read_only() const -> bool override { return false; } - [[nodiscard]] auto - remove_directory(const std::string &api_path) -> api_error override; + [[nodiscard]] auto remove_directory(const std::string &api_path) + -> api_error override; - [[nodiscard]] auto - remove_file(const std::string &api_path) -> api_error override; + [[nodiscard]] auto remove_file(const std::string &api_path) + -> api_error override; - [[nodiscard]] auto - remove_item_meta(const std::string &api_path, - const std::string &key) -> api_error override; + [[nodiscard]] auto remove_item_meta(const std::string &api_path, + const std::string &key) + -> api_error override; - [[nodiscard]] auto - set_item_meta(const std::string &api_path, const std::string &key, - const std::string &value) -> api_error override; + [[nodiscard]] auto set_item_meta(const std::string &api_path, + const std::string &key, + const std::string &value) + -> api_error override; - [[nodiscard]] auto - set_item_meta(const std::string &api_path, - const api_meta_map &meta) -> api_error override; + [[nodiscard]] auto set_item_meta(const std::string &api_path, + const api_meta_map &meta) + -> api_error override; [[nodiscard]] auto start(api_item_added_callback api_item_added, i_file_manager *mgr) -> bool override; void stop() override; - [[nodiscard]] auto - upload_file(const std::string &api_path, const std::string &source_path, - stop_type &stop_requested) -> api_error override; + [[nodiscard]] auto upload_file(const std::string &api_path, + const std::string &source_path, + stop_type &stop_requested) + -> api_error override; }; } // namespace repertory diff --git a/repertory/librepertory/include/providers/encrypt/encrypt_provider.hpp b/repertory/librepertory/include/providers/encrypt/encrypt_provider.hpp index 718caff2..cf878933 100644 --- a/repertory/librepertory/include/providers/encrypt/encrypt_provider.hpp +++ b/repertory/librepertory/include/providers/encrypt/encrypt_provider.hpp @@ -111,7 +111,8 @@ public: [[nodiscard]] auto get_file(const std::string &api_path, api_file &file) const -> api_error override; - [[nodiscard]] auto get_file_list(api_file_list &list) const + [[nodiscard]] auto get_file_list(api_file_list &list, + std::string &marker) const -> api_error override; [[nodiscard]] auto get_file_size(const std::string &api_path, diff --git a/repertory/librepertory/include/providers/i_provider.hpp b/repertory/librepertory/include/providers/i_provider.hpp index 5643ca90..a7a3d94b 100644 --- a/repertory/librepertory/include/providers/i_provider.hpp +++ b/repertory/librepertory/include/providers/i_provider.hpp @@ -58,7 +58,8 @@ public: [[nodiscard]] virtual auto get_file(const std::string &api_path, api_file &file) const -> api_error = 0; - [[nodiscard]] virtual auto get_file_list(api_file_list &list) const + [[nodiscard]] virtual auto get_file_list(api_file_list &list, + std::string &marker) const -> api_error = 0; [[nodiscard]] virtual auto get_file_size(const std::string &api_path, diff --git a/repertory/librepertory/include/providers/meta_db.hpp b/repertory/librepertory/include/providers/meta_db.hpp index c2cd9dc8..5f6ce24c 100644 --- a/repertory/librepertory/include/providers/meta_db.hpp +++ b/repertory/librepertory/include/providers/meta_db.hpp @@ -63,14 +63,16 @@ public: [[nodiscard]] auto get_total_item_count() const -> std::uint64_t; + [[nodiscard]] auto get_total_size() const -> std::uint64_t; + void remove_api_path(const std::string &api_path); [[nodiscard]] auto remove_item_meta(const std::string &api_path, const std::string &key) -> api_error; - [[nodiscard]] auto - rename_item_meta(const std::string &from_api_path, - const std::string &to_api_path) -> api_error; + [[nodiscard]] auto rename_item_meta(const std::string &from_api_path, + const std::string &to_api_path) + -> api_error; [[nodiscard]] auto set_item_meta(const std::string &api_path, const std::string &key, diff --git a/repertory/librepertory/include/providers/s3/s3_provider.hpp b/repertory/librepertory/include/providers/s3/s3_provider.hpp index 23b191e2..82e5df4c 100644 --- a/repertory/librepertory/include/providers/s3/s3_provider.hpp +++ b/repertory/librepertory/include/providers/s3/s3_provider.hpp @@ -47,70 +47,73 @@ public: auto operator=(s3_provider &&) -> s3_provider & = delete; private: - [[nodiscard]] auto - add_if_not_found(api_file &file, - const std::string &object_name) const -> api_error; + [[nodiscard]] auto add_if_not_found(api_file &file, + const std::string &object_name) const + -> api_error; - [[nodiscard]] auto - create_file_extra(const std::string &api_path, - api_meta_map &meta) -> api_error override; + [[nodiscard]] auto create_file_extra(const std::string &api_path, + api_meta_map &meta) + -> api_error override; - [[nodiscard]] auto - create_path_directories(const std::string &api_path, - const std::string &key) const -> api_error; + [[nodiscard]] auto create_path_directories(const std::string &api_path, + const std::string &key) const + -> api_error; - [[nodiscard]] auto - decrypt_object_name(std::string &object_name) const -> api_error; + [[nodiscard]] auto decrypt_object_name(std::string &object_name) const + -> api_error; - [[nodiscard]] auto - get_last_modified(bool directory, - const std::string &api_path) const -> std::uint64_t; + [[nodiscard]] auto get_last_modified(bool directory, + const std::string &api_path) const + -> std::uint64_t; [[nodiscard]] auto get_object_info(bool directory, const std::string &api_path, bool &is_encrypted, std::string &object_name, head_object_result &result) const -> api_error; - [[nodiscard]] auto get_object_list( - std::string &response_data, long &response_code, - std::optional delimiter = std::nullopt, - std::optional prefix = std::nullopt, - std::optional token = std::nullopt) const -> bool; + [[nodiscard]] auto + get_object_list(std::string &response_data, long &response_code, + std::optional delimiter = std::nullopt, + std::optional prefix = std::nullopt, + std::optional token = std::nullopt) const + -> bool; protected: - [[nodiscard]] auto - create_directory_impl(const std::string &api_path, - api_meta_map &meta) -> api_error override; + [[nodiscard]] auto create_directory_impl(const std::string &api_path, + api_meta_map &meta) + -> api_error override; [[nodiscard]] auto get_directory_items_impl(const std::string &api_path, directory_item_list &list) const -> api_error override; - [[nodiscard]] auto - get_used_drive_space_impl() const -> std::uint64_t override; + [[nodiscard]] auto get_used_drive_space_impl() const + -> std::uint64_t override; - [[nodiscard]] auto - remove_directory_impl(const std::string &api_path) -> api_error override; + [[nodiscard]] auto remove_directory_impl(const std::string &api_path) + -> api_error override; - [[nodiscard]] auto - remove_file_impl(const std::string &api_path) -> api_error override; + [[nodiscard]] auto remove_file_impl(const std::string &api_path) + -> api_error override; - [[nodiscard]] auto - upload_file_impl(const std::string &api_path, const std::string &source_path, - stop_type &stop_requested) -> api_error override; + [[nodiscard]] auto upload_file_impl(const std::string &api_path, + const std::string &source_path, + stop_type &stop_requested) + -> api_error override; public: - [[nodiscard]] static auto - convert_api_date(std::string_view date) -> std::uint64_t; + [[nodiscard]] static auto convert_api_date(std::string_view date) + -> std::uint64_t; [[nodiscard]] auto get_directory_item_count(const std::string &api_path) const -> std::uint64_t override; - [[nodiscard]] auto get_file(const std::string &api_path, - api_file &file) const -> api_error override; + [[nodiscard]] auto get_file(const std::string &api_path, api_file &file) const + -> api_error override; - [[nodiscard]] auto - get_file_list(api_file_list &list) const -> api_error override; + [[nodiscard]] auto get_file_list(api_file_list &list, + std::string &marker) const + -> api_error override; [[nodiscard]] auto get_total_drive_space() const -> std::uint64_t override; @@ -121,8 +124,8 @@ public: [[nodiscard]] auto is_directory(const std::string &api_path, bool &exists) const -> api_error override; - [[nodiscard]] auto is_file(const std::string &api_path, - bool &exists) const -> api_error override; + [[nodiscard]] auto is_file(const std::string &api_path, bool &exists) const + -> api_error override; [[nodiscard]] auto is_online() const -> bool override; @@ -130,14 +133,15 @@ public: return false; }; - [[nodiscard]] auto - read_file_bytes(const std::string &api_path, std::size_t size, - std::uint64_t offset, data_buffer &data, - stop_type &stop_requested) -> api_error override; + [[nodiscard]] auto read_file_bytes(const std::string &api_path, + std::size_t size, std::uint64_t offset, + data_buffer &data, + stop_type &stop_requested) + -> api_error override; - [[nodiscard]] auto - rename_file(const std::string &from_api_path, - const std::string &to_api_path) -> api_error override; + [[nodiscard]] auto rename_file(const std::string &from_api_path, + const std::string &to_api_path) + -> api_error override; [[nodiscard]] auto start(api_item_added_callback api_item_added, i_file_manager *mgr) -> bool override; diff --git a/repertory/librepertory/include/providers/sia/sia_provider.hpp b/repertory/librepertory/include/providers/sia/sia_provider.hpp index f693104e..1903d8d6 100644 --- a/repertory/librepertory/include/providers/sia/sia_provider.hpp +++ b/repertory/librepertory/include/providers/sia/sia_provider.hpp @@ -82,7 +82,8 @@ public: [[nodiscard]] auto get_file(const std::string &api_path, api_file &file) const -> api_error override; - [[nodiscard]] auto get_file_list(api_file_list &list) const + [[nodiscard]] auto get_file_list(api_file_list &list, + std::string &marker) const -> api_error override; [[nodiscard]] auto get_provider_type() const -> provider_type override { diff --git a/repertory/librepertory/include/types/repertory.hpp b/repertory/librepertory/include/types/repertory.hpp index a06be51a..da798253 100644 --- a/repertory/librepertory/include/types/repertory.hpp +++ b/repertory/librepertory/include/types/repertory.hpp @@ -79,6 +79,7 @@ enum class api_error { invalid_version, item_exists, item_not_found, + more_data, no_disk_space, not_implemented, not_supported, @@ -95,16 +96,16 @@ enum class api_error { [[nodiscard]] auto api_error_from_string(std::string_view str) -> api_error; -[[nodiscard]] auto -api_error_to_string(const api_error &error) -> const std::string &; +[[nodiscard]] auto api_error_to_string(const api_error &error) + -> const std::string &; enum class download_type { direct, fallback, ring_buffer }; -[[nodiscard]] auto -download_type_from_string(std::string type, - const download_type &default_type) -> download_type; +[[nodiscard]] auto download_type_from_string(std::string type, + const download_type &default_type) + -> download_type; -[[nodiscard]] auto -download_type_to_string(const download_type &type) -> std::string; +[[nodiscard]] auto download_type_to_string(const download_type &type) + -> std::string; enum class exit_code : std::int32_t { success, diff --git a/repertory/librepertory/src/drives/fuse/fuse_drive.cpp b/repertory/librepertory/src/drives/fuse/fuse_drive.cpp index 37b2418e..dcd12d99 100644 --- a/repertory/librepertory/src/drives/fuse/fuse_drive.cpp +++ b/repertory/librepertory/src/drives/fuse/fuse_drive.cpp @@ -81,8 +81,8 @@ auto fuse_drive::chown_impl(std::string api_path, uid_t uid, gid_t gid, struct fuse_file_info * /*file_info*/) -> api_error { #else -auto fuse_drive::chown_impl(std::string api_path, uid_t uid, - gid_t gid) -> api_error { +auto fuse_drive::chown_impl(std::string api_path, uid_t uid, gid_t gid) + -> api_error { #endif return check_and_perform( api_path, X_OK, [&](api_meta_map &meta) -> api_error { @@ -481,8 +481,8 @@ auto fuse_drive::getattr_impl(std::string api_path, struct stat *unix_st, struct fuse_file_info * /*file_info*/) -> api_error { #else -auto fuse_drive::getattr_impl(std::string api_path, - struct stat *unix_st) -> api_error { +auto fuse_drive::getattr_impl(std::string api_path, struct stat *unix_st) + -> api_error { #endif auto parent = utils::path::get_parent_api_path(api_path); @@ -565,8 +565,8 @@ auto fuse_drive::getxtimes_impl(std::string api_path, struct timespec *bkuptime, #endif // __APPLE__ #if FUSE_USE_VERSION >= 30 -auto fuse_drive::init_impl(struct fuse_conn_info *conn, - struct fuse_config *cfg) -> void * { +auto fuse_drive::init_impl(struct fuse_conn_info *conn, struct fuse_config *cfg) + -> void * { #else void *fuse_drive::init_impl(struct fuse_conn_info *conn) { #endif @@ -588,8 +588,6 @@ void *fuse_drive::init_impl(struct fuse_conn_info *conn) { event_system::instance().start(); was_mounted_ = true; - polling::instance().start(&config_); - fm_ = std::make_unique(config_, provider_); server_ = std::make_unique(config_, provider_, *fm_); if (not provider_.is_read_only()) { @@ -622,6 +620,9 @@ void *fuse_drive::init_impl(struct fuse_conn_info *conn) { if (not lock_data_.set_mount_state(true, get_mount_location(), getpid())) { utils::error::raise_error(function_name, "failed to set mount state"); } + + polling::instance().start(&config_); + event_system::instance().raise(get_mount_location()); } catch (const std::exception &e) { utils::error::raise_error(function_name, e, "exception during fuse init"); @@ -799,8 +800,9 @@ auto fuse_drive::release_impl(std::string /*api_path*/, return api_error::success; } -auto fuse_drive::releasedir_impl( - std::string /*api_path*/, struct fuse_file_info *file_info) -> api_error { +auto fuse_drive::releasedir_impl(std::string /*api_path*/, + struct fuse_file_info *file_info) + -> api_error { auto iter = directory_cache_->get_directory(file_info->fh); if (iter == nullptr) { return api_error::invalid_handle; @@ -818,8 +820,8 @@ auto fuse_drive::rename_directory(const std::string &from_api_path, } auto fuse_drive::rename_file(const std::string &from_api_path, - const std::string &to_api_path, - bool overwrite) -> int { + const std::string &to_api_path, bool overwrite) + -> int { auto res = fm_->rename_file(from_api_path, to_api_path, overwrite); errno = std::abs(utils::from_api_error(res)); return (res == api_error::success) ? 0 : -1; @@ -829,8 +831,8 @@ auto fuse_drive::rename_file(const std::string &from_api_path, auto fuse_drive::rename_impl(std::string from_api_path, std::string to_api_path, unsigned int /*flags*/) -> api_error { #else -auto fuse_drive::rename_impl(std::string from_api_path, - std::string to_api_path) -> api_error { +auto fuse_drive::rename_impl(std::string from_api_path, std::string to_api_path) + -> api_error { #endif auto res = check_parent_access(to_api_path, W_OK | X_OK); if (res != api_error::success) { @@ -944,15 +946,15 @@ auto fuse_drive::getxattr_impl(std::string api_path, const char *name, } #else // __APPLE__ auto fuse_drive::getxattr_impl(std::string api_path, const char *name, - char *value, size_t size, - int &attribute_size) -> api_error { + char *value, size_t size, int &attribute_size) + -> api_error { return getxattr_common(api_path, name, value, size, attribute_size, nullptr); } #endif // __APPLE__ auto fuse_drive::listxattr_impl(std::string api_path, char *buffer, size_t size, - int &required_size, - bool &return_size) -> api_error { + int &required_size, bool &return_size) + -> api_error { auto check_size = (size == 0); auto res = check_parent_access(api_path, X_OK); @@ -992,8 +994,8 @@ auto fuse_drive::listxattr_impl(std::string api_path, char *buffer, size_t size, return res; } -auto fuse_drive::removexattr_impl(std::string api_path, - const char *name) -> api_error { +auto fuse_drive::removexattr_impl(std::string api_path, const char *name) + -> api_error { std::string attribute_name; #if defined(__APPLE__) auto res = parse_xattr_parameters(name, 0, attribute_name, api_path); @@ -1021,8 +1023,8 @@ auto fuse_drive::setxattr_impl(std::string api_path, const char *name, uint32_t position) -> api_error { #else // __APPLE__ auto fuse_drive::setxattr_impl(std::string api_path, const char *name, - const char *value, size_t size, - int flags) -> api_error { + const char *value, size_t size, int flags) + -> api_error { #endif std::string attribute_name; #if defined(__APPLE__) @@ -1100,8 +1102,8 @@ void fuse_drive::set_item_meta(const std::string &api_path, } #if defined(__APPLE__) -auto fuse_drive::setattr_x_impl(std::string api_path, - struct setattr_x *attr) -> api_error { +auto fuse_drive::setattr_x_impl(std::string api_path, struct setattr_x *attr) + -> api_error { bool exists{}; auto res = provider_.is_file(api_path, exists); if (res != api_error::success) { @@ -1155,7 +1157,7 @@ auto fuse_drive::setattr_x_impl(std::string api_path, ts[0].tv_sec = attr->acctime.tv_sec; ts[0].tv_nsec = attr->acctime.tv_nsec; } else { - struct timeval tv {}; + struct timeval tv{}; gettimeofday(&tv, NULL); ts[0].tv_sec = tv.tv_sec; ts[0].tv_nsec = tv.tv_usec * 1000; @@ -1200,8 +1202,9 @@ auto fuse_drive::setattr_x_impl(std::string api_path, return api_error::success; } -auto fuse_drive::setbkuptime_impl( - std::string api_path, const struct timespec *bkuptime) -> api_error { +auto fuse_drive::setbkuptime_impl(std::string api_path, + const struct timespec *bkuptime) + -> api_error { return check_and_perform( api_path, X_OK, [&](api_meta_map &meta) -> api_error { auto nanos = bkuptime->tv_nsec + @@ -1237,8 +1240,8 @@ auto fuse_drive::setvolname_impl(const char * /*volname*/) -> api_error { return api_error::success; } -auto fuse_drive::statfs_x_impl(std::string /*api_path*/, - struct statfs *stbuf) -> api_error { +auto fuse_drive::statfs_x_impl(std::string /*api_path*/, struct statfs *stbuf) + -> api_error { if (statfs(&config_.get_cache_directory()[0], stbuf) != 0) { return api_error::os_error; } @@ -1263,8 +1266,8 @@ auto fuse_drive::statfs_x_impl(std::string /*api_path*/, return api_error::success; } #else // __APPLE__ -auto fuse_drive::statfs_impl(std::string /*api_path*/, - struct statvfs *stbuf) -> api_error { +auto fuse_drive::statfs_impl(std::string /*api_path*/, struct statvfs *stbuf) + -> api_error { if (statvfs(config_.get_cache_directory().data(), stbuf) != 0) { return api_error::os_error; } @@ -1344,8 +1347,8 @@ auto fuse_drive::utimens_impl(std::string api_path, const struct timespec tv[2], struct fuse_file_info * /*file_info*/) -> api_error { #else -auto fuse_drive::utimens_impl(std::string api_path, - const struct timespec tv[2]) -> api_error { +auto fuse_drive::utimens_impl(std::string api_path, const struct timespec tv[2]) + -> api_error { #endif api_meta_map meta; auto res = provider_.get_item_meta(api_path, meta); diff --git a/repertory/librepertory/src/drives/winfsp/winfsp_drive.cpp b/repertory/librepertory/src/drives/winfsp/winfsp_drive.cpp index 854ab901..e5f6e4bc 100644 --- a/repertory/librepertory/src/drives/winfsp/winfsp_drive.cpp +++ b/repertory/librepertory/src/drives/winfsp/winfsp_drive.cpp @@ -619,7 +619,6 @@ auto winfsp_drive::Mounted(PVOID host) -> NTSTATUS { auto *file_system_host{ reinterpret_cast(host), }; - polling::instance().start(&config_); fm_ = std::make_unique(config_, provider_); server_ = std::make_unique(config_, provider_, *fm_); if (not provider_.is_read_only()) { @@ -652,6 +651,7 @@ auto winfsp_drive::Mounted(PVOID host) -> NTSTATUS { utils::error::raise_error(function_name, "failed to set mount state"); } + polling::instance().start(&config_); event_system::instance().raise(mount_location); } catch (const std::exception &e) { utils::error::raise_error(function_name, e, "exception occurred"); diff --git a/repertory/librepertory/src/file_manager/file_manager.cpp b/repertory/librepertory/src/file_manager/file_manager.cpp index 33a05b7c..dc6d1c5b 100644 --- a/repertory/librepertory/src/file_manager/file_manager.cpp +++ b/repertory/librepertory/src/file_manager/file_manager.cpp @@ -1096,18 +1096,4 @@ void file_manager::upload_handler() { upload_notify_.notify_all(); } } - -void file_manager::update_used_space(std::uint64_t &used_space) const { - recur_mutex_lock open_lock(open_file_mtx_); - for (auto &&item : open_file_lookup_) { - std::uint64_t file_size{}; - auto res = provider_.get_file_size(item.second->get_api_path(), file_size); - if ((res == api_error::success) && - (file_size != item.second->get_file_size()) && - (used_space >= file_size)) { - used_space -= file_size; - used_space += item.second->get_file_size(); - } - } -} } // namespace repertory diff --git a/repertory/librepertory/src/providers/base_provider.cpp b/repertory/librepertory/src/providers/base_provider.cpp index 95fcf03e..1c235685 100644 --- a/repertory/librepertory/src/providers/base_provider.cpp +++ b/repertory/librepertory/src/providers/base_provider.cpp @@ -33,8 +33,8 @@ namespace repertory { auto base_provider::create_api_file(std::string path, std::string key, - std::uint64_t size, - std::uint64_t file_time) -> api_file { + std::uint64_t size, std::uint64_t file_time) + -> api_file { api_file file{}; file.api_path = utils::path::create_api_path(path); file.api_parent = utils::path::get_parent_api_path(file.api_path); @@ -66,8 +66,8 @@ auto base_provider::create_api_file(std::string path, std::uint64_t size, } auto base_provider::create_directory_clone_source_meta( - const std::string &source_api_path, - const std::string &api_path) -> api_error { + const std::string &source_api_path, const std::string &api_path) + -> api_error { REPERTORY_USES_FUNCTION_NAME(); bool exists{}; @@ -164,8 +164,8 @@ auto base_provider::create_directory(const std::string &api_path, return set_item_meta(api_path, meta); } -auto base_provider::create_file(const std::string &api_path, - api_meta_map &meta) -> api_error { +auto base_provider::create_file(const std::string &api_path, api_meta_map &meta) + -> api_error { REPERTORY_USES_FUNCTION_NAME(); bool exists{}; @@ -222,8 +222,9 @@ auto base_provider::create_file(const std::string &api_path, return api_error::error; } -auto base_provider::get_api_path_from_source( - const std::string &source_path, std::string &api_path) const -> api_error { +auto base_provider::get_api_path_from_source(const std::string &source_path, + std::string &api_path) const + -> api_error { REPERTORY_USES_FUNCTION_NAME(); if (source_path.empty()) { @@ -236,8 +237,9 @@ auto base_provider::get_api_path_from_source( return db3_->get_api_path(source_path, api_path); } -auto base_provider::get_directory_items( - const std::string &api_path, directory_item_list &list) const -> api_error { +auto base_provider::get_directory_items(const std::string &api_path, + directory_item_list &list) const + -> api_error { REPERTORY_USES_FUNCTION_NAME(); bool exists{}; @@ -301,9 +303,10 @@ auto base_provider::get_file_size(const std::string &api_path, return api_error::success; } -auto base_provider::get_filesystem_item( - const std::string &api_path, bool directory, - filesystem_item &fsi) const -> api_error { +auto base_provider::get_filesystem_item(const std::string &api_path, + bool directory, + filesystem_item &fsi) const + -> api_error { bool exists{}; auto res = is_directory(api_path, exists); if (res != api_error::success) { @@ -336,9 +339,10 @@ auto base_provider::get_filesystem_item( return api_error::success; } -auto base_provider::get_filesystem_item_and_file( - const std::string &api_path, api_file &file, - filesystem_item &fsi) const -> api_error { +auto base_provider::get_filesystem_item_and_file(const std::string &api_path, + api_file &file, + filesystem_item &fsi) const + -> api_error { auto res = get_file(api_path, file); if (res != api_error::success) { return res; @@ -399,11 +403,16 @@ auto base_provider::get_total_item_count() const -> std::uint64_t { } auto base_provider::get_used_drive_space() const -> std::uint64_t { - if (used_space == 0U) { - update_used_drive_space(); + REPERTORY_USES_FUNCTION_NAME(); + + try { + return get_used_drive_space_impl(); + } catch (const std::exception &ex) { + utils::error::raise_error(function_name, ex, + "failed to get used drive space"); } - return used_space; + return 0U; } auto base_provider::is_file_writeable(const std::string &api_path) const @@ -424,6 +433,10 @@ void base_provider::remove_deleted_files(bool source_only) { auto source_list = utils::file::directory{config_.get_cache_directory()}.get_files(); for (auto &&source_file : source_list) { + if (stop_requested_) { + return; + } + filesystem_item fsi{}; if (get_filesystem_item_from_source_path(source_file->get_path(), fsi) != api_error::item_not_found) { @@ -466,16 +479,26 @@ void base_provider::remove_deleted_files(bool source_only) { std::string source_path; }; - api_file_list list{}; - auto res = get_file_list(list); - if (res != api_error::success) { - utils::error::raise_error(function_name, res, "failed to get file list"); - return; + { + api_file_list list{}; + std::string marker; + auto res{api_error::more_data}; + while (not stop_requested_ && res == api_error::more_data) { + res = get_file_list(list, marker); + if (res != api_error::success && res != api_error::more_data) { + utils::error::raise_error(function_name, res, + "failed to get file list"); + return; + } + } } std::vector removed_list{}; - for (auto &&api_path : db3_->get_api_path_list()) { + if (stop_requested_) { + return; + } + api_meta_map meta{}; if (get_item_meta(api_path, meta) != api_error::success) { continue; @@ -510,6 +533,10 @@ void base_provider::remove_deleted_files(bool source_only) { utils::path::combine(config_.get_data_directory(), {"orphaned"}); for (auto &&item : removed_list) { + if (stop_requested_) { + return; + } + if (item.directory) { continue; } @@ -559,6 +586,10 @@ void base_provider::remove_deleted_files(bool source_only) { } for (auto &&item : removed_list) { + if (stop_requested_) { + return; + } + if (not item.directory) { continue; } @@ -664,6 +695,8 @@ auto base_provider::set_item_meta(const std::string &api_path, auto base_provider::start(api_item_added_callback api_item_added, i_file_manager *mgr) -> bool { + stop_requested_ = false; + api_item_added_ = api_item_added; fm_ = mgr; @@ -702,30 +735,15 @@ auto base_provider::start(api_item_added_callback api_item_added, polling::instance().set_callback({"check_deleted", polling::frequency::low, [this]() { remove_deleted_files(false); }}); - polling::instance().set_callback({"drive_space", polling::frequency::low, - [this]() { update_used_drive_space(); }}); return true; } void base_provider::stop() { + stop_requested_ = true; polling::instance().remove_callback("check_deleted"); - polling::instance().remove_callback("drive_space"); db3_.reset(); } -void base_provider::update_used_drive_space() const { - REPERTORY_USES_FUNCTION_NAME(); - - try { - auto next_used_space = get_used_drive_space_impl(); - get_file_mgr()->update_used_space(next_used_space); - used_space = next_used_space; - } catch (const std::exception &ex) { - utils::error::raise_error(function_name, ex, - "failed to get used drive space"); - } -} - auto base_provider::upload_file(const std::string &api_path, const std::string &source_path, stop_type &stop_requested) -> api_error { diff --git a/repertory/librepertory/src/providers/encrypt/encrypt_provider.cpp b/repertory/librepertory/src/providers/encrypt/encrypt_provider.cpp index 73c6f122..d6271fa2 100644 --- a/repertory/librepertory/src/providers/encrypt/encrypt_provider.cpp +++ b/repertory/librepertory/src/providers/encrypt/encrypt_provider.cpp @@ -405,7 +405,9 @@ auto encrypt_provider::get_file(const std::string &api_path, return api_error::error; } -auto encrypt_provider::get_file_list(api_file_list &list) const -> api_error { +auto encrypt_provider::get_file_list(api_file_list &list, + std::string & /* marker */) const + -> api_error { REPERTORY_USES_FUNCTION_NAME(); const auto cfg = config_.get_encrypt_config(); diff --git a/repertory/librepertory/src/providers/meta_db.cpp b/repertory/librepertory/src/providers/meta_db.cpp index 7e4bd4e6..b2aa1489 100644 --- a/repertory/librepertory/src/providers/meta_db.cpp +++ b/repertory/librepertory/src/providers/meta_db.cpp @@ -90,8 +90,8 @@ auto meta_db::get_api_path_list() -> std::vector { return ret; } -auto meta_db::get_item_meta(const std::string &api_path, - api_meta_map &meta) -> api_error { +auto meta_db::get_item_meta(const std::string &api_path, api_meta_map &meta) + -> api_error { REPERTORY_USES_FUNCTION_NAME(); auto result = utils::db::sqlite::db_select{*db_, table_name} @@ -218,6 +218,43 @@ auto meta_db::get_total_item_count() const -> std::uint64_t { return ret; } +auto meta_db::get_total_size() const -> std::uint64_t { + REPERTORY_USES_FUNCTION_NAME(); + + std::uint64_t ret{}; + + try { + auto result = utils::db::sqlite::db_select{*db_, table_name} + .column("api_path") + .where("directory") + .equals(0) + .go(); + + while (result.has_row()) { + std::optional row; + if (not result.get_row(row)) { + continue; + } + + if (not row.has_value()) { + continue; + } + + auto api_path = row->get_column("api_path").get_value(); + std::string size_str; + if (get_item_meta(api_path, META_SIZE, size_str) != api_error::success) { + continue; + } + + ret += utils::string::to_uint64(size_str); + } + } catch (const std::exception &e) { + utils::error::raise_error(function_name, e, "failed to get total size"); + } + + return ret; +} + void meta_db::remove_api_path(const std::string &api_path) { REPERTORY_USES_FUNCTION_NAME(); @@ -274,8 +311,8 @@ auto meta_db::set_item_meta(const std::string &api_path, return update_item_meta(api_path, existing_meta); } -auto meta_db::update_item_meta(const std::string &api_path, - api_meta_map meta) -> api_error { +auto meta_db::update_item_meta(const std::string &api_path, api_meta_map meta) + -> api_error { REPERTORY_USES_FUNCTION_NAME(); auto directory = utils::string::to_bool(meta[META_DIRECTORY]); diff --git a/repertory/librepertory/src/providers/s3/s3_provider.cpp b/repertory/librepertory/src/providers/s3/s3_provider.cpp index 7cab515b..3eedbc07 100644 --- a/repertory/librepertory/src/providers/s3/s3_provider.cpp +++ b/repertory/librepertory/src/providers/s3/s3_provider.cpp @@ -482,85 +482,81 @@ auto s3_provider::get_file(const std::string &api_path, api_file &file) const return api_error::error; } -auto s3_provider::get_file_list(api_file_list &list) const -> api_error { +auto s3_provider::get_file_list(api_file_list &list, std::string &marker) const + -> api_error { REPERTORY_USES_FUNCTION_NAME(); - auto grab_more{true}; - std::string token{}; - while (grab_more) { - std::string response_data; - long response_code{}; - if (not get_object_list(response_data, response_code, std::nullopt, - std::nullopt, token)) { - return api_error::comm_error; - } + std::string response_data; + long response_code{}; + if (not get_object_list(response_data, response_code, std::nullopt, + std::nullopt, token)) { + return api_error::comm_error; + } - if (response_code != http_error_codes::ok) { - utils::error::raise_error(function_name, response_code, - "failed to get file list"); - return api_error::comm_error; - } + if (response_code != http_error_codes::ok) { + utils::error::raise_error(function_name, response_code, + "failed to get file list"); + return api_error::comm_error; + } - pugi::xml_document doc; - auto res = doc.load_string(response_data.c_str()); - if (res.status != pugi::xml_parse_status::status_ok) { - utils::error::raise_error(function_name, res.status, - "failed to parse xml document"); - return api_error::comm_error; - } + pugi::xml_document doc; + auto res = doc.load_string(response_data.c_str()); + if (res.status != pugi::xml_parse_status::status_ok) { + utils::error::raise_error(function_name, res.status, + "failed to parse xml document"); + return api_error::comm_error; + } - grab_more = doc.select_node("/ListBucketResult/IsTruncated") - .node() - .text() - .as_bool(); - if (grab_more) { - token = doc.select_node("/ListBucketResult/NextContinuationToken") - .node() - .text() - .as_string(); - } + auto grab_more = + doc.select_node("/ListBucketResult/IsTruncated").node().text().as_bool(); + if (grab_more) { + marker = doc.select_node("/ListBucketResult/NextContinuationToken") + .node() + .text() + .as_string(); + } - auto node_list = doc.select_nodes("/ListBucketResult/Contents"); - for (auto &&node : node_list) { - auto api_path = - std::string{node.node().select_node("Key").node().text().as_string()}; - if (not utils::string::ends_with(api_path, "/")) { - auto is_encrypted = - not get_config().get_s3_config().encryption_token.empty(); - if (is_encrypted) { - auto err = decrypt_object_name(api_path); - if (err != api_error::success) { - return err; - } - } - - auto size = node.node().select_node("Size").node().text().as_ullong(); - - api_file file{}; - file.api_path = utils::path::create_api_path(api_path); - file.api_parent = utils::path::get_parent_api_path(file.api_path); - file.accessed_date = file.changed_date = file.creation_date = - file.modified_date = - convert_api_date(node.node() - .select_node("LastModified") - .node() - .text() - .as_string()); - file.file_size = is_encrypted ? utils::encryption::encrypting_reader:: - calculate_decrypted_size(size) - : size; - file.key = is_encrypted ? utils::path::create_api_path(api_path) : ""; - auto err = add_if_not_found(file, api_path); + auto node_list = doc.select_nodes("/ListBucketResult/Contents"); + for (auto &&node : node_list) { + auto api_path = + std::string{node.node().select_node("Key").node().text().as_string()}; + if (not utils::string::ends_with(api_path, "/")) { + auto is_encrypted = + not get_config().get_s3_config().encryption_token.empty(); + if (is_encrypted) { + auto err = decrypt_object_name(api_path); if (err != api_error::success) { return err; } - - list.push_back(std::move(file)); } + + auto size = node.node().select_node("Size").node().text().as_ullong(); + + api_file file{}; + file.api_path = utils::path::create_api_path(api_path); + file.api_parent = utils::path::get_parent_api_path(file.api_path); + file.accessed_date = file.changed_date = file.creation_date = + file.modified_date = convert_api_date(node.node() + .select_node("LastModified") + .node() + .text() + .as_string()); + file.file_size = + is_encrypted + ? utils::encryption::encrypting_reader::calculate_decrypted_size( + size) + : size; + file.key = is_encrypted ? utils::path::create_api_path(api_path) : ""; + auto err = add_if_not_found(file, api_path); + if (err != api_error::success) { + return err; + } + + list.push_back(std::move(file)); } } - return api_error::success; + return grab_more ? api_error::more_data : api_error::success; } auto s3_provider::get_last_modified(bool directory, @@ -662,53 +658,7 @@ auto s3_provider::get_total_drive_space() const -> std::uint64_t { } auto s3_provider::get_used_drive_space_impl() const -> std::uint64_t { - auto grab_more{true}; - std::string token{}; - std::uint64_t total_size{}; - while (grab_more) { - std::string response_data; - long response_code{}; - if (not get_object_list(response_data, response_code, std::nullopt, - std::nullopt, token)) { - return total_size; - } - - if (response_code != http_error_codes::ok) { - return total_size; - } - - pugi::xml_document doc; - auto res = doc.load_string(response_data.c_str()); - if (res.status != pugi::xml_parse_status::status_ok) { - return total_size; - } - - grab_more = doc.select_node("/ListBucketResult/IsTruncated") - .node() - .text() - .as_bool(); - if (grab_more) { - token = doc.select_node("/ListBucketResult/NextContinuationToken") - .node() - .text() - .as_string(); - } - - const auto cfg = get_config().get_s3_config(); - const auto is_encrypted = not cfg.encryption_token.empty(); - - auto node_list = doc.select_nodes("/ListBucketResult/Contents"); - total_size = std::accumulate( - node_list.begin(), node_list.end(), total_size, - [&is_encrypted](std::uint64_t total, auto node) -> std::uint64_t { - auto size = node.node().select_node("Size").node().text().as_ullong(); - return total + (is_encrypted ? utils::encryption::encrypting_reader:: - calculate_decrypted_size(size) - : size); - }); - } - - return total_size; + return get_db().get_total_size(); } auto s3_provider::is_directory(const std::string &api_path, bool &exists) const diff --git a/repertory/librepertory/src/providers/sia/sia_provider.cpp b/repertory/librepertory/src/providers/sia/sia_provider.cpp index bbf9977a..ee2c44ac 100644 --- a/repertory/librepertory/src/providers/sia/sia_provider.cpp +++ b/repertory/librepertory/src/providers/sia/sia_provider.cpp @@ -214,7 +214,9 @@ auto sia_provider::get_file(const std::string &api_path, api_file &file) const return api_error::success; } -auto sia_provider::get_file_list(api_file_list &list) const -> api_error { +auto sia_provider::get_file_list(api_file_list &list, + std::string & /* marker */) const + -> api_error { REPERTORY_USES_FUNCTION_NAME(); using dir_func = std::function; @@ -394,34 +396,7 @@ auto sia_provider::get_total_drive_space() const -> std::uint64_t { } auto sia_provider::get_used_drive_space_impl() const -> std::uint64_t { - REPERTORY_USES_FUNCTION_NAME(); - - curl::requests::http_get get{}; - get.allow_timeout = true; - get.path = "/api/bus/stats/objects"; - get.query["bucket"] = get_bucket(get_config().get_sia_config()); - - json object_data{}; - get.response_handler = [&object_data](const data_buffer &data, - long response_code) { - if (response_code == http_error_codes::ok) { - object_data = nlohmann::json::parse(data.begin(), data.end()); - } - }; - - long response_code{}; - stop_type stop_requested{}; - if (not get_comm().make_request(get, response_code, stop_requested)) { - return 0U; - } - - if (response_code != http_error_codes::ok) { - utils::error::raise_error(function_name, response_code, - "failed to get used drive space"); - return 0U; - } - - return object_data["totalObjectsSize"].get(); + return get_db().get_total_size(); } auto sia_provider::is_directory(const std::string &api_path, bool &exists) const diff --git a/repertory/librepertory/src/types/repertory.cpp b/repertory/librepertory/src/types/repertory.cpp index df5332f6..5ff96f35 100644 --- a/repertory/librepertory/src/types/repertory.cpp +++ b/repertory/librepertory/src/types/repertory.cpp @@ -25,8 +25,9 @@ #include "utils/string.hpp" namespace repertory { -auto download_type_from_string( - std::string type, const download_type &default_type) -> download_type { +auto download_type_from_string(std::string type, + const download_type &default_type) + -> download_type { type = utils::string::to_lower(utils::string::trim(type)); if (type == "direct") { return download_type::direct; @@ -84,6 +85,7 @@ static const std::unordered_map LOOKUP = { {api_error::invalid_version, "invalid_version"}, {api_error::item_exists, "item_exists"}, {api_error::item_not_found, "item_not_found"}, + {api_error::more_data, "more_data"}, {api_error::no_disk_space, "no_disk_space"}, {api_error::not_implemented, "not_implemented"}, {api_error::not_supported, "not_supported"}, diff --git a/repertory/librepertory/src/utils/polling.cpp b/repertory/librepertory/src/utils/polling.cpp index 8b42c84d..08afa131 100644 --- a/repertory/librepertory/src/utils/polling.cpp +++ b/repertory/librepertory/src/utils/polling.cpp @@ -31,33 +31,35 @@ void polling::frequency_thread( while (not stop_requested_) { std::deque> futures; unique_mutex_lock lock(mutex_); - if (not stop_requested_ && - notify_.wait_for(lock, std::chrono::seconds(get_frequency_seconds())) == - std::cv_status::timeout) { - for (const auto &item : items_) { - if (item.second.freq == freq) { - futures.emplace_back( - std::async(std::launch::async, [this, &freq, item]() -> void { - if (config_->get_event_level() == event_level::trace || - freq != frequency::second) { - event_system::instance().raise( - item.first); - } - item.second.action(); - if (config_->get_event_level() == event_level::trace || - freq != frequency::second) { - event_system::instance().raise(item.first); - } - })); - } - } - lock.unlock(); - - while (not futures.empty()) { - futures.front().wait(); - futures.pop_front(); + for (const auto &item : items_) { + if (item.second.freq == freq) { + futures.emplace_back( + std::async(std::launch::async, [this, &freq, item]() -> void { + if (config_->get_event_level() == event_level::trace || + freq != frequency::second) { + event_system::instance().raise(item.first); + } + item.second.action(); + if (config_->get_event_level() == event_level::trace || + freq != frequency::second) { + event_system::instance().raise(item.first); + } + })); } } + lock.unlock(); + + while (not futures.empty()) { + futures.front().wait(); + futures.pop_front(); + } + + lock.lock(); + if (stop_requested_) { + continue; + } + + notify_.wait_for(lock, std::chrono::seconds(get_frequency_seconds())); } } @@ -73,50 +75,53 @@ void polling::set_callback(const polling_item &item) { void polling::start(app_config *config) { mutex_lock lock(start_stop_mutex_); - if (not high_frequency_thread_) { - event_system::instance().raise("polling"); - config_ = config; - stop_requested_ = false; - high_frequency_thread_ = std::make_unique([this]() -> void { - this->frequency_thread( - [this]() -> std::uint32_t { - return config_->get_high_frequency_interval_secs(); - }, - frequency::high); - }); - low_frequency_thread_ = std::make_unique([this]() -> void { - this->frequency_thread( - [this]() -> std::uint32_t { - return config_->get_low_frequency_interval_secs(); - }, - frequency::low); - }); - second_frequency_thread_ = std::make_unique([this]() -> void { - this->frequency_thread([]() -> std::uint32_t { return 1U; }, - frequency::second); - }); + if (high_frequency_thread_) { + return; } + + event_system::instance().raise("polling"); + config_ = config; + stop_requested_ = false; + high_frequency_thread_ = std::make_unique([this]() -> void { + this->frequency_thread( + [this]() -> std::uint32_t { + return config_->get_high_frequency_interval_secs(); + }, + frequency::high); + }); + low_frequency_thread_ = std::make_unique([this]() -> void { + this->frequency_thread( + [this]() -> std::uint32_t { + return config_->get_low_frequency_interval_secs(); + }, + frequency::low); + }); + second_frequency_thread_ = std::make_unique([this]() -> void { + this->frequency_thread([]() -> std::uint32_t { return 1U; }, + frequency::second); + }); } void polling::stop() { - if (high_frequency_thread_) { - event_system::instance().raise("polling"); - mutex_lock lock(start_stop_mutex_); - if (high_frequency_thread_) { - { - stop_requested_ = true; - - mutex_lock lock2(mutex_); - notify_.notify_all(); - } - high_frequency_thread_->join(); - low_frequency_thread_->join(); - second_frequency_thread_->join(); - high_frequency_thread_.reset(); - low_frequency_thread_.reset(); - second_frequency_thread_.reset(); - } - event_system::instance().raise("polling"); + mutex_lock lock(start_stop_mutex_); + if (not high_frequency_thread_) { + return; } + + event_system::instance().raise("polling"); + stop_requested_ = true; + + unique_mutex_lock lock2(mutex_); + notify_.notify_all(); + lock2.unlock(); + + high_frequency_thread_->join(); + low_frequency_thread_->join(); + second_frequency_thread_->join(); + high_frequency_thread_.reset(); + low_frequency_thread_.reset(); + second_frequency_thread_.reset(); + + event_system::instance().raise("polling"); } } // namespace repertory diff --git a/repertory/repertory_test/include/mocks/mock_provider.hpp b/repertory/repertory_test/include/mocks/mock_provider.hpp index beb78a7c..e398afda 100644 --- a/repertory/repertory_test/include/mocks/mock_provider.hpp +++ b/repertory/repertory_test/include/mocks/mock_provider.hpp @@ -60,8 +60,8 @@ public: MOCK_METHOD(api_error, get_file, (const std::string &api_path, api_file &file), (const, override)); - MOCK_METHOD(api_error, get_file_list, (api_file_list & list), - (const, override)); + MOCK_METHOD(api_error, get_file_list, + (api_file_list & list, std::string &marker), (const, override)); MOCK_METHOD(api_error, get_file_size, (const std::string &api_path, std::uint64_t &file_size), diff --git a/repertory/repertory_test/src/providers_test.cpp b/repertory/repertory_test/src/providers_test.cpp index 33fb737b..bfd7b2c5 100644 --- a/repertory/repertory_test/src/providers_test.cpp +++ b/repertory/repertory_test/src/providers_test.cpp @@ -547,7 +547,8 @@ static void get_file_fails_if_item_is_directory(const app_config &cfg, static void get_file_list(const app_config &cfg, i_provider &provider) { api_file_list list{}; - EXPECT_EQ(api_error::success, provider.get_file_list(list)); + std::string marker; + EXPECT_EQ(api_error::success, provider.get_file_list(list, marker)); if (provider.get_provider_type() == provider_type::encrypt) { EXPECT_EQ(std::size_t(2U), list.size());