refactor

repertory/librepertory/include/file_manager/file_manager.hpp

@@ -161,8 +161,6 @@ public:
   void start();
 
   void stop();
-
-  void update_used_space(std::uint64_t &used_space) const override;
 };
 } // namespace repertory
 

repertory/librepertory/include/file_manager/i_file_manager.hpp

@@ -31,21 +31,20 @@ class i_file_manager {
   INTERFACE_SETUP(i_file_manager);
 
 public:
-  [[nodiscard]] virtual auto
-  evict_file(const std::string &api_path) -> bool = 0;
-
-  [[nodiscard]] virtual auto get_directory_items(
-      const std::string &api_path) const -> directory_item_list = 0;
+  [[nodiscard]] virtual auto evict_file(const std::string &api_path)
+      -> bool = 0;
 
   [[nodiscard]] virtual auto
-  get_open_files() const -> std::unordered_map<std::string, std::size_t> = 0;
+  get_directory_items(const std::string &api_path) const
+      -> directory_item_list = 0;
+
+  [[nodiscard]] virtual auto get_open_files() const
+      -> std::unordered_map<std::string, std::size_t> = 0;
 
   [[nodiscard]] virtual auto has_no_open_file_handles() const -> bool = 0;
 
-  [[nodiscard]] virtual auto
-  is_processing(const std::string &api_path) const -> bool = 0;
-
-  virtual void update_used_space(std::uint64_t &used_space) const = 0;
+  [[nodiscard]] virtual auto is_processing(const std::string &api_path) const
+      -> bool = 0;
 };
 } // namespace repertory
 

repertory/librepertory/include/providers/base_provider.hpp

@@ -44,25 +44,24 @@ private:
   api_item_added_callback api_item_added_;
   std::unique_ptr<meta_db> db3_;
   i_file_manager *fm_{};
-  mutable std::atomic<std::uint64_t> used_space;
+  std::atomic<bool> stop_requested_{false};
 
 private:
   void remove_deleted_files(bool source_only);
 
-  void update_used_drive_space() const;
-
 protected:
-  [[nodiscard]] static auto
-  create_api_file(std::string path, std::string key, std::uint64_t size,
-                  std::uint64_t file_time) -> api_file;
+  [[nodiscard]] static auto create_api_file(std::string path, std::string key,
+                                            std::uint64_t size,
+                                            std::uint64_t file_time)
+      -> api_file;
 
   [[nodiscard]] static auto create_api_file(std::string path,
                                             std::uint64_t size,
                                             api_meta_map &meta) -> api_file;
 
-  [[nodiscard]] virtual auto
-  create_directory_impl(const std::string &api_path,
-                        api_meta_map &meta) -> api_error = 0;
+  [[nodiscard]] virtual auto create_directory_impl(const std::string &api_path,
+                                                   api_meta_map &meta)
+      -> api_error = 0;
 
   [[nodiscard]] virtual auto
   create_file_extra(const std::string & /* api_path */,
@@ -74,8 +73,8 @@ protected:
     return api_item_added_;
   }
 
-  [[nodiscard]] auto
-  get_api_item_added() const -> const api_item_added_callback & {
+  [[nodiscard]] auto get_api_item_added() const
+      -> const api_item_added_callback & {
     return api_item_added_;
   }
 
@@ -89,6 +88,8 @@ protected:
 
   [[nodiscard]] auto get_db() -> meta_db & { return *db3_; }
 
+  [[nodiscard]] auto get_db() const -> const meta_db & { return *db3_; }
+
   [[nodiscard]] virtual auto
   get_directory_items_impl(const std::string &api_path,
                            directory_item_list &list) const -> api_error = 0;
@@ -99,23 +100,22 @@ protected:
     return fm_;
   }
 
-  [[nodiscard]] virtual auto
-  get_used_drive_space_impl() const -> std::uint64_t = 0;
+  [[nodiscard]] virtual auto remove_directory_impl(const std::string &api_path)
+      -> api_error = 0;
 
-  [[nodiscard]] virtual auto
-  remove_directory_impl(const std::string &api_path) -> api_error = 0;
+  [[nodiscard]] virtual auto remove_file_impl(const std::string &api_path)
+      -> api_error = 0;
 
-  [[nodiscard]] virtual auto
-  remove_file_impl(const std::string &api_path) -> api_error = 0;
-
-  [[nodiscard]] virtual auto
-  upload_file_impl(const std::string &api_path, const std::string &source_path,
-                   stop_type &stop_requested) -> api_error = 0;
+  [[nodiscard]] virtual auto upload_file_impl(const std::string &api_path,
+                                              const std::string &source_path,
+                                              stop_type &stop_requested)
+      -> api_error = 0;
 
 public:
-  [[nodiscard]] auto create_directory_clone_source_meta(
-      const std::string &source_api_path,
-      const std::string &api_path) -> api_error override;
+  [[nodiscard]] auto
+  create_directory_clone_source_meta(const std::string &source_api_path,
+                                     const std::string &api_path)
+      -> api_error override;
 
   [[nodiscard]] auto create_directory(const std::string &api_path,
                                       api_meta_map &meta) -> api_error override;
@@ -123,76 +123,82 @@ public:
   [[nodiscard]] auto create_file(const std::string &api_path,
                                  api_meta_map &meta) -> api_error override;
 
-  [[nodiscard]] auto
-  get_api_path_from_source(const std::string &source_path,
-                           std::string &api_path) const -> api_error override;
+  [[nodiscard]] auto get_api_path_from_source(const std::string &source_path,
+                                              std::string &api_path) const
+      -> api_error override;
+
+  [[nodiscard]] auto get_directory_items(const std::string &api_path,
+                                         directory_item_list &list) const
+      -> api_error override;
+
+  [[nodiscard]] auto get_file_size(const std::string &api_path,
+                                   std::uint64_t &file_size) const
+      -> api_error override;
+
+  [[nodiscard]] auto get_filesystem_item(const std::string &api_path,
+                                         bool directory,
+                                         filesystem_item &fsi) const
+      -> api_error override;
+
+  [[nodiscard]] auto get_filesystem_item_and_file(const std::string &api_path,
+                                                  api_file &f,
+                                                  filesystem_item &fsi) const
+      -> api_error override;
 
   [[nodiscard]] auto
-  get_directory_items(const std::string &api_path,
-                      directory_item_list &list) const -> api_error override;
+  get_filesystem_item_from_source_path(const std::string &source_path,
+                                       filesystem_item &fsi) const
+      -> api_error override;
 
-  [[nodiscard]] auto
-  get_file_size(const std::string &api_path,
-                std::uint64_t &file_size) const -> api_error override;
+  [[nodiscard]] auto get_item_meta(const std::string &api_path,
+                                   api_meta_map &meta) const
+      -> api_error override;
 
-  [[nodiscard]] auto
-  get_filesystem_item(const std::string &api_path, bool directory,
-                      filesystem_item &fsi) const -> api_error override;
+  [[nodiscard]] auto get_item_meta(const std::string &api_path,
+                                   const std::string &key,
+                                   std::string &value) const
+      -> api_error override;
 
-  [[nodiscard]] auto get_filesystem_item_and_file(
-      const std::string &api_path, api_file &f,
-      filesystem_item &fsi) const -> api_error override;
-
-  [[nodiscard]] auto get_filesystem_item_from_source_path(
-      const std::string &source_path,
-      filesystem_item &fsi) const -> api_error override;
-
-  [[nodiscard]] auto
-  get_item_meta(const std::string &api_path,
-                api_meta_map &meta) const -> api_error override;
-
-  [[nodiscard]] auto
-  get_item_meta(const std::string &api_path, const std::string &key,
-                std::string &value) const -> api_error override;
-
-  [[nodiscard]] auto
-  get_pinned_files() const -> std::vector<std::string> override;
+  [[nodiscard]] auto get_pinned_files() const
+      -> std::vector<std::string> override;
 
   [[nodiscard]] auto get_total_item_count() const -> std::uint64_t override;
 
   [[nodiscard]] auto get_used_drive_space() const -> std::uint64_t override;
 
-  [[nodiscard]] auto
-  is_file_writeable(const std::string &api_path) const -> bool override;
+  [[nodiscard]] auto is_file_writeable(const std::string &api_path) const
+      -> bool override;
 
   [[nodiscard]] auto is_read_only() const -> bool override { return false; }
 
-  [[nodiscard]] auto
-  remove_directory(const std::string &api_path) -> api_error override;
+  [[nodiscard]] auto remove_directory(const std::string &api_path)
+      -> api_error override;
 
-  [[nodiscard]] auto
-  remove_file(const std::string &api_path) -> api_error override;
+  [[nodiscard]] auto remove_file(const std::string &api_path)
+      -> api_error override;
 
-  [[nodiscard]] auto
-  remove_item_meta(const std::string &api_path,
-                   const std::string &key) -> api_error override;
+  [[nodiscard]] auto remove_item_meta(const std::string &api_path,
+                                      const std::string &key)
+      -> api_error override;
 
-  [[nodiscard]] auto
-  set_item_meta(const std::string &api_path, const std::string &key,
-                const std::string &value) -> api_error override;
+  [[nodiscard]] auto set_item_meta(const std::string &api_path,
+                                   const std::string &key,
+                                   const std::string &value)
+      -> api_error override;
 
-  [[nodiscard]] auto
-  set_item_meta(const std::string &api_path,
-                const api_meta_map &meta) -> api_error override;
+  [[nodiscard]] auto set_item_meta(const std::string &api_path,
+                                   const api_meta_map &meta)
+      -> api_error override;
 
   [[nodiscard]] auto start(api_item_added_callback api_item_added,
                            i_file_manager *mgr) -> bool override;
 
   void stop() override;
 
-  [[nodiscard]] auto
-  upload_file(const std::string &api_path, const std::string &source_path,
-              stop_type &stop_requested) -> api_error override;
+  [[nodiscard]] auto upload_file(const std::string &api_path,
+                                 const std::string &source_path,
+                                 stop_type &stop_requested)
+      -> api_error override;
 };
 } // namespace repertory
 

repertory/librepertory/include/providers/encrypt/encrypt_provider.hpp

@@ -111,7 +111,8 @@ public:
   [[nodiscard]] auto get_file(const std::string &api_path, api_file &file) const
       -> api_error override;
 
-  [[nodiscard]] auto get_file_list(api_file_list &list) const
+  [[nodiscard]] auto get_file_list(api_file_list &list,
+                                   std::string &marker) const
       -> api_error override;
 
   [[nodiscard]] auto get_file_size(const std::string &api_path,

repertory/librepertory/include/providers/i_provider.hpp

@@ -58,7 +58,8 @@ public:
   [[nodiscard]] virtual auto get_file(const std::string &api_path,
                                       api_file &file) const -> api_error = 0;
 
-  [[nodiscard]] virtual auto get_file_list(api_file_list &list) const
+  [[nodiscard]] virtual auto get_file_list(api_file_list &list,
+                                           std::string &marker) const
       -> api_error = 0;
 
   [[nodiscard]] virtual auto get_file_size(const std::string &api_path,

repertory/librepertory/include/providers/meta_db.hpp

@@ -63,14 +63,16 @@ public:
 
   [[nodiscard]] auto get_total_item_count() const -> std::uint64_t;
 
+  [[nodiscard]] auto get_total_size() const -> std::uint64_t;
+
   void remove_api_path(const std::string &api_path);
 
   [[nodiscard]] auto remove_item_meta(const std::string &api_path,
                                       const std::string &key) -> api_error;
 
-  [[nodiscard]] auto
-  rename_item_meta(const std::string &from_api_path,
-                   const std::string &to_api_path) -> api_error;
+  [[nodiscard]] auto rename_item_meta(const std::string &from_api_path,
+                                      const std::string &to_api_path)
+      -> api_error;
 
   [[nodiscard]] auto set_item_meta(const std::string &api_path,
                                    const std::string &key,

repertory/librepertory/include/providers/s3/s3_provider.hpp

@@ -47,70 +47,70 @@ public:
   auto operator=(s3_provider &&) -> s3_provider & = delete;
 
 private:
-  [[nodiscard]] auto
-  add_if_not_found(api_file &file,
-                   const std::string &object_name) const -> api_error;
+  [[nodiscard]] auto add_if_not_found(api_file &file,
+                                      const std::string &object_name) const
+      -> api_error;
 
-  [[nodiscard]] auto
-  create_file_extra(const std::string &api_path,
-                    api_meta_map &meta) -> api_error override;
+  [[nodiscard]] auto create_file_extra(const std::string &api_path,
+                                       api_meta_map &meta)
+      -> api_error override;
 
-  [[nodiscard]] auto
-  create_path_directories(const std::string &api_path,
-                          const std::string &key) const -> api_error;
+  [[nodiscard]] auto create_path_directories(const std::string &api_path,
+                                             const std::string &key) const
+      -> api_error;
 
-  [[nodiscard]] auto
-  decrypt_object_name(std::string &object_name) const -> api_error;
+  [[nodiscard]] auto decrypt_object_name(std::string &object_name) const
+      -> api_error;
 
-  [[nodiscard]] auto
-  get_last_modified(bool directory,
-                    const std::string &api_path) const -> std::uint64_t;
+  [[nodiscard]] auto get_last_modified(bool directory,
+                                       const std::string &api_path) const
+      -> std::uint64_t;
 
   [[nodiscard]] auto
   get_object_info(bool directory, const std::string &api_path,
                   bool &is_encrypted, std::string &object_name,
                   head_object_result &result) const -> api_error;
 
-  [[nodiscard]] auto get_object_list(
-      std::string &response_data, long &response_code,
-      std::optional<std::string> delimiter = std::nullopt,
-      std::optional<std::string> prefix = std::nullopt,
-      std::optional<std::string> token = std::nullopt) const -> bool;
+  [[nodiscard]] auto
+  get_object_list(std::string &response_data, long &response_code,
+                  std::optional<std::string> delimiter = std::nullopt,
+                  std::optional<std::string> prefix = std::nullopt,
+                  std::optional<std::string> token = std::nullopt) const
+      -> bool;
 
 protected:
-  [[nodiscard]] auto
-  create_directory_impl(const std::string &api_path,
-                        api_meta_map &meta) -> api_error override;
+  [[nodiscard]] auto create_directory_impl(const std::string &api_path,
+                                           api_meta_map &meta)
+      -> api_error override;
 
   [[nodiscard]] auto get_directory_items_impl(const std::string &api_path,
                                               directory_item_list &list) const
       -> api_error override;
 
-  [[nodiscard]] auto
-  get_used_drive_space_impl() const -> std::uint64_t override;
+  [[nodiscard]] auto remove_directory_impl(const std::string &api_path)
+      -> api_error override;
 
-  [[nodiscard]] auto
-  remove_directory_impl(const std::string &api_path) -> api_error override;
+  [[nodiscard]] auto remove_file_impl(const std::string &api_path)
+      -> api_error override;
 
-  [[nodiscard]] auto
-  remove_file_impl(const std::string &api_path) -> api_error override;
-
-  [[nodiscard]] auto
-  upload_file_impl(const std::string &api_path, const std::string &source_path,
-                   stop_type &stop_requested) -> api_error override;
+  [[nodiscard]] auto upload_file_impl(const std::string &api_path,
+                                      const std::string &source_path,
+                                      stop_type &stop_requested)
+      -> api_error override;
 
 public:
-  [[nodiscard]] static auto
-  convert_api_date(std::string_view date) -> std::uint64_t;
+  [[nodiscard]] static auto convert_api_date(std::string_view date)
+      -> std::uint64_t;
 
   [[nodiscard]] auto get_directory_item_count(const std::string &api_path) const
       -> std::uint64_t override;
 
-  [[nodiscard]] auto get_file(const std::string &api_path,
-                              api_file &file) const -> api_error override;
+  [[nodiscard]] auto get_file(const std::string &api_path, api_file &file) const
+      -> api_error override;
 
-  [[nodiscard]] auto
-  get_file_list(api_file_list &list) const -> api_error override;
+  [[nodiscard]] auto get_file_list(api_file_list &list,
+                                   std::string &marker) const
+      -> api_error override;
 
   [[nodiscard]] auto get_total_drive_space() const -> std::uint64_t override;
 
@@ -121,8 +121,8 @@ public:
   [[nodiscard]] auto is_directory(const std::string &api_path,
                                   bool &exists) const -> api_error override;
 
-  [[nodiscard]] auto is_file(const std::string &api_path,
-                             bool &exists) const -> api_error override;
+  [[nodiscard]] auto is_file(const std::string &api_path, bool &exists) const
+      -> api_error override;
 
   [[nodiscard]] auto is_online() const -> bool override;
 
@@ -130,14 +130,15 @@ public:
     return false;
   };
 
-  [[nodiscard]] auto
-  read_file_bytes(const std::string &api_path, std::size_t size,
-                  std::uint64_t offset, data_buffer &data,
-                  stop_type &stop_requested) -> api_error override;
+  [[nodiscard]] auto read_file_bytes(const std::string &api_path,
+                                     std::size_t size, std::uint64_t offset,
+                                     data_buffer &data,
+                                     stop_type &stop_requested)
+      -> api_error override;
 
-  [[nodiscard]] auto
-  rename_file(const std::string &from_api_path,
-              const std::string &to_api_path) -> api_error override;
+  [[nodiscard]] auto rename_file(const std::string &from_api_path,
+                                 const std::string &to_api_path)
+      -> api_error override;
 
   [[nodiscard]] auto start(api_item_added_callback api_item_added,
                            i_file_manager *mgr) -> bool override;

repertory/librepertory/include/providers/sia/sia_provider.hpp

@@ -61,9 +61,6 @@ protected:
                                               directory_item_list &list) const
       -> api_error override;
 
-  [[nodiscard]] auto get_used_drive_space_impl() const
-      -> std::uint64_t override;
-
   [[nodiscard]] auto remove_directory_impl(const std::string &api_path)
       -> api_error override;
 
@@ -82,7 +79,8 @@ public:
   [[nodiscard]] auto get_file(const std::string &api_path, api_file &file) const
       -> api_error override;
 
-  [[nodiscard]] auto get_file_list(api_file_list &list) const
+  [[nodiscard]] auto get_file_list(api_file_list &list,
+                                   std::string &marker) const
       -> api_error override;
 
   [[nodiscard]] auto get_provider_type() const -> provider_type override {

repertory/librepertory/include/types/repertory.hpp

@@ -79,6 +79,7 @@ enum class api_error {
   invalid_version,
   item_exists,
   item_not_found,
+  more_data,
   no_disk_space,
   not_implemented,
   not_supported,
@@ -95,16 +96,16 @@ enum class api_error {
 
 [[nodiscard]] auto api_error_from_string(std::string_view str) -> api_error;
 
-[[nodiscard]] auto
-api_error_to_string(const api_error &error) -> const std::string &;
+[[nodiscard]] auto api_error_to_string(const api_error &error)
+    -> const std::string &;
 
 enum class download_type { direct, fallback, ring_buffer };
-[[nodiscard]] auto
-download_type_from_string(std::string type,
-                          const download_type &default_type) -> download_type;
+[[nodiscard]] auto download_type_from_string(std::string type,
+                                             const download_type &default_type)
+    -> download_type;
 
-[[nodiscard]] auto
-download_type_to_string(const download_type &type) -> std::string;
+[[nodiscard]] auto download_type_to_string(const download_type &type)
+    -> std::string;
 
 enum class exit_code : std::int32_t {
   success,

repertory/librepertory/src/drives/eviction.cpp

@@ -38,18 +38,6 @@ auto eviction::check_minimum_requirements(const std::string &file_path)
 
   auto check_file = utils::file::file{file_path};
 
-  auto opt_size = check_file.size();
-  if (not opt_size.has_value()) {
-    utils::error::raise_error(function_name, utils::get_last_error_code(),
-                              file_path, "failed to get file size");
-    return false;
-  }
-
-  auto file_size{opt_size.value()};
-  if (file_size == 0U) {
-    return false;
-  }
-
   auto reference_time =
       check_file.get_time(config_.get_eviction_uses_accessed_time()
                               ? utils::file::time_type::accessed

repertory/librepertory/src/drives/fuse/fuse_drive.cpp

@@ -81,8 +81,8 @@ auto fuse_drive::chown_impl(std::string api_path, uid_t uid, gid_t gid,
                             struct fuse_file_info * /*file_info*/)
     -> api_error {
 #else
-auto fuse_drive::chown_impl(std::string api_path, uid_t uid,
-                            gid_t gid) -> api_error {
+auto fuse_drive::chown_impl(std::string api_path, uid_t uid, gid_t gid)
+    -> api_error {
 #endif
   return check_and_perform(
       api_path, X_OK, [&](api_meta_map &meta) -> api_error {
@@ -481,8 +481,8 @@ auto fuse_drive::getattr_impl(std::string api_path, struct stat *unix_st,
                               struct fuse_file_info * /*file_info*/)
     -> api_error {
 #else
-auto fuse_drive::getattr_impl(std::string api_path,
-                              struct stat *unix_st) -> api_error {
+auto fuse_drive::getattr_impl(std::string api_path, struct stat *unix_st)
+    -> api_error {
 #endif
   auto parent = utils::path::get_parent_api_path(api_path);
 
@@ -565,8 +565,8 @@ auto fuse_drive::getxtimes_impl(std::string api_path, struct timespec *bkuptime,
 #endif // __APPLE__
 
 #if FUSE_USE_VERSION >= 30
-auto fuse_drive::init_impl(struct fuse_conn_info *conn,
-                           struct fuse_config *cfg) -> void * {
+auto fuse_drive::init_impl(struct fuse_conn_info *conn, struct fuse_config *cfg)
+    -> void * {
 #else
 void *fuse_drive::init_impl(struct fuse_conn_info *conn) {
 #endif
@@ -588,8 +588,6 @@ void *fuse_drive::init_impl(struct fuse_conn_info *conn) {
   event_system::instance().start();
   was_mounted_ = true;
 
-  polling::instance().start(&config_);
-
   fm_ = std::make_unique<file_manager>(config_, provider_);
   server_ = std::make_unique<full_server>(config_, provider_, *fm_);
   if (not provider_.is_read_only()) {
@@ -622,6 +620,9 @@ void *fuse_drive::init_impl(struct fuse_conn_info *conn) {
     if (not lock_data_.set_mount_state(true, get_mount_location(), getpid())) {
       utils::error::raise_error(function_name, "failed to set mount state");
     }
+
+    polling::instance().start(&config_);
+
     event_system::instance().raise<drive_mounted>(get_mount_location());
   } catch (const std::exception &e) {
     utils::error::raise_error(function_name, e, "exception during fuse init");
@@ -799,8 +800,9 @@ auto fuse_drive::release_impl(std::string /*api_path*/,
   return api_error::success;
 }
 
-auto fuse_drive::releasedir_impl(
-    std::string /*api_path*/, struct fuse_file_info *file_info) -> api_error {
+auto fuse_drive::releasedir_impl(std::string /*api_path*/,
+                                 struct fuse_file_info *file_info)
+    -> api_error {
   auto iter = directory_cache_->get_directory(file_info->fh);
   if (iter == nullptr) {
     return api_error::invalid_handle;
@@ -818,8 +820,8 @@ auto fuse_drive::rename_directory(const std::string &from_api_path,
 }
 
 auto fuse_drive::rename_file(const std::string &from_api_path,
-                             const std::string &to_api_path,
-                             bool overwrite) -> int {
+                             const std::string &to_api_path, bool overwrite)
+    -> int {
   auto res = fm_->rename_file(from_api_path, to_api_path, overwrite);
   errno = std::abs(utils::from_api_error(res));
   return (res == api_error::success) ? 0 : -1;
@@ -829,8 +831,8 @@ auto fuse_drive::rename_file(const std::string &from_api_path,
 auto fuse_drive::rename_impl(std::string from_api_path, std::string to_api_path,
                              unsigned int /*flags*/) -> api_error {
 #else
-auto fuse_drive::rename_impl(std::string from_api_path,
-                             std::string to_api_path) -> api_error {
+auto fuse_drive::rename_impl(std::string from_api_path, std::string to_api_path)
+    -> api_error {
 #endif
   auto res = check_parent_access(to_api_path, W_OK | X_OK);
   if (res != api_error::success) {
@@ -944,15 +946,15 @@ auto fuse_drive::getxattr_impl(std::string api_path, const char *name,
 }
 #else  // __APPLE__
 auto fuse_drive::getxattr_impl(std::string api_path, const char *name,
-                               char *value, size_t size,
-                               int &attribute_size) -> api_error {
+                               char *value, size_t size, int &attribute_size)
+    -> api_error {
   return getxattr_common(api_path, name, value, size, attribute_size, nullptr);
 }
 #endif // __APPLE__
 
 auto fuse_drive::listxattr_impl(std::string api_path, char *buffer, size_t size,
-                                int &required_size,
-                                bool &return_size) -> api_error {
+                                int &required_size, bool &return_size)
+    -> api_error {
   auto check_size = (size == 0);
 
   auto res = check_parent_access(api_path, X_OK);
@@ -992,8 +994,8 @@ auto fuse_drive::listxattr_impl(std::string api_path, char *buffer, size_t size,
   return res;
 }
 
-auto fuse_drive::removexattr_impl(std::string api_path,
-                                  const char *name) -> api_error {
+auto fuse_drive::removexattr_impl(std::string api_path, const char *name)
+    -> api_error {
   std::string attribute_name;
 #if defined(__APPLE__)
   auto res = parse_xattr_parameters(name, 0, attribute_name, api_path);
@@ -1021,8 +1023,8 @@ auto fuse_drive::setxattr_impl(std::string api_path, const char *name,
                                uint32_t position) -> api_error {
 #else // __APPLE__
 auto fuse_drive::setxattr_impl(std::string api_path, const char *name,
-                               const char *value, size_t size,
-                               int flags) -> api_error {
+                               const char *value, size_t size, int flags)
+    -> api_error {
 #endif
   std::string attribute_name;
 #if defined(__APPLE__)
@@ -1100,8 +1102,8 @@ void fuse_drive::set_item_meta(const std::string &api_path,
 }
 
 #if defined(__APPLE__)
-auto fuse_drive::setattr_x_impl(std::string api_path,
-                                struct setattr_x *attr) -> api_error {
+auto fuse_drive::setattr_x_impl(std::string api_path, struct setattr_x *attr)
+    -> api_error {
   bool exists{};
   auto res = provider_.is_file(api_path, exists);
   if (res != api_error::success) {
@@ -1155,7 +1157,7 @@ auto fuse_drive::setattr_x_impl(std::string api_path,
       ts[0].tv_sec = attr->acctime.tv_sec;
       ts[0].tv_nsec = attr->acctime.tv_nsec;
     } else {
-      struct timeval tv {};
+      struct timeval tv{};
       gettimeofday(&tv, NULL);
       ts[0].tv_sec = tv.tv_sec;
       ts[0].tv_nsec = tv.tv_usec * 1000;
@@ -1200,8 +1202,9 @@ auto fuse_drive::setattr_x_impl(std::string api_path,
   return api_error::success;
 }
 
-auto fuse_drive::setbkuptime_impl(
-    std::string api_path, const struct timespec *bkuptime) -> api_error {
+auto fuse_drive::setbkuptime_impl(std::string api_path,
+                                  const struct timespec *bkuptime)
+    -> api_error {
   return check_and_perform(
       api_path, X_OK, [&](api_meta_map &meta) -> api_error {
         auto nanos = bkuptime->tv_nsec +
@@ -1237,8 +1240,8 @@ auto fuse_drive::setvolname_impl(const char * /*volname*/) -> api_error {
   return api_error::success;
 }
 
-auto fuse_drive::statfs_x_impl(std::string /*api_path*/,
-                               struct statfs *stbuf) -> api_error {
+auto fuse_drive::statfs_x_impl(std::string /*api_path*/, struct statfs *stbuf)
+    -> api_error {
   if (statfs(&config_.get_cache_directory()[0], stbuf) != 0) {
     return api_error::os_error;
   }
@@ -1263,8 +1266,8 @@ auto fuse_drive::statfs_x_impl(std::string /*api_path*/,
   return api_error::success;
 }
 #else  // __APPLE__
-auto fuse_drive::statfs_impl(std::string /*api_path*/,
-                             struct statvfs *stbuf) -> api_error {
+auto fuse_drive::statfs_impl(std::string /*api_path*/, struct statvfs *stbuf)
+    -> api_error {
   if (statvfs(config_.get_cache_directory().data(), stbuf) != 0) {
     return api_error::os_error;
   }
@@ -1344,8 +1347,8 @@ auto fuse_drive::utimens_impl(std::string api_path, const struct timespec tv[2],
                               struct fuse_file_info * /*file_info*/)
     -> api_error {
 #else
-auto fuse_drive::utimens_impl(std::string api_path,
-                              const struct timespec tv[2]) -> api_error {
+auto fuse_drive::utimens_impl(std::string api_path, const struct timespec tv[2])
+    -> api_error {
 #endif
   api_meta_map meta;
   auto res = provider_.get_item_meta(api_path, meta);

repertory/librepertory/src/drives/winfsp/winfsp_drive.cpp

@@ -619,7 +619,6 @@ auto winfsp_drive::Mounted(PVOID host) -> NTSTATUS {
   auto *file_system_host{
       reinterpret_cast<FileSystemHost *>(host),
   };
-  polling::instance().start(&config_);
   fm_ = std::make_unique<file_manager>(config_, provider_);
   server_ = std::make_unique<full_server>(config_, provider_, *fm_);
   if (not provider_.is_read_only()) {
@@ -652,6 +651,7 @@ auto winfsp_drive::Mounted(PVOID host) -> NTSTATUS {
       utils::error::raise_error(function_name, "failed to set mount state");
     }
 
+    polling::instance().start(&config_);
     event_system::instance().raise<drive_mounted>(mount_location);
   } catch (const std::exception &e) {
     utils::error::raise_error(function_name, e, "exception occurred");

repertory/librepertory/src/file_manager/file_manager.cpp

@@ -1096,18 +1096,4 @@ void file_manager::upload_handler() {
     upload_notify_.notify_all();
   }
 }
-
-void file_manager::update_used_space(std::uint64_t &used_space) const {
-  recur_mutex_lock open_lock(open_file_mtx_);
-  for (auto &&item : open_file_lookup_) {
-    std::uint64_t file_size{};
-    auto res = provider_.get_file_size(item.second->get_api_path(), file_size);
-    if ((res == api_error::success) &&
-        (file_size != item.second->get_file_size()) &&
-        (used_space >= file_size)) {
-      used_space -= file_size;
-      used_space += item.second->get_file_size();
-    }
-  }
-}
 } // namespace repertory

repertory/librepertory/src/providers/base_provider.cpp

@@ -33,8 +33,8 @@
 
 namespace repertory {
 auto base_provider::create_api_file(std::string path, std::string key,
-                                    std::uint64_t size,
-                                    std::uint64_t file_time) -> api_file {
+                                    std::uint64_t size, std::uint64_t file_time)
+    -> api_file {
   api_file file{};
   file.api_path = utils::path::create_api_path(path);
   file.api_parent = utils::path::get_parent_api_path(file.api_path);
@@ -66,8 +66,8 @@ auto base_provider::create_api_file(std::string path, std::uint64_t size,
 }
 
 auto base_provider::create_directory_clone_source_meta(
-    const std::string &source_api_path,
-    const std::string &api_path) -> api_error {
+    const std::string &source_api_path, const std::string &api_path)
+    -> api_error {
   REPERTORY_USES_FUNCTION_NAME();
 
   bool exists{};
@@ -164,8 +164,8 @@ auto base_provider::create_directory(const std::string &api_path,
   return set_item_meta(api_path, meta);
 }
 
-auto base_provider::create_file(const std::string &api_path,
-                                api_meta_map &meta) -> api_error {
+auto base_provider::create_file(const std::string &api_path, api_meta_map &meta)
+    -> api_error {
   REPERTORY_USES_FUNCTION_NAME();
 
   bool exists{};
@@ -222,8 +222,9 @@ auto base_provider::create_file(const std::string &api_path,
   return api_error::error;
 }
 
-auto base_provider::get_api_path_from_source(
-    const std::string &source_path, std::string &api_path) const -> api_error {
+auto base_provider::get_api_path_from_source(const std::string &source_path,
+                                             std::string &api_path) const
+    -> api_error {
   REPERTORY_USES_FUNCTION_NAME();
 
   if (source_path.empty()) {
@@ -236,8 +237,9 @@ auto base_provider::get_api_path_from_source(
   return db3_->get_api_path(source_path, api_path);
 }
 
-auto base_provider::get_directory_items(
-    const std::string &api_path, directory_item_list &list) const -> api_error {
+auto base_provider::get_directory_items(const std::string &api_path,
+                                        directory_item_list &list) const
+    -> api_error {
   REPERTORY_USES_FUNCTION_NAME();
 
   bool exists{};
@@ -301,9 +303,10 @@ auto base_provider::get_file_size(const std::string &api_path,
   return api_error::success;
 }
 
-auto base_provider::get_filesystem_item(
-    const std::string &api_path, bool directory,
-    filesystem_item &fsi) const -> api_error {
+auto base_provider::get_filesystem_item(const std::string &api_path,
+                                        bool directory,
+                                        filesystem_item &fsi) const
+    -> api_error {
   bool exists{};
   auto res = is_directory(api_path, exists);
   if (res != api_error::success) {
@@ -336,9 +339,10 @@ auto base_provider::get_filesystem_item(
   return api_error::success;
 }
 
-auto base_provider::get_filesystem_item_and_file(
-    const std::string &api_path, api_file &file,
-    filesystem_item &fsi) const -> api_error {
+auto base_provider::get_filesystem_item_and_file(const std::string &api_path,
+                                                 api_file &file,
+                                                 filesystem_item &fsi) const
+    -> api_error {
   auto res = get_file(api_path, file);
   if (res != api_error::success) {
     return res;
@@ -399,11 +403,7 @@ auto base_provider::get_total_item_count() const -> std::uint64_t {
 }
 
 auto base_provider::get_used_drive_space() const -> std::uint64_t {
-  if (used_space == 0U) {
-    update_used_drive_space();
-  }
-
-  return used_space;
+  return db3_->get_total_size();
 }
 
 auto base_provider::is_file_writeable(const std::string &api_path) const
@@ -424,6 +424,10 @@ void base_provider::remove_deleted_files(bool source_only) {
     auto source_list =
         utils::file::directory{config_.get_cache_directory()}.get_files();
     for (auto &&source_file : source_list) {
+      if (stop_requested_) {
+        return;
+      }
+
       filesystem_item fsi{};
       if (get_filesystem_item_from_source_path(source_file->get_path(), fsi) !=
           api_error::item_not_found) {
@@ -466,16 +470,26 @@ void base_provider::remove_deleted_files(bool source_only) {
     std::string source_path;
   };
 
-  api_file_list list{};
-  auto res = get_file_list(list);
-  if (res != api_error::success) {
-    utils::error::raise_error(function_name, res, "failed to get file list");
-    return;
+  {
+    api_file_list list{};
+    std::string marker;
+    auto res{api_error::more_data};
+    while (not stop_requested_ && res == api_error::more_data) {
+      res = get_file_list(list, marker);
+      if (res != api_error::success && res != api_error::more_data) {
+        utils::error::raise_error(function_name, res,
+                                  "failed to get file list");
+        return;
+      }
+    }
   }
 
   std::vector<removed_item> removed_list{};
-
   for (auto &&api_path : db3_->get_api_path_list()) {
+    if (stop_requested_) {
+      return;
+    }
+
     api_meta_map meta{};
     if (get_item_meta(api_path, meta) != api_error::success) {
       continue;
@@ -510,6 +524,10 @@ void base_provider::remove_deleted_files(bool source_only) {
       utils::path::combine(config_.get_data_directory(), {"orphaned"});
 
   for (auto &&item : removed_list) {
+    if (stop_requested_) {
+      return;
+    }
+
     if (item.directory) {
       continue;
     }
@@ -559,6 +577,10 @@ void base_provider::remove_deleted_files(bool source_only) {
   }
 
   for (auto &&item : removed_list) {
+    if (stop_requested_) {
+      return;
+    }
+
     if (not item.directory) {
       continue;
     }
@@ -664,6 +686,8 @@ auto base_provider::set_item_meta(const std::string &api_path,
 
 auto base_provider::start(api_item_added_callback api_item_added,
                           i_file_manager *mgr) -> bool {
+  stop_requested_ = false;
+
   api_item_added_ = api_item_added;
   fm_ = mgr;
 
@@ -702,30 +726,15 @@ auto base_provider::start(api_item_added_callback api_item_added,
 
   polling::instance().set_callback({"check_deleted", polling::frequency::low,
                                     [this]() { remove_deleted_files(false); }});
-  polling::instance().set_callback({"drive_space", polling::frequency::low,
-                                    [this]() { update_used_drive_space(); }});
   return true;
 }
 
 void base_provider::stop() {
+  stop_requested_ = true;
   polling::instance().remove_callback("check_deleted");
-  polling::instance().remove_callback("drive_space");
   db3_.reset();
 }
 
-void base_provider::update_used_drive_space() const {
-  REPERTORY_USES_FUNCTION_NAME();
-
-  try {
-    auto next_used_space = get_used_drive_space_impl();
-    get_file_mgr()->update_used_space(next_used_space);
-    used_space = next_used_space;
-  } catch (const std::exception &ex) {
-    utils::error::raise_error(function_name, ex,
-                              "failed to get used drive space");
-  }
-}
-
 auto base_provider::upload_file(const std::string &api_path,
                                 const std::string &source_path,
                                 stop_type &stop_requested) -> api_error {

repertory/librepertory/src/providers/encrypt/encrypt_provider.cpp

@@ -405,7 +405,9 @@ auto encrypt_provider::get_file(const std::string &api_path,
   return api_error::error;
 }
 
-auto encrypt_provider::get_file_list(api_file_list &list) const -> api_error {
+auto encrypt_provider::get_file_list(api_file_list &list,
+                                     std::string & /* marker */) const
+    -> api_error {
   REPERTORY_USES_FUNCTION_NAME();
 
   const auto cfg = config_.get_encrypt_config();

repertory/librepertory/src/providers/meta_db.cpp

@@ -44,13 +44,14 @@ meta_db::meta_db(const app_config &cfg) {
            "data TEXT, "
            "directory INTEGER, "
            "pinned INTEGER, "
+           "size INTEGER, "
            "source_path TEXT"
            ");"},
       },
   };
 
   db_ = utils::db::sqlite::create_db(
-      utils::path::combine(cfg.get_data_directory(), {"meta.db"}),
+      utils::path::combine(cfg.get_data_directory(), {"provider_meta.db"}),
       sql_create_tables);
 }
 
@@ -90,8 +91,8 @@ auto meta_db::get_api_path_list() -> std::vector<std::string> {
   return ret;
 }
 
-auto meta_db::get_item_meta(const std::string &api_path,
-                            api_meta_map &meta) -> api_error {
+auto meta_db::get_item_meta(const std::string &api_path, api_meta_map &meta)
+    -> api_error {
   REPERTORY_USES_FUNCTION_NAME();
 
   auto result = utils::db::sqlite::db_select{*db_, table_name}
@@ -114,9 +115,10 @@ auto meta_db::get_item_meta(const std::string &api_path,
           row->get_column("directory").get_value<std::int64_t>() == 1);
       meta[META_PINNED] = utils::string::from_bool(
           row->get_column("pinned").get_value<std::int64_t>() == 1);
+      meta[META_SIZE] = std::to_string(static_cast<std::uint64_t>(
+          row->get_column("size").get_value<std::int64_t>()));
       meta[META_SOURCE] =
           row->get_column("source_path").get_value<std::string>();
-
       return api_error::success;
     }
 
@@ -156,6 +158,9 @@ auto meta_db::get_item_meta(const std::string &api_path, const std::string &key,
           : key == META_DIRECTORY
               ? utils::string::from_bool(
                     row->get_column("directory").get_value<std::int64_t>() == 1)
+          : key == META_SIZE
+              ? std::to_string(static_cast<std::uint64_t>(
+                    row->get_column("size").get_value<std::int64_t>()))
               : json::parse(
                     row->get_column("data").get_value<std::string>())[key]
                     .get<std::string>();
@@ -218,6 +223,28 @@ auto meta_db::get_total_item_count() const -> std::uint64_t {
   return ret;
 }
 
+auto meta_db::get_total_size() const -> std::uint64_t {
+  REPERTORY_USES_FUNCTION_NAME();
+
+  try {
+    auto result = utils::db::sqlite::db_select{*db_, table_name}
+                      .column("SUM(size) as total_size")
+                      .where("directory")
+                      .equals(0)
+                      .go();
+
+    std::optional<utils::db::sqlite::db_result::row> row;
+    if (result.get_row(row) && row.has_value()) {
+      return static_cast<std::uint64_t>(
+          row->get_column("total_size").get_value<std::int64_t>());
+    }
+  } catch (const std::exception &e) {
+    utils::error::raise_error(function_name, e, "failed to get total size");
+  }
+
+  return 0U;
+}
+
 void meta_db::remove_api_path(const std::string &api_path) {
   REPERTORY_USES_FUNCTION_NAME();
 
@@ -274,16 +301,19 @@ auto meta_db::set_item_meta(const std::string &api_path,
   return update_item_meta(api_path, existing_meta);
 }
 
-auto meta_db::update_item_meta(const std::string &api_path,
-                               api_meta_map meta) -> api_error {
+auto meta_db::update_item_meta(const std::string &api_path, api_meta_map meta)
+    -> api_error {
   REPERTORY_USES_FUNCTION_NAME();
 
   auto directory = utils::string::to_bool(meta[META_DIRECTORY]);
   auto pinned = utils::string::to_bool(meta[META_PINNED]);
+  auto size =
+      directory ? std::uint64_t(0U) : utils::string::to_uint64(meta[META_SIZE]);
   auto source_path = meta[META_SOURCE];
 
   meta.erase(META_DIRECTORY);
   meta.erase(META_PINNED);
+  meta.erase(META_SIZE);
   meta.erase(META_SOURCE);
 
   auto result = utils::db::sqlite::db_insert{*db_, table_name}
@@ -292,6 +322,7 @@ auto meta_db::update_item_meta(const std::string &api_path,
                     .column_value("data", nlohmann::json(meta).dump())
                     .column_value("directory", directory ? 1 : 0)
                     .column_value("pinned", pinned ? 1 : 0)
+                    .column_value("size", static_cast<std::int64_t>(size))
                     .column_value("source_path", source_path)
                     .go();
   if (not result.ok()) {

repertory/librepertory/src/providers/s3/s3_provider.cpp

@@ -43,8 +43,9 @@ s3_provider::s3_provider(app_config &config, i_http_comm &comm)
   get_comm().enable_s3_path_style(config.get_s3_config().use_path_style);
 }
 
-auto s3_provider::add_if_not_found(
-    api_file &file, const std::string &object_name) const -> api_error {
+auto s3_provider::add_if_not_found(api_file &file,
+                                   const std::string &object_name) const
+    -> api_error {
   api_meta_map meta{};
   if (get_item_meta(file.api_path, meta) == api_error::item_not_found) {
     auto err = create_path_directories(
@@ -70,7 +71,7 @@ auto s3_provider::convert_api_date(std::string_view date) -> std::uint64_t {
                 utils::string::split(date_parts.at(1U), 'Z', true).at(0U)) *
                 1000000UL;
 
-  struct tm tm1 {};
+  struct tm tm1{};
 #if defined(_WIN32)
   utils::time::strptime(date_time.c_str(), "%Y-%m-%dT%T", &tm1);
   return nanos + utils::time::windows_time_t_to_unix_time(_mkgmtime(&tm1));
@@ -85,8 +86,8 @@ auto s3_provider::create_directory_impl(const std::string &api_path,
                                         api_meta_map &meta) -> api_error {
   REPERTORY_USES_FUNCTION_NAME();
 
-  const auto cfg = get_config().get_s3_config();
-  const auto is_encrypted = not cfg.encryption_token.empty();
+  auto cfg = get_config().get_s3_config();
+  auto is_encrypted = not cfg.encryption_token.empty();
   stop_type stop_requested{false};
 
   if (is_encrypted) {
@@ -109,7 +110,7 @@ auto s3_provider::create_directory_impl(const std::string &api_path,
                              {utils::collection::to_hex_string(result)}));
   }
 
-  const auto object_name =
+  auto object_name =
       utils::path::create_api_path(is_encrypted ? meta[META_KEY] : api_path);
 
   curl::requests::http_put_file put_file{};
@@ -161,17 +162,18 @@ auto s3_provider::create_file_extra(const std::string &api_path,
   return api_error::success;
 }
 
-auto s3_provider::create_path_directories(
-    const std::string &api_path, const std::string &key) const -> api_error {
+auto s3_provider::create_path_directories(const std::string &api_path,
+                                          const std::string &key) const
+    -> api_error {
   if (api_path == "/") {
     return api_error::success;
   }
 
-  const auto encryption_token = get_config().get_s3_config().encryption_token;
-  const auto is_encrypted = not encryption_token.empty();
+  auto encryption_token = get_config().get_s3_config().encryption_token;
+  auto is_encrypted = not encryption_token.empty();
 
-  const auto path_parts = utils::string::split(api_path, '/', false);
-  const auto key_parts = utils::string::split(key, '/', false);
+  auto path_parts = utils::string::split(api_path, '/', false);
+  auto key_parts = utils::string::split(key, '/', false);
 
   if (is_encrypted && key_parts.size() != path_parts.size()) {
     return api_error::error;
@@ -223,8 +225,8 @@ auto s3_provider::get_directory_item_count(const std::string &api_path) const
   REPERTORY_USES_FUNCTION_NAME();
 
   try {
-    const auto cfg = get_config().get_s3_config();
-    const auto is_encrypted = not cfg.encryption_token.empty();
+    auto cfg = get_config().get_s3_config();
+    auto is_encrypted = not cfg.encryption_token.empty();
     std::string key;
     if (is_encrypted) {
       auto res = get_item_meta(api_path, META_KEY, key);
@@ -233,7 +235,7 @@ auto s3_provider::get_directory_item_count(const std::string &api_path) const
       }
     }
 
-    const auto object_name =
+    auto object_name =
         api_path == "/"
             ? ""
             : utils::path::create_api_path(is_encrypted ? key : api_path);
@@ -295,12 +297,13 @@ auto s3_provider::get_directory_item_count(const std::string &api_path) const
   return 0U;
 }
 
-auto s3_provider::get_directory_items_impl(
-    const std::string &api_path, directory_item_list &list) const -> api_error {
+auto s3_provider::get_directory_items_impl(const std::string &api_path,
+                                           directory_item_list &list) const
+    -> api_error {
   REPERTORY_USES_FUNCTION_NAME();
 
-  const auto cfg = get_config().get_s3_config();
-  const auto is_encrypted = not cfg.encryption_token.empty();
+  auto cfg = get_config().get_s3_config();
+  auto is_encrypted = not cfg.encryption_token.empty();
 
   auto ret = api_error::success;
   std::string key;
@@ -311,7 +314,7 @@ auto s3_provider::get_directory_items_impl(
     }
   }
 
-  const auto object_name =
+  auto object_name =
       api_path == "/"
           ? ""
           : utils::path::create_api_path(is_encrypted ? key : api_path);
@@ -414,26 +417,35 @@ auto s3_provider::get_directory_items_impl(
     for (auto &&node : node_list) {
       auto child_object_name = utils::path::create_api_path(
           node.node().select_node("Key").node().text().as_string());
-      if (child_object_name != utils::path::create_api_path(prefix)) {
-        auto size = node.node().select_node("Size").node().text().as_ullong();
-        auto last_modified = convert_api_date(
-            node.node().select_node("LastModified").node().text().as_string());
-        add_directory_item(
-            false, child_object_name, last_modified,
-            [&is_encrypted, &size](const directory_item &) -> std::uint64_t {
-              return is_encrypted ? utils::encryption::encrypting_reader::
-                                        calculate_decrypted_size(size)
-                                  : size;
-            });
+      if (child_object_name == utils::path::create_api_path(prefix)) {
+        continue;
       }
+
+      auto size = node.node().select_node("Size").node().text().as_ullong();
+      auto last_modified = convert_api_date(
+          node.node().select_node("LastModified").node().text().as_string());
+      add_directory_item(false, child_object_name, last_modified,
+                         [this, &is_encrypted, &size](
+                             const directory_item &dir_item) -> std::uint64_t {
+                           std::string size_str;
+                           if (get_item_meta(dir_item.api_path, META_SIZE,
+                                             size_str) == api_error::success) {
+                             return utils::string::to_uint64(size_str);
+                           }
+
+                           return is_encrypted
+                                      ? utils::encryption::encrypting_reader::
+                                            calculate_decrypted_size(size)
+                                      : size;
+                         });
     }
   }
 
   return ret;
 }
 
-auto s3_provider::get_file(const std::string &api_path,
-                           api_file &file) const -> api_error {
+auto s3_provider::get_file(const std::string &api_path, api_file &file) const
+    -> api_error {
   REPERTORY_USES_FUNCTION_NAME();
 
   try {
@@ -450,12 +462,20 @@ auto s3_provider::get_file(const std::string &api_path,
     file.api_parent = utils::path::get_parent_api_path(api_path);
     file.accessed_date = file.changed_date = file.creation_date =
         file.modified_date = result.last_modified;
-    file.file_size =
-        is_encrypted
-            ? utils::encryption::encrypting_reader::calculate_decrypted_size(
-                  result.content_length)
-            : result.content_length;
     file.key = is_encrypted ? utils::path::create_api_path(api_path) : "";
+
+    std::string size_str;
+    if (get_item_meta(file.api_path, META_SIZE, size_str) ==
+        api_error::success) {
+      file.file_size = utils::string::to_uint64(size_str);
+    } else {
+      file.file_size =
+          is_encrypted
+              ? utils::encryption::encrypting_reader::calculate_decrypted_size(
+                    result.content_length)
+              : result.content_length;
+    }
+
     return add_if_not_found(file, object_name);
   } catch (const std::exception &e) {
     utils::error::raise_error(function_name, e, "exception occurred");
@@ -464,89 +484,86 @@ auto s3_provider::get_file(const std::string &api_path,
   return api_error::error;
 }
 
-auto s3_provider::get_file_list(api_file_list &list) const -> api_error {
+auto s3_provider::get_file_list(api_file_list &list, std::string &marker) const
+    -> api_error {
   REPERTORY_USES_FUNCTION_NAME();
 
-  auto grab_more{true};
-  std::string token{};
-  while (grab_more) {
-    std::string response_data;
-    long response_code{};
-    if (not get_object_list(response_data, response_code, std::nullopt,
-                            std::nullopt, token)) {
-      return api_error::comm_error;
-    }
+  std::string response_data;
+  long response_code{};
+  if (not get_object_list(response_data, response_code, std::nullopt,
+                          std::nullopt, marker)) {
+    return api_error::comm_error;
+  }
+  marker.clear();
 
-    if (response_code != http_error_codes::ok) {
-      utils::error::raise_error(function_name, response_code,
-                                "failed to get file list");
-      return api_error::comm_error;
-    }
-
-    pugi::xml_document doc;
-    auto res = doc.load_string(response_data.c_str());
-    if (res.status != pugi::xml_parse_status::status_ok) {
-      utils::error::raise_error(function_name, res.status,
-                                "failed to parse xml document");
-      return api_error::comm_error;
-    }
-
-    grab_more = doc.select_node("/ListBucketResult/IsTruncated")
-                    .node()
-                    .text()
-                    .as_bool();
-    if (grab_more) {
-      token = doc.select_node("/ListBucketResult/NextContinuationToken")
-                  .node()
-                  .text()
-                  .as_string();
-    }
-
-    auto node_list = doc.select_nodes("/ListBucketResult/Contents");
-    for (auto &&node : node_list) {
-      auto api_path =
-          std::string{node.node().select_node("Key").node().text().as_string()};
-      if (not utils::string::ends_with(api_path, "/")) {
-        auto is_encrypted =
-            not get_config().get_s3_config().encryption_token.empty();
-        if (is_encrypted) {
-          auto err = decrypt_object_name(api_path);
-          if (err != api_error::success) {
-            return err;
-          }
-        }
-
-        auto size = node.node().select_node("Size").node().text().as_ullong();
-
-        api_file file{};
-        file.api_path = utils::path::create_api_path(api_path);
-        file.api_parent = utils::path::get_parent_api_path(file.api_path);
-        file.accessed_date = file.changed_date = file.creation_date =
-            file.modified_date =
-                convert_api_date(node.node()
-                                     .select_node("LastModified")
-                                     .node()
-                                     .text()
-                                     .as_string());
-        file.file_size = is_encrypted ? utils::encryption::encrypting_reader::
-                                            calculate_decrypted_size(size)
-                                      : size;
-        file.key = is_encrypted ? utils::path::create_api_path(api_path) : "";
-        auto err = add_if_not_found(file, api_path);
-        if (err != api_error::success) {
-          return err;
-        }
-
-        list.push_back(std::move(file));
-      }
-    }
+  if (response_code != http_error_codes::ok) {
+    utils::error::raise_error(function_name, response_code,
+                              "failed to get file list");
+    return api_error::comm_error;
   }
 
-  return api_error::success;
+  pugi::xml_document doc;
+  auto res = doc.load_string(response_data.c_str());
+  if (res.status != pugi::xml_parse_status::status_ok) {
+    utils::error::raise_error(function_name, res.status,
+                              "failed to parse xml document");
+    return api_error::comm_error;
+  }
+
+  auto grab_more =
+      doc.select_node("/ListBucketResult/IsTruncated").node().text().as_bool();
+  if (grab_more) {
+    marker = doc.select_node("/ListBucketResult/NextContinuationToken")
+                 .node()
+                 .text()
+                 .as_string();
+  }
+
+  auto node_list = doc.select_nodes("/ListBucketResult/Contents");
+  for (auto &&node : node_list) {
+    auto api_path =
+        std::string{node.node().select_node("Key").node().text().as_string()};
+    if (utils::string::ends_with(api_path, "/")) {
+      continue;
+    }
+
+    auto is_encrypted =
+        not get_config().get_s3_config().encryption_token.empty();
+    if (is_encrypted) {
+      auto err = decrypt_object_name(api_path);
+      if (err != api_error::success) {
+        return err;
+      }
+    }
+
+    auto size = node.node().select_node("Size").node().text().as_ullong();
+
+    api_file file{};
+    file.api_path = utils::path::create_api_path(api_path);
+    file.api_parent = utils::path::get_parent_api_path(file.api_path);
+    file.accessed_date = file.changed_date = file.creation_date =
+        file.modified_date = convert_api_date(
+            node.node().select_node("LastModified").node().text().as_string());
+    file.file_size =
+        is_encrypted
+            ? utils::encryption::encrypting_reader::calculate_decrypted_size(
+                  size)
+            : size;
+    file.key = is_encrypted ? utils::path::create_api_path(api_path) : "";
+    auto err = add_if_not_found(file, api_path);
+    if (err != api_error::success) {
+      return err;
+    }
+
+    list.push_back(std::move(file));
+  }
+
+  return grab_more ? api_error::more_data : api_error::success;
 }
 
-auto s3_provider::get_last_modified(
-    bool directory, const std::string &api_path) const -> std::uint64_t {
+auto s3_provider::get_last_modified(bool directory,
+                                    const std::string &api_path) const
+    -> std::uint64_t {
   bool is_encrypted{};
   std::string object_name;
   head_object_result result{};
@@ -556,13 +573,14 @@ auto s3_provider::get_last_modified(
              : utils::time::get_time_now();
 }
 
-auto s3_provider::get_object_info(
-    bool directory, const std::string &api_path, bool &is_encrypted,
-    std::string &object_name, head_object_result &result) const -> api_error {
+auto s3_provider::get_object_info(bool directory, const std::string &api_path,
+                                  bool &is_encrypted, std::string &object_name,
+                                  head_object_result &result) const
+    -> api_error {
   REPERTORY_USES_FUNCTION_NAME();
 
   try {
-    const auto cfg = get_config().get_s3_config();
+    auto cfg = get_config().get_s3_config();
     is_encrypted = not cfg.encryption_token.empty();
 
     std::string key;
@@ -607,10 +625,12 @@ auto s3_provider::get_object_info(
   return api_error::error;
 }
 
-auto s3_provider::get_object_list(
-    std::string &response_data, long &response_code,
-    std::optional<std::string> delimiter, std::optional<std::string> prefix,
-    std::optional<std::string> token) const -> bool {
+auto s3_provider::get_object_list(std::string &response_data,
+                                  long &response_code,
+                                  std::optional<std::string> delimiter,
+                                  std::optional<std::string> prefix,
+                                  std::optional<std::string> token) const
+    -> bool {
   curl::requests::http_get get{};
   get.allow_timeout = true;
   get.aws_service = "aws:amz:" + get_config().get_s3_config().region + ":s3";
@@ -639,58 +659,8 @@ auto s3_provider::get_total_drive_space() const -> std::uint64_t {
   return std::numeric_limits<std::int64_t>::max() / std::int64_t(2);
 }
 
-auto s3_provider::get_used_drive_space_impl() const -> std::uint64_t {
-  auto grab_more{true};
-  std::string token{};
-  std::uint64_t total_size{};
-  while (grab_more) {
-    std::string response_data;
-    long response_code{};
-    if (not get_object_list(response_data, response_code, std::nullopt,
-                            std::nullopt, token)) {
-      return total_size;
-    }
-
-    if (response_code != http_error_codes::ok) {
-      return total_size;
-    }
-
-    pugi::xml_document doc;
-    auto res = doc.load_string(response_data.c_str());
-    if (res.status != pugi::xml_parse_status::status_ok) {
-      return total_size;
-    }
-
-    grab_more = doc.select_node("/ListBucketResult/IsTruncated")
-                    .node()
-                    .text()
-                    .as_bool();
-    if (grab_more) {
-      token = doc.select_node("/ListBucketResult/NextContinuationToken")
-                  .node()
-                  .text()
-                  .as_string();
-    }
-
-    const auto cfg = get_config().get_s3_config();
-    const auto is_encrypted = not cfg.encryption_token.empty();
-
-    auto node_list = doc.select_nodes("/ListBucketResult/Contents");
-    total_size = std::accumulate(
-        node_list.begin(), node_list.end(), total_size,
-        [&is_encrypted](std::uint64_t total, auto node) -> std::uint64_t {
-          auto size = node.node().select_node("Size").node().text().as_ullong();
-          return total + (is_encrypted ? utils::encryption::encrypting_reader::
-                                             calculate_decrypted_size(size)
-                                       : size);
-        });
-  }
-
-  return total_size;
-}
-
-auto s3_provider::is_directory(const std::string &api_path,
-                               bool &exists) const -> api_error {
+auto s3_provider::is_directory(const std::string &api_path, bool &exists) const
+    -> api_error {
   REPERTORY_USES_FUNCTION_NAME();
 
   exists = false;
@@ -717,8 +687,8 @@ auto s3_provider::is_directory(const std::string &api_path,
   return api_error::error;
 }
 
-auto s3_provider::is_file(const std::string &api_path,
-                          bool &exists) const -> api_error {
+auto s3_provider::is_file(const std::string &api_path, bool &exists) const
+    -> api_error {
   REPERTORY_USES_FUNCTION_NAME();
 
   exists = false;
@@ -755,8 +725,8 @@ auto s3_provider::read_file_bytes(const std::string &api_path, std::size_t size,
   REPERTORY_USES_FUNCTION_NAME();
 
   try {
-    const auto cfg = get_config().get_s3_config();
-    const auto is_encrypted = not cfg.encryption_token.empty();
+    auto cfg = get_config().get_s3_config();
+    auto is_encrypted = not cfg.encryption_token.empty();
     std::string key;
     if (is_encrypted) {
       auto res = get_item_meta(api_path, META_KEY, key);
@@ -765,7 +735,7 @@ auto s3_provider::read_file_bytes(const std::string &api_path, std::size_t size,
       }
     }
 
-    const auto object_name =
+    auto object_name =
         utils::path::create_api_path(is_encrypted ? key : api_path);
 
     const auto read_bytes =
@@ -825,30 +795,31 @@ auto s3_provider::read_file_bytes(const std::string &api_path, std::size_t size,
       return res;
     };
 
-    if (is_encrypted) {
-      std::string temp;
-      auto res = get_item_meta(api_path, META_SIZE, temp);
-      if (res != api_error::success) {
-        return res;
-      }
-      const auto total_size = utils::string::to_uint64(temp);
-
-      return utils::encryption::read_encrypted_range(
-                 {offset, offset + size - 1U},
-                 utils::encryption::generate_key<utils::encryption::hash_256_t>(
-                     cfg.encryption_token),
-                 [&](data_buffer &ct_buffer, std::uint64_t start_offset,
-                     std::uint64_t end_offset) -> bool {
-                   return read_bytes((end_offset - start_offset + 1U),
-                                     start_offset,
-                                     ct_buffer) == api_error::success;
-                 },
-                 total_size, data)
-                 ? api_error::success
-                 : api_error::decryption_error;
+    if (not is_encrypted) {
+      return read_bytes(size, offset, data);
     }
 
-    return read_bytes(size, offset, data);
+    std::string temp;
+    auto res = get_item_meta(api_path, META_SIZE, temp);
+    if (res != api_error::success) {
+      return res;
+    }
+
+    auto total_size = utils::string::to_uint64(temp);
+    return utils::encryption::read_encrypted_range(
+               {offset, offset + size - 1U},
+               utils::encryption::generate_key<utils::encryption::hash_256_t>(
+                   cfg.encryption_token),
+               [&](data_buffer &ct_buffer, std::uint64_t start_offset,
+                   std::uint64_t end_offset) -> bool {
+                 return read_bytes((end_offset - start_offset + 1U),
+                                   start_offset,
+                                   ct_buffer) == api_error::success;
+               },
+               total_size, data)
+               ? api_error::success
+               : api_error::decryption_error;
+
   } catch (const std::exception &e) {
     utils::error::raise_error(function_name, e, "exception occurred");
   }
@@ -860,8 +831,8 @@ auto s3_provider::remove_directory_impl(const std::string &api_path)
     -> api_error {
   REPERTORY_USES_FUNCTION_NAME();
 
-  const auto cfg = get_config().get_s3_config();
-  const auto is_encrypted = not cfg.encryption_token.empty();
+  auto cfg = get_config().get_s3_config();
+  auto is_encrypted = not cfg.encryption_token.empty();
 
   std::string key;
   if (is_encrypted) {
@@ -871,7 +842,7 @@ auto s3_provider::remove_directory_impl(const std::string &api_path)
     }
   }
 
-  const auto object_name =
+  auto object_name =
       utils::path::create_api_path(is_encrypted ? key : api_path);
 
   curl::requests::http_delete del{};
@@ -902,8 +873,8 @@ auto s3_provider::remove_directory_impl(const std::string &api_path)
 auto s3_provider::remove_file_impl(const std::string &api_path) -> api_error {
   REPERTORY_USES_FUNCTION_NAME();
 
-  const auto cfg = get_config().get_s3_config();
-  const auto is_encrypted = not cfg.encryption_token.empty();
+  auto cfg = get_config().get_s3_config();
+  auto is_encrypted = not cfg.encryption_token.empty();
 
   std::string key;
   if (is_encrypted) {
@@ -913,7 +884,7 @@ auto s3_provider::remove_file_impl(const std::string &api_path) -> api_error {
     }
   }
 
-  const auto object_name =
+  auto object_name =
       utils::path::create_api_path(is_encrypted ? key : api_path);
 
   curl::requests::http_delete del{};
@@ -973,8 +944,8 @@ auto s3_provider::upload_file_impl(const std::string &api_path,
     file_size = opt_size.value();
   }
 
-  const auto cfg = get_config().get_s3_config();
-  const auto is_encrypted = not cfg.encryption_token.empty();
+  auto cfg = get_config().get_s3_config();
+  auto is_encrypted = not cfg.encryption_token.empty();
 
   std::string key;
   if (is_encrypted) {
@@ -984,7 +955,7 @@ auto s3_provider::upload_file_impl(const std::string &api_path,
     }
   }
 
-  const auto object_name =
+  auto object_name =
       utils::path::create_api_path(is_encrypted ? key : api_path);
 
   curl::requests::http_put_file put_file{};

repertory/librepertory/src/providers/sia/sia_provider.cpp

@@ -214,7 +214,9 @@ auto sia_provider::get_file(const std::string &api_path, api_file &file) const
   return api_error::success;
 }
 
-auto sia_provider::get_file_list(api_file_list &list) const -> api_error {
+auto sia_provider::get_file_list(api_file_list &list,
+                                 std::string & /* marker */) const
+    -> api_error {
   REPERTORY_USES_FUNCTION_NAME();
 
   using dir_func = std::function<api_error(std::string api_path)>;
@@ -393,37 +395,6 @@ auto sia_provider::get_total_drive_space() const -> std::uint64_t {
   return 0U;
 }
 
-auto sia_provider::get_used_drive_space_impl() const -> std::uint64_t {
-  REPERTORY_USES_FUNCTION_NAME();
-
-  curl::requests::http_get get{};
-  get.allow_timeout = true;
-  get.path = "/api/bus/stats/objects";
-  get.query["bucket"] = get_bucket(get_config().get_sia_config());
-
-  json object_data{};
-  get.response_handler = [&object_data](const data_buffer &data,
-                                        long response_code) {
-    if (response_code == http_error_codes::ok) {
-      object_data = nlohmann::json::parse(data.begin(), data.end());
-    }
-  };
-
-  long response_code{};
-  stop_type stop_requested{};
-  if (not get_comm().make_request(get, response_code, stop_requested)) {
-    return 0U;
-  }
-
-  if (response_code != http_error_codes::ok) {
-    utils::error::raise_error(function_name, response_code,
-                              "failed to get used drive space");
-    return 0U;
-  }
-
-  return object_data["totalObjectsSize"].get<std::uint64_t>();
-}
-
 auto sia_provider::is_directory(const std::string &api_path, bool &exists) const
     -> api_error {
   REPERTORY_USES_FUNCTION_NAME();

repertory/librepertory/src/types/repertory.cpp

@@ -25,8 +25,9 @@
 #include "utils/string.hpp"
 
 namespace repertory {
-auto download_type_from_string(
-    std::string type, const download_type &default_type) -> download_type {
+auto download_type_from_string(std::string type,
+                               const download_type &default_type)
+    -> download_type {
   type = utils::string::to_lower(utils::string::trim(type));
   if (type == "direct") {
     return download_type::direct;
@@ -84,6 +85,7 @@ static const std::unordered_map<api_error, std::string> LOOKUP = {
     {api_error::invalid_version, "invalid_version"},
     {api_error::item_exists, "item_exists"},
     {api_error::item_not_found, "item_not_found"},
+    {api_error::more_data, "more_data"},
     {api_error::no_disk_space, "no_disk_space"},
     {api_error::not_implemented, "not_implemented"},
     {api_error::not_supported, "not_supported"},

repertory/librepertory/src/utils/polling.cpp

@@ -31,33 +31,35 @@ void polling::frequency_thread(
   while (not stop_requested_) {
     std::deque<std::future<void>> futures;
     unique_mutex_lock lock(mutex_);
-    if (not stop_requested_ &&
-        notify_.wait_for(lock, std::chrono::seconds(get_frequency_seconds())) ==
-            std::cv_status::timeout) {
-      for (const auto &item : items_) {
-        if (item.second.freq == freq) {
-          futures.emplace_back(
-              std::async(std::launch::async, [this, &freq, item]() -> void {
-                if (config_->get_event_level() == event_level::trace ||
-                    freq != frequency::second) {
-                  event_system::instance().raise<polling_item_begin>(
-                      item.first);
-                }
-                item.second.action();
-                if (config_->get_event_level() == event_level::trace ||
-                    freq != frequency::second) {
-                  event_system::instance().raise<polling_item_end>(item.first);
-                }
-              }));
-        }
-      }
-      lock.unlock();
-
-      while (not futures.empty()) {
-        futures.front().wait();
-        futures.pop_front();
+    for (const auto &item : items_) {
+      if (item.second.freq == freq) {
+        futures.emplace_back(
+            std::async(std::launch::async, [this, &freq, item]() -> void {
+              if (config_->get_event_level() == event_level::trace ||
+                  freq != frequency::second) {
+                event_system::instance().raise<polling_item_begin>(item.first);
+              }
+              item.second.action();
+              if (config_->get_event_level() == event_level::trace ||
+                  freq != frequency::second) {
+                event_system::instance().raise<polling_item_end>(item.first);
+              }
+            }));
       }
     }
+    lock.unlock();
+
+    while (not futures.empty()) {
+      futures.front().wait();
+      futures.pop_front();
+    }
+
+    lock.lock();
+    if (stop_requested_) {
+      continue;
+    }
+
+    notify_.wait_for(lock, std::chrono::seconds(get_frequency_seconds()));
   }
 }
 
@@ -73,50 +75,53 @@ void polling::set_callback(const polling_item &item) {
 
 void polling::start(app_config *config) {
   mutex_lock lock(start_stop_mutex_);
-  if (not high_frequency_thread_) {
-    event_system::instance().raise<service_started>("polling");
-    config_ = config;
-    stop_requested_ = false;
-    high_frequency_thread_ = std::make_unique<std::thread>([this]() -> void {
-      this->frequency_thread(
-          [this]() -> std::uint32_t {
-            return config_->get_high_frequency_interval_secs();
-          },
-          frequency::high);
-    });
-    low_frequency_thread_ = std::make_unique<std::thread>([this]() -> void {
-      this->frequency_thread(
-          [this]() -> std::uint32_t {
-            return config_->get_low_frequency_interval_secs();
-          },
-          frequency::low);
-    });
-    second_frequency_thread_ = std::make_unique<std::thread>([this]() -> void {
-      this->frequency_thread([]() -> std::uint32_t { return 1U; },
-                             frequency::second);
-    });
+  if (high_frequency_thread_) {
+    return;
   }
+
+  event_system::instance().raise<service_started>("polling");
+  config_ = config;
+  stop_requested_ = false;
+  high_frequency_thread_ = std::make_unique<std::thread>([this]() -> void {
+    this->frequency_thread(
+        [this]() -> std::uint32_t {
+          return config_->get_high_frequency_interval_secs();
+        },
+        frequency::high);
+  });
+  low_frequency_thread_ = std::make_unique<std::thread>([this]() -> void {
+    this->frequency_thread(
+        [this]() -> std::uint32_t {
+          return config_->get_low_frequency_interval_secs();
+        },
+        frequency::low);
+  });
+  second_frequency_thread_ = std::make_unique<std::thread>([this]() -> void {
+    this->frequency_thread([]() -> std::uint32_t { return 1U; },
+                           frequency::second);
+  });
 }
 
 void polling::stop() {
-  if (high_frequency_thread_) {
-    event_system::instance().raise<service_shutdown_begin>("polling");
-    mutex_lock lock(start_stop_mutex_);
-    if (high_frequency_thread_) {
-      {
-        stop_requested_ = true;
-
-        mutex_lock lock2(mutex_);
-        notify_.notify_all();
-      }
-      high_frequency_thread_->join();
-      low_frequency_thread_->join();
-      second_frequency_thread_->join();
-      high_frequency_thread_.reset();
-      low_frequency_thread_.reset();
-      second_frequency_thread_.reset();
-    }
-    event_system::instance().raise<service_shutdown_end>("polling");
+  mutex_lock lock(start_stop_mutex_);
+  if (not high_frequency_thread_) {
+    return;
   }
+
+  event_system::instance().raise<service_shutdown_begin>("polling");
+  stop_requested_ = true;
+
+  unique_mutex_lock lock2(mutex_);
+  notify_.notify_all();
+  lock2.unlock();
+
+  high_frequency_thread_->join();
+  low_frequency_thread_->join();
+  second_frequency_thread_->join();
+  high_frequency_thread_.reset();
+  low_frequency_thread_.reset();
+  second_frequency_thread_.reset();
+
+  event_system::instance().raise<service_shutdown_end>("polling");
 }
 } // namespace repertory

repertory/repertory_test/include/mocks/mock_provider.hpp

@@ -60,8 +60,8 @@ public:
   MOCK_METHOD(api_error, get_file,
               (const std::string &api_path, api_file &file), (const, override));
 
-  MOCK_METHOD(api_error, get_file_list, (api_file_list & list),
-              (const, override));
+  MOCK_METHOD(api_error, get_file_list,
+              (api_file_list & list, std::string &marker), (const, override));
 
   MOCK_METHOD(api_error, get_file_size,
               (const std::string &api_path, std::uint64_t &file_size),

repertory/repertory_test/src/providers_test.cpp

@@ -547,7 +547,8 @@ static void get_file_fails_if_item_is_directory(const app_config &cfg,
 
 static void get_file_list(const app_config &cfg, i_provider &provider) {
   api_file_list list{};
-  EXPECT_EQ(api_error::success, provider.get_file_list(list));
+  std::string marker;
+  EXPECT_EQ(api_error::success, provider.get_file_list(list, marker));
   if (provider.get_provider_type() == provider_type::encrypt) {
     EXPECT_EQ(std::size_t(2U), list.size());