From 62857e1372a529ca7cf4a55d8933153842229e0f Mon Sep 17 00:00:00 2001
From: "Scott E. Graves" <scott.e.graves@protonmail.com>
Date: Tue, 3 Dec 2024 14:02:51 -0600
Subject: [PATCH] refactor

---
 .../include/providers/base_provider.hpp       | 162 ++++++----
 .../src/providers/base_provider.cpp           | 281 ++++++++++--------
 2 files changed, 254 insertions(+), 189 deletions(-)

diff --git a/repertory/librepertory/include/providers/base_provider.hpp b/repertory/librepertory/include/providers/base_provider.hpp
index 874b3812..59ae4da2 100644
--- a/repertory/librepertory/include/providers/base_provider.hpp
+++ b/repertory/librepertory/include/providers/base_provider.hpp
@@ -32,6 +32,13 @@ class i_file_manager;
 class i_http_comm;
 
 class base_provider : public i_provider {
+private:
+  struct removed_item final {
+    std::string api_path;
+    bool directory{};
+    std::string source_path;
+  };
+
 public:
   base_provider(app_config &config, i_http_comm &comm)
       : config_(config), comm_(comm) {}
@@ -46,20 +53,35 @@ private:
   i_file_manager *fm_{};
 
 private:
-  void remove_deleted_files(const stop_type &stop_requested);
+  void add_all_items(const stop_type &stop_requested);
+
+  void get_removed_items(std::deque<removed_item> &directories,
+                         std::deque<removed_item> &files,
+                         const stop_type &stop_requested) const;
+
+  void process_removed_directories(std::deque<removed_item> &removed_list,
+                                   const stop_type &stop_requested);
+
+  void process_removed_files(std::deque<removed_item> &removed_list,
+                             const stop_type &stop_requested);
+
+  void remove_deleted_items(const stop_type &stop_requested);
+
+  void remove_unmatched_source_files(const stop_type &stop_requested);
 
 protected:
-  [[nodiscard]] static auto
-  create_api_file(std::string path, std::string key, std::uint64_t size,
-                  std::uint64_t file_time) -> api_file;
+  [[nodiscard]] static auto create_api_file(std::string path, std::string key,
+                                            std::uint64_t size,
+                                            std::uint64_t file_time)
+      -> api_file;
 
   [[nodiscard]] static auto create_api_file(std::string path,
                                             std::uint64_t size,
                                             api_meta_map &meta) -> api_file;
 
-  [[nodiscard]] virtual auto
-  create_directory_impl(const std::string &api_path,
-                        api_meta_map &meta) -> api_error = 0;
+  [[nodiscard]] virtual auto create_directory_impl(const std::string &api_path,
+                                                   api_meta_map &meta)
+      -> api_error = 0;
 
   [[nodiscard]] virtual auto
   create_file_extra(const std::string & /* api_path */,
@@ -71,8 +93,8 @@ protected:
     return api_item_added_;
   }
 
-  [[nodiscard]] auto
-  get_api_item_added() const -> const api_item_added_callback & {
+  [[nodiscard]] auto get_api_item_added() const
+      -> const api_item_added_callback & {
     return api_item_added_;
   }
 
@@ -98,20 +120,22 @@ protected:
     return fm_;
   }
 
-  [[nodiscard]] virtual auto
-  remove_directory_impl(const std::string &api_path) -> api_error = 0;
+  [[nodiscard]] virtual auto remove_directory_impl(const std::string &api_path)
+      -> api_error = 0;
 
-  [[nodiscard]] virtual auto
-  remove_file_impl(const std::string &api_path) -> api_error = 0;
+  [[nodiscard]] virtual auto remove_file_impl(const std::string &api_path)
+      -> api_error = 0;
 
-  [[nodiscard]] virtual auto
-  upload_file_impl(const std::string &api_path, const std::string &source_path,
-                   stop_type &stop_requested) -> api_error = 0;
+  [[nodiscard]] virtual auto upload_file_impl(const std::string &api_path,
+                                              const std::string &source_path,
+                                              stop_type &stop_requested)
+      -> api_error = 0;
 
 public:
-  [[nodiscard]] auto create_directory_clone_source_meta(
-      const std::string &source_api_path,
-      const std::string &api_path) -> api_error override;
+  [[nodiscard]] auto
+  create_directory_clone_source_meta(const std::string &source_api_path,
+                                     const std::string &api_path)
+      -> api_error override;
 
   [[nodiscard]] auto create_directory(const std::string &api_path,
                                       api_meta_map &meta) -> api_error override;
@@ -119,76 +143,82 @@ public:
   [[nodiscard]] auto create_file(const std::string &api_path,
                                  api_meta_map &meta) -> api_error override;
 
-  [[nodiscard]] auto
-  get_api_path_from_source(const std::string &source_path,
-                           std::string &api_path) const -> api_error override;
+  [[nodiscard]] auto get_api_path_from_source(const std::string &source_path,
+                                              std::string &api_path) const
+      -> api_error override;
+
+  [[nodiscard]] auto get_directory_items(const std::string &api_path,
+                                         directory_item_list &list) const
+      -> api_error override;
+
+  [[nodiscard]] auto get_file_size(const std::string &api_path,
+                                   std::uint64_t &file_size) const
+      -> api_error override;
+
+  [[nodiscard]] auto get_filesystem_item(const std::string &api_path,
+                                         bool directory,
+                                         filesystem_item &fsi) const
+      -> api_error override;
+
+  [[nodiscard]] auto get_filesystem_item_and_file(const std::string &api_path,
+                                                  api_file &f,
+                                                  filesystem_item &fsi) const
+      -> api_error override;
 
   [[nodiscard]] auto
-  get_directory_items(const std::string &api_path,
-                      directory_item_list &list) const -> api_error override;
+  get_filesystem_item_from_source_path(const std::string &source_path,
+                                       filesystem_item &fsi) const
+      -> api_error override;
 
-  [[nodiscard]] auto
-  get_file_size(const std::string &api_path,
-                std::uint64_t &file_size) const -> api_error override;
+  [[nodiscard]] auto get_item_meta(const std::string &api_path,
+                                   api_meta_map &meta) const
+      -> api_error override;
 
-  [[nodiscard]] auto
-  get_filesystem_item(const std::string &api_path, bool directory,
-                      filesystem_item &fsi) const -> api_error override;
+  [[nodiscard]] auto get_item_meta(const std::string &api_path,
+                                   const std::string &key,
+                                   std::string &value) const
+      -> api_error override;
 
-  [[nodiscard]] auto get_filesystem_item_and_file(
-      const std::string &api_path, api_file &f,
-      filesystem_item &fsi) const -> api_error override;
-
-  [[nodiscard]] auto get_filesystem_item_from_source_path(
-      const std::string &source_path,
-      filesystem_item &fsi) const -> api_error override;
-
-  [[nodiscard]] auto
-  get_item_meta(const std::string &api_path,
-                api_meta_map &meta) const -> api_error override;
-
-  [[nodiscard]] auto
-  get_item_meta(const std::string &api_path, const std::string &key,
-                std::string &value) const -> api_error override;
-
-  [[nodiscard]] auto
-  get_pinned_files() const -> std::vector<std::string> override;
+  [[nodiscard]] auto get_pinned_files() const
+      -> std::vector<std::string> override;
 
   [[nodiscard]] auto get_total_item_count() const -> std::uint64_t override;
 
   [[nodiscard]] auto get_used_drive_space() const -> std::uint64_t override;
 
-  [[nodiscard]] auto
-  is_file_writeable(const std::string &api_path) const -> bool override;
+  [[nodiscard]] auto is_file_writeable(const std::string &api_path) const
+      -> bool override;
 
   [[nodiscard]] auto is_read_only() const -> bool override { return false; }
 
-  [[nodiscard]] auto
-  remove_directory(const std::string &api_path) -> api_error override;
+  [[nodiscard]] auto remove_directory(const std::string &api_path)
+      -> api_error override;
 
-  [[nodiscard]] auto
-  remove_file(const std::string &api_path) -> api_error override;
+  [[nodiscard]] auto remove_file(const std::string &api_path)
+      -> api_error override;
 
-  [[nodiscard]] auto
-  remove_item_meta(const std::string &api_path,
-                   const std::string &key) -> api_error override;
+  [[nodiscard]] auto remove_item_meta(const std::string &api_path,
+                                      const std::string &key)
+      -> api_error override;
 
-  [[nodiscard]] auto
-  set_item_meta(const std::string &api_path, const std::string &key,
-                const std::string &value) -> api_error override;
+  [[nodiscard]] auto set_item_meta(const std::string &api_path,
+                                   const std::string &key,
+                                   const std::string &value)
+      -> api_error override;
 
-  [[nodiscard]] auto
-  set_item_meta(const std::string &api_path,
-                const api_meta_map &meta) -> api_error override;
+  [[nodiscard]] auto set_item_meta(const std::string &api_path,
+                                   const api_meta_map &meta)
+      -> api_error override;
 
   [[nodiscard]] auto start(api_item_added_callback api_item_added,
                            i_file_manager *mgr) -> bool override;
 
   void stop() override;
 
-  [[nodiscard]] auto
-  upload_file(const std::string &api_path, const std::string &source_path,
-              stop_type &stop_requested) -> api_error override;
+  [[nodiscard]] auto upload_file(const std::string &api_path,
+                                 const std::string &source_path,
+                                 stop_type &stop_requested)
+      -> api_error override;
 };
 } // namespace repertory
 
diff --git a/repertory/librepertory/src/providers/base_provider.cpp b/repertory/librepertory/src/providers/base_provider.cpp
index f35c4dfc..5a46e393 100644
--- a/repertory/librepertory/src/providers/base_provider.cpp
+++ b/repertory/librepertory/src/providers/base_provider.cpp
@@ -32,9 +32,23 @@
 #include "utils/time.hpp"
 
 namespace repertory {
+void base_provider::add_all_items(const stop_type &stop_requested) {
+  REPERTORY_USES_FUNCTION_NAME();
+
+  api_file_list list{};
+  std::string marker;
+  auto res{api_error::more_data};
+  while (not stop_requested && res == api_error::more_data) {
+    res = get_file_list(list, marker);
+    if (res != api_error::success && res != api_error::more_data) {
+      utils::error::raise_error(function_name, res, "failed to get file list");
+    }
+  }
+}
+
 auto base_provider::create_api_file(std::string path, std::string key,
-                                    std::uint64_t size,
-                                    std::uint64_t file_time) -> api_file {
+                                    std::uint64_t size, std::uint64_t file_time)
+    -> api_file {
   api_file file{};
   file.api_path = utils::path::create_api_path(path);
   file.api_parent = utils::path::get_parent_api_path(file.api_path);
@@ -66,8 +80,8 @@ auto base_provider::create_api_file(std::string path, std::uint64_t size,
 }
 
 auto base_provider::create_directory_clone_source_meta(
-    const std::string &source_api_path,
-    const std::string &api_path) -> api_error {
+    const std::string &source_api_path, const std::string &api_path)
+    -> api_error {
   REPERTORY_USES_FUNCTION_NAME();
 
   bool exists{};
@@ -164,8 +178,8 @@ auto base_provider::create_directory(const std::string &api_path,
   return set_item_meta(api_path, meta);
 }
 
-auto base_provider::create_file(const std::string &api_path,
-                                api_meta_map &meta) -> api_error {
+auto base_provider::create_file(const std::string &api_path, api_meta_map &meta)
+    -> api_error {
   REPERTORY_USES_FUNCTION_NAME();
 
   bool exists{};
@@ -222,8 +236,9 @@ auto base_provider::create_file(const std::string &api_path,
   return api_error::error;
 }
 
-auto base_provider::get_api_path_from_source(
-    const std::string &source_path, std::string &api_path) const -> api_error {
+auto base_provider::get_api_path_from_source(const std::string &source_path,
+                                             std::string &api_path) const
+    -> api_error {
   REPERTORY_USES_FUNCTION_NAME();
 
   if (source_path.empty()) {
@@ -236,8 +251,9 @@ auto base_provider::get_api_path_from_source(
   return db3_->get_api_path(source_path, api_path);
 }
 
-auto base_provider::get_directory_items(
-    const std::string &api_path, directory_item_list &list) const -> api_error {
+auto base_provider::get_directory_items(const std::string &api_path,
+                                        directory_item_list &list) const
+    -> api_error {
   REPERTORY_USES_FUNCTION_NAME();
 
   bool exists{};
@@ -301,9 +317,10 @@ auto base_provider::get_file_size(const std::string &api_path,
   return api_error::success;
 }
 
-auto base_provider::get_filesystem_item(
-    const std::string &api_path, bool directory,
-    filesystem_item &fsi) const -> api_error {
+auto base_provider::get_filesystem_item(const std::string &api_path,
+                                        bool directory,
+                                        filesystem_item &fsi) const
+    -> api_error {
   bool exists{};
   auto res = is_directory(api_path, exists);
   if (res != api_error::success) {
@@ -336,9 +353,10 @@ auto base_provider::get_filesystem_item(
   return api_error::success;
 }
 
-auto base_provider::get_filesystem_item_and_file(
-    const std::string &api_path, api_file &file,
-    filesystem_item &fsi) const -> api_error {
+auto base_provider::get_filesystem_item_and_file(const std::string &api_path,
+                                                 api_file &file,
+                                                 filesystem_item &fsi) const
+    -> api_error {
   auto res = get_file(api_path, file);
   if (res != api_error::success) {
     return res;
@@ -394,6 +412,45 @@ auto base_provider::get_pinned_files() const -> std::vector<std::string> {
   return db3_->get_pinned_files();
 }
 
+void base_provider::get_removed_items(std::deque<removed_item> &directories,
+                                      std::deque<removed_item> &files,
+                                      const stop_type &stop_requested) const {
+  for (auto &&api_path : db3_->get_api_path_list()) {
+    if (stop_requested) {
+      return;
+    }
+
+    api_meta_map meta{};
+    if (get_item_meta(api_path, meta) != api_error::success) {
+      continue;
+    }
+
+    if (utils::string::to_bool(meta[META_DIRECTORY])) {
+      bool exists{};
+      if (is_directory(api_path, exists) != api_error::success) {
+        continue;
+      }
+
+      if (not exists) {
+        directories.emplace_back(removed_item{api_path, true, ""});
+      }
+
+      continue;
+    }
+
+    bool exists{};
+    if (is_file(api_path, exists) != api_error::success) {
+      continue;
+    }
+
+    if (exists) {
+      continue;
+    }
+
+    files.emplace_back(removed_item{api_path, false, meta[META_SOURCE]});
+  }
+}
+
 auto base_provider::get_total_item_count() const -> std::uint64_t {
   return db3_->get_total_item_count();
 }
@@ -413,108 +470,29 @@ auto base_provider::is_file_writeable(const std::string &api_path) const
   return not exists;
 }
 
-void base_provider::remove_deleted_files(const stop_type &stop_requested) {
-  REPERTORY_USES_FUNCTION_NAME();
-
-  if (not is_read_only()) {
-    auto source_list =
-        utils::file::directory{config_.get_cache_directory()}.get_files();
-    for (auto &&source_file : source_list) {
-      if (stop_requested) {
-        return;
-      }
-
-      filesystem_item fsi{};
-      if (get_filesystem_item_from_source_path(source_file->get_path(), fsi) !=
-          api_error::item_not_found) {
-        continue;
-      }
-
-      auto reference_time =
-          source_file->get_time(config_.get_eviction_uses_accessed_time()
-                                    ? utils::file::time_type::accessed
-                                    : utils::file::time_type::modified);
-      if (not reference_time.has_value()) {
-        continue;
-      }
-
-      auto delay = (config_.get_eviction_delay_mins() * 60UL) *
-                   utils::time::NANOS_PER_SECOND;
-      if ((reference_time.value() + static_cast<std::uint64_t>(delay)) >=
-          utils::time::get_time_now()) {
-        continue;
-      }
-
-      event_system::instance().raise<orphaned_source_file_detected>(
-          source_file->get_path());
-      if (not source_file->remove()) {
-        continue;
-      }
-
-      event_system::instance().raise<orphaned_source_file_removed>(
-          source_file->get_path());
-    }
-  }
-
-  struct removed_item final {
-    std::string api_path;
-    bool directory{};
-    std::string source_path;
-  };
-
-  {
-    api_file_list list{};
-    std::string marker;
-    auto res{api_error::more_data};
-    while (not stop_requested && res == api_error::more_data) {
-      res = get_file_list(list, marker);
-      if (res != api_error::success && res != api_error::more_data) {
-        utils::error::raise_error(function_name, res,
-                                  "failed to get file list");
-        return;
-      }
-    }
-  }
-
-  std::vector<removed_item> removed_list{};
-  for (auto &&api_path : db3_->get_api_path_list()) {
+void base_provider::process_removed_directories(
+    std::deque<removed_item> &removed_list, const stop_type &stop_requested) {
+  for (auto &&item : removed_list) {
     if (stop_requested) {
       return;
     }
 
-    api_meta_map meta{};
-    if (get_item_meta(api_path, meta) != api_error::success) {
+    if (not item.directory) {
       continue;
     }
 
-    if (utils::string::to_bool(meta[META_DIRECTORY])) {
-      bool exists{};
-      if (is_directory(api_path, exists) != api_error::success) {
-        continue;
-      }
-
-      if (not exists) {
-        removed_list.emplace_back(removed_item{api_path, true, ""});
-      }
-
-      continue;
-    }
-
-    bool exists{};
-    if (is_file(api_path, exists) != api_error::success) {
-      continue;
-    }
-
-    if (exists) {
-      continue;
-    }
-
-    removed_list.emplace_back(removed_item{api_path, false, meta[META_SOURCE]});
+    db3_->remove_api_path(item.api_path);
+    event_system::instance().raise<directory_removed_externally>(
+        item.api_path, item.source_path);
   }
+}
 
-  const auto orphaned_directory =
+void base_provider::process_removed_files(
+    std::deque<removed_item> &removed_list, const stop_type &stop_requested) {
+  REPERTORY_USES_FUNCTION_NAME();
+
+  auto orphaned_directory =
       utils::path::combine(config_.get_data_directory(), {"orphaned"});
-
   for (auto &&item : removed_list) {
     if (stop_requested) {
       return;
@@ -535,8 +513,8 @@ void base_provider::remove_deleted_files(const stop_type &stop_requested) {
       continue;
     }
 
-    const auto parts = utils::string::split(item.api_path, '/', false);
-    const auto orphaned_file = utils::path::combine(
+    auto parts = utils::string::split(item.api_path, '/', false);
+    auto orphaned_file = utils::path::combine(
         orphaned_directory, {utils::path::strip_to_file_name(item.source_path) +
                              '_' + parts[parts.size() - 1U]});
 
@@ -567,20 +545,32 @@ void base_provider::remove_deleted_files(const stop_type &stop_requested) {
     event_system::instance().raise<file_removed_externally>(item.api_path,
                                                             item.source_path);
   }
+}
 
-  for (auto &&item : removed_list) {
-    if (stop_requested) {
-      return;
-    }
-
-    if (not item.directory) {
-      continue;
-    }
-
-    db3_->remove_api_path(item.api_path);
-    event_system::instance().raise<directory_removed_externally>(
-        item.api_path, item.source_path);
+void base_provider::remove_deleted_items(const stop_type &stop_requested) {
+  add_all_items(stop_requested);
+  if (stop_requested) {
+    return;
   }
+
+  remove_unmatched_source_files(stop_requested);
+  if (stop_requested) {
+    return;
+  }
+
+  std::deque<removed_item> directories;
+  std::deque<removed_item> files;
+  get_removed_items(directories, files, stop_requested);
+  if (stop_requested) {
+    return;
+  }
+
+  process_removed_files(files, stop_requested);
+  if (stop_requested) {
+    return;
+  }
+
+  process_removed_directories(directories, stop_requested);
 }
 
 auto base_provider::remove_file(const std::string &api_path) -> api_error {
@@ -665,6 +655,51 @@ auto base_provider::remove_item_meta(const std::string &api_path,
   return db3_->remove_item_meta(api_path, key);
 }
 
+void base_provider::remove_unmatched_source_files(
+    const stop_type &stop_requested) {
+  if (is_read_only()) {
+    return;
+  }
+
+  auto source_list =
+      utils::file::directory{config_.get_cache_directory()}.get_files();
+  for (auto &&source_file : source_list) {
+    if (stop_requested) {
+      return;
+    }
+
+    filesystem_item fsi{};
+    if (get_filesystem_item_from_source_path(source_file->get_path(), fsi) !=
+        api_error::item_not_found) {
+      continue;
+    }
+
+    auto reference_time =
+        source_file->get_time(config_.get_eviction_uses_accessed_time()
+                                  ? utils::file::time_type::accessed
+                                  : utils::file::time_type::modified);
+    if (not reference_time.has_value()) {
+      continue;
+    }
+
+    auto delay = (config_.get_eviction_delay_mins() * 60UL) *
+                 utils::time::NANOS_PER_SECOND;
+    if ((reference_time.value() + static_cast<std::uint64_t>(delay)) >=
+        utils::time::get_time_now()) {
+      continue;
+    }
+
+    event_system::instance().raise<orphaned_source_file_detected>(
+        source_file->get_path());
+    if (not source_file->remove()) {
+      continue;
+    }
+
+    event_system::instance().raise<orphaned_source_file_removed>(
+        source_file->get_path());
+  }
+}
+
 auto base_provider::set_item_meta(const std::string &api_path,
                                   const std::string &key,
                                   const std::string &value) -> api_error {
@@ -715,7 +750,7 @@ auto base_provider::start(api_item_added_callback api_item_added,
   polling::instance().set_callback({
       "check_deleted",
       polling::frequency::low,
-      [this](auto &&stop_requested) { remove_deleted_files(stop_requested); },
+      [this](auto &&stop_requested) { remove_deleted_items(stop_requested); },
   });
   return true;
 }