refactor cleanup

repertory/librepertory/include/db/i_meta_db.hpp

@@ -31,6 +31,10 @@ class i_meta_db {
 public:
   virtual void clear() = 0;
 
+  virtual void enumerate_api_path_list(
+      std::function<void(const std::vector<std::string> &)> callback,
+      const stop_type &stop_requested) const = 0;
+
   [[nodiscard]] virtual auto get_api_path(const std::string &source_path,
                                           std::string &api_path) const
       -> api_error = 0;

repertory/librepertory/include/db/impl/rdb_meta_db.hpp

@@ -80,6 +80,10 @@ private:
 public:
   void clear() override;
 
+  void enumerate_api_path_list(
+      std::function<void(const std::vector<std::string> &)> callback,
+      const stop_type &stop_requested) const override;
+
   [[nodiscard]] auto get_api_path(const std::string &source_path,
                                   std::string &api_path) const
       -> api_error override;

repertory/librepertory/include/db/impl/sqlite_meta_db.hpp

@@ -50,6 +50,10 @@ private:
 public:
   void clear() override;
 
+  void enumerate_api_path_list(
+      std::function<void(const std::vector<std::string> &)> callback,
+      const stop_type &stop_requested) const override;
+
   [[nodiscard]] auto get_api_path(const std::string &source_path,
                                   std::string &api_path) const
       -> api_error override;

repertory/librepertory/src/db/impl/rdb_meta_db.cpp

@@ -58,6 +58,24 @@ void rdb_meta_db::create_or_open(bool clear) {
 
 void rdb_meta_db::clear() { create_or_open(true); }
 
+void rdb_meta_db::enumerate_api_path_list(
+    std::function<void(const std::vector<std::string> &)> callback,
+    const stop_type &stop_requested) const {
+  std::vector<std::string> list{};
+
+  auto iter = create_iterator(meta_family_);
+  for (iter->SeekToFirst(); not stop_requested && iter->Valid(); iter->Next()) {
+    list.push_back(iter->key().ToString());
+
+    if (list.size() < 100U) {
+      continue;
+    }
+
+    callback(list);
+    list.clear();
+  }
+}
+
 auto rdb_meta_db::create_iterator(rocksdb::ColumnFamilyHandle *family) const
     -> std::shared_ptr<rocksdb::Iterator> {
   return std::shared_ptr<rocksdb::Iterator>(

repertory/librepertory/src/db/impl/sqlite_meta_db.cpp

@@ -75,6 +75,28 @@ void sqlite_meta_db::clear() {
                                 std::to_string(result.get_error()));
 }
 
+void sqlite_meta_db::enumerate_api_path_list(
+    bool directories,
+    std::function<void(const std::vector<std::string> &)> callback,
+    const stop_type &stop_requested) const {
+  auto result =
+      utils::db::sqlite::db_select{*db_, table_name}.column("api_path").go();
+
+  std::vector<std::string> list{};
+  while (result.has_row() && not stop_requested) {
+    std::optional<utils::db::sqlite::db_result::row> row;
+    if (result.get_row(row) && row.has_value()) {
+      list.push_back(row->get_column("api_path").get_value<std::string>());
+      if (list.size() < 100U) {
+        continue;
+      }
+
+      callback(list);
+      list.clear();
+    }
+  }
+}
+
 auto sqlite_meta_db::get_api_path(const std::string &source_path,
                                   std::string &api_path) const -> api_error {
   auto result = utils::db::sqlite::db_select{*db_, table_name}

repertory/librepertory/src/providers/base_provider.cpp

@@ -514,7 +514,8 @@ void base_provider::process_removed_files(std::deque<removed_item> removed_list,
 }
 
 void base_provider::process_removed_items(const stop_type &stop_requested) {
-  auto list = db3_->get_api_path_list();
+  db3_->enumerate_api_path_list(
+      [this, &stop_requested](auto &&list) {
         [[maybe_unused]] auto res =
             std::all_of(list.begin(), list.end(), [&](auto &&api_path) -> bool {
               if (stop_requested) {
@@ -529,25 +530,24 @@ void base_provider::process_removed_items(const stop_type &stop_requested) {
                     }
 
                     if (utils::string::to_bool(meta[META_DIRECTORY])) {
+                      bool exists{};
+                      if (is_directory(api_path, exists) !=
+                          api_error::success) {
+                        return;
+                      }
+
+                      if (exists) {
+                        return;
+                      }
+
+                      process_removed_directories(
+                          {
+                              removed_item{api_path, true, ""},
+                          },
+                          stop_requested2);
+
                       return;
                     }
-              //   bool exists{};
-              //   if (is_directory(api_path, exists) != api_error::success) {
-              //     return;
-              //   }
-              //
-              //   if (exists) {
-              //     return;
-              //   }
-              //
-              //   // process_removed_directories(
-              //   //     {
-              //   //         removed_item{api_path, true, ""},
-              //   //     },
-              //   //     stop_requested2);
-              //
-              //   return;
-              // }
 
                     bool exists{};
                     if (is_file(api_path, exists) != api_error::success) {
@@ -568,6 +568,8 @@ void base_provider::process_removed_items(const stop_type &stop_requested) {
 
               return not stop_requested;
             });
+      },
+      stop_requested);
 }
 
 void base_provider::remove_deleted_items(const stop_type &stop_requested) {