refactor cleanup

repertory/librepertory/include/db/i_file_db.hpp

@@ -56,6 +56,10 @@ public:
 
   [[nodiscard]] virtual auto count() const -> std::uint64_t = 0;
 
+  virtual void enumerate_api_path_list(
+      std::function<void(const std::vector<i_file_db::file_info> &)> callback,
+      const stop_type &stop_requested) const = 0;
+
   [[nodiscard]] virtual auto get_api_path(const std::string &source_path,
                                           std::string &api_path) const
       -> api_error = 0;

repertory/librepertory/include/db/impl/rdb_file_db.hpp

@@ -78,6 +78,10 @@ public:
 
   [[nodiscard]] auto count() const -> std::uint64_t override;
 
+  void enumerate_api_path_list(
+      std::function<void(const std::vector<i_file_db::file_info> &)> callback,
+      const stop_type &stop_requested) const override;
+
   [[nodiscard]] auto get_api_path(const std::string &source_path,
                                   std::string &api_path) const
       -> api_error override;

repertory/librepertory/include/db/impl/sqlite_file_db.hpp

@@ -53,6 +53,10 @@ public:
 
   [[nodiscard]] auto count() const -> std::uint64_t override;
 
+  void enumerate_api_path_list(
+      std::function<void(const std::vector<i_file_db::file_info> &)> callback,
+      const stop_type &stop_requested) const override;
+
   [[nodiscard]] auto get_api_path(const std::string &source_path,
                                   std::string &api_path) const
       -> api_error override;

repertory/librepertory/src/db/impl/rdb_file_db.cpp

@@ -149,6 +149,54 @@ auto rdb_file_db::count() const -> std::uint64_t {
   return ret;
 }
 
+void rdb_file_db::enumerate_api_path_list(
+    std::function<void(const std::vector<i_file_db::file_info> &)> callback,
+    const stop_type &stop_requested) const {
+  std::vector<i_file_db::file_info> list;
+  {
+    auto iter = create_iterator(file_family_);
+    for (iter->SeekToFirst(); not stop_requested && iter->Valid();
+         iter->Next()) {
+      if (stop_requested) {
+        break;
+      }
+
+      auto json_data = json::parse(iter->value().ToString());
+      list.emplace_back(i_file_db::file_info{
+          iter->key().ToString(),
+          false,
+          json_data.at("source_path").get<std::string>(),
+      });
+
+      if (list.size() < 100U) {
+        continue;
+      }
+
+      callback(list);
+      list.clear();
+    }
+  }
+
+  {
+    auto iter = create_iterator(directory_family_);
+    for (iter->SeekToFirst(); not stop_requested && iter->Valid();
+         iter->Next()) {
+      list.emplace_back(i_file_db::file_info{
+          iter->key().ToString(),
+          true,
+          iter->value().ToString(),
+      });
+
+      if (list.size() < 100U) {
+        continue;
+      }
+
+      callback(list);
+      list.clear();
+    }
+  }
+}
+
 auto rdb_file_db::get_api_path(const std::string &source_path,
                                std::string &api_path) const -> api_error {
   REPERTORY_USES_FUNCTION_NAME();

repertory/librepertory/src/db/impl/rdb_meta_db.cpp

@@ -58,6 +58,12 @@ void rdb_meta_db::create_or_open(bool clear) {
 
 void rdb_meta_db::clear() { create_or_open(true); }
 
+auto rdb_meta_db::create_iterator(rocksdb::ColumnFamilyHandle *family) const
+    -> std::shared_ptr<rocksdb::Iterator> {
+  return std::shared_ptr<rocksdb::Iterator>(
+      db_->NewIterator(rocksdb::ReadOptions{}, family));
+}
+
 void rdb_meta_db::enumerate_api_path_list(
     std::function<void(const std::vector<std::string> &)> callback,
     const stop_type &stop_requested) const {
@@ -76,12 +82,6 @@ void rdb_meta_db::enumerate_api_path_list(
   }
 }
 
-auto rdb_meta_db::create_iterator(rocksdb::ColumnFamilyHandle *family) const
-    -> std::shared_ptr<rocksdb::Iterator> {
-  return std::shared_ptr<rocksdb::Iterator>(
-      db_->NewIterator(rocksdb::ReadOptions{}, family));
-}
-
 auto rdb_meta_db::get_api_path(const std::string &source_path,
                                std::string &api_path) const -> api_error {
   REPERTORY_USES_FUNCTION_NAME();

repertory/librepertory/src/db/impl/sqlite_file_db.cpp

@@ -133,6 +133,33 @@ auto sqlite_file_db::count() const -> std::uint64_t {
   return 0U;
 }
 
+void sqlite_file_db::enumerate_api_path_list(
+    std::function<void(const std::vector<i_file_db::file_info> &)> callback,
+    const stop_type &stop_requested) const {
+  std::vector<i_file_db::file_info> list;
+
+  auto result = utils::db::sqlite::db_select{*db_, file_table}.go();
+  while (result.has_row() && not stop_requested) {
+    std::optional<utils::db::sqlite::db_result::row> row;
+    if (result.get_row(row) && row.has_value()) {
+      list.emplace_back(i_file_db::file_info{
+          .api_path = row->get_column("api_path").get_value<std::string>(),
+          .directory =
+              row->get_column("directory").get_value<std::int64_t>() == 1,
+          .source_path =
+              row->get_column("source_path").get_value<std::string>(),
+      });
+
+      if (list.size() < 100U) {
+        continue;
+      }
+
+      callback(list);
+      list.clear();
+    }
+  }
+}
+
 auto sqlite_file_db::get_api_path(const std::string &source_path,
                                   std::string &api_path) const -> api_error {
   auto result = utils::db::sqlite::db_select{*db_, file_table}
@@ -287,11 +314,7 @@ auto sqlite_file_db::get_item_list(const stop_type &stop_requested) const
   std::vector<i_file_db::file_info> ret;
 
   auto result = utils::db::sqlite::db_select{*db_, file_table}.go();
-  while (result.has_row()) {
+  while (result.has_row() && not stop_requested) {
-    if (stop_requested) {
-      break;
-    }
-
     std::optional<utils::db::sqlite::db_result::row> row;
     if (result.get_row(row) && row.has_value()) {
       ret.emplace_back(i_file_db::file_info{
@@ -302,8 +325,6 @@ auto sqlite_file_db::get_item_list(const stop_type &stop_requested) const
               row->get_column("source_path").get_value<std::string>(),
       });
     }
-
-    result.next_row();
   }
 
   return ret;

repertory/librepertory/src/db/impl/sqlite_meta_db.cpp

@@ -76,7 +76,6 @@ void sqlite_meta_db::clear() {
 }
 
 void sqlite_meta_db::enumerate_api_path_list(
-    bool directories,
     std::function<void(const std::vector<std::string> &)> callback,
     const stop_type &stop_requested) const {
   auto result =
@@ -87,6 +86,7 @@ void sqlite_meta_db::enumerate_api_path_list(
     std::optional<utils::db::sqlite::db_result::row> row;
     if (result.get_row(row) && row.has_value()) {
       list.push_back(row->get_column("api_path").get_value<std::string>());
+
       if (list.size() < 100U) {
         continue;
       }

repertory/librepertory/src/providers/encrypt/encrypt_provider.cpp

@@ -806,34 +806,37 @@ auto encrypt_provider::read_file_bytes(const std::string &api_path,
 }
 
 void encrypt_provider::remove_deleted_files(const stop_type &stop_requested) {
-  std::vector<i_file_db::file_info> removed_list{};
+  db_->enumerate_api_path_list(
+      [this, &stop_requested](auto &&list) {
+        std::vector<i_file_db::file_info> removed_list{};
+        for (const auto &item : list) {
+          if (stop_requested) {
+            return;
+          }
 
-  for (const auto &item : db_->get_item_list(stop_requested)) {
+          if (not utils::path::exists(item.source_path)) {
-    if (stop_requested) {
+            removed_list.emplace_back(item);
-      return;
+          }
-    }
+        }
 
-    if (not utils::path::exists(item.source_path)) {
+        for (const auto &item : removed_list) {
-      removed_list.emplace_back(item);
+          if (stop_requested) {
-    }
+            return;
-  }
+          }
 
-  for (const auto &item : removed_list) {
+          // TODO handle error
-    if (stop_requested) {
+          auto del_res{db_->remove_item(item.api_path)};
-      return;
+          if (item.directory) {
-    }
+            event_system::instance().raise<directory_removed_externally>(
+                item.api_path, item.source_path);
+            continue;
+          }
 
-    // TODO handle error
+          event_system::instance().raise<file_removed_externally>(
-    auto del_res{db_->remove_item(item.api_path)};
+              item.api_path, item.source_path);
-    if (item.directory) {
+        }
-      event_system::instance().raise<directory_removed_externally>(
+      },
-          item.api_path, item.source_path);
+      stop_requested);
-      continue;
-    }
-
-    event_system::instance().raise<file_removed_externally>(item.api_path,
-                                                            item.source_path);
-  }
 }
 
 auto encrypt_provider::start(api_item_added_callback /*api_item_added*/,