updated CHANGELOG.md

CHANGELOG.md

@@ -16,6 +16,7 @@
 ### Changes from v2.0.2-rc
 
 * Always use direct for read-only providers
+* Fixed externally removed files not being processed during cleanup
 * Fixed http headers not being added for requests
 * Fixed incorrect `stat` values for remote mounts
 * Fixed invalid directory nullptr error on remote mounts

repertory/librepertory/src/db/impl/rdb_file_db.cpp

@@ -171,6 +171,10 @@ void rdb_file_db::enumerate_item_list(
       callback(list);
       list.clear();
     }
+
+    if (not list.empty()) {
+      callback(list);
+    }
   }
 
   {
@@ -190,6 +194,10 @@ void rdb_file_db::enumerate_item_list(
       callback(list);
       list.clear();
     }
+
+    if (not list.empty()) {
+      callback(list);
+    }
   }
 }
 
@@ -203,9 +211,8 @@ auto rdb_file_db::get_api_path(const std::string &source_path,
   });
 }
 
-auto rdb_file_db::get_directory_api_path(const std::string &source_path,
-                                         std::string &api_path) const
-    -> api_error {
+auto rdb_file_db::get_directory_api_path(
+    const std::string &source_path, std::string &api_path) const -> api_error {
   REPERTORY_USES_FUNCTION_NAME();
 
   auto result = perform_action(function_name, [&]() -> rocksdb::Status {
@@ -228,9 +235,8 @@ auto rdb_file_db::get_directory_api_path(const std::string &source_path,
                                              : result;
 }
 
-auto rdb_file_db::get_directory_source_path(const std::string &api_path,
-                                            std::string &source_path) const
-    -> api_error {
+auto rdb_file_db::get_directory_source_path(
+    const std::string &api_path, std::string &source_path) const -> api_error {
   REPERTORY_USES_FUNCTION_NAME();
 
   auto result = perform_action(function_name, [&]() -> rocksdb::Status {
@@ -291,9 +297,8 @@ auto rdb_file_db::get_file_data(const std::string &api_path,
   return result;
 }
 
-auto rdb_file_db::get_file_source_path(const std::string &api_path,
-                                       std::string &source_path) const
-    -> api_error {
+auto rdb_file_db::get_file_source_path(
+    const std::string &api_path, std::string &source_path) const -> api_error {
   REPERTORY_USES_FUNCTION_NAME();
 
   auto result = perform_action(function_name, [&]() -> rocksdb::Status {

repertory/librepertory/src/db/impl/rdb_meta_db.cpp

@@ -81,6 +81,10 @@ void rdb_meta_db::enumerate_api_path_list(
     callback(list);
     list.clear();
   }
+
+  if (not list.empty()) {
+    callback(list);
+  }
 }
 
 auto rdb_meta_db::get_api_path(const std::string &source_path,
@@ -314,10 +318,9 @@ void rdb_meta_db::remove_api_path(const std::string &api_path) {
   }
 }
 
-auto rdb_meta_db::remove_api_path(const std::string &api_path,
-                                  const std::string &source_path,
-                                  rocksdb::Transaction *txn)
-    -> rocksdb::Status {
+auto rdb_meta_db::remove_api_path(
+    const std::string &api_path, const std::string &source_path,
+    rocksdb::Transaction *txn) -> rocksdb::Status {
   auto txn_res = txn->Delete(pinned_family_, api_path);
   if (not txn_res.ok()) {
     return txn_res;

repertory/librepertory/src/db/impl/sqlite_file_db.cpp

@@ -65,9 +65,8 @@ sqlite_file_db::sqlite_file_db(const app_config &cfg) {
 
 sqlite_file_db::~sqlite_file_db() { db_.reset(); }
 
-auto sqlite_file_db::add_directory(const std::string &api_path,
-                                   const std::string &source_path)
-    -> api_error {
+auto sqlite_file_db::add_directory(
+    const std::string &api_path, const std::string &source_path) -> api_error {
   REPERTORY_USES_FUNCTION_NAME();
 
   auto result = utils::db::sqlite::db_insert{*db_, file_table}
@@ -158,6 +157,10 @@ void sqlite_file_db::enumerate_item_list(
       list.clear();
     }
   }
+
+  if (not list.empty()) {
+    callback(list);
+  }
 }
 
 auto sqlite_file_db::get_api_path(const std::string &source_path,
@@ -179,9 +182,8 @@ auto sqlite_file_db::get_api_path(const std::string &source_path,
   return api_error::item_not_found;
 }
 
-auto sqlite_file_db::get_directory_api_path(const std::string &source_path,
-                                            std::string &api_path) const
-    -> api_error {
+auto sqlite_file_db::get_directory_api_path(
+    const std::string &source_path, std::string &api_path) const -> api_error {
   auto result = utils::db::sqlite::db_select{*db_, file_table}
                     .column("api_path")
                     .where("source_path")
@@ -202,9 +204,8 @@ auto sqlite_file_db::get_directory_api_path(const std::string &source_path,
   return api_error::directory_not_found;
 }
 
-auto sqlite_file_db::get_directory_source_path(const std::string &api_path,
-                                               std::string &source_path) const
-    -> api_error {
+auto sqlite_file_db::get_directory_source_path(
+    const std::string &api_path, std::string &source_path) const -> api_error {
   auto result = utils::db::sqlite::db_select{*db_, file_table}
                     .column("source_path")
                     .where("api_path")
@@ -225,9 +226,8 @@ auto sqlite_file_db::get_directory_source_path(const std::string &api_path,
   return api_error::directory_not_found;
 }
 
-auto sqlite_file_db::get_file_api_path(const std::string &source_path,
-                                       std::string &api_path) const
-    -> api_error {
+auto sqlite_file_db::get_file_api_path(
+    const std::string &source_path, std::string &api_path) const -> api_error {
   auto result = utils::db::sqlite::db_select{*db_, file_table}
                     .column("api_path")
                     .where("source_path")
@@ -286,9 +286,8 @@ auto sqlite_file_db::get_file_data(const std::string &api_path,
   return api_error::item_not_found;
 }
 
-auto sqlite_file_db::get_file_source_path(const std::string &api_path,
-                                          std::string &source_path) const
-    -> api_error {
+auto sqlite_file_db::get_file_source_path(
+    const std::string &api_path, std::string &source_path) const -> api_error {
   auto result = utils::db::sqlite::db_select{*db_, file_table}
                     .column("source_path")
                     .where("api_path")
@@ -330,9 +329,8 @@ auto sqlite_file_db::get_item_list(stop_type_callback stop_requested_cb) const
   return ret;
 }
 
-auto sqlite_file_db::get_source_path(const std::string &api_path,
-                                     std::string &source_path) const
-    -> api_error {
+auto sqlite_file_db::get_source_path(
+    const std::string &api_path, std::string &source_path) const -> api_error {
   auto result = utils::db::sqlite::db_select{*db_, file_table}
                     .column("source_path")
                     .where("api_path")

repertory/librepertory/src/db/impl/sqlite_meta_db.cpp

@@ -95,6 +95,10 @@ void sqlite_meta_db::enumerate_api_path_list(
       list.clear();
     }
   }
+
+  if (not list.empty()) {
+    callback(list);
+  }
 }
 
 auto sqlite_meta_db::get_api_path(const std::string &source_path,

repertory/librepertory/src/providers/base_provider.cpp

@@ -41,13 +41,13 @@
 #include "file_manager/cache_size_mgr.hpp"
 #include "file_manager/i_file_manager.hpp"
 #include "platform/platform.hpp"
+#include "utils/config.hpp"
 #include "utils/error_utils.hpp"
 #include "utils/file_utils.hpp"
 #include "utils/path.hpp"
 #include "utils/polling.hpp"
 #include "utils/tasks.hpp"
 #include "utils/time.hpp"
-#include <utils/config.hpp>
 
 namespace repertory {
 void base_provider::add_all_items(stop_type &stop_requested) {
@@ -549,6 +549,8 @@ void base_provider::process_removed_files(std::deque<removed_item> removed_list,
 }
 
 void base_provider::process_removed_items(stop_type &stop_requested) {
+  REPERTORY_USES_FUNCTION_NAME();
+
   const auto get_stop_requested = [&stop_requested]() -> bool {
     return stop_requested || app_config::get_stop_requested();
   };
@@ -564,7 +566,8 @@ void base_provider::process_removed_items(stop_type &stop_requested) {
               tasks::instance().schedule({
                   [this, api_path](auto &&task_stopped) {
                     api_meta_map meta{};
-                    if (get_item_meta(api_path, meta) != api_error::success) {
+                    auto result = get_item_meta(api_path, meta);
+                    if (result != api_error::success) {
                       return;
                     }
 
@@ -612,6 +615,8 @@ void base_provider::process_removed_items(stop_type &stop_requested) {
 }
 
 void base_provider::remove_deleted_items(stop_type &stop_requested) {
+  REPERTORY_USES_FUNCTION_NAME();
+
   const auto get_stop_requested = [&stop_requested]() -> bool {
     return stop_requested || app_config::get_stop_requested();
   };

repertory/librepertory/src/utils/tasks.cpp

@@ -84,7 +84,7 @@ void tasks::start(app_config *config) {
   stop_requested_ = false;
   tasks_.clear();
 
-  for (std::uint32_t idx = 0U; idx < std::thread::hardware_concurrency();
+  for (std::uint32_t idx = 0U; idx < std::thread::hardware_concurrency() * 2U;
        ++idx) {
     task_threads_.emplace_back(
         std::make_unique<std::jthread>([this]() { task_thread(); }));