From f7b829f71c99c98157a71b61fa3bc814721440cc Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Sat, 25 Jan 2025 05:59:44 -0600 Subject: [PATCH] fix check deleted --- .../librepertory/src/db/impl/rdb_file_db.cpp | 23 +++++--- .../librepertory/src/db/impl/rdb_meta_db.cpp | 11 ++-- .../src/db/impl/sqlite_file_db.cpp | 34 ++++++----- .../src/db/impl/sqlite_meta_db.cpp | 4 ++ .../src/providers/base_provider.cpp | 56 ++++++++++++++++++- repertory/librepertory/src/utils/tasks.cpp | 2 +- 6 files changed, 96 insertions(+), 34 deletions(-) diff --git a/repertory/librepertory/src/db/impl/rdb_file_db.cpp b/repertory/librepertory/src/db/impl/rdb_file_db.cpp index 4765f781..255f60f6 100644 --- a/repertory/librepertory/src/db/impl/rdb_file_db.cpp +++ b/repertory/librepertory/src/db/impl/rdb_file_db.cpp @@ -171,6 +171,10 @@ void rdb_file_db::enumerate_item_list( callback(list); list.clear(); } + + if (not list.empty()) { + callback(list); + } } { @@ -190,6 +194,10 @@ void rdb_file_db::enumerate_item_list( callback(list); list.clear(); } + + if (not list.empty()) { + callback(list); + } } } @@ -203,9 +211,8 @@ auto rdb_file_db::get_api_path(const std::string &source_path, }); } -auto rdb_file_db::get_directory_api_path(const std::string &source_path, - std::string &api_path) const - -> api_error { +auto rdb_file_db::get_directory_api_path( + const std::string &source_path, std::string &api_path) const -> api_error { REPERTORY_USES_FUNCTION_NAME(); auto result = perform_action(function_name, [&]() -> rocksdb::Status { @@ -228,9 +235,8 @@ auto rdb_file_db::get_directory_api_path(const std::string &source_path, : result; } -auto rdb_file_db::get_directory_source_path(const std::string &api_path, - std::string &source_path) const - -> api_error { +auto rdb_file_db::get_directory_source_path( + const std::string &api_path, std::string &source_path) const -> api_error { REPERTORY_USES_FUNCTION_NAME(); auto result = perform_action(function_name, [&]() -> rocksdb::Status { @@ -291,9 +297,8 @@ auto rdb_file_db::get_file_data(const std::string &api_path, return result; } -auto rdb_file_db::get_file_source_path(const std::string &api_path, - std::string &source_path) const - -> api_error { +auto rdb_file_db::get_file_source_path( + const std::string &api_path, std::string &source_path) const -> api_error { REPERTORY_USES_FUNCTION_NAME(); auto result = perform_action(function_name, [&]() -> rocksdb::Status { diff --git a/repertory/librepertory/src/db/impl/rdb_meta_db.cpp b/repertory/librepertory/src/db/impl/rdb_meta_db.cpp index 4598f361..538f3d90 100644 --- a/repertory/librepertory/src/db/impl/rdb_meta_db.cpp +++ b/repertory/librepertory/src/db/impl/rdb_meta_db.cpp @@ -81,6 +81,10 @@ void rdb_meta_db::enumerate_api_path_list( callback(list); list.clear(); } + + if (not list.empty()) { + callback(list); + } } auto rdb_meta_db::get_api_path(const std::string &source_path, @@ -314,10 +318,9 @@ void rdb_meta_db::remove_api_path(const std::string &api_path) { } } -auto rdb_meta_db::remove_api_path(const std::string &api_path, - const std::string &source_path, - rocksdb::Transaction *txn) - -> rocksdb::Status { +auto rdb_meta_db::remove_api_path( + const std::string &api_path, const std::string &source_path, + rocksdb::Transaction *txn) -> rocksdb::Status { auto txn_res = txn->Delete(pinned_family_, api_path); if (not txn_res.ok()) { return txn_res; diff --git a/repertory/librepertory/src/db/impl/sqlite_file_db.cpp b/repertory/librepertory/src/db/impl/sqlite_file_db.cpp index b4baecac..c7cdd024 100644 --- a/repertory/librepertory/src/db/impl/sqlite_file_db.cpp +++ b/repertory/librepertory/src/db/impl/sqlite_file_db.cpp @@ -65,9 +65,8 @@ sqlite_file_db::sqlite_file_db(const app_config &cfg) { sqlite_file_db::~sqlite_file_db() { db_.reset(); } -auto sqlite_file_db::add_directory(const std::string &api_path, - const std::string &source_path) - -> api_error { +auto sqlite_file_db::add_directory( + const std::string &api_path, const std::string &source_path) -> api_error { REPERTORY_USES_FUNCTION_NAME(); auto result = utils::db::sqlite::db_insert{*db_, file_table} @@ -158,6 +157,10 @@ void sqlite_file_db::enumerate_item_list( list.clear(); } } + + if (not list.empty()) { + callback(list); + } } auto sqlite_file_db::get_api_path(const std::string &source_path, @@ -179,9 +182,8 @@ auto sqlite_file_db::get_api_path(const std::string &source_path, return api_error::item_not_found; } -auto sqlite_file_db::get_directory_api_path(const std::string &source_path, - std::string &api_path) const - -> api_error { +auto sqlite_file_db::get_directory_api_path( + const std::string &source_path, std::string &api_path) const -> api_error { auto result = utils::db::sqlite::db_select{*db_, file_table} .column("api_path") .where("source_path") @@ -202,9 +204,8 @@ auto sqlite_file_db::get_directory_api_path(const std::string &source_path, return api_error::directory_not_found; } -auto sqlite_file_db::get_directory_source_path(const std::string &api_path, - std::string &source_path) const - -> api_error { +auto sqlite_file_db::get_directory_source_path( + const std::string &api_path, std::string &source_path) const -> api_error { auto result = utils::db::sqlite::db_select{*db_, file_table} .column("source_path") .where("api_path") @@ -225,9 +226,8 @@ auto sqlite_file_db::get_directory_source_path(const std::string &api_path, return api_error::directory_not_found; } -auto sqlite_file_db::get_file_api_path(const std::string &source_path, - std::string &api_path) const - -> api_error { +auto sqlite_file_db::get_file_api_path( + const std::string &source_path, std::string &api_path) const -> api_error { auto result = utils::db::sqlite::db_select{*db_, file_table} .column("api_path") .where("source_path") @@ -286,9 +286,8 @@ auto sqlite_file_db::get_file_data(const std::string &api_path, return api_error::item_not_found; } -auto sqlite_file_db::get_file_source_path(const std::string &api_path, - std::string &source_path) const - -> api_error { +auto sqlite_file_db::get_file_source_path( + const std::string &api_path, std::string &source_path) const -> api_error { auto result = utils::db::sqlite::db_select{*db_, file_table} .column("source_path") .where("api_path") @@ -330,9 +329,8 @@ auto sqlite_file_db::get_item_list(stop_type_callback stop_requested_cb) const return ret; } -auto sqlite_file_db::get_source_path(const std::string &api_path, - std::string &source_path) const - -> api_error { +auto sqlite_file_db::get_source_path( + const std::string &api_path, std::string &source_path) const -> api_error { auto result = utils::db::sqlite::db_select{*db_, file_table} .column("source_path") .where("api_path") diff --git a/repertory/librepertory/src/db/impl/sqlite_meta_db.cpp b/repertory/librepertory/src/db/impl/sqlite_meta_db.cpp index d1ef1918..3bf829a7 100644 --- a/repertory/librepertory/src/db/impl/sqlite_meta_db.cpp +++ b/repertory/librepertory/src/db/impl/sqlite_meta_db.cpp @@ -95,6 +95,10 @@ void sqlite_meta_db::enumerate_api_path_list( list.clear(); } } + + if (not list.empty()) { + callback(list); + } } auto sqlite_meta_db::get_api_path(const std::string &source_path, diff --git a/repertory/librepertory/src/providers/base_provider.cpp b/repertory/librepertory/src/providers/base_provider.cpp index b80f2556..12588961 100644 --- a/repertory/librepertory/src/providers/base_provider.cpp +++ b/repertory/librepertory/src/providers/base_provider.cpp @@ -24,6 +24,7 @@ #include "app_config.hpp" #include "db/meta_db.hpp" #include "events/event_system.hpp" +#include "events/types/debug_log.hpp" #include "events/types/directory_remove_failed.hpp" #include "events/types/directory_removed.hpp" #include "events/types/directory_removed_externally.hpp" @@ -41,13 +42,13 @@ #include "file_manager/cache_size_mgr.hpp" #include "file_manager/i_file_manager.hpp" #include "platform/platform.hpp" +#include "utils/config.hpp" #include "utils/error_utils.hpp" #include "utils/file_utils.hpp" #include "utils/path.hpp" #include "utils/polling.hpp" #include "utils/tasks.hpp" #include "utils/time.hpp" -#include namespace repertory { void base_provider::add_all_items(stop_type &stop_requested) { @@ -57,6 +58,8 @@ void base_provider::add_all_items(stop_type &stop_requested) { REPERTORY_USES_FUNCTION_NAME(); + event_system::instance().raise(function_name, "get list|begin"); + api_file_list list{}; std::string marker; auto res{api_error::more_data}; @@ -66,6 +69,9 @@ void base_provider::add_all_items(stop_type &stop_requested) { utils::error::raise_error(function_name, res, "failed to get file list"); } } + + event_system::instance().raise( + function_name, fmt::format("get list|end|{}", list.size())); } auto base_provider::create_api_file(std::string path, std::string key, @@ -549,12 +555,20 @@ void base_provider::process_removed_files(std::deque removed_list, } void base_provider::process_removed_items(stop_type &stop_requested) { + REPERTORY_USES_FUNCTION_NAME(); + const auto get_stop_requested = [&stop_requested]() -> bool { return stop_requested || app_config::get_stop_requested(); }; + event_system::instance().raise(function_name, + "checking removed items"); db3_->enumerate_api_path_list( [this, &get_stop_requested](auto &&list) { + event_system::instance().raise( + function_name, + fmt::format("enumerating removed items|{}", list.size())); + [[maybe_unused]] auto res = std::all_of(list.begin(), list.end(), [&](auto &&api_path) -> bool { if (get_stop_requested()) { @@ -563,22 +577,44 @@ void base_provider::process_removed_items(stop_type &stop_requested) { tasks::instance().schedule({ [this, api_path](auto &&task_stopped) { + event_system::instance().raise( + function_name, + fmt::format("checking item|{}", api_path)); + api_meta_map meta{}; - if (get_item_meta(api_path, meta) != api_error::success) { + auto result = get_item_meta(api_path, meta); + if (result != api_error::success) { + event_system::instance().raise( + function_name, + fmt::format("meta not found|{}|error|{}", api_path, + api_error_to_string(result))); return; } if (utils::string::to_bool(meta[META_DIRECTORY])) { + event_system::instance().raise( + function_name, + fmt::format("checking directory|{}", api_path)); + bool exists{}; if (is_directory(api_path, exists) != api_error::success) { return; } + event_system::instance().raise( + function_name, + fmt::format("checking directory|{}|exists|{}", + api_path, exists)); if (exists) { return; } + event_system::instance().raise( + function_name, + fmt::format( + "processing removed directory|{}|exists|{}", + api_path, exists)); process_removed_directories( { removed_item{api_path, true, ""}, @@ -588,15 +624,26 @@ void base_provider::process_removed_items(stop_type &stop_requested) { return; } + event_system::instance().raise( + function_name, + fmt::format("checking file|{}", api_path)); + bool exists{}; if (is_file(api_path, exists) != api_error::success) { return; } + event_system::instance().raise( + function_name, fmt::format("checking file|{}|exists|{}", + api_path, exists)); if (exists) { return; } + event_system::instance().raise( + function_name, + fmt::format("processing removed file|{}|exists|{}", + api_path, exists)); process_removed_files( { removed_item{api_path, false, meta[META_SOURCE]}, @@ -612,20 +659,25 @@ void base_provider::process_removed_items(stop_type &stop_requested) { } void base_provider::remove_deleted_items(stop_type &stop_requested) { + REPERTORY_USES_FUNCTION_NAME(); + const auto get_stop_requested = [&stop_requested]() -> bool { return stop_requested || app_config::get_stop_requested(); }; + event_system::instance().raise(function_name, "add all"); add_all_items(stop_requested); if (get_stop_requested()) { return; } + event_system::instance().raise(function_name, "remove unmatched"); remove_unmatched_source_files(stop_requested); if (get_stop_requested()) { return; } + event_system::instance().raise(function_name, "remove items"); process_removed_items(stop_requested); } diff --git a/repertory/librepertory/src/utils/tasks.cpp b/repertory/librepertory/src/utils/tasks.cpp index 2926ef41..f2184bc6 100644 --- a/repertory/librepertory/src/utils/tasks.cpp +++ b/repertory/librepertory/src/utils/tasks.cpp @@ -84,7 +84,7 @@ void tasks::start(app_config *config) { stop_requested_ = false; tasks_.clear(); - for (std::uint32_t idx = 0U; idx < std::thread::hardware_concurrency(); + for (std::uint32_t idx = 0U; idx < std::thread::hardware_concurrency() * 2U; ++idx) { task_threads_.emplace_back( std::make_unique([this]() { task_thread(); }));