diff --git a/repertory/librepertory/include/db/i_meta_db.hpp b/repertory/librepertory/include/db/i_meta_db.hpp index f97ceb43..2f6f0da8 100644 --- a/repertory/librepertory/include/db/i_meta_db.hpp +++ b/repertory/librepertory/include/db/i_meta_db.hpp @@ -31,6 +31,10 @@ class i_meta_db { public: virtual void clear() = 0; + virtual void enumerate_api_path_list( + std::function &)> callback, + const stop_type &stop_requested) const = 0; + [[nodiscard]] virtual auto get_api_path(const std::string &source_path, std::string &api_path) const -> api_error = 0; diff --git a/repertory/librepertory/include/db/impl/rdb_meta_db.hpp b/repertory/librepertory/include/db/impl/rdb_meta_db.hpp index bd25e452..83dc7b95 100644 --- a/repertory/librepertory/include/db/impl/rdb_meta_db.hpp +++ b/repertory/librepertory/include/db/impl/rdb_meta_db.hpp @@ -80,6 +80,10 @@ private: public: void clear() override; + void enumerate_api_path_list( + std::function &)> callback, + const stop_type &stop_requested) const override; + [[nodiscard]] auto get_api_path(const std::string &source_path, std::string &api_path) const -> api_error override; diff --git a/repertory/librepertory/include/db/impl/sqlite_meta_db.hpp b/repertory/librepertory/include/db/impl/sqlite_meta_db.hpp index 92fcea65..c895d20f 100644 --- a/repertory/librepertory/include/db/impl/sqlite_meta_db.hpp +++ b/repertory/librepertory/include/db/impl/sqlite_meta_db.hpp @@ -50,6 +50,10 @@ private: public: void clear() override; + void enumerate_api_path_list( + std::function &)> callback, + const stop_type &stop_requested) const override; + [[nodiscard]] auto get_api_path(const std::string &source_path, std::string &api_path) const -> api_error override; diff --git a/repertory/librepertory/src/db/impl/rdb_meta_db.cpp b/repertory/librepertory/src/db/impl/rdb_meta_db.cpp index 2cf51119..cb129796 100644 --- a/repertory/librepertory/src/db/impl/rdb_meta_db.cpp +++ b/repertory/librepertory/src/db/impl/rdb_meta_db.cpp @@ -58,6 +58,24 @@ void rdb_meta_db::create_or_open(bool clear) { void rdb_meta_db::clear() { create_or_open(true); } +void rdb_meta_db::enumerate_api_path_list( + std::function &)> callback, + const stop_type &stop_requested) const { + std::vector list{}; + + auto iter = create_iterator(meta_family_); + for (iter->SeekToFirst(); not stop_requested && iter->Valid(); iter->Next()) { + list.push_back(iter->key().ToString()); + + if (list.size() < 100U) { + continue; + } + + callback(list); + list.clear(); + } +} + auto rdb_meta_db::create_iterator(rocksdb::ColumnFamilyHandle *family) const -> std::shared_ptr { return std::shared_ptr( diff --git a/repertory/librepertory/src/db/impl/sqlite_meta_db.cpp b/repertory/librepertory/src/db/impl/sqlite_meta_db.cpp index f07fd780..bc31f87f 100644 --- a/repertory/librepertory/src/db/impl/sqlite_meta_db.cpp +++ b/repertory/librepertory/src/db/impl/sqlite_meta_db.cpp @@ -75,6 +75,28 @@ void sqlite_meta_db::clear() { std::to_string(result.get_error())); } +void sqlite_meta_db::enumerate_api_path_list( + bool directories, + std::function &)> callback, + const stop_type &stop_requested) const { + auto result = + utils::db::sqlite::db_select{*db_, table_name}.column("api_path").go(); + + std::vector list{}; + while (result.has_row() && not stop_requested) { + std::optional row; + if (result.get_row(row) && row.has_value()) { + list.push_back(row->get_column("api_path").get_value()); + if (list.size() < 100U) { + continue; + } + + callback(list); + list.clear(); + } + } +} + auto sqlite_meta_db::get_api_path(const std::string &source_path, std::string &api_path) const -> api_error { auto result = utils::db::sqlite::db_select{*db_, table_name} diff --git a/repertory/librepertory/src/providers/base_provider.cpp b/repertory/librepertory/src/providers/base_provider.cpp index bc3d3365..8e3fe622 100644 --- a/repertory/librepertory/src/providers/base_provider.cpp +++ b/repertory/librepertory/src/providers/base_provider.cpp @@ -514,60 +514,62 @@ void base_provider::process_removed_files(std::deque removed_list, } void base_provider::process_removed_items(const stop_type &stop_requested) { - auto list = db3_->get_api_path_list(); - [[maybe_unused]] auto res = - std::all_of(list.begin(), list.end(), [&](auto &&api_path) -> bool { - if (stop_requested) { - return false; - } - - tasks::instance().schedule({ - [this, api_path](auto &&task_stopped) { - api_meta_map meta{}; - if (get_item_meta(api_path, meta) != api_error::success) { - return; + db3_->enumerate_api_path_list( + [this, &stop_requested](auto &&list) { + [[maybe_unused]] auto res = + std::all_of(list.begin(), list.end(), [&](auto &&api_path) -> bool { + if (stop_requested) { + return false; } - if (utils::string::to_bool(meta[META_DIRECTORY])) { - return; - } - // bool exists{}; - // if (is_directory(api_path, exists) != api_error::success) { - // return; - // } - // - // if (exists) { - // return; - // } - // - // // process_removed_directories( - // // { - // // removed_item{api_path, true, ""}, - // // }, - // // stop_requested2); - // - // return; - // } + tasks::instance().schedule({ + [this, api_path](auto &&task_stopped) { + api_meta_map meta{}; + if (get_item_meta(api_path, meta) != api_error::success) { + return; + } - bool exists{}; - if (is_file(api_path, exists) != api_error::success) { - return; - } + if (utils::string::to_bool(meta[META_DIRECTORY])) { + bool exists{}; + if (is_directory(api_path, exists) != + api_error::success) { + return; + } - if (exists) { - return; - } + if (exists) { + return; + } - process_removed_files( - { - removed_item{api_path, false, meta[META_SOURCE]}, + process_removed_directories( + { + removed_item{api_path, true, ""}, + }, + stop_requested2); + + return; + } + + bool exists{}; + if (is_file(api_path, exists) != api_error::success) { + return; + } + + if (exists) { + return; + } + + process_removed_files( + { + removed_item{api_path, false, meta[META_SOURCE]}, + }, + task_stopped); }, - task_stopped); - }, - }); + }); - return not stop_requested; - }); + return not stop_requested; + }); + }, + stop_requested); } void base_provider::remove_deleted_items(const stop_type &stop_requested) {