diff --git a/repertory/librepertory/include/db/i_file_db.hpp b/repertory/librepertory/include/db/i_file_db.hpp index 33d0443c..e1779ce9 100644 --- a/repertory/librepertory/include/db/i_file_db.hpp +++ b/repertory/librepertory/include/db/i_file_db.hpp @@ -56,6 +56,10 @@ public: [[nodiscard]] virtual auto count() const -> std::uint64_t = 0; + virtual void enumerate_api_path_list( + std::function &)> callback, + const stop_type &stop_requested) const = 0; + [[nodiscard]] virtual auto get_api_path(const std::string &source_path, std::string &api_path) const -> api_error = 0; diff --git a/repertory/librepertory/include/db/impl/rdb_file_db.hpp b/repertory/librepertory/include/db/impl/rdb_file_db.hpp index e42e8977..e9955db3 100644 --- a/repertory/librepertory/include/db/impl/rdb_file_db.hpp +++ b/repertory/librepertory/include/db/impl/rdb_file_db.hpp @@ -78,6 +78,10 @@ public: [[nodiscard]] auto count() const -> std::uint64_t override; + void enumerate_api_path_list( + std::function &)> callback, + const stop_type &stop_requested) const override; + [[nodiscard]] auto get_api_path(const std::string &source_path, std::string &api_path) const -> api_error override; diff --git a/repertory/librepertory/include/db/impl/sqlite_file_db.hpp b/repertory/librepertory/include/db/impl/sqlite_file_db.hpp index ecfd057f..8e6d2c30 100644 --- a/repertory/librepertory/include/db/impl/sqlite_file_db.hpp +++ b/repertory/librepertory/include/db/impl/sqlite_file_db.hpp @@ -53,6 +53,10 @@ public: [[nodiscard]] auto count() const -> std::uint64_t override; + void enumerate_api_path_list( + std::function &)> callback, + const stop_type &stop_requested) const override; + [[nodiscard]] auto get_api_path(const std::string &source_path, std::string &api_path) const -> api_error override; diff --git a/repertory/librepertory/src/db/impl/rdb_file_db.cpp b/repertory/librepertory/src/db/impl/rdb_file_db.cpp index 59f1f717..c521386a 100644 --- a/repertory/librepertory/src/db/impl/rdb_file_db.cpp +++ b/repertory/librepertory/src/db/impl/rdb_file_db.cpp @@ -149,6 +149,54 @@ auto rdb_file_db::count() const -> std::uint64_t { return ret; } +void rdb_file_db::enumerate_api_path_list( + std::function &)> callback, + const stop_type &stop_requested) const { + std::vector list; + { + auto iter = create_iterator(file_family_); + for (iter->SeekToFirst(); not stop_requested && iter->Valid(); + iter->Next()) { + if (stop_requested) { + break; + } + + auto json_data = json::parse(iter->value().ToString()); + list.emplace_back(i_file_db::file_info{ + iter->key().ToString(), + false, + json_data.at("source_path").get(), + }); + + if (list.size() < 100U) { + continue; + } + + callback(list); + list.clear(); + } + } + + { + auto iter = create_iterator(directory_family_); + for (iter->SeekToFirst(); not stop_requested && iter->Valid(); + iter->Next()) { + list.emplace_back(i_file_db::file_info{ + iter->key().ToString(), + true, + iter->value().ToString(), + }); + + if (list.size() < 100U) { + continue; + } + + callback(list); + list.clear(); + } + } +} + auto rdb_file_db::get_api_path(const std::string &source_path, std::string &api_path) const -> api_error { REPERTORY_USES_FUNCTION_NAME(); diff --git a/repertory/librepertory/src/db/impl/rdb_meta_db.cpp b/repertory/librepertory/src/db/impl/rdb_meta_db.cpp index cb129796..74f83fe9 100644 --- a/repertory/librepertory/src/db/impl/rdb_meta_db.cpp +++ b/repertory/librepertory/src/db/impl/rdb_meta_db.cpp @@ -58,6 +58,12 @@ void rdb_meta_db::create_or_open(bool clear) { void rdb_meta_db::clear() { create_or_open(true); } +auto rdb_meta_db::create_iterator(rocksdb::ColumnFamilyHandle *family) const + -> std::shared_ptr { + return std::shared_ptr( + db_->NewIterator(rocksdb::ReadOptions{}, family)); +} + void rdb_meta_db::enumerate_api_path_list( std::function &)> callback, const stop_type &stop_requested) const { @@ -76,12 +82,6 @@ void rdb_meta_db::enumerate_api_path_list( } } -auto rdb_meta_db::create_iterator(rocksdb::ColumnFamilyHandle *family) const - -> std::shared_ptr { - return std::shared_ptr( - db_->NewIterator(rocksdb::ReadOptions{}, family)); -} - auto rdb_meta_db::get_api_path(const std::string &source_path, std::string &api_path) const -> api_error { REPERTORY_USES_FUNCTION_NAME(); diff --git a/repertory/librepertory/src/db/impl/sqlite_file_db.cpp b/repertory/librepertory/src/db/impl/sqlite_file_db.cpp index ce7662d5..bbbe7891 100644 --- a/repertory/librepertory/src/db/impl/sqlite_file_db.cpp +++ b/repertory/librepertory/src/db/impl/sqlite_file_db.cpp @@ -133,6 +133,33 @@ auto sqlite_file_db::count() const -> std::uint64_t { return 0U; } +void sqlite_file_db::enumerate_api_path_list( + std::function &)> callback, + const stop_type &stop_requested) const { + std::vector list; + + auto result = utils::db::sqlite::db_select{*db_, file_table}.go(); + while (result.has_row() && not stop_requested) { + std::optional row; + if (result.get_row(row) && row.has_value()) { + list.emplace_back(i_file_db::file_info{ + .api_path = row->get_column("api_path").get_value(), + .directory = + row->get_column("directory").get_value() == 1, + .source_path = + row->get_column("source_path").get_value(), + }); + + if (list.size() < 100U) { + continue; + } + + callback(list); + list.clear(); + } + } +} + auto sqlite_file_db::get_api_path(const std::string &source_path, std::string &api_path) const -> api_error { auto result = utils::db::sqlite::db_select{*db_, file_table} @@ -287,11 +314,7 @@ auto sqlite_file_db::get_item_list(const stop_type &stop_requested) const std::vector ret; auto result = utils::db::sqlite::db_select{*db_, file_table}.go(); - while (result.has_row()) { - if (stop_requested) { - break; - } - + while (result.has_row() && not stop_requested) { std::optional row; if (result.get_row(row) && row.has_value()) { ret.emplace_back(i_file_db::file_info{ @@ -302,8 +325,6 @@ auto sqlite_file_db::get_item_list(const stop_type &stop_requested) const row->get_column("source_path").get_value(), }); } - - result.next_row(); } return ret; diff --git a/repertory/librepertory/src/db/impl/sqlite_meta_db.cpp b/repertory/librepertory/src/db/impl/sqlite_meta_db.cpp index bc31f87f..e26c3937 100644 --- a/repertory/librepertory/src/db/impl/sqlite_meta_db.cpp +++ b/repertory/librepertory/src/db/impl/sqlite_meta_db.cpp @@ -76,7 +76,6 @@ void sqlite_meta_db::clear() { } void sqlite_meta_db::enumerate_api_path_list( - bool directories, std::function &)> callback, const stop_type &stop_requested) const { auto result = @@ -87,6 +86,7 @@ void sqlite_meta_db::enumerate_api_path_list( std::optional row; if (result.get_row(row) && row.has_value()) { list.push_back(row->get_column("api_path").get_value()); + if (list.size() < 100U) { continue; } diff --git a/repertory/librepertory/src/providers/encrypt/encrypt_provider.cpp b/repertory/librepertory/src/providers/encrypt/encrypt_provider.cpp index dde78f14..cc6a4156 100644 --- a/repertory/librepertory/src/providers/encrypt/encrypt_provider.cpp +++ b/repertory/librepertory/src/providers/encrypt/encrypt_provider.cpp @@ -806,34 +806,37 @@ auto encrypt_provider::read_file_bytes(const std::string &api_path, } void encrypt_provider::remove_deleted_files(const stop_type &stop_requested) { - std::vector removed_list{}; + db_->enumerate_api_path_list( + [this, &stop_requested](auto &&list) { + std::vector removed_list{}; + for (const auto &item : list) { + if (stop_requested) { + return; + } - for (const auto &item : db_->get_item_list(stop_requested)) { - if (stop_requested) { - return; - } + if (not utils::path::exists(item.source_path)) { + removed_list.emplace_back(item); + } + } - if (not utils::path::exists(item.source_path)) { - removed_list.emplace_back(item); - } - } + for (const auto &item : removed_list) { + if (stop_requested) { + return; + } - for (const auto &item : removed_list) { - if (stop_requested) { - return; - } + // TODO handle error + auto del_res{db_->remove_item(item.api_path)}; + if (item.directory) { + event_system::instance().raise( + item.api_path, item.source_path); + continue; + } - // TODO handle error - auto del_res{db_->remove_item(item.api_path)}; - if (item.directory) { - event_system::instance().raise( - item.api_path, item.source_path); - continue; - } - - event_system::instance().raise(item.api_path, - item.source_path); - } + event_system::instance().raise( + item.api_path, item.source_path); + } + }, + stop_requested); } auto encrypt_provider::start(api_item_added_callback /*api_item_added*/,