BlockStorage/repertory
1
0
Fork 0
Code Issues 8 Pull Requests Releases 6 Wiki Activity

fix check deleted

Browse Source
This commit is contained in:
Scott E. Graves 2025-01-25 05:59:44 -06:00
parent 412e03898d
commit f7b829f71c
6 changed files with 96 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
repertory/librepertory/src/db/impl/rdb_file_db.cpp
View File

@ -171,6 +171,10 @@ void rdb_file_db::enumerate_item_list(
callback(list);
list.clear();
}
if (not list.empty()) {
callback(list);
}
}
{
@ -190,6 +194,10 @@ void rdb_file_db::enumerate_item_list(
callback(list);
list.clear();
}
if (not list.empty()) {
callback(list);
}
}
}
@ -203,9 +211,8 @@ auto rdb_file_db::get_api_path(const std::string &source_path,
});
}
auto rdb_file_db::get_directory_api_path(const std::string &source_path,
std::string &api_path) const
-> api_error {
auto rdb_file_db::get_directory_api_path(
const std::string &source_path, std::string &api_path) const -> api_error {
REPERTORY_USES_FUNCTION_NAME();
auto result = perform_action(function_name, [&]() -> rocksdb::Status {
@ -228,9 +235,8 @@ auto rdb_file_db::get_directory_api_path(const std::string &source_path,
: result;
}
auto rdb_file_db::get_directory_source_path(const std::string &api_path,
std::string &source_path) const
-> api_error {
auto rdb_file_db::get_directory_source_path(
const std::string &api_path, std::string &source_path) const -> api_error {
REPERTORY_USES_FUNCTION_NAME();
auto result = perform_action(function_name, [&]() -> rocksdb::Status {
@ -291,9 +297,8 @@ auto rdb_file_db::get_file_data(const std::string &api_path,
return result;
}
auto rdb_file_db::get_file_source_path(const std::string &api_path,
std::string &source_path) const
-> api_error {
auto rdb_file_db::get_file_source_path(
const std::string &api_path, std::string &source_path) const -> api_error {
REPERTORY_USES_FUNCTION_NAME();
auto result = perform_action(function_name, [&]() -> rocksdb::Status {

11
repertory/librepertory/src/db/impl/rdb_meta_db.cpp
View File

@ -81,6 +81,10 @@ void rdb_meta_db::enumerate_api_path_list(
callback(list);
list.clear();
}
if (not list.empty()) {
callback(list);
}
}
auto rdb_meta_db::get_api_path(const std::string &source_path,
@ -314,10 +318,9 @@ void rdb_meta_db::remove_api_path(const std::string &api_path) {
}
}
auto rdb_meta_db::remove_api_path(const std::string &api_path,
const std::string &source_path,
rocksdb::Transaction *txn)
-> rocksdb::Status {
auto rdb_meta_db::remove_api_path(
const std::string &api_path, const std::string &source_path,
rocksdb::Transaction *txn) -> rocksdb::Status {
auto txn_res = txn->Delete(pinned_family_, api_path);
if (not txn_res.ok()) {
return txn_res;

34
repertory/librepertory/src/db/impl/sqlite_file_db.cpp
View File

@ -65,9 +65,8 @@ sqlite_file_db::sqlite_file_db(const app_config &cfg) {
sqlite_file_db::~sqlite_file_db() { db_.reset(); }
auto sqlite_file_db::add_directory(const std::string &api_path,
const std::string &source_path)
-> api_error {
auto sqlite_file_db::add_directory(
const std::string &api_path, const std::string &source_path) -> api_error {
REPERTORY_USES_FUNCTION_NAME();
auto result = utils::db::sqlite::db_insert{*db_, file_table}
@ -158,6 +157,10 @@ void sqlite_file_db::enumerate_item_list(
list.clear();
}
}
if (not list.empty()) {
callback(list);
}
}
auto sqlite_file_db::get_api_path(const std::string &source_path,
@ -179,9 +182,8 @@ auto sqlite_file_db::get_api_path(const std::string &source_path,
return api_error::item_not_found;
}
auto sqlite_file_db::get_directory_api_path(const std::string &source_path,
std::string &api_path) const
-> api_error {
auto sqlite_file_db::get_directory_api_path(
const std::string &source_path, std::string &api_path) const -> api_error {
auto result = utils::db::sqlite::db_select{*db_, file_table}
.column("api_path")
.where("source_path")
@ -202,9 +204,8 @@ auto sqlite_file_db::get_directory_api_path(const std::string &source_path,
return api_error::directory_not_found;
}
auto sqlite_file_db::get_directory_source_path(const std::string &api_path,
std::string &source_path) const
-> api_error {
auto sqlite_file_db::get_directory_source_path(
const std::string &api_path, std::string &source_path) const -> api_error {
auto result = utils::db::sqlite::db_select{*db_, file_table}
.column("source_path")
.where("api_path")
@ -225,9 +226,8 @@ auto sqlite_file_db::get_directory_source_path(const std::string &api_path,
return api_error::directory_not_found;
}
auto sqlite_file_db::get_file_api_path(const std::string &source_path,
std::string &api_path) const
-> api_error {
auto sqlite_file_db::get_file_api_path(
const std::string &source_path, std::string &api_path) const -> api_error {
auto result = utils::db::sqlite::db_select{*db_, file_table}
.column("api_path")
.where("source_path")
@ -286,9 +286,8 @@ auto sqlite_file_db::get_file_data(const std::string &api_path,
return api_error::item_not_found;
}
auto sqlite_file_db::get_file_source_path(const std::string &api_path,
std::string &source_path) const
-> api_error {
auto sqlite_file_db::get_file_source_path(
const std::string &api_path, std::string &source_path) const -> api_error {
auto result = utils::db::sqlite::db_select{*db_, file_table}
.column("source_path")
.where("api_path")
@ -330,9 +329,8 @@ auto sqlite_file_db::get_item_list(stop_type_callback stop_requested_cb) const
return ret;
}
auto sqlite_file_db::get_source_path(const std::string &api_path,
std::string &source_path) const
-> api_error {
auto sqlite_file_db::get_source_path(
const std::string &api_path, std::string &source_path) const -> api_error {
auto result = utils::db::sqlite::db_select{*db_, file_table}
.column("source_path")
.where("api_path")

4
repertory/librepertory/src/db/impl/sqlite_meta_db.cpp
View File

@ -95,6 +95,10 @@ void sqlite_meta_db::enumerate_api_path_list(
list.clear();
}
}
if (not list.empty()) {
callback(list);
}
}
auto sqlite_meta_db::get_api_path(const std::string &source_path,

56
repertory/librepertory/src/providers/base_provider.cpp
View File

@ -24,6 +24,7 @@
#include "app_config.hpp"
#include "db/meta_db.hpp"
#include "events/event_system.hpp"
#include "events/types/debug_log.hpp"
#include "events/types/directory_remove_failed.hpp"
#include "events/types/directory_removed.hpp"
#include "events/types/directory_removed_externally.hpp"
@ -41,13 +42,13 @@
#include "file_manager/cache_size_mgr.hpp"
#include "file_manager/i_file_manager.hpp"
#include "platform/platform.hpp"
#include "utils/config.hpp"
#include "utils/error_utils.hpp"
#include "utils/file_utils.hpp"
#include "utils/path.hpp"
#include "utils/polling.hpp"
#include "utils/tasks.hpp"
#include "utils/time.hpp"
#include <utils/config.hpp>
namespace repertory {
void base_provider::add_all_items(stop_type &stop_requested) {
@ -57,6 +58,8 @@ void base_provider::add_all_items(stop_type &stop_requested) {
REPERTORY_USES_FUNCTION_NAME();
event_system::instance().raise<debug_log>(function_name, "get list|begin");
api_file_list list{};
std::string marker;
auto res{api_error::more_data};
@ -66,6 +69,9 @@ void base_provider::add_all_items(stop_type &stop_requested) {
utils::error::raise_error(function_name, res, "failed to get file list");
}
}
event_system::instance().raise<debug_log>(
function_name, fmt::format("get list|end|{}", list.size()));
}
auto base_provider::create_api_file(std::string path, std::string key,
@ -549,12 +555,20 @@ void base_provider::process_removed_files(std::deque<removed_item> removed_list,
}
void base_provider::process_removed_items(stop_type &stop_requested) {
REPERTORY_USES_FUNCTION_NAME();
const auto get_stop_requested = [&stop_requested]() -> bool {
return stop_requested || app_config::get_stop_requested();
};
event_system::instance().raise<debug_log>(function_name,
"checking removed items");
db3_->enumerate_api_path_list(
[this, &get_stop_requested](auto &&list) {
event_system::instance().raise<debug_log>(
function_name,
fmt::format("enumerating removed items|{}", list.size()));
[[maybe_unused]] auto res =
std::all_of(list.begin(), list.end(), [&](auto &&api_path) -> bool {
if (get_stop_requested()) {
@ -563,22 +577,44 @@ void base_provider::process_removed_items(stop_type &stop_requested) {
tasks::instance().schedule({
[this, api_path](auto &&task_stopped) {
event_system::instance().raise<debug_log>(
function_name,
fmt::format("checking item|{}", api_path));
api_meta_map meta{};
if (get_item_meta(api_path, meta) != api_error::success) {
auto result = get_item_meta(api_path, meta);
if (result != api_error::success) {
event_system::instance().raise<debug_log>(
function_name,
fmt::format("meta not found|{}|error|{}", api_path,
api_error_to_string(result)));
return;
}
if (utils::string::to_bool(meta[META_DIRECTORY])) {
event_system::instance().raise<debug_log>(
function_name,
fmt::format("checking directory|{}", api_path));
bool exists{};
if (is_directory(api_path, exists) !=
api_error::success) {
return;
}
event_system::instance().raise<debug_log>(
function_name,
fmt::format("checking directory|{}|exists|{}",
api_path, exists));
if (exists) {
return;
}
event_system::instance().raise<debug_log>(
function_name,
fmt::format(
"processing removed directory|{}|exists|{}",
api_path, exists));
process_removed_directories(
{
removed_item{api_path, true, ""},
@ -588,15 +624,26 @@ void base_provider::process_removed_items(stop_type &stop_requested) {
return;
}
event_system::instance().raise<debug_log>(
function_name,
fmt::format("checking file|{}", api_path));
bool exists{};
if (is_file(api_path, exists) != api_error::success) {
return;
}
event_system::instance().raise<debug_log>(
function_name, fmt::format("checking file|{}|exists|{}",
api_path, exists));
if (exists) {
return;
}
event_system::instance().raise<debug_log>(
function_name,
fmt::format("processing removed file|{}|exists|{}",
api_path, exists));
process_removed_files(
{
removed_item{api_path, false, meta[META_SOURCE]},
@ -612,20 +659,25 @@ void base_provider::process_removed_items(stop_type &stop_requested) {
}
void base_provider::remove_deleted_items(stop_type &stop_requested) {
REPERTORY_USES_FUNCTION_NAME();
const auto get_stop_requested = [&stop_requested]() -> bool {
return stop_requested || app_config::get_stop_requested();
};
event_system::instance().raise<debug_log>(function_name, "add all");
add_all_items(stop_requested);
if (get_stop_requested()) {
return;
}
event_system::instance().raise<debug_log>(function_name, "remove unmatched");
remove_unmatched_source_files(stop_requested);
if (get_stop_requested()) {
return;
}
event_system::instance().raise<debug_log>(function_name, "remove items");
process_removed_items(stop_requested);
}

2
repertory/librepertory/src/utils/tasks.cpp
View File

@ -84,7 +84,7 @@ void tasks::start(app_config *config) {
stop_requested_ = false;
tasks_.clear();
for (std::uint32_t idx = 0U; idx < std::thread::hardware_concurrency();
for (std::uint32_t idx = 0U; idx < std::thread::hardware_concurrency() * 2U;
++idx) {
task_threads_.emplace_back(
std::make_unique<std::jthread>([this]() { task_thread(); }));