BlockStorage/repertory
1
0
Fork 0
Code Issues 8 Pull Requests Releases 6 Wiki Activity

begin working on better handling of removed files

Browse Source
This commit is contained in:
Scott E. Graves
2025-01-02 10:27:52 -06:00
parent c3978562d5
commit c27528fe73
7 changed files with 86 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
repertory/librepertory/include/db/i_file_db.hpp
View File

@ -80,7 +80,8 @@ public:
get_file_source_path(const std::string &api_path, get_file_source_path(const std::string &api_path,
std::string &source_path) const -> api_error = 0; std::string &source_path) const -> api_error = 0;
[[nodiscard]] virtual auto get_item_list() const [[nodiscard]] virtual auto
get_item_list(const stop_type &stop_requested) const
-> std::vector<file_info> = 0; -> std::vector<file_info> = 0;
[[nodiscard]] virtual auto get_source_path(const std::string &api_path, [[nodiscard]] virtual auto get_source_path(const std::string &api_path,

54
repertory/librepertory/include/db/impl/rdb_file_db.hpp
View File

@ -67,50 +67,50 @@ private:
rocksdb::Transaction *txn) -> rocksdb::Status; rocksdb::Transaction *txn) -> rocksdb::Status;
public: public:
[[nodiscard]] auto [[nodiscard]] auto add_directory(const std::string &api_path,
add_directory(const std::string &api_path, const std::string &source_path)
const std::string &source_path) -> api_error override; -> api_error override;
[[nodiscard]] auto [[nodiscard]] auto add_or_update_file(const i_file_db::file_data &data)
add_or_update_file(const i_file_db::file_data &data) -> api_error override; -> api_error override;
void clear() override; void clear() override;
[[nodiscard]] auto count() const -> std::uint64_t override; [[nodiscard]] auto count() const -> std::uint64_t override;
[[nodiscard]] auto [[nodiscard]] auto get_api_path(const std::string &source_path,
get_api_path(const std::string &source_path, std::string &api_path) const
std::string &api_path) const -> api_error override; -> api_error override;
[[nodiscard]] auto [[nodiscard]] auto get_directory_api_path(const std::string &source_path,
get_directory_api_path(const std::string &source_path, std::string &api_path) const
std::string &api_path) const -> api_error override; -> api_error override;
[[nodiscard]] auto get_directory_source_path(const std::string &api_path, [[nodiscard]] auto get_directory_source_path(const std::string &api_path,
std::string &source_path) const std::string &source_path) const
-> api_error override; -> api_error override;
[[nodiscard]] auto [[nodiscard]] auto get_file_api_path(const std::string &source_path,
get_file_api_path(const std::string &source_path, std::string &api_path) const
std::string &api_path) const -> api_error override; -> api_error override;
[[nodiscard]] auto [[nodiscard]] auto get_file_data(const std::string &api_path,
get_file_data(const std::string &api_path, i_file_db::file_data &data) const
i_file_db::file_data &data) const -> api_error override; -> api_error override;
[[nodiscard]] auto [[nodiscard]] auto get_file_source_path(const std::string &api_path,
get_file_source_path(const std::string &api_path, std::string &source_path) const
std::string &source_path) const -> api_error override; -> api_error override;
[[nodiscard]] auto [[nodiscard]] auto get_item_list(const stop_type &stop_requested) const
get_item_list() const -> std::vector<i_file_db::file_info> override; -> std::vector<i_file_db::file_info> override;
[[nodiscard]] auto [[nodiscard]] auto get_source_path(const std::string &api_path,
get_source_path(const std::string &api_path, std::string &source_path) const
std::string &source_path) const -> api_error override; -> api_error override;
[[nodiscard]] auto [[nodiscard]] auto remove_item(const std::string &api_path)
remove_item(const std::string &api_path) -> api_error override; -> api_error override;
}; };
} // namespace repertory } // namespace repertory

2
repertory/librepertory/include/db/impl/sqlite_file_db.hpp
View File

@ -77,7 +77,7 @@ public:
std::string &source_path) const std::string &source_path) const
-> api_error override; -> api_error override;
[[nodiscard]] auto get_item_list() const [[nodiscard]] auto get_item_list(const stop_type &stop_requested) const
-> std::vector<i_file_db::file_info> override; -> std::vector<i_file_db::file_info> override;
[[nodiscard]] auto get_source_path(const std::string &api_path, [[nodiscard]] auto get_source_path(const std::string &api_path,

11
repertory/librepertory/src/db/impl/rdb_file_db.cpp
View File

@ -267,11 +267,16 @@ auto rdb_file_db::get_file_source_path(const std::string &api_path,
return result; return result;
} }
auto rdb_file_db::get_item_list() const -> std::vector<i_file_db::file_info> { auto rdb_file_db::get_item_list(const stop_type &stop_requested) const
-> std::vector<i_file_db::file_info> {
std::vector<i_file_db::file_info> ret{}; std::vector<i_file_db::file_info> ret{};
{ {
auto iter = create_iterator(directory_family_); auto iter = create_iterator(directory_family_);
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
if (stop_requested) {
break;
}
ret.emplace_back(i_file_db::file_info{ ret.emplace_back(i_file_db::file_info{
iter->key().ToString(), iter->key().ToString(),
true, true,
@ -283,6 +288,10 @@ auto rdb_file_db::get_item_list() const -> std::vector<i_file_db::file_info> {
{ {
auto iter = create_iterator(file_family_); auto iter = create_iterator(file_family_);
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
if (stop_requested) {
break;
}
auto json_data = json::parse(iter->value().ToString()); auto json_data = json::parse(iter->value().ToString());
ret.emplace_back(i_file_db::file_info{ ret.emplace_back(i_file_db::file_info{
iter->key().ToString(), iter->key().ToString(),

6
repertory/librepertory/src/db/impl/sqlite_file_db.cpp
View File

@ -282,12 +282,16 @@ auto sqlite_file_db::get_file_source_path(const std::string &api_path,
return api_error::item_not_found; return api_error::item_not_found;
} }
auto sqlite_file_db::get_item_list() const auto sqlite_file_db::get_item_list(const stop_type &stop_requested) const
-> std::vector<i_file_db::file_info> { -> std::vector<i_file_db::file_info> {
std::vector<i_file_db::file_info> ret; std::vector<i_file_db::file_info> ret;
auto result = utils::db::sqlite::db_select{*db_, file_table}.go(); auto result = utils::db::sqlite::db_select{*db_, file_table}.go();
while (result.has_row()) { while (result.has_row()) {
if (stop_requested) {
break;
}
std::optional<utils::db::sqlite::db_result::row> row; std::optional<utils::db::sqlite::db_result::row> row;
if (result.get_row(row) && row.has_value()) { if (result.get_row(row) && row.has_value()) {
ret.emplace_back(i_file_db::file_info{ ret.emplace_back(i_file_db::file_info{

62
repertory/librepertory/src/providers/encrypt/encrypt_provider.cpp
View File

@ -38,9 +38,10 @@ namespace repertory {
encrypt_provider::encrypt_provider(app_config &config) encrypt_provider::encrypt_provider(app_config &config)
: config_(config), encrypt_config_(config.get_encrypt_config()) {} : config_(config), encrypt_config_(config.get_encrypt_config()) {}
auto encrypt_provider::create_api_file( auto encrypt_provider::create_api_file(const std::string &api_path,
const std::string &api_path, bool directory, bool directory,
const std::string &source_path) -> api_file { const std::string &source_path)
-> api_file {
auto times = utils::file::get_times(source_path); auto times = utils::file::get_times(source_path);
if (not times.has_value()) { if (not times.has_value()) {
throw std::runtime_error("failed to get file times"); throw std::runtime_error("failed to get file times");
@ -66,10 +67,10 @@ auto encrypt_provider::create_api_file(
void encrypt_provider::create_item_meta(api_meta_map &meta, bool directory, void encrypt_provider::create_item_meta(api_meta_map &meta, bool directory,
const api_file &file) { const api_file &file) {
#if defined(_WIN32) #if defined(_WIN32)
struct _stat64 buf {}; struct _stat64 buf{};
_stat64(file.source_path.c_str(), &buf); _stat64(file.source_path.c_str(), &buf);
#else // !defined(_WIN32) #else // !defined(_WIN32)
struct stat buf {}; struct stat buf{};
stat(file.source_path.c_str(), &buf); stat(file.source_path.c_str(), &buf);
#endif // defined(_WIN32) #endif // defined(_WIN32)
@ -149,8 +150,9 @@ auto encrypt_provider::do_fs_operation(
return callback(cfg, source_path); return callback(cfg, source_path);
} }
auto encrypt_provider::get_api_path_from_source( auto encrypt_provider::get_api_path_from_source(const std::string &source_path,
const std::string &source_path, std::string &api_path) const -> api_error { std::string &api_path) const
-> api_error {
REPERTORY_USES_FUNCTION_NAME(); REPERTORY_USES_FUNCTION_NAME();
try { try {
@ -189,8 +191,9 @@ auto encrypt_provider::get_directory_item_count(
return count; return count;
} }
auto encrypt_provider::get_directory_items( auto encrypt_provider::get_directory_items(const std::string &api_path,
const std::string &api_path, directory_item_list &list) const -> api_error { directory_item_list &list) const
-> api_error {
REPERTORY_USES_FUNCTION_NAME(); REPERTORY_USES_FUNCTION_NAME();
return do_fs_operation( return do_fs_operation(
@ -318,8 +321,9 @@ auto encrypt_provider::get_file(const std::string &api_path,
return api_error::error; return api_error::error;
} }
auto encrypt_provider::get_file_list( auto encrypt_provider::get_file_list(api_file_list &list,
api_file_list &list, std::string & /* marker */) const -> api_error { std::string & /* marker */) const
-> api_error {
REPERTORY_USES_FUNCTION_NAME(); REPERTORY_USES_FUNCTION_NAME();
const auto &cfg = get_encrypt_config(); const auto &cfg = get_encrypt_config();
@ -342,8 +346,9 @@ auto encrypt_provider::get_file_list(
return api_error::error; return api_error::error;
} }
auto encrypt_provider::get_file_size( auto encrypt_provider::get_file_size(const std::string &api_path,
const std::string &api_path, std::uint64_t &file_size) const -> api_error { std::uint64_t &file_size) const
-> api_error {
REPERTORY_USES_FUNCTION_NAME(); REPERTORY_USES_FUNCTION_NAME();
try { try {
@ -364,9 +369,10 @@ auto encrypt_provider::get_file_size(
return api_error::error; return api_error::error;
} }
auto encrypt_provider::get_filesystem_item( auto encrypt_provider::get_filesystem_item(const std::string &api_path,
const std::string &api_path, bool directory, bool directory,
filesystem_item &fsi) const -> api_error { filesystem_item &fsi) const
-> api_error {
std::string source_path; std::string source_path;
if (directory) { if (directory) {
auto result = db_->get_directory_source_path(api_path, source_path); auto result = db_->get_directory_source_path(api_path, source_path);
@ -417,9 +423,10 @@ auto encrypt_provider::get_filesystem_item_from_source_path(
return get_filesystem_item(api_path, false, fsi); return get_filesystem_item(api_path, false, fsi);
} }
auto encrypt_provider::get_filesystem_item_and_file( auto encrypt_provider::get_filesystem_item_and_file(const std::string &api_path,
const std::string &api_path, api_file &file, api_file &file,
filesystem_item &fsi) const -> api_error { filesystem_item &fsi) const
-> api_error {
REPERTORY_USES_FUNCTION_NAME(); REPERTORY_USES_FUNCTION_NAME();
try { try {
@ -544,8 +551,8 @@ auto encrypt_provider::is_directory(const std::string &api_path,
return api_error::error; return api_error::error;
} }
auto encrypt_provider::is_file(const std::string &api_path, auto encrypt_provider::is_file(const std::string &api_path, bool &exists) const
bool &exists) const -> api_error { -> api_error {
REPERTORY_USES_FUNCTION_NAME(); REPERTORY_USES_FUNCTION_NAME();
try { try {
@ -790,7 +797,7 @@ auto encrypt_provider::read_file_bytes(const std::string &api_path,
void encrypt_provider::remove_deleted_files(const stop_type &stop_requested) { void encrypt_provider::remove_deleted_files(const stop_type &stop_requested) {
std::vector<i_file_db::file_info> removed_list{}; std::vector<i_file_db::file_info> removed_list{};
for (const auto &item : db_->get_item_list()) { for (const auto &item : db_->get_item_list(stop_requested)) {
if (stop_requested) { if (stop_requested) {
return; return;
} }
@ -856,11 +863,12 @@ auto encrypt_provider::start(api_item_added_callback /*api_item_added*/,
} }
} }
polling::instance().set_callback({ // polling::instance().set_callback({
"check_deleted", // "check_deleted",
polling::frequency::low, // polling::frequency::low,
[this](auto &&stop_requested) { remove_deleted_files(stop_requested); }, // [this](auto &&stop_requested) { remove_deleted_files(stop_requested);
}); // },
// });
event_system::instance().raise<service_started>("encrypt_provider"); event_system::instance().raise<service_started>("encrypt_provider");
return true; return true;

10
repertory/repertory_test/src/file_db_test.cpp
View File

@ -30,7 +30,8 @@ TYPED_TEST(file_db_test, can_add_and_remove_directory) {
EXPECT_EQ(api_error::success, this->file_db->add_directory("/", "c:\\test")); EXPECT_EQ(api_error::success, this->file_db->add_directory("/", "c:\\test"));
auto list = this->file_db->get_item_list(); stop_type stop_requested{false};
auto list = this->file_db->get_item_list(stop_requested);
EXPECT_EQ(1U, list.size()); EXPECT_EQ(1U, list.size());
EXPECT_STREQ("/", list.at(0U).api_path.c_str()); EXPECT_STREQ("/", list.at(0U).api_path.c_str());
EXPECT_TRUE(list.at(0U).directory); EXPECT_TRUE(list.at(0U).directory);
@ -38,7 +39,7 @@ TYPED_TEST(file_db_test, can_add_and_remove_directory) {
EXPECT_EQ(api_error::success, this->file_db->remove_item("/")); EXPECT_EQ(api_error::success, this->file_db->remove_item("/"));
list = this->file_db->get_item_list(); list = this->file_db->get_item_list(stop_requested);
EXPECT_EQ(0U, list.size()); EXPECT_EQ(0U, list.size());
} }
@ -52,7 +53,8 @@ TYPED_TEST(file_db_test, can_add_and_remove_file) {
"c:\\test\\file.txt", "c:\\test\\file.txt",
})); }));
auto list = this->file_db->get_item_list(); stop_type stop_requested{false};
auto list = this->file_db->get_item_list(stop_requested);
EXPECT_EQ(1U, list.size()); EXPECT_EQ(1U, list.size());
EXPECT_STREQ("/file", list.at(0U).api_path.c_str()); EXPECT_STREQ("/file", list.at(0U).api_path.c_str());
EXPECT_FALSE(list.at(0U).directory); EXPECT_FALSE(list.at(0U).directory);
@ -60,7 +62,7 @@ TYPED_TEST(file_db_test, can_add_and_remove_file) {
EXPECT_EQ(api_error::success, this->file_db->remove_item("/file")); EXPECT_EQ(api_error::success, this->file_db->remove_item("/file"));
list = this->file_db->get_item_list(); list = this->file_db->get_item_list(stop_requested);
EXPECT_EQ(0U, list.size()); EXPECT_EQ(0U, list.size());
} }