BlockStorage/repertory
1
0
Fork 0
Code Issues 8 Pull Requests Releases 6 Wiki Activity

begin working on better handling of removed files

Browse Source
This commit is contained in:
Scott E. Graves
2025-01-02 10:27:52 -06:00
parent c3978562d5
commit c27528fe73
7 changed files with 86 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
repertory/librepertory/include/db/i_file_db.hpp
View File

@ -80,7 +80,8 @@ public:
get_file_source_path(const std::string &api_path,
std::string &source_path) const -> api_error = 0;
[[nodiscard]] virtual auto get_item_list() const
[[nodiscard]] virtual auto
get_item_list(const stop_type &stop_requested) const
-> std::vector<file_info> = 0;
[[nodiscard]] virtual auto get_source_path(const std::string &api_path,

54
repertory/librepertory/include/db/impl/rdb_file_db.hpp
View File

@ -67,50 +67,50 @@ private:
rocksdb::Transaction *txn) -> rocksdb::Status;
public:
[[nodiscard]] auto
add_directory(const std::string &api_path,
const std::string &source_path) -> api_error override;
[[nodiscard]] auto add_directory(const std::string &api_path,
const std::string &source_path)
-> api_error override;
[[nodiscard]] auto
add_or_update_file(const i_file_db::file_data &data) -> api_error override;
[[nodiscard]] auto add_or_update_file(const i_file_db::file_data &data)
-> api_error override;
void clear() override;
[[nodiscard]] auto count() const -> std::uint64_t override;
[[nodiscard]] auto
get_api_path(const std::string &source_path,
std::string &api_path) const -> api_error override;
[[nodiscard]] auto get_api_path(const std::string &source_path,
std::string &api_path) const
-> api_error override;
[[nodiscard]] auto
get_directory_api_path(const std::string &source_path,
std::string &api_path) const -> api_error override;
[[nodiscard]] auto get_directory_api_path(const std::string &source_path,
std::string &api_path) const
-> api_error override;
[[nodiscard]] auto get_directory_source_path(const std::string &api_path,
std::string &source_path) const
-> api_error override;
[[nodiscard]] auto
get_file_api_path(const std::string &source_path,
std::string &api_path) const -> api_error override;
[[nodiscard]] auto get_file_api_path(const std::string &source_path,
std::string &api_path) const
-> api_error override;
[[nodiscard]] auto
get_file_data(const std::string &api_path,
i_file_db::file_data &data) const -> api_error override;
[[nodiscard]] auto get_file_data(const std::string &api_path,
i_file_db::file_data &data) const
-> api_error override;
[[nodiscard]] auto
get_file_source_path(const std::string &api_path,
std::string &source_path) const -> api_error override;
[[nodiscard]] auto get_file_source_path(const std::string &api_path,
std::string &source_path) const
-> api_error override;
[[nodiscard]] auto
get_item_list() const -> std::vector<i_file_db::file_info> override;
[[nodiscard]] auto get_item_list(const stop_type &stop_requested) const
-> std::vector<i_file_db::file_info> override;
[[nodiscard]] auto
get_source_path(const std::string &api_path,
std::string &source_path) const -> api_error override;
[[nodiscard]] auto get_source_path(const std::string &api_path,
std::string &source_path) const
-> api_error override;
[[nodiscard]] auto
remove_item(const std::string &api_path) -> api_error override;
[[nodiscard]] auto remove_item(const std::string &api_path)
-> api_error override;
};
} // namespace repertory

2
repertory/librepertory/include/db/impl/sqlite_file_db.hpp
View File

@ -77,7 +77,7 @@ public:
std::string &source_path) const
-> api_error override;
[[nodiscard]] auto get_item_list() const
[[nodiscard]] auto get_item_list(const stop_type &stop_requested) const
-> std::vector<i_file_db::file_info> override;
[[nodiscard]] auto get_source_path(const std::string &api_path,

11
repertory/librepertory/src/db/impl/rdb_file_db.cpp
View File

@ -267,11 +267,16 @@ auto rdb_file_db::get_file_source_path(const std::string &api_path,
return result;
}
auto rdb_file_db::get_item_list() const -> std::vector<i_file_db::file_info> {
auto rdb_file_db::get_item_list(const stop_type &stop_requested) const
-> std::vector<i_file_db::file_info> {
std::vector<i_file_db::file_info> ret{};
{
auto iter = create_iterator(directory_family_);
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
if (stop_requested) {
break;
}
ret.emplace_back(i_file_db::file_info{
iter->key().ToString(),
true,
@ -283,6 +288,10 @@ auto rdb_file_db::get_item_list() const -> std::vector<i_file_db::file_info> {
{
auto iter = create_iterator(file_family_);
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
if (stop_requested) {
break;
}
auto json_data = json::parse(iter->value().ToString());
ret.emplace_back(i_file_db::file_info{
iter->key().ToString(),

6
repertory/librepertory/src/db/impl/sqlite_file_db.cpp
View File

@ -282,12 +282,16 @@ auto sqlite_file_db::get_file_source_path(const std::string &api_path,
return api_error::item_not_found;
}
auto sqlite_file_db::get_item_list() const
auto sqlite_file_db::get_item_list(const stop_type &stop_requested) const
-> std::vector<i_file_db::file_info> {
std::vector<i_file_db::file_info> ret;
auto result = utils::db::sqlite::db_select{*db_, file_table}.go();
while (result.has_row()) {
if (stop_requested) {
break;
}
std::optional<utils::db::sqlite::db_result::row> row;
if (result.get_row(row) && row.has_value()) {
ret.emplace_back(i_file_db::file_info{

62
repertory/librepertory/src/providers/encrypt/encrypt_provider.cpp
View File

@ -38,9 +38,10 @@ namespace repertory {
encrypt_provider::encrypt_provider(app_config &config)
: config_(config), encrypt_config_(config.get_encrypt_config()) {}
auto encrypt_provider::create_api_file(
const std::string &api_path, bool directory,
const std::string &source_path) -> api_file {
auto encrypt_provider::create_api_file(const std::string &api_path,
bool directory,
const std::string &source_path)
-> api_file {
auto times = utils::file::get_times(source_path);
if (not times.has_value()) {
throw std::runtime_error("failed to get file times");
@ -66,10 +67,10 @@ auto encrypt_provider::create_api_file(
void encrypt_provider::create_item_meta(api_meta_map &meta, bool directory,
const api_file &file) {
#if defined(_WIN32)
struct _stat64 buf {};
struct _stat64 buf{};
_stat64(file.source_path.c_str(), &buf);
#else // !defined(_WIN32)
struct stat buf {};
struct stat buf{};
stat(file.source_path.c_str(), &buf);
#endif // defined(_WIN32)
@ -149,8 +150,9 @@ auto encrypt_provider::do_fs_operation(
return callback(cfg, source_path);
}
auto encrypt_provider::get_api_path_from_source(
const std::string &source_path, std::string &api_path) const -> api_error {
auto encrypt_provider::get_api_path_from_source(const std::string &source_path,
std::string &api_path) const
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
try {
@ -189,8 +191,9 @@ auto encrypt_provider::get_directory_item_count(
return count;
}
auto encrypt_provider::get_directory_items(
const std::string &api_path, directory_item_list &list) const -> api_error {
auto encrypt_provider::get_directory_items(const std::string &api_path,
directory_item_list &list) const
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
return do_fs_operation(
@ -318,8 +321,9 @@ auto encrypt_provider::get_file(const std::string &api_path,
return api_error::error;
}
auto encrypt_provider::get_file_list(
api_file_list &list, std::string & /* marker */) const -> api_error {
auto encrypt_provider::get_file_list(api_file_list &list,
std::string & /* marker */) const
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
const auto &cfg = get_encrypt_config();
@ -342,8 +346,9 @@ auto encrypt_provider::get_file_list(
return api_error::error;
}
auto encrypt_provider::get_file_size(
const std::string &api_path, std::uint64_t &file_size) const -> api_error {
auto encrypt_provider::get_file_size(const std::string &api_path,
std::uint64_t &file_size) const
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
try {
@ -364,9 +369,10 @@ auto encrypt_provider::get_file_size(
return api_error::error;
}
auto encrypt_provider::get_filesystem_item(
const std::string &api_path, bool directory,
filesystem_item &fsi) const -> api_error {
auto encrypt_provider::get_filesystem_item(const std::string &api_path,
bool directory,
filesystem_item &fsi) const
-> api_error {
std::string source_path;
if (directory) {
auto result = db_->get_directory_source_path(api_path, source_path);
@ -417,9 +423,10 @@ auto encrypt_provider::get_filesystem_item_from_source_path(
return get_filesystem_item(api_path, false, fsi);
}
auto encrypt_provider::get_filesystem_item_and_file(
const std::string &api_path, api_file &file,
filesystem_item &fsi) const -> api_error {
auto encrypt_provider::get_filesystem_item_and_file(const std::string &api_path,
api_file &file,
filesystem_item &fsi) const
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
try {
@ -544,8 +551,8 @@ auto encrypt_provider::is_directory(const std::string &api_path,
return api_error::error;
}
auto encrypt_provider::is_file(const std::string &api_path,
bool &exists) const -> api_error {
auto encrypt_provider::is_file(const std::string &api_path, bool &exists) const
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
try {
@ -790,7 +797,7 @@ auto encrypt_provider::read_file_bytes(const std::string &api_path,
void encrypt_provider::remove_deleted_files(const stop_type &stop_requested) {
std::vector<i_file_db::file_info> removed_list{};
for (const auto &item : db_->get_item_list()) {
for (const auto &item : db_->get_item_list(stop_requested)) {
if (stop_requested) {
return;
}
@ -856,11 +863,12 @@ auto encrypt_provider::start(api_item_added_callback /*api_item_added*/,
}
}
polling::instance().set_callback({
"check_deleted",
polling::frequency::low,
[this](auto &&stop_requested) { remove_deleted_files(stop_requested); },
});
// polling::instance().set_callback({
// "check_deleted",
// polling::frequency::low,
// [this](auto &&stop_requested) { remove_deleted_files(stop_requested);
// },
// });
event_system::instance().raise<service_started>("encrypt_provider");
return true;

10
repertory/repertory_test/src/file_db_test.cpp
View File

@ -30,7 +30,8 @@ TYPED_TEST(file_db_test, can_add_and_remove_directory) {
EXPECT_EQ(api_error::success, this->file_db->add_directory("/", "c:\\test"));
auto list = this->file_db->get_item_list();
stop_type stop_requested{false};
auto list = this->file_db->get_item_list(stop_requested);
EXPECT_EQ(1U, list.size());
EXPECT_STREQ("/", list.at(0U).api_path.c_str());
EXPECT_TRUE(list.at(0U).directory);
@ -38,7 +39,7 @@ TYPED_TEST(file_db_test, can_add_and_remove_directory) {
EXPECT_EQ(api_error::success, this->file_db->remove_item("/"));
list = this->file_db->get_item_list();
list = this->file_db->get_item_list(stop_requested);
EXPECT_EQ(0U, list.size());
}
@ -52,7 +53,8 @@ TYPED_TEST(file_db_test, can_add_and_remove_file) {
"c:\\test\\file.txt",
}));
auto list = this->file_db->get_item_list();
stop_type stop_requested{false};
auto list = this->file_db->get_item_list(stop_requested);
EXPECT_EQ(1U, list.size());
EXPECT_STREQ("/file", list.at(0U).api_path.c_str());
EXPECT_FALSE(list.at(0U).directory);
@ -60,7 +62,7 @@ TYPED_TEST(file_db_test, can_add_and_remove_file) {
EXPECT_EQ(api_error::success, this->file_db->remove_item("/file"));
list = this->file_db->get_item_list();
list = this->file_db->get_item_list(stop_requested);
EXPECT_EQ(0U, list.size());
}