BlockStorage/repertory
1
0
Fork 0
Code Issues 8 Pull Requests Releases 6 Wiki Activity

refactor cleanup

Browse Source
This commit is contained in:
Scott E. Graves
2025-01-02 14:01:16 -06:00
parent a25dd8178e
commit eaa6a6f92e
8 changed files with 122 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
repertory/librepertory/include/db/i_file_db.hpp
View File

@ -56,6 +56,10 @@ public:
[[nodiscard]] virtual auto count() const -> std::uint64_t = 0;
virtual void enumerate_api_path_list(
std::function<void(const std::vector<i_file_db::file_info> &)> callback,
const stop_type &stop_requested) const = 0;
[[nodiscard]] virtual auto get_api_path(const std::string &source_path,
std::string &api_path) const
-> api_error = 0;

4
repertory/librepertory/include/db/impl/rdb_file_db.hpp
View File

@ -78,6 +78,10 @@ public:
[[nodiscard]] auto count() const -> std::uint64_t override;
void enumerate_api_path_list(
std::function<void(const std::vector<i_file_db::file_info> &)> callback,
const stop_type &stop_requested) const override;
[[nodiscard]] auto get_api_path(const std::string &source_path,
std::string &api_path) const
-> api_error override;

4
repertory/librepertory/include/db/impl/sqlite_file_db.hpp
View File

@ -53,6 +53,10 @@ public:
[[nodiscard]] auto count() const -> std::uint64_t override;
void enumerate_api_path_list(
std::function<void(const std::vector<i_file_db::file_info> &)> callback,
const stop_type &stop_requested) const override;
[[nodiscard]] auto get_api_path(const std::string &source_path,
std::string &api_path) const
-> api_error override;

48
repertory/librepertory/src/db/impl/rdb_file_db.cpp
View File

@ -149,6 +149,54 @@ auto rdb_file_db::count() const -> std::uint64_t {
return ret;
}
void rdb_file_db::enumerate_api_path_list(
std::function<void(const std::vector<i_file_db::file_info> &)> callback,
const stop_type &stop_requested) const {
std::vector<i_file_db::file_info> list;
{
auto iter = create_iterator(file_family_);
for (iter->SeekToFirst(); not stop_requested && iter->Valid();
iter->Next()) {
if (stop_requested) {
break;
}
auto json_data = json::parse(iter->value().ToString());
list.emplace_back(i_file_db::file_info{
iter->key().ToString(),
false,
json_data.at("source_path").get<std::string>(),
});
if (list.size() < 100U) {
continue;
}
callback(list);
list.clear();
}
}
{
auto iter = create_iterator(directory_family_);
for (iter->SeekToFirst(); not stop_requested && iter->Valid();
iter->Next()) {
list.emplace_back(i_file_db::file_info{
iter->key().ToString(),
true,
iter->value().ToString(),
});
if (list.size() < 100U) {
continue;
}
callback(list);
list.clear();
}
}
}
auto rdb_file_db::get_api_path(const std::string &source_path,
std::string &api_path) const -> api_error {
REPERTORY_USES_FUNCTION_NAME();

12
repertory/librepertory/src/db/impl/rdb_meta_db.cpp
View File

@ -58,6 +58,12 @@ void rdb_meta_db::create_or_open(bool clear) {
void rdb_meta_db::clear() { create_or_open(true); }
auto rdb_meta_db::create_iterator(rocksdb::ColumnFamilyHandle *family) const
-> std::shared_ptr<rocksdb::Iterator> {
return std::shared_ptr<rocksdb::Iterator>(
db_->NewIterator(rocksdb::ReadOptions{}, family));
}
void rdb_meta_db::enumerate_api_path_list(
std::function<void(const std::vector<std::string> &)> callback,
const stop_type &stop_requested) const {
@ -76,12 +82,6 @@ void rdb_meta_db::enumerate_api_path_list(
}
}
auto rdb_meta_db::create_iterator(rocksdb::ColumnFamilyHandle *family) const
-> std::shared_ptr<rocksdb::Iterator> {
return std::shared_ptr<rocksdb::Iterator>(
db_->NewIterator(rocksdb::ReadOptions{}, family));
}
auto rdb_meta_db::get_api_path(const std::string &source_path,
std::string &api_path) const -> api_error {
REPERTORY_USES_FUNCTION_NAME();

35
repertory/librepertory/src/db/impl/sqlite_file_db.cpp
View File

@ -133,6 +133,33 @@ auto sqlite_file_db::count() const -> std::uint64_t {
return 0U;
}
void sqlite_file_db::enumerate_api_path_list(
std::function<void(const std::vector<i_file_db::file_info> &)> callback,
const stop_type &stop_requested) const {
std::vector<i_file_db::file_info> list;
auto result = utils::db::sqlite::db_select{*db_, file_table}.go();
while (result.has_row() && not stop_requested) {
std::optional<utils::db::sqlite::db_result::row> row;
if (result.get_row(row) && row.has_value()) {
list.emplace_back(i_file_db::file_info{
.api_path = row->get_column("api_path").get_value<std::string>(),
.directory =
row->get_column("directory").get_value<std::int64_t>() == 1,
.source_path =
row->get_column("source_path").get_value<std::string>(),
});
if (list.size() < 100U) {
continue;
}
callback(list);
list.clear();
}
}
}
auto sqlite_file_db::get_api_path(const std::string &source_path,
std::string &api_path) const -> api_error {
auto result = utils::db::sqlite::db_select{*db_, file_table}
@ -287,11 +314,7 @@ auto sqlite_file_db::get_item_list(const stop_type &stop_requested) const
std::vector<i_file_db::file_info> ret;
auto result = utils::db::sqlite::db_select{*db_, file_table}.go();
while (result.has_row()) {
if (stop_requested) {
break;
}
while (result.has_row() && not stop_requested) {
std::optional<utils::db::sqlite::db_result::row> row;
if (result.get_row(row) && row.has_value()) {
ret.emplace_back(i_file_db::file_info{
@ -302,8 +325,6 @@ auto sqlite_file_db::get_item_list(const stop_type &stop_requested) const
row->get_column("source_path").get_value<std::string>(),
});
}
result.next_row();
}
return ret;

2
repertory/librepertory/src/db/impl/sqlite_meta_db.cpp
View File

@ -76,7 +76,6 @@ void sqlite_meta_db::clear() {
}
void sqlite_meta_db::enumerate_api_path_list(
bool directories,
std::function<void(const std::vector<std::string> &)> callback,
const stop_type &stop_requested) const {
auto result =
@ -87,6 +86,7 @@ void sqlite_meta_db::enumerate_api_path_list(
std::optional<utils::db::sqlite::db_result::row> row;
if (result.get_row(row) && row.has_value()) {
list.push_back(row->get_column("api_path").get_value<std::string>());
if (list.size() < 100U) {
continue;
}

51
repertory/librepertory/src/providers/encrypt/encrypt_provider.cpp
View File

@ -806,34 +806,37 @@ auto encrypt_provider::read_file_bytes(const std::string &api_path,
}
void encrypt_provider::remove_deleted_files(const stop_type &stop_requested) {
std::vector<i_file_db::file_info> removed_list{};
db_->enumerate_api_path_list(
[this, &stop_requested](auto &&list) {
std::vector<i_file_db::file_info> removed_list{};
for (const auto &item : list) {
if (stop_requested) {
return;
}
for (const auto &item : db_->get_item_list(stop_requested)) {
if (stop_requested) {
return;
}
if (not utils::path::exists(item.source_path)) {
removed_list.emplace_back(item);
}
}
if (not utils::path::exists(item.source_path)) {
removed_list.emplace_back(item);
}
}
for (const auto &item : removed_list) {
if (stop_requested) {
return;
}
for (const auto &item : removed_list) {
if (stop_requested) {
return;
}
// TODO handle error
auto del_res{db_->remove_item(item.api_path)};
if (item.directory) {
event_system::instance().raise<directory_removed_externally>(
item.api_path, item.source_path);
continue;
}
// TODO handle error
auto del_res{db_->remove_item(item.api_path)};
if (item.directory) {
event_system::instance().raise<directory_removed_externally>(
item.api_path, item.source_path);
continue;
}
event_system::instance().raise<file_removed_externally>(item.api_path,
item.source_path);
}
event_system::instance().raise<file_removed_externally>(
item.api_path, item.source_path);
}
},
stop_requested);
}
auto encrypt_provider::start(api_item_added_callback /*api_item_added*/,