BlockStorage/repertory
1
0
Fork 0
Code Issues 7 Pull Requests Releases 6 Wiki Activity

refactor

Browse Source
This commit is contained in:
Scott E. Graves 2024-12-03 14:02:51 -06:00
parent ae3b592cf6
commit 62857e1372
2 changed files with 254 additions and 189 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

162
repertory/librepertory/include/providers/base_provider.hpp
View File

@ -32,6 +32,13 @@ class i_file_manager;
class i_http_comm;
class base_provider : public i_provider {
private:
struct removed_item final {
std::string api_path;
bool directory{};
std::string source_path;
};
public:
base_provider(app_config &config, i_http_comm &comm)
: config_(config), comm_(comm) {}
@ -46,20 +53,35 @@ private:
i_file_manager *fm_{};
private:
void remove_deleted_files(const stop_type &stop_requested);
void add_all_items(const stop_type &stop_requested);
void get_removed_items(std::deque<removed_item> &directories,
std::deque<removed_item> &files,
const stop_type &stop_requested) const;
void process_removed_directories(std::deque<removed_item> &removed_list,
const stop_type &stop_requested);
void process_removed_files(std::deque<removed_item> &removed_list,
const stop_type &stop_requested);
void remove_deleted_items(const stop_type &stop_requested);
void remove_unmatched_source_files(const stop_type &stop_requested);
protected:
[[nodiscard]] static auto
create_api_file(std::string path, std::string key, std::uint64_t size,
std::uint64_t file_time) -> api_file;
[[nodiscard]] static auto create_api_file(std::string path, std::string key,
std::uint64_t size,
std::uint64_t file_time)
-> api_file;
[[nodiscard]] static auto create_api_file(std::string path,
std::uint64_t size,
api_meta_map &meta) -> api_file;
[[nodiscard]] virtual auto
create_directory_impl(const std::string &api_path,
api_meta_map &meta) -> api_error = 0;
[[nodiscard]] virtual auto create_directory_impl(const std::string &api_path,
api_meta_map &meta)
-> api_error = 0;
[[nodiscard]] virtual auto
create_file_extra(const std::string & /* api_path */,
@ -71,8 +93,8 @@ protected:
return api_item_added_;
}
[[nodiscard]] auto
get_api_item_added() const -> const api_item_added_callback & {
[[nodiscard]] auto get_api_item_added() const
-> const api_item_added_callback & {
return api_item_added_;
}
@ -98,20 +120,22 @@ protected:
return fm_;
}
[[nodiscard]] virtual auto
remove_directory_impl(const std::string &api_path) -> api_error = 0;
[[nodiscard]] virtual auto remove_directory_impl(const std::string &api_path)
-> api_error = 0;
[[nodiscard]] virtual auto
remove_file_impl(const std::string &api_path) -> api_error = 0;
[[nodiscard]] virtual auto remove_file_impl(const std::string &api_path)
-> api_error = 0;
[[nodiscard]] virtual auto
upload_file_impl(const std::string &api_path, const std::string &source_path,
stop_type &stop_requested) -> api_error = 0;
[[nodiscard]] virtual auto upload_file_impl(const std::string &api_path,
const std::string &source_path,
stop_type &stop_requested)
-> api_error = 0;
public:
[[nodiscard]] auto create_directory_clone_source_meta(
const std::string &source_api_path,
const std::string &api_path) -> api_error override;
[[nodiscard]] auto
create_directory_clone_source_meta(const std::string &source_api_path,
const std::string &api_path)
-> api_error override;
[[nodiscard]] auto create_directory(const std::string &api_path,
api_meta_map &meta) -> api_error override;
@ -119,76 +143,82 @@ public:
[[nodiscard]] auto create_file(const std::string &api_path,
api_meta_map &meta) -> api_error override;
[[nodiscard]] auto
get_api_path_from_source(const std::string &source_path,
std::string &api_path) const -> api_error override;
[[nodiscard]] auto get_api_path_from_source(const std::string &source_path,
std::string &api_path) const
-> api_error override;
[[nodiscard]] auto get_directory_items(const std::string &api_path,
directory_item_list &list) const
-> api_error override;
[[nodiscard]] auto get_file_size(const std::string &api_path,
std::uint64_t &file_size) const
-> api_error override;
[[nodiscard]] auto get_filesystem_item(const std::string &api_path,
bool directory,
filesystem_item &fsi) const
-> api_error override;
[[nodiscard]] auto get_filesystem_item_and_file(const std::string &api_path,
api_file &f,
filesystem_item &fsi) const
-> api_error override;
[[nodiscard]] auto
get_directory_items(const std::string &api_path,
directory_item_list &list) const -> api_error override;
get_filesystem_item_from_source_path(const std::string &source_path,
filesystem_item &fsi) const
-> api_error override;
[[nodiscard]] auto
get_file_size(const std::string &api_path,
std::uint64_t &file_size) const -> api_error override;
[[nodiscard]] auto get_item_meta(const std::string &api_path,
api_meta_map &meta) const
-> api_error override;
[[nodiscard]] auto
get_filesystem_item(const std::string &api_path, bool directory,
filesystem_item &fsi) const -> api_error override;
[[nodiscard]] auto get_item_meta(const std::string &api_path,
const std::string &key,
std::string &value) const
-> api_error override;
[[nodiscard]] auto get_filesystem_item_and_file(
const std::string &api_path, api_file &f,
filesystem_item &fsi) const -> api_error override;
[[nodiscard]] auto get_filesystem_item_from_source_path(
const std::string &source_path,
filesystem_item &fsi) const -> api_error override;
[[nodiscard]] auto
get_item_meta(const std::string &api_path,
api_meta_map &meta) const -> api_error override;
[[nodiscard]] auto
get_item_meta(const std::string &api_path, const std::string &key,
std::string &value) const -> api_error override;
[[nodiscard]] auto
get_pinned_files() const -> std::vector<std::string> override;
[[nodiscard]] auto get_pinned_files() const
-> std::vector<std::string> override;
[[nodiscard]] auto get_total_item_count() const -> std::uint64_t override;
[[nodiscard]] auto get_used_drive_space() const -> std::uint64_t override;
[[nodiscard]] auto
is_file_writeable(const std::string &api_path) const -> bool override;
[[nodiscard]] auto is_file_writeable(const std::string &api_path) const
-> bool override;
[[nodiscard]] auto is_read_only() const -> bool override { return false; }
[[nodiscard]] auto
remove_directory(const std::string &api_path) -> api_error override;
[[nodiscard]] auto remove_directory(const std::string &api_path)
-> api_error override;
[[nodiscard]] auto
remove_file(const std::string &api_path) -> api_error override;
[[nodiscard]] auto remove_file(const std::string &api_path)
-> api_error override;
[[nodiscard]] auto
remove_item_meta(const std::string &api_path,
const std::string &key) -> api_error override;
[[nodiscard]] auto remove_item_meta(const std::string &api_path,
const std::string &key)
-> api_error override;
[[nodiscard]] auto
set_item_meta(const std::string &api_path, const std::string &key,
const std::string &value) -> api_error override;
[[nodiscard]] auto set_item_meta(const std::string &api_path,
const std::string &key,
const std::string &value)
-> api_error override;
[[nodiscard]] auto
set_item_meta(const std::string &api_path,
const api_meta_map &meta) -> api_error override;
[[nodiscard]] auto set_item_meta(const std::string &api_path,
const api_meta_map &meta)
-> api_error override;
[[nodiscard]] auto start(api_item_added_callback api_item_added,
i_file_manager *mgr) -> bool override;
void stop() override;
[[nodiscard]] auto
upload_file(const std::string &api_path, const std::string &source_path,
stop_type &stop_requested) -> api_error override;
[[nodiscard]] auto upload_file(const std::string &api_path,
const std::string &source_path,
stop_type &stop_requested)
-> api_error override;
};
} // namespace repertory

281
repertory/librepertory/src/providers/base_provider.cpp
View File

@ -32,9 +32,23 @@
#include "utils/time.hpp"
namespace repertory {
void base_provider::add_all_items(const stop_type &stop_requested) {
REPERTORY_USES_FUNCTION_NAME();
api_file_list list{};
std::string marker;
auto res{api_error::more_data};
while (not stop_requested && res == api_error::more_data) {
res = get_file_list(list, marker);
if (res != api_error::success && res != api_error::more_data) {
utils::error::raise_error(function_name, res, "failed to get file list");
}
}
}
auto base_provider::create_api_file(std::string path, std::string key,
std::uint64_t size,
std::uint64_t file_time) -> api_file {
std::uint64_t size, std::uint64_t file_time)
-> api_file {
api_file file{};
file.api_path = utils::path::create_api_path(path);
file.api_parent = utils::path::get_parent_api_path(file.api_path);
@ -66,8 +80,8 @@ auto base_provider::create_api_file(std::string path, std::uint64_t size,
}
auto base_provider::create_directory_clone_source_meta(
const std::string &source_api_path,
const std::string &api_path) -> api_error {
const std::string &source_api_path, const std::string &api_path)
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
bool exists{};
@ -164,8 +178,8 @@ auto base_provider::create_directory(const std::string &api_path,
return set_item_meta(api_path, meta);
}
auto base_provider::create_file(const std::string &api_path,
api_meta_map &meta) -> api_error {
auto base_provider::create_file(const std::string &api_path, api_meta_map &meta)
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
bool exists{};
@ -222,8 +236,9 @@ auto base_provider::create_file(const std::string &api_path,
return api_error::error;
}
auto base_provider::get_api_path_from_source(
const std::string &source_path, std::string &api_path) const -> api_error {
auto base_provider::get_api_path_from_source(const std::string &source_path,
std::string &api_path) const
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
if (source_path.empty()) {
@ -236,8 +251,9 @@ auto base_provider::get_api_path_from_source(
return db3_->get_api_path(source_path, api_path);
}
auto base_provider::get_directory_items(
const std::string &api_path, directory_item_list &list) const -> api_error {
auto base_provider::get_directory_items(const std::string &api_path,
directory_item_list &list) const
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
bool exists{};
@ -301,9 +317,10 @@ auto base_provider::get_file_size(const std::string &api_path,
return api_error::success;
}
auto base_provider::get_filesystem_item(
const std::string &api_path, bool directory,
filesystem_item &fsi) const -> api_error {
auto base_provider::get_filesystem_item(const std::string &api_path,
bool directory,
filesystem_item &fsi) const
-> api_error {
bool exists{};
auto res = is_directory(api_path, exists);
if (res != api_error::success) {
@ -336,9 +353,10 @@ auto base_provider::get_filesystem_item(
return api_error::success;
}
auto base_provider::get_filesystem_item_and_file(
const std::string &api_path, api_file &file,
filesystem_item &fsi) const -> api_error {
auto base_provider::get_filesystem_item_and_file(const std::string &api_path,
api_file &file,
filesystem_item &fsi) const
-> api_error {
auto res = get_file(api_path, file);
if (res != api_error::success) {
return res;
@ -394,6 +412,45 @@ auto base_provider::get_pinned_files() const -> std::vector<std::string> {
return db3_->get_pinned_files();
}
void base_provider::get_removed_items(std::deque<removed_item> &directories,
std::deque<removed_item> &files,
const stop_type &stop_requested) const {
for (auto &&api_path : db3_->get_api_path_list()) {
if (stop_requested) {
return;
}
api_meta_map meta{};
if (get_item_meta(api_path, meta) != api_error::success) {
continue;
}
if (utils::string::to_bool(meta[META_DIRECTORY])) {
bool exists{};
if (is_directory(api_path, exists) != api_error::success) {
continue;
}
if (not exists) {
directories.emplace_back(removed_item{api_path, true, ""});
}
continue;
}
bool exists{};
if (is_file(api_path, exists) != api_error::success) {
continue;
}
if (exists) {
continue;
}
files.emplace_back(removed_item{api_path, false, meta[META_SOURCE]});
}
}
auto base_provider::get_total_item_count() const -> std::uint64_t {
return db3_->get_total_item_count();
}
@ -413,108 +470,29 @@ auto base_provider::is_file_writeable(const std::string &api_path) const
return not exists;
}
void base_provider::remove_deleted_files(const stop_type &stop_requested) {
REPERTORY_USES_FUNCTION_NAME();
if (not is_read_only()) {
auto source_list =
utils::file::directory{config_.get_cache_directory()}.get_files();
for (auto &&source_file : source_list) {
if (stop_requested) {
return;
}
filesystem_item fsi{};
if (get_filesystem_item_from_source_path(source_file->get_path(), fsi) !=
api_error::item_not_found) {
continue;
}
auto reference_time =
source_file->get_time(config_.get_eviction_uses_accessed_time()
? utils::file::time_type::accessed
: utils::file::time_type::modified);
if (not reference_time.has_value()) {
continue;
}
auto delay = (config_.get_eviction_delay_mins() * 60UL) *
utils::time::NANOS_PER_SECOND;
if ((reference_time.value() + static_cast<std::uint64_t>(delay)) >=
utils::time::get_time_now()) {
continue;
}
event_system::instance().raise<orphaned_source_file_detected>(
source_file->get_path());
if (not source_file->remove()) {
continue;
}
event_system::instance().raise<orphaned_source_file_removed>(
source_file->get_path());
}
}
struct removed_item final {
std::string api_path;
bool directory{};
std::string source_path;
};
{
api_file_list list{};
std::string marker;
auto res{api_error::more_data};
while (not stop_requested && res == api_error::more_data) {
res = get_file_list(list, marker);
if (res != api_error::success && res != api_error::more_data) {
utils::error::raise_error(function_name, res,
"failed to get file list");
return;
}
}
}
std::vector<removed_item> removed_list{};
for (auto &&api_path : db3_->get_api_path_list()) {
void base_provider::process_removed_directories(
std::deque<removed_item> &removed_list, const stop_type &stop_requested) {
for (auto &&item : removed_list) {
if (stop_requested) {
return;
}
api_meta_map meta{};
if (get_item_meta(api_path, meta) != api_error::success) {
if (not item.directory) {
continue;
}
if (utils::string::to_bool(meta[META_DIRECTORY])) {
bool exists{};
if (is_directory(api_path, exists) != api_error::success) {
continue;
}
if (not exists) {
removed_list.emplace_back(removed_item{api_path, true, ""});
}
continue;
}
bool exists{};
if (is_file(api_path, exists) != api_error::success) {
continue;
}
if (exists) {
continue;
}
removed_list.emplace_back(removed_item{api_path, false, meta[META_SOURCE]});
db3_->remove_api_path(item.api_path);
event_system::instance().raise<directory_removed_externally>(
item.api_path, item.source_path);
}
}
const auto orphaned_directory =
void base_provider::process_removed_files(
std::deque<removed_item> &removed_list, const stop_type &stop_requested) {
REPERTORY_USES_FUNCTION_NAME();
auto orphaned_directory =
utils::path::combine(config_.get_data_directory(), {"orphaned"});
for (auto &&item : removed_list) {
if (stop_requested) {
return;
@ -535,8 +513,8 @@ void base_provider::remove_deleted_files(const stop_type &stop_requested) {
continue;
}
const auto parts = utils::string::split(item.api_path, '/', false);
const auto orphaned_file = utils::path::combine(
auto parts = utils::string::split(item.api_path, '/', false);
auto orphaned_file = utils::path::combine(
orphaned_directory, {utils::path::strip_to_file_name(item.source_path) +
'_' + parts[parts.size() - 1U]});
@ -567,20 +545,32 @@ void base_provider::remove_deleted_files(const stop_type &stop_requested) {
event_system::instance().raise<file_removed_externally>(item.api_path,
item.source_path);
}
}
for (auto &&item : removed_list) {
if (stop_requested) {
return;
}
if (not item.directory) {
continue;
}
db3_->remove_api_path(item.api_path);
event_system::instance().raise<directory_removed_externally>(
item.api_path, item.source_path);
void base_provider::remove_deleted_items(const stop_type &stop_requested) {
add_all_items(stop_requested);
if (stop_requested) {
return;
}
remove_unmatched_source_files(stop_requested);
if (stop_requested) {
return;
}
std::deque<removed_item> directories;
std::deque<removed_item> files;
get_removed_items(directories, files, stop_requested);
if (stop_requested) {
return;
}
process_removed_files(files, stop_requested);
if (stop_requested) {
return;
}
process_removed_directories(directories, stop_requested);
}
auto base_provider::remove_file(const std::string &api_path) -> api_error {
@ -665,6 +655,51 @@ auto base_provider::remove_item_meta(const std::string &api_path,
return db3_->remove_item_meta(api_path, key);
}
void base_provider::remove_unmatched_source_files(
const stop_type &stop_requested) {
if (is_read_only()) {
return;
}
auto source_list =
utils::file::directory{config_.get_cache_directory()}.get_files();
for (auto &&source_file : source_list) {
if (stop_requested) {
return;
}
filesystem_item fsi{};
if (get_filesystem_item_from_source_path(source_file->get_path(), fsi) !=
api_error::item_not_found) {
continue;
}
auto reference_time =
source_file->get_time(config_.get_eviction_uses_accessed_time()
? utils::file::time_type::accessed
: utils::file::time_type::modified);
if (not reference_time.has_value()) {
continue;
}
auto delay = (config_.get_eviction_delay_mins() * 60UL) *
utils::time::NANOS_PER_SECOND;
if ((reference_time.value() + static_cast<std::uint64_t>(delay)) >=
utils::time::get_time_now()) {
continue;
}
event_system::instance().raise<orphaned_source_file_detected>(
source_file->get_path());
if (not source_file->remove()) {
continue;
}
event_system::instance().raise<orphaned_source_file_removed>(
source_file->get_path());
}
}
auto base_provider::set_item_meta(const std::string &api_path,
const std::string &key,
const std::string &value) -> api_error {
@ -715,7 +750,7 @@ auto base_provider::start(api_item_added_callback api_item_added,
polling::instance().set_callback({
"check_deleted",
polling::frequency::low,
[this](auto &&stop_requested) { remove_deleted_files(stop_requested); },
[this](auto &&stop_requested) { remove_deleted_items(stop_requested); },
});
return true;
}