From 72314606f35f5427f5625fd0f90cf9eb77a9bbe0 Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Sun, 12 Nov 2023 11:45:54 -0600 Subject: [PATCH] extract common behavior --- include/providers/base_provider.hpp | 146 +++++++ include/providers/s3/s3_provider.hpp | 74 +--- include/providers/sia/sia_provider.hpp | 77 +--- src/providers/base_provider.cpp | 529 +++++++++++++++++++++++ src/providers/s3/s3_provider.cpp | 541 ++---------------------- src/providers/sia/sia_provider.cpp | 556 ++----------------------- 6 files changed, 753 insertions(+), 1170 deletions(-) create mode 100644 include/providers/base_provider.hpp create mode 100644 src/providers/base_provider.cpp diff --git a/include/providers/base_provider.hpp b/include/providers/base_provider.hpp new file mode 100644 index 00000000..f00733fe --- /dev/null +++ b/include/providers/base_provider.hpp @@ -0,0 +1,146 @@ +/* + Copyright <2018-2023> + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ +#ifndef INCLUDE_PROVIDERS_BASE_PROVIDER_HPP_ +#define INCLUDE_PROVIDERS_BASE_PROVIDER_HPP_ + +#include "providers/i_provider.hpp" +#include "types/repertory.hpp" + +namespace repertory { +class app_config; +class i_file_manager; +class i_http_comm; + +class base_provider : public i_provider { +public: + base_provider(app_config &config, i_http_comm &comm) + : config_(config), comm_(comm) {} + +private: + app_config &config_; + i_http_comm &comm_; + +private: + api_item_added_callback api_item_added_; + std::unique_ptr db_; + std::string DB_NAME = "meta_db"; + i_file_manager *fm_{}; + +private: + void remove_deleted_files(); + +protected: + [[nodiscard]] static auto create_api_file(std::string path, std::string key, + std::uint64_t size) -> api_file; + + [[nodiscard]] static auto create_api_file(std::string path, + std::uint64_t size, + api_meta_map &meta) -> api_file; + + [[nodiscard]] auto get_api_item_added() -> api_item_added_callback & { + return api_item_added_; + } + + [[nodiscard]] auto get_api_item_added() const + -> const api_item_added_callback & { + return api_item_added_; + } + + [[nodiscard]] auto get_comm() const -> i_http_comm & { return comm_; } + + [[nodiscard]] auto get_config() -> app_config & { return config_; } + + [[nodiscard]] auto get_config() const -> const app_config & { + return config_; + } + + [[nodiscard]] auto get_db() const -> rocksdb::DB * { return db_.get(); } + + [[nodiscard]] auto get_file_mgr() -> i_file_manager * { return fm_; } + + [[nodiscard]] auto get_file_mgr() const -> const i_file_manager * { + return fm_; + } + +public: + [[nodiscard]] auto + create_directory_clone_source_meta(const std::string &source_api_path, + const std::string &api_path) + -> api_error override; + + [[nodiscard]] auto get_api_path_from_source(const std::string &source_path, + std::string &api_path) const + -> api_error override; + + [[nodiscard]] auto get_filesystem_item(const std::string &api_path, + bool directory, + filesystem_item &fsi) const + -> api_error override; + + [[nodiscard]] auto get_filesystem_item_and_file(const std::string &api_path, + api_file &f, + filesystem_item &fsi) const + -> api_error override; + + [[nodiscard]] auto + get_filesystem_item_from_source_path(const std::string &source_path, + filesystem_item &fsi) const + -> api_error override; + + [[nodiscard]] auto get_item_meta(const std::string &api_path, + api_meta_map &meta) const + -> api_error override; + + [[nodiscard]] auto get_item_meta(const std::string &api_path, + const std::string &key, + std::string &value) const + -> api_error override; + + [[nodiscard]] auto get_pinned_files() const + -> std::vector override; + + [[nodiscard]] auto get_total_item_count() const -> std::uint64_t override; + + [[nodiscard]] auto is_file_writeable(const std::string &api_path) const + -> bool override; + + [[nodiscard]] auto remove_item_meta(const std::string &api_path, + const std::string &key) + -> api_error override; + + [[nodiscard]] auto set_item_meta(const std::string &api_path, + const std::string &key, + const std::string &value) + -> api_error override; + + [[nodiscard]] auto set_item_meta(const std::string &api_path, + const api_meta_map &meta) + -> api_error override; + + [[nodiscard]] auto start(api_item_added_callback api_item_added, + i_file_manager *mgr) -> bool override; + + void stop() override; +}; +} // namespace repertory + +#endif // INCLUDE_PROVIDERS_BASE_PROVIDER_HPP_ diff --git a/include/providers/s3/s3_provider.hpp b/include/providers/s3/s3_provider.hpp index 2ff78853..377400d5 100644 --- a/include/providers/s3/s3_provider.hpp +++ b/include/providers/s3/s3_provider.hpp @@ -23,7 +23,7 @@ #define INCLUDE_PROVIDERS_S3_S3_PROVIDER_HPP_ #if defined(REPERTORY_ENABLE_S3) -#include "providers/i_provider.hpp" +#include "providers/base_provider.hpp" #include "types/repertory.hpp" namespace repertory { @@ -32,7 +32,7 @@ class i_file_manager; class i_http_comm; struct head_object_result; -class s3_provider final : public i_provider { +class s3_provider final : public base_provider { public: s3_provider(app_config &config, i_http_comm &comm); @@ -44,25 +44,11 @@ public: auto operator=(const s3_provider &) -> s3_provider & = delete; auto operator=(s3_provider &&) -> s3_provider & = delete; -private: - app_config &config_; - i_http_comm &comm_; - -private: - api_item_added_callback api_item_added_; - std::unique_ptr db_; - std::string DB_NAME = "meta_db"; - i_file_manager *fm_{}; - private: [[nodiscard]] auto add_if_not_found(api_file &file, const std::string &object_name) const -> api_error; - [[nodiscard]] static auto create_api_file(const std::string &path, - const std::string &key, - std::uint64_t size) -> api_file; - [[nodiscard]] auto create_path_directories(const std::string &api_path, const std::string &key) const -> api_error; @@ -75,24 +61,13 @@ private: bool &is_encrypted, std::string &object_name, head_object_result &result) const -> api_error; - void remove_deleted_files(); - public: [[nodiscard]] auto create_directory(const std::string &api_path, api_meta_map &meta) -> api_error override; - [[nodiscard]] auto - create_directory_clone_source_meta(const std::string &source_api_path, - const std::string &api_path) - -> api_error override; - [[nodiscard]] auto create_file(const std::string &api_path, api_meta_map &meta) -> api_error override; - [[nodiscard]] auto get_api_path_from_source(const std::string &source_path, - std::string &api_path) const - -> api_error override; - [[nodiscard]] auto get_directory_item_count(const std::string &api_path) const -> std::uint64_t override; @@ -110,37 +85,8 @@ public: std::uint64_t &file_size) const -> api_error override; - [[nodiscard]] auto get_filesystem_item(const std::string &api_path, - bool directory, - filesystem_item &fsi) const - -> api_error override; - - [[nodiscard]] auto get_filesystem_item_and_file(const std::string &api_path, - api_file &file, - filesystem_item &fsi) const - -> api_error override; - - [[nodiscard]] auto - get_filesystem_item_from_source_path(const std::string &source_path, - filesystem_item &fsi) const - -> api_error override; - - [[nodiscard]] auto get_pinned_files() const - -> std::vector override; - - [[nodiscard]] auto get_item_meta(const std::string &api_path, - api_meta_map &meta) const - -> api_error override; - - [[nodiscard]] auto get_item_meta(const std::string &api_path, - const std::string &key, - std::string &value) const - -> api_error override; - [[nodiscard]] auto get_total_drive_space() const -> std::uint64_t override; - [[nodiscard]] auto get_total_item_count() const -> std::uint64_t override; - [[nodiscard]] auto get_provider_type() const -> provider_type override { return provider_type::s3; } @@ -155,9 +101,6 @@ public: [[nodiscard]] auto is_file(const std::string &api_path, bool &exists) const -> api_error override; - [[nodiscard]] auto is_file_writeable(const std::string &api_path) const - -> bool override; - [[nodiscard]] auto is_online() const -> bool override; [[nodiscard]] auto is_rename_supported() const -> bool override { @@ -176,23 +119,10 @@ public: [[nodiscard]] auto remove_file(const std::string &api_path) -> api_error override; - [[nodiscard]] auto remove_item_meta(const std::string &api_path, - const std::string &key) - -> api_error override; - [[nodiscard]] auto rename_file(const std::string &from_api_path, const std::string &to_api_path) -> api_error override; - [[nodiscard]] auto set_item_meta(const std::string &api_path, - const std::string &key, - const std::string &value) - -> api_error override; - - [[nodiscard]] auto set_item_meta(const std::string &api_path, - const api_meta_map &meta) - -> api_error override; - [[nodiscard]] auto start(api_item_added_callback api_item_added, i_file_manager *mgr) -> bool override; diff --git a/include/providers/sia/sia_provider.hpp b/include/providers/sia/sia_provider.hpp index 6acde062..1e0250a5 100644 --- a/include/providers/sia/sia_provider.hpp +++ b/include/providers/sia/sia_provider.hpp @@ -22,7 +22,7 @@ #ifndef INCLUDE_PROVIDERS_SIA_SIA_PROVIDER_HPP_ #define INCLUDE_PROVIDERS_SIA_SIA_PROVIDER_HPP_ -#include "providers/i_provider.hpp" +#include "providers/base_provider.hpp" #include "types/repertory.hpp" namespace repertory { @@ -30,7 +30,7 @@ class app_config; class i_file_manager; class i_http_comm; -class sia_provider : public i_provider { +class sia_provider : public base_provider { public: sia_provider(app_config &config, i_http_comm &comm); @@ -43,50 +43,22 @@ public: auto operator=(sia_provider &&) -> sia_provider & = delete; private: - app_config &config_; - i_http_comm &comm_; - -private: - api_item_added_callback api_item_added_; - std::unique_ptr db_; - std::string DB_NAME = "meta_db"; - i_file_manager *fm_{}; - -private: - [[nodiscard]] static auto create_api_file(std::string path, - std::uint64_t size) -> api_file; - - [[nodiscard]] static auto create_api_file(std::string path, - std::uint64_t size, - api_meta_map &meta) -> api_file; - [[nodiscard]] auto get_object_info(const std::string &api_path, json &object_info) const -> api_error; [[nodiscard]] auto get_object_list(const std::string &api_path, nlohmann::json &object_list) const -> bool; - void remove_deleted_files(); - public: [[nodiscard]] auto create_directory(const std::string &api_path, api_meta_map &meta) -> api_error override; - [[nodiscard]] auto - create_directory_clone_source_meta(const std::string &source_api_path, - const std::string &api_path) - -> api_error override; - [[nodiscard]] auto get_directory_item_count(const std::string &api_path) const -> std::uint64_t override; [[nodiscard]] auto create_file(const std::string &api_path, api_meta_map &meta) -> api_error override; - [[nodiscard]] auto get_api_path_from_source(const std::string &source_path, - std::string &api_path) const - -> api_error override; - [[nodiscard]] auto get_directory_items(const std::string &api_path, directory_item_list &list) const -> api_error override; @@ -101,41 +73,12 @@ public: std::uint64_t &file_size) const -> api_error override; - [[nodiscard]] auto get_filesystem_item(const std::string &api_path, - bool directory, - filesystem_item &fsi) const - -> api_error override; - - [[nodiscard]] auto get_filesystem_item_and_file(const std::string &api_path, - api_file &f, - filesystem_item &fsi) const - -> api_error override; - - [[nodiscard]] auto - get_filesystem_item_from_source_path(const std::string &source_path, - filesystem_item &fsi) const - -> api_error override; - - [[nodiscard]] auto get_item_meta(const std::string &api_path, - api_meta_map &meta) const - -> api_error override; - - [[nodiscard]] auto get_item_meta(const std::string &api_path, - const std::string &key, - std::string &value) const - -> api_error override; - - [[nodiscard]] auto get_pinned_files() const - -> std::vector override; - [[nodiscard]] auto get_provider_type() const -> provider_type override { return provider_type::sia; } [[nodiscard]] auto get_total_drive_space() const -> std::uint64_t override; - [[nodiscard]] auto get_total_item_count() const -> std::uint64_t override; - [[nodiscard]] auto get_used_drive_space() const -> std::uint64_t override; [[nodiscard]] auto is_direct_only() const -> bool override { return false; } @@ -146,9 +89,6 @@ public: [[nodiscard]] auto is_file(const std::string &api_path, bool &exists) const -> api_error override; - [[nodiscard]] auto is_file_writeable(const std::string &api_path) const - -> bool override; - [[nodiscard]] auto is_online() const -> bool override; [[nodiscard]] auto is_rename_supported() const -> bool override { @@ -167,23 +107,10 @@ public: [[nodiscard]] auto remove_file(const std::string &api_path) -> api_error override; - [[nodiscard]] auto remove_item_meta(const std::string &api_path, - const std::string &key) - -> api_error override; - [[nodiscard]] auto rename_file(const std::string &from_api_path, const std::string &to_api_path) -> api_error override; - [[nodiscard]] auto set_item_meta(const std::string &api_path, - const std::string &key, - const std::string &value) - -> api_error override; - - [[nodiscard]] auto set_item_meta(const std::string &api_path, - const api_meta_map &meta) - -> api_error override; - [[nodiscard]] auto start(api_item_added_callback api_item_added, i_file_manager *mgr) -> bool override; diff --git a/src/providers/base_provider.cpp b/src/providers/base_provider.cpp new file mode 100644 index 00000000..fab317c5 --- /dev/null +++ b/src/providers/base_provider.cpp @@ -0,0 +1,529 @@ +/* + Copyright <2018-2023> + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ +#include "providers/base_provider.hpp" + +#include "app_config.hpp" +#include "events/event_system.hpp" +#include "events/events.hpp" +#include "file_manager/i_file_manager.hpp" +#include "utils/file_utils.hpp" +#include "utils/path_utils.hpp" +#include "utils/polling.hpp" +#include "utils/rocksdb_utils.hpp" + +namespace repertory { +auto base_provider::create_api_file(std::string path, std::string key, + std::uint64_t size) -> api_file { + api_file file{}; + file.api_path = utils::path::create_api_path(path); + file.api_parent = utils::path::get_parent_api_path(file.api_path); + file.accessed_date = utils::get_file_time_now(); + file.changed_date = utils::get_file_time_now(); + file.creation_date = utils::get_file_time_now(); + file.modified_date = utils::get_file_time_now(); + file.key = key; + file.file_size = size; + return file; +} + +auto base_provider::create_api_file(std::string path, std::uint64_t size, + api_meta_map &meta) -> api_file { + auto current_size = utils::string::to_uint64(meta[META_SIZE]); + if (current_size == 0U) { + current_size = size; + } + + api_file file{}; + file.api_path = utils::path::create_api_path(path); + file.api_parent = utils::path::get_parent_api_path(file.api_path); + file.accessed_date = utils::string::to_uint64(meta[META_ACCESSED]); + file.changed_date = utils::string::to_uint64(meta[META_CHANGED]); + file.creation_date = utils::string::to_uint64(meta[META_CREATION]); + file.file_size = current_size; + file.modified_date = utils::string::to_uint64(meta[META_MODIFIED]); + return file; +} + +auto base_provider::create_directory_clone_source_meta( + const std::string &source_api_path, const std::string &api_path) + -> api_error { + bool exists{}; + auto res = is_file(source_api_path, exists); + if (res != api_error::success) { + return res; + } + if (exists) { + utils::error::raise_api_path_error(__FUNCTION__, api_path, + api_error::item_exists, + "failed to create directory"); + return api_error::item_exists; + } + + res = is_directory(api_path, exists); + if (res != api_error::success) { + return res; + } + if (exists) { + utils::error::raise_api_path_error(__FUNCTION__, api_path, + api_error::directory_exists, + "failed to create directory"); + return api_error::directory_exists; + } + + res = is_file(api_path, exists); + if (res != api_error::success) { + return res; + } + if (exists) { + utils::error::raise_api_path_error(__FUNCTION__, api_path, + api_error::item_exists, + "failed to create directory"); + return api_error::item_exists; + } + + api_meta_map meta{}; + res = get_item_meta(source_api_path, meta); + if (res != api_error::success) { + if (res == api_error::item_not_found) { + res = api_error::directory_not_found; + } + utils::error::raise_api_path_error(__FUNCTION__, api_path, res, + "failed to create directory"); + return res; + } + + return create_directory(api_path, meta); +} + +auto base_provider::get_api_path_from_source(const std::string &source_path, + std::string &api_path) const + -> api_error { + if (source_path.empty()) { + utils::error::raise_api_path_error(__FUNCTION__, api_path, + api_error::item_not_found, + "failed to source path from api path"); + return api_error::item_not_found; + } + + auto iterator = std::unique_ptr( + db_->NewIterator(rocksdb::ReadOptions())); + for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) { + std::string current_source_path{}; + if (get_item_meta(iterator->key().ToString(), META_SOURCE, + current_source_path) != api_error::success) { + continue; + } + + if (current_source_path.empty()) { + continue; + } + + if (current_source_path == source_path) { + api_path = iterator->key().ToString(); + return api_error::success; + } + } + + return api_error::item_not_found; +} + +auto base_provider::get_filesystem_item(const std::string &api_path, + bool directory, + filesystem_item &fsi) const + -> api_error { + bool exists{}; + auto res = is_directory(api_path, exists); + if (res != api_error::success) { + return res; + } + if (directory && not exists) { + return api_error::directory_not_found; + } + + res = is_file(api_path, exists); + if (res != api_error::success) { + return res; + } + if (not directory && not exists) { + return api_error::item_not_found; + } + + api_meta_map meta{}; + res = get_item_meta(api_path, meta); + if (res != api_error::success) { + return res; + } + + fsi.api_parent = utils::path::get_parent_api_path(api_path); + fsi.api_path = api_path; + fsi.directory = directory; + fsi.size = fsi.directory ? 0U : utils::string::to_uint64(meta[META_SIZE]); + fsi.source_path = meta[META_SOURCE]; + + return api_error::success; +} + +auto base_provider::get_filesystem_item_and_file(const std::string &api_path, + api_file &file, + filesystem_item &fsi) const + -> api_error { + auto res = get_file(api_path, file); + if (res != api_error::success) { + return res; + } + + api_meta_map meta{}; + res = get_item_meta(api_path, meta); + if (res != api_error::success) { + return res; + } + + fsi.api_parent = utils::path::get_parent_api_path(api_path); + fsi.api_path = api_path; + fsi.directory = false; + fsi.size = utils::string::to_uint64(meta[META_SIZE]); + fsi.source_path = meta[META_SOURCE]; + + return api_error::success; +} + +auto base_provider::get_filesystem_item_from_source_path( + const std::string &source_path, filesystem_item &fsi) const -> api_error { + std::string api_path{}; + auto res = get_api_path_from_source(source_path, api_path); + if (res != api_error::success) { + return res; + } + + bool exists{}; + res = is_directory(api_path, exists); + if (res != api_error::success) { + return res; + } + if (exists) { + return api_error::directory_exists; + } + + return get_filesystem_item(api_path, false, fsi); +} + +auto base_provider::get_item_meta(const std::string &api_path, + api_meta_map &meta) const -> api_error { + std::string meta_value{}; + db_->Get(rocksdb::ReadOptions(), api_path, &meta_value); + if (meta_value.empty()) { + return api_error::item_not_found; + } + + try { + meta = json::parse(meta_value).get(); + + return api_error::success; + } catch (const std::exception &e) { + utils::error::raise_api_path_error(__FUNCTION__, api_path, e, + "failed to get item meta"); + } + + return api_error::error; +} + +auto base_provider::get_item_meta(const std::string &api_path, + const std::string &key, + std::string &value) const -> api_error { + std::string meta_value{}; + db_->Get(rocksdb::ReadOptions(), api_path, &meta_value); + if (meta_value.empty()) { + return api_error::item_not_found; + } + + try { + value = json::parse(meta_value)[key]; + return api_error::success; + } catch (const std::exception &e) { + utils::error::raise_api_path_error(__FUNCTION__, api_path, e, + "failed to get item meta"); + } + + return api_error::error; +} + +auto base_provider::get_pinned_files() const -> std::vector { + std::vector ret{}; + + auto iterator = std::unique_ptr( + db_->NewIterator(rocksdb::ReadOptions())); + for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) { + std::string pinned{}; + if (get_item_meta(iterator->key().ToString(), META_PINNED, pinned) != + api_error::success) { + continue; + } + if (pinned.empty() || not utils::string::to_bool(pinned)) { + continue; + } + ret.emplace_back(iterator->key().ToString()); + } + + return ret; +} + +auto base_provider::get_total_item_count() const -> std::uint64_t { + std::uint64_t ret{}; + auto iterator = std::unique_ptr( + db_->NewIterator(rocksdb::ReadOptions())); + for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) { + ret++; + } + return ret; +} + +auto base_provider::is_file_writeable(const std::string &api_path) const + -> bool { + bool exists{}; + auto res = is_directory(api_path, exists); + if (res != api_error::success) { + return false; + } + + return not exists; +} + +void base_provider::remove_deleted_files() { + struct removed_item { + std::string api_path{}; + bool directory{}; + std::string source_path{}; + }; + + api_file_list list{}; + auto res = get_file_list(list); + if (res != api_error::success) { + utils::error::raise_error(__FUNCTION__, res, "failed to get file list"); + return; + } + + std::vector removed_list{}; + auto iterator = std::unique_ptr( + db_->NewIterator(rocksdb::ReadOptions())); + for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) { + api_meta_map meta{}; + if (get_item_meta(iterator->key().ToString(), meta) == api_error::success) { + if (utils::string::to_bool(meta[META_DIRECTORY])) { + bool exists{}; + if (is_directory(iterator->key().ToString(), exists) != + api_error::success) { + continue; + } + if (not exists) { + removed_list.emplace_back( + removed_item{iterator->key().ToString(), true, ""}); + } + continue; + } + + bool exists{}; + if (is_file(iterator->key().ToString(), exists) != api_error::success) { + continue; + } + if (not exists) { + removed_list.emplace_back( + removed_item{iterator->key().ToString(), false, meta[META_SOURCE]}); + } + } + } + + for (const auto &item : removed_list) { + if (not item.directory) { + if (utils::file::is_file(item.source_path)) { + const auto orphaned_directory = + utils::path::combine(config_.get_data_directory(), {"orphaned"}); + if (utils::file::create_full_directory_path(orphaned_directory)) { + const auto parts = utils::string::split(item.api_path, '/', false); + const auto orphaned_file = utils::path::combine( + orphaned_directory, + {utils::path::strip_to_file_name(item.source_path) + '_' + + parts[parts.size() - 1U]}); + + event_system::instance().raise( + item.source_path); + if (utils::file::reset_modified_time(item.source_path) && + utils::file::copy_file(item.source_path, orphaned_file)) { + event_system::instance().raise( + item.source_path, orphaned_file); + } else { + event_system::instance().raise( + item.source_path, orphaned_file, + std::to_string(utils::get_last_error_code())); + } + } else { + utils::error::raise_error( + __FUNCTION__, std::to_string(utils::get_last_error_code()), + "failed to create orphaned director|sp|" + orphaned_directory); + continue; + } + } + + if (fm_->evict_file(item.api_path)) { + db_->Delete(rocksdb::WriteOptions(), item.api_path); + event_system::instance().raise( + item.api_path, item.source_path); + } + } + } + + for (const auto &item : removed_list) { + if (item.directory) { + db_->Delete(rocksdb::WriteOptions(), item.api_path); + event_system::instance().raise( + item.api_path, item.source_path); + } + } +} + +auto base_provider::remove_item_meta(const std::string &api_path, + const std::string &key) -> api_error { + api_meta_map meta{}; + auto res = get_item_meta(api_path, meta); + if (res != api_error::success) { + return res; + } + + meta.erase(key); + + auto res2 = db_->Put(rocksdb::WriteOptions(), api_path, json(meta).dump()); + if (not res2.ok()) { + utils::error::raise_api_path_error(__FUNCTION__, api_path, res2.code(), + "failed to remove item meta"); + return api_error::error; + } + + return api_error::success; +} + +auto base_provider::set_item_meta(const std::string &api_path, + const std::string &key, + const std::string &value) -> api_error { + json meta_json{}; + std::string meta_value{}; + db_->Get(rocksdb::ReadOptions(), api_path, &meta_value); + if (not meta_value.empty()) { + try { + meta_json = json::parse(meta_value); + } catch (const std::exception &e) { + utils::error::raise_api_path_error(__FUNCTION__, api_path, e, + "failed to set item meta"); + return api_error::error; + } + } + + meta_json[key] = value; + + const auto res = + db_->Put(rocksdb::WriteOptions(), api_path, meta_json.dump()); + if (not res.ok()) { + utils::error::raise_api_path_error(__FUNCTION__, api_path, res.code(), + "failed to set item meta"); + return api_error::error; + } + + return api_error::success; +} + +auto base_provider::set_item_meta(const std::string &api_path, + const api_meta_map &meta) -> api_error { + json meta_json{}; + std::string meta_value{}; + db_->Get(rocksdb::ReadOptions(), api_path, &meta_value); + if (not meta_value.empty()) { + try { + meta_json = json::parse(meta_value); + } catch (const std::exception &e) { + utils::error::raise_api_path_error(__FUNCTION__, api_path, e, + "failed to set item meta"); + return api_error::error; + } + } + + for (const auto &kv : meta) { + meta_json[kv.first] = kv.second; + } + + const auto res = + db_->Put(rocksdb::WriteOptions(), api_path, meta_json.dump()); + if (not res.ok()) { + utils::error::raise_api_path_error(__FUNCTION__, api_path, res.code(), + "failed to set item meta"); + return api_error::error; + } + + return api_error::success; +} + +auto base_provider::start(api_item_added_callback api_item_added, + i_file_manager *mgr) -> bool { + event_system::instance().raise("sia_provider"); + utils::db::create_rocksdb(config_, DB_NAME, db_); + + api_item_added_ = api_item_added; + fm_ = mgr; + + api_meta_map meta{}; + if (get_item_meta("/", meta) == api_error::item_not_found) { + auto dir = create_api_file("/", "", 0U); + api_item_added_(true, dir); + } + + auto online = false; + auto unmount_requested = false; + { + repertory::event_consumer consumer( + "unmount_requested", + [&unmount_requested](const event &) { unmount_requested = true; }); + for (std::uint16_t i = 0U; not online && not unmount_requested && + (i < config_.get_online_check_retry_secs()); + i++) { + online = is_online(); + if (not online) { + event_system::instance().raise( + config_.get_host_config().host_name_or_ip, + config_.get_host_config().api_port); + std::this_thread::sleep_for(1s); + } + } + } + + if (online && not unmount_requested) { + polling::instance().set_callback({"check_deleted", polling::frequency::low, + [this]() { remove_deleted_files(); }}); + return true; + } + + return false; +} + +void base_provider::stop() { + event_system::instance().raise("sia_provider"); + polling::instance().remove_callback("check_deleted"); + db_.reset(); + event_system::instance().raise("sia_provider"); +} +} // namespace repertory diff --git a/src/providers/s3/s3_provider.cpp b/src/providers/s3/s3_provider.cpp index 800122da..bda8eaf6 100644 --- a/src/providers/s3/s3_provider.cpp +++ b/src/providers/s3/s3_provider.cpp @@ -69,8 +69,8 @@ get_object_list(i_http_comm &client, const s3_config &config, } // namespace s3_provider::s3_provider(app_config &config, i_http_comm &comm) - : config_(config), comm_(comm) { - comm_.enable_s3_path_style(config.get_s3_config().use_path_style); + : base_provider(config, comm) { + get_comm().enable_s3_path_style(config.get_s3_config().use_path_style); } auto s3_provider::add_if_not_found(api_file &file, @@ -84,27 +84,12 @@ auto s3_provider::add_if_not_found(api_file &file, return err; } - api_item_added_(false, file); + get_api_item_added()(false, file); } return api_error::success; } -auto s3_provider::create_api_file(const std::string &path, - const std::string &key, std::uint64_t size) - -> api_file { - api_file file{}; - file.api_path = utils::path::create_api_path(path); - file.api_parent = utils::path::get_parent_api_path(file.api_path); - file.accessed_date = utils::get_file_time_now(); - file.changed_date = utils::get_file_time_now(); - file.creation_date = utils::get_file_time_now(); - file.file_size = size; - file.key = key; - file.modified_date = utils::get_file_time_now(); - return file; -} - auto s3_provider::create_directory(const std::string &api_path, api_meta_map &meta) -> api_error { @@ -132,7 +117,7 @@ auto s3_provider::create_directory(const std::string &api_path, } try { - const auto cfg = config_.get_s3_config(); + const auto cfg = get_config().get_s3_config(); const auto is_encrypted = not cfg.encryption_token.empty(); stop_type stop_requested{false}; @@ -165,7 +150,7 @@ auto s3_provider::create_directory(const std::string &api_path, put_file.path = object_name + '/'; long response_code{}; - if (not comm_.make_request(put_file, response_code, stop_requested)) { + if (not get_comm().make_request(put_file, response_code, stop_requested)) { utils::error::raise_api_path_error(__FUNCTION__, api_path, api_error::comm_error, "failed to create directory"); @@ -187,57 +172,6 @@ auto s3_provider::create_directory(const std::string &api_path, return set_item_meta(api_path, meta); } -auto s3_provider::create_directory_clone_source_meta( - const std::string &source_api_path, const std::string &api_path) - -> api_error { - bool exists{}; - auto res = is_file(source_api_path, exists); - if (res != api_error::success) { - return res; - } - if (exists) { - utils::error::raise_api_path_error(__FUNCTION__, api_path, - api_error::item_exists, - "failed to create directory"); - return api_error::item_exists; - } - - res = is_directory(api_path, exists); - if (res != api_error::success) { - return res; - } - if (exists) { - utils::error::raise_api_path_error(__FUNCTION__, api_path, - api_error::directory_exists, - "failed to create directory"); - return api_error::directory_exists; - } - - res = is_file(api_path, exists); - if (res != api_error::success) { - return res; - } - if (exists) { - utils::error::raise_api_path_error(__FUNCTION__, api_path, - api_error::item_exists, - "failed to create directory"); - return api_error::item_exists; - } - - api_meta_map meta{}; - res = get_item_meta(source_api_path, meta); - if (res != api_error::success) { - if (res == api_error::item_not_found) { - res = api_error::directory_not_found; - } - utils::error::raise_api_path_error(__FUNCTION__, api_path, res, - "failed to create directory"); - return res; - } - - return create_directory(api_path, meta); -} - auto s3_provider::create_file(const std::string &api_path, api_meta_map &meta) -> api_error { bool exists{}; @@ -265,7 +199,7 @@ auto s3_provider::create_file(const std::string &api_path, api_meta_map &meta) stop_type stop_requested{false}; try { - if (not config_.get_s3_config().encryption_token.empty()) { + if (not get_config().get_s3_config().encryption_token.empty()) { std::string encrypted_file_path; res = get_item_meta(utils::path::get_parent_api_path(api_path), META_KEY, encrypted_file_path); @@ -277,7 +211,7 @@ auto s3_provider::create_file(const std::string &api_path, api_meta_map &meta) data_buffer result; utils::encryption::encrypt_data( - config_.get_s3_config().encryption_token, + get_config().get_s3_config().encryption_token, *(utils::string::split(api_path, '/', false).end() - 1U), result); meta[META_KEY] = utils::path::create_api_path(utils::path::combine( @@ -295,7 +229,7 @@ auto s3_provider::create_file(const std::string &api_path, api_meta_map &meta) res = upload_file(api_path, meta[META_SOURCE], stop_requested); if (res != api_error::success) { - db_->Delete(rocksdb::WriteOptions(), api_path); + get_db()->Delete(rocksdb::WriteOptions(), api_path); } return res; @@ -314,7 +248,7 @@ auto s3_provider::create_path_directories(const std::string &api_path, return api_error::success; } - const auto encryption_token = config_.get_s3_config().encryption_token; + const auto encryption_token = get_config().get_s3_config().encryption_token; const auto is_encrypted = not encryption_token.empty(); const auto path_parts = utils::string::split(api_path, '/', false); @@ -338,7 +272,7 @@ auto s3_provider::create_path_directories(const std::string &api_path, auto res = get_item_meta(cur_path, meta); if (res == api_error::item_not_found) { auto dir = create_api_file(cur_path, cur_key, 0U); - api_item_added_(true, dir); + get_api_item_added()(true, dir); continue; } @@ -355,7 +289,7 @@ auto s3_provider::decrypt_object_name(std::string &object_name) const auto parts = utils::string::split(object_name, '/', false); for (auto &part : parts) { auto err = utils::encryption::decrypt_file_name( - config_.get_s3_config().encryption_token, part); + get_config().get_s3_config().encryption_token, part); if (err != api_error::success) { return err; } @@ -365,42 +299,10 @@ auto s3_provider::decrypt_object_name(std::string &object_name) const return api_error::success; } -auto s3_provider::get_api_path_from_source(const std::string &source_path, - std::string &api_path) const - -> api_error { - if (source_path.empty()) { - utils::error::raise_api_path_error(__FUNCTION__, api_path, - api_error::item_not_found, - "failed to source path from api path"); - return api_error::item_not_found; - } - - auto iterator = std::unique_ptr( - db_->NewIterator(rocksdb::ReadOptions())); - for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) { - std::string current_source_path{}; - if (get_item_meta(iterator->key().ToString(), META_SOURCE, - current_source_path) != api_error::success) { - continue; - } - - if (current_source_path.empty()) { - continue; - } - - if (current_source_path == source_path) { - api_path = iterator->key().ToString(); - return api_error::success; - } - } - - return api_error::item_not_found; -} - auto s3_provider::get_directory_item_count(const std::string &api_path) const -> std::uint64_t { try { - const auto cfg = config_.get_s3_config(); + const auto cfg = get_config().get_s3_config(); const auto is_encrypted = not cfg.encryption_token.empty(); std::string key; if (is_encrypted) { @@ -419,8 +321,8 @@ auto s3_provider::get_directory_item_count(const std::string &api_path) const long response_code{}; auto prefix = object_name.empty() ? object_name : object_name + "/"; - if (not get_object_list(comm_, config_.get_s3_config(), response_data, - response_code, "/", prefix)) { + if (not get_object_list(get_comm(), get_config().get_s3_config(), + response_data, response_code, "/", prefix)) { return 0U; } @@ -460,7 +362,7 @@ auto s3_provider::get_directory_items(const std::string &api_path, directory_item_list &list) const -> api_error { try { - const auto cfg = config_.get_s3_config(); + const auto cfg = get_config().get_s3_config(); const auto is_encrypted = not cfg.encryption_token.empty(); std::string key; if (is_encrypted) { @@ -479,8 +381,8 @@ auto s3_provider::get_directory_items(const std::string &api_path, long response_code{}; auto prefix = object_name.empty() ? object_name : object_name + "/"; - if (not get_object_list(comm_, config_.get_s3_config(), response_data, - response_code, "/", prefix)) { + if (not get_object_list(get_comm(), get_config().get_s3_config(), + response_data, response_code, "/", prefix)) { return api_error::comm_error; } @@ -629,8 +531,8 @@ auto s3_provider::get_file(const std::string &api_path, api_file &file) const auto s3_provider::get_file_list(api_file_list &list) const -> api_error { std::string response_data; long response_code{}; - if (not get_object_list(comm_, config_.get_s3_config(), response_data, - response_code)) { + if (not get_object_list(get_comm(), get_config().get_s3_config(), + response_data, response_code)) { return api_error::comm_error; } @@ -649,7 +551,8 @@ auto s3_provider::get_file_list(api_file_list &list) const -> api_error { auto api_path = std::string{node.node().select_node("Key").node().text().as_string()}; if (not utils::string::ends_with(api_path, "/")) { - auto is_encrypted = not config_.get_s3_config().encryption_token.empty(); + auto is_encrypted = + not get_config().get_s3_config().encryption_token.empty(); if (is_encrypted) { auto err = decrypt_object_name(api_path); if (err != api_error::success) { @@ -707,91 +610,12 @@ auto s3_provider::get_file_size(const std::string &api_path, return api_error::success; } -auto s3_provider::get_filesystem_item(const std::string &api_path, - bool directory, - filesystem_item &fsi) const -> api_error { - bool exists{}; - auto res = is_directory(api_path, exists); - if (res != api_error::success) { - return res; - } - if (directory && not exists) { - return api_error::directory_not_found; - } - - res = is_file(api_path, exists); - if (res != api_error::success) { - return res; - } - if (not directory && not exists) { - return api_error::item_not_found; - } - - api_meta_map meta{}; - res = get_item_meta(api_path, meta); - if (res != api_error::success) { - return res; - } - - fsi.api_parent = utils::path::get_parent_api_path(api_path); - fsi.api_path = api_path; - fsi.directory = directory; - fsi.size = fsi.directory ? 0U : utils::string::to_uint64(meta[META_SIZE]); - fsi.source_path = meta[META_SOURCE]; - - return api_error::success; -} - -auto s3_provider::get_filesystem_item_and_file(const std::string &api_path, - api_file &file, - filesystem_item &fsi) const - -> api_error { - auto res = get_file(api_path, file); - if (res != api_error::success) { - return res; - } - - api_meta_map meta{}; - res = get_item_meta(api_path, meta); - if (res != api_error::success) { - return res; - } - - fsi.api_parent = utils::path::get_parent_api_path(api_path); - fsi.api_path = api_path; - fsi.directory = false; - fsi.size = utils::string::to_uint64(meta[META_SIZE]); - fsi.source_path = meta[META_SOURCE]; - - return api_error::success; -} - -auto s3_provider::get_filesystem_item_from_source_path( - const std::string &source_path, filesystem_item &fsi) const -> api_error { - std::string api_path{}; - auto res = get_api_path_from_source(source_path, api_path); - if (res != api_error::success) { - return res; - } - - bool exists{}; - res = is_directory(api_path, exists); - if (res != api_error::success) { - return res; - } - if (exists) { - return api_error::directory_exists; - } - - return get_filesystem_item(api_path, false, fsi); -} - auto s3_provider::get_object_info(bool directory, const std::string &api_path, bool &is_encrypted, std::string &object_name, head_object_result &result) const -> api_error { try { - const auto cfg = config_.get_s3_config(); + const auto cfg = get_config().get_s3_config(); is_encrypted = not cfg.encryption_token.empty(); std::string key; @@ -812,7 +636,7 @@ auto s3_provider::get_object_info(bool directory, const std::string &api_path, stop_type stop_requested{false}; long response_code{}; - if (not comm_.make_request(head, response_code, stop_requested)) { + if (not get_comm().make_request(head, response_code, stop_requested)) { return api_error::comm_error; } @@ -834,85 +658,15 @@ auto s3_provider::get_object_info(bool directory, const std::string &api_path, return api_error::error; } -auto s3_provider::get_pinned_files() const -> std::vector { - std::vector ret{}; - - auto iterator = std::unique_ptr( - db_->NewIterator(rocksdb::ReadOptions())); - for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) { - std::string pinned{}; - if (get_item_meta(iterator->key().ToString(), META_PINNED, pinned) != - api_error::success) { - continue; - } - if (pinned.empty() || not utils::string::to_bool(pinned)) { - continue; - } - ret.emplace_back(iterator->key().ToString()); - } - - return ret; -} - -auto s3_provider::get_item_meta(const std::string &api_path, - api_meta_map &meta) const -> api_error { - std::string meta_value{}; - db_->Get(rocksdb::ReadOptions(), api_path, &meta_value); - if (meta_value.empty()) { - return api_error::item_not_found; - } - - try { - meta = json::parse(meta_value).get(); - - return api_error::success; - } catch (const std::exception &e) { - utils::error::raise_api_path_error(__FUNCTION__, api_path, e, - "failed to get item meta"); - } - - return api_error::error; -} - -auto s3_provider::get_item_meta(const std::string &api_path, - const std::string &key, - std::string &value) const -> api_error { - std::string meta_value{}; - db_->Get(rocksdb::ReadOptions(), api_path, &meta_value); - if (meta_value.empty()) { - return api_error::item_not_found; - } - - try { - value = json::parse(meta_value)[key]; - return api_error::success; - } catch (const std::exception &e) { - utils::error::raise_api_path_error(__FUNCTION__, api_path, e, - "failed to get item meta"); - } - - return api_error::error; -} - auto s3_provider::get_total_drive_space() const -> std::uint64_t { return std::numeric_limits::max() / std::int64_t(2); } -auto s3_provider::get_total_item_count() const -> std::uint64_t { - std::uint64_t ret{}; - auto iterator = std::unique_ptr( - db_->NewIterator(rocksdb::ReadOptions())); - for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) { - ret++; - } - return ret; -} - auto s3_provider::get_used_drive_space() const -> std::uint64_t { std::string response_data; long response_code{}; - if (not get_object_list(comm_, config_.get_s3_config(), response_data, - response_code)) { + if (not get_object_list(get_comm(), get_config().get_s3_config(), + response_data, response_code)) { return 0U; } @@ -926,7 +680,7 @@ auto s3_provider::get_used_drive_space() const -> std::uint64_t { return 0U; } - const auto cfg = config_.get_s3_config(); + const auto cfg = get_config().get_s3_config(); const auto is_encrypted = not cfg.encryption_token.empty(); auto node_list = doc.select_nodes("/ListBucketResult/Contents"); @@ -991,16 +745,6 @@ auto s3_provider::is_file(const std::string &api_path, bool &exists) const return api_error::error; } -auto s3_provider::is_file_writeable(const std::string &api_path) const -> bool { - bool exists{}; - auto res = is_directory(api_path, exists); - if (res != api_error::success) { - return false; - } - - return not exists; -} - auto s3_provider::is_online() const -> bool { // TODO implement this return true; @@ -1010,7 +754,7 @@ auto s3_provider::read_file_bytes(const std::string &api_path, std::size_t size, std::uint64_t offset, data_buffer &data, stop_type &stop_requested) -> api_error { try { - const auto cfg = config_.get_s3_config(); + const auto cfg = get_config().get_s3_config(); const auto is_encrypted = not cfg.encryption_token.empty(); std::string key; if (is_encrypted) { @@ -1030,7 +774,7 @@ auto s3_provider::read_file_bytes(const std::string &api_path, std::size_t size, auto res = api_error::error; for (std::uint32_t i = 0U; not stop_requested && res != api_error::success && - i < config_.get_retry_read_count() + 1U; + i < get_config().get_retry_read_count() + 1U; i++) { curl::requests::http_get get{}; get.aws_service = "aws:amz:" + cfg.region + ":s3"; @@ -1063,7 +807,7 @@ auto s3_provider::read_file_bytes(const std::string &api_path, std::size_t size, std::this_thread::sleep_for(1s); }; - if (not comm_.make_request(get, response_code, stop_requested)) { + if (not get_comm().make_request(get, response_code, stop_requested)) { notify_retry(); continue; } @@ -1107,98 +851,6 @@ auto s3_provider::read_file_bytes(const std::string &api_path, std::size_t size, return api_error::error; } -void s3_provider::remove_deleted_files() { - struct removed_item { - std::string api_path{}; - bool directory{}; - std::string source_path{}; - }; - - api_file_list list{}; - auto res = get_file_list(list); - if (res != api_error::success) { - utils::error::raise_error(__FUNCTION__, res, "failed to get file list"); - return; - } - - std::vector removed_list{}; - auto iterator = std::unique_ptr( - db_->NewIterator(rocksdb::ReadOptions())); - for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) { - api_meta_map meta{}; - if (get_item_meta(iterator->key().ToString(), meta) == api_error::success) { - if (utils::string::to_bool(meta[META_DIRECTORY])) { - bool exists{}; - if (is_directory(iterator->key().ToString(), exists) != - api_error::success) { - continue; - } - if (not exists) { - removed_list.emplace_back( - removed_item{iterator->key().ToString(), true, ""}); - } - continue; - } - - bool exists{}; - if (is_file(iterator->key().ToString(), exists) != api_error::success) { - continue; - } - if (not exists) { - removed_list.emplace_back( - removed_item{iterator->key().ToString(), false, meta[META_SOURCE]}); - } - } - } - - for (const auto &item : removed_list) { - if (not item.directory) { - if (utils::file::is_file(item.source_path)) { - const auto orphaned_directory = - utils::path::combine(config_.get_data_directory(), {"orphaned"}); - if (utils::file::create_full_directory_path(orphaned_directory)) { - const auto parts = utils::string::split(item.api_path, '/', false); - const auto orphaned_file = utils::path::combine( - orphaned_directory, - {utils::path::strip_to_file_name(item.source_path) + '_' + - parts[parts.size() - 1U]}); - - event_system::instance().raise( - item.source_path); - if (utils::file::reset_modified_time(item.source_path) && - utils::file::copy_file(item.source_path, orphaned_file)) { - event_system::instance().raise( - item.source_path, orphaned_file); - } else { - event_system::instance().raise( - item.source_path, orphaned_file, - std::to_string(utils::get_last_error_code())); - } - } else { - utils::error::raise_error( - __FUNCTION__, std::to_string(utils::get_last_error_code()), - "failed to create orphaned director|sp|" + orphaned_directory); - continue; - } - } - - if (fm_->evict_file(item.api_path)) { - db_->Delete(rocksdb::WriteOptions(), item.api_path); - event_system::instance().raise( - item.api_path, item.source_path); - } - } - } - - for (const auto &item : removed_list) { - if (item.directory) { - db_->Delete(rocksdb::WriteOptions(), item.api_path); - event_system::instance().raise( - item.api_path, item.source_path); - } - } -} - auto s3_provider::remove_directory(const std::string &api_path) -> api_error { const auto notify_end = [&api_path](api_error error) -> api_error { if (error == api_error::success) { @@ -1219,7 +871,7 @@ auto s3_provider::remove_directory(const std::string &api_path) -> api_error { return notify_end(api_error::item_not_found); } - const auto cfg = config_.get_s3_config(); + const auto cfg = get_config().get_s3_config(); const auto is_encrypted = not cfg.encryption_token.empty(); std::string key; @@ -1240,7 +892,7 @@ auto s3_provider::remove_directory(const std::string &api_path) -> api_error { long response_code{}; stop_type stop_requested{}; - if (not comm_.make_request(del, response_code, stop_requested)) { + if (not get_comm().make_request(del, response_code, stop_requested)) { utils::error::raise_api_path_error(__FUNCTION__, api_path, api_error::comm_error, "failed to remove directory"); @@ -1255,7 +907,7 @@ auto s3_provider::remove_directory(const std::string &api_path) -> api_error { return notify_end(api_error::comm_error); } - auto status = db_->Delete(rocksdb::WriteOptions(), api_path); + auto status = get_db()->Delete(rocksdb::WriteOptions(), api_path); if (not status.ok()) { utils::error::raise_api_path_error(__FUNCTION__, api_path, status.code(), "failed to remove directory"); @@ -1284,7 +936,7 @@ auto s3_provider::remove_file(const std::string &api_path) -> api_error { api_meta_map meta{}; auto res = get_item_meta(api_path, meta); - auto res2 = db_->Delete(rocksdb::WriteOptions(), api_path); + auto res2 = get_db()->Delete(rocksdb::WriteOptions(), api_path); if (not res2.ok()) { utils::error::raise_api_path_error(function_name, api_path, res2.code(), "failed to remove file"); @@ -1314,7 +966,7 @@ auto s3_provider::remove_file(const std::string &api_path) -> api_error { return remove_file_meta(); } - const auto cfg = config_.get_s3_config(); + const auto cfg = get_config().get_s3_config(); const auto is_encrypted = not cfg.encryption_token.empty(); std::string key; @@ -1335,7 +987,7 @@ auto s3_provider::remove_file(const std::string &api_path) -> api_error { long response_code{}; stop_type stop_requested{}; - if (not comm_.make_request(del, response_code, stop_requested)) { + if (not get_comm().make_request(del, response_code, stop_requested)) { utils::error::raise_api_path_error( __FUNCTION__, api_path, api_error::comm_error, "failed to remove file"); return notify_end(api_error::comm_error); @@ -1352,136 +1004,21 @@ auto s3_provider::remove_file(const std::string &api_path) -> api_error { return remove_file_meta(); } -auto s3_provider::remove_item_meta(const std::string &api_path, - const std::string &key) -> api_error { - api_meta_map meta{}; - auto res = get_item_meta(api_path, meta); - if (res != api_error::success) { - return res; - } - - meta.erase(key); - - auto status = db_->Put(rocksdb::WriteOptions(), api_path, json(meta).dump()); - if (not status.ok()) { - utils::error::raise_api_path_error(__FUNCTION__, api_path, status.code(), - "failed to remove item meta"); - return api_error::error; - } - - return api_error::success; -} - auto s3_provider::rename_file(const std::string & /* from_api_path */, const std::string & /* to_api_path */) -> api_error { return api_error::not_implemented; } -auto s3_provider::set_item_meta(const std::string &api_path, - const std::string &key, - const std::string &value) -> api_error { - json meta_json{}; - std::string meta_value{}; - db_->Get(rocksdb::ReadOptions(), api_path, &meta_value); - if (not meta_value.empty()) { - try { - meta_json = json::parse(meta_value); - } catch (const std::exception &e) { - utils::error::raise_api_path_error(__FUNCTION__, api_path, e, - "failed to set item meta"); - return api_error::error; - } - } - - meta_json[key] = value; - - auto status = db_->Put(rocksdb::WriteOptions(), api_path, meta_json.dump()); - if (not status.ok()) { - utils::error::raise_api_path_error(__FUNCTION__, api_path, status.code(), - "failed to set item meta"); - return api_error::error; - } - - return api_error::success; -} - -auto s3_provider::set_item_meta(const std::string &api_path, - const api_meta_map &meta) -> api_error { - json meta_json{}; - std::string meta_value{}; - db_->Get(rocksdb::ReadOptions(), api_path, &meta_value); - if (not meta_value.empty()) { - try { - meta_json = json::parse(meta_value); - } catch (const std::exception &e) { - utils::error::raise_api_path_error(__FUNCTION__, api_path, e, - "failed to set item meta"); - return api_error::error; - } - } - - for (const auto &item : meta) { - meta_json[item.first] = item.second; - } - - auto status = db_->Put(rocksdb::WriteOptions(), api_path, meta_json.dump()); - if (not status.ok()) { - utils::error::raise_api_path_error(__FUNCTION__, api_path, status.code(), - "failed to set item meta"); - return api_error::error; - } - - return api_error::success; -} - auto s3_provider::start(api_item_added_callback api_item_added, i_file_manager *mgr) -> bool { event_system::instance().raise("s3_provider"); - utils::db::create_rocksdb(config_, DB_NAME, db_); - - api_item_added_ = api_item_added; - fm_ = mgr; - - api_meta_map meta{}; - if (get_item_meta("/", meta) == api_error::item_not_found) { - auto dir = create_api_file("/", "", 0U); - api_item_added_(true, dir); - } - - auto online{false}; - auto unmount_requested{false}; - { - repertory::event_consumer consumer( - "unmount_requested", - [&unmount_requested](const event &) { unmount_requested = true; }); - for (std::uint16_t i = 0U; not online && not unmount_requested && - (i < config_.get_online_check_retry_secs()); - i++) { - online = is_online(); - if (not online) { - auto cfg = curl_comm::create_host_config( - config_.get_s3_config(), config_.get_s3_config().use_path_style); - event_system::instance().raise(cfg.host_name_or_ip, - cfg.api_port); - std::this_thread::sleep_for(1s); - } - } - } - - if (online && not unmount_requested) { - polling::instance().set_callback({"check_deleted", polling::frequency::low, - [this]() { remove_deleted_files(); }}); - return true; - } - - return false; + return base_provider::start(api_item_added, mgr); } void s3_provider::stop() { event_system::instance().raise("s3_provider"); - polling::instance().remove_callback("check_deleted"); - db_.reset(); + base_provider::stop(); event_system::instance().raise("s3_provider"); } @@ -1503,7 +1040,7 @@ auto s3_provider::upload_file(const std::string &api_path, return notify_end(api_error::comm_error); } - const auto cfg = config_.get_s3_config(); + const auto cfg = get_config().get_s3_config(); const auto is_encrypted = not cfg.encryption_token.empty(); std::string key; @@ -1530,7 +1067,7 @@ auto s3_provider::upload_file(const std::string &api_path, } long response_code{}; - if (not comm_.make_request(put_file, response_code, stop_requested)) { + if (not get_comm().make_request(put_file, response_code, stop_requested)) { return notify_end(api_error::comm_error); } diff --git a/src/providers/sia/sia_provider.cpp b/src/providers/sia/sia_provider.cpp index b07cb1cd..980ad4e6 100644 --- a/src/providers/sia/sia_provider.cpp +++ b/src/providers/sia/sia_provider.cpp @@ -25,6 +25,7 @@ #include "comm/i_http_comm.hpp" #include "events/events.hpp" #include "file_manager/i_file_manager.hpp" +#include "providers/base_provider.hpp" #include "types/repertory.hpp" #include "utils/error_utils.hpp" #include "utils/file_utils.hpp" @@ -36,7 +37,7 @@ namespace repertory { sia_provider::sia_provider(app_config &config, i_http_comm &comm) - : config_(config), comm_(comm) {} + : base_provider(config, comm) {} auto sia_provider::get_object_info(const std::string &api_path, json &object_info) const -> api_error { @@ -54,7 +55,7 @@ auto sia_provider::get_object_info(const std::string &api_path, long response_code{}; stop_type stop_requested{}; - if (not comm_.make_request(get, response_code, stop_requested)) { + if (not get_comm().make_request(get, response_code, stop_requested)) { return api_error::comm_error; } @@ -92,7 +93,7 @@ auto sia_provider::get_object_list(const std::string &api_path, long response_code{}; stop_type stop_requested{}; - if (not comm_.make_request(get, response_code, stop_requested)) { + if (not get_comm().make_request(get, response_code, stop_requested)) { utils::error::raise_api_path_error(__FUNCTION__, api_path, api_error::comm_error, "failed to get object list"); @@ -108,37 +109,6 @@ auto sia_provider::get_object_list(const std::string &api_path, return true; } -auto sia_provider::create_api_file(std::string path, std::uint64_t size) - -> api_file { - api_file file{}; - file.api_path = utils::path::create_api_path(path); - file.api_parent = utils::path::get_parent_api_path(file.api_path); - file.accessed_date = utils::get_file_time_now(); - file.changed_date = utils::get_file_time_now(); - file.creation_date = utils::get_file_time_now(); - file.modified_date = utils::get_file_time_now(); - file.file_size = size; - return file; -} - -auto sia_provider::create_api_file(std::string path, std::uint64_t size, - api_meta_map &meta) -> api_file { - auto current_size = utils::string::to_uint64(meta[META_SIZE]); - if (current_size == 0U) { - current_size = size; - } - - api_file file{}; - file.api_path = utils::path::create_api_path(path); - file.api_parent = utils::path::get_parent_api_path(file.api_path); - file.accessed_date = utils::string::to_uint64(meta[META_ACCESSED]); - file.changed_date = utils::string::to_uint64(meta[META_CHANGED]); - file.creation_date = utils::string::to_uint64(meta[META_CREATION]); - file.file_size = current_size; - file.modified_date = utils::string::to_uint64(meta[META_MODIFIED]); - return file; -} - auto sia_provider::create_directory(const std::string &api_path, api_meta_map &meta) -> api_error { bool exists{}; @@ -170,7 +140,7 @@ auto sia_provider::create_directory(const std::string &api_path, long response_code{}; stop_type stop_requested{}; - if (not comm_.make_request(put_file, response_code, stop_requested)) { + if (not get_comm().make_request(put_file, response_code, stop_requested)) { utils::error::raise_api_path_error(__FUNCTION__, api_path, api_error::comm_error, "failed to create directory"); @@ -192,57 +162,6 @@ auto sia_provider::create_directory(const std::string &api_path, return set_item_meta(api_path, meta); } -auto sia_provider::create_directory_clone_source_meta( - const std::string &source_api_path, const std::string &api_path) - -> api_error { - bool exists{}; - auto res = is_file(source_api_path, exists); - if (res != api_error::success) { - return res; - } - if (exists) { - utils::error::raise_api_path_error(__FUNCTION__, api_path, - api_error::item_exists, - "failed to create directory"); - return api_error::item_exists; - } - - res = is_directory(api_path, exists); - if (res != api_error::success) { - return res; - } - if (exists) { - utils::error::raise_api_path_error(__FUNCTION__, api_path, - api_error::directory_exists, - "failed to create directory"); - return api_error::directory_exists; - } - - res = is_file(api_path, exists); - if (res != api_error::success) { - return res; - } - if (exists) { - utils::error::raise_api_path_error(__FUNCTION__, api_path, - api_error::item_exists, - "failed to create directory"); - return api_error::item_exists; - } - - api_meta_map meta{}; - res = get_item_meta(source_api_path, meta); - if (res != api_error::success) { - if (res == api_error::item_not_found) { - res = api_error::directory_not_found; - } - utils::error::raise_api_path_error(__FUNCTION__, api_path, res, - "failed to create directory"); - return res; - } - - return create_directory(api_path, meta); -} - auto sia_provider::create_file(const std::string &api_path, api_meta_map &meta) -> api_error { bool exists{}; @@ -274,7 +193,7 @@ auto sia_provider::create_file(const std::string &api_path, api_meta_map &meta) long response_code{}; stop_type stop_requested{}; - if (not comm_.make_request(put_file, response_code, stop_requested)) { + if (not get_comm().make_request(put_file, response_code, stop_requested)) { utils::error::raise_api_path_error(__FUNCTION__, api_path, api_error::comm_error, "failed to create file"); @@ -296,38 +215,6 @@ auto sia_provider::create_file(const std::string &api_path, api_meta_map &meta) return set_item_meta(api_path, meta); } -auto sia_provider::get_api_path_from_source(const std::string &source_path, - std::string &api_path) const - -> api_error { - if (source_path.empty()) { - utils::error::raise_api_path_error(__FUNCTION__, api_path, - api_error::item_not_found, - "failed to source path from api path"); - return api_error::item_not_found; - } - - auto iterator = std::unique_ptr( - db_->NewIterator(rocksdb::ReadOptions())); - for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) { - std::string current_source_path{}; - if (get_item_meta(iterator->key().ToString(), META_SOURCE, - current_source_path) != api_error::success) { - continue; - } - - if (current_source_path.empty()) { - continue; - } - - if (current_source_path == source_path) { - api_path = iterator->key().ToString(); - return api_error::success; - } - } - - return api_error::item_not_found; -} - auto sia_provider::get_directory_item_count(const std::string &api_path) const -> std::uint64_t { try { @@ -399,9 +286,9 @@ auto sia_provider::get_directory_items(const std::string &api_path, if (get_item_meta(entry_api_path, meta) == api_error::item_not_found) { file = create_api_file( - entry_api_path, + entry_api_path, "", directory ? 0U : entry["size"].get()); - api_item_added_(directory, file); + get_api_item_added()(directory, file); res = get_item_meta(entry_api_path, meta); if (res != api_error::success) { utils::error::raise_error(__FUNCTION__, res, @@ -414,14 +301,14 @@ auto sia_provider::get_directory_items(const std::string &api_path, directory ? 0U : entry["size"].get(), meta); } - directory_item di{}; - di.api_parent = file.api_parent; - di.api_path = file.api_path; - di.directory = directory; - di.meta = meta; - di.resolved = true; - di.size = file.file_size; - list.emplace_back(std::move(di)); + directory_item dir_item{}; + dir_item.api_parent = file.api_parent; + dir_item.api_path = file.api_path; + dir_item.directory = directory; + dir_item.meta = meta; + dir_item.resolved = true; + dir_item.size = file.file_size; + list.emplace_back(std::move(dir_item)); } catch (const std::exception &e) { utils::error::raise_api_path_error(__FUNCTION__, api_path, e, "failed to process entry|" + @@ -472,8 +359,8 @@ auto sia_provider::get_file(const std::string &api_path, api_file &file) const api_meta_map meta{}; if (get_item_meta(api_path, meta) == api_error::item_not_found) { - file = create_api_file(api_path, size); - api_item_added_(false, file); + file = create_api_file(api_path, "", size); + get_api_item_added()(false, file); } else { file = create_api_file(api_path, size, meta); } @@ -503,8 +390,8 @@ auto sia_provider::get_file_list(api_file_list &list) const -> api_error { api_meta_map meta{}; if (get_item_meta(entry_api_path, meta) == api_error::item_not_found) { - auto dir = create_api_file(entry_api_path, 0U); - api_item_added_(true, dir); + auto dir = create_api_file(entry_api_path, "", 0U); + get_api_item_added()(true, dir); } auto res = get_files_in_dir(entry_api_path); @@ -518,9 +405,9 @@ auto sia_provider::get_file_list(api_file_list &list) const -> api_error { api_meta_map meta{}; if (get_item_meta(entry_api_path, meta) == api_error::item_not_found) { - file = create_api_file(entry_api_path, + file = create_api_file(entry_api_path, "", entry["size"].get()); - api_item_added_(false, file); + get_api_item_added()(false, file); } else { file = create_api_file(entry_api_path, entry["size"].get(), meta); @@ -563,146 +450,6 @@ auto sia_provider::get_file_size(const std::string &api_path, return api_error::success; } -auto sia_provider::get_filesystem_item(const std::string &api_path, - bool directory, - filesystem_item &fsi) const - -> api_error { - bool exists{}; - auto res = is_directory(api_path, exists); - if (res != api_error::success) { - return res; - } - if (directory && not exists) { - return api_error::directory_not_found; - } - - res = is_file(api_path, exists); - if (res != api_error::success) { - return res; - } - if (not directory && not exists) { - return api_error::item_not_found; - } - - api_meta_map meta{}; - res = get_item_meta(api_path, meta); - if (res != api_error::success) { - return res; - } - - fsi.api_parent = utils::path::get_parent_api_path(api_path); - fsi.api_path = api_path; - fsi.directory = directory; - fsi.size = fsi.directory ? 0U : utils::string::to_uint64(meta[META_SIZE]); - fsi.source_path = meta[META_SOURCE]; - - return api_error::success; -} - -auto sia_provider::get_filesystem_item_and_file(const std::string &api_path, - api_file &file, - filesystem_item &fsi) const - -> api_error { - auto res = get_file(api_path, file); - if (res != api_error::success) { - return res; - } - - api_meta_map meta{}; - res = get_item_meta(api_path, meta); - if (res != api_error::success) { - return res; - } - - fsi.api_parent = utils::path::get_parent_api_path(api_path); - fsi.api_path = api_path; - fsi.directory = false; - fsi.size = utils::string::to_uint64(meta[META_SIZE]); - fsi.source_path = meta[META_SOURCE]; - - return api_error::success; -} - -auto sia_provider::get_filesystem_item_from_source_path( - const std::string &source_path, filesystem_item &fsi) const -> api_error { - std::string api_path{}; - auto res = get_api_path_from_source(source_path, api_path); - if (res != api_error::success) { - return res; - } - - bool exists{}; - res = is_directory(api_path, exists); - if (res != api_error::success) { - return res; - } - if (exists) { - return api_error::directory_exists; - } - - return get_filesystem_item(api_path, false, fsi); -} - -auto sia_provider::get_item_meta(const std::string &api_path, - api_meta_map &meta) const -> api_error { - std::string meta_value{}; - db_->Get(rocksdb::ReadOptions(), api_path, &meta_value); - if (meta_value.empty()) { - return api_error::item_not_found; - } - - try { - meta = json::parse(meta_value).get(); - - return api_error::success; - } catch (const std::exception &e) { - utils::error::raise_api_path_error(__FUNCTION__, api_path, e, - "failed to get item meta"); - } - - return api_error::error; -} - -auto sia_provider::get_item_meta(const std::string &api_path, - const std::string &key, - std::string &value) const -> api_error { - std::string meta_value{}; - db_->Get(rocksdb::ReadOptions(), api_path, &meta_value); - if (meta_value.empty()) { - return api_error::item_not_found; - } - - try { - value = json::parse(meta_value)[key]; - return api_error::success; - } catch (const std::exception &e) { - utils::error::raise_api_path_error(__FUNCTION__, api_path, e, - "failed to get item meta"); - } - - return api_error::error; -} - -auto sia_provider::get_pinned_files() const -> std::vector { - std::vector ret{}; - - auto iterator = std::unique_ptr( - db_->NewIterator(rocksdb::ReadOptions())); - for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) { - std::string pinned{}; - if (get_item_meta(iterator->key().ToString(), META_PINNED, pinned) != - api_error::success) { - continue; - } - if (pinned.empty() || not utils::string::to_bool(pinned)) { - continue; - } - ret.emplace_back(iterator->key().ToString()); - } - - return ret; -} - auto sia_provider::get_total_drive_space() const -> std::uint64_t { try { curl::requests::http_get get{}; @@ -719,7 +466,7 @@ auto sia_provider::get_total_drive_space() const -> std::uint64_t { long response_code{}; stop_type stop_requested{}; - if (not comm_.make_request(get, response_code, stop_requested)) { + if (not get_comm().make_request(get, response_code, stop_requested)) { return 0U; } @@ -738,16 +485,6 @@ auto sia_provider::get_total_drive_space() const -> std::uint64_t { return 0U; } -auto sia_provider::get_total_item_count() const -> std::uint64_t { - std::uint64_t ret{}; - auto iterator = std::unique_ptr( - db_->NewIterator(rocksdb::ReadOptions())); - for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) { - ret++; - } - return ret; -} - auto sia_provider::get_used_drive_space() const -> std::uint64_t { // TODO adjust size based on open files try { @@ -765,7 +502,7 @@ auto sia_provider::get_used_drive_space() const -> std::uint64_t { long response_code{}; stop_type stop_requested{}; - if (not comm_.make_request(get, response_code, stop_requested)) { + if (not get_comm().make_request(get, response_code, stop_requested)) { return 0U; } @@ -776,7 +513,7 @@ auto sia_provider::get_used_drive_space() const -> std::uint64_t { } auto used_space = object_data["totalObjectsSize"].get(); - fm_->update_used_space(used_space); + get_file_mgr()->update_used_space(used_space); return used_space; } catch (const std::exception &ex) { utils::error::raise_error(__FUNCTION__, ex, @@ -845,17 +582,6 @@ auto sia_provider::is_file(const std::string &api_path, bool &exists) const return api_error::error; } -auto sia_provider::is_file_writeable(const std::string &api_path) const - -> bool { - bool exists{}; - auto res = is_directory(api_path, exists); - if (res != api_error::success) { - return false; - } - - return not exists; -} - auto sia_provider::is_online() const -> bool { try { curl::requests::http_get get{}; @@ -872,7 +598,7 @@ auto sia_provider::is_online() const -> bool { long response_code{}; stop_type stop_requested{}; - if (not comm_.make_request(get, response_code, stop_requested)) { + if (not get_comm().make_request(get, response_code, stop_requested)) { utils::error::raise_error(__FUNCTION__, api_error::comm_error, "failed to determine if provider is online"); return false; @@ -910,7 +636,7 @@ auto sia_provider::read_file_bytes(const std::string &api_path, auto res = api_error::comm_error; for (std::uint32_t i = 0U; not stop_requested && res != api_error::success && - i < config_.get_retry_read_count() + 1U; + i < get_config().get_retry_read_count() + 1U; i++) { long response_code{}; const auto notify_retry = [&]() { @@ -930,7 +656,7 @@ auto sia_provider::read_file_bytes(const std::string &api_path, std::this_thread::sleep_for(1s); }; - if (not comm_.make_request(get, response_code, stop_requested)) { + if (not get_comm().make_request(get, response_code, stop_requested)) { notify_retry(); continue; } @@ -947,98 +673,6 @@ auto sia_provider::read_file_bytes(const std::string &api_path, return res; } -void sia_provider::remove_deleted_files() { - struct removed_item { - std::string api_path{}; - bool directory{}; - std::string source_path{}; - }; - - api_file_list list{}; - auto res = get_file_list(list); - if (res != api_error::success) { - utils::error::raise_error(__FUNCTION__, res, "failed to get file list"); - return; - } - - std::vector removed_list{}; - auto iterator = std::unique_ptr( - db_->NewIterator(rocksdb::ReadOptions())); - for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) { - api_meta_map meta{}; - if (get_item_meta(iterator->key().ToString(), meta) == api_error::success) { - if (utils::string::to_bool(meta[META_DIRECTORY])) { - bool exists{}; - if (is_directory(iterator->key().ToString(), exists) != - api_error::success) { - continue; - } - if (not exists) { - removed_list.emplace_back( - removed_item{iterator->key().ToString(), true, ""}); - } - continue; - } - - bool exists{}; - if (is_file(iterator->key().ToString(), exists) != api_error::success) { - continue; - } - if (not exists) { - removed_list.emplace_back( - removed_item{iterator->key().ToString(), false, meta[META_SOURCE]}); - } - } - } - - for (const auto &item : removed_list) { - if (not item.directory) { - if (utils::file::is_file(item.source_path)) { - const auto orphaned_directory = - utils::path::combine(config_.get_data_directory(), {"orphaned"}); - if (utils::file::create_full_directory_path(orphaned_directory)) { - const auto parts = utils::string::split(item.api_path, '/', false); - const auto orphaned_file = utils::path::combine( - orphaned_directory, - {utils::path::strip_to_file_name(item.source_path) + '_' + - parts[parts.size() - 1U]}); - - event_system::instance().raise( - item.source_path); - if (utils::file::reset_modified_time(item.source_path) && - utils::file::copy_file(item.source_path, orphaned_file)) { - event_system::instance().raise( - item.source_path, orphaned_file); - } else { - event_system::instance().raise( - item.source_path, orphaned_file, - std::to_string(utils::get_last_error_code())); - } - } else { - utils::error::raise_error( - __FUNCTION__, std::to_string(utils::get_last_error_code()), - "failed to create orphaned director|sp|" + orphaned_directory); - continue; - } - } - - if (fm_->evict_file(item.api_path)) { - db_->Delete(rocksdb::WriteOptions(), item.api_path); - event_system::instance().raise( - item.api_path, item.source_path); - } - } - } - - for (const auto &item : removed_list) { - if (item.directory) { - db_->Delete(rocksdb::WriteOptions(), item.api_path); - event_system::instance().raise( - item.api_path, item.source_path); - } - } -} - auto sia_provider::remove_directory(const std::string &api_path) -> api_error { const auto notify_end = [&api_path](api_error error) -> api_error { if (error == api_error::success) { @@ -1065,7 +699,7 @@ auto sia_provider::remove_directory(const std::string &api_path) -> api_error { long response_code{}; stop_type stop_requested{}; - if (not comm_.make_request(del, response_code, stop_requested)) { + if (not get_comm().make_request(del, response_code, stop_requested)) { utils::error::raise_api_path_error(__FUNCTION__, api_path, api_error::comm_error, "failed to remove directory"); @@ -1078,7 +712,7 @@ auto sia_provider::remove_directory(const std::string &api_path) -> api_error { return notify_end(api_error::comm_error); } - auto res2 = db_->Delete(rocksdb::WriteOptions(), api_path); + auto res2 = get_db()->Delete(rocksdb::WriteOptions(), api_path); if (not res2.ok()) { utils::error::raise_api_path_error(__FUNCTION__, api_path, res2.code(), "failed to remove directory"); @@ -1104,7 +738,7 @@ auto sia_provider::remove_file(const std::string &api_path) -> api_error { api_meta_map meta{}; auto res = get_item_meta(api_path, meta); - auto res2 = db_->Delete(rocksdb::WriteOptions(), api_path); + auto res2 = get_db()->Delete(rocksdb::WriteOptions(), api_path); if (not res2.ok()) { utils::error::raise_api_path_error(__FUNCTION__, api_path, res2.code(), "failed to remove file"); @@ -1140,7 +774,7 @@ auto sia_provider::remove_file(const std::string &api_path) -> api_error { long response_code{}; stop_type stop_requested{}; - if (not comm_.make_request(del, response_code, stop_requested)) { + if (not get_comm().make_request(del, response_code, stop_requested)) { utils::error::raise_api_path_error( __FUNCTION__, api_path, api_error::comm_error, "failed to remove file"); return notify_end(api_error::comm_error); @@ -1156,139 +790,19 @@ auto sia_provider::remove_file(const std::string &api_path) -> api_error { return remove_file_meta(); } -auto sia_provider::remove_item_meta(const std::string &api_path, - const std::string &key) -> api_error { - api_meta_map meta{}; - auto res = get_item_meta(api_path, meta); - if (res != api_error::success) { - return res; - } - - meta.erase(key); - - auto res2 = db_->Put(rocksdb::WriteOptions(), api_path, json(meta).dump()); - if (not res2.ok()) { - utils::error::raise_api_path_error(__FUNCTION__, api_path, res2.code(), - "failed to remove item meta"); - return api_error::error; - } - - return api_error::success; -} - auto sia_provider::rename_file(const std::string & /*from_api_path*/, const std::string & /*to_api_path*/) -> api_error { return api_error::not_implemented; } -auto sia_provider::set_item_meta(const std::string &api_path, - const std::string &key, - const std::string &value) -> api_error { - json meta_json{}; - std::string meta_value{}; - db_->Get(rocksdb::ReadOptions(), api_path, &meta_value); - if (not meta_value.empty()) { - try { - meta_json = json::parse(meta_value); - } catch (const std::exception &e) { - utils::error::raise_api_path_error(__FUNCTION__, api_path, e, - "failed to set item meta"); - return api_error::error; - } - } - - meta_json[key] = value; - - const auto res = - db_->Put(rocksdb::WriteOptions(), api_path, meta_json.dump()); - if (not res.ok()) { - utils::error::raise_api_path_error(__FUNCTION__, api_path, res.code(), - "failed to set item meta"); - return api_error::error; - } - - return api_error::success; -} - -auto sia_provider::set_item_meta(const std::string &api_path, - const api_meta_map &meta) -> api_error { - json meta_json{}; - std::string meta_value{}; - db_->Get(rocksdb::ReadOptions(), api_path, &meta_value); - if (not meta_value.empty()) { - try { - meta_json = json::parse(meta_value); - } catch (const std::exception &e) { - utils::error::raise_api_path_error(__FUNCTION__, api_path, e, - "failed to set item meta"); - return api_error::error; - } - } - - for (const auto &kv : meta) { - meta_json[kv.first] = kv.second; - } - - const auto res = - db_->Put(rocksdb::WriteOptions(), api_path, meta_json.dump()); - if (not res.ok()) { - utils::error::raise_api_path_error(__FUNCTION__, api_path, res.code(), - "failed to set item meta"); - return api_error::error; - } - - return api_error::success; -} - auto sia_provider::start(api_item_added_callback api_item_added, i_file_manager *mgr) -> bool { event_system::instance().raise("sia_provider"); - utils::db::create_rocksdb(config_, DB_NAME, db_); - - api_item_added_ = api_item_added; - fm_ = mgr; - - api_meta_map meta{}; - if (get_item_meta("/", meta) == api_error::item_not_found) { - auto dir = create_api_file("/", 0U); - api_item_added_(true, dir); - } - - auto online = false; - auto unmount_requested = false; - { - repertory::event_consumer ec( - "unmount_requested", - [&unmount_requested](const event &) { unmount_requested = true; }); - for (std::uint16_t i = 0u; not online && not unmount_requested && - (i < config_.get_online_check_retry_secs()); - i++) { - online = is_online(); - if (not online) { - event_system::instance().raise( - config_.get_host_config().host_name_or_ip, - config_.get_host_config().api_port); - std::this_thread::sleep_for(1s); - } - } - } - - if (online && not unmount_requested) { - polling::instance().set_callback({"check_deleted", polling::frequency::low, - [this]() { remove_deleted_files(); }}); - return true; - } - - return false; + return base_provider::start(api_item_added, mgr); } -void sia_provider::stop() { - event_system::instance().raise("sia_provider"); - polling::instance().remove_callback("check_deleted"); - db_.reset(); - event_system::instance().raise("sia_provider"); -} +void sia_provider::stop() { return base_provider::stop(); } auto sia_provider::upload_file(const std::string &api_path, const std::string &source_path, @@ -1308,7 +822,7 @@ auto sia_provider::upload_file(const std::string &api_path, put_file.source_path = source_path; long response_code{}; - if (not comm_.make_request(put_file, response_code, stop_requested)) { + if (not get_comm().make_request(put_file, response_code, stop_requested)) { utils::error::raise_api_path_error(__FUNCTION__, api_path, source_path, api_error::comm_error, "failed to upload file");