extract common behavior

include/providers/base_provider.hpp

@@ -0,0 +1,146 @@
+/*
+  Copyright <2018-2023> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all
+  copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  SOFTWARE.
+*/
+#ifndef INCLUDE_PROVIDERS_BASE_PROVIDER_HPP_
+#define INCLUDE_PROVIDERS_BASE_PROVIDER_HPP_
+
+#include "providers/i_provider.hpp"
+#include "types/repertory.hpp"
+
+namespace repertory {
+class app_config;
+class i_file_manager;
+class i_http_comm;
+
+class base_provider : public i_provider {
+public:
+  base_provider(app_config &config, i_http_comm &comm)
+      : config_(config), comm_(comm) {}
+
+private:
+  app_config &config_;
+  i_http_comm &comm_;
+
+private:
+  api_item_added_callback api_item_added_;
+  std::unique_ptr<rocksdb::DB> db_;
+  std::string DB_NAME = "meta_db";
+  i_file_manager *fm_{};
+
+private:
+  void remove_deleted_files();
+
+protected:
+  [[nodiscard]] static auto create_api_file(std::string path, std::string key,
+                                            std::uint64_t size) -> api_file;
+
+  [[nodiscard]] static auto create_api_file(std::string path,
+                                            std::uint64_t size,
+                                            api_meta_map &meta) -> api_file;
+
+  [[nodiscard]] auto get_api_item_added() -> api_item_added_callback & {
+    return api_item_added_;
+  }
+
+  [[nodiscard]] auto get_api_item_added() const
+      -> const api_item_added_callback & {
+    return api_item_added_;
+  }
+
+  [[nodiscard]] auto get_comm() const -> i_http_comm & { return comm_; }
+
+  [[nodiscard]] auto get_config() -> app_config & { return config_; }
+
+  [[nodiscard]] auto get_config() const -> const app_config & {
+    return config_;
+  }
+
+  [[nodiscard]] auto get_db() const -> rocksdb::DB * { return db_.get(); }
+
+  [[nodiscard]] auto get_file_mgr() -> i_file_manager * { return fm_; }
+
+  [[nodiscard]] auto get_file_mgr() const -> const i_file_manager * {
+    return fm_;
+  }
+
+public:
+  [[nodiscard]] auto
+  create_directory_clone_source_meta(const std::string &source_api_path,
+                                     const std::string &api_path)
+      -> api_error override;
+
+  [[nodiscard]] auto get_api_path_from_source(const std::string &source_path,
+                                              std::string &api_path) const
+      -> api_error override;
+
+  [[nodiscard]] auto get_filesystem_item(const std::string &api_path,
+                                         bool directory,
+                                         filesystem_item &fsi) const
+      -> api_error override;
+
+  [[nodiscard]] auto get_filesystem_item_and_file(const std::string &api_path,
+                                                  api_file &f,
+                                                  filesystem_item &fsi) const
+      -> api_error override;
+
+  [[nodiscard]] auto
+  get_filesystem_item_from_source_path(const std::string &source_path,
+                                       filesystem_item &fsi) const
+      -> api_error override;
+
+  [[nodiscard]] auto get_item_meta(const std::string &api_path,
+                                   api_meta_map &meta) const
+      -> api_error override;
+
+  [[nodiscard]] auto get_item_meta(const std::string &api_path,
+                                   const std::string &key,
+                                   std::string &value) const
+      -> api_error override;
+
+  [[nodiscard]] auto get_pinned_files() const
+      -> std::vector<std::string> override;
+
+  [[nodiscard]] auto get_total_item_count() const -> std::uint64_t override;
+
+  [[nodiscard]] auto is_file_writeable(const std::string &api_path) const
+      -> bool override;
+
+  [[nodiscard]] auto remove_item_meta(const std::string &api_path,
+                                      const std::string &key)
+      -> api_error override;
+
+  [[nodiscard]] auto set_item_meta(const std::string &api_path,
+                                   const std::string &key,
+                                   const std::string &value)
+      -> api_error override;
+
+  [[nodiscard]] auto set_item_meta(const std::string &api_path,
+                                   const api_meta_map &meta)
+      -> api_error override;
+
+  [[nodiscard]] auto start(api_item_added_callback api_item_added,
+                           i_file_manager *mgr) -> bool override;
+
+  void stop() override;
+};
+} // namespace repertory
+
+#endif // INCLUDE_PROVIDERS_BASE_PROVIDER_HPP_

include/providers/s3/s3_provider.hpp

@@ -23,7 +23,7 @@
 #define INCLUDE_PROVIDERS_S3_S3_PROVIDER_HPP_
 #if defined(REPERTORY_ENABLE_S3)
 
-#include "providers/i_provider.hpp"
+#include "providers/base_provider.hpp"
 #include "types/repertory.hpp"
 
 namespace repertory {
@@ -32,7 +32,7 @@ class i_file_manager;
 class i_http_comm;
 struct head_object_result;
 
-class s3_provider final : public i_provider {
+class s3_provider final : public base_provider {
 public:
   s3_provider(app_config &config, i_http_comm &comm);
 
@@ -44,25 +44,11 @@ public:
   auto operator=(const s3_provider &) -> s3_provider & = delete;
   auto operator=(s3_provider &&) -> s3_provider & = delete;
 
-private:
-  app_config &config_;
-  i_http_comm &comm_;
-
-private:
-  api_item_added_callback api_item_added_;
-  std::unique_ptr<rocksdb::DB> db_;
-  std::string DB_NAME = "meta_db";
-  i_file_manager *fm_{};
-
 private:
   [[nodiscard]] auto add_if_not_found(api_file &file,
                                       const std::string &object_name) const
       -> api_error;
 
-  [[nodiscard]] static auto create_api_file(const std::string &path,
-                                            const std::string &key,
-                                            std::uint64_t size) -> api_file;
-
   [[nodiscard]] auto create_path_directories(const std::string &api_path,
                                              const std::string &key) const
       -> api_error;
@@ -75,24 +61,13 @@ private:
                   bool &is_encrypted, std::string &object_name,
                   head_object_result &result) const -> api_error;
 
-  void remove_deleted_files();
-
 public:
   [[nodiscard]] auto create_directory(const std::string &api_path,
                                       api_meta_map &meta) -> api_error override;
 
-  [[nodiscard]] auto
-  create_directory_clone_source_meta(const std::string &source_api_path,
-                                     const std::string &api_path)
-      -> api_error override;
-
   [[nodiscard]] auto create_file(const std::string &api_path,
                                  api_meta_map &meta) -> api_error override;
 
-  [[nodiscard]] auto get_api_path_from_source(const std::string &source_path,
-                                              std::string &api_path) const
-      -> api_error override;
-
   [[nodiscard]] auto get_directory_item_count(const std::string &api_path) const
       -> std::uint64_t override;
 
@@ -110,37 +85,8 @@ public:
                                    std::uint64_t &file_size) const
       -> api_error override;
 
-  [[nodiscard]] auto get_filesystem_item(const std::string &api_path,
-                                         bool directory,
-                                         filesystem_item &fsi) const
-      -> api_error override;
-
-  [[nodiscard]] auto get_filesystem_item_and_file(const std::string &api_path,
-                                                  api_file &file,
-                                                  filesystem_item &fsi) const
-      -> api_error override;
-
-  [[nodiscard]] auto
-  get_filesystem_item_from_source_path(const std::string &source_path,
-                                       filesystem_item &fsi) const
-      -> api_error override;
-
-  [[nodiscard]] auto get_pinned_files() const
-      -> std::vector<std::string> override;
-
-  [[nodiscard]] auto get_item_meta(const std::string &api_path,
-                                   api_meta_map &meta) const
-      -> api_error override;
-
-  [[nodiscard]] auto get_item_meta(const std::string &api_path,
-                                   const std::string &key,
-                                   std::string &value) const
-      -> api_error override;
-
   [[nodiscard]] auto get_total_drive_space() const -> std::uint64_t override;
 
-  [[nodiscard]] auto get_total_item_count() const -> std::uint64_t override;
-
   [[nodiscard]] auto get_provider_type() const -> provider_type override {
     return provider_type::s3;
   }
@@ -155,9 +101,6 @@ public:
   [[nodiscard]] auto is_file(const std::string &api_path, bool &exists) const
       -> api_error override;
 
-  [[nodiscard]] auto is_file_writeable(const std::string &api_path) const
-      -> bool override;
-
   [[nodiscard]] auto is_online() const -> bool override;
 
   [[nodiscard]] auto is_rename_supported() const -> bool override {
@@ -176,23 +119,10 @@ public:
   [[nodiscard]] auto remove_file(const std::string &api_path)
       -> api_error override;
 
-  [[nodiscard]] auto remove_item_meta(const std::string &api_path,
-                                      const std::string &key)
-      -> api_error override;
-
   [[nodiscard]] auto rename_file(const std::string &from_api_path,
                                  const std::string &to_api_path)
       -> api_error override;
 
-  [[nodiscard]] auto set_item_meta(const std::string &api_path,
-                                   const std::string &key,
-                                   const std::string &value)
-      -> api_error override;
-
-  [[nodiscard]] auto set_item_meta(const std::string &api_path,
-                                   const api_meta_map &meta)
-      -> api_error override;
-
   [[nodiscard]] auto start(api_item_added_callback api_item_added,
                            i_file_manager *mgr) -> bool override;
 

include/providers/sia/sia_provider.hpp

@@ -22,7 +22,7 @@
 #ifndef INCLUDE_PROVIDERS_SIA_SIA_PROVIDER_HPP_
 #define INCLUDE_PROVIDERS_SIA_SIA_PROVIDER_HPP_
 
-#include "providers/i_provider.hpp"
+#include "providers/base_provider.hpp"
 #include "types/repertory.hpp"
 
 namespace repertory {
@@ -30,7 +30,7 @@ class app_config;
 class i_file_manager;
 class i_http_comm;
 
-class sia_provider : public i_provider {
+class sia_provider : public base_provider {
 public:
   sia_provider(app_config &config, i_http_comm &comm);
 
@@ -43,50 +43,22 @@ public:
   auto operator=(sia_provider &&) -> sia_provider & = delete;
 
 private:
-  app_config &config_;
-  i_http_comm &comm_;
-
-private:
-  api_item_added_callback api_item_added_;
-  std::unique_ptr<rocksdb::DB> db_;
-  std::string DB_NAME = "meta_db";
-  i_file_manager *fm_{};
-
-private:
-  [[nodiscard]] static auto create_api_file(std::string path,
-                                            std::uint64_t size) -> api_file;
-
-  [[nodiscard]] static auto create_api_file(std::string path,
-                                            std::uint64_t size,
-                                            api_meta_map &meta) -> api_file;
-
   [[nodiscard]] auto get_object_info(const std::string &api_path,
                                      json &object_info) const -> api_error;
 
   [[nodiscard]] auto get_object_list(const std::string &api_path,
                                      nlohmann::json &object_list) const -> bool;
 
-  void remove_deleted_files();
-
 public:
   [[nodiscard]] auto create_directory(const std::string &api_path,
                                       api_meta_map &meta) -> api_error override;
 
-  [[nodiscard]] auto
-  create_directory_clone_source_meta(const std::string &source_api_path,
-                                     const std::string &api_path)
-      -> api_error override;
-
   [[nodiscard]] auto get_directory_item_count(const std::string &api_path) const
       -> std::uint64_t override;
 
   [[nodiscard]] auto create_file(const std::string &api_path,
                                  api_meta_map &meta) -> api_error override;
 
-  [[nodiscard]] auto get_api_path_from_source(const std::string &source_path,
-                                              std::string &api_path) const
-      -> api_error override;
-
   [[nodiscard]] auto get_directory_items(const std::string &api_path,
                                          directory_item_list &list) const
       -> api_error override;
@@ -101,41 +73,12 @@ public:
                                    std::uint64_t &file_size) const
       -> api_error override;
 
-  [[nodiscard]] auto get_filesystem_item(const std::string &api_path,
-                                         bool directory,
-                                         filesystem_item &fsi) const
-      -> api_error override;
-
-  [[nodiscard]] auto get_filesystem_item_and_file(const std::string &api_path,
-                                                  api_file &f,
-                                                  filesystem_item &fsi) const
-      -> api_error override;
-
-  [[nodiscard]] auto
-  get_filesystem_item_from_source_path(const std::string &source_path,
-                                       filesystem_item &fsi) const
-      -> api_error override;
-
-  [[nodiscard]] auto get_item_meta(const std::string &api_path,
-                                   api_meta_map &meta) const
-      -> api_error override;
-
-  [[nodiscard]] auto get_item_meta(const std::string &api_path,
-                                   const std::string &key,
-                                   std::string &value) const
-      -> api_error override;
-
-  [[nodiscard]] auto get_pinned_files() const
-      -> std::vector<std::string> override;
-
   [[nodiscard]] auto get_provider_type() const -> provider_type override {
     return provider_type::sia;
   }
 
   [[nodiscard]] auto get_total_drive_space() const -> std::uint64_t override;
 
-  [[nodiscard]] auto get_total_item_count() const -> std::uint64_t override;
-
   [[nodiscard]] auto get_used_drive_space() const -> std::uint64_t override;
 
   [[nodiscard]] auto is_direct_only() const -> bool override { return false; }
@@ -146,9 +89,6 @@ public:
   [[nodiscard]] auto is_file(const std::string &api_path, bool &exists) const
       -> api_error override;
 
-  [[nodiscard]] auto is_file_writeable(const std::string &api_path) const
-      -> bool override;
-
   [[nodiscard]] auto is_online() const -> bool override;
 
   [[nodiscard]] auto is_rename_supported() const -> bool override {
@@ -167,23 +107,10 @@ public:
   [[nodiscard]] auto remove_file(const std::string &api_path)
       -> api_error override;
 
-  [[nodiscard]] auto remove_item_meta(const std::string &api_path,
-                                      const std::string &key)
-      -> api_error override;
-
   [[nodiscard]] auto rename_file(const std::string &from_api_path,
                                  const std::string &to_api_path)
       -> api_error override;
 
-  [[nodiscard]] auto set_item_meta(const std::string &api_path,
-                                   const std::string &key,
-                                   const std::string &value)
-      -> api_error override;
-
-  [[nodiscard]] auto set_item_meta(const std::string &api_path,
-                                   const api_meta_map &meta)
-      -> api_error override;
-
   [[nodiscard]] auto start(api_item_added_callback api_item_added,
                            i_file_manager *mgr) -> bool override;
 

src/providers/base_provider.cpp

@@ -0,0 +1,529 @@
+/*
+  Copyright <2018-2023> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all
+  copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  SOFTWARE.
+*/
+#include "providers/base_provider.hpp"
+
+#include "app_config.hpp"
+#include "events/event_system.hpp"
+#include "events/events.hpp"
+#include "file_manager/i_file_manager.hpp"
+#include "utils/file_utils.hpp"
+#include "utils/path_utils.hpp"
+#include "utils/polling.hpp"
+#include "utils/rocksdb_utils.hpp"
+
+namespace repertory {
+auto base_provider::create_api_file(std::string path, std::string key,
+                                    std::uint64_t size) -> api_file {
+  api_file file{};
+  file.api_path = utils::path::create_api_path(path);
+  file.api_parent = utils::path::get_parent_api_path(file.api_path);
+  file.accessed_date = utils::get_file_time_now();
+  file.changed_date = utils::get_file_time_now();
+  file.creation_date = utils::get_file_time_now();
+  file.modified_date = utils::get_file_time_now();
+  file.key = key;
+  file.file_size = size;
+  return file;
+}
+
+auto base_provider::create_api_file(std::string path, std::uint64_t size,
+                                    api_meta_map &meta) -> api_file {
+  auto current_size = utils::string::to_uint64(meta[META_SIZE]);
+  if (current_size == 0U) {
+    current_size = size;
+  }
+
+  api_file file{};
+  file.api_path = utils::path::create_api_path(path);
+  file.api_parent = utils::path::get_parent_api_path(file.api_path);
+  file.accessed_date = utils::string::to_uint64(meta[META_ACCESSED]);
+  file.changed_date = utils::string::to_uint64(meta[META_CHANGED]);
+  file.creation_date = utils::string::to_uint64(meta[META_CREATION]);
+  file.file_size = current_size;
+  file.modified_date = utils::string::to_uint64(meta[META_MODIFIED]);
+  return file;
+}
+
+auto base_provider::create_directory_clone_source_meta(
+    const std::string &source_api_path, const std::string &api_path)
+    -> api_error {
+  bool exists{};
+  auto res = is_file(source_api_path, exists);
+  if (res != api_error::success) {
+    return res;
+  }
+  if (exists) {
+    utils::error::raise_api_path_error(__FUNCTION__, api_path,
+                                       api_error::item_exists,
+                                       "failed to create directory");
+    return api_error::item_exists;
+  }
+
+  res = is_directory(api_path, exists);
+  if (res != api_error::success) {
+    return res;
+  }
+  if (exists) {
+    utils::error::raise_api_path_error(__FUNCTION__, api_path,
+                                       api_error::directory_exists,
+                                       "failed to create directory");
+    return api_error::directory_exists;
+  }
+
+  res = is_file(api_path, exists);
+  if (res != api_error::success) {
+    return res;
+  }
+  if (exists) {
+    utils::error::raise_api_path_error(__FUNCTION__, api_path,
+                                       api_error::item_exists,
+                                       "failed to create directory");
+    return api_error::item_exists;
+  }
+
+  api_meta_map meta{};
+  res = get_item_meta(source_api_path, meta);
+  if (res != api_error::success) {
+    if (res == api_error::item_not_found) {
+      res = api_error::directory_not_found;
+    }
+    utils::error::raise_api_path_error(__FUNCTION__, api_path, res,
+                                       "failed to create directory");
+    return res;
+  }
+
+  return create_directory(api_path, meta);
+}
+
+auto base_provider::get_api_path_from_source(const std::string &source_path,
+                                             std::string &api_path) const
+    -> api_error {
+  if (source_path.empty()) {
+    utils::error::raise_api_path_error(__FUNCTION__, api_path,
+                                       api_error::item_not_found,
+                                       "failed to source path from api path");
+    return api_error::item_not_found;
+  }
+
+  auto iterator = std::unique_ptr<rocksdb::Iterator>(
+      db_->NewIterator(rocksdb::ReadOptions()));
+  for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
+    std::string current_source_path{};
+    if (get_item_meta(iterator->key().ToString(), META_SOURCE,
+                      current_source_path) != api_error::success) {
+      continue;
+    }
+
+    if (current_source_path.empty()) {
+      continue;
+    }
+
+    if (current_source_path == source_path) {
+      api_path = iterator->key().ToString();
+      return api_error::success;
+    }
+  }
+
+  return api_error::item_not_found;
+}
+
+auto base_provider::get_filesystem_item(const std::string &api_path,
+                                        bool directory,
+                                        filesystem_item &fsi) const
+    -> api_error {
+  bool exists{};
+  auto res = is_directory(api_path, exists);
+  if (res != api_error::success) {
+    return res;
+  }
+  if (directory && not exists) {
+    return api_error::directory_not_found;
+  }
+
+  res = is_file(api_path, exists);
+  if (res != api_error::success) {
+    return res;
+  }
+  if (not directory && not exists) {
+    return api_error::item_not_found;
+  }
+
+  api_meta_map meta{};
+  res = get_item_meta(api_path, meta);
+  if (res != api_error::success) {
+    return res;
+  }
+
+  fsi.api_parent = utils::path::get_parent_api_path(api_path);
+  fsi.api_path = api_path;
+  fsi.directory = directory;
+  fsi.size = fsi.directory ? 0U : utils::string::to_uint64(meta[META_SIZE]);
+  fsi.source_path = meta[META_SOURCE];
+
+  return api_error::success;
+}
+
+auto base_provider::get_filesystem_item_and_file(const std::string &api_path,
+                                                 api_file &file,
+                                                 filesystem_item &fsi) const
+    -> api_error {
+  auto res = get_file(api_path, file);
+  if (res != api_error::success) {
+    return res;
+  }
+
+  api_meta_map meta{};
+  res = get_item_meta(api_path, meta);
+  if (res != api_error::success) {
+    return res;
+  }
+
+  fsi.api_parent = utils::path::get_parent_api_path(api_path);
+  fsi.api_path = api_path;
+  fsi.directory = false;
+  fsi.size = utils::string::to_uint64(meta[META_SIZE]);
+  fsi.source_path = meta[META_SOURCE];
+
+  return api_error::success;
+}
+
+auto base_provider::get_filesystem_item_from_source_path(
+    const std::string &source_path, filesystem_item &fsi) const -> api_error {
+  std::string api_path{};
+  auto res = get_api_path_from_source(source_path, api_path);
+  if (res != api_error::success) {
+    return res;
+  }
+
+  bool exists{};
+  res = is_directory(api_path, exists);
+  if (res != api_error::success) {
+    return res;
+  }
+  if (exists) {
+    return api_error::directory_exists;
+  }
+
+  return get_filesystem_item(api_path, false, fsi);
+}
+
+auto base_provider::get_item_meta(const std::string &api_path,
+                                  api_meta_map &meta) const -> api_error {
+  std::string meta_value{};
+  db_->Get(rocksdb::ReadOptions(), api_path, &meta_value);
+  if (meta_value.empty()) {
+    return api_error::item_not_found;
+  }
+
+  try {
+    meta = json::parse(meta_value).get<api_meta_map>();
+
+    return api_error::success;
+  } catch (const std::exception &e) {
+    utils::error::raise_api_path_error(__FUNCTION__, api_path, e,
+                                       "failed to get item meta");
+  }
+
+  return api_error::error;
+}
+
+auto base_provider::get_item_meta(const std::string &api_path,
+                                  const std::string &key,
+                                  std::string &value) const -> api_error {
+  std::string meta_value{};
+  db_->Get(rocksdb::ReadOptions(), api_path, &meta_value);
+  if (meta_value.empty()) {
+    return api_error::item_not_found;
+  }
+
+  try {
+    value = json::parse(meta_value)[key];
+    return api_error::success;
+  } catch (const std::exception &e) {
+    utils::error::raise_api_path_error(__FUNCTION__, api_path, e,
+                                       "failed to get item meta");
+  }
+
+  return api_error::error;
+}
+
+auto base_provider::get_pinned_files() const -> std::vector<std::string> {
+  std::vector<std::string> ret{};
+
+  auto iterator = std::unique_ptr<rocksdb::Iterator>(
+      db_->NewIterator(rocksdb::ReadOptions()));
+  for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
+    std::string pinned{};
+    if (get_item_meta(iterator->key().ToString(), META_PINNED, pinned) !=
+        api_error::success) {
+      continue;
+    }
+    if (pinned.empty() || not utils::string::to_bool(pinned)) {
+      continue;
+    }
+    ret.emplace_back(iterator->key().ToString());
+  }
+
+  return ret;
+}
+
+auto base_provider::get_total_item_count() const -> std::uint64_t {
+  std::uint64_t ret{};
+  auto iterator = std::unique_ptr<rocksdb::Iterator>(
+      db_->NewIterator(rocksdb::ReadOptions()));
+  for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
+    ret++;
+  }
+  return ret;
+}
+
+auto base_provider::is_file_writeable(const std::string &api_path) const
+    -> bool {
+  bool exists{};
+  auto res = is_directory(api_path, exists);
+  if (res != api_error::success) {
+    return false;
+  }
+
+  return not exists;
+}
+
+void base_provider::remove_deleted_files() {
+  struct removed_item {
+    std::string api_path{};
+    bool directory{};
+    std::string source_path{};
+  };
+
+  api_file_list list{};
+  auto res = get_file_list(list);
+  if (res != api_error::success) {
+    utils::error::raise_error(__FUNCTION__, res, "failed to get file list");
+    return;
+  }
+
+  std::vector<removed_item> removed_list{};
+  auto iterator = std::unique_ptr<rocksdb::Iterator>(
+      db_->NewIterator(rocksdb::ReadOptions()));
+  for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
+    api_meta_map meta{};
+    if (get_item_meta(iterator->key().ToString(), meta) == api_error::success) {
+      if (utils::string::to_bool(meta[META_DIRECTORY])) {
+        bool exists{};
+        if (is_directory(iterator->key().ToString(), exists) !=
+            api_error::success) {
+          continue;
+        }
+        if (not exists) {
+          removed_list.emplace_back(
+              removed_item{iterator->key().ToString(), true, ""});
+        }
+        continue;
+      }
+
+      bool exists{};
+      if (is_file(iterator->key().ToString(), exists) != api_error::success) {
+        continue;
+      }
+      if (not exists) {
+        removed_list.emplace_back(
+            removed_item{iterator->key().ToString(), false, meta[META_SOURCE]});
+      }
+    }
+  }
+
+  for (const auto &item : removed_list) {
+    if (not item.directory) {
+      if (utils::file::is_file(item.source_path)) {
+        const auto orphaned_directory =
+            utils::path::combine(config_.get_data_directory(), {"orphaned"});
+        if (utils::file::create_full_directory_path(orphaned_directory)) {
+          const auto parts = utils::string::split(item.api_path, '/', false);
+          const auto orphaned_file = utils::path::combine(
+              orphaned_directory,
+              {utils::path::strip_to_file_name(item.source_path) + '_' +
+               parts[parts.size() - 1U]});
+
+          event_system::instance().raise<orphaned_file_detected>(
+              item.source_path);
+          if (utils::file::reset_modified_time(item.source_path) &&
+              utils::file::copy_file(item.source_path, orphaned_file)) {
+            event_system::instance().raise<orphaned_file_processed>(
+                item.source_path, orphaned_file);
+          } else {
+            event_system::instance().raise<orphaned_file_processing_failed>(
+                item.source_path, orphaned_file,
+                std::to_string(utils::get_last_error_code()));
+          }
+        } else {
+          utils::error::raise_error(
+              __FUNCTION__, std::to_string(utils::get_last_error_code()),
+              "failed to create orphaned director|sp|" + orphaned_directory);
+          continue;
+        }
+      }
+
+      if (fm_->evict_file(item.api_path)) {
+        db_->Delete(rocksdb::WriteOptions(), item.api_path);
+        event_system::instance().raise<file_removed_externally>(
+            item.api_path, item.source_path);
+      }
+    }
+  }
+
+  for (const auto &item : removed_list) {
+    if (item.directory) {
+      db_->Delete(rocksdb::WriteOptions(), item.api_path);
+      event_system::instance().raise<directory_removed_externally>(
+          item.api_path, item.source_path);
+    }
+  }
+}
+
+auto base_provider::remove_item_meta(const std::string &api_path,
+                                     const std::string &key) -> api_error {
+  api_meta_map meta{};
+  auto res = get_item_meta(api_path, meta);
+  if (res != api_error::success) {
+    return res;
+  }
+
+  meta.erase(key);
+
+  auto res2 = db_->Put(rocksdb::WriteOptions(), api_path, json(meta).dump());
+  if (not res2.ok()) {
+    utils::error::raise_api_path_error(__FUNCTION__, api_path, res2.code(),
+                                       "failed to remove item meta");
+    return api_error::error;
+  }
+
+  return api_error::success;
+}
+
+auto base_provider::set_item_meta(const std::string &api_path,
+                                  const std::string &key,
+                                  const std::string &value) -> api_error {
+  json meta_json{};
+  std::string meta_value{};
+  db_->Get(rocksdb::ReadOptions(), api_path, &meta_value);
+  if (not meta_value.empty()) {
+    try {
+      meta_json = json::parse(meta_value);
+    } catch (const std::exception &e) {
+      utils::error::raise_api_path_error(__FUNCTION__, api_path, e,
+                                         "failed to set item meta");
+      return api_error::error;
+    }
+  }
+
+  meta_json[key] = value;
+
+  const auto res =
+      db_->Put(rocksdb::WriteOptions(), api_path, meta_json.dump());
+  if (not res.ok()) {
+    utils::error::raise_api_path_error(__FUNCTION__, api_path, res.code(),
+                                       "failed to set item meta");
+    return api_error::error;
+  }
+
+  return api_error::success;
+}
+
+auto base_provider::set_item_meta(const std::string &api_path,
+                                  const api_meta_map &meta) -> api_error {
+  json meta_json{};
+  std::string meta_value{};
+  db_->Get(rocksdb::ReadOptions(), api_path, &meta_value);
+  if (not meta_value.empty()) {
+    try {
+      meta_json = json::parse(meta_value);
+    } catch (const std::exception &e) {
+      utils::error::raise_api_path_error(__FUNCTION__, api_path, e,
+                                         "failed to set item meta");
+      return api_error::error;
+    }
+  }
+
+  for (const auto &kv : meta) {
+    meta_json[kv.first] = kv.second;
+  }
+
+  const auto res =
+      db_->Put(rocksdb::WriteOptions(), api_path, meta_json.dump());
+  if (not res.ok()) {
+    utils::error::raise_api_path_error(__FUNCTION__, api_path, res.code(),
+                                       "failed to set item meta");
+    return api_error::error;
+  }
+
+  return api_error::success;
+}
+
+auto base_provider::start(api_item_added_callback api_item_added,
+                          i_file_manager *mgr) -> bool {
+  event_system::instance().raise<service_started>("sia_provider");
+  utils::db::create_rocksdb(config_, DB_NAME, db_);
+
+  api_item_added_ = api_item_added;
+  fm_ = mgr;
+
+  api_meta_map meta{};
+  if (get_item_meta("/", meta) == api_error::item_not_found) {
+    auto dir = create_api_file("/", "", 0U);
+    api_item_added_(true, dir);
+  }
+
+  auto online = false;
+  auto unmount_requested = false;
+  {
+    repertory::event_consumer consumer(
+        "unmount_requested",
+        [&unmount_requested](const event &) { unmount_requested = true; });
+    for (std::uint16_t i = 0U; not online && not unmount_requested &&
+                               (i < config_.get_online_check_retry_secs());
+         i++) {
+      online = is_online();
+      if (not online) {
+        event_system::instance().raise<provider_offline>(
+            config_.get_host_config().host_name_or_ip,
+            config_.get_host_config().api_port);
+        std::this_thread::sleep_for(1s);
+      }
+    }
+  }
+
+  if (online && not unmount_requested) {
+    polling::instance().set_callback({"check_deleted", polling::frequency::low,
+                                      [this]() { remove_deleted_files(); }});
+    return true;
+  }
+
+  return false;
+}
+
+void base_provider::stop() {
+  event_system::instance().raise<service_shutdown_begin>("sia_provider");
+  polling::instance().remove_callback("check_deleted");
+  db_.reset();
+  event_system::instance().raise<service_shutdown_end>("sia_provider");
+}
+} // namespace repertory

src/providers/s3/s3_provider.cpp

@@ -69,8 +69,8 @@ get_object_list(i_http_comm &client, const s3_config &config,
 } // namespace
 
 s3_provider::s3_provider(app_config &config, i_http_comm &comm)
-    : config_(config), comm_(comm) {
+    : base_provider(config, comm) {
-  comm_.enable_s3_path_style(config.get_s3_config().use_path_style);
+  get_comm().enable_s3_path_style(config.get_s3_config().use_path_style);
 }
 
 auto s3_provider::add_if_not_found(api_file &file,
@@ -84,27 +84,12 @@ auto s3_provider::add_if_not_found(api_file &file,
       return err;
     }
 
-    api_item_added_(false, file);
+    get_api_item_added()(false, file);
   }
 
   return api_error::success;
 }
 
-auto s3_provider::create_api_file(const std::string &path,
-                                  const std::string &key, std::uint64_t size)
-    -> api_file {
-  api_file file{};
-  file.api_path = utils::path::create_api_path(path);
-  file.api_parent = utils::path::get_parent_api_path(file.api_path);
-  file.accessed_date = utils::get_file_time_now();
-  file.changed_date = utils::get_file_time_now();
-  file.creation_date = utils::get_file_time_now();
-  file.file_size = size;
-  file.key = key;
-  file.modified_date = utils::get_file_time_now();
-  return file;
-}
-
 auto s3_provider::create_directory(const std::string &api_path,
                                    api_meta_map &meta) -> api_error {
 
@@ -132,7 +117,7 @@ auto s3_provider::create_directory(const std::string &api_path,
   }
 
   try {
-    const auto cfg = config_.get_s3_config();
+    const auto cfg = get_config().get_s3_config();
     const auto is_encrypted = not cfg.encryption_token.empty();
     stop_type stop_requested{false};
 
@@ -165,7 +150,7 @@ auto s3_provider::create_directory(const std::string &api_path,
     put_file.path = object_name + '/';
 
     long response_code{};
-    if (not comm_.make_request(put_file, response_code, stop_requested)) {
+    if (not get_comm().make_request(put_file, response_code, stop_requested)) {
       utils::error::raise_api_path_error(__FUNCTION__, api_path,
                                          api_error::comm_error,
                                          "failed to create directory");
@@ -187,57 +172,6 @@ auto s3_provider::create_directory(const std::string &api_path,
   return set_item_meta(api_path, meta);
 }
 
-auto s3_provider::create_directory_clone_source_meta(
-    const std::string &source_api_path, const std::string &api_path)
-    -> api_error {
-  bool exists{};
-  auto res = is_file(source_api_path, exists);
-  if (res != api_error::success) {
-    return res;
-  }
-  if (exists) {
-    utils::error::raise_api_path_error(__FUNCTION__, api_path,
-                                       api_error::item_exists,
-                                       "failed to create directory");
-    return api_error::item_exists;
-  }
-
-  res = is_directory(api_path, exists);
-  if (res != api_error::success) {
-    return res;
-  }
-  if (exists) {
-    utils::error::raise_api_path_error(__FUNCTION__, api_path,
-                                       api_error::directory_exists,
-                                       "failed to create directory");
-    return api_error::directory_exists;
-  }
-
-  res = is_file(api_path, exists);
-  if (res != api_error::success) {
-    return res;
-  }
-  if (exists) {
-    utils::error::raise_api_path_error(__FUNCTION__, api_path,
-                                       api_error::item_exists,
-                                       "failed to create directory");
-    return api_error::item_exists;
-  }
-
-  api_meta_map meta{};
-  res = get_item_meta(source_api_path, meta);
-  if (res != api_error::success) {
-    if (res == api_error::item_not_found) {
-      res = api_error::directory_not_found;
-    }
-    utils::error::raise_api_path_error(__FUNCTION__, api_path, res,
-                                       "failed to create directory");
-    return res;
-  }
-
-  return create_directory(api_path, meta);
-}
-
 auto s3_provider::create_file(const std::string &api_path, api_meta_map &meta)
     -> api_error {
   bool exists{};
@@ -265,7 +199,7 @@ auto s3_provider::create_file(const std::string &api_path, api_meta_map &meta)
 
   stop_type stop_requested{false};
   try {
-    if (not config_.get_s3_config().encryption_token.empty()) {
+    if (not get_config().get_s3_config().encryption_token.empty()) {
       std::string encrypted_file_path;
       res = get_item_meta(utils::path::get_parent_api_path(api_path), META_KEY,
                           encrypted_file_path);
@@ -277,7 +211,7 @@ auto s3_provider::create_file(const std::string &api_path, api_meta_map &meta)
 
       data_buffer result;
       utils::encryption::encrypt_data(
-          config_.get_s3_config().encryption_token,
+          get_config().get_s3_config().encryption_token,
           *(utils::string::split(api_path, '/', false).end() - 1U), result);
 
       meta[META_KEY] = utils::path::create_api_path(utils::path::combine(
@@ -295,7 +229,7 @@ auto s3_provider::create_file(const std::string &api_path, api_meta_map &meta)
 
     res = upload_file(api_path, meta[META_SOURCE], stop_requested);
     if (res != api_error::success) {
-      db_->Delete(rocksdb::WriteOptions(), api_path);
+      get_db()->Delete(rocksdb::WriteOptions(), api_path);
     }
 
     return res;
@@ -314,7 +248,7 @@ auto s3_provider::create_path_directories(const std::string &api_path,
     return api_error::success;
   }
 
-  const auto encryption_token = config_.get_s3_config().encryption_token;
+  const auto encryption_token = get_config().get_s3_config().encryption_token;
   const auto is_encrypted = not encryption_token.empty();
 
   const auto path_parts = utils::string::split(api_path, '/', false);
@@ -338,7 +272,7 @@ auto s3_provider::create_path_directories(const std::string &api_path,
     auto res = get_item_meta(cur_path, meta);
     if (res == api_error::item_not_found) {
       auto dir = create_api_file(cur_path, cur_key, 0U);
-      api_item_added_(true, dir);
+      get_api_item_added()(true, dir);
       continue;
     }
 
@@ -355,7 +289,7 @@ auto s3_provider::decrypt_object_name(std::string &object_name) const
   auto parts = utils::string::split(object_name, '/', false);
   for (auto &part : parts) {
     auto err = utils::encryption::decrypt_file_name(
-        config_.get_s3_config().encryption_token, part);
+        get_config().get_s3_config().encryption_token, part);
     if (err != api_error::success) {
       return err;
     }
@@ -365,42 +299,10 @@ auto s3_provider::decrypt_object_name(std::string &object_name) const
   return api_error::success;
 }
 
-auto s3_provider::get_api_path_from_source(const std::string &source_path,
-                                           std::string &api_path) const
-    -> api_error {
-  if (source_path.empty()) {
-    utils::error::raise_api_path_error(__FUNCTION__, api_path,
-                                       api_error::item_not_found,
-                                       "failed to source path from api path");
-    return api_error::item_not_found;
-  }
-
-  auto iterator = std::unique_ptr<rocksdb::Iterator>(
-      db_->NewIterator(rocksdb::ReadOptions()));
-  for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
-    std::string current_source_path{};
-    if (get_item_meta(iterator->key().ToString(), META_SOURCE,
-                      current_source_path) != api_error::success) {
-      continue;
-    }
-
-    if (current_source_path.empty()) {
-      continue;
-    }
-
-    if (current_source_path == source_path) {
-      api_path = iterator->key().ToString();
-      return api_error::success;
-    }
-  }
-
-  return api_error::item_not_found;
-}
-
 auto s3_provider::get_directory_item_count(const std::string &api_path) const
     -> std::uint64_t {
   try {
-    const auto cfg = config_.get_s3_config();
+    const auto cfg = get_config().get_s3_config();
     const auto is_encrypted = not cfg.encryption_token.empty();
     std::string key;
     if (is_encrypted) {
@@ -419,8 +321,8 @@ auto s3_provider::get_directory_item_count(const std::string &api_path) const
     long response_code{};
     auto prefix = object_name.empty() ? object_name : object_name + "/";
 
-    if (not get_object_list(comm_, config_.get_s3_config(), response_data,
+    if (not get_object_list(get_comm(), get_config().get_s3_config(),
-                            response_code, "/", prefix)) {
+                            response_data, response_code, "/", prefix)) {
       return 0U;
     }
 
@@ -460,7 +362,7 @@ auto s3_provider::get_directory_items(const std::string &api_path,
                                       directory_item_list &list) const
     -> api_error {
   try {
-    const auto cfg = config_.get_s3_config();
+    const auto cfg = get_config().get_s3_config();
     const auto is_encrypted = not cfg.encryption_token.empty();
     std::string key;
     if (is_encrypted) {
@@ -479,8 +381,8 @@ auto s3_provider::get_directory_items(const std::string &api_path,
     long response_code{};
     auto prefix = object_name.empty() ? object_name : object_name + "/";
 
-    if (not get_object_list(comm_, config_.get_s3_config(), response_data,
+    if (not get_object_list(get_comm(), get_config().get_s3_config(),
-                            response_code, "/", prefix)) {
+                            response_data, response_code, "/", prefix)) {
       return api_error::comm_error;
     }
 
@@ -629,8 +531,8 @@ auto s3_provider::get_file(const std::string &api_path, api_file &file) const
 auto s3_provider::get_file_list(api_file_list &list) const -> api_error {
   std::string response_data;
   long response_code{};
-  if (not get_object_list(comm_, config_.get_s3_config(), response_data,
+  if (not get_object_list(get_comm(), get_config().get_s3_config(),
-                          response_code)) {
+                          response_data, response_code)) {
     return api_error::comm_error;
   }
 
@@ -649,7 +551,8 @@ auto s3_provider::get_file_list(api_file_list &list) const -> api_error {
     auto api_path =
         std::string{node.node().select_node("Key").node().text().as_string()};
     if (not utils::string::ends_with(api_path, "/")) {
-      auto is_encrypted = not config_.get_s3_config().encryption_token.empty();
+      auto is_encrypted =
+          not get_config().get_s3_config().encryption_token.empty();
       if (is_encrypted) {
         auto err = decrypt_object_name(api_path);
         if (err != api_error::success) {
@@ -707,91 +610,12 @@ auto s3_provider::get_file_size(const std::string &api_path,
   return api_error::success;
 }
 
-auto s3_provider::get_filesystem_item(const std::string &api_path,
-                                      bool directory,
-                                      filesystem_item &fsi) const -> api_error {
-  bool exists{};
-  auto res = is_directory(api_path, exists);
-  if (res != api_error::success) {
-    return res;
-  }
-  if (directory && not exists) {
-    return api_error::directory_not_found;
-  }
-
-  res = is_file(api_path, exists);
-  if (res != api_error::success) {
-    return res;
-  }
-  if (not directory && not exists) {
-    return api_error::item_not_found;
-  }
-
-  api_meta_map meta{};
-  res = get_item_meta(api_path, meta);
-  if (res != api_error::success) {
-    return res;
-  }
-
-  fsi.api_parent = utils::path::get_parent_api_path(api_path);
-  fsi.api_path = api_path;
-  fsi.directory = directory;
-  fsi.size = fsi.directory ? 0U : utils::string::to_uint64(meta[META_SIZE]);
-  fsi.source_path = meta[META_SOURCE];
-
-  return api_error::success;
-}
-
-auto s3_provider::get_filesystem_item_and_file(const std::string &api_path,
-                                               api_file &file,
-                                               filesystem_item &fsi) const
-    -> api_error {
-  auto res = get_file(api_path, file);
-  if (res != api_error::success) {
-    return res;
-  }
-
-  api_meta_map meta{};
-  res = get_item_meta(api_path, meta);
-  if (res != api_error::success) {
-    return res;
-  }
-
-  fsi.api_parent = utils::path::get_parent_api_path(api_path);
-  fsi.api_path = api_path;
-  fsi.directory = false;
-  fsi.size = utils::string::to_uint64(meta[META_SIZE]);
-  fsi.source_path = meta[META_SOURCE];
-
-  return api_error::success;
-}
-
-auto s3_provider::get_filesystem_item_from_source_path(
-    const std::string &source_path, filesystem_item &fsi) const -> api_error {
-  std::string api_path{};
-  auto res = get_api_path_from_source(source_path, api_path);
-  if (res != api_error::success) {
-    return res;
-  }
-
-  bool exists{};
-  res = is_directory(api_path, exists);
-  if (res != api_error::success) {
-    return res;
-  }
-  if (exists) {
-    return api_error::directory_exists;
-  }
-
-  return get_filesystem_item(api_path, false, fsi);
-}
-
 auto s3_provider::get_object_info(bool directory, const std::string &api_path,
                                   bool &is_encrypted, std::string &object_name,
                                   head_object_result &result) const
     -> api_error {
   try {
-    const auto cfg = config_.get_s3_config();
+    const auto cfg = get_config().get_s3_config();
     is_encrypted = not cfg.encryption_token.empty();
 
     std::string key;
@@ -812,7 +636,7 @@ auto s3_provider::get_object_info(bool directory, const std::string &api_path,
 
     stop_type stop_requested{false};
     long response_code{};
-    if (not comm_.make_request(head, response_code, stop_requested)) {
+    if (not get_comm().make_request(head, response_code, stop_requested)) {
       return api_error::comm_error;
     }
 
@@ -834,85 +658,15 @@ auto s3_provider::get_object_info(bool directory, const std::string &api_path,
   return api_error::error;
 }
 
-auto s3_provider::get_pinned_files() const -> std::vector<std::string> {
-  std::vector<std::string> ret{};
-
-  auto iterator = std::unique_ptr<rocksdb::Iterator>(
-      db_->NewIterator(rocksdb::ReadOptions()));
-  for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
-    std::string pinned{};
-    if (get_item_meta(iterator->key().ToString(), META_PINNED, pinned) !=
-        api_error::success) {
-      continue;
-    }
-    if (pinned.empty() || not utils::string::to_bool(pinned)) {
-      continue;
-    }
-    ret.emplace_back(iterator->key().ToString());
-  }
-
-  return ret;
-}
-
-auto s3_provider::get_item_meta(const std::string &api_path,
-                                api_meta_map &meta) const -> api_error {
-  std::string meta_value{};
-  db_->Get(rocksdb::ReadOptions(), api_path, &meta_value);
-  if (meta_value.empty()) {
-    return api_error::item_not_found;
-  }
-
-  try {
-    meta = json::parse(meta_value).get<api_meta_map>();
-
-    return api_error::success;
-  } catch (const std::exception &e) {
-    utils::error::raise_api_path_error(__FUNCTION__, api_path, e,
-                                       "failed to get item meta");
-  }
-
-  return api_error::error;
-}
-
-auto s3_provider::get_item_meta(const std::string &api_path,
-                                const std::string &key,
-                                std::string &value) const -> api_error {
-  std::string meta_value{};
-  db_->Get(rocksdb::ReadOptions(), api_path, &meta_value);
-  if (meta_value.empty()) {
-    return api_error::item_not_found;
-  }
-
-  try {
-    value = json::parse(meta_value)[key];
-    return api_error::success;
-  } catch (const std::exception &e) {
-    utils::error::raise_api_path_error(__FUNCTION__, api_path, e,
-                                       "failed to get item meta");
-  }
-
-  return api_error::error;
-}
-
 auto s3_provider::get_total_drive_space() const -> std::uint64_t {
   return std::numeric_limits<std::int64_t>::max() / std::int64_t(2);
 }
 
-auto s3_provider::get_total_item_count() const -> std::uint64_t {
-  std::uint64_t ret{};
-  auto iterator = std::unique_ptr<rocksdb::Iterator>(
-      db_->NewIterator(rocksdb::ReadOptions()));
-  for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
-    ret++;
-  }
-  return ret;
-}
-
 auto s3_provider::get_used_drive_space() const -> std::uint64_t {
   std::string response_data;
   long response_code{};
-  if (not get_object_list(comm_, config_.get_s3_config(), response_data,
+  if (not get_object_list(get_comm(), get_config().get_s3_config(),
-                          response_code)) {
+                          response_data, response_code)) {
     return 0U;
   }
 
@@ -926,7 +680,7 @@ auto s3_provider::get_used_drive_space() const -> std::uint64_t {
     return 0U;
   }
 
-  const auto cfg = config_.get_s3_config();
+  const auto cfg = get_config().get_s3_config();
   const auto is_encrypted = not cfg.encryption_token.empty();
 
   auto node_list = doc.select_nodes("/ListBucketResult/Contents");
@@ -991,16 +745,6 @@ auto s3_provider::is_file(const std::string &api_path, bool &exists) const
   return api_error::error;
 }
 
-auto s3_provider::is_file_writeable(const std::string &api_path) const -> bool {
-  bool exists{};
-  auto res = is_directory(api_path, exists);
-  if (res != api_error::success) {
-    return false;
-  }
-
-  return not exists;
-}
-
 auto s3_provider::is_online() const -> bool {
   // TODO implement this
   return true;
@@ -1010,7 +754,7 @@ auto s3_provider::read_file_bytes(const std::string &api_path, std::size_t size,
                                   std::uint64_t offset, data_buffer &data,
                                   stop_type &stop_requested) -> api_error {
   try {
-    const auto cfg = config_.get_s3_config();
+    const auto cfg = get_config().get_s3_config();
     const auto is_encrypted = not cfg.encryption_token.empty();
     std::string key;
     if (is_encrypted) {
@@ -1030,7 +774,7 @@ auto s3_provider::read_file_bytes(const std::string &api_path, std::size_t size,
       auto res = api_error::error;
       for (std::uint32_t i = 0U;
            not stop_requested && res != api_error::success &&
-           i < config_.get_retry_read_count() + 1U;
+           i < get_config().get_retry_read_count() + 1U;
            i++) {
         curl::requests::http_get get{};
         get.aws_service = "aws:amz:" + cfg.region + ":s3";
@@ -1063,7 +807,7 @@ auto s3_provider::read_file_bytes(const std::string &api_path, std::size_t size,
           std::this_thread::sleep_for(1s);
         };
 
-        if (not comm_.make_request(get, response_code, stop_requested)) {
+        if (not get_comm().make_request(get, response_code, stop_requested)) {
           notify_retry();
           continue;
         }
@@ -1107,98 +851,6 @@ auto s3_provider::read_file_bytes(const std::string &api_path, std::size_t size,
   return api_error::error;
 }
 
-void s3_provider::remove_deleted_files() {
-  struct removed_item {
-    std::string api_path{};
-    bool directory{};
-    std::string source_path{};
-  };
-
-  api_file_list list{};
-  auto res = get_file_list(list);
-  if (res != api_error::success) {
-    utils::error::raise_error(__FUNCTION__, res, "failed to get file list");
-    return;
-  }
-
-  std::vector<removed_item> removed_list{};
-  auto iterator = std::unique_ptr<rocksdb::Iterator>(
-      db_->NewIterator(rocksdb::ReadOptions()));
-  for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
-    api_meta_map meta{};
-    if (get_item_meta(iterator->key().ToString(), meta) == api_error::success) {
-      if (utils::string::to_bool(meta[META_DIRECTORY])) {
-        bool exists{};
-        if (is_directory(iterator->key().ToString(), exists) !=
-            api_error::success) {
-          continue;
-        }
-        if (not exists) {
-          removed_list.emplace_back(
-              removed_item{iterator->key().ToString(), true, ""});
-        }
-        continue;
-      }
-
-      bool exists{};
-      if (is_file(iterator->key().ToString(), exists) != api_error::success) {
-        continue;
-      }
-      if (not exists) {
-        removed_list.emplace_back(
-            removed_item{iterator->key().ToString(), false, meta[META_SOURCE]});
-      }
-    }
-  }
-
-  for (const auto &item : removed_list) {
-    if (not item.directory) {
-      if (utils::file::is_file(item.source_path)) {
-        const auto orphaned_directory =
-            utils::path::combine(config_.get_data_directory(), {"orphaned"});
-        if (utils::file::create_full_directory_path(orphaned_directory)) {
-          const auto parts = utils::string::split(item.api_path, '/', false);
-          const auto orphaned_file = utils::path::combine(
-              orphaned_directory,
-              {utils::path::strip_to_file_name(item.source_path) + '_' +
-               parts[parts.size() - 1U]});
-
-          event_system::instance().raise<orphaned_file_detected>(
-              item.source_path);
-          if (utils::file::reset_modified_time(item.source_path) &&
-              utils::file::copy_file(item.source_path, orphaned_file)) {
-            event_system::instance().raise<orphaned_file_processed>(
-                item.source_path, orphaned_file);
-          } else {
-            event_system::instance().raise<orphaned_file_processing_failed>(
-                item.source_path, orphaned_file,
-                std::to_string(utils::get_last_error_code()));
-          }
-        } else {
-          utils::error::raise_error(
-              __FUNCTION__, std::to_string(utils::get_last_error_code()),
-              "failed to create orphaned director|sp|" + orphaned_directory);
-          continue;
-        }
-      }
-
-      if (fm_->evict_file(item.api_path)) {
-        db_->Delete(rocksdb::WriteOptions(), item.api_path);
-        event_system::instance().raise<file_removed_externally>(
-            item.api_path, item.source_path);
-      }
-    }
-  }
-
-  for (const auto &item : removed_list) {
-    if (item.directory) {
-      db_->Delete(rocksdb::WriteOptions(), item.api_path);
-      event_system::instance().raise<directory_removed_externally>(
-          item.api_path, item.source_path);
-    }
-  }
-}
-
 auto s3_provider::remove_directory(const std::string &api_path) -> api_error {
   const auto notify_end = [&api_path](api_error error) -> api_error {
     if (error == api_error::success) {
@@ -1219,7 +871,7 @@ auto s3_provider::remove_directory(const std::string &api_path) -> api_error {
     return notify_end(api_error::item_not_found);
   }
 
-  const auto cfg = config_.get_s3_config();
+  const auto cfg = get_config().get_s3_config();
   const auto is_encrypted = not cfg.encryption_token.empty();
 
   std::string key;
@@ -1240,7 +892,7 @@ auto s3_provider::remove_directory(const std::string &api_path) -> api_error {
 
   long response_code{};
   stop_type stop_requested{};
-  if (not comm_.make_request(del, response_code, stop_requested)) {
+  if (not get_comm().make_request(del, response_code, stop_requested)) {
     utils::error::raise_api_path_error(__FUNCTION__, api_path,
                                        api_error::comm_error,
                                        "failed to remove directory");
@@ -1255,7 +907,7 @@ auto s3_provider::remove_directory(const std::string &api_path) -> api_error {
     return notify_end(api_error::comm_error);
   }
 
-  auto status = db_->Delete(rocksdb::WriteOptions(), api_path);
+  auto status = get_db()->Delete(rocksdb::WriteOptions(), api_path);
   if (not status.ok()) {
     utils::error::raise_api_path_error(__FUNCTION__, api_path, status.code(),
                                        "failed to remove directory");
@@ -1284,7 +936,7 @@ auto s3_provider::remove_file(const std::string &api_path) -> api_error {
     api_meta_map meta{};
     auto res = get_item_meta(api_path, meta);
 
-    auto res2 = db_->Delete(rocksdb::WriteOptions(), api_path);
+    auto res2 = get_db()->Delete(rocksdb::WriteOptions(), api_path);
     if (not res2.ok()) {
       utils::error::raise_api_path_error(function_name, api_path, res2.code(),
                                          "failed to remove file");
@@ -1314,7 +966,7 @@ auto s3_provider::remove_file(const std::string &api_path) -> api_error {
     return remove_file_meta();
   }
 
-  const auto cfg = config_.get_s3_config();
+  const auto cfg = get_config().get_s3_config();
   const auto is_encrypted = not cfg.encryption_token.empty();
 
   std::string key;
@@ -1335,7 +987,7 @@ auto s3_provider::remove_file(const std::string &api_path) -> api_error {
 
   long response_code{};
   stop_type stop_requested{};
-  if (not comm_.make_request(del, response_code, stop_requested)) {
+  if (not get_comm().make_request(del, response_code, stop_requested)) {
     utils::error::raise_api_path_error(
         __FUNCTION__, api_path, api_error::comm_error, "failed to remove file");
     return notify_end(api_error::comm_error);
@@ -1352,136 +1004,21 @@ auto s3_provider::remove_file(const std::string &api_path) -> api_error {
   return remove_file_meta();
 }
 
-auto s3_provider::remove_item_meta(const std::string &api_path,
-                                   const std::string &key) -> api_error {
-  api_meta_map meta{};
-  auto res = get_item_meta(api_path, meta);
-  if (res != api_error::success) {
-    return res;
-  }
-
-  meta.erase(key);
-
-  auto status = db_->Put(rocksdb::WriteOptions(), api_path, json(meta).dump());
-  if (not status.ok()) {
-    utils::error::raise_api_path_error(__FUNCTION__, api_path, status.code(),
-                                       "failed to remove item meta");
-    return api_error::error;
-  }
-
-  return api_error::success;
-}
-
 auto s3_provider::rename_file(const std::string & /* from_api_path */,
                               const std::string & /* to_api_path */)
     -> api_error {
   return api_error::not_implemented;
 }
 
-auto s3_provider::set_item_meta(const std::string &api_path,
-                                const std::string &key,
-                                const std::string &value) -> api_error {
-  json meta_json{};
-  std::string meta_value{};
-  db_->Get(rocksdb::ReadOptions(), api_path, &meta_value);
-  if (not meta_value.empty()) {
-    try {
-      meta_json = json::parse(meta_value);
-    } catch (const std::exception &e) {
-      utils::error::raise_api_path_error(__FUNCTION__, api_path, e,
-                                         "failed to set item meta");
-      return api_error::error;
-    }
-  }
-
-  meta_json[key] = value;
-
-  auto status = db_->Put(rocksdb::WriteOptions(), api_path, meta_json.dump());
-  if (not status.ok()) {
-    utils::error::raise_api_path_error(__FUNCTION__, api_path, status.code(),
-                                       "failed to set item meta");
-    return api_error::error;
-  }
-
-  return api_error::success;
-}
-
-auto s3_provider::set_item_meta(const std::string &api_path,
-                                const api_meta_map &meta) -> api_error {
-  json meta_json{};
-  std::string meta_value{};
-  db_->Get(rocksdb::ReadOptions(), api_path, &meta_value);
-  if (not meta_value.empty()) {
-    try {
-      meta_json = json::parse(meta_value);
-    } catch (const std::exception &e) {
-      utils::error::raise_api_path_error(__FUNCTION__, api_path, e,
-                                         "failed to set item meta");
-      return api_error::error;
-    }
-  }
-
-  for (const auto &item : meta) {
-    meta_json[item.first] = item.second;
-  }
-
-  auto status = db_->Put(rocksdb::WriteOptions(), api_path, meta_json.dump());
-  if (not status.ok()) {
-    utils::error::raise_api_path_error(__FUNCTION__, api_path, status.code(),
-                                       "failed to set item meta");
-    return api_error::error;
-  }
-
-  return api_error::success;
-}
-
 auto s3_provider::start(api_item_added_callback api_item_added,
                         i_file_manager *mgr) -> bool {
   event_system::instance().raise<service_started>("s3_provider");
-  utils::db::create_rocksdb(config_, DB_NAME, db_);
+  return base_provider::start(api_item_added, mgr);
-
-  api_item_added_ = api_item_added;
-  fm_ = mgr;
-
-  api_meta_map meta{};
-  if (get_item_meta("/", meta) == api_error::item_not_found) {
-    auto dir = create_api_file("/", "", 0U);
-    api_item_added_(true, dir);
-  }
-
-  auto online{false};
-  auto unmount_requested{false};
-  {
-    repertory::event_consumer consumer(
-        "unmount_requested",
-        [&unmount_requested](const event &) { unmount_requested = true; });
-    for (std::uint16_t i = 0U; not online && not unmount_requested &&
-                               (i < config_.get_online_check_retry_secs());
-         i++) {
-      online = is_online();
-      if (not online) {
-        auto cfg = curl_comm::create_host_config(
-            config_.get_s3_config(), config_.get_s3_config().use_path_style);
-        event_system::instance().raise<provider_offline>(cfg.host_name_or_ip,
-                                                         cfg.api_port);
-        std::this_thread::sleep_for(1s);
-      }
-    }
-  }
-
-  if (online && not unmount_requested) {
-    polling::instance().set_callback({"check_deleted", polling::frequency::low,
-                                      [this]() { remove_deleted_files(); }});
-    return true;
-  }
-
-  return false;
 }
 
 void s3_provider::stop() {
   event_system::instance().raise<service_shutdown_begin>("s3_provider");
-  polling::instance().remove_callback("check_deleted");
+  base_provider::stop();
-  db_.reset();
   event_system::instance().raise<service_shutdown_end>("s3_provider");
 }
 
@@ -1503,7 +1040,7 @@ auto s3_provider::upload_file(const std::string &api_path,
       return notify_end(api_error::comm_error);
     }
 
-    const auto cfg = config_.get_s3_config();
+    const auto cfg = get_config().get_s3_config();
     const auto is_encrypted = not cfg.encryption_token.empty();
 
     std::string key;
@@ -1530,7 +1067,7 @@ auto s3_provider::upload_file(const std::string &api_path,
     }
 
     long response_code{};
-    if (not comm_.make_request(put_file, response_code, stop_requested)) {
+    if (not get_comm().make_request(put_file, response_code, stop_requested)) {
       return notify_end(api_error::comm_error);
     }
 

src/providers/sia/sia_provider.cpp

@@ -25,6 +25,7 @@
 #include "comm/i_http_comm.hpp"
 #include "events/events.hpp"
 #include "file_manager/i_file_manager.hpp"
+#include "providers/base_provider.hpp"
 #include "types/repertory.hpp"
 #include "utils/error_utils.hpp"
 #include "utils/file_utils.hpp"
@@ -36,7 +37,7 @@
 
 namespace repertory {
 sia_provider::sia_provider(app_config &config, i_http_comm &comm)
-    : config_(config), comm_(comm) {}
+    : base_provider(config, comm) {}
 
 auto sia_provider::get_object_info(const std::string &api_path,
                                    json &object_info) const -> api_error {
@@ -54,7 +55,7 @@ auto sia_provider::get_object_info(const std::string &api_path,
 
     long response_code{};
     stop_type stop_requested{};
-    if (not comm_.make_request(get, response_code, stop_requested)) {
+    if (not get_comm().make_request(get, response_code, stop_requested)) {
       return api_error::comm_error;
     }
 
@@ -92,7 +93,7 @@ auto sia_provider::get_object_list(const std::string &api_path,
 
   long response_code{};
   stop_type stop_requested{};
-  if (not comm_.make_request(get, response_code, stop_requested)) {
+  if (not get_comm().make_request(get, response_code, stop_requested)) {
     utils::error::raise_api_path_error(__FUNCTION__, api_path,
                                        api_error::comm_error,
                                        "failed to get object list");
@@ -108,37 +109,6 @@ auto sia_provider::get_object_list(const std::string &api_path,
   return true;
 }
 
-auto sia_provider::create_api_file(std::string path, std::uint64_t size)
-    -> api_file {
-  api_file file{};
-  file.api_path = utils::path::create_api_path(path);
-  file.api_parent = utils::path::get_parent_api_path(file.api_path);
-  file.accessed_date = utils::get_file_time_now();
-  file.changed_date = utils::get_file_time_now();
-  file.creation_date = utils::get_file_time_now();
-  file.modified_date = utils::get_file_time_now();
-  file.file_size = size;
-  return file;
-}
-
-auto sia_provider::create_api_file(std::string path, std::uint64_t size,
-                                   api_meta_map &meta) -> api_file {
-  auto current_size = utils::string::to_uint64(meta[META_SIZE]);
-  if (current_size == 0U) {
-    current_size = size;
-  }
-
-  api_file file{};
-  file.api_path = utils::path::create_api_path(path);
-  file.api_parent = utils::path::get_parent_api_path(file.api_path);
-  file.accessed_date = utils::string::to_uint64(meta[META_ACCESSED]);
-  file.changed_date = utils::string::to_uint64(meta[META_CHANGED]);
-  file.creation_date = utils::string::to_uint64(meta[META_CREATION]);
-  file.file_size = current_size;
-  file.modified_date = utils::string::to_uint64(meta[META_MODIFIED]);
-  return file;
-}
-
 auto sia_provider::create_directory(const std::string &api_path,
                                     api_meta_map &meta) -> api_error {
   bool exists{};
@@ -170,7 +140,7 @@ auto sia_provider::create_directory(const std::string &api_path,
 
     long response_code{};
     stop_type stop_requested{};
-    if (not comm_.make_request(put_file, response_code, stop_requested)) {
+    if (not get_comm().make_request(put_file, response_code, stop_requested)) {
       utils::error::raise_api_path_error(__FUNCTION__, api_path,
                                          api_error::comm_error,
                                          "failed to create directory");
@@ -192,57 +162,6 @@ auto sia_provider::create_directory(const std::string &api_path,
   return set_item_meta(api_path, meta);
 }
 
-auto sia_provider::create_directory_clone_source_meta(
-    const std::string &source_api_path, const std::string &api_path)
-    -> api_error {
-  bool exists{};
-  auto res = is_file(source_api_path, exists);
-  if (res != api_error::success) {
-    return res;
-  }
-  if (exists) {
-    utils::error::raise_api_path_error(__FUNCTION__, api_path,
-                                       api_error::item_exists,
-                                       "failed to create directory");
-    return api_error::item_exists;
-  }
-
-  res = is_directory(api_path, exists);
-  if (res != api_error::success) {
-    return res;
-  }
-  if (exists) {
-    utils::error::raise_api_path_error(__FUNCTION__, api_path,
-                                       api_error::directory_exists,
-                                       "failed to create directory");
-    return api_error::directory_exists;
-  }
-
-  res = is_file(api_path, exists);
-  if (res != api_error::success) {
-    return res;
-  }
-  if (exists) {
-    utils::error::raise_api_path_error(__FUNCTION__, api_path,
-                                       api_error::item_exists,
-                                       "failed to create directory");
-    return api_error::item_exists;
-  }
-
-  api_meta_map meta{};
-  res = get_item_meta(source_api_path, meta);
-  if (res != api_error::success) {
-    if (res == api_error::item_not_found) {
-      res = api_error::directory_not_found;
-    }
-    utils::error::raise_api_path_error(__FUNCTION__, api_path, res,
-                                       "failed to create directory");
-    return res;
-  }
-
-  return create_directory(api_path, meta);
-}
-
 auto sia_provider::create_file(const std::string &api_path, api_meta_map &meta)
     -> api_error {
   bool exists{};
@@ -274,7 +193,7 @@ auto sia_provider::create_file(const std::string &api_path, api_meta_map &meta)
 
     long response_code{};
     stop_type stop_requested{};
-    if (not comm_.make_request(put_file, response_code, stop_requested)) {
+    if (not get_comm().make_request(put_file, response_code, stop_requested)) {
       utils::error::raise_api_path_error(__FUNCTION__, api_path,
                                          api_error::comm_error,
                                          "failed to create file");
@@ -296,38 +215,6 @@ auto sia_provider::create_file(const std::string &api_path, api_meta_map &meta)
   return set_item_meta(api_path, meta);
 }
 
-auto sia_provider::get_api_path_from_source(const std::string &source_path,
-                                            std::string &api_path) const
-    -> api_error {
-  if (source_path.empty()) {
-    utils::error::raise_api_path_error(__FUNCTION__, api_path,
-                                       api_error::item_not_found,
-                                       "failed to source path from api path");
-    return api_error::item_not_found;
-  }
-
-  auto iterator = std::unique_ptr<rocksdb::Iterator>(
-      db_->NewIterator(rocksdb::ReadOptions()));
-  for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
-    std::string current_source_path{};
-    if (get_item_meta(iterator->key().ToString(), META_SOURCE,
-                      current_source_path) != api_error::success) {
-      continue;
-    }
-
-    if (current_source_path.empty()) {
-      continue;
-    }
-
-    if (current_source_path == source_path) {
-      api_path = iterator->key().ToString();
-      return api_error::success;
-    }
-  }
-
-  return api_error::item_not_found;
-}
-
 auto sia_provider::get_directory_item_count(const std::string &api_path) const
     -> std::uint64_t {
   try {
@@ -399,9 +286,9 @@ auto sia_provider::get_directory_items(const std::string &api_path,
           if (get_item_meta(entry_api_path, meta) ==
               api_error::item_not_found) {
             file = create_api_file(
-                entry_api_path,
+                entry_api_path, "",
                 directory ? 0U : entry["size"].get<std::uint64_t>());
-            api_item_added_(directory, file);
+            get_api_item_added()(directory, file);
             res = get_item_meta(entry_api_path, meta);
             if (res != api_error::success) {
               utils::error::raise_error(__FUNCTION__, res,
@@ -414,14 +301,14 @@ auto sia_provider::get_directory_items(const std::string &api_path,
                 directory ? 0U : entry["size"].get<std::uint64_t>(), meta);
           }
 
-          directory_item di{};
+          directory_item dir_item{};
-          di.api_parent = file.api_parent;
+          dir_item.api_parent = file.api_parent;
-          di.api_path = file.api_path;
+          dir_item.api_path = file.api_path;
-          di.directory = directory;
+          dir_item.directory = directory;
-          di.meta = meta;
+          dir_item.meta = meta;
-          di.resolved = true;
+          dir_item.resolved = true;
-          di.size = file.file_size;
+          dir_item.size = file.file_size;
-          list.emplace_back(std::move(di));
+          list.emplace_back(std::move(dir_item));
         } catch (const std::exception &e) {
           utils::error::raise_api_path_error(__FUNCTION__, api_path, e,
                                              "failed to process entry|" +
@@ -472,8 +359,8 @@ auto sia_provider::get_file(const std::string &api_path, api_file &file) const
 
   api_meta_map meta{};
   if (get_item_meta(api_path, meta) == api_error::item_not_found) {
-    file = create_api_file(api_path, size);
+    file = create_api_file(api_path, "", size);
-    api_item_added_(false, file);
+    get_api_item_added()(false, file);
   } else {
     file = create_api_file(api_path, size, meta);
   }
@@ -503,8 +390,8 @@ auto sia_provider::get_file_list(api_file_list &list) const -> api_error {
             api_meta_map meta{};
             if (get_item_meta(entry_api_path, meta) ==
                 api_error::item_not_found) {
-              auto dir = create_api_file(entry_api_path, 0U);
+              auto dir = create_api_file(entry_api_path, "", 0U);
-              api_item_added_(true, dir);
+              get_api_item_added()(true, dir);
             }
 
             auto res = get_files_in_dir(entry_api_path);
@@ -518,9 +405,9 @@ auto sia_provider::get_file_list(api_file_list &list) const -> api_error {
           api_meta_map meta{};
           if (get_item_meta(entry_api_path, meta) ==
               api_error::item_not_found) {
-            file = create_api_file(entry_api_path,
+            file = create_api_file(entry_api_path, "",
                                    entry["size"].get<std::uint64_t>());
-            api_item_added_(false, file);
+            get_api_item_added()(false, file);
           } else {
             file = create_api_file(entry_api_path,
                                    entry["size"].get<std::uint64_t>(), meta);
@@ -563,146 +450,6 @@ auto sia_provider::get_file_size(const std::string &api_path,
   return api_error::success;
 }
 
-auto sia_provider::get_filesystem_item(const std::string &api_path,
-                                       bool directory,
-                                       filesystem_item &fsi) const
-    -> api_error {
-  bool exists{};
-  auto res = is_directory(api_path, exists);
-  if (res != api_error::success) {
-    return res;
-  }
-  if (directory && not exists) {
-    return api_error::directory_not_found;
-  }
-
-  res = is_file(api_path, exists);
-  if (res != api_error::success) {
-    return res;
-  }
-  if (not directory && not exists) {
-    return api_error::item_not_found;
-  }
-
-  api_meta_map meta{};
-  res = get_item_meta(api_path, meta);
-  if (res != api_error::success) {
-    return res;
-  }
-
-  fsi.api_parent = utils::path::get_parent_api_path(api_path);
-  fsi.api_path = api_path;
-  fsi.directory = directory;
-  fsi.size = fsi.directory ? 0U : utils::string::to_uint64(meta[META_SIZE]);
-  fsi.source_path = meta[META_SOURCE];
-
-  return api_error::success;
-}
-
-auto sia_provider::get_filesystem_item_and_file(const std::string &api_path,
-                                                api_file &file,
-                                                filesystem_item &fsi) const
-    -> api_error {
-  auto res = get_file(api_path, file);
-  if (res != api_error::success) {
-    return res;
-  }
-
-  api_meta_map meta{};
-  res = get_item_meta(api_path, meta);
-  if (res != api_error::success) {
-    return res;
-  }
-
-  fsi.api_parent = utils::path::get_parent_api_path(api_path);
-  fsi.api_path = api_path;
-  fsi.directory = false;
-  fsi.size = utils::string::to_uint64(meta[META_SIZE]);
-  fsi.source_path = meta[META_SOURCE];
-
-  return api_error::success;
-}
-
-auto sia_provider::get_filesystem_item_from_source_path(
-    const std::string &source_path, filesystem_item &fsi) const -> api_error {
-  std::string api_path{};
-  auto res = get_api_path_from_source(source_path, api_path);
-  if (res != api_error::success) {
-    return res;
-  }
-
-  bool exists{};
-  res = is_directory(api_path, exists);
-  if (res != api_error::success) {
-    return res;
-  }
-  if (exists) {
-    return api_error::directory_exists;
-  }
-
-  return get_filesystem_item(api_path, false, fsi);
-}
-
-auto sia_provider::get_item_meta(const std::string &api_path,
-                                 api_meta_map &meta) const -> api_error {
-  std::string meta_value{};
-  db_->Get(rocksdb::ReadOptions(), api_path, &meta_value);
-  if (meta_value.empty()) {
-    return api_error::item_not_found;
-  }
-
-  try {
-    meta = json::parse(meta_value).get<api_meta_map>();
-
-    return api_error::success;
-  } catch (const std::exception &e) {
-    utils::error::raise_api_path_error(__FUNCTION__, api_path, e,
-                                       "failed to get item meta");
-  }
-
-  return api_error::error;
-}
-
-auto sia_provider::get_item_meta(const std::string &api_path,
-                                 const std::string &key,
-                                 std::string &value) const -> api_error {
-  std::string meta_value{};
-  db_->Get(rocksdb::ReadOptions(), api_path, &meta_value);
-  if (meta_value.empty()) {
-    return api_error::item_not_found;
-  }
-
-  try {
-    value = json::parse(meta_value)[key];
-    return api_error::success;
-  } catch (const std::exception &e) {
-    utils::error::raise_api_path_error(__FUNCTION__, api_path, e,
-                                       "failed to get item meta");
-  }
-
-  return api_error::error;
-}
-
-auto sia_provider::get_pinned_files() const -> std::vector<std::string> {
-  std::vector<std::string> ret{};
-
-  auto iterator = std::unique_ptr<rocksdb::Iterator>(
-      db_->NewIterator(rocksdb::ReadOptions()));
-  for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
-    std::string pinned{};
-    if (get_item_meta(iterator->key().ToString(), META_PINNED, pinned) !=
-        api_error::success) {
-      continue;
-    }
-    if (pinned.empty() || not utils::string::to_bool(pinned)) {
-      continue;
-    }
-    ret.emplace_back(iterator->key().ToString());
-  }
-
-  return ret;
-}
-
 auto sia_provider::get_total_drive_space() const -> std::uint64_t {
   try {
     curl::requests::http_get get{};
@@ -719,7 +466,7 @@ auto sia_provider::get_total_drive_space() const -> std::uint64_t {
 
     long response_code{};
     stop_type stop_requested{};
-    if (not comm_.make_request(get, response_code, stop_requested)) {
+    if (not get_comm().make_request(get, response_code, stop_requested)) {
       return 0U;
     }
 
@@ -738,16 +485,6 @@ auto sia_provider::get_total_drive_space() const -> std::uint64_t {
   return 0U;
 }
 
-auto sia_provider::get_total_item_count() const -> std::uint64_t {
-  std::uint64_t ret{};
-  auto iterator = std::unique_ptr<rocksdb::Iterator>(
-      db_->NewIterator(rocksdb::ReadOptions()));
-  for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
-    ret++;
-  }
-  return ret;
-}
-
 auto sia_provider::get_used_drive_space() const -> std::uint64_t {
   // TODO adjust size based on open files
   try {
@@ -765,7 +502,7 @@ auto sia_provider::get_used_drive_space() const -> std::uint64_t {
 
     long response_code{};
     stop_type stop_requested{};
-    if (not comm_.make_request(get, response_code, stop_requested)) {
+    if (not get_comm().make_request(get, response_code, stop_requested)) {
       return 0U;
     }
 
@@ -776,7 +513,7 @@ auto sia_provider::get_used_drive_space() const -> std::uint64_t {
     }
 
     auto used_space = object_data["totalObjectsSize"].get<std::uint64_t>();
-    fm_->update_used_space(used_space);
+    get_file_mgr()->update_used_space(used_space);
     return used_space;
   } catch (const std::exception &ex) {
     utils::error::raise_error(__FUNCTION__, ex,
@@ -845,17 +582,6 @@ auto sia_provider::is_file(const std::string &api_path, bool &exists) const
   return api_error::error;
 }
 
-auto sia_provider::is_file_writeable(const std::string &api_path) const
-    -> bool {
-  bool exists{};
-  auto res = is_directory(api_path, exists);
-  if (res != api_error::success) {
-    return false;
-  }
-
-  return not exists;
-}
-
 auto sia_provider::is_online() const -> bool {
   try {
     curl::requests::http_get get{};
@@ -872,7 +598,7 @@ auto sia_provider::is_online() const -> bool {
 
     long response_code{};
     stop_type stop_requested{};
-    if (not comm_.make_request(get, response_code, stop_requested)) {
+    if (not get_comm().make_request(get, response_code, stop_requested)) {
       utils::error::raise_error(__FUNCTION__, api_error::comm_error,
                                 "failed to determine if provider is online");
       return false;
@@ -910,7 +636,7 @@ auto sia_provider::read_file_bytes(const std::string &api_path,
 
   auto res = api_error::comm_error;
   for (std::uint32_t i = 0U; not stop_requested && res != api_error::success &&
-                             i < config_.get_retry_read_count() + 1U;
+                             i < get_config().get_retry_read_count() + 1U;
        i++) {
     long response_code{};
     const auto notify_retry = [&]() {
@@ -930,7 +656,7 @@ auto sia_provider::read_file_bytes(const std::string &api_path,
       std::this_thread::sleep_for(1s);
     };
 
-    if (not comm_.make_request(get, response_code, stop_requested)) {
+    if (not get_comm().make_request(get, response_code, stop_requested)) {
       notify_retry();
       continue;
     }
@@ -947,98 +673,6 @@ auto sia_provider::read_file_bytes(const std::string &api_path,
   return res;
 }
 
-void sia_provider::remove_deleted_files() {
-  struct removed_item {
-    std::string api_path{};
-    bool directory{};
-    std::string source_path{};
-  };
-
-  api_file_list list{};
-  auto res = get_file_list(list);
-  if (res != api_error::success) {
-    utils::error::raise_error(__FUNCTION__, res, "failed to get file list");
-    return;
-  }
-
-  std::vector<removed_item> removed_list{};
-  auto iterator = std::unique_ptr<rocksdb::Iterator>(
-      db_->NewIterator(rocksdb::ReadOptions()));
-  for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
-    api_meta_map meta{};
-    if (get_item_meta(iterator->key().ToString(), meta) == api_error::success) {
-      if (utils::string::to_bool(meta[META_DIRECTORY])) {
-        bool exists{};
-        if (is_directory(iterator->key().ToString(), exists) !=
-            api_error::success) {
-          continue;
-        }
-        if (not exists) {
-          removed_list.emplace_back(
-              removed_item{iterator->key().ToString(), true, ""});
-        }
-        continue;
-      }
-
-      bool exists{};
-      if (is_file(iterator->key().ToString(), exists) != api_error::success) {
-        continue;
-      }
-      if (not exists) {
-        removed_list.emplace_back(
-            removed_item{iterator->key().ToString(), false, meta[META_SOURCE]});
-      }
-    }
-  }
-
-  for (const auto &item : removed_list) {
-    if (not item.directory) {
-      if (utils::file::is_file(item.source_path)) {
-        const auto orphaned_directory =
-            utils::path::combine(config_.get_data_directory(), {"orphaned"});
-        if (utils::file::create_full_directory_path(orphaned_directory)) {
-          const auto parts = utils::string::split(item.api_path, '/', false);
-          const auto orphaned_file = utils::path::combine(
-              orphaned_directory,
-              {utils::path::strip_to_file_name(item.source_path) + '_' +
-               parts[parts.size() - 1U]});
-
-          event_system::instance().raise<orphaned_file_detected>(
-              item.source_path);
-          if (utils::file::reset_modified_time(item.source_path) &&
-              utils::file::copy_file(item.source_path, orphaned_file)) {
-            event_system::instance().raise<orphaned_file_processed>(
-                item.source_path, orphaned_file);
-          } else {
-            event_system::instance().raise<orphaned_file_processing_failed>(
-                item.source_path, orphaned_file,
-                std::to_string(utils::get_last_error_code()));
-          }
-        } else {
-          utils::error::raise_error(
-              __FUNCTION__, std::to_string(utils::get_last_error_code()),
-              "failed to create orphaned director|sp|" + orphaned_directory);
-          continue;
-        }
-      }
-
-      if (fm_->evict_file(item.api_path)) {
-        db_->Delete(rocksdb::WriteOptions(), item.api_path);
-        event_system::instance().raise<file_removed_externally>(
-            item.api_path, item.source_path);
-      }
-    }
-  }
-
-  for (const auto &item : removed_list) {
-    if (item.directory) {
-      db_->Delete(rocksdb::WriteOptions(), item.api_path);
-      event_system::instance().raise<directory_removed_externally>(
-          item.api_path, item.source_path);
-    }
-  }
-}
-
 auto sia_provider::remove_directory(const std::string &api_path) -> api_error {
   const auto notify_end = [&api_path](api_error error) -> api_error {
     if (error == api_error::success) {
@@ -1065,7 +699,7 @@ auto sia_provider::remove_directory(const std::string &api_path) -> api_error {
 
   long response_code{};
   stop_type stop_requested{};
-  if (not comm_.make_request(del, response_code, stop_requested)) {
+  if (not get_comm().make_request(del, response_code, stop_requested)) {
     utils::error::raise_api_path_error(__FUNCTION__, api_path,
                                        api_error::comm_error,
                                        "failed to remove directory");
@@ -1078,7 +712,7 @@ auto sia_provider::remove_directory(const std::string &api_path) -> api_error {
     return notify_end(api_error::comm_error);
   }
 
-  auto res2 = db_->Delete(rocksdb::WriteOptions(), api_path);
+  auto res2 = get_db()->Delete(rocksdb::WriteOptions(), api_path);
   if (not res2.ok()) {
     utils::error::raise_api_path_error(__FUNCTION__, api_path, res2.code(),
                                        "failed to remove directory");
@@ -1104,7 +738,7 @@ auto sia_provider::remove_file(const std::string &api_path) -> api_error {
     api_meta_map meta{};
     auto res = get_item_meta(api_path, meta);
 
-    auto res2 = db_->Delete(rocksdb::WriteOptions(), api_path);
+    auto res2 = get_db()->Delete(rocksdb::WriteOptions(), api_path);
     if (not res2.ok()) {
       utils::error::raise_api_path_error(__FUNCTION__, api_path, res2.code(),
                                          "failed to remove file");
@@ -1140,7 +774,7 @@ auto sia_provider::remove_file(const std::string &api_path) -> api_error {
 
   long response_code{};
   stop_type stop_requested{};
-  if (not comm_.make_request(del, response_code, stop_requested)) {
+  if (not get_comm().make_request(del, response_code, stop_requested)) {
     utils::error::raise_api_path_error(
         __FUNCTION__, api_path, api_error::comm_error, "failed to remove file");
     return notify_end(api_error::comm_error);
@@ -1156,139 +790,19 @@ auto sia_provider::remove_file(const std::string &api_path) -> api_error {
   return remove_file_meta();
 }
 
-auto sia_provider::remove_item_meta(const std::string &api_path,
-                                    const std::string &key) -> api_error {
-  api_meta_map meta{};
-  auto res = get_item_meta(api_path, meta);
-  if (res != api_error::success) {
-    return res;
-  }
-
-  meta.erase(key);
-
-  auto res2 = db_->Put(rocksdb::WriteOptions(), api_path, json(meta).dump());
-  if (not res2.ok()) {
-    utils::error::raise_api_path_error(__FUNCTION__, api_path, res2.code(),
-                                       "failed to remove item meta");
-    return api_error::error;
-  }
-
-  return api_error::success;
-}
-
 auto sia_provider::rename_file(const std::string & /*from_api_path*/,
                                const std::string & /*to_api_path*/)
     -> api_error {
   return api_error::not_implemented;
 }
 
-auto sia_provider::set_item_meta(const std::string &api_path,
-                                 const std::string &key,
-                                 const std::string &value) -> api_error {
-  json meta_json{};
-  std::string meta_value{};
-  db_->Get(rocksdb::ReadOptions(), api_path, &meta_value);
-  if (not meta_value.empty()) {
-    try {
-      meta_json = json::parse(meta_value);
-    } catch (const std::exception &e) {
-      utils::error::raise_api_path_error(__FUNCTION__, api_path, e,
-                                         "failed to set item meta");
-      return api_error::error;
-    }
-  }
-
-  meta_json[key] = value;
-
-  const auto res =
-      db_->Put(rocksdb::WriteOptions(), api_path, meta_json.dump());
-  if (not res.ok()) {
-    utils::error::raise_api_path_error(__FUNCTION__, api_path, res.code(),
-                                       "failed to set item meta");
-    return api_error::error;
-  }
-
-  return api_error::success;
-}
-
-auto sia_provider::set_item_meta(const std::string &api_path,
-                                 const api_meta_map &meta) -> api_error {
-  json meta_json{};
-  std::string meta_value{};
-  db_->Get(rocksdb::ReadOptions(), api_path, &meta_value);
-  if (not meta_value.empty()) {
-    try {
-      meta_json = json::parse(meta_value);
-    } catch (const std::exception &e) {
-      utils::error::raise_api_path_error(__FUNCTION__, api_path, e,
-                                         "failed to set item meta");
-      return api_error::error;
-    }
-  }
-
-  for (const auto &kv : meta) {
-    meta_json[kv.first] = kv.second;
-  }
-
-  const auto res =
-      db_->Put(rocksdb::WriteOptions(), api_path, meta_json.dump());
-  if (not res.ok()) {
-    utils::error::raise_api_path_error(__FUNCTION__, api_path, res.code(),
-                                       "failed to set item meta");
-    return api_error::error;
-  }
-
-  return api_error::success;
-}
-
 auto sia_provider::start(api_item_added_callback api_item_added,
                          i_file_manager *mgr) -> bool {
   event_system::instance().raise<service_started>("sia_provider");
-  utils::db::create_rocksdb(config_, DB_NAME, db_);
+  return base_provider::start(api_item_added, mgr);
-
-  api_item_added_ = api_item_added;
-  fm_ = mgr;
-
-  api_meta_map meta{};
-  if (get_item_meta("/", meta) == api_error::item_not_found) {
-    auto dir = create_api_file("/", 0U);
-    api_item_added_(true, dir);
-  }
-
-  auto online = false;
-  auto unmount_requested = false;
-  {
-    repertory::event_consumer ec(
-        "unmount_requested",
-        [&unmount_requested](const event &) { unmount_requested = true; });
-    for (std::uint16_t i = 0u; not online && not unmount_requested &&
-                               (i < config_.get_online_check_retry_secs());
-         i++) {
-      online = is_online();
-      if (not online) {
-        event_system::instance().raise<provider_offline>(
-            config_.get_host_config().host_name_or_ip,
-            config_.get_host_config().api_port);
-        std::this_thread::sleep_for(1s);
-      }
-    }
-  }
-
-  if (online && not unmount_requested) {
-    polling::instance().set_callback({"check_deleted", polling::frequency::low,
-                                      [this]() { remove_deleted_files(); }});
-    return true;
-  }
-
-  return false;
 }
 
-void sia_provider::stop() {
+void sia_provider::stop() { return base_provider::stop(); }
-  event_system::instance().raise<service_shutdown_begin>("sia_provider");
-  polling::instance().remove_callback("check_deleted");
-  db_.reset();
-  event_system::instance().raise<service_shutdown_end>("sia_provider");
-}
 
 auto sia_provider::upload_file(const std::string &api_path,
                                const std::string &source_path,
@@ -1308,7 +822,7 @@ auto sia_provider::upload_file(const std::string &api_path,
     put_file.source_path = source_path;
 
     long response_code{};
-    if (not comm_.make_request(put_file, response_code, stop_requested)) {
+    if (not get_comm().make_request(put_file, response_code, stop_requested)) {
       utils::error::raise_api_path_error(__FUNCTION__, api_path, source_path,
                                          api_error::comm_error,
                                          "failed to upload file");