v2.0.2-rc (#27)

## v2.0.2-rc

### BREAKING CHANGES

* Refactored `config.json` - will need to verify configuration settings prior to mounting

### Issues

* \#12 \[Unit Test\] Complete all providers unit tests
* \#14 \[Unit Test\] SQLite mini-ORM unit tests and cleanup
* \#16 Add support for bucket name in Sia provider
* \#17 Update to common c++ build system
  * A single 64-bit Linux Jenkins server is used to build all Linux and Windows versions
  * All dependency sources are now included
  * MSVC is no longer supported
  * MSYS2 is required for building Windows binaries on Windows
  * OS X support is temporarily disabled
* \#19 \[bug\] Rename file is broken for files that are existing
* \#23 \[bug\] Incorrect file size displayed while upload is pending
* \#24 RocksDB implementations should be transactional
* \#25 Writes should block when maximum cache size is reached
* \#26 Complete ring buffer and direct download support

### Changes from v2.0.1-rc

* Ability to choose between RocksDB and SQLite databases
* Added direct reads and implemented download fallback
* Corrected file times on S3 and Sia providers
* Corrected handling of `chown()` and `chmod()`
* Fixed erroneous download of chunks after resize

Reviewed-on: #27

repertory/librepertory/include/providers/base_provider.hpp

@@ -0,0 +1,225 @@
+/*
+  Copyright <2018-2024> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all
+  copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  SOFTWARE.
+*/
+#ifndef REPERTORY_INCLUDE_PROVIDERS_BASE_PROVIDER_HPP_
+#define REPERTORY_INCLUDE_PROVIDERS_BASE_PROVIDER_HPP_
+
+#include "db/i_meta_db.hpp"
+#include "providers/i_provider.hpp"
+#include "types/repertory.hpp"
+
+namespace repertory {
+class app_config;
+class i_file_manager;
+class i_http_comm;
+
+class base_provider : public i_provider {
+private:
+  struct removed_item final {
+    std::string api_path;
+    bool directory{};
+    std::string source_path;
+  };
+
+public:
+  base_provider(app_config &config, i_http_comm &comm)
+      : config_(config), comm_(comm) {}
+
+private:
+  app_config &config_;
+  i_http_comm &comm_;
+
+private:
+  api_item_added_callback api_item_added_;
+  std::unique_ptr<i_meta_db> db3_;
+  i_file_manager *fm_{};
+
+private:
+  void add_all_items(const stop_type &stop_requested);
+
+  void process_removed_directories(std::deque<removed_item> removed_list,
+                                   const stop_type &stop_requested);
+
+  void process_removed_files(std::deque<removed_item> removed_list,
+                             const stop_type &stop_requested);
+
+  void process_removed_items(const stop_type &stop_requested);
+
+  void remove_deleted_items(const stop_type &stop_requested);
+
+  void remove_unmatched_source_files(const stop_type &stop_requested);
+
+protected:
+  [[nodiscard]] static auto create_api_file(std::string path, std::string key,
+                                            std::uint64_t size,
+                                            std::uint64_t file_time)
+      -> api_file;
+
+  [[nodiscard]] static auto create_api_file(std::string path,
+                                            std::uint64_t size,
+                                            api_meta_map &meta) -> api_file;
+
+  [[nodiscard]] virtual auto create_directory_impl(const std::string &api_path,
+                                                   api_meta_map &meta)
+      -> api_error = 0;
+
+  [[nodiscard]] virtual auto
+  create_file_extra(const std::string & /* api_path */,
+                    api_meta_map & /* meta */) -> api_error {
+    return api_error::success;
+  }
+
+  [[nodiscard]] auto get_api_item_added() -> api_item_added_callback & {
+    return api_item_added_;
+  }
+
+  [[nodiscard]] auto get_api_item_added() const
+      -> const api_item_added_callback & {
+    return api_item_added_;
+  }
+
+  [[nodiscard]] auto get_comm() -> i_http_comm & { return comm_; }
+
+  [[nodiscard]] auto get_comm() const -> const i_http_comm & { return comm_; }
+
+  [[nodiscard]] auto get_config() -> app_config & { return config_; }
+
+  [[nodiscard]] auto get_config() const -> const app_config & {
+    return config_;
+  }
+
+  [[nodiscard]] auto get_db() -> i_meta_db & { return *db3_; }
+
+  [[nodiscard]] auto get_db() const -> const i_meta_db & { return *db3_; }
+
+  [[nodiscard]] virtual auto
+  get_directory_items_impl(const std::string &api_path,
+                           directory_item_list &list) const -> api_error = 0;
+
+  [[nodiscard]] auto get_file_mgr() -> i_file_manager * { return fm_; }
+
+  [[nodiscard]] auto get_file_mgr() const -> const i_file_manager * {
+    return fm_;
+  }
+
+  [[nodiscard]] virtual auto remove_directory_impl(const std::string &api_path)
+      -> api_error = 0;
+
+  [[nodiscard]] virtual auto remove_file_impl(const std::string &api_path)
+      -> api_error = 0;
+
+  [[nodiscard]] virtual auto upload_file_impl(const std::string &api_path,
+                                              const std::string &source_path,
+                                              stop_type &stop_requested)
+      -> api_error = 0;
+
+public:
+  [[nodiscard]] auto
+  create_directory_clone_source_meta(const std::string &source_api_path,
+                                     const std::string &api_path)
+      -> api_error override;
+
+  [[nodiscard]] auto create_directory(const std::string &api_path,
+                                      api_meta_map &meta) -> api_error override;
+
+  [[nodiscard]] auto create_file(const std::string &api_path,
+                                 api_meta_map &meta) -> api_error override;
+
+  [[nodiscard]] auto get_api_path_from_source(const std::string &source_path,
+                                              std::string &api_path) const
+      -> api_error override;
+
+  [[nodiscard]] auto get_directory_items(const std::string &api_path,
+                                         directory_item_list &list) const
+      -> api_error override;
+
+  [[nodiscard]] auto get_file_size(const std::string &api_path,
+                                   std::uint64_t &file_size) const
+      -> api_error override;
+
+  [[nodiscard]] auto get_filesystem_item(const std::string &api_path,
+                                         bool directory,
+                                         filesystem_item &fsi) const
+      -> api_error override;
+
+  [[nodiscard]] auto get_filesystem_item_and_file(const std::string &api_path,
+                                                  api_file &f,
+                                                  filesystem_item &fsi) const
+      -> api_error override;
+
+  [[nodiscard]] auto
+  get_filesystem_item_from_source_path(const std::string &source_path,
+                                       filesystem_item &fsi) const
+      -> api_error override;
+
+  [[nodiscard]] auto get_item_meta(const std::string &api_path,
+                                   api_meta_map &meta) const
+      -> api_error override;
+
+  [[nodiscard]] auto get_item_meta(const std::string &api_path,
+                                   const std::string &key,
+                                   std::string &value) const
+      -> api_error override;
+
+  [[nodiscard]] auto get_pinned_files() const
+      -> std::vector<std::string> override;
+
+  [[nodiscard]] auto get_total_item_count() const -> std::uint64_t override;
+
+  [[nodiscard]] auto get_used_drive_space() const -> std::uint64_t override;
+
+  [[nodiscard]] auto is_file_writeable(const std::string &api_path) const
+      -> bool override;
+
+  [[nodiscard]] auto is_read_only() const -> bool override { return false; }
+
+  [[nodiscard]] auto remove_directory(const std::string &api_path)
+      -> api_error override;
+
+  [[nodiscard]] auto remove_file(const std::string &api_path)
+      -> api_error override;
+
+  [[nodiscard]] auto remove_item_meta(const std::string &api_path,
+                                      const std::string &key)
+      -> api_error override;
+
+  [[nodiscard]] auto set_item_meta(const std::string &api_path,
+                                   const std::string &key,
+                                   const std::string &value)
+      -> api_error override;
+
+  [[nodiscard]] auto set_item_meta(const std::string &api_path,
+                                   const api_meta_map &meta)
+      -> api_error override;
+
+  [[nodiscard]] auto start(api_item_added_callback api_item_added,
+                           i_file_manager *mgr) -> bool override;
+
+  void stop() override;
+
+  [[nodiscard]] auto upload_file(const std::string &api_path,
+                                 const std::string &source_path,
+                                 stop_type &stop_requested)
+      -> api_error override;
+};
+} // namespace repertory
+
+#endif // REPERTORY_INCLUDE_PROVIDERS_BASE_PROVIDER_HPP_

repertory/librepertory/include/providers/encrypt/encrypt_provider.hpp

@@ -0,0 +1,236 @@
+/*
+  Copyright <2018-2024> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all
+  copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  SOFTWARE.
+*/
+#ifndef REPERTORY_INCLUDE_PROVIDERS_ENCRYPT_ENCRYPT_PROVIDER_HPP_
+#define REPERTORY_INCLUDE_PROVIDERS_ENCRYPT_ENCRYPT_PROVIDER_HPP_
+
+#include "app_config.hpp"
+#include "db/i_file_db.hpp"
+#include "providers/i_provider.hpp"
+#include "utils/encrypting_reader.hpp"
+
+namespace repertory {
+class encrypt_provider final : public i_provider {
+public:
+  static const constexpr auto type{provider_type::encrypt};
+
+public:
+  explicit encrypt_provider(app_config &config);
+
+  ~encrypt_provider() override = default;
+
+public:
+  encrypt_provider(const encrypt_provider &) = delete;
+  encrypt_provider(encrypt_provider &&) = delete;
+  auto operator=(const encrypt_provider &) -> encrypt_provider & = delete;
+  auto operator=(encrypt_provider &&) -> encrypt_provider & = delete;
+
+private:
+  struct reader_info final {
+    std::chrono::system_clock::time_point last_access_time{
+        std::chrono::system_clock::now(),
+    };
+    std::unique_ptr<utils::encryption::encrypting_reader> reader;
+    std::mutex reader_mtx;
+  };
+
+private:
+  app_config &config_;
+  encrypt_config encrypt_config_;
+
+private:
+  std::unique_ptr<i_file_db> db_{nullptr};
+  i_file_manager *fm_{nullptr};
+  std::unordered_map<std::string, std::shared_ptr<reader_info>> reader_lookup_;
+  std::recursive_mutex reader_lookup_mtx_;
+
+private:
+  static auto create_api_file(const std::string &api_path, bool directory,
+                              const std::string &source_path) -> api_file;
+
+  static void create_item_meta(api_meta_map &meta, bool directory,
+                               const api_file &file);
+
+  auto do_fs_operation(const std::string &api_path, bool directory,
+                       std::function<api_error(const encrypt_config &cfg,
+                                               const std::string &source_path)>
+                           callback) const -> api_error;
+
+  [[nodiscard]] auto get_encrypt_config() const -> const encrypt_config & {
+    return encrypt_config_;
+  }
+
+  auto process_directory_entry(const utils::file::i_fs_item &dir_entry,
+                               const encrypt_config &cfg,
+                               std::string &api_path) const -> bool;
+
+  void remove_deleted_files(const stop_type &stop_requested);
+
+public:
+  [[nodiscard]] auto create_directory(const std::string &api_path,
+                                      api_meta_map &meta) -> api_error override;
+
+  [[nodiscard]] auto
+  create_directory_clone_source_meta(const std::string & /*source_api_path*/,
+                                     const std::string & /*api_path*/)
+      -> api_error override {
+    return api_error::not_implemented;
+  }
+
+  [[nodiscard]] auto create_file(const std::string & /*api_path*/,
+                                 api_meta_map & /*meta*/)
+      -> api_error override {
+    return api_error::not_implemented;
+  }
+
+  [[nodiscard]] auto
+  get_api_path_from_source(const std::string & /*source_path*/,
+                           std::string & /*api_path*/) const
+      -> api_error override;
+
+  [[nodiscard]] auto get_directory_item_count(const std::string &api_path) const
+      -> std::uint64_t override;
+
+  [[nodiscard]] auto get_directory_items(const std::string &api_path,
+                                         directory_item_list &list) const
+      -> api_error override;
+
+  [[nodiscard]] auto get_file(const std::string &api_path, api_file &file) const
+      -> api_error override;
+
+  [[nodiscard]] auto get_file_list(api_file_list &list,
+                                   std::string &marker) const
+      -> api_error override;
+
+  [[nodiscard]] auto get_file_size(const std::string &api_path,
+                                   std::uint64_t &file_size) const
+      -> api_error override;
+
+  [[nodiscard]] auto get_filesystem_item(const std::string &api_path,
+                                         bool directory,
+                                         filesystem_item &fsi) const
+      -> api_error override;
+
+  [[nodiscard]] auto get_filesystem_item_and_file(const std::string &api_path,
+                                                  api_file &file,
+                                                  filesystem_item &fsi) const
+      -> api_error override;
+
+  [[nodiscard]] auto
+  get_filesystem_item_from_source_path(const std::string &source_path,
+                                       filesystem_item &fsi) const
+      -> api_error override;
+
+  [[nodiscard]] auto get_pinned_files() const
+      -> std::vector<std::string> override;
+
+  [[nodiscard]] auto get_item_meta(const std::string &api_path,
+                                   api_meta_map &meta) const
+      -> api_error override;
+
+  [[nodiscard]] auto get_item_meta(const std::string &api_path,
+                                   const std::string &key,
+                                   std::string &value) const
+      -> api_error override;
+
+  [[nodiscard]] auto get_total_drive_space() const -> std::uint64_t override;
+
+  [[nodiscard]] auto get_total_item_count() const -> std::uint64_t override;
+
+  [[nodiscard]] auto get_provider_type() const -> provider_type override {
+    return type;
+  }
+
+  [[nodiscard]] auto get_used_drive_space() const -> std::uint64_t override;
+
+  [[nodiscard]] auto is_directory(const std::string &api_path,
+                                  bool &exists) const -> api_error override;
+
+  [[nodiscard]] auto is_file(const std::string &api_path, bool &exists) const
+      -> api_error override;
+
+  [[nodiscard]] auto is_file_writeable(const std::string &api_path) const
+      -> bool override;
+
+  [[nodiscard]] auto is_online() const -> bool override;
+
+  [[nodiscard]] auto is_read_only() const -> bool override { return true; }
+
+  [[nodiscard]] auto is_rename_supported() const -> bool override {
+    return false;
+  }
+
+  [[nodiscard]] auto read_file_bytes(const std::string &api_path,
+                                     std::size_t size, std::uint64_t offset,
+                                     data_buffer &data,
+                                     stop_type &stop_requested)
+      -> api_error override;
+
+  [[nodiscard]] auto remove_directory(const std::string & /*api_path*/)
+      -> api_error override {
+    return api_error::not_implemented;
+  }
+
+  [[nodiscard]] auto remove_file(const std::string & /*api_path*/)
+      -> api_error override {
+    return api_error::not_implemented;
+  }
+
+  [[nodiscard]] auto remove_item_meta(const std::string & /*api_path*/,
+                                      const std::string & /*key*/)
+      -> api_error override {
+    return api_error::success;
+  }
+
+  [[nodiscard]] auto rename_file(const std::string & /*from_api_path*/,
+                                 const std::string & /*to_api_path*/)
+      -> api_error override {
+    return api_error::not_implemented;
+  }
+
+  [[nodiscard]] auto set_item_meta(const std::string & /*api_path*/,
+                                   const std::string & /*key*/,
+                                   const std::string & /*value*/)
+      -> api_error override {
+    return api_error::success;
+  }
+
+  [[nodiscard]] auto set_item_meta(const std::string & /*api_path*/,
+                                   const api_meta_map & /*meta*/)
+      -> api_error override {
+    return api_error::success;
+  }
+
+  [[nodiscard]] auto start(api_item_added_callback api_item_added,
+                           i_file_manager *mgr) -> bool override;
+
+  void stop() override;
+
+  [[nodiscard]] auto upload_file(const std::string & /*api_path*/,
+                                 const std::string & /*source_path*/,
+                                 stop_type & /*stop_requested*/)
+      -> api_error override {
+    return api_error::not_implemented;
+  }
+};
+} // namespace repertory
+
+#endif // REPERTORY_INCLUDE_PROVIDERS_ENCRYPT_ENCRYPT_PROVIDER_HPP_

repertory/librepertory/include/providers/i_provider.hpp

@@ -0,0 +1,158 @@
+/*
+  Copyright <2018-2024> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all
+  copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  SOFTWARE.
+*/
+#ifndef REPERTORY_INCLUDE_PROVIDERS_I_PROVIDER_HPP_
+#define REPERTORY_INCLUDE_PROVIDERS_I_PROVIDER_HPP_
+
+#include "types/repertory.hpp"
+
+namespace repertory {
+class i_file_manager;
+
+class i_provider {
+  INTERFACE_SETUP(i_provider);
+
+public:
+  [[nodiscard]] virtual auto create_directory(const std::string &api_path,
+                                              api_meta_map &meta)
+      -> api_error = 0;
+
+  [[nodiscard]] virtual auto
+  create_directory_clone_source_meta(const std::string &source_api_path,
+                                     const std::string &api_path)
+      -> api_error = 0;
+
+  [[nodiscard]] virtual auto create_file(const std::string &api_path,
+                                         api_meta_map &meta) -> api_error = 0;
+
+  [[nodiscard]] virtual auto
+  get_api_path_from_source(const std::string &source_path,
+                           std::string &api_path) const -> api_error = 0;
+
+  [[nodiscard]] virtual auto
+  get_directory_item_count(const std::string &api_path) const
+      -> std::uint64_t = 0;
+
+  [[nodiscard]] virtual auto
+  get_directory_items(const std::string &api_path,
+                      directory_item_list &list) const -> api_error = 0;
+
+  [[nodiscard]] virtual auto get_file(const std::string &api_path,
+                                      api_file &file) const -> api_error = 0;
+
+  [[nodiscard]] virtual auto get_file_list(api_file_list &list,
+                                           std::string &marker) const
+      -> api_error = 0;
+
+  [[nodiscard]] virtual auto get_file_size(const std::string &api_path,
+                                           std::uint64_t &file_size) const
+      -> api_error = 0;
+
+  [[nodiscard]] virtual auto get_filesystem_item(const std::string &api_path,
+                                                 bool directory,
+                                                 filesystem_item &fsi) const
+      -> api_error = 0;
+
+  [[nodiscard]] virtual auto
+  get_filesystem_item_and_file(const std::string &api_path, api_file &file,
+                               filesystem_item &fsi) const -> api_error = 0;
+
+  [[nodiscard]] virtual auto
+  get_filesystem_item_from_source_path(const std::string &source_path,
+                                       filesystem_item &fsi) const
+      -> api_error = 0;
+
+  [[nodiscard]] virtual auto get_item_meta(const std::string &api_path,
+                                           api_meta_map &meta) const
+      -> api_error = 0;
+
+  [[nodiscard]] virtual auto get_item_meta(const std::string &api_path,
+                                           const std::string &key,
+                                           std::string &value) const
+      -> api_error = 0;
+
+  [[nodiscard]] virtual auto get_pinned_files() const
+      -> std::vector<std::string> = 0;
+
+  [[nodiscard]] virtual auto get_provider_type() const -> provider_type = 0;
+
+  [[nodiscard]] virtual auto get_total_drive_space() const -> std::uint64_t = 0;
+
+  [[nodiscard]] virtual auto get_total_item_count() const -> std::uint64_t = 0;
+
+  [[nodiscard]] virtual auto get_used_drive_space() const -> std::uint64_t = 0;
+
+  [[nodiscard]] virtual auto is_directory(const std::string &api_path,
+                                          bool &exists) const -> api_error = 0;
+
+  [[nodiscard]] virtual auto is_file(const std::string &api_path,
+                                     bool &exists) const -> api_error = 0;
+
+  [[nodiscard]] virtual auto
+  is_file_writeable(const std::string &api_path) const -> bool = 0;
+
+  [[nodiscard]] virtual auto is_online() const -> bool = 0;
+
+  [[nodiscard]] virtual auto is_read_only() const -> bool = 0;
+
+  [[nodiscard]] virtual auto is_rename_supported() const -> bool = 0;
+
+  [[nodiscard]] virtual auto
+  read_file_bytes(const std::string &api_path, std::size_t size,
+                  std::uint64_t offset, data_buffer &data,
+                  stop_type &stop_requested) -> api_error = 0;
+
+  [[nodiscard]] virtual auto remove_directory(const std::string &api_path)
+      -> api_error = 0;
+
+  [[nodiscard]] virtual auto remove_file(const std::string &api_path)
+      -> api_error = 0;
+
+  [[nodiscard]] virtual auto remove_item_meta(const std::string &api_path,
+                                              const std::string &key)
+      -> api_error = 0;
+
+  [[nodiscard]] virtual auto rename_file(const std::string &from_api_path,
+                                         const std::string &to_api_path)
+      -> api_error = 0;
+
+  [[nodiscard]] virtual auto set_item_meta(const std::string &api_path,
+                                           const std::string &key,
+                                           const std::string &value)
+      -> api_error = 0;
+
+  [[nodiscard]] virtual auto set_item_meta(const std::string &api_path,
+                                           const api_meta_map &meta)
+      -> api_error = 0;
+
+  [[nodiscard]] virtual auto start(api_item_added_callback api_item_added,
+                                   i_file_manager *mgr) -> bool = 0;
+
+  virtual void stop() = 0;
+
+  [[nodiscard]] virtual auto upload_file(const std::string &api_path,
+                                         const std::string &source_path,
+                                         stop_type &stop_requested)
+      -> api_error = 0;
+};
+} // namespace repertory
+
+#endif // REPERTORY_INCLUDE_PROVIDERS_I_PROVIDER_HPP_

repertory/librepertory/include/providers/provider.hpp

@@ -0,0 +1,36 @@
+/*
+  Copyright <2018-2024> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all
+  copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  SOFTWARE.
+*/
+#ifndef REPERTORY_INCLUDE_PROVIDERS_PROVIDER_HPP_
+#define REPERTORY_INCLUDE_PROVIDERS_PROVIDER_HPP_
+
+#include "types/repertory.hpp"
+
+namespace repertory {
+class app_config;
+class i_provider;
+
+[[nodiscard]] auto
+create_provider(const provider_type &prov,
+                app_config &config) -> std::unique_ptr<i_provider>;
+} // namespace repertory
+
+#endif // REPERTORY_INCLUDE_PROVIDERS_PROVIDER_HPP_

repertory/librepertory/include/providers/s3/s3_provider.hpp

@@ -0,0 +1,157 @@
+/*
+  Copyright <2018-2024> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all
+  copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  SOFTWARE.
+*/
+#ifndef REPERTORY_INCLUDE_PROVIDERS_S3_S3_PROVIDER_HPP_
+#define REPERTORY_INCLUDE_PROVIDERS_S3_S3_PROVIDER_HPP_
+
+#include "providers/base_provider.hpp"
+#include "types/repertory.hpp"
+
+namespace repertory {
+class app_config;
+class i_file_manager;
+class i_http_comm;
+struct head_object_result;
+
+class s3_provider final : public base_provider {
+public:
+  static const constexpr auto type{provider_type::s3};
+
+public:
+  s3_provider(app_config &config, i_http_comm &comm);
+
+  ~s3_provider() override = default;
+
+public:
+  s3_provider(const s3_provider &) = delete;
+  s3_provider(s3_provider &&) = delete;
+  auto operator=(const s3_provider &) -> s3_provider & = delete;
+  auto operator=(s3_provider &&) -> s3_provider & = delete;
+
+private:
+  s3_config s3_config_;
+
+private:
+  [[nodiscard]] auto add_if_not_found(api_file &file,
+                                      const std::string &object_name) const
+      -> api_error;
+
+  [[nodiscard]] auto create_file_extra(const std::string &api_path,
+                                       api_meta_map &meta)
+      -> api_error override;
+
+  [[nodiscard]] auto create_path_directories(const std::string &api_path,
+                                             const std::string &key) const
+      -> api_error;
+
+  [[nodiscard]] auto decrypt_object_name(std::string &object_name) const
+      -> api_error;
+
+  [[nodiscard]] auto get_last_modified(bool directory,
+                                       const std::string &api_path) const
+      -> std::uint64_t;
+
+  [[nodiscard]] auto
+  get_object_info(bool directory, const std::string &api_path,
+                  bool &is_encrypted, std::string &object_name,
+                  head_object_result &result) const -> api_error;
+
+  [[nodiscard]] auto
+  get_object_list(std::string &response_data, long &response_code,
+                  std::optional<std::string> delimiter = std::nullopt,
+                  std::optional<std::string> prefix = std::nullopt,
+                  std::optional<std::string> token = std::nullopt) const
+      -> bool;
+
+  [[nodiscard]] auto get_s3_config() const -> const s3_config & {
+    return s3_config_;
+  }
+
+protected:
+  [[nodiscard]] auto create_directory_impl(const std::string &api_path,
+                                           api_meta_map &meta)
+      -> api_error override;
+
+  [[nodiscard]] auto get_directory_items_impl(const std::string &api_path,
+                                              directory_item_list &list) const
+      -> api_error override;
+
+  [[nodiscard]] auto remove_directory_impl(const std::string &api_path)
+      -> api_error override;
+
+  [[nodiscard]] auto remove_file_impl(const std::string &api_path)
+      -> api_error override;
+
+  [[nodiscard]] auto upload_file_impl(const std::string &api_path,
+                                      const std::string &source_path,
+                                      stop_type &stop_requested)
+      -> api_error override;
+
+public:
+  [[nodiscard]] static auto convert_api_date(std::string_view date)
+      -> std::uint64_t;
+
+  [[nodiscard]] auto get_directory_item_count(const std::string &api_path) const
+      -> std::uint64_t override;
+
+  [[nodiscard]] auto get_file(const std::string &api_path, api_file &file) const
+      -> api_error override;
+
+  [[nodiscard]] auto get_file_list(api_file_list &list,
+                                   std::string &marker) const
+      -> api_error override;
+
+  [[nodiscard]] auto get_total_drive_space() const -> std::uint64_t override;
+
+  [[nodiscard]] auto get_provider_type() const -> provider_type override {
+    return type;
+  }
+
+  [[nodiscard]] auto is_directory(const std::string &api_path,
+                                  bool &exists) const -> api_error override;
+
+  [[nodiscard]] auto is_file(const std::string &api_path, bool &exists) const
+      -> api_error override;
+
+  [[nodiscard]] auto is_online() const -> bool override;
+
+  [[nodiscard]] auto is_rename_supported() const -> bool override {
+    return false;
+  };
+
+  [[nodiscard]] auto read_file_bytes(const std::string &api_path,
+                                     std::size_t size, std::uint64_t offset,
+                                     data_buffer &data,
+                                     stop_type &stop_requested)
+      -> api_error override;
+
+  [[nodiscard]] auto rename_file(const std::string &from_api_path,
+                                 const std::string &to_api_path)
+      -> api_error override;
+
+  [[nodiscard]] auto start(api_item_added_callback api_item_added,
+                           i_file_manager *mgr) -> bool override;
+
+  void stop() override;
+};
+} // namespace repertory
+
+#endif // REPERTORY_INCLUDE_PROVIDERS_S3_S3_PROVIDER_HPP_

repertory/librepertory/include/providers/sia/sia_provider.hpp

@@ -0,0 +1,128 @@
+/*
+  Copyright <2018-2024> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all
+  copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  SOFTWARE.
+*/
+#ifndef REPERTORY_INCLUDE_PROVIDERS_SIA_SIA_PROVIDER_HPP_
+#define REPERTORY_INCLUDE_PROVIDERS_SIA_SIA_PROVIDER_HPP_
+
+#include "providers/base_provider.hpp"
+#include "types/repertory.hpp"
+
+namespace repertory {
+class app_config;
+class i_file_manager;
+class i_http_comm;
+
+class sia_provider : public base_provider {
+public:
+  static const constexpr auto type{provider_type::sia};
+
+public:
+  sia_provider(app_config &config, i_http_comm &comm);
+
+  ~sia_provider() override = default;
+
+public:
+  sia_provider(const sia_provider &) = delete;
+  sia_provider(sia_provider &&) = delete;
+  auto operator=(const sia_provider &) -> sia_provider & = delete;
+  auto operator=(sia_provider &&) -> sia_provider & = delete;
+
+private:
+  sia_config sia_config_;
+
+private:
+  [[nodiscard]] auto get_object_info(const std::string &api_path,
+                                     json &object_info) const -> api_error;
+
+  [[nodiscard]] auto get_object_list(const std::string &api_path,
+                                     nlohmann::json &object_list) const -> bool;
+
+  [[nodiscard]] auto get_sia_config() const -> const auto & {
+    return sia_config_;
+  }
+
+protected:
+  [[nodiscard]] auto create_directory_impl(const std::string &api_path,
+                                           api_meta_map &meta)
+      -> api_error override;
+
+  [[nodiscard]] auto get_directory_items_impl(const std::string &api_path,
+                                              directory_item_list &list) const
+      -> api_error override;
+
+  [[nodiscard]] auto remove_directory_impl(const std::string &api_path)
+      -> api_error override;
+
+  [[nodiscard]] auto remove_file_impl(const std::string &api_path)
+      -> api_error override;
+
+  [[nodiscard]] auto upload_file_impl(const std::string &api_path,
+                                      const std::string &source_path,
+                                      stop_type &stop_requested)
+      -> api_error override;
+
+public:
+  [[nodiscard]] auto get_directory_item_count(const std::string &api_path) const
+      -> std::uint64_t override;
+
+  [[nodiscard]] auto get_file(const std::string &api_path, api_file &file) const
+      -> api_error override;
+
+  [[nodiscard]] auto get_file_list(api_file_list &list,
+                                   std::string &marker) const
+      -> api_error override;
+
+  [[nodiscard]] auto get_provider_type() const -> provider_type override {
+    return type;
+  }
+
+  [[nodiscard]] auto get_total_drive_space() const -> std::uint64_t override;
+
+  [[nodiscard]] auto is_directory(const std::string &api_path,
+                                  bool &exists) const -> api_error override;
+
+  [[nodiscard]] auto is_file(const std::string &api_path, bool &exists) const
+      -> api_error override;
+
+  [[nodiscard]] auto is_online() const -> bool override;
+
+  [[nodiscard]] auto is_rename_supported() const -> bool override {
+    return true;
+  }
+
+  [[nodiscard]] auto read_file_bytes(const std::string &api_path,
+                                     std::size_t size, std::uint64_t offset,
+                                     data_buffer &buffer,
+                                     stop_type &stop_requested)
+      -> api_error override;
+
+  [[nodiscard]] auto rename_file(const std::string &from_api_path,
+                                 const std::string &to_api_path)
+      -> api_error override;
+
+  [[nodiscard]] auto start(api_item_added_callback api_item_added,
+                           i_file_manager *mgr) -> bool override;
+
+  void stop() override;
+};
+} // namespace repertory
+
+#endif // REPERTORY_INCLUDE_PROVIDERS_SIA_SIA_PROVIDER_HPP_