initial commit

include/providers/base_provider.hpp

@@ -0,0 +1,123 @@
+/*
+  Copyright <2018-2022> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
+  associated documentation files (the "Software"), to deal in the Software without restriction,
+  including without limitation the rights to use, copy, modify, merge, publish, distribute,
+  sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all copies or
+  substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
+  NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
+  OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/
+#ifndef INCLUDE_PROVIDERS_BASE_PROVIDER_HPP_
+#define INCLUDE_PROVIDERS_BASE_PROVIDER_HPP_
+
+#include "common.hpp"
+#include "db/meta_db.hpp"
+#include "providers/i_provider.hpp"
+
+namespace repertory {
+class app_config;
+class base_provider : public i_provider {
+public:
+  explicit base_provider(app_config &config);
+
+  ~base_provider() override = default;
+
+private:
+  app_config &config_;
+
+protected:
+  api_item_added_callback api_item_added_;
+  meta_db meta_db_;
+  mutable std::recursive_mutex notify_added_mutex_;
+  i_open_file_table *oft_ = nullptr;
+
+protected:
+  app_config &get_config() { return config_; }
+
+  app_config &get_config() const { return config_; }
+
+  virtual std::string format_api_path(std::string api_path) const { return api_path; }
+
+  virtual std::string &restore_api_path(std::string &api_path) const { return api_path; }
+
+  void notify_directory_added(const std::string &api_path, const std::string &api_parent) const {
+    const_cast<base_provider *>(this)->notify_directory_added(api_path, api_parent);
+  }
+
+  virtual void notify_directory_added(const std::string &api_path, const std::string &api_parent);
+
+  api_error notify_file_added(const std::string &api_path, const std::string &api_parent,
+                              const std::uint64_t &size) const {
+    return const_cast<base_provider *>(this)->notify_file_added(api_path, api_parent, size);
+  }
+
+  virtual api_error notify_file_added(const std::string &api_path, const std::string &api_parent,
+                                      const std::uint64_t &size) = 0;
+
+  void remove_item_meta(const std::string &api_path) {
+    return meta_db_.remove_item_meta(format_api_path(api_path));
+  }
+
+  void update_filesystem_item(const bool &directory, const api_error &error,
+                              const std::string &api_path, filesystem_item &fsi) const;
+
+public:
+  api_error create_directory_clone_source_meta(const std::string &source_api_path,
+                                               const std::string &api_path) override;
+
+  api_error create_file(const std::string &api_path, api_meta_map &meta) override;
+
+  api_error get_api_path_from_source(const std::string &source_path,
+                                     std::string &api_path) const override;
+
+  api_error get_filesystem_item(const std::string &api_path, const bool &directory,
+                                filesystem_item &fsi) const override;
+
+  api_error get_filesystem_item_and_file(const std::string &api_path, api_file &file,
+                                         filesystem_item &fsi) const override;
+
+  api_error get_filesystem_item_from_source_path(const std::string &source_path,
+                                                 filesystem_item &fsi) const override;
+
+  api_error get_item_meta(const std::string &api_path, api_meta_map &meta) const override;
+
+  api_error get_item_meta(const std::string &api_path, const std::string &key,
+                          std::string &value) const override;
+
+  std::vector<std::string> get_pinned_files() const override { return meta_db_.get_pinned_files(); }
+
+  std::uint64_t get_used_drive_space() const override;
+
+  bool is_file_writeable(const std::string &) const override { return true; }
+
+  api_error remove_item_meta(const std::string &api_path, const std::string &key) override {
+    return meta_db_.remove_item_meta(format_api_path(api_path), key);
+  }
+
+  api_error set_item_meta(const std::string &api_path, const std::string &key,
+                          const std::string &value) override {
+    return meta_db_.set_item_meta(format_api_path(api_path), key, value);
+  }
+
+  api_error set_item_meta(const std::string &api_path, const api_meta_map &meta) override {
+    return meta_db_.set_item_meta(format_api_path(api_path), meta);
+  }
+
+  api_error set_source_path(const std::string &api_path, const std::string &source_path) override {
+    return meta_db_.set_source_path(format_api_path(api_path), source_path);
+  }
+
+  bool start(api_item_added_callback api_item_added, i_open_file_table *oft) override;
+};
+} // namespace repertory
+
+#endif // INCLUDE_PROVIDERS_BASE_PROVIDER_HPP_

include/providers/i_provider.hpp

@@ -0,0 +1,115 @@
+/*
+  Copyright <2018-2022> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
+  associated documentation files (the "Software"), to deal in the Software without restriction,
+  including without limitation the rights to use, copy, modify, merge, publish, distribute,
+  sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all copies or
+  substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
+  NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
+  OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/
+#ifndef INCLUDE_PROVIDERS_I_PROVIDER_HPP_
+#define INCLUDE_PROVIDERS_I_PROVIDER_HPP_
+
+#include "common.hpp"
+#include "types/repertory.hpp"
+
+namespace repertory {
+class i_open_file_table;
+class i_provider {
+  INTERFACE_SETUP(i_provider);
+
+public:
+  virtual api_error create_directory(const std::string &api_path, const api_meta_map &meta) = 0;
+
+  virtual api_error create_directory_clone_source_meta(const std::string &source_api_path,
+                                                       const std::string &api_path) = 0;
+
+  virtual api_error create_file(const std::string &api_path, api_meta_map &meta) = 0;
+
+  virtual api_error get_api_path_from_source(const std::string &source_path,
+                                             std::string &api_path) const = 0;
+
+  virtual api_error get_file_list(api_file_list &list) const = 0;
+
+  virtual std::uint64_t get_directory_item_count(const std::string &api_path) const = 0;
+
+  virtual api_error get_directory_items(const std::string &api_path,
+                                        directory_item_list &list) const = 0;
+
+  virtual api_error get_file(const std::string &api_path, api_file &file) const = 0;
+
+  virtual api_error get_file_size(const std::string &api_path, std::uint64_t &file_size) const = 0;
+
+  virtual api_error get_filesystem_item(const std::string &api_path, const bool &directory,
+                                        filesystem_item &fsi) const = 0;
+
+  virtual api_error get_filesystem_item_and_file(const std::string &api_path, api_file &file,
+                                                 filesystem_item &fsi) const = 0;
+
+  virtual api_error get_filesystem_item_from_source_path(const std::string &source_path,
+                                                         filesystem_item &fsi) const = 0;
+
+  virtual std::vector<std::string> get_pinned_files() const = 0;
+
+  virtual api_error get_item_meta(const std::string &api_path, api_meta_map &meta) const = 0;
+
+  virtual api_error get_item_meta(const std::string &api_path, const std::string &key,
+                                  std::string &value) const = 0;
+
+  virtual std::uint64_t get_total_drive_space() const = 0;
+
+  virtual std::uint64_t get_total_item_count() const = 0;
+
+  virtual provider_type get_provider_type() const = 0;
+
+  virtual std::uint64_t get_used_drive_space() const = 0;
+
+  virtual bool is_directory(const std::string &api_path) const = 0;
+
+  virtual bool is_file(const std::string &api_path) const = 0;
+
+  virtual bool is_file_writeable(const std::string &api_path) const = 0;
+
+  virtual bool is_online() const = 0;
+
+  virtual bool is_rename_supported() const = 0;
+
+  virtual api_error read_file_bytes(const std::string &api_path, const std::size_t &size,
+                                    const std::uint64_t &offset, std::vector<char> &data,
+                                    const bool &stop_requested) = 0;
+
+  virtual api_error remove_directory(const std::string &api_path) = 0;
+
+  virtual api_error remove_file(const std::string &api_path) = 0;
+
+  virtual api_error remove_item_meta(const std::string &api_path, const std::string &key) = 0;
+
+  virtual api_error rename_file(const std::string &fromApiPath, const std::string &toApiPath) = 0;
+
+  virtual api_error set_item_meta(const std::string &api_path, const std::string &key,
+                                  const std::string &value) = 0;
+
+  virtual api_error set_item_meta(const std::string &api_path, const api_meta_map &meta) = 0;
+
+  virtual api_error set_source_path(const std::string &api_path,
+                                    const std::string &source_path) = 0;
+
+  virtual bool start(api_item_added_callback api_item_added, i_open_file_table *oft) = 0;
+
+  virtual void stop() = 0;
+
+  virtual api_error upload_file(const std::string &api_path, const std::string &source_path,
+                               const std::string &encryption_token) = 0;
+};
+} // namespace repertory
+
+#endif // INCLUDE_PROVIDERS_I_PROVIDER_HPP_

include/providers/passthrough/passthroughprovider.hpp

@@ -0,0 +1,115 @@
+/*
+  Copyright <2018-2022> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
+  associated documentation files (the "Software"), to deal in the Software without restriction,
+  including without limitation the rights to use, copy, modify, merge, publish, distribute,
+  sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all copies or
+  substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
+  NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
+  OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/
+#ifndef INCLUDE_PROVIDERS_PASSTHROUGH_PASSTHROUGHPROVIDER_HPP_
+#define INCLUDE_PROVIDERS_PASSTHROUGH_PASSTHROUGHPROVIDER_HPP_
+#if defined(REPERTORY_TESTING_NEW)
+
+#include "common.hpp"
+#include "providers/basebase_provider.hpp"
+
+namespace repertory {
+class app_config;
+class CPassthroughProvider final : public base_provider {
+public:
+  explicit CPassthroughProvider(app_config &config)
+      : base_provider(config), passthroughLocation_("" /*config.GetPassthroughConfig().Location*/) {
+  }
+  ~CPassthroughProvider() override = default;
+
+private:
+  const std::string passthroughLocation_;
+
+private:
+  std::string ConstructFullPath(const std::string &api_path) const;
+
+protected:
+  api_error notify_file_added(const std::string &api_path, const std::string &api_parent,
+                              const std::uint64_t &size) override;
+
+public:
+  api_error CreateDirectory(const std::string &api_path, const api_meta_map &metaMap) override;
+
+  api_error get_file_list(ApiFileList &fileList) const override;
+
+  std::uint64_t get_directory_item_count(const std::string &api_path) const override;
+
+  api_error get_directory_items(const std::string &api_path,
+                                directory_item_list &list) const override;
+
+  api_error GetFile(const std::string &api_path, ApiFile &file) const override;
+
+  api_error GetFileSize(const std::string &api_path, std::uint64_t &fileSize) const override;
+
+  api_error get_filesystem_item(const std::string &api_path, const bool &directory,
+                                FileSystemItem &fileSystemItem) const override;
+
+  api_error get_filesystem_item_from_source_path(const std::string &sourceFilePath,
+                                                 FileSystemItem &fileSystemItem) const override;
+
+  api_error get_item_meta(const std::string &api_path, api_meta_map &meta) const override;
+
+  api_error get_item_meta(const std::string &api_path, const std::string &key,
+                          std::string &value) const override;
+
+  provider_type get_provider_type() const override { return provider_type::passthrough; }
+
+  std::uint64_t get_total_drive_space() const override;
+
+  std::uint64_t get_total_item_count() const override;
+
+  std::uint64_t get_used_drive_space() const override;
+
+  bool IsDirectory(const std::string &api_path) const override;
+
+  bool IsFile(const std::string &api_path) const override;
+
+  bool IsOnline() const override { return true; }
+
+  bool is_rename_supported() const override { return true; }
+
+  api_error read_file_bytes(const std::string &apiFilepath, const std::size_t &size,
+                            const std::uint64_t &offset, std::vector<char> &data,
+                            const bool &stop_requested) override;
+
+  api_error RemoveDirectory(const std::string &api_path) override;
+
+  api_error RemoveFile(const std::string &api_path) override;
+
+  api_error RenameFile(const std::string &fromApiPath, const std::string &toApiPath) override;
+
+  api_error remove_item_meta(const std::string &api_path, const std::string &key) override;
+
+  api_error set_item_meta(const std::string &api_path, const std::string &key,
+                          const std::string &value) override;
+
+  api_error set_item_meta(const std::string &api_path, const api_meta_map &meta) override;
+
+  api_error set_source_path(const std::string &api_path, const std::string &sourcePath) override;
+
+  bool Start(api_item_added_callback apiItemAdded, i_open_file_table *openFileTable) override;
+
+  void Stop() override;
+
+  api_error upload_file(const std::string &api_path, const std::string &sourcePath,
+                        const std::string &encryptionToken) override;
+};
+} // namespace repertory
+
+#endif // REPERTORY_TESTING_NEW
+#endif // INCLUDE_PROVIDERS_PASSTHROUGH_PASSTHROUGHPROVIDER_HPP_

include/providers/provider.hpp

@@ -0,0 +1,31 @@
+/*
+  Copyright <2018-2022> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
+  associated documentation files (the "Software"), to deal in the Software without restriction,
+  including without limitation the rights to use, copy, modify, merge, publish, distribute,
+  sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all copies or
+  substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
+  NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
+  OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/
+#ifndef INCLUDE_PROVIDERS_PROVIDER_HPP_
+#define INCLUDE_PROVIDERS_PROVIDER_HPP_
+
+#include "common.hpp"
+#include "types/repertory.hpp"
+
+namespace repertory {
+class app_config;
+class i_provider;
+std::unique_ptr<i_provider> create_provider(const provider_type &pt, app_config &config);
+} // namespace repertory
+
+#endif // INCLUDE_PROVIDERS_PROVIDER_HPP_

include/providers/s3/s3_provider.hpp

@@ -0,0 +1,114 @@
+/*
+  Copyright <2018-2022> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
+  associated documentation files (the "Software"), to deal in the Software without restriction,
+  including without limitation the rights to use, copy, modify, merge, publish, distribute,
+  sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all copies or
+  substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
+  NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
+  OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/
+#ifndef INCLUDE_PROVIDERS_S3_S3_PROVIDER_HPP_
+#define INCLUDE_PROVIDERS_S3_S3_PROVIDER_HPP_
+#if defined(REPERTORY_ENABLE_S3)
+
+#include "common.hpp"
+#include "db/directory_db.hpp"
+#include "providers/base_provider.hpp"
+#include "upload/upload_manager.hpp"
+
+namespace repertory {
+class app_config;
+class i_s3_comm;
+class s3_provider final : public base_provider {
+public:
+  s3_provider(app_config &config, i_s3_comm &s3_comm);
+
+private:
+  i_s3_comm &s3_comm_;
+  directory_db directory_db_;
+  upload_manager upload_manager_;
+  std::uint64_t total_item_count_ = 0u;
+
+private:
+  void build_directories();
+
+  void update_item_meta(directory_item &di, const bool &format_path) const;
+
+  void upload_completed(const std::string &, const std::string &, const json &) {}
+
+  api_error upload_handler(const upload_manager::upload &upload, json &, json &);
+
+protected:
+  std::string format_api_path(std::string api_path) const override;
+
+  void notify_directory_added(const std::string &api_path, const std::string &api_parent) override;
+
+  api_error notify_file_added(const std::string &api_path, const std::string &api_parent,
+                              const std::uint64_t &size) override;
+
+  std::string &restore_api_path(std::string &api_path) const override;
+
+public:
+  api_error create_directory(const std::string &api_path, const api_meta_map &meta) override;
+
+  api_error create_file(const std::string &api_path, api_meta_map &meta) override;
+
+  api_error get_file_list(api_file_list &list) const override;
+
+  std::uint64_t get_directory_item_count(const std::string &api_path) const override;
+
+  api_error get_directory_items(const std::string &api_path,
+                                directory_item_list &list) const override;
+
+  api_error get_file(const std::string &api_path, api_file &file) const override;
+
+  api_error get_file_size(const std::string &api_path, std::uint64_t &file_size) const override;
+
+  provider_type get_provider_type() const override { return provider_type::s3; }
+
+  std::uint64_t get_total_drive_space() const override {
+    return std::numeric_limits<std::int64_t>::max() / std::int64_t(2);
+  }
+
+  std::uint64_t get_total_item_count() const override { return total_item_count_; }
+
+  bool is_directory(const std::string &api_path) const override;
+
+  bool is_file(const std::string &api_path) const override;
+
+  bool is_online() const override;
+
+  bool is_processing(const std::string &api_path) const;
+
+  bool is_rename_supported() const override { return false; }
+
+  api_error read_file_bytes(const std::string &api_path, const std::size_t &size,
+                            const std::uint64_t &offset, std::vector<char> &data,
+                            const bool &stop_requested) override;
+
+  api_error remove_directory(const std::string &api_path) override;
+
+  api_error remove_file(const std::string &api_path) override;
+
+  api_error rename_file(const std::string &from_api_path, const std::string &to_api_path) override;
+
+  bool start(api_item_added_callback api_item_added, i_open_file_table *oft) override;
+
+  void stop() override;
+
+  api_error upload_file(const std::string &api_path, const std::string &source_path,
+                        const std::string &encryption_token) override;
+};
+} // namespace repertory
+
+#endif // REPERTORY_ENABLE_S3
+#endif // INCLUDE_PROVIDERS_S3_S3_PROVIDER_HPP_

include/providers/sia/sia_provider.hpp

@@ -0,0 +1,136 @@
+/*
+  Copyright <2018-2022> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
+  associated documentation files (the "Software"), to deal in the Software without restriction,
+  including without limitation the rights to use, copy, modify, merge, publish, distribute,
+  sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all copies or
+  substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
+  NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
+  OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/
+#ifndef INCLUDE_PROVIDERS_SIA_SIAPROVIDER_HPP_
+#define INCLUDE_PROVIDERS_SIA_SIAPROVIDER_HPP_
+
+#include "common.hpp"
+#include "providers/base_provider.hpp"
+
+namespace repertory {
+class app_config;
+class i_comm;
+class sia_provider : public base_provider {
+public:
+  sia_provider(app_config &config, i_comm &comm);
+
+  ~sia_provider() override = default;
+
+private:
+  i_comm &comm_;
+  std::uint64_t total_drive_space_ = 0u;
+  bool stop_requested_ = false;
+  std::mutex start_stop_mutex_;
+  std::condition_variable start_stop_notify_;
+  std::unique_ptr<std::thread> drive_space_thread_;
+
+private:
+  api_item_added_callback &get_api_item_added() { return api_item_added_; }
+
+  i_comm &get_comm() { return comm_; }
+
+  i_comm &get_comm() const { return comm_; } // TODO Fix non-const reference return
+
+private:
+  void calculate_total_drive_space();
+
+  api_error calculate_used_drive_space(std::uint64_t &used_space);
+
+  void drive_space_thread();
+
+  bool processed_orphaned_file(const std::string &source_path,
+                               const std::string &api_path = "") const;
+
+  void remove_deleted_files();
+
+  void remove_expired_orphaned_files();
+
+  void remove_unknown_source_files();
+
+protected:
+  api_error check_file_exists(const std::string &api_path) const;
+
+  bool check_directory_found(const json &error) const;
+
+  bool check_not_found(const json &error) const;
+
+  virtual void cleanup();
+
+  void create_api_file(const json &json_file, const std::string &path_name, api_file &file) const;
+
+  api_error get_directory(const std::string &api_path, json &result, json &error) const;
+
+  api_error notify_file_added(const std::string &api_path, const std::string &api_parent,
+                              const std::uint64_t &size) override;
+
+  void set_api_file_dates(const json &file, api_file &apiFile) const;
+
+public:
+  api_error create_directory(const std::string &api_path, const api_meta_map &meta) override;
+
+  std::uint64_t get_directory_item_count(const std::string &api_path) const override;
+
+  api_error get_directory_items(const std::string &api_path,
+                                directory_item_list &list) const override;
+
+  api_error get_file(const std::string &api_path, api_file &file) const override;
+
+  api_error get_file_list(api_file_list &list) const override;
+
+  api_error get_file_size(const std::string &api_path, std::uint64_t &file_size) const override;
+
+  api_error get_filesystem_item(const std::string &api_path, const bool &directory,
+                                filesystem_item &fsi) const override;
+
+  api_error get_filesystem_item_and_file(const std::string &api_path, api_file &apiFile,
+                                         filesystem_item &fsi) const override;
+
+  provider_type get_provider_type() const override { return provider_type::sia; }
+
+  std::uint64_t get_total_drive_space() const override { return total_drive_space_; }
+
+  std::uint64_t get_total_item_count() const override;
+
+  bool is_directory(const std::string &api_path) const override;
+
+  bool is_file(const std::string &api_path) const override;
+
+  bool is_online() const override;
+
+  bool is_rename_supported() const override { return true; }
+
+  api_error read_file_bytes(const std::string &api_path, const std::size_t &size,
+                            const std::uint64_t &offset, std::vector<char> &buffer,
+                            const bool &stop_requested) override;
+
+  api_error remove_directory(const std::string &api_path) override;
+
+  api_error remove_file(const std::string &api_path) override;
+
+  api_error rename_file(const std::string &from_api_path, const std::string &to_api_path) override;
+
+  bool start(api_item_added_callback api_item_added, i_open_file_table *oft) override;
+
+  void stop() override;
+
+  api_error upload_file(const std::string &api_path, const std::string &source_path,
+                        const std::string &) override;
+};
+} // namespace repertory
+
+#endif // INCLUDE_PROVIDERS_SIA_SIAPROVIDER_HPP_

include/providers/skynet/skynet_provider.hpp

@@ -0,0 +1,139 @@
+/*
+  Copyright <2018-2022> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
+  associated documentation files (the "Software"), to deal in the Software without restriction,
+  including without limitation the rights to use, copy, modify, merge, publish, distribute,
+  sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all copies or
+  substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
+  NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
+  OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/
+#ifndef INCLUDE_PROVIDERS_SKYNET_SKYNET_PROVIDER_HPP_
+#define INCLUDE_PROVIDERS_SKYNET_SKYNET_PROVIDER_HPP_
+#if defined(REPERTORY_ENABLE_SKYNET)
+
+#include "common.hpp"
+#include "db/directory_db.hpp"
+#include "db/meta_db.hpp"
+#include "events/event_system.hpp"
+#include "providers/base_provider.hpp"
+#include "upload/upload_manager.hpp"
+
+namespace repertory {
+class app_config;
+class i_comm;
+class skynet_provider final : public base_provider {
+  E_CONSUMER();
+
+public:
+  skynet_provider(app_config &config, i_comm &comm);
+
+  ~skynet_provider() override { E_CONSUMER_RELEASE(); }
+
+private:
+  i_comm &comm_;
+  directory_db directory_db_;
+  upload_manager upload_manager_;
+  bool stop_requested_ = false;
+  std::atomic<std::size_t> next_download_index_;
+  std::atomic<std::size_t> next_upload_index_;
+  std::shared_ptr<std::vector<host_config>> download_list_;
+  std::shared_ptr<std::vector<host_config>> upload_list_;
+  mutable std::mutex portal_mutex_;
+
+private:
+  std::size_t get_retry_count() const;
+
+  void populate_api_file(api_file &file) const;
+
+  void process_export(json &result, const std::string &api_path) const;
+
+  void upload_completed(const std::string &api_path, const std::string &, const json &data);
+
+  api_error upload_handler(const upload_manager::upload &upload, json &data, json &error);
+
+protected:
+  void notify_directory_added(const std::string &api_path, const std::string &api_parent) override;
+
+  api_error notify_file_added(const std::string &, const std::string &,
+                              const std::uint64_t &) override;
+
+public:
+  api_error create_directory(const std::string &api_path, const api_meta_map &meta) override;
+
+  api_error create_file(const std::string &api_path, api_meta_map &meta) override;
+
+  json export_all() const;
+
+  json export_list(const std::vector<std::string> &api_path_list) const;
+
+  std::uint64_t get_directory_item_count(const std::string &api_path) const override;
+
+  api_error get_directory_items(const std::string &api_path,
+                                directory_item_list &list) const override;
+
+  api_error get_file(const std::string &api_path, api_file &file) const override;
+
+  api_error get_file_list(api_file_list &list) const override;
+
+  api_error get_file_size(const std::string &api_path, std::uint64_t &file_size) const override;
+
+  host_config get_host_config(const bool &upload);
+
+  provider_type get_provider_type() const override { return provider_type::skynet; }
+
+  api_error get_skynet_metadata(const std::string &skylink, json &json_meta);
+
+  std::uint64_t get_total_drive_space() const override {
+    return std::numeric_limits<std::int64_t>::max() / std::int64_t(2);
+  }
+
+  std::uint64_t get_total_item_count() const override {
+    return directory_db_.get_total_item_count();
+  }
+
+  api_error import_skylink(const skylink_import &si);
+
+  bool is_directory(const std::string &api_path) const override;
+
+  bool is_file(const std::string &api_path) const override;
+
+  bool is_file_writeable(const std::string &api_path) const override;
+
+  bool is_online() const override { return true; }
+
+  bool is_processing(const std::string &api_path) const;
+
+  bool is_rename_supported() const override { return true; }
+
+  api_error read_file_bytes(const std::string &api_path, const std::size_t &size,
+                            const std::uint64_t &offset, std::vector<char> &data,
+                            const bool &stop_requested) override;
+
+  api_error remove_directory(const std::string &api_path) override;
+
+  api_error remove_file(const std::string &api_path) override;
+
+  api_error rename_file(const std::string &from_api_path, const std::string &to_api_path) override;
+
+  bool start(api_item_added_callback api_item_added, i_open_file_table *oft) override;
+
+  void stop() override;
+
+  bool update_portal_list();
+
+  api_error upload_file(const std::string &api_path, const std::string &source_path,
+                        const std::string &encryption_token) override;
+};
+} // namespace repertory
+
+#endif // REPERTORY_ENABLE_SKYNET
+#endif // INCLUDE_PROVIDERS_SKYNET_SKYNET_PROVIDER_HPP_