initial commit

include/download/buffered_reader.hpp

@@ -0,0 +1,78 @@
+/*
+  Copyright <2018-2022> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
+  associated documentation files (the "Software"), to deal in the Software without restriction,
+  including without limitation the rights to use, copy, modify, merge, publish, distribute,
+  sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all copies or
+  substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
+  NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
+  OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/
+#ifndef INCLUDE_DOWNLOAD_BUFFERED_READER_HPP_
+#define INCLUDE_DOWNLOAD_BUFFERED_READER_HPP_
+
+#include "common.hpp"
+#include "types/repertory.hpp"
+
+namespace repertory {
+class app_config;
+class buffered_reader final {
+public:
+  buffered_reader(const app_config &config, const filesystem_item &fsi,
+                  const api_reader_callback &api_reader, const std::size_t &chunk_size,
+                  const std::size_t &total_chunks, const std::size_t &start_chunk);
+
+  ~buffered_reader();
+
+private:
+  const filesystem_item &fsi_;
+  const api_reader_callback &api_reader_;
+  const std::size_t chunk_size_;
+  const std::size_t total_chunks_;
+  boost::dynamic_bitset<> ring_state_;
+
+  api_error error_ = api_error::success;
+  std::unique_ptr<std::vector<char>> first_chunk_data_;
+  std::unique_ptr<std::vector<char>> last_chunk_data_;
+  std::size_t read_chunk_index_ = 0u;
+  std::mutex read_mutex_;
+  std::condition_variable read_notify_;
+  std::size_t read_offset_ = 0u;
+  bool reset_reader_ = false;
+  std::vector<std::vector<char>> ring_data_;
+  std::unique_ptr<std::thread> reader_thread_;
+  bool stop_requested_ = false;
+  std::mutex write_mutex_;
+  std::size_t write_chunk_index_ = 0u;
+
+private:
+  bool is_active() const { return not stop_requested_ && (error_ == api_error::success); }
+
+  void reader_thread();
+
+public:
+  std::size_t get_chunk_size() const { return chunk_size_; }
+
+  void get_first_chunk(std::vector<char> *&data) { data = first_chunk_data_.get(); }
+
+  void get_last_chunk(std::vector<char> *&data) { data = last_chunk_data_.get(); }
+
+  bool has_first_chunk() const { return static_cast<bool>(first_chunk_data_); }
+
+  bool has_last_chunk() const { return static_cast<bool>(last_chunk_data_); }
+
+  void notify_stop_requested();
+
+  api_error read_chunk(const std::size_t &chunk_index, std::vector<char> &data);
+};
+} // namespace repertory
+
+#endif // INCLUDE_DOWNLOAD_BUFFERED_READER_HPP_

include/download/direct_download.hpp

@@ -0,0 +1,99 @@
+/*
+  Copyright <2018-2022> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
+  associated documentation files (the "Software"), to deal in the Software without restriction,
+  including without limitation the rights to use, copy, modify, merge, publish, distribute,
+  sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all copies or
+  substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
+  NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
+  OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/
+#ifndef INCLUDE_DOWNLOAD_DIRECT_DOWNLOAD_HPP_
+#define INCLUDE_DOWNLOAD_DIRECT_DOWNLOAD_HPP_
+
+#include "common.hpp"
+#include "download/i_download.hpp"
+#include "types/repertory.hpp"
+
+namespace repertory {
+class buffered_reader;
+class app_config;
+class direct_download final : public virtual i_download {
+public:
+  direct_download(const app_config &config, filesystem_item fsi,
+                  const api_reader_callback &api_reader, const std::uint64_t &handle);
+
+  ~direct_download() override;
+
+private:
+  const app_config &config_;
+  const filesystem_item fsi_;
+  const api_reader_callback &api_reader_;
+  const std::uint64_t handle_;
+
+  api_error error_ = api_error::success;
+  std::unique_ptr<buffered_reader> buffered_reader_;
+  bool disable_download_end_ = false;
+  bool download_end_notified_ = false;
+  double progress_ = 0.0;
+  std::mutex read_mutex_;
+  bool stop_requested_ = false;
+
+private:
+  bool is_active() const { return not stop_requested_ && (error_ == api_error::success); }
+
+  void notify_download_end();
+
+  void set_api_error(const api_error &error);
+
+public:
+  api_error allocate(const std::uint64_t & /*handle*/, const std::uint64_t & /*size*/,
+                     const allocator_callback & /*allocator*/,
+                     const completer_callback & /*completer*/) override {
+    return api_error::not_implemented;
+  }
+
+  api_error download_all() override { return api_error::not_implemented; }
+
+  api_error get_result() const override { return error_; }
+
+  std::string get_source_path() const override { return ""; }
+
+  void get_state_information(filesystem_item &, std::size_t &, std::size_t &,
+                             boost::dynamic_bitset<> &, boost::dynamic_bitset<> &) override {}
+
+  bool get_write_supported() const override { return false; }
+
+  void notify_stop_requested() override;
+
+  bool pause() override { return false; }
+
+  api_error read_bytes(const std::uint64_t &, std::size_t read_size,
+                       const std::uint64_t &read_offset, std::vector<char> &data) override;
+
+  void reset_timeout(const bool &) override {}
+
+  void resume() override {}
+
+  void set_api_path(const std::string &) override {}
+
+  void set_disable_download_end(const bool &disable) override { disable_download_end_ = disable; }
+
+  api_error write_bytes(const std::uint64_t &, const std::uint64_t &, std::vector<char>,
+                        std::size_t &bytes_written,
+                        const completer_callback & /*completer*/) override {
+    bytes_written = 0u;
+    return api_error::not_implemented;
+  }
+};
+} // namespace repertory
+
+#endif // INCLUDE_DOWNLOAD_DIRECT_DOWNLOAD_HPP_

include/download/download.hpp

@@ -0,0 +1,191 @@
+/*
+  Copyright <2018-2022> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
+  associated documentation files (the "Software"), to deal in the Software without restriction,
+  including without limitation the rights to use, copy, modify, merge, publish, distribute,
+  sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all copies or
+  substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
+  NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
+  OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/
+#ifndef INCLUDE_DOWNLOAD_DOWNLOAD_HPP_
+#define INCLUDE_DOWNLOAD_DOWNLOAD_HPP_
+
+#include "common.hpp"
+#include "app_config.hpp"
+#include "download/i_download.hpp"
+#include "types/repertory.hpp"
+#include "utils/native_file.hpp"
+
+namespace repertory {
+class i_open_file_table;
+class download final : public virtual i_download {
+private:
+  struct active_chunk {
+    explicit active_chunk(std::thread worker) : worker(std::move(worker)) {}
+
+    std::thread worker;
+    std::mutex mutex;
+    std::condition_variable notify;
+  };
+  typedef std::shared_ptr<active_chunk> active_chunk_ptr;
+
+  struct read_data {
+    read_data(std::vector<char> &data, const std::uint64_t &offset)
+        : complete(false), data(data), offset(offset) {}
+
+    bool complete;
+    std::vector<char> &data;
+    const std::uint64_t offset;
+    std::mutex mutex;
+    std::condition_variable notify;
+  };
+  typedef std::shared_ptr<read_data> read_data_ptr;
+
+  struct write_data {
+    write_data(const std::size_t &chunk, std::vector<char> data, const std::uint64_t &offset)
+        : chunk_index(chunk), data(std::move(data)), offset(offset) {}
+    write_data(const std::uint64_t &offset, std::vector<char> data)
+        : chunk_index(0u), data(std::move(data)), offset(offset), from_read(false) {}
+
+    std::size_t chunk_index;
+    std::vector<char> data;
+    const std::uint64_t offset;
+    bool from_read = true;
+    bool complete = false;
+    std::mutex mutex;
+    std::condition_variable notify;
+    std::size_t written = 0u;
+  };
+  typedef std::shared_ptr<write_data> write_data_ptr;
+
+public:
+  download(const app_config &config, filesystem_item &fsi, const api_reader_callback &api_reader,
+           const std::size_t &chunk_size, i_open_file_table &oft);
+
+  download(const app_config &config, filesystem_item &fsi, const api_reader_callback &api_reader,
+           const std::size_t &chunk_size, std::size_t &last_chunk_size,
+           boost::dynamic_bitset<> &read_state, boost::dynamic_bitset<> &write_state,
+           i_open_file_table &oft);
+
+  ~download() override;
+
+private:
+  // Constructor initialization
+  const app_config &config_;
+  filesystem_item fsi_;
+  const api_reader_callback &api_reader_;
+  i_open_file_table &oft_;
+  const std::size_t chunk_size_;
+  boost::dynamic_bitset<> read_chunk_state_;
+  std::size_t last_chunk_size_;
+  boost::dynamic_bitset<> write_chunk_state_;
+
+  // Default initialization
+  bool auto_close_ = false;
+  std::unordered_map<std::size_t, active_chunk_ptr> active_chunks_;
+  api_error error_ = api_error::success;
+  std::vector<std::thread> background_workers_;
+  std::size_t current_chunk_index_ = 0u;
+  bool disable_download_end_ = false;
+  std::unique_ptr<std::thread> io_thread_;
+  std::uint64_t open_file_handle_ = REPERTORY_API_INVALID_HANDLE;
+  bool paused_ = false;
+  bool processed_ = false;
+  std::condition_variable processed_notify_;
+  double progress_ = 0.0;
+  std::size_t read_offset_ = 0u;
+  std::deque<read_data_ptr> read_queue_;
+  native_file_ptr read_write_file_;
+  std::mutex read_write_mutex_;
+  std::condition_variable read_write_notify_;
+  bool stop_requested_ = false;
+  std::chrono::system_clock::time_point timeout_ =
+      std::chrono::system_clock::now() +
+      std::chrono::seconds(config_.get_chunk_downloader_timeout_secs());
+  std::deque<write_data_ptr> write_queue_;
+
+private:
+  void create_active_chunk(std::size_t chunk_index);
+
+  void download_chunk(std::size_t chunk_index, bool inactive_only);
+
+  bool get_complete() const;
+
+  bool get_timeout_enabled() const;
+
+  void handle_active_chunk_complete(std::size_t chunk_index, unique_mutex_lock &lock);
+
+  void initialize_download(filesystem_item &fsi, const bool &delete_existing);
+
+  void io_data_worker();
+
+  void notify_progress();
+
+  void process_download_complete(unique_mutex_lock &lock);
+
+  void process_read_queue(unique_mutex_lock &lock);
+
+  void process_timeout(unique_mutex_lock &lock);
+
+  void process_write_queue(unique_mutex_lock &lock);
+
+  void read_ahead_worker();
+
+  void read_behind_worker();
+
+  void read_end_worker();
+
+  void shutdown(unique_mutex_lock &lock);
+
+  void wait_for_io(unique_mutex_lock &lock);
+
+public:
+  api_error allocate(const std::uint64_t &handle, const std::uint64_t &size,
+                     const allocator_callback &allocator,
+                     const completer_callback &completer) override;
+
+  api_error download_all() override;
+
+  api_error get_result() const override { return error_; }
+
+  std::string get_source_path() const override { return fsi_.source_path; }
+
+  void get_state_information(filesystem_item &fsi, std::size_t &chunk_size,
+                             std::size_t &last_chunk_size, boost::dynamic_bitset<> &read_state,
+                             boost::dynamic_bitset<> &write_state) override;
+
+  bool get_write_supported() const override { return true; }
+
+  bool is_active() const { return not read_chunk_state_.all(); }
+
+  void notify_stop_requested() override;
+
+  bool pause() override;
+
+  api_error read_bytes(const std::uint64_t &handle, std::size_t read_size,
+                       const std::uint64_t &read_offset, std::vector<char> &data) override;
+
+  void reset_timeout(const bool &) override;
+
+  void resume() override;
+
+  void set_api_path(const std::string &api_path) override;
+
+  void set_disable_download_end(const bool &disable) override { disable_download_end_ = disable; }
+
+  api_error write_bytes(const std::uint64_t &handle, const std::uint64_t &write_offset,
+                        std::vector<char> data, std::size_t &bytes_written,
+                        const completer_callback &completer) override;
+};
+} // namespace repertory
+
+#endif // INCLUDE_DOWNLOAD_DOWNLOAD_HPP_

include/download/download_manager.hpp

@@ -0,0 +1,102 @@
+/*
+  Copyright <2018-2022> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
+  associated documentation files (the "Software"), to deal in the Software without restriction,
+  including without limitation the rights to use, copy, modify, merge, publish, distribute,
+  sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all copies or
+  substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
+  NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
+  OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/
+#ifndef INCLUDE_DOWNLOAD_DOWNLOAD_MANAGER_HPP_
+#define INCLUDE_DOWNLOAD_DOWNLOAD_MANAGER_HPP_
+
+#include "common.hpp"
+#include "download/i_download.hpp"
+#include "download/i_download_manager.hpp"
+#include "events/event_system.hpp"
+
+namespace repertory {
+class app_config;
+class download_end;
+class filesystem_item_handle_closed;
+class i_open_file_table;
+class download_manager final : public virtual i_download_manager {
+  E_CONSUMER();
+
+public:
+  download_manager(const app_config &config, api_reader_callback api_reader,
+                   const bool &force_download = false);
+
+  ~download_manager() override;
+
+private:
+  const app_config &config_;
+  const api_reader_callback api_reader_;
+  bool force_download_;
+  i_open_file_table *oft_ = nullptr;
+  mutable std::recursive_mutex download_mutex_;
+  std::unordered_map<std::string, std::unordered_map<std::uint64_t, download_ptr>> download_lookup_;
+  std::recursive_mutex start_stop_mutex_;
+  bool stop_requested_ = true;
+  std::unique_ptr<rocksdb::DB> restore_db_;
+
+private:
+  bool contains_handle(const std::string &api_path, const std::uint64_t &handle) const;
+
+  download_ptr get_download(std::uint64_t handle, filesystem_item &fsi,
+                            const bool &write_Supported);
+
+  void handle_download_end(const download_end &de);
+
+  void on_handle_closed(const filesystem_item_handle_closed &handle_closed);
+
+  void reset_timeout(const std::string &api_path, const bool &file_closed);
+
+  void start_incomplete();
+
+public:
+  api_error allocate(const std::uint64_t &handle, filesystem_item &fsi, const std::uint64_t &size,
+                     const i_download::allocator_callback &allocator) override;
+
+  bool contains_restore(const std::string &api_path) const override;
+
+  api_error download_file(const std::uint64_t &handle, filesystem_item &fsi) override;
+
+  std::size_t get_download_count() const { return download_lookup_.size(); }
+
+  std::string get_source_path(const std::string &api_path) const;
+
+  bool is_processing(const std::string &api_path) const override;
+
+  bool pause_download(const std::string &api_path) override;
+
+  api_error read_bytes(const std::uint64_t &handle, filesystem_item &fsi, std::size_t read_size,
+                       const std::uint64_t &read_offset, std::vector<char> &data) override;
+
+  void rename_download(const std::string &from_api_path, const std::string &to_api_path) override;
+
+  api_error resize(const std::uint64_t &handle, filesystem_item &fsi,
+                   const std::uint64_t &size) override;
+
+  void resume_download(const std::string &api_path) override;
+
+  void start(i_open_file_table *oft);
+
+  void stop();
+
+  api_error write_bytes(const std::uint64_t &handle, filesystem_item &fsi,
+                        const std::uint64_t &write_offset, std::vector<char> data,
+                        std::size_t &bytes_written) override;
+};
+} // namespace repertory
+
+#endif // INCLUDE_DOWNLOAD_DOWNLOAD_MANAGER_HPP_

include/download/events.hpp

@@ -0,0 +1,85 @@
+/*
+  Copyright <2018-2022> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
+  associated documentation files (the "Software"), to deal in the Software without restriction,
+  including without limitation the rights to use, copy, modify, merge, publish, distribute,
+  sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all copies or
+  substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
+  NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
+  OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/
+#ifndef INCLUDE_DOWNLOAD_EVENTS_HPP_
+#define INCLUDE_DOWNLOAD_EVENTS_HPP_
+
+#include "common.hpp"
+#include "events/events.hpp"
+#include "types/repertory.hpp"
+
+namespace repertory {
+// clang-format off
+E_SIMPLE2(download_begin, normal, true,
+  std::string, api_path, ap, E_STRING,
+  std::string, dest_path, dest, E_STRING
+);
+
+E_SIMPLE4(download_end, normal, true,
+  std::string, api_path, ap, E_STRING,
+  std::string, dest_path, dest, E_STRING,
+  std::uint64_t, handle, handle, E_FROM_UINT64,
+  api_error, result, result, E_FROM_API_FILE_ERROR
+);
+
+E_SIMPLE2(download_paused, normal, true,
+  std::string, api_path, ap, E_STRING,
+  std::string, dest_path, dest, E_STRING
+);
+
+E_SIMPLE3(download_progress, normal, true,
+  std::string, api_path, ap, E_STRING,
+  std::string, dest_path, dest, E_STRING,
+  double, progress, prog, E_DOUBLE_PRECISE
+);
+
+E_SIMPLE2(download_restored, normal, true,
+  std::string, api_path, ap, E_STRING,
+  std::string, dest_path, dest, E_STRING
+);
+
+E_SIMPLE3(download_restore_failed, error, true,
+  std::string, api_path, ap, E_STRING,
+  std::string, dest_path, dest, E_STRING,
+  std::string, error, err, E_STRING
+);
+
+E_SIMPLE2(download_resumed, normal, true,
+  std::string, api_path, ap, E_STRING,
+  std::string, dest_path, dest, E_STRING
+);
+
+E_SIMPLE2(download_stored, normal, true,
+  std::string, api_path, ap, E_STRING,
+  std::string, dest_path, dest, E_STRING
+);
+
+E_SIMPLE3(download_store_failed, error, true,
+  std::string, api_path, ap, E_STRING,
+  std::string, dest_path, dest, E_STRING,
+  std::string, error, err, E_STRING
+);
+
+E_SIMPLE2(download_timeout, warn, true,
+  std::string, api_path, ap, E_STRING,
+  std::string, dest_path, dest, E_STRING
+);
+// clang-format on
+} // namespace repertory
+
+#endif // INCLUDE_DOWNLOAD_EVENTS_HPP_

include/download/i_download.hpp

@@ -0,0 +1,76 @@
+/*
+  Copyright <2018-2022> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
+  associated documentation files (the "Software"), to deal in the Software without restriction,
+  including without limitation the rights to use, copy, modify, merge, publish, distribute,
+  sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all copies or
+  substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
+  NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
+  OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/
+#ifndef INCLUDE_DOWNLOAD_I_DOWNLOAD_HPP_
+#define INCLUDE_DOWNLOAD_I_DOWNLOAD_HPP_
+
+#include "common.hpp"
+#include "types/repertory.hpp"
+
+namespace repertory {
+class i_download {
+  INTERFACE_SETUP(i_download);
+
+public:
+  typedef std::function<api_error()> allocator_callback;
+
+  typedef std::function<void(std::uint64_t old_size, std::uint64_t new_size, bool changed)>
+      completer_callback;
+
+public:
+  virtual api_error allocate(const std::uint64_t &handle, const std::uint64_t &size,
+                             const allocator_callback &allocator,
+                             const completer_callback &completer) = 0;
+
+  virtual api_error download_all() = 0;
+
+  virtual api_error get_result() const = 0;
+
+  virtual std::string get_source_path() const = 0;
+
+  virtual void get_state_information(filesystem_item &fsi, std::size_t &chunk_size,
+                                     std::size_t &last_chunk_size,
+                                     boost::dynamic_bitset<> &read_state,
+                                     boost::dynamic_bitset<> &write_state) = 0;
+
+  virtual bool get_write_supported() const = 0;
+
+  virtual void notify_stop_requested() = 0;
+
+  virtual bool pause() = 0;
+
+  virtual api_error read_bytes(const std::uint64_t &handle, std::size_t read_size,
+                               const std::uint64_t &read_offset, std::vector<char> &data) = 0;
+
+  virtual void reset_timeout(const bool &file_closed) = 0;
+
+  virtual void resume() = 0;
+
+  virtual void set_api_path(const std::string &api_path) = 0;
+
+  virtual void set_disable_download_end(const bool &disable) = 0;
+
+  virtual api_error write_bytes(const std::uint64_t &handle, const std::uint64_t &write_offset,
+                                std::vector<char> data, std::size_t &bytes_written,
+                                const completer_callback &completer) = 0;
+};
+
+typedef std::shared_ptr<i_download> download_ptr;
+} // namespace repertory
+
+#endif // INCLUDE_DOWNLOAD_I_DOWNLOAD_HPP_

include/download/i_download_manager.hpp

@@ -0,0 +1,61 @@
+/*
+  Copyright <2018-2022> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
+  associated documentation files (the "Software"), to deal in the Software without restriction,
+  including without limitation the rights to use, copy, modify, merge, publish, distribute,
+  sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all copies or
+  substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
+  NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
+  OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/
+#ifndef INCLUDE_DOWNLOAD_I_DOWNLOAD_MANAGER_HPP_
+#define INCLUDE_DOWNLOAD_I_DOWNLOAD_MANAGER_HPP_
+
+#include "common.hpp"
+#include "download/i_download.hpp"
+#include "types/repertory.hpp"
+
+namespace repertory {
+class i_download_manager {
+  INTERFACE_SETUP(i_download_manager);
+
+public:
+  virtual api_error allocate(const std::uint64_t &handle, filesystem_item &fsi,
+                             const std::uint64_t &size,
+                             const i_download::allocator_callback &allocator) = 0;
+
+  virtual bool contains_restore(const std::string &api_path) const = 0;
+
+  virtual api_error download_file(const std::uint64_t &handle, filesystem_item &fsi) = 0;
+
+  virtual bool is_processing(const std::string &api_path) const = 0;
+
+  virtual bool pause_download(const std::string &api_path) = 0;
+
+  virtual api_error read_bytes(const std::uint64_t &handle, filesystem_item &fsi,
+                               std::size_t read_size, const std::uint64_t &read_offset,
+                               std::vector<char> &data) = 0;
+
+  virtual void rename_download(const std::string &from_api_path,
+                               const std::string &to_api_path) = 0;
+
+  virtual api_error resize(const std::uint64_t &handle, filesystem_item &fsi,
+                           const std::uint64_t &size) = 0;
+
+  virtual void resume_download(const std::string &api_path) = 0;
+
+  virtual api_error write_bytes(const std::uint64_t &handle, filesystem_item &fsi,
+                                const std::uint64_t &write_offset, std::vector<char> data,
+                                std::size_t &bytes_written) = 0;
+};
+} // namespace repertory
+
+#endif // INCLUDE_DOWNLOAD_I_DOWNLOAD_MANAGER_HPP_

include/download/reader_pool.hpp

@@ -0,0 +1,87 @@
+/*
+  Copyright <2018-2022> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
+  associated documentation files (the "Software"), to deal in the Software without restriction,
+  including without limitation the rights to use, copy, modify, merge, publish, distribute,
+  sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all copies or
+  substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
+  NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
+  OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/
+#ifndef INCLUDE_DOWNLOAD_READER_POOL_HPP_
+#define INCLUDE_DOWNLOAD_READER_POOL_HPP_
+
+#include "common.hpp"
+#include "types/repertory.hpp"
+
+namespace repertory {
+class reader_pool final {
+public:
+  typedef std::function<void(api_error error)> completed_callback;
+
+private:
+  struct pool_work_item {
+    pool_work_item(std::string api_path, const std::size_t &read_size,
+                   const std::uint64_t &read_offset, std::vector<char> &data,
+                   completed_callback completed)
+        : api_path(std::move(api_path)), read_size(read_size), read_offset(read_offset), data(data),
+          completed(completed) {}
+
+    std::string api_path;
+    std::size_t read_size;
+    std::uint64_t read_offset;
+    std::vector<char> &data;
+    completed_callback completed;
+  };
+
+public:
+  reader_pool(const std::size_t &pool_size, const api_reader_callback &api_reader)
+      : pool_size_(pool_size), api_reader_(api_reader) {
+    start();
+  }
+
+  ~reader_pool() { stop(); }
+
+private:
+  const std::size_t pool_size_;
+  const api_reader_callback &api_reader_;
+  bool paused_ = false;
+  bool restart_active_ = false;
+  bool stop_requested_ = false;
+  std::mutex work_mutex_;
+  std::condition_variable work_notify_;
+  std::deque<std::shared_ptr<pool_work_item>> work_queue_;
+  std::vector<std::thread> work_threads_;
+  std::uint16_t active_count_ = 0u;
+
+private:
+  void process_work_item(pool_work_item &work);
+
+  void start();
+
+  void stop();
+
+  void wait_for_resume(unique_mutex_lock &lock);
+
+public:
+  void pause();
+
+  void queue_read_bytes(const std::string &api_path, const std::size_t &read_size,
+                        const std::uint64_t &read_offset, std::vector<char> &data,
+                        completed_callback completed);
+
+  void restart();
+
+  void resume();
+};
+} // namespace repertory
+
+#endif // INCLUDE_DOWNLOAD_READER_POOL_HPP_

include/download/ring_download.hpp

@@ -0,0 +1,136 @@
+/*
+  Copyright <2018-2022> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
+  associated documentation files (the "Software"), to deal in the Software without restriction,
+  including without limitation the rights to use, copy, modify, merge, publish, distribute,
+  sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all copies or
+  substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
+  NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
+  OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/
+#ifndef INCLUDE_DOWNLOAD_RING_DOWNLOAD_HPP_
+#define INCLUDE_DOWNLOAD_RING_DOWNLOAD_HPP_
+
+#include "common.hpp"
+#include "download/i_download.hpp"
+#include "utils/native_file.hpp"
+#include "types/repertory.hpp"
+
+namespace repertory {
+class buffered_reader;
+class app_config;
+class ring_download final : public virtual i_download {
+private:
+  struct io_action {
+    io_action(std::mutex &mtx, std::condition_variable &cv, std::function<void()> action)
+        : mutex(mtx), notify(cv), action(action) {}
+
+    std::mutex &mutex;
+    std::condition_variable &notify;
+    std::function<void()> action;
+  };
+
+public:
+  ring_download(const app_config &config, filesystem_item fsi,
+                const api_reader_callback &api_reader, const std::uint64_t &handle,
+                const std::size_t &chunk_size, const std::size_t &ring_buffer_size);
+
+  ~ring_download() override;
+
+private:
+  const app_config &config_;
+  const filesystem_item fsi_;
+  const api_reader_callback &api_reader_;
+  const std::uint64_t handle_;
+  const std::size_t chunk_size_;
+  boost::dynamic_bitset<> ring_state_;
+  const std::size_t total_chunks_ = 0u;
+
+  api_error error_ = api_error::success;
+  native_file_ptr buffer_file_;
+  std::string buffer_file_path_;
+  buffered_reader *buffered_reader_ = nullptr;
+  bool disable_download_end_ = false;
+  std::size_t head_chunk_index_ = 0ull;
+  std::mutex io_mutex_;
+  std::condition_variable io_notify_;
+  std::deque<std::unique_ptr<io_action>> io_queue_;
+  std::unique_ptr<std::thread> io_thread_;
+  std::size_t read_chunk_ = 0ull;
+  std::unique_ptr<std::thread> buffer_thread_;
+  std::mutex read_mutex_;
+  std::condition_variable read_notify_;
+  bool stop_requested_ = false;
+  std::size_t write_chunk_ = 0ull;
+  std::mutex write_mutex_;
+
+private:
+  void buffer_thread(std::size_t start_chunk_index);
+
+  void io_thread();
+
+  bool is_active() const { return not stop_requested_ && (error_ == api_error::success); }
+
+  bool queue_io_item(std::mutex &m, std::condition_variable &cv, const bool &is_read,
+                     std::function<void()> action);
+
+  void read(std::size_t read_chunk_index, std::size_t read_size, std::size_t read_offset,
+            std::vector<char> &data);
+
+  void set_api_error(const api_error &error);
+
+  void start(const std::size_t &start_chunk_index);
+
+  void stop();
+
+public:
+  api_error allocate(const std::uint64_t & /*handle*/, const std::uint64_t & /*size*/,
+                     const allocator_callback & /*allocator*/,
+                     const completer_callback & /*completer*/) override {
+    return api_error::not_implemented;
+  }
+
+  api_error download_all() override { return api_error::not_implemented; }
+
+  api_error get_result() const override { return error_; }
+
+  std::string get_source_path() const override { return buffer_file_path_; }
+
+  void get_state_information(filesystem_item &, std::size_t &, std::size_t &,
+                             boost::dynamic_bitset<> &, boost::dynamic_bitset<> &) override {}
+
+  bool get_write_supported() const override { return false; }
+
+  void notify_stop_requested() override;
+
+  bool pause() override { return false; }
+
+  api_error read_bytes(const std::uint64_t &, std::size_t read_size,
+                       const std::uint64_t &read_offset, std::vector<char> &data) override;
+
+  void reset_timeout(const bool &) override {}
+
+  void resume() override {}
+
+  void set_api_path(const std::string &) override {}
+
+  void set_disable_download_end(const bool &disable) override { disable_download_end_ = disable; }
+
+  api_error write_bytes(const std::uint64_t &, const std::uint64_t &, std::vector<char>,
+                        std::size_t &bytes_written,
+                        const completer_callback & /*completer*/) override {
+    bytes_written = 0u;
+    return api_error::not_implemented;
+  }
+};
+} // namespace repertory
+
+#endif // INCLUDE_DOWNLOAD_RING_DOWNLOAD_HPP_

include/download/utils.hpp

@@ -0,0 +1,41 @@
+/*
+  Copyright <2018-2022> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
+  associated documentation files (the "Software"), to deal in the Software without restriction,
+  including without limitation the rights to use, copy, modify, merge, publish, distribute,
+  sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all copies or
+  substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
+  NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
+  OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/
+#ifndef INCLUDE_DOWNLOAD_UTILS_HPP_
+#define INCLUDE_DOWNLOAD_UTILS_HPP_
+
+#include "common.hpp"
+#include "app_config.hpp"
+
+namespace repertory::utils::download {
+template <typename event_type>
+void notify_progress(const app_config &config, const std::string &api_path,
+                     const std::string &source_path, const double &current, const double &total,
+                     double &progress) {
+  if (config.get_event_level() >= event_type::level) {
+    const double next_progress = (current / total) * 100.0;
+    if ((next_progress == 0.0) || (next_progress >= (progress + 0.2)) ||
+        ((next_progress == 100.00) && (next_progress != progress))) {
+      progress = next_progress;
+      event_system::instance().raise<event_type>(api_path, source_path, progress);
+    }
+  }
+}
+} // namespace repertory::utils::download
+
+#endif // INCLUDE_DOWNLOAD_UTILS_HPP_