initial commit

src/comm/aws_s3/aws_s3_comm.cpp

@@ -0,0 +1,677 @@
+/*
+  Copyright <2018-2022> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
+  associated documentation files (the "Software"), to deal in the Software without restriction,
+  including without limitation the rights to use, copy, modify, merge, publish, distribute,
+  sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all copies or
+  substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
+  NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
+  OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/
+#if defined(REPERTORY_ENABLE_S3)
+#include "comm/aws_s3/aws_s3_comm.hpp"
+#include "app_config.hpp"
+#include "events/events.hpp"
+#include "events/event_system.hpp"
+#include "providers/i_provider.hpp"
+#include "types/repertory.hpp"
+#include "utils/encryption.hpp"
+#include "utils/path_utils.hpp"
+#include "utils/polling.hpp"
+#include "utils/string_utils.hpp"
+
+namespace repertory {
+static const i_s3_comm::get_key_callback empty_key = []() { return ""; };
+
+aws_s3_comm::aws_s3_comm(const app_config &config)
+    : config_(config), s3_config_(config.get_s3_config()) {
+  s3_config_.bucket = utils::string::trim(s3_config_.bucket);
+  Aws::InitAPI(sdk_options_);
+
+  if (config_.get_event_level() >= event_level::debug) {
+    Aws::Utils::Logging::InitializeAWSLogging(
+        Aws::MakeShared<Aws::Utils::Logging::DefaultLogSystem>(
+            "Repertory", Aws::Utils::Logging::LogLevel::Trace,
+            utils::path::combine(config_.get_log_directory(), {"aws_sdk_"})));
+  }
+
+  Aws::Auth::AWSCredentials credentials;
+  credentials.SetAWSAccessKeyId(s3_config_.access_key);
+  credentials.SetAWSSecretKey(s3_config_.secret_key);
+
+  Aws::Client::ClientConfiguration configuration;
+  configuration.endpointOverride = s3_config_.url;
+  configuration.httpRequestTimeoutMs = s3_config_.timeout_ms;
+  configuration.region = s3_config_.region;
+  configuration.requestTimeoutMs = s3_config_.timeout_ms;
+
+  configuration.connectTimeoutMs = s3_config_.timeout_ms;
+  // TODO make configurable
+  const auto enable_path_style = utils::string::begins_with(s3_config_.url, "http://localhost") ||
+                                 utils::string::begins_with(s3_config_.url, "https://localhost") ||
+                                 utils::string::begins_with(s3_config_.url, "http://127.0.0.1") ||
+                                 utils::string::begins_with(s3_config_.url, "https://127.0.0.1");
+
+  s3_client_ = std::make_unique<Aws::S3::S3Client>(
+      credentials, configuration, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
+      not enable_path_style);
+  polling::instance().set_callback(
+      {"s3_directory_cache", false, [this]() { this->clear_expired_directories(); }});
+}
+
+aws_s3_comm::~aws_s3_comm() {
+  polling::instance().remove_callback("s3_directory_cache");
+  Aws::ShutdownAPI(sdk_options_);
+  Aws::Utils::Logging::ShutdownAWSLogging();
+  s3_client_.reset();
+}
+
+void aws_s3_comm::clear_expired_directories() {
+  recur_mutex_lock l(cached_directories_mutex_);
+  std::vector<std::string> expired_list;
+  for (const auto &kv : cached_directories_) {
+    if (kv.second.expiration <= std::chrono::system_clock::now()) {
+      expired_list.emplace_back(kv.first);
+    }
+  }
+
+  for (const auto &expired : expired_list) {
+    event_system::instance().raise<debug_log>(__FUNCTION__, expired, "expired");
+    cached_directories_.erase(expired);
+  }
+}
+
+api_error aws_s3_comm::create_bucket(const std::string &api_path) {
+  std::string bucket_name, object_name;
+  get_bucket_name_and_object_name(api_path, empty_key, bucket_name, object_name);
+
+  if (config_.get_event_level() >= event_level::debug) {
+    event_system::instance().raise<debug_log>(__FUNCTION__, bucket_name, "begin");
+  }
+
+  auto ret = api_error::access_denied;
+  if (s3_config_.bucket.empty()) {
+    Aws::S3::Model::CreateBucketRequest request{};
+    request.SetBucket(bucket_name);
+    const auto outcome = s3_client_->CreateBucket(request);
+    if (outcome.IsSuccess()) {
+      remove_cached_directory(utils::path::get_parent_api_path(api_path));
+      ret = api_error::success;
+    } else {
+      const auto &error = outcome.GetError();
+      event_system::instance().raise<repertory_exception>(
+          __FUNCTION__, error.GetExceptionName() + "|" + error.GetMessage());
+      ret = api_error::comm_error;
+    }
+  }
+
+  if (config_.get_event_level() >= event_level::debug) {
+    event_system::instance().raise<debug_log>(__FUNCTION__, bucket_name,
+                                              "end|" + std::to_string(static_cast<int>(ret)));
+  }
+  return ret;
+}
+
+bool aws_s3_comm::exists(const std::string &api_path, const get_key_callback &get_key) const {
+  if (config_.get_event_level() >= event_level::debug) {
+    event_system::instance().raise<debug_log>(__FUNCTION__, api_path, "begin");
+  }
+  if (api_path == "/") {
+    return false;
+  }
+
+  auto ret = false;
+  if (not get_cached_file_exists(api_path, ret)) {
+    std::string bucket_name, object_name;
+    get_bucket_name_and_object_name(api_path, get_key, bucket_name, object_name);
+
+    Aws::S3::Model::HeadObjectRequest request{};
+    request.SetBucket(bucket_name);
+    request.SetKey(object_name);
+    ret = s3_client_->HeadObject(request).IsSuccess();
+  }
+
+  if (config_.get_event_level() >= event_level::debug) {
+    event_system::instance().raise<debug_log>(__FUNCTION__, api_path, "end|" + std::to_string(ret));
+  }
+  return ret;
+}
+
+void aws_s3_comm::get_bucket_name_and_object_name(const std::string &api_path,
+                                                  const get_key_callback &get_key,
+                                                  std::string &bucket_name,
+                                                  std::string &object_name) const {
+  bucket_name = s3_config_.bucket;
+  object_name = api_path.substr(1);
+
+  if (bucket_name.empty()) {
+    bucket_name = utils::string::split(api_path, '/')[1];
+    object_name = object_name.substr(bucket_name.size());
+    if (not object_name.empty() && (object_name[0] == '/')) {
+      object_name = object_name.substr(1);
+    }
+  }
+
+  const auto key = get_key();
+  if (not key.empty()) {
+    auto parts = utils::string::split(object_name, '/', false);
+    parts[parts.size() - 1u] = key;
+    object_name = utils::string::join(parts, '/');
+  }
+}
+
+bool aws_s3_comm::get_cached_directory_item_count(const std::string &api_path,
+                                                  std::size_t &count) const {
+  recur_mutex_lock l(cached_directories_mutex_);
+  if (cached_directories_.find(api_path) != cached_directories_.end()) {
+    count = cached_directories_.at(api_path).items.size();
+    return true;
+  }
+
+  return false;
+}
+
+bool aws_s3_comm::get_cached_directory_items(const std::string &api_path,
+                                             const meta_provider_callback &meta_provider,
+                                             directory_item_list &list) const {
+  unique_recur_mutex_lock l(cached_directories_mutex_);
+  if (cached_directories_.find(api_path) != cached_directories_.end()) {
+    auto &cachedEntry = cached_directories_.at(api_path);
+    list = cachedEntry.items;
+    cached_directories_[api_path].reset_timeout(
+        std::chrono::seconds(config_.get_s3_config().cache_timeout_secs));
+    l.unlock();
+
+    for (auto &item : list) {
+      meta_provider(item, false);
+    }
+    return true;
+  }
+  return false;
+}
+
+bool aws_s3_comm::get_cached_file_exists(const std::string &api_path, bool &exists) const {
+  exists = false;
+
+  unique_recur_mutex_lock l(cached_directories_mutex_);
+  const auto parent_api_path = utils::path::get_parent_api_path(api_path);
+  if (cached_directories_.find(parent_api_path) != cached_directories_.end()) {
+    auto &entry = cached_directories_.at(parent_api_path);
+    exists =
+        std::find_if(entry.items.begin(), entry.items.end(), [&api_path](const auto &item) -> bool {
+          return not item.directory && (api_path == item.api_path);
+        }) != entry.items.end();
+    if (exists) {
+      cached_directories_[api_path].reset_timeout(
+          std::chrono::seconds(config_.get_s3_config().cache_timeout_secs));
+      return true;
+    }
+  }
+
+  return false;
+}
+
+std::size_t
+aws_s3_comm::get_directory_item_count(const std::string &api_path,
+                                      const meta_provider_callback &meta_provider) const {
+  if (config_.get_event_level() >= event_level::debug) {
+    event_system::instance().raise<debug_log>(__FUNCTION__, api_path, "begin");
+  }
+
+  std::size_t ret = 0u;
+  if (not(s3_config_.bucket.empty() && (api_path == "/"))) {
+    if (not get_cached_directory_item_count(api_path, ret)) {
+      directory_item_list list;
+      grab_directory_items(api_path, meta_provider, list);
+      return list.size();
+    }
+  }
+
+  if (config_.get_event_level() >= event_level::debug) {
+    event_system::instance().raise<debug_log>(__FUNCTION__, api_path, "end|" + std::to_string(ret));
+  }
+  return ret;
+}
+
+api_error aws_s3_comm::get_directory_items(const std::string &api_path,
+                                           const meta_provider_callback &meta_provider,
+                                           directory_item_list &list) const {
+  if (config_.get_event_level() >= event_level::debug) {
+    event_system::instance().raise<debug_log>(__FUNCTION__, api_path, "begin");
+  }
+
+  auto ret = api_error::success;
+  if (not get_cached_directory_items(api_path, meta_provider, list)) {
+    ret = grab_directory_items(api_path, meta_provider, list);
+  }
+
+  if (config_.get_event_level() >= event_level::debug) {
+    event_system::instance().raise<debug_log>(__FUNCTION__, api_path,
+                                              "end|" + std::to_string(std::uint8_t(ret)));
+  }
+  return ret;
+}
+
+api_error aws_s3_comm::get_file(const std::string &api_path, const get_key_callback &get_key,
+                                const get_name_callback &get_name,
+                                const get_token_callback &get_token, api_file &file) const {
+  if (config_.get_event_level() >= event_level::debug) {
+    event_system::instance().raise<debug_log>(__FUNCTION__, api_path, "begin");
+  }
+  auto ret = api_error::success;
+
+  std::string bucket_name, object_name;
+  get_bucket_name_and_object_name(api_path, get_key, bucket_name, object_name);
+
+  Aws::S3::Model::HeadObjectRequest request{};
+  request.SetBucket(bucket_name);
+  request.SetKey(object_name);
+
+  const auto outcome = s3_client_->HeadObject(request);
+  if (outcome.IsSuccess()) {
+    const auto key = get_key();
+    auto object = outcome.GetResult();
+    object_name = get_name(key, object_name);
+    file.accessed_date = utils::get_file_time_now();
+    file.api_path = utils::path::create_api_path(utils::path::combine(bucket_name, {object_name}));
+    file.api_parent = utils::path::get_parent_api_path(file.api_path);
+    file.changed_date = object.GetLastModified().Millis() * 1000u * 1000u;
+    file.created_date = object.GetLastModified().Millis() * 1000u * 1000u;
+    file.encryption_token = get_token();
+    if (file.encryption_token.empty()) {
+      file.file_size = object.GetContentLength();
+    } else {
+      file.file_size =
+          utils::encryption::encrypting_reader::calculate_decrypted_size(object.GetContentLength());
+    }
+    file.modified_date = object.GetLastModified().Millis() * 1000u * 1000u;
+    file.recoverable = true;
+    file.redundancy = 3.0;
+  } else {
+    const auto &error = outcome.GetError();
+    event_system::instance().raise<repertory_exception>(__FUNCTION__, error.GetExceptionName() +
+                                                                          "|" + error.GetMessage());
+    ret = api_error::comm_error;
+  }
+
+  if (config_.get_event_level() >= event_level::debug) {
+    event_system::instance().raise<debug_log>(__FUNCTION__, api_path,
+                                              "end|" + std::to_string(std::uint8_t(ret)));
+  }
+  return ret;
+}
+
+api_error aws_s3_comm::get_file_list(const get_api_file_token_callback &get_api_file_token,
+                                     const get_name_callback &get_name, api_file_list &list) const {
+  if (config_.get_event_level() >= event_level::debug) {
+    event_system::instance().raise<debug_log>(__FUNCTION__, "/", "begin");
+  }
+  list.clear();
+  auto ret = api_error::success;
+
+  const auto bucket_name = s3_config_.bucket;
+  if (bucket_name.empty()) {
+    const auto outcome = s3_client_->ListBuckets();
+    if (outcome.IsSuccess()) {
+      const auto &bucket_list = outcome.GetResult().GetBuckets();
+      for (const auto &bucket : bucket_list) {
+        get_file_list(bucket.GetName(), get_api_file_token, get_name, list);
+      }
+    } else {
+      const auto &error = outcome.GetError();
+      event_system::instance().raise<repertory_exception>(
+          __FUNCTION__, error.GetExceptionName() + "|" + error.GetMessage());
+      ret = api_error::comm_error;
+    }
+  } else {
+    ret = get_file_list("", get_api_file_token, get_name, list);
+  }
+
+  return ret;
+}
+
+api_error aws_s3_comm::get_file_list(const std::string &bucket_name,
+                                     const get_api_file_token_callback &get_api_file_token,
+                                     const get_name_callback &get_name, api_file_list &list) const {
+  auto ret = api_error::success;
+
+  Aws::S3::Model::ListObjectsRequest request{};
+  request.SetBucket(bucket_name.empty() ? s3_config_.bucket : bucket_name);
+
+  const auto outcome = s3_client_->ListObjects(request);
+  if (outcome.IsSuccess()) {
+    const auto &object_list = outcome.GetResult().GetContents();
+    for (auto const &object : object_list) {
+      api_file file{};
+      file.accessed_date = utils::get_file_time_now();
+
+      std::string object_name = object.GetKey();
+      object_name =
+          get_name(*(utils::string::split(object_name, '/', false).end() - 1u), object_name);
+      file.api_path = utils::path::create_api_path(utils::path::combine(
+          bucket_name.empty() ? s3_config_.bucket : bucket_name, {object_name}));
+      file.api_parent = utils::path::get_parent_api_path(file.api_path);
+      file.changed_date = object.GetLastModified().Millis() * 1000u * 1000u;
+      file.created_date = object.GetLastModified().Millis() * 1000u * 1000u;
+      file.encryption_token = get_api_file_token(file.api_path);
+      if (file.encryption_token.empty()) {
+        file.file_size = object.GetSize();
+      } else {
+        file.file_size =
+            object.GetSize() -
+            (utils::divide_with_ceiling(
+                 static_cast<std::uint64_t>(object.GetSize()),
+                 static_cast<std::uint64_t>(
+                     utils::encryption::encrypting_reader::get_encrypted_chunk_size())) *
+             utils::encryption::encrypting_reader::get_header_size());
+      }
+      file.modified_date = object.GetLastModified().Millis() * 1000u * 1000u;
+      file.recoverable = true;
+      file.redundancy = 3.0;
+      list.emplace_back(std::move(file));
+    }
+  } else {
+    const auto &error = outcome.GetError();
+    event_system::instance().raise<repertory_exception>(__FUNCTION__, error.GetExceptionName() +
+                                                                          "|" + error.GetMessage());
+    ret = api_error::comm_error;
+  }
+
+  return ret;
+}
+
+api_error aws_s3_comm::grab_directory_items(const std::string &api_path,
+                                            const meta_provider_callback &meta_provider,
+                                            directory_item_list &list) const {
+  auto ret = api_error::success;
+  if (s3_config_.bucket.empty() && (api_path == "/")) {
+    const auto outcome = s3_client_->ListBuckets();
+    if (outcome.IsSuccess()) {
+      const auto &bucket_list = outcome.GetResult().GetBuckets();
+      for (const auto &bucket : bucket_list) {
+        directory_item di{};
+        di.api_path = utils::path::create_api_path(utils::path::combine("/", {bucket.GetName()}));
+        di.api_parent = utils::path::get_parent_api_path(di.api_path);
+        di.directory = true;
+        di.size = get_directory_item_count(di.api_path, meta_provider);
+        meta_provider(di, true);
+        list.emplace_back(std::move(di));
+      }
+
+      set_cached_directory_items(api_path, list);
+    } else {
+      const auto &error = outcome.GetError();
+      event_system::instance().raise<repertory_exception>(
+          __FUNCTION__, error.GetExceptionName() + "|" + error.GetMessage());
+      ret = api_error::comm_error;
+    }
+  } else {
+    std::string bucket_name, object_name;
+    get_bucket_name_and_object_name(api_path, empty_key, bucket_name, object_name);
+
+    Aws::S3::Model::ListObjectsRequest request{};
+    request.SetBucket(bucket_name);
+    request.SetDelimiter("/");
+    request.SetPrefix(object_name.empty() ? object_name : object_name + "/");
+
+    const auto outcome = s3_client_->ListObjects(request);
+    if (outcome.IsSuccess()) {
+      const auto &object_list = outcome.GetResult().GetContents();
+      for (auto const &object : object_list) {
+        directory_item item{};
+        item.api_path =
+            utils::path::create_api_path(utils::path::combine(bucket_name, {object.GetKey()}));
+        item.api_parent = utils::path::get_parent_api_path(item.api_path);
+        item.directory = false;
+        item.size = object.GetSize();
+        meta_provider(item, true);
+        list.emplace_back(std::move(item));
+      }
+
+      set_cached_directory_items(api_path, list);
+    } else {
+      const auto &error = outcome.GetError();
+      event_system::instance().raise<repertory_exception>(
+          __FUNCTION__, error.GetExceptionName() + "|" + error.GetMessage());
+      ret = api_error::comm_error;
+    }
+  }
+
+  return ret;
+}
+
+api_error aws_s3_comm::read_file_bytes(const std::string &api_path, const std::size_t &size,
+                                       const std::uint64_t &offset, std::vector<char> &data,
+                                       const get_key_callback &get_key,
+                                       const get_size_callback &get_size,
+                                       const get_token_callback &get_token,
+                                       const bool &stop_requested) const {
+  auto ret = api_error::success;
+  data.clear();
+
+  std::string bucket_name, object_name;
+  get_bucket_name_and_object_name(api_path, get_key, bucket_name, object_name);
+
+  const auto encryption_token = get_token();
+  const auto data_size = get_size();
+  if (encryption_token.empty()) {
+    Aws::S3::Model::GetObjectRequest request{};
+    request.SetBucket(bucket_name);
+    request.SetKey(object_name);
+    request.SetResponseContentType("application/octet-stream");
+    request.SetRange("bytes=" + utils::string::from_uint64(offset) + "-" +
+                     utils::string::from_uint64(offset + size - 1u));
+    request.SetContinueRequestHandler(
+        [&stop_requested](const Aws::Http::HttpRequest *) { return not stop_requested; });
+
+    auto outcome = s3_client_->GetObject(request);
+    if (outcome.IsSuccess()) {
+      auto result = outcome.GetResultWithOwnership();
+      const auto len = result.GetContentLength();
+      data.resize(len);
+      result.GetBody().read(&data[0], len);
+    } else {
+      const auto &error = outcome.GetError();
+      event_system::instance().raise<repertory_exception>(
+          __FUNCTION__, error.GetExceptionName() + "|" + error.GetMessage());
+      ret = api_error::comm_error;
+    }
+  } else {
+    const auto key = utils::encryption::generate_key(encryption_token);
+    ret = utils::encryption::read_encrypted_range(
+        {offset, offset + size - 1}, key,
+        [&](std::vector<char> &ct, const std::uint64_t &start_offset,
+            const std::uint64_t &end_offset) -> api_error {
+          return read_file_bytes(
+              api_path, (end_offset - start_offset + 1u), start_offset, ct, get_key, get_size,
+              []() -> std::string { return ""; }, stop_requested);
+        },
+        data_size, data);
+  }
+
+  return ret;
+}
+
+api_error aws_s3_comm::remove_bucket(const std::string &api_path) {
+  std::string bucket_name, object_name;
+  get_bucket_name_and_object_name(api_path, empty_key, bucket_name, object_name);
+
+  if (config_.get_event_level() >= event_level::debug) {
+    event_system::instance().raise<debug_log>(__FUNCTION__, bucket_name, "begin");
+  }
+
+  auto ret = api_error::access_denied;
+  if (s3_config_.bucket.empty()) {
+    Aws::S3::Model::DeleteBucketRequest request{};
+    request.SetBucket(bucket_name);
+    const auto outcome = s3_client_->DeleteBucket(request);
+    if (outcome.IsSuccess()) {
+      remove_cached_directory(api_path);
+      ret = api_error::success;
+    } else {
+      const auto &error = outcome.GetError();
+      event_system::instance().raise<repertory_exception>(
+          __FUNCTION__, error.GetExceptionName() + "|" + error.GetMessage());
+      ret = api_error::comm_error;
+    }
+  }
+
+  if (config_.get_event_level() >= event_level::debug) {
+    event_system::instance().raise<debug_log>(__FUNCTION__, bucket_name,
+                                              "end|" + std::to_string(static_cast<int>(ret)));
+  }
+  return ret;
+}
+
+void aws_s3_comm::remove_cached_directory(const std::string &api_path) {
+  recur_mutex_lock l(cached_directories_mutex_);
+  cached_directories_.erase(api_path);
+}
+
+api_error aws_s3_comm::remove_file(const std::string &api_path, const get_key_callback &get_key) {
+  if (config_.get_event_level() >= event_level::debug) {
+    event_system::instance().raise<debug_log>(__FUNCTION__, api_path, "begin");
+  }
+  auto ret = api_error::success;
+
+  std::string bucket_name, object_name;
+  get_bucket_name_and_object_name(api_path, get_key, bucket_name, object_name);
+
+  Aws::S3::Model::DeleteObjectRequest request{};
+  request.SetBucket(bucket_name);
+  request.SetKey(object_name);
+  const auto outcome = s3_client_->DeleteObject(request);
+  if (outcome.IsSuccess()) {
+    remove_cached_directory(utils::path::get_parent_api_path(api_path));
+  } else {
+    const auto &error = outcome.GetError();
+    event_system::instance().raise<repertory_exception>(__FUNCTION__, error.GetExceptionName() +
+                                                                          "|" + error.GetMessage());
+    ret = api_error::comm_error;
+  }
+
+  if (config_.get_event_level() >= event_level::debug) {
+    event_system::instance().raise<debug_log>(__FUNCTION__, api_path,
+                                              "end|" + std::to_string(std::uint8_t(ret)));
+  }
+  return ret;
+}
+
+api_error aws_s3_comm::rename_file(const std::string & /*api_path*/,
+                                   const std::string & /*new_api_path*/) {
+  return api_error::not_implemented;
+  /*   if (config_.get_event_level() >= event_level::debug) { */
+  /*     event_system::instance().raise<debug_log>(__FUNCTION__, api_path, "begin"); */
+  /*   } */
+  /*   auto ret = api_error::success; */
+  /*  */
+  /*   std::string bucket_name, object_name; */
+  /*   get_bucket_name_and_object_name(api_path, bucket_name, object_name); */
+  /*  */
+  /*   std::string new_object_name; */
+  /*   get_bucket_name_and_object_name(new_api_path, bucket_name, new_object_name); */
+  /*  */
+  /*   Aws::S3::Model::CopyObjectRequest request{}; */
+  /*   request.SetBucket(bucket_name); */
+  /*   request.SetCopySource(bucket_name + '/' + object_name); */
+  /*   request.SetKey(new_object_name); */
+  /*  */
+  /*   const auto outcome = s3_client_->CopyObject(request); */
+  /*   if (outcome.IsSuccess()) { */
+  /*     ret = remove_file(api_path); */
+  /*   } else { */
+  /*     const auto &error = outcome.GetError(); */
+  /*     event_system::instance().raise<repertory_exception>(__FUNCTION__, error.GetExceptionName()
+   * +
+   * "|" + */
+  /*                                                                         error.GetMessage()); */
+  /*     ret = api_error::comm_error; */
+  /*   } */
+  /*  */
+  /*   if (config_.get_event_level() >= event_level::debug) { */
+  /*     event_system::instance().raise<debug_log>(__FUNCTION__, api_path, */
+  /*                                             "end|" + std::to_string(std::uint8_t(ret))); */
+  /*   } */
+  /*   return ret; */
+}
+
+void aws_s3_comm::set_cached_directory_items(const std::string &api_path,
+                                             directory_item_list list) const {
+  recur_mutex_lock l(cached_directories_mutex_);
+  cached_directories_[api_path].items = std::move(list);
+  cached_directories_[api_path].reset_timeout(
+      std::chrono::seconds(config_.get_s3_config().cache_timeout_secs));
+}
+
+api_error aws_s3_comm::upload_file(const std::string &api_path, const std::string &source_path,
+                                   const std::string &encryption_token,
+                                   const get_key_callback &get_key, const set_key_callback &set_key,
+                                   const bool &stop_requested) {
+  static const auto no_stop = false;
+  if (config_.get_event_level() >= event_level::debug) {
+    event_system::instance().raise<debug_log>(__FUNCTION__, api_path, "begin");
+  }
+  auto ret = api_error::success;
+
+  std::string bucket_name, object_name;
+  get_bucket_name_and_object_name(api_path, get_key, bucket_name, object_name);
+
+  std::shared_ptr<Aws::IOStream> file_stream;
+  if (encryption_token.empty()) {
+    file_stream = Aws::MakeShared<Aws::FStream>(&source_path[0], &source_path[0],
+                                                std::ios_base::in | std::ios_base::binary);
+  } else {
+    const auto file_name = ([&api_path]() -> std::string {
+      return *(utils::string::split(api_path, '/', false).end() - 1u);
+    })();
+
+    const auto reader =
+        utils::encryption::encrypting_reader(file_name, source_path, no_stop, encryption_token, -1);
+    auto key = get_key();
+    if (key.empty()) {
+      key = reader.get_encrypted_file_name();
+      set_key(key);
+
+      auto parts = utils::string::split(object_name, '/', false);
+      parts[parts.size() - 1u] = key;
+      object_name = utils::string::join(parts, '/');
+    }
+
+    file_stream = reader.create_iostream();
+  }
+  file_stream->seekg(0);
+
+  Aws::S3::Model::PutObjectRequest request{};
+  request.SetBucket(bucket_name);
+  request.SetKey(object_name);
+  request.SetBody(file_stream);
+  request.SetContinueRequestHandler(
+      [&stop_requested](const Aws::Http::HttpRequest *) { return not stop_requested; });
+
+  const auto outcome = s3_client_->PutObject(request);
+  if (outcome.IsSuccess()) {
+    remove_cached_directory(utils::path::get_parent_api_path(api_path));
+  } else {
+    const auto &error = outcome.GetError();
+    event_system::instance().raise<repertory_exception>(__FUNCTION__, error.GetExceptionName() +
+                                                                          "|" + error.GetMessage());
+    ret = api_error::upload_failed;
+  }
+
+  if (config_.get_event_level() >= event_level::debug) {
+    event_system::instance().raise<debug_log>(__FUNCTION__, api_path,
+                                              "end|" + std::to_string(std::uint8_t(ret)));
+  }
+  return ret;
+}
+} // namespace repertory
+
+#endif // REPERTORY_ENABLE_S3

src/comm/curl/curl_comm.cpp

@@ -0,0 +1,849 @@
+/*
+  Copyright <2018-2022> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
+  associated documentation files (the "Software"), to deal in the Software without restriction,
+  including without limitation the rights to use, copy, modify, merge, publish, distribute,
+  sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all copies or
+  substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
+  NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
+  OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/
+#include "comm/curl/curl_comm.hpp"
+#include "comm/curl/curl_resolver.hpp"
+#include "curl/curl.h"
+#include "types/repertory.hpp"
+#include "utils/Base64.hpp"
+#include "utils/encryption.hpp"
+#include "utils/encrypting_reader.hpp"
+#include "utils/file_utils.hpp"
+#include "utils/path_utils.hpp"
+
+namespace repertory {
+struct curl_setup {
+  bool allow_timeout = false;
+  const host_config &hc;
+  const http_parameters *parameters = nullptr;
+  http_headers *headers = nullptr;
+  bool post = false;
+  raw_write_data *write_data = nullptr;
+  std::string *write_string = nullptr;
+  std::unique_ptr<curl_resolver> resolver;
+};
+
+struct raw_write_data {
+  std::vector<char> *buffer;
+  const bool &stop_requested;
+};
+
+struct read_data {
+  const bool *stop_requested = nullptr;
+  native_file *nf = nullptr;
+  std::uint64_t offset = 0u;
+};
+
+CURL *curl_comm::common_curl_setup(const std::string &path, curl_setup &setup, std::string &url,
+                                   std::string &fields) {
+  return common_curl_setup(utils::create_curl(), path, setup, url, fields);
+}
+
+CURL *curl_comm::common_curl_setup(CURL *curl_handle, const std::string &path, curl_setup &setup,
+                                   std::string &url, std::string &fields) {
+  url = construct_url(curl_handle, path, setup.hc);
+  if (setup.post) {
+    curl_easy_setopt(curl_handle, CURLOPT_HTTPGET, 0);
+    curl_easy_setopt(curl_handle, CURLOPT_POST, 1L);
+    if (setup.parameters && not setup.parameters->empty()) {
+      for (const auto &param : *setup.parameters) {
+        if (not fields.empty()) {
+          fields += "&";
+        }
+        if (param.first ==
+            "new" + app_config::get_provider_path_name(config_.get_provider_type())) {
+          fields += (param.first + "=" + url_encode(curl_handle, param.second));
+        } else {
+          fields += (param.first + "=" + param.second);
+        }
+      }
+      curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, &fields[0]);
+    } else {
+      curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, "");
+    }
+  } else {
+    curl_easy_setopt(curl_handle, CURLOPT_POST, 0);
+    curl_easy_setopt(curl_handle, CURLOPT_HTTPGET, 1L);
+    if (setup.parameters && not setup.parameters->empty()) {
+      url += "?";
+      for (const auto &param : *setup.parameters) {
+        if (url[url.size() - 1] != '?') {
+          url += "&";
+        }
+        url += (param.first + "=" + url_encode(curl_handle, param.second));
+      }
+    }
+  }
+
+  if (not setup.hc.agent_string.empty()) {
+    curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, &setup.hc.agent_string[0]);
+  }
+
+  if (setup.write_data) {
+    setup.write_data->buffer->clear();
+    curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, setup.write_data);
+    curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, write_data_callback_);
+  } else if (setup.write_string) {
+    setup.write_string->clear();
+    curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, setup.write_string);
+    curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, write_string_callback_);
+  }
+
+  curl_easy_setopt(curl_handle, CURLOPT_URL, url.c_str());
+
+  if (setup.allow_timeout) {
+    curl_easy_setopt(curl_handle, CURLOPT_TIMEOUT_MS, setup.hc.timeout_ms);
+  }
+
+  if (not setup.hc.api_password.empty()) {
+    curl_easy_setopt(curl_handle, CURLOPT_USERNAME, "");
+    curl_easy_setopt(curl_handle, CURLOPT_PASSWORD, &setup.hc.api_password[0]);
+  }
+
+  if (setup.headers) {
+    curl_easy_setopt(curl_handle, CURLOPT_HEADERDATA, setup.headers);
+    curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, write_header_callback_);
+  }
+
+  std::vector<std::string> items = {"localhost:" + std::to_string(setup.hc.api_port) +
+                                    ":127.0.0.1"};
+  setup.resolver = std::make_unique<curl_resolver>(curl_handle, items);
+
+  return curl_handle;
+}
+
+std::string curl_comm::construct_url(CURL *curl_handle, const std::string &relative_path,
+                                     const host_config &hc) {
+  auto custom_port = (((hc.protocol == "http") && (hc.api_port == 80u)) ||
+                      ((hc.protocol == "https") && (hc.api_port == 443u)))
+                         ? ""
+                         : ":" + std::to_string(hc.api_port);
+  auto ret = hc.protocol + "://" + utils::string::trim_copy(hc.host_name_or_ip) + custom_port;
+  auto path = utils::path::combine("/", {hc.path});
+  if (relative_path.empty()) {
+    ret += utils::path::create_api_path(path);
+    if (utils::string::ends_with(hc.path, "/")) {
+      ret += '/';
+    }
+  } else {
+    path = utils::path::combine(path, {url_encode(curl_handle, relative_path, true)});
+    ret += utils::path::create_api_path(path);
+    if (utils::string::ends_with(relative_path, "/")) {
+      ret += '/';
+    }
+  }
+  return ret;
+}
+
+bool curl_comm::create_auth_session(CURL *&curl_handle, const app_config &config, host_config hc,
+                                    std::string &session) {
+  auto ret = true;
+  if (not curl_handle) {
+    curl_handle = utils::create_curl();
+  }
+
+  if (not hc.auth_url.empty() && not hc.auth_user.empty()) {
+    event_system::instance().raise<comm_auth_begin>(hc.auth_url, hc.auth_user);
+    ret = false;
+    session = utils::create_uuid_string();
+    const auto cookie_path = utils::path::combine(config.get_data_directory(), {session + ".txt"});
+    http_headers headers{};
+    auto url = utils::string::right_trim(utils::string::trim(hc.auth_url), '/') + "/api/login";
+
+    if (not hc.agent_string.empty()) {
+      curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, &hc.agent_string[0]);
+    }
+
+    if (not hc.api_password.empty()) {
+      curl_easy_setopt(curl_handle, CURLOPT_USERNAME, "");
+      curl_easy_setopt(curl_handle, CURLOPT_PASSWORD, &hc.api_password[0]);
+    }
+
+    struct curl_slist *hs = nullptr;
+    hs = curl_slist_append(hs, "Content-Type: application/json");
+    curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, hs);
+
+    const auto payload = json({
+                                  {"email", hc.auth_user},
+                                  {"password", hc.auth_password},
+                              })
+                             .dump(2);
+    curl_easy_setopt(curl_handle, CURLOPT_COOKIEFILE, cookie_path.c_str());
+    curl_easy_setopt(curl_handle, CURLOPT_COOKIEJAR, cookie_path.c_str());
+    curl_easy_setopt(curl_handle, CURLOPT_HEADERDATA, &headers);
+    curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, write_header_callback_);
+    curl_easy_setopt(curl_handle, CURLOPT_HTTPGET, 0L);
+    curl_easy_setopt(curl_handle, CURLOPT_POST, 1L);
+    curl_easy_setopt(curl_handle, CURLOPT_TIMEOUT_MS, hc.timeout_ms);
+    curl_easy_setopt(curl_handle, CURLOPT_URL, url.c_str());
+    curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, payload.c_str());
+
+    long code{};
+    auto res = curl_easy_perform(curl_handle);
+    curl_easy_getinfo(curl_handle, CURLINFO_HTTP_CODE, &code);
+    if ((res == CURLE_OK) && ((code >= 200) && (code < 300))) {
+      auto cookie = headers["set-cookie"];
+      if (cookie.empty()) {
+        code = 400;
+      } else {
+        cookie = headers["set-cookie"];
+        if (cookie.empty() || not utils::string::contains(cookie, "skynet-jwt=")) {
+          code = 401;
+        } else {
+          ret = true;
+        }
+      }
+    }
+
+    if (ret) {
+      curl_easy_setopt(curl_handle, CURLOPT_COOKIELIST, "FLUSH");
+      update_auth_session(utils::reset_curl(curl_handle), config, session);
+    } else {
+      curl_easy_cleanup(curl_handle);
+      release_auth_session(config, hc, session);
+    }
+    curl_slist_free_all(hs);
+
+    event_system::instance().raise<comm_auth_end>(hc.auth_url, hc.auth_user, res, code);
+  }
+
+  return ret;
+}
+
+api_error curl_comm::get_or_post(const host_config &hc, const bool &post, const std::string &path,
+                                 const http_parameters &parameters, json &data, json &error,
+                                 http_headers *headers, std::function<void(CURL *curl_handle)> cb) {
+  std::string result;
+  auto setup = curl_setup{not post, hc, &parameters, headers, post, nullptr, &result, nullptr};
+
+  std::string fields;
+  std::string url;
+  auto *curl_handle = common_curl_setup(path, setup, url, fields);
+  if (cb) {
+    cb(curl_handle);
+  }
+
+  if (config_.get_event_level() >= event_level::verbose) {
+    if (post) {
+      event_system::instance().raise<comm_post_begin>(url, fields);
+    } else {
+      event_system::instance().raise<comm_get_begin>(url);
+    }
+  }
+
+  std::unique_ptr<std::chrono::system_clock::time_point> tp(nullptr);
+  if (config_.get_enable_comm_duration_events()) {
+    tp = std::make_unique<std::chrono::system_clock::time_point>(std::chrono::system_clock::now());
+  }
+
+  const auto curl_code = curl_easy_perform(curl_handle);
+  if (curl_code == CURLE_OPERATION_TIMEDOUT) {
+    event_system::instance().raise<repertory_exception>(__FUNCTION__, "CURL timeout: " + path);
+  }
+
+  long http_code = -1;
+  curl_easy_getinfo(curl_handle, CURLINFO_RESPONSE_CODE, &http_code);
+  if (tp != nullptr) {
+    const auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(
+        std::chrono::system_clock::now() - *tp);
+    event_system::instance().raise<comm_duration>(url, std::to_string(duration.count()));
+  }
+
+  const auto ret = process_json_response(url, curl_code, http_code, result, data, error);
+  if (config_.get_event_level() >= event_level::verbose) {
+    if (post) {
+      event_system::instance().raise<comm_post_end>(
+          url, curl_code, http_code, ((ret == api_error::success) ? data.dump(2) : error.dump(2)));
+    } else {
+      event_system::instance().raise<comm_get_end>(
+          url, curl_code, http_code, ((ret == api_error::success) ? data.dump(2) : error.dump(2)));
+    }
+  }
+
+  curl_easy_cleanup(curl_handle);
+
+  return ret;
+}
+
+api_error curl_comm::get_range(const host_config &hc, const std::string &path,
+                               const std::uint64_t &data_size, const http_parameters &parameters,
+                               const std::string &encryption_token, std::vector<char> &data,
+                               const http_ranges &ranges, json &error, http_headers *headers,
+                               const bool &stop_requested) {
+  if (encryption_token.empty()) {
+    return get_range_unencrypted(hc, path, parameters, data, ranges, error, headers,
+                                 stop_requested);
+  }
+
+  if (ranges.empty()) {
+    return api_error::error;
+  }
+
+  const auto key = utils::encryption::generate_key(encryption_token);
+  for (const auto &range : ranges) {
+    const auto result = utils::encryption::read_encrypted_range(
+        range, key,
+        [&](std::vector<char> &ct, const std::uint64_t &start_offset,
+            const std::uint64_t &end_offset) -> api_error {
+          const auto ret =
+              get_range_unencrypted(hc, path, parameters, ct, {{start_offset, end_offset}}, error,
+                                    headers, stop_requested);
+          headers = nullptr;
+          return ret;
+        },
+        data_size, data);
+    if (result != api_error::success) {
+      return result;
+    }
+  }
+
+  return api_error::success;
+}
+
+api_error curl_comm::get_range_unencrypted(const host_config &hc, const std::string &path,
+                                           const http_parameters &parameters,
+                                           std::vector<char> &data, const http_ranges &ranges,
+                                           json &error, http_headers *headers,
+                                           const bool &stop_requested) {
+  raw_write_data wd = {&data, stop_requested};
+
+  auto setup = curl_setup{false, hc, &parameters, headers, false, &wd, nullptr, nullptr};
+
+  std::string fields;
+  std::string url;
+  auto *curl_handle = common_curl_setup(path, setup, url, fields);
+
+  std::string range_list;
+  if (not ranges.empty()) {
+    range_list = std::accumulate(
+        std::next(ranges.begin()), ranges.end(), http_range_to_string(ranges[0]),
+        [](const auto &l, const auto &r) { return l + ',' + http_range_to_string(r); });
+    curl_easy_setopt(curl_handle, CURLOPT_RANGE, &range_list[0]);
+  }
+
+  return execute_binary_operation<comm_get_range_begin, comm_get_range_end>(curl_handle, url, data,
+                                                                            error, stop_requested);
+}
+
+api_error curl_comm::get_raw(const host_config &hc, const std::string &path,
+                             const http_parameters &parameters, std::vector<char> &data,
+                             json &error, const bool &stop_requested) {
+  raw_write_data wd = {&data, stop_requested};
+
+  auto setup = curl_setup{false, hc, &parameters, nullptr, false, &wd, nullptr, nullptr};
+
+  std::string fields;
+  std::string url;
+  return execute_binary_operation<comm_get_range_begin, comm_get_range_end>(
+      common_curl_setup(path, setup, url, fields), url, data, error, stop_requested);
+}
+
+std::string curl_comm::http_range_to_string(const http_range &range) {
+  return std::to_string(range.begin) + '-' + std::to_string(range.end);
+}
+
+api_error curl_comm::post_file(const host_config &hc, const std::string &path,
+                               const std::string &source_path, const http_parameters &parameters,
+                               json &data, json &error, const bool &stop_requested) {
+  auto ret = api_error::os_error;
+  std::uint64_t file_size{};
+  if (utils::file::get_file_size(source_path, file_size)) {
+    std::string result;
+    auto setup = curl_setup{false, hc, &parameters, nullptr, true, nullptr, &result, nullptr};
+
+    std::string fields;
+    std::string url;
+    auto *curl_handle = common_curl_setup(path, setup, url, fields);
+    curl_easy_setopt(curl_handle, CURLOPT_UPLOAD, 1L);
+    curl_easy_setopt(curl_handle, CURLOPT_INFILESIZE_LARGE, file_size);
+
+    native_file::native_file_ptr nf;
+    native_file::create_or_open(source_path, nf);
+    if (nf) {
+      read_data rd = {&stop_requested};
+      rd.nf = nf.get();
+      curl_easy_setopt(curl_handle, CURLOPT_READDATA, &rd);
+      curl_easy_setopt(curl_handle, CURLOPT_READFUNCTION, read_data_callback_);
+
+      ret = execute_json_operation<comm_post_file_begin, comm_post_file_end>(
+          curl_handle, url, result, data, error, stop_requested);
+      nf->close();
+    }
+  }
+
+  return ret;
+}
+
+api_error curl_comm::post_multipart_file(const host_config &hc, const std::string &path,
+                                         const std::string &file_name,
+                                         const std::string &source_path,
+                                         const std::string &encryption_token, json &data,
+                                         json &error, const bool &stop_requested) {
+  std::string result;
+  std::uint64_t file_size{};
+  std::unique_ptr<utils::encryption::encrypting_reader> reader;
+  if (encryption_token.empty()) {
+    if (not utils::file::get_file_size(source_path, file_size)) {
+      return api_error::os_error;
+    }
+  } else {
+    try {
+      reader = std::make_unique<utils::encryption::encrypting_reader>(
+          file_name, source_path, stop_requested, encryption_token);
+      file_size = reader->get_total_size();
+    } catch (const std::exception &e) {
+      event_system::instance().raise<repertory_exception>(__FUNCTION__, e.what());
+      return api_error::error;
+    }
+  }
+
+  std::string session;
+  CURL *curl_handle = nullptr;
+  if (not session_manager_.create_auth_session(curl_handle, config_, hc, session)) {
+    return api_error::access_denied;
+  }
+
+  if (config_.get_provider_type() == provider_type::skynet) {
+    curl_easy_cleanup(curl_handle);
+
+    const auto fn = reader ? reader->get_encrypted_file_name() : file_name;
+    const auto fs = reader ? reader->get_total_size() : file_size;
+
+    std::string location;
+    if (tus_upload_create(hc, fn, fs, location)) {
+      std::string skylink;
+      if (tus_upload(hc, source_path, fn, fs, location, skylink, stop_requested, reader.get())) {
+        data["skylink"] = skylink;
+        return api_error::success;
+      }
+    }
+
+    return api_error::comm_error;
+  }
+
+  auto setup = curl_setup{false, hc, nullptr, nullptr, true, nullptr, &result, nullptr};
+
+  std::string fields;
+  std::string url;
+  common_curl_setup(curl_handle, path, setup, url, fields);
+
+  auto *curl_mime = curl_mime_init(curl_handle);
+  auto *mime_part = curl_mime_addpart(curl_mime);
+  curl_mime_name(mime_part, "file");
+
+  auto ret = api_error::success;
+  if (encryption_token.empty()) {
+    curl_mime_filename(mime_part, &file_name[0]);
+    curl_mime_filedata(mime_part, &source_path[0]);
+  } else {
+    try {
+      curl_mime_filename(mime_part, reader->get_encrypted_file_name().c_str());
+      curl_mime_data_cb(
+          mime_part, reader->get_total_size(),
+          static_cast<curl_read_callback>(utils::encryption::encrypting_reader::reader_function),
+          nullptr, nullptr, reader.get());
+    } catch (const std::exception &e) {
+      event_system::instance().raise<repertory_exception>(__FUNCTION__, e.what());
+      ret = api_error::error;
+    }
+  }
+
+  if (ret == api_error::success) {
+    curl_easy_setopt(curl_handle, CURLOPT_MIMEPOST, curl_mime);
+
+    ret = execute_json_operation<comm_post_multi_part_file_begin, comm_post_multi_part_file_end>(
+        curl_handle, url, result, data, error, stop_requested,
+        (ret == api_error::success) ? CURLE_OK : CURLE_SEND_ERROR);
+  }
+
+  curl_mime_free(curl_mime);
+  curl_easy_cleanup(curl_handle);
+
+  /*   if (not session.empty() && (ret == api_error::success)) { */
+  /*     auto pin_hc = hc; */
+  /*  */
+  /*     const auto skylink = data["skylink"].get<std::string>(); */
+  /*     utils::string::replace(pin_hc.path, "/skyfile", "/pin/" + skylink); */
+  /*  */
+  /*     ret = api_error::comm_error; */
+  /*     for (std::uint8_t i = 0u; not stop_requested && (i < 30u) && (ret != api_error::success);
+   * i++) { */
+  /*       if (i) { */
+  /*         event_system::instance().raise<repertory_exception>( */
+  /*             __FUNCTION__, "RETRY [" + std::to_string(i) + "] Pin failed for file: " +
+   * file_name); */
+  /*         std::this_thread::sleep_for(1s); */
+  /*       } */
+  /*  */
+  /*       json response; */
+  /*       http_headers headers{}; */
+  /*       ret = get_or_post(pin_hc, true, "", {}, response, error, &headers, [&](CURL *curl_handle)
+   * { */
+  /*         session_manager_.update_auth_session(curl_handle, config_, hc); */
+  /*       }); */
+  /*     } */
+  /*   } */
+
+  session_manager_.release_auth_session(config_, hc, session);
+
+  return ret;
+}
+
+api_error curl_comm::process_binary_response(const std::string &url, const CURLcode &res,
+                                             const long &http_code, std::vector<char> data,
+                                             json &error) {
+  const auto ret = process_response(
+      url, res, http_code, data.size(),
+      [&]() -> std::string { return (data.empty() ? "" : std::string(&data[0], data.size())); },
+      nullptr, error);
+
+  if (ret != api_error::success) {
+    data.clear();
+  }
+
+  return ret;
+}
+
+api_error curl_comm::process_json_response(const std::string &url, const CURLcode &res,
+                                           const long &http_code, const std::string &result,
+                                           json &data, json &error) {
+  const auto ret = process_response(
+      url, res, http_code, result.size(), [&]() -> std::string { return result; },
+      [&]() {
+        if (result.length()) {
+          data = json::parse(result.c_str());
+        }
+      },
+      error);
+
+  if (ret != api_error::success) {
+    data.clear();
+  }
+
+  return ret;
+}
+
+api_error curl_comm::process_response(const std::string &url, const CURLcode &res,
+                                      const long &http_code, const std::size_t &data_size,
+                                      const std::function<std::string()> &to_string_convertor,
+                                      const std::function<void()> &success_handler,
+                                      json &error) const {
+  auto ret = api_error::success;
+
+  auto construct_error = [&]() {
+    ret = api_error::comm_error;
+    const auto *curl_string = curl_easy_strerror(res);
+    std::string error_string(curl_string ? curl_string : "");
+    error["message"] = error_string + ":" + std::to_string(http_code);
+    error["url"] = url;
+  };
+
+  if ((res == CURLE_OK) && ((http_code >= 200) && (http_code < 300))) {
+    if (success_handler) {
+      success_handler();
+    }
+  } else if (data_size == 0u) {
+    construct_error();
+  } else {
+    try {
+      const auto tmp = json::parse(to_string_convertor().c_str());
+      if (tmp.find("message") != tmp.end()) {
+        ret = api_error::comm_error;
+        error = tmp;
+        error["url"] = url;
+      } else {
+        construct_error();
+      }
+    } catch (...) {
+      construct_error();
+    }
+  }
+
+  return ret;
+}
+
+void curl_comm::release_auth_session(const app_config &config, host_config hc,
+                                     const std::string &session) {
+  if (not hc.auth_url.empty() && not hc.auth_user.empty()) {
+    event_system::instance().raise<comm_auth_logout_begin>(hc.auth_url, hc.auth_user);
+    const auto cookie_path = utils::path::combine(config.get_data_directory(), {session + ".txt"});
+    const auto url =
+        utils::string::right_trim(utils::string::trim(hc.auth_url), '/') + "/api/logout";
+
+    auto *curl_handle = utils::create_curl();
+    if (not hc.agent_string.empty()) {
+      curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, &hc.agent_string[0]);
+    }
+
+    if (not hc.api_password.empty()) {
+      curl_easy_setopt(curl_handle, CURLOPT_USERNAME, "");
+      curl_easy_setopt(curl_handle, CURLOPT_PASSWORD, &hc.api_password[0]);
+    }
+
+    curl_easy_setopt(curl_handle, CURLOPT_COOKIEFILE, cookie_path.c_str());
+    curl_easy_setopt(curl_handle, CURLOPT_COOKIEJAR, cookie_path.c_str());
+    curl_easy_setopt(curl_handle, CURLOPT_HTTPGET, 0L);
+    curl_easy_setopt(curl_handle, CURLOPT_POST, 1L);
+    curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, "");
+    curl_easy_setopt(curl_handle, CURLOPT_URL, url.c_str());
+    curl_easy_setopt(curl_handle, CURLOPT_TIMEOUT_MS, hc.timeout_ms);
+    curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, write_null_callback_);
+
+    const auto curl_code = curl_easy_perform(curl_handle);
+
+    long http_code = -1;
+    curl_easy_getinfo(curl_handle, CURLINFO_HTTP_CODE, &http_code);
+    curl_easy_cleanup(curl_handle);
+
+    event_system::instance().raise<comm_auth_logout_end>(hc.auth_url, hc.auth_user, curl_code,
+                                                         http_code);
+
+    utils::file::delete_file(cookie_path);
+  }
+}
+
+bool curl_comm::tus_upload(host_config hc, const std::string &source_path,
+                           const std::string &file_name, std::uint64_t file_size,
+                           const std::string &location, std::string &skylink,
+                           const bool &stop_requested,
+                           utils::encryption::encrypting_reader *reader) {
+  static const constexpr std::uint64_t max_skynet_file_size = (1ull << 22ull) * 10ull;
+
+  auto tus_hc = hc;
+  utils::string::replace(tus_hc.path, "/skyfile", "/tus");
+
+  auto ret = true;
+  std::uint64_t offset = 0u;
+  native_file::native_file_ptr nf;
+  while (ret && file_size) {
+    const auto chunk_size = std::min(max_skynet_file_size, file_size);
+
+    auto *curl_handle = utils::create_curl();
+    curl_easy_setopt(curl_handle, CURLOPT_CUSTOMREQUEST, "PATCH");
+    curl_easy_setopt(curl_handle, CURLOPT_HTTPGET, 0);
+    curl_easy_setopt(curl_handle, CURLOPT_NOBODY, 0);
+    const auto upload_offset = "Upload-Offset: " + std::to_string(offset);
+    const auto content_length = "Content-Length: " + std::to_string(chunk_size);
+
+    struct curl_slist *hs = nullptr;
+    hs = curl_slist_append(hs, "tus-resumable: 1.0.0");
+    hs = curl_slist_append(hs, "Content-Type: application/offset+octet-stream");
+    hs = curl_slist_append(hs, content_length.c_str());
+    hs = curl_slist_append(hs, upload_offset.c_str());
+    curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, hs);
+
+    curl_easy_setopt(curl_handle, CURLOPT_INFILESIZE_LARGE, chunk_size);
+    curl_easy_setopt(curl_handle, CURLOPT_UPLOAD, 1L);
+
+    session_manager_.update_auth_session(curl_handle, config_, hc);
+
+    read_data rd = {&stop_requested};
+    if (reader) {
+      curl_easy_setopt(curl_handle, CURLOPT_READDATA, reader);
+      curl_easy_setopt(
+          curl_handle, CURLOPT_READFUNCTION,
+          static_cast<curl_read_callback>(utils::encryption::encrypting_reader::reader_function));
+    } else {
+      if (not nf) {
+        native_file::create_or_open(source_path, nf);
+      }
+
+      if (nf) {
+        rd.nf = nf.get();
+        rd.offset = offset;
+        curl_easy_setopt(curl_handle, CURLOPT_READDATA, &rd);
+        curl_easy_setopt(curl_handle, CURLOPT_READFUNCTION, read_data_callback_);
+      } else {
+        ret = false;
+      }
+    }
+
+    if (ret) {
+      event_system::instance().raise<comm_tus_upload_begin>(file_name, location, file_size, offset);
+      curl_easy_setopt(curl_handle, CURLOPT_URL, location.c_str());
+
+      const auto curl_code = curl_easy_perform(curl_handle);
+
+      long http_code = -1;
+      curl_easy_getinfo(curl_handle, CURLINFO_HTTP_CODE, &http_code);
+
+      event_system::instance().raise<comm_tus_upload_end>(file_name, location, file_size, offset,
+                                                          curl_code, http_code);
+      if ((ret = ((curl_code == CURLE_OK) && (http_code >= 200) && (http_code < 300)))) {
+        file_size -= chunk_size;
+        offset += chunk_size;
+      }
+    }
+
+    curl_easy_cleanup(curl_handle);
+    curl_slist_free_all(hs);
+  }
+
+  if (nf) {
+    nf->close();
+  }
+
+  if (ret) {
+    auto *curl_handle = utils::create_curl();
+
+    http_headers headers{};
+    curl_easy_setopt(curl_handle, CURLOPT_CUSTOMREQUEST, "HEAD");
+    curl_easy_setopt(curl_handle, CURLOPT_NOBODY, 1L);
+    curl_easy_setopt(curl_handle, CURLOPT_HEADERDATA, &headers);
+    curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, write_header_callback_);
+
+    session_manager_.update_auth_session(curl_handle, config_, hc);
+
+    struct curl_slist *hs = nullptr;
+    hs = curl_slist_append(hs, "tus-resumable: 1.0.0");
+    curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, hs);
+
+    curl_easy_setopt(curl_handle, CURLOPT_URL, location.c_str());
+
+    const auto res = curl_easy_perform(curl_handle);
+    long http_code = -1;
+    curl_easy_getinfo(curl_handle, CURLINFO_HTTP_CODE, &http_code);
+
+    ret = ((res == CURLE_OK) && (http_code >= 200) && (http_code < 300));
+
+    skylink = headers["skynet-skylink"];
+
+    curl_easy_cleanup(curl_handle);
+    curl_slist_free_all(hs);
+  }
+
+  return ret;
+}
+
+bool curl_comm::tus_upload_create(host_config hc, const std::string &fileName,
+                                  const std::uint64_t &fileSize, std::string &location) {
+  auto tus_hc = hc;
+  utils::string::replace(tus_hc.path, "/skyfile", "/tus");
+
+  auto *curl_handle = utils::create_curl();
+
+  const auto url = construct_url(curl_handle, "", tus_hc);
+  event_system::instance().raise<comm_tus_upload_create_begin>(fileName, url);
+
+  http_headers headers{};
+  curl_easy_setopt(curl_handle, CURLOPT_HTTPGET, 0);
+  curl_easy_setopt(curl_handle, CURLOPT_POST, 1L);
+  curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, "");
+  curl_easy_setopt(curl_handle, CURLOPT_HEADERDATA, &headers);
+  curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, write_header_callback_);
+
+  const auto upload_length = "upload-length: " + utils::string::from_uint64(fileSize);
+  const auto upload_metadata = "upload-metadata: filename " + macaron::Base64::Encode(fileName) +
+                               ",filetype " + macaron::Base64::Encode("application/octet-stream");
+
+  struct curl_slist *hs = nullptr;
+  hs = curl_slist_append(hs, "tus-resumable: 1.0.0");
+  hs = curl_slist_append(hs, "Content-Length: 0");
+  hs = curl_slist_append(hs, upload_length.c_str());
+  hs = curl_slist_append(hs, upload_metadata.c_str());
+  curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, hs);
+
+  session_manager_.update_auth_session(curl_handle, config_, hc);
+
+  curl_easy_setopt(curl_handle, CURLOPT_URL, url.c_str());
+
+  const auto res = curl_easy_perform(curl_handle);
+
+  long http_code = -1;
+  curl_easy_getinfo(curl_handle, CURLINFO_HTTP_CODE, &http_code);
+
+  const auto ret = (res == CURLE_OK && http_code == 201);
+  if (ret) {
+    location = headers["location"];
+  }
+
+  curl_easy_cleanup(curl_handle);
+  curl_slist_free_all(hs);
+
+  event_system::instance().raise<comm_tus_upload_create_end>(fileName, url, res, http_code);
+
+  return ret;
+}
+
+void curl_comm::update_auth_session(CURL *curl_handle, const app_config &config,
+                                    const std::string &session) {
+  const auto cookie_path = utils::path::combine(config.get_data_directory(), {session + ".txt"});
+  curl_easy_setopt(curl_handle, CURLOPT_COOKIEFILE, cookie_path.c_str());
+  curl_easy_setopt(curl_handle, CURLOPT_COOKIEJAR, cookie_path.c_str());
+}
+
+std::string curl_comm::url_encode(CURL *curl_handle, const std::string &data,
+                                  const bool &allow_slash) {
+  auto *value = curl_easy_escape(curl_handle, data.c_str(), 0);
+  std::string ret = value;
+  curl_free(value);
+
+  if (allow_slash) {
+    utils::string::replace(ret, "%2F", "/");
+  }
+  return ret;
+}
+
+curl_comm::curl_read_callback curl_comm::read_data_callback_ =
+    static_cast<curl_comm::curl_read_callback>(
+        [](char *buffer, size_t size, size_t nitems, void *instream) -> size_t {
+          auto *rd = reinterpret_cast<read_data *>(instream);
+          std::size_t bytes_read{};
+          const auto ret = rd->nf->read_bytes(buffer, size * nitems, rd->offset, bytes_read);
+          if (ret) {
+            rd->offset += bytes_read;
+          }
+          return ret && not *rd->stop_requested ? bytes_read : CURL_READFUNC_ABORT;
+        });
+
+curl_comm::curl_write_callback curl_comm::write_data_callback_ =
+    static_cast<curl_comm::curl_write_callback>(
+        [](char *buffer, size_t size, size_t nitems, void *outstream) -> size_t {
+          auto *wd = reinterpret_cast<raw_write_data *>(outstream);
+          std::copy(buffer, buffer + (size * nitems), std::back_inserter(*wd->buffer));
+          return wd->stop_requested ? 0 : size * nitems;
+        });
+
+curl_comm::curl_write_callback curl_comm::write_header_callback_ =
+    static_cast<curl_comm::curl_write_callback>(
+        [](char *buffer, size_t size, size_t nitems, void *outstream) -> size_t {
+          auto &headers = *reinterpret_cast<http_headers *>(outstream);
+          const auto header = std::string(buffer, size * nitems);
+          const auto parts = utils::string::split(header, ':');
+          if (parts.size() > 1u) {
+            auto data = header.substr(parts[0u].size() + 1u);
+            utils::string::left_trim(data);
+            utils::string::right_trim(data, '\r');
+            utils::string::right_trim(data, '\n');
+            utils::string::right_trim(data, '\r');
+            headers[utils::string::to_lower(parts[0u])] = data;
+          }
+          return size * nitems;
+        });
+
+curl_comm::curl_write_callback curl_comm::write_null_callback_ =
+    static_cast<curl_comm::curl_write_callback>(
+        [](char *, size_t size, size_t nitems, void *) -> size_t { return size * nitems; });
+
+curl_comm::curl_write_callback curl_comm::write_string_callback_ =
+    static_cast<curl_comm::curl_write_callback>(
+        [](char *buffer, size_t size, size_t nitems, void *outstream) -> size_t {
+          (*reinterpret_cast<std::string *>(outstream)) += std::string(buffer, size * nitems);
+          return size * nitems;
+        });
+} // namespace repertory

src/comm/curl/curl_resolver.cpp

@@ -0,0 +1,43 @@
+/*
+  Copyright <2018-2022> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
+  associated documentation files (the "Software"), to deal in the Software without restriction,
+  including without limitation the rights to use, copy, modify, merge, publish, distribute,
+  sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all copies or
+  substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
+  NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
+  OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/
+#include "comm/curl/curl_resolver.hpp"
+
+namespace repertory {
+curl_resolver::curl_resolver(CURL *handle, std::vector<std::string> items, const bool &ignore_root)
+    : items_(std::move(items)) {
+#ifndef _WIN32
+  if (ignore_root && (getuid() == 0)) {
+    items_.clear();
+  }
+#endif
+  for (const auto &item : items_) {
+    host_list_ = curl_slist_append(host_list_, &item[0u]);
+  }
+
+  if (host_list_) {
+    curl_easy_setopt(handle, CURLOPT_RESOLVE, host_list_);
+  }
+}
+
+curl_resolver::~curl_resolver() {
+  if (host_list_) {
+    curl_slist_free_all(host_list_);
+  }
+}
+} // namespace repertory

src/comm/curl/multi_request.cpp

@@ -0,0 +1,58 @@
+/*
+  Copyright <2018-2022> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
+  associated documentation files (the "Software"), to deal in the Software without restriction,
+  including without limitation the rights to use, copy, modify, merge, publish, distribute,
+  sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all copies or
+  substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
+  NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
+  OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/
+#include "comm/curl/multi_request.hpp"
+#include "utils/utils.hpp"
+
+namespace repertory {
+multi_request::multi_request(CURL *curl_handle, const bool &stop_requested)
+    : curl_handle_(curl_handle), stop_requested_(stop_requested), multi_handle_(curl_multi_init()) {
+  curl_multi_add_handle(multi_handle_, curl_handle);
+}
+
+multi_request::~multi_request() {
+  curl_multi_remove_handle(multi_handle_, curl_handle_);
+  curl_easy_cleanup(curl_handle_);
+  curl_multi_cleanup(multi_handle_);
+}
+
+void multi_request::get_result(CURLcode &curl_code, long &http_code) {
+  curl_code = CURLcode::CURLE_ABORTED_BY_CALLBACK;
+  http_code = -1;
+
+  auto error = false;
+  int running_handles = 0;
+  curl_multi_perform(multi_handle_, &running_handles);
+  while (not error && (running_handles > 0) && not stop_requested_) {
+    int ignored;
+    curl_multi_wait(multi_handle_, nullptr, 0, 100, &ignored);
+
+    const auto ret = curl_multi_perform(multi_handle_, &running_handles);
+    error = (ret != CURLM_CALL_MULTI_PERFORM) && (ret != CURLM_OK);
+  }
+
+  if (not stop_requested_) {
+    int remaining_messages = 0;
+    auto *multi_result = curl_multi_info_read(multi_handle_, &remaining_messages);
+    if (multi_result && (multi_result->msg == CURLMSG_DONE)) {
+      curl_easy_getinfo(multi_result->easy_handle, CURLINFO_RESPONSE_CODE, &http_code);
+      curl_code = multi_result->data.result;
+    }
+  }
+}
+} // namespace repertory

src/comm/curl/session_manager.cpp

@@ -0,0 +1,73 @@
+/*
+  Copyright <2018-2022> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
+  associated documentation files (the "Software"), to deal in the Software without restriction,
+  including without limitation the rights to use, copy, modify, merge, publish, distribute,
+  sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all copies or
+  substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
+  NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
+  OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/
+#include "comm/curl/session_manager.hpp"
+#include "app_config.hpp"
+#include "comm/curl/curl_comm.hpp"
+#include "utils/utils.hpp"
+
+namespace repertory {
+bool session_manager::create_auth_session(CURL *&curl_handle, const app_config &config,
+                                          host_config hc, std::string &session) {
+  auto ret = true;
+  if (not curl_handle) {
+    curl_handle = utils::create_curl();
+  }
+
+  if (not hc.auth_url.empty() && not hc.auth_user.empty()) {
+    mutex_lock l(session_mutex_);
+    if (session_.empty()) {
+      if ((ret = curl_comm::create_auth_session(curl_handle, config, hc, session))) {
+        session_ = session;
+      }
+    } else {
+      session = session_;
+      curl_comm::update_auth_session(curl_handle, config, session_);
+    }
+
+    if (ret) {
+      session_count_++;
+    }
+  }
+
+  return ret;
+}
+
+void session_manager::release_auth_session(const app_config &config, host_config hc,
+                                           const std::string &session) {
+  if (not hc.auth_url.empty() && not hc.auth_user.empty()) {
+    mutex_lock l(session_mutex_);
+    if (not session_.empty() && (session == session_) && session_count_) {
+      if (not --session_count_) {
+        curl_comm::release_auth_session(config, hc, session_);
+        session_.clear();
+      }
+    }
+  }
+}
+
+void session_manager::update_auth_session(CURL *curl_handle, const app_config &config,
+                                          const host_config &hc) {
+  mutex_lock l(session_mutex_);
+  if (hc.auth_url.empty() || hc.auth_user.empty() || session_.empty()) {
+    return;
+  }
+
+  curl_comm::update_auth_session(curl_handle, config, session_);
+}
+} // namespace repertory

src/comm/packet/client_pool.cpp

@@ -0,0 +1,150 @@
+/*
+  Copyright <2018-2022> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
+  associated documentation files (the "Software"), to deal in the Software without restriction,
+  including without limitation the rights to use, copy, modify, merge, publish, distribute,
+  sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all copies or
+  substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
+  NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
+  OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/
+#include "comm/packet/client_pool.hpp"
+#include "events/events.hpp"
+#include "events/event_system.hpp"
+
+namespace repertory {
+void client_pool::pool::execute(const std::uint64_t &thread_id, const worker_callback &worker,
+                                const worker_complete_callback &worker_complete) {
+  const auto index = static_cast<std::size_t>(thread_id % pool_queues_.size());
+  auto wi = std::make_shared<work_item>(worker, worker_complete);
+  auto &pool_queue = pool_queues_[index];
+
+  unique_mutex_lock queue_lock(pool_queue->mutex);
+  pool_queue->queue.emplace_back(wi);
+  pool_queue->notify.notify_all();
+  queue_lock.unlock();
+}
+
+client_pool::pool::pool(const std::uint8_t &pool_size) {
+  thread_index_ = 0u;
+  for (std::uint8_t i = 0u; i < pool_size; i++) {
+    pool_queues_.emplace_back(std::make_unique<work_queue>());
+  }
+
+  for (std::size_t i = 0u; i < pool_queues_.size(); i++) {
+    pool_threads_.emplace_back(std::thread([this]() {
+      const auto thread_index = thread_index_++;
+
+      auto &pool_queue = pool_queues_[thread_index];
+      auto &queue = pool_queue->queue;
+      auto &queue_mutex = pool_queue->mutex;
+      auto &queue_notify = pool_queue->notify;
+
+      unique_mutex_lock queue_lock(queue_mutex);
+      queue_notify.notify_all();
+      queue_lock.unlock();
+      while (not shutdown_) {
+        queue_lock.lock();
+        if (queue.empty()) {
+          queue_notify.wait(queue_lock);
+        }
+
+        while (not queue.empty()) {
+          auto workItem = queue.front();
+          queue.pop_front();
+          queue_notify.notify_all();
+          queue_lock.unlock();
+
+          try {
+            const auto result = workItem->work();
+            workItem->work_complete(result);
+          } catch (const std::exception &e) {
+            workItem->work_complete(utils::translate_api_error(api_error::error));
+            event_system::instance().raise<repertory_exception>(__FUNCTION__,
+                                                              e.what() ? e.what() : "unknown");
+          }
+
+          queue_lock.lock();
+        }
+
+        queue_notify.notify_all();
+        queue_lock.unlock();
+      }
+
+      queue_lock.lock();
+      while (not queue.empty()) {
+        auto wi = queue.front();
+        queue.pop_front();
+        queue_notify.notify_all();
+        queue_lock.unlock();
+
+        wi->work_complete(utils::translate_api_error(api_error::download_stopped));
+
+        queue_lock.lock();
+      }
+
+      queue_notify.notify_all();
+      queue_lock.unlock();
+    }));
+  }
+}
+
+void client_pool::pool::shutdown() {
+  shutdown_ = true;
+
+  for (auto &pool_queue : pool_queues_) {
+    unique_mutex_lock l(pool_queue->mutex);
+    pool_queue->notify.notify_all();
+  }
+
+  for (auto &thread : pool_threads_) {
+    thread.join();
+  }
+
+  pool_queues_.clear();
+  pool_threads_.clear();
+}
+
+void client_pool::execute(const std::string &client_id, const std::uint64_t &thread_id,
+                          const worker_callback &worker,
+                          const worker_complete_callback &worker_complete) {
+  unique_mutex_lock pool_lock(pool_mutex_);
+  if (shutdown_) {
+    pool_lock.unlock();
+    throw std::runtime_error("Client pool is shutdown");
+  }
+
+  if (not pool_lookup_[client_id]) {
+    pool_lookup_[client_id] = std::make_shared<pool>(pool_size_);
+  }
+
+  pool_lookup_[client_id]->execute(thread_id, worker, worker_complete);
+  pool_lock.unlock();
+}
+
+void client_pool::remove_client(const std::string &client_id) {
+  mutex_lock pool_lock(pool_mutex_);
+  pool_lookup_.erase(client_id);
+}
+
+void client_pool::shutdown() {
+  unique_mutex_lock pool_lock(pool_mutex_);
+  if (not shutdown_) {
+    shutdown_ = true;
+    event_system::instance().raise<service_shutdown>("client_pool");
+    for (auto &kv : pool_lookup_) {
+      kv.second->shutdown();
+    }
+    pool_lookup_.clear();
+  }
+  pool_lock.unlock();
+}
+} // namespace repertory

src/comm/packet/packet.cpp

@@ -0,0 +1,555 @@
+/*
+  Copyright <2018-2022> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
+  associated documentation files (the "Software"), to deal in the Software without restriction,
+  including without limitation the rights to use, copy, modify, merge, publish, distribute,
+  sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all copies or
+  substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
+  NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
+  OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/
+#include "comm/packet/packet.hpp"
+#include "events/events.hpp"
+#include "events/event_system.hpp"
+#include "types/remote.hpp"
+#include "types/repertory.hpp"
+#include "utils/encryption.hpp"
+#include "utils/utils.hpp"
+
+namespace repertory {
+void packet::clear() {
+  buffer_.clear();
+  decode_offset_ = 0u;
+}
+
+packet::error_type packet::decode(std::string &data) {
+  const auto *str = &buffer_[decode_offset_];
+  const auto length = strnlen(str, buffer_.size() - decode_offset_);
+  data = std::string(str, length);
+  decode_offset_ += (length + 1);
+
+  return utils::translate_api_error(api_error::success);
+}
+
+packet::error_type packet::decode(std::wstring &data) {
+  std::string utf8_string;
+  const auto ret = decode(utf8_string);
+  if (ret == 0) {
+    data = utils::string::from_utf8(utf8_string);
+  }
+
+  return utils::translate_api_error(api_error::success);
+}
+
+packet::error_type packet::decode(void *&ptr) {
+  return decode(reinterpret_cast<std::uint64_t &>(ptr));
+}
+
+packet::error_type packet::decode(void *buffer, const size_t &size) {
+  if (size) {
+    const auto read_size = utils::calculate_read_size(buffer_.size(), size, decode_offset_);
+    if (read_size == size) {
+      memcpy(buffer, &buffer_[decode_offset_], size);
+      decode_offset_ += size;
+      return utils::translate_api_error(api_error::success);
+    }
+
+    return ((decode_offset_ + size) > buffer_.size())
+               ? utils::translate_api_error(api_error::buffer_overflow)
+               : utils::translate_api_error(api_error::buffer_too_small);
+  }
+  return utils::translate_api_error(api_error::success);
+}
+
+packet::error_type packet::decode(std::int8_t &i) {
+  const auto ret = decode(&i, sizeof(i));
+  if (ret == 0) {
+    boost::endian::big_to_native_inplace(i);
+  }
+  return ret;
+}
+
+packet::error_type packet::decode(std::uint8_t &i) {
+  const auto ret = decode(&i, sizeof(i));
+  if (ret == 0) {
+    boost::endian::big_to_native_inplace(i);
+  }
+  return ret;
+}
+
+packet::error_type packet::decode(std::int16_t &i) {
+  const auto ret = decode(&i, sizeof(i));
+  if (ret == 0) {
+    boost::endian::big_to_native_inplace(i);
+  }
+  return ret;
+}
+
+packet::error_type packet::decode(std::uint16_t &i) {
+  const auto ret = decode(&i, sizeof(i));
+  if (ret == 0) {
+    boost::endian::big_to_native_inplace(i);
+  }
+  return ret;
+}
+
+packet::error_type packet::decode(std::int32_t &i) {
+  const auto ret = decode(&i, sizeof(i));
+  if (ret == 0) {
+    boost::endian::big_to_native_inplace(i);
+  }
+  return ret;
+}
+
+packet::error_type packet::decode(std::uint32_t &i) {
+  const auto ret = decode(&i, sizeof(i));
+  if (ret == 0) {
+    boost::endian::big_to_native_inplace(i);
+  }
+  return ret;
+}
+
+packet::error_type packet::decode(std::int64_t &i) {
+  const auto ret = decode(&i, sizeof(i));
+  if (ret == 0) {
+    boost::endian::big_to_native_inplace(i);
+  }
+  return ret;
+}
+
+packet::error_type packet::decode(std::uint64_t &i) {
+  const auto ret = decode(&i, sizeof(i));
+  if (ret == 0) {
+    boost::endian::big_to_native_inplace(i);
+  }
+  return ret;
+}
+
+packet::error_type packet::decode(remote::setattr_x &i) {
+  const auto ret = decode(&i, sizeof(i));
+  if (ret == 0) {
+    boost::endian::big_to_native_inplace(i.acctime);
+    boost::endian::big_to_native_inplace(i.bkuptime);
+    boost::endian::big_to_native_inplace(i.chgtime);
+    boost::endian::big_to_native_inplace(i.crtime);
+    boost::endian::big_to_native_inplace(i.flags);
+    boost::endian::big_to_native_inplace(i.gid);
+    boost::endian::big_to_native_inplace(i.mode);
+    boost::endian::big_to_native_inplace(i.modtime);
+    boost::endian::big_to_native_inplace(i.size);
+    boost::endian::big_to_native_inplace(i.uid);
+    boost::endian::big_to_native_inplace(i.valid);
+  }
+
+  return ret;
+}
+
+packet::error_type packet::decode(remote::stat &i) {
+  const auto ret = decode(&i, sizeof(i));
+  if (ret == 0) {
+    boost::endian::big_to_native_inplace(i.st_mode);
+    boost::endian::big_to_native_inplace(i.st_nlink);
+    boost::endian::big_to_native_inplace(i.st_uid);
+    boost::endian::big_to_native_inplace(i.st_gid);
+    boost::endian::big_to_native_inplace(i.st_atimespec);
+    boost::endian::big_to_native_inplace(i.st_mtimespec);
+    boost::endian::big_to_native_inplace(i.st_ctimespec);
+    boost::endian::big_to_native_inplace(i.st_birthtimespec);
+    boost::endian::big_to_native_inplace(i.st_size);
+    boost::endian::big_to_native_inplace(i.st_blocks);
+    boost::endian::big_to_native_inplace(i.st_blksize);
+    boost::endian::big_to_native_inplace(i.st_flags);
+  }
+
+  return ret;
+}
+
+packet::error_type packet::decode(remote::statfs &i) {
+  const auto ret = decode(&i, sizeof(i));
+  if (ret == 0) {
+    boost::endian::big_to_native_inplace(i.f_bavail);
+    boost::endian::big_to_native_inplace(i.f_bfree);
+    boost::endian::big_to_native_inplace(i.f_blocks);
+    boost::endian::big_to_native_inplace(i.f_favail);
+    boost::endian::big_to_native_inplace(i.f_ffree);
+    boost::endian::big_to_native_inplace(i.f_files);
+  }
+  return ret;
+}
+
+packet::error_type packet::decode(remote::statfs_x &i) {
+  auto ret = decode(*dynamic_cast<remote::statfs *>(&i));
+  if (ret == 0) {
+    ret = decode(&i.f_mntfromname[0], 1024);
+  }
+  return ret;
+}
+
+packet::error_type packet::decode(remote::file_info &i) {
+  const auto ret = decode(&i, sizeof(i));
+  if (ret == 0) {
+    boost::endian::big_to_native_inplace(i.AllocationSize);
+    boost::endian::big_to_native_inplace(i.ChangeTime);
+    boost::endian::big_to_native_inplace(i.CreationTime);
+    boost::endian::big_to_native_inplace(i.EaSize);
+    boost::endian::big_to_native_inplace(i.FileAttributes);
+    boost::endian::big_to_native_inplace(i.FileSize);
+    boost::endian::big_to_native_inplace(i.HardLinks);
+    boost::endian::big_to_native_inplace(i.IndexNumber);
+    boost::endian::big_to_native_inplace(i.LastAccessTime);
+    boost::endian::big_to_native_inplace(i.LastWriteTime);
+    boost::endian::big_to_native_inplace(i.ReparseTag);
+  }
+
+  return ret;
+}
+
+int packet::decode_json(packet &response, json &json_data) {
+  int ret = 0;
+  std::string data;
+  if ((ret = response.decode(data)) == 0) {
+    try {
+      json_data = json::parse(data);
+    } catch (const std::exception &e) {
+      event_system::instance().raise<repertory_exception>(
+          __FUNCTION__, e.what() ? e.what() : "Failed to parse JSON string");
+      ret = -EIO;
+    }
+  }
+
+  return ret;
+}
+
+packet::error_type packet::decrypt(const std::string &token) {
+  auto ret = utils::translate_api_error(api_error::success);
+  try {
+    std::vector<char> result;
+    if (not utils::encryption::decrypt_data(token, &buffer_[decode_offset_],
+                                            buffer_.size() - decode_offset_, result)) {
+      throw std::runtime_error("Decryption failed");
+    }
+    buffer_ = std::move(result);
+    decode_offset_ = 0;
+  } catch (const std::exception &e) {
+    event_system::instance().raise<repertory_exception>(__FUNCTION__, e.what());
+    ret = utils::translate_api_error(api_error::error);
+  }
+
+  return ret;
+}
+
+void packet::encode(const void *buffer, const std::size_t &size, bool should_reserve) {
+  if (size) {
+    if (should_reserve) {
+      buffer_.reserve(buffer_.size() + size);
+    }
+    const auto *char_buffer = reinterpret_cast<const char *>(buffer);
+    buffer_.insert(buffer_.end(), char_buffer, char_buffer + size);
+  }
+}
+
+void packet::encode(const std::string &str) {
+  const auto len = strnlen(&str[0], str.size());
+  buffer_.reserve(len + 1 + buffer_.size());
+  encode(&str[0], len, false);
+  buffer_.emplace_back(0);
+}
+
+void packet::encode(wchar_t *str) { encode(utils::string::to_utf8(str ? str : L"")); }
+
+void packet::encode(const wchar_t *str) { encode(utils::string::to_utf8(str ? str : L"")); }
+
+void packet::encode(const std::wstring &str) { encode(utils::string::to_utf8(str)); }
+
+void packet::encode(std::int8_t i) {
+  boost::endian::native_to_big_inplace(i);
+  encode(&i, sizeof(i), true);
+}
+
+void packet::encode(std::uint8_t i) {
+  boost::endian::native_to_big_inplace(i);
+  encode(&i, sizeof(i), true);
+}
+
+void packet::encode(std::int16_t i) {
+  boost::endian::native_to_big_inplace(i);
+  encode(&i, sizeof(i), true);
+}
+
+void packet::encode(std::uint16_t i) {
+  boost::endian::native_to_big_inplace(i);
+  encode(&i, sizeof(i), true);
+}
+
+void packet::encode(std::int32_t i) {
+  boost::endian::native_to_big_inplace(i);
+  encode(&i, sizeof(i), true);
+}
+
+void packet::encode(std::uint32_t i) {
+  boost::endian::native_to_big_inplace(i);
+  encode(&i, sizeof(i), true);
+}
+
+void packet::encode(std::int64_t i) {
+  boost::endian::native_to_big_inplace(i);
+  encode(&i, sizeof(i), true);
+}
+
+void packet::encode(std::uint64_t i) {
+  boost::endian::native_to_big_inplace(i);
+  encode(&i, sizeof(i), true);
+}
+
+void packet::encode(remote::setattr_x i) {
+  boost::endian::native_to_big_inplace(i.acctime);
+  boost::endian::native_to_big_inplace(i.bkuptime);
+  boost::endian::native_to_big_inplace(i.chgtime);
+  boost::endian::native_to_big_inplace(i.crtime);
+  boost::endian::native_to_big_inplace(i.flags);
+  boost::endian::native_to_big_inplace(i.gid);
+  boost::endian::native_to_big_inplace(i.mode);
+  boost::endian::native_to_big_inplace(i.modtime);
+  boost::endian::native_to_big_inplace(i.size);
+  boost::endian::native_to_big_inplace(i.uid);
+  boost::endian::native_to_big_inplace(i.valid);
+
+  encode(&i, sizeof(i), true);
+}
+
+void packet::encode(remote::stat i) {
+  boost::endian::native_to_big_inplace(i.st_mode);
+  boost::endian::native_to_big_inplace(i.st_nlink);
+  boost::endian::native_to_big_inplace(i.st_uid);
+  boost::endian::native_to_big_inplace(i.st_gid);
+  boost::endian::native_to_big_inplace(i.st_atimespec);
+  boost::endian::native_to_big_inplace(i.st_mtimespec);
+  boost::endian::native_to_big_inplace(i.st_ctimespec);
+  boost::endian::native_to_big_inplace(i.st_birthtimespec);
+  boost::endian::native_to_big_inplace(i.st_size);
+  boost::endian::native_to_big_inplace(i.st_blocks);
+  boost::endian::native_to_big_inplace(i.st_blksize);
+  boost::endian::native_to_big_inplace(i.st_flags);
+
+  encode(&i, sizeof(i), true);
+}
+
+void packet::encode(remote::statfs i, bool should_reserve) {
+  boost::endian::native_to_big_inplace(i.f_bavail);
+  boost::endian::native_to_big_inplace(i.f_bfree);
+  boost::endian::native_to_big_inplace(i.f_blocks);
+  boost::endian::native_to_big_inplace(i.f_favail);
+  boost::endian::native_to_big_inplace(i.f_ffree);
+  boost::endian::native_to_big_inplace(i.f_files);
+
+  encode(&i, sizeof(remote::statfs), should_reserve);
+}
+
+void packet::encode(remote::statfs_x i) {
+  buffer_.reserve(buffer_.size() + sizeof(remote::statfs) + 1024);
+  encode(*dynamic_cast<remote::statfs *>(&i), false);
+  encode(&i.f_mntfromname[0], 1024, false);
+}
+
+void packet::encode(remote::file_info i) {
+  boost::endian::native_to_big_inplace(i.FileAttributes);
+  boost::endian::native_to_big_inplace(i.ReparseTag);
+  boost::endian::native_to_big_inplace(i.AllocationSize);
+  boost::endian::native_to_big_inplace(i.FileSize);
+  boost::endian::native_to_big_inplace(i.CreationTime);
+  boost::endian::native_to_big_inplace(i.LastAccessTime);
+  boost::endian::native_to_big_inplace(i.LastWriteTime);
+  boost::endian::native_to_big_inplace(i.ChangeTime);
+  boost::endian::native_to_big_inplace(i.IndexNumber);
+  boost::endian::native_to_big_inplace(i.HardLinks);
+  boost::endian::native_to_big_inplace(i.EaSize);
+
+  encode(&i, sizeof(i), true);
+}
+
+void packet::encode_top(const void *buffer, const std::size_t &size, bool should_reserve) {
+  if (size) {
+    if (should_reserve) {
+      buffer_.reserve(buffer_.size() + size);
+    }
+    const auto *char_buffer = reinterpret_cast<const char *>(buffer);
+    buffer_.insert(buffer_.begin(), char_buffer, char_buffer + size);
+  }
+}
+
+void packet::encode_top(const std::string &str) {
+  const auto len = strnlen(&str[0], str.size());
+  buffer_.reserve(len + 1 + buffer_.size());
+  encode_top(&str[0], len, false);
+  buffer_.insert(buffer_.begin() + len, 0);
+}
+
+void packet::encode_top(const std::wstring &str) { encode_top(utils::string::to_utf8(str)); }
+
+void packet::encode_top(std::int8_t i) {
+  boost::endian::native_to_big_inplace(i);
+  encode_top(&i, sizeof(i), true);
+}
+
+void packet::encode_top(std::uint8_t i) {
+  boost::endian::native_to_big_inplace(i);
+  encode_top(&i, sizeof(i), true);
+}
+
+void packet::encode_top(std::int16_t i) {
+  boost::endian::native_to_big_inplace(i);
+  encode_top(&i, sizeof(i), true);
+}
+
+void packet::encode_top(std::uint16_t i) {
+  boost::endian::native_to_big_inplace(i);
+  encode_top(&i, sizeof(i), true);
+}
+
+void packet::encode_top(std::int32_t i) {
+  boost::endian::native_to_big_inplace(i);
+  encode_top(&i, sizeof(i), true);
+}
+
+void packet::encode_top(std::uint32_t i) {
+  boost::endian::native_to_big_inplace(i);
+  encode_top(&i, sizeof(i), true);
+}
+
+void packet::encode_top(std::int64_t i) {
+  boost::endian::native_to_big_inplace(i);
+  encode_top(&i, sizeof(i), true);
+}
+
+void packet::encode_top(std::uint64_t i) {
+  boost::endian::native_to_big_inplace(i);
+  encode_top(&i, sizeof(i), true);
+}
+
+void packet::encode_top(remote::setattr_x i) {
+  boost::endian::native_to_big_inplace(i.acctime);
+  boost::endian::native_to_big_inplace(i.bkuptime);
+  boost::endian::native_to_big_inplace(i.chgtime);
+  boost::endian::native_to_big_inplace(i.crtime);
+  boost::endian::native_to_big_inplace(i.flags);
+  boost::endian::native_to_big_inplace(i.gid);
+  boost::endian::native_to_big_inplace(i.mode);
+  boost::endian::native_to_big_inplace(i.modtime);
+  boost::endian::native_to_big_inplace(i.size);
+  boost::endian::native_to_big_inplace(i.uid);
+  boost::endian::native_to_big_inplace(i.valid);
+
+  encode_top(&i, sizeof(i), true);
+}
+
+void packet::encode_top(remote::stat i) {
+  boost::endian::native_to_big_inplace(i.st_mode);
+  boost::endian::native_to_big_inplace(i.st_nlink);
+  boost::endian::native_to_big_inplace(i.st_uid);
+  boost::endian::native_to_big_inplace(i.st_gid);
+  boost::endian::native_to_big_inplace(i.st_atimespec);
+  boost::endian::native_to_big_inplace(i.st_mtimespec);
+  boost::endian::native_to_big_inplace(i.st_ctimespec);
+  boost::endian::native_to_big_inplace(i.st_birthtimespec);
+  boost::endian::native_to_big_inplace(i.st_size);
+  boost::endian::native_to_big_inplace(i.st_blocks);
+  boost::endian::native_to_big_inplace(i.st_blksize);
+  boost::endian::native_to_big_inplace(i.st_flags);
+
+  encode_top(&i, sizeof(i), true);
+}
+
+void packet::encode_top(remote::statfs i, bool should_reserve) {
+  boost::endian::native_to_big_inplace(i.f_bavail);
+  boost::endian::native_to_big_inplace(i.f_bfree);
+  boost::endian::native_to_big_inplace(i.f_blocks);
+  boost::endian::native_to_big_inplace(i.f_favail);
+  boost::endian::native_to_big_inplace(i.f_ffree);
+  boost::endian::native_to_big_inplace(i.f_files);
+
+  encode_top(&i, sizeof(remote::statfs), should_reserve);
+}
+
+void packet::encode_top(remote::statfs_x i) {
+  buffer_.reserve(buffer_.size() + sizeof(remote::statfs) + 1024);
+  encode_top(&i.f_mntfromname[0], 1024, false);
+  encode_top(*dynamic_cast<remote::statfs *>(&i), false);
+}
+
+void packet::encode_top(remote::file_info i) {
+  boost::endian::native_to_big_inplace(i.FileAttributes);
+  boost::endian::native_to_big_inplace(i.ReparseTag);
+  boost::endian::native_to_big_inplace(i.AllocationSize);
+  boost::endian::native_to_big_inplace(i.FileSize);
+  boost::endian::native_to_big_inplace(i.CreationTime);
+  boost::endian::native_to_big_inplace(i.LastAccessTime);
+  boost::endian::native_to_big_inplace(i.LastWriteTime);
+  boost::endian::native_to_big_inplace(i.ChangeTime);
+  boost::endian::native_to_big_inplace(i.IndexNumber);
+  boost::endian::native_to_big_inplace(i.HardLinks);
+  boost::endian::native_to_big_inplace(i.EaSize);
+
+  encode_top(&i, sizeof(i), true);
+}
+
+void packet::encrypt(const std::string &token) {
+  try {
+    std::vector<char> result;
+    utils::encryption::encrypt_data(token, buffer_, result);
+    buffer_ = std::move(result);
+    encode_top(static_cast<std::uint32_t>(buffer_.size()));
+  } catch (const std::exception &e) {
+    event_system::instance().raise<repertory_exception>(__FUNCTION__, e.what());
+  }
+}
+
+void packet::transfer_into(std::vector<char> &buffer) {
+  buffer = std::move(buffer_);
+  buffer_ = std::vector<char>();
+  decode_offset_ = 0;
+}
+
+packet &packet::operator=(const std::vector<char> &buffer) noexcept {
+  if (&buffer_ != &buffer) {
+    buffer_ = buffer;
+    decode_offset_ = 0;
+  }
+
+  return *this;
+}
+
+packet &packet::operator=(std::vector<char> &&buffer) noexcept {
+  if (&buffer_ != &buffer) {
+    buffer_ = std::move(buffer);
+    decode_offset_ = 0;
+  }
+
+  return *this;
+}
+
+packet &packet::operator=(const packet &p) noexcept {
+  if (this != &p) {
+    buffer_ = p.buffer_;
+    decode_offset_ = p.decode_offset_;
+  }
+
+  return *this;
+}
+
+packet &packet::operator=(packet &&p) noexcept {
+  if (this != &p) {
+    buffer_ = std::move(p.buffer_);
+    decode_offset_ = p.decode_offset_;
+  }
+
+  return *this;
+}
+} // namespace repertory

src/comm/packet/packet_client.cpp

@@ -0,0 +1,227 @@
+/*
+  Copyright <2018-2022> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
+  associated documentation files (the "Software"), to deal in the Software without restriction,
+  including without limitation the rights to use, copy, modify, merge, publish, distribute,
+  sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all copies or
+  substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
+  NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
+  OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/
+#include "comm/packet/packet_client.hpp"
+#include "events/events.hpp"
+#include "types/repertory.hpp"
+#include "utils/timeout.hpp"
+
+namespace repertory {
+// clang-format off
+E_SIMPLE2(packet_client_timeout, error, true, 
+  std::string, event_name, en, E_STRING, 
+  std::string, message, msg, E_STRING
+);
+// clang-format on
+
+packet_client::packet_client(std::string host_name_or_ip, const std::uint8_t &max_connections,
+                             const std::uint16_t &port, const std::uint16_t &receive_timeout,
+                             const std::uint16_t &send_timeout, std::string encryption_token)
+    : io_context_(),
+      host_name_or_ip_(std::move(host_name_or_ip)),
+      max_connections_(max_connections ? max_connections : 20u),
+      port_(port),
+      receive_timeout_(receive_timeout),
+      send_timeout_(send_timeout),
+      encryption_token_(std::move(encryption_token)),
+      unique_id_(utils::create_uuid_string()) {}
+
+packet_client::~packet_client() {
+  allow_connections_ = false;
+  close_all();
+  io_context_.stop();
+}
+
+void packet_client::close(client &client) const {
+  try {
+    boost::system::error_code ec;
+    client.socket.close(ec);
+  } catch (...) {
+  }
+}
+
+void packet_client::close_all() {
+  unique_mutex_lock clients_lock(clients_mutex_);
+  for (auto &c : clients_) {
+    close(*c.get());
+  }
+
+  clients_.clear();
+  unique_id_ = utils::create_uuid_string();
+}
+
+bool packet_client::connect(client &c) {
+  auto ret = false;
+  try {
+    resolve();
+    boost::asio::connect(c.socket, resolve_results_);
+    c.socket.set_option(boost::asio::ip::tcp::no_delay(true));
+    c.socket.set_option(boost::asio::socket_base::linger(false, 0));
+
+    packet response;
+    read_packet(c, response);
+
+    ret = true;
+  } catch (const std::exception &e) {
+    event_system::instance().raise<repertory_exception>(__FUNCTION__, e.what());
+  }
+  return ret;
+}
+
+std::shared_ptr<packet_client::client> packet_client::get_client() {
+  std::shared_ptr<client> ret;
+
+  unique_mutex_lock clients_lock(clients_mutex_);
+  if (allow_connections_) {
+    if (clients_.empty()) {
+      clients_lock.unlock();
+      ret = std::make_shared<client>(io_context_);
+      connect(*ret);
+    } else {
+      ret = clients_[0u];
+      utils::remove_element_from(clients_, ret);
+      clients_lock.unlock();
+    }
+  }
+
+  return ret;
+}
+
+void packet_client::put_client(std::shared_ptr<client> &c) {
+  mutex_lock clientsLock(clients_mutex_);
+  if (clients_.size() < max_connections_) {
+    clients_.emplace_back(c);
+  }
+}
+
+packet::error_type packet_client::read_packet(client &c, packet &response) {
+  std::vector<char> buffer(sizeof(std::uint32_t));
+  const auto read_buffer = [&]() {
+    std::uint32_t offset = 0u;
+    while (offset < buffer.size()) {
+      const auto bytes_read =
+          boost::asio::read(c.socket, boost::asio::buffer(&buffer[offset], buffer.size() - offset));
+      if (bytes_read <= 0) {
+        throw std::runtime_error("Read failed: " + std::to_string(bytes_read));
+      }
+      offset += static_cast<std::uint32_t>(bytes_read);
+    }
+  };
+  read_buffer();
+
+  const auto size = boost::endian::big_to_native(*reinterpret_cast<std::uint32_t *>(&buffer[0u]));
+  buffer.resize(size);
+
+  read_buffer();
+  response = std::move(buffer);
+
+  auto ret = response.decrypt(encryption_token_);
+  if (ret == 0) {
+    ret = response.decode(c.nonce);
+  }
+
+  return ret;
+}
+
+void packet_client::resolve() {
+  if (resolve_results_.empty()) {
+    resolve_results_ =
+        tcp::resolver(io_context_).resolve({host_name_or_ip_, std::to_string(port_)});
+  }
+}
+
+packet::error_type packet_client::send(const std::string &method, std::uint32_t &service_flags) {
+  packet request;
+  return send(method, request, service_flags);
+}
+
+packet::error_type packet_client::send(const std::string &method, packet &request,
+                                       std::uint32_t &service_flags) {
+  packet response;
+  return send(method, request, response, service_flags);
+}
+
+packet::error_type packet_client::send(const std::string &method, packet &request, packet &response,
+                                       std::uint32_t &service_flags) {
+  auto success = false;
+  packet::error_type ret = utils::translate_api_error(api_error::error);
+  request.encode_top(method);
+  request.encode_top(utils::get_thread_id());
+  request.encode_top(unique_id_);
+  request.encode_top(PACKET_SERVICE_FLAGS);
+  request.encode_top(get_repertory_version());
+
+  static const auto max_attempts = 2;
+  for (auto i = 1; allow_connections_ && not success && (i <= max_attempts); i++) {
+    auto c = get_client();
+    if (c) {
+      try {
+        request.encode_top(c->nonce);
+        request.encrypt(encryption_token_);
+
+        timeout request_timeout(
+            [this, method, c]() {
+              event_system::instance().raise<packet_client_timeout>("Request", method);
+              close(*c.get());
+            },
+            std::chrono::seconds(send_timeout_));
+
+        std::uint32_t offset = 0u;
+        while (offset < request.get_size()) {
+          const auto bytes_written = boost::asio::write(
+              c->socket, boost::asio::buffer(&request[offset], request.get_size() - offset));
+          if (bytes_written <= 0) {
+            throw std::runtime_error("Write failed: " + std::to_string(bytes_written));
+          }
+          offset += static_cast<std::uint32_t>(bytes_written);
+        }
+        request_timeout.disable();
+
+        timeout response_timeout(
+            [this, method, c]() {
+              event_system::instance().raise<packet_client_timeout>("Response", method);
+              close(*c.get());
+            },
+            std::chrono::seconds(receive_timeout_));
+
+        ret = read_packet(*c, response);
+        response_timeout.disable();
+        if (ret == 0) {
+          response.decode(service_flags);
+          response.decode(ret);
+          success = true;
+          put_client(c);
+        }
+      } catch (const std::exception &e) {
+        event_system::instance().raise<repertory_exception>(__FUNCTION__, e.what());
+        close_all();
+        if (allow_connections_ && (i < max_attempts)) {
+          std::this_thread::sleep_for(1s);
+        }
+      }
+    }
+
+    if (not allow_connections_) {
+      ret = utils::translate_api_error(api_error::error);
+      success = true;
+    }
+  }
+
+  return CONVERT_STATUS_NOT_IMPLEMENTED(ret);
+}
+} // namespace repertory

src/comm/packet/packet_server.cpp

@@ -0,0 +1,225 @@
+/*
+  Copyright <2018-2022> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
+  associated documentation files (the "Software"), to deal in the Software without restriction,
+  including without limitation the rights to use, copy, modify, merge, publish, distribute,
+  sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all copies or
+  substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
+  NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
+  OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/
+#include "comm/packet/packet_server.hpp"
+#include "events/events.hpp"
+#include "events/event_system.hpp"
+#include "comm/packet/packet.hpp"
+#include "types/repertory.hpp"
+#include "utils/utils.hpp"
+
+namespace repertory {
+packet_server::packet_server(const std::uint16_t &port, std::string token, std::uint8_t pool_size,
+                             closed_callback closed, message_handler_callback message_handler)
+    : encryption_token_(std::move(token)), closed_(closed), message_handler_(message_handler) {
+  initialize(port, pool_size);
+}
+
+packet_server::~packet_server() {
+  event_system::instance().raise<service_shutdown>("packet_server");
+  std::thread([this]() {
+    for (std::size_t i = 0u; i < service_threads_.size(); i++) {
+      io_context_.stop();
+    }
+  }).detach();
+
+  server_thread_->join();
+  server_thread_.reset();
+}
+
+void packet_server::add_client(connection &c, const std::string &client_id) {
+  c.client_id = client_id;
+
+  recur_mutex_lock connection_lock(connection_mutex_);
+  if (connection_lookup_.find(client_id) == connection_lookup_.end()) {
+    connection_lookup_[client_id] = 1u;
+  } else {
+    connection_lookup_[client_id]++;
+  }
+}
+
+void packet_server::initialize(const uint16_t &port, uint8_t pool_size) {
+  pool_size = std::max(uint8_t(1u), pool_size);
+  server_thread_ = std::make_unique<std::thread>([this, port, pool_size]() {
+    tcp::acceptor acceptor(io_context_);
+    try {
+      const auto endpoint = tcp::endpoint(tcp::v4(), port);
+      acceptor.open(endpoint.protocol());
+      acceptor.set_option(socket_base::reuse_address(true));
+      acceptor.bind(endpoint);
+      acceptor.listen();
+    } catch (const std::exception &e) {
+      event_system::instance().raise<repertory_exception>(__FUNCTION__, e.what());
+    }
+    listen_for_connection(acceptor);
+
+    for (std::uint8_t i = 0u; i < pool_size; i++) {
+      service_threads_.emplace_back(std::thread([this]() { io_context_.run(); }));
+    }
+
+    for (auto &th : service_threads_) {
+      th.join();
+    }
+  });
+}
+
+void packet_server::listen_for_connection(tcp::acceptor &acceptor) {
+  auto c = std::make_shared<packet_server::connection>(io_context_, acceptor);
+  acceptor.async_accept(
+      c->socket, boost::bind(&packet_server::on_accept, this, c, boost::asio::placeholders::error));
+}
+
+void packet_server::on_accept(std::shared_ptr<connection> c, boost::system::error_code ec) {
+  listen_for_connection(c->acceptor);
+  if (ec) {
+    event_system::instance().raise<repertory_exception>(__FUNCTION__, ec.message());
+    std::this_thread::sleep_for(1s);
+  } else {
+    c->socket.set_option(boost::asio::ip::tcp::no_delay(true));
+    c->socket.set_option(boost::asio::socket_base::linger(false, 0));
+
+    c->generate_nonce();
+
+    packet response;
+    send_response(c, 0, response);
+  }
+}
+
+void packet_server::read_header(std::shared_ptr<connection> c) {
+  c->buffer.resize(sizeof(std::uint32_t));
+  boost::asio::async_read(c->socket, boost::asio::buffer(&c->buffer[0u], c->buffer.size()),
+                          [this, c](boost::system::error_code ec, std::size_t) {
+                            if (ec) {
+                              remove_client(*c);
+                              event_system::instance().raise<repertory_exception>(__FUNCTION__,
+                                                                                ec.message());
+                            } else {
+                              auto to_read = *reinterpret_cast<std::uint32_t *>(&c->buffer[0u]);
+                              boost::endian::big_to_native_inplace(to_read);
+                              read_packet(c, to_read);
+                            }
+                          });
+}
+
+void packet_server::read_packet(std::shared_ptr<connection> c, const std::uint32_t &data_size) {
+  try {
+    const auto read_buffer = [&]() {
+      std::uint32_t offset = 0u;
+      while (offset < c->buffer.size()) {
+        const auto bytes_read = boost::asio::read(
+            c->socket, boost::asio::buffer(&c->buffer[offset], c->buffer.size() - offset));
+        if (bytes_read <= 0) {
+          throw std::runtime_error("read failed: " + std::to_string(bytes_read));
+        }
+        offset += static_cast<std::uint32_t>(bytes_read);
+      }
+    };
+
+    auto should_send_response = true;
+    auto response = std::make_shared<packet>();
+    c->buffer.resize(data_size);
+    read_buffer();
+
+    packet::error_type ret;
+    auto request = std::make_shared<packet>(c->buffer);
+    if (request->decrypt(encryption_token_) == 0) {
+      std::string nonce;
+      if ((ret = request->decode(nonce)) == 0) {
+        if (nonce != c->nonce) {
+          throw std::runtime_error("invalid nonce");
+        }
+        c->generate_nonce();
+
+        std::string version;
+        if ((ret = request->decode(version)) == 0) {
+          if (utils::compare_version_strings(version, MIN_REMOTE_VERSION) >= 0) {
+            std::uint32_t service_flags = 0u;
+            DECODE_OR_IGNORE(request, service_flags);
+
+            std::string client_id;
+            DECODE_OR_IGNORE(request, client_id);
+
+            std::uint64_t thread_id = 0u;
+            DECODE_OR_IGNORE(request, thread_id);
+
+            std::string method;
+            DECODE_OR_IGNORE(request, method);
+
+            if (ret == 0) {
+              if (c->client_id.empty()) {
+                add_client(*c, client_id);
+              }
+
+              should_send_response = false;
+              message_handler_(service_flags, client_id, thread_id, method, request.get(),
+                               *response,
+                               [this, c, request, response](const packet::error_type &result) {
+                                 this->send_response(c, result, *response);
+                               });
+            }
+          } else {
+            ret = utils::translate_api_error(api_error::incompatible_version);
+          }
+        } else {
+          ret = utils::translate_api_error(api_error::invalid_version);
+        }
+      } else {
+        throw std::runtime_error("invalid nonce");
+      }
+    } else {
+      throw std::runtime_error("decryption failed");
+    }
+    if (should_send_response) {
+      send_response(c, ret, *response);
+    }
+  } catch (const std::exception &e) {
+    remove_client(*c);
+    event_system::instance().raise<repertory_exception>(__FUNCTION__, e.what());
+  }
+}
+
+void packet_server::remove_client(connection &c) {
+  if (not c.client_id.empty()) {
+    recur_mutex_lock connection_lock(connection_mutex_);
+    if (not --connection_lookup_[c.client_id]) {
+      connection_lookup_.erase(c.client_id);
+      closed_(c.client_id);
+    }
+  }
+}
+
+void packet_server::send_response(std::shared_ptr<connection> c, const packet::error_type &result,
+                                  packet &response) {
+  response.encode_top(result);
+  response.encode_top(PACKET_SERVICE_FLAGS);
+  response.encode_top(c->nonce);
+  response.encrypt(encryption_token_);
+  response.transfer_into(c->buffer);
+
+  boost::asio::async_write(c->socket, boost::asio::buffer(c->buffer),
+                           [this, c](boost::system::error_code ec, std::size_t /*length*/) {
+                             if (ec) {
+                               remove_client(*c);
+                               event_system::instance().raise<repertory_exception>(__FUNCTION__,
+                                                                                 ec.message());
+                             } else {
+                               read_header(c);
+                             }
+                           });
+}
+} // namespace repertory