2.0.0-rc (#9)

### Issues

* \#1 \[bug\] Unable to mount S3 due to 'item_not_found' exception
* \#2 Require bucket name for S3 mounts
* \#3 \[bug\] File size is not being updated in S3 mount
* \#4 Upgrade to libfuse-3.x.x
* \#5 Switch to renterd for Sia support
* \#6 Switch to cpp-httplib to further reduce dependencies
* \#7 Remove global_data and calculate used disk space per provider
* \#8 Switch to libcurl for S3 mount support

### Changes from v1.x.x

* Added read-only encrypt provider
  * Pass-through mount point that transparently encrypts source data using `XChaCha20-Poly1305`
* Added S3 encryption support via `XChaCha20-Poly1305`
* Added replay protection to remote mounts
* Added support base64 writes in remote FUSE
* Created static linked Linux binaries for `amd64` and `aarch64` using `musl-libc`
* Removed legacy Sia renter support
* Removed Skynet support
* Fixed multiple remote mount WinFSP API issues on \*NIX servers
* Implemented chunked read and write
  * Writes for non-cached files are performed in chunks of 8Mib
* Removed `repertory-ui` support
* Removed `FreeBSD` support
* Switched to `libsodium` over `CryptoPP`
* Switched to `XChaCha20-Poly1305` for remote mounts
* Updated `GoogleTest` to v1.14.0
* Updated `JSON for Modern C++` to v3.11.2
* Updated `OpenSSL` to v1.1.1w
* Updated `RocksDB` to v8.5.3
* Updated `WinFSP` to 2023
* Updated `boost` to v1.78.0
* Updated `cURL` to v8.3.0
* Updated `zlib` to v1.3
* Use `upload_manager` for all providers
  * Adds a delay to uploads to prevent excessive API calls
  * Supports re-upload after mount restart for incomplete uploads
  * NOTE: Uploads for all providers are full file (no resume support)
    * Multipart upload support is planned for S3

Reviewed-on: #9

src/utils/utils.cpp

@@ -1,27 +1,30 @@
 /*
-  Copyright <2018-2022> <scott.e.graves@protonmail.com>
+  Copyright <2018-2023> <scott.e.graves@protonmail.com>
 
-  Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
-  associated documentation files (the "Software"), to deal in the Software without restriction,
-  including without limitation the rights to use, copy, modify, merge, publish, distribute,
-  sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
   furnished to do so, subject to the following conditions:
 
-  The above copyright notice and this permission notice shall be included in all copies or
-  substantial portions of the Software.
+  The above copyright notice and this permission notice shall be included in all
+  copies or substantial portions of the Software.
 
-  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
-  NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
-  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
-  OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  SOFTWARE.
 */
 #include "utils/utils.hpp"
+
 #include "app_config.hpp"
-#include "events/events.hpp"
 #include "events/event_system.hpp"
+#include "events/events.hpp"
 #include "providers/i_provider.hpp"
-#include "types/skynet.hpp"
 #include "types/startup_exception.hpp"
 #include "utils/com_init_wrapper.hpp"
 #include "utils/native_file.hpp"
@@ -29,18 +32,9 @@
 #include "utils/string_utils.hpp"
 
 namespace repertory::utils {
-hastings api_currency_to_hastings(const api_currency &currency) {
-  ttmath::Parser<api_currency> parser;
-  parser.Parse(currency.ToString() + " * (10 ^ 24)");
-  ttmath::Conv conv;
-  conv.scient_from = 256;
-  conv.base = 10u;
-  conv.round = 0;
-  return parser.stack[0u].value.ToString(conv);
-}
-
-void calculate_allocation_size(const bool &directory, const std::uint64_t &file_size,
-                               UINT64 allocation_size, std::string &allocation_meta_size) {
+void calculate_allocation_size(bool directory, std::uint64_t file_size,
+                               UINT64 allocation_size,
+                               std::string &allocation_meta_size) {
   if (directory) {
     allocation_meta_size = "0";
     return;
@@ -49,20 +43,24 @@ void calculate_allocation_size(const bool &directory, const std::uint64_t &file_
   if (file_size > allocation_size) {
     allocation_size = file_size;
   }
-  allocation_size = ((allocation_size == 0u) ? WINFSP_ALLOCATION_UNIT : allocation_size);
   allocation_size =
-      utils::divide_with_ceiling(allocation_size, WINFSP_ALLOCATION_UNIT) * WINFSP_ALLOCATION_UNIT;
+      ((allocation_size == 0u) ? WINFSP_ALLOCATION_UNIT : allocation_size);
+  allocation_size =
+      utils::divide_with_ceiling(allocation_size, WINFSP_ALLOCATION_UNIT) *
+      WINFSP_ALLOCATION_UNIT;
   allocation_meta_size = std::to_string(allocation_size);
 }
 
-std::size_t calculate_read_size(const uint64_t &total_size, const std::size_t &read_size,
-                                const uint64_t &offset) {
-  return static_cast<std::size_t>(((offset + read_size) > total_size)
-                                      ? ((offset < total_size) ? total_size - offset : 0u)
-                                      : read_size);
+auto calculate_read_size(const uint64_t &total_size, std::size_t read_size,
+                         const uint64_t &offset) -> std::size_t {
+  return static_cast<std::size_t>(
+      ((offset + read_size) > total_size)
+          ? ((offset < total_size) ? total_size - offset : 0u)
+          : read_size);
 }
 
-int compare_version_strings(std::string version1, std::string version2) {
+auto compare_version_strings(std::string version1, std::string version2)
+    -> int {
   if (utils::string::contains(version1, "-")) {
     version1 = utils::string::split(version1, '-')[0u];
   }
@@ -94,44 +92,33 @@ int compare_version_strings(std::string version1, std::string version2) {
   return 0;
 }
 
-std::uint64_t convert_api_date(const std::string &date) {
-  //"2019-02-21T02:24:37.653091916-06:00"
-  const auto parts = utils::string::split(date, '.');
-  const auto dt = parts[0];
-  const auto nanos = utils::string::to_uint64(utils::string::split(parts[1u], '-')[0u]);
+auto convert_api_date(const std::string &date) -> std::uint64_t {
+  // 2009-10-12T17:50:30.000Z
+  const auto date_parts = utils::string::split(date, '.');
+  const auto date_time = date_parts[0U];
+  const auto nanos =
+      utils::string::to_uint64(utils::string::split(date_parts[1U], 'Z')[0U]);
 
   struct tm tm1 {};
 #ifdef _WIN32
-  auto convert_time = [](time_t t) -> std::uint64_t {
-    const auto ll = Int32x32To64(t, 10000000) + 116444736000000000ull;
-    ULARGE_INTEGER ft{};
-    ft.LowPart = static_cast<DWORD>(ll);
-    ft.HighPart = static_cast<DWORD>(ll >> 32);
-    return ft.QuadPart;
-  };
-
-  const auto parts2 = utils::string::split(dt, 'T');
-  const auto date_parts = utils::string::split(parts2[0u], '-');
-  const auto time_parts = utils::string::split(parts2[1u], ':');
-  tm1.tm_year = utils::string::to_int32(date_parts[0u]) - 1900;
-  tm1.tm_mon = utils::string::to_int32(date_parts[1u]) - 1;
-  tm1.tm_mday = utils::string::to_int32(date_parts[2u]);
-  tm1.tm_hour = utils::string::to_uint32(time_parts[0u]);
-  tm1.tm_min = utils::string::to_uint32(time_parts[1u]);
-  tm1.tm_sec = utils::string::to_uint32(time_parts[2u]);
-  tm1.tm_wday = -1;
-  tm1.tm_yday = -1;
-  tm1.tm_isdst = -1;
-  return (nanos / 100) + convert_time(mktime(&tm1));
+  utils::strptime(date_time.c_str(), "%Y-%m-%dT%T", &tm1);
 #else
-  strptime(&dt[0], "%Y-%m-%dT%T", &tm1);
-  return nanos + (mktime(&tm1) * NANOS_PER_SECOND);
+  strptime(date_time.c_str(), "%Y-%m-%dT%T", &tm1);
 #endif
+  return nanos + (mktime(&tm1) * NANOS_PER_SECOND);
 }
 
-CURL *create_curl() { return reset_curl(curl_easy_init()); }
+auto create_curl() -> CURL * {
+  static std::recursive_mutex mtx;
 
-std::string create_uuid_string() {
+  unique_recur_mutex_lock l(mtx);
+  curl_global_init(CURL_GLOBAL_DEFAULT);
+  l.unlock();
+
+  return reset_curl(curl_easy_init());
+}
+
+auto create_uuid_string() -> std::string {
 #ifdef _WIN32
   UUID guid{};
   UuidCreate(&guid);
@@ -161,11 +148,13 @@ std::string create_uuid_string() {
 #endif
 }
 
-std::string create_volume_label(const provider_type &pt) {
+auto create_volume_label(const provider_type &pt) -> std::string {
   return "repertory_" + app_config::get_provider_name(pt);
 }
 
-download_type download_type_from_string(std::string type, const download_type &default_type) {
+auto download_type_from_string(std::string type,
+                               const download_type &default_type)
+    -> download_type {
   type = utils::string::to_lower(utils::string::trim(type));
   if (type == "direct") {
     return download_type::direct;
@@ -178,7 +167,7 @@ download_type download_type_from_string(std::string type, const download_type &d
   return default_type;
 }
 
-std::string download_type_to_string(const download_type &type) {
+auto download_type_to_string(const download_type &type) -> std::string {
   switch (type) {
   case download_type::direct:
     return "direct";
@@ -193,7 +182,7 @@ std::string download_type_to_string(const download_type &type) {
 
 #ifdef _WIN32
 // https://www.frenk.com/2009/12/convert-filetime-to-unix-timestamp/
-remote::file_time filetime_to_unix_time(const FILETIME &ft) {
+auto filetime_to_unix_time(const FILETIME &ft) -> remote::file_time {
   LARGE_INTEGER date{};
   date.HighPart = ft.dwHighDateTime;
   date.LowPart = ft.dwLowDateTime;
@@ -209,7 +198,7 @@ void unix_time_to_filetime(const remote::file_time &ts, FILETIME &ft) {
 }
 #endif
 
-std::string generate_random_string(const std::uint16_t &length) {
+auto generate_random_string(std::uint16_t length) -> std::string {
   srand(static_cast<unsigned int>(get_time_now()));
 
   std::string ret;
@@ -217,13 +206,18 @@ std::string generate_random_string(const std::uint16_t &length) {
   for (std::uint16_t i = 0u; i < length; i++) {
     do {
       ret[i] = static_cast<char>(rand() % 74 + 48);
-    } while (((ret[i] >= 91) && (ret[i] <= 96)) || ((ret[i] >= 58) && (ret[i] <= 64)));
+    } while (((ret[i] >= 91) && (ret[i] <= 96)) ||
+             ((ret[i] >= 58) && (ret[i] <= 64)));
   }
 
   return ret;
 }
 
-std::string get_environment_variable(const std::string &variable) {
+auto get_attributes_from_meta(const api_meta_map &meta) -> DWORD {
+  return static_cast<DWORD>(utils::string::to_uint32(meta.at(META_ATTRIBUTES)));
+}
+
+auto get_environment_variable(const std::string &variable) -> std::string {
 #ifdef _WIN32
   std::string value;
   auto sz = ::GetEnvironmentVariable(&variable[0], nullptr, 0);
@@ -239,7 +233,7 @@ std::string get_environment_variable(const std::string &variable) {
 #endif
 }
 
-std::uint64_t get_file_time_now() {
+auto get_file_time_now() -> std::uint64_t {
 #ifdef _WIN32
   SYSTEMTIME st{};
   ::GetSystemTime(&st);
@@ -254,7 +248,8 @@ std::uint64_t get_file_time_now() {
 void get_local_time_now(struct tm &local_time) {
   memset(&local_time, 0, sizeof(local_time));
 
-  const auto now = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now());
+  const auto now =
+      std::chrono::system_clock::to_time_t(std::chrono::system_clock::now());
 #ifdef _WIN32
   localtime_s(&local_time, &now);
 #else
@@ -265,10 +260,8 @@ void get_local_time_now(struct tm &local_time) {
 #endif
 }
 
-#ifdef _WIN32
-#endif
-
-bool get_next_available_port(std::uint16_t first_port, std::uint16_t &available_port) {
+auto get_next_available_port(std::uint16_t first_port,
+                             std::uint16_t &available_port) -> bool {
   using namespace boost::asio;
   using ip::tcp;
   boost::system::error_code ec;
@@ -285,28 +278,25 @@ bool get_next_available_port(std::uint16_t first_port, std::uint16_t &available_
   return not ec;
 }
 
-std::uint64_t get_time_now() {
+auto get_time_now() -> std::uint64_t {
 #ifdef _WIN32
   return static_cast<std::uint64_t>(
       std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()));
 #else
 #if __APPLE__
-  return std::chrono::nanoseconds(std::chrono::system_clock::now().time_since_epoch()).count();
+  return std::chrono::nanoseconds(
+             std::chrono::system_clock::now().time_since_epoch())
+      .count();
 #else
   return static_cast<std::uint64_t>(
-      std::chrono::nanoseconds(std::chrono::high_resolution_clock::now().time_since_epoch())
+      std::chrono::nanoseconds(
+          std::chrono::high_resolution_clock::now().time_since_epoch())
           .count());
 #endif
 #endif
 }
 
-api_currency hastings_string_to_api_currency(const std::string &amount) {
-  ttmath::Parser<api_currency> parser;
-  parser.Parse(amount + " / (10 ^ 24)");
-  return parser.stack[0].value;
-}
-
-CURL *reset_curl(CURL *curl_handle) {
+auto reset_curl(CURL *curl_handle) -> CURL * {
   curl_easy_reset(curl_handle);
 #if __APPLE__
   curl_easy_setopt(curl_handle, CURLOPT_NOSIGNAL, 1);
@@ -314,56 +304,40 @@ CURL *reset_curl(CURL *curl_handle) {
   return curl_handle;
 }
 
-bool retryable_action(const std::function<bool()> &action) {
+auto retryable_action(const std::function<bool()> &action) -> bool {
   auto succeeded = false;
-  for (auto i = 0; not(succeeded = action()) && (i < 10); i++) {
+  for (std::uint8_t i = 0u; not(succeeded = action()) && (i < 20u); i++) {
     std::this_thread::sleep_for(100ms);
   }
   return succeeded;
 }
 
-/* bool parse_url(const std::string &url, HostConfig &hc) { */
-/*   static const auto pathRegex = std::regex( */
-/*       R"(^((\w+):)?(\/\/((\w+)?(:(\w+))?@)?([^\/\?:]+)(:(\d+))?)?(\/?([^\/\?#][^\?#]*)?)?(\?([^#]+))?(#(\w*))?$)");
- */
-/*  */
-/*   std::smatch results; */
-/*   if (std::regex_search(url, results, pathRegex)) { */
-/*     hc.HostNameOrIp = results[8u].str(); */
-/*     hc.Path = utils::path::create_api_path(results[11u].str()); */
-/*     hc.Protocol = utils::string::to_lower(results[2u].str()); */
-/*     hc.ApiPort = results[10u].str().empty() ? ((hc.Protocol == "https") ? 443u : 80u) */
-/*                                             : utils::string::to_uint16(results[10u].str()); */
-/*     return not hc.HostNameOrIp.empty() && not hc.Protocol.empty() && */
-/*            ((hc.Protocol == "http") || (hc.Protocol == "https")); */
-/*   } */
-/*  */
-/*   return false; */
-/* } */
-
-void spin_wait_for_mutex(std::function<bool()> complete, std::condition_variable &cv,
-                         std::mutex &mtx, const std::string &text) {
+void spin_wait_for_mutex(std::function<bool()> complete,
+                         std::condition_variable &cv, std::mutex &mtx,
+                         const std::string &text) {
   while (not complete()) {
     unique_mutex_lock l(mtx);
     if (not complete()) {
       if (not text.empty()) {
-        /* event_system::instance().raise<DebugLog>(__FUNCTION__, "spin_wait_for_mutex", text); */
+        /* event_system::instance().raise<DebugLog>(__FUNCTION__,
+         * "spin_wait_for_mutex", text); */
       }
-      cv.wait_for(l, 5s);
+      cv.wait_for(l, 1s);
     }
     l.unlock();
   }
 }
 
-void spin_wait_for_mutex(bool &complete, std::condition_variable &cv, std::mutex &mtx,
-                         const std::string &text) {
+void spin_wait_for_mutex(bool &complete, std::condition_variable &cv,
+                         std::mutex &mtx, const std::string &text) {
   while (not complete) {
     unique_mutex_lock l(mtx);
     if (not complete) {
       if (not text.empty()) {
-        /* event_system::instance().raise<DebugLog>(__FUNCTION__, "spin_wait_for_mutex", text); */
+        /* event_system::instance().raise<DebugLog>(__FUNCTION__,
+         * "spin_wait_for_mutex", text); */
       }
-      cv.wait_for(l, 5s);
+      cv.wait_for(l, 1s);
     }
     l.unlock();
   }