2.0.0-rc (#9)

### Issues

* \#1 \[bug\] Unable to mount S3 due to 'item_not_found' exception
* \#2 Require bucket name for S3 mounts
* \#3 \[bug\] File size is not being updated in S3 mount
* \#4 Upgrade to libfuse-3.x.x
* \#5 Switch to renterd for Sia support
* \#6 Switch to cpp-httplib to further reduce dependencies
* \#7 Remove global_data and calculate used disk space per provider
* \#8 Switch to libcurl for S3 mount support

### Changes from v1.x.x

* Added read-only encrypt provider
  * Pass-through mount point that transparently encrypts source data using `XChaCha20-Poly1305`
* Added S3 encryption support via `XChaCha20-Poly1305`
* Added replay protection to remote mounts
* Added support base64 writes in remote FUSE
* Created static linked Linux binaries for `amd64` and `aarch64` using `musl-libc`
* Removed legacy Sia renter support
* Removed Skynet support
* Fixed multiple remote mount WinFSP API issues on \*NIX servers
* Implemented chunked read and write
  * Writes for non-cached files are performed in chunks of 8Mib
* Removed `repertory-ui` support
* Removed `FreeBSD` support
* Switched to `libsodium` over `CryptoPP`
* Switched to `XChaCha20-Poly1305` for remote mounts
* Updated `GoogleTest` to v1.14.0
* Updated `JSON for Modern C++` to v3.11.2
* Updated `OpenSSL` to v1.1.1w
* Updated `RocksDB` to v8.5.3
* Updated `WinFSP` to 2023
* Updated `boost` to v1.78.0
* Updated `cURL` to v8.3.0
* Updated `zlib` to v1.3
* Use `upload_manager` for all providers
  * Adds a delay to uploads to prevent excessive API calls
  * Supports re-upload after mount restart for incomplete uploads
  * NOTE: Uploads for all providers are full file (no resume support)
    * Multipart upload support is planned for S3

Reviewed-on: #9

src/utils/native_file.cpp

@@ -1,26 +1,41 @@
 /*
-  Copyright <2018-2022> <scott.e.graves@protonmail.com>
+  Copyright <2018-2023> <scott.e.graves@protonmail.com>
 
-  Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
-  associated documentation files (the "Software"), to deal in the Software without restriction,
-  including without limitation the rights to use, copy, modify, merge, publish, distribute,
-  sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
   furnished to do so, subject to the following conditions:
 
-  The above copyright notice and this permission notice shall be included in all copies or
-  substantial portions of the Software.
+  The above copyright notice and this permission notice shall be included in all
+  copies or substantial portions of the Software.
 
-  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
-  NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
-  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
-  OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  SOFTWARE.
 */
 #include "utils/native_file.hpp"
+
+#include "types/repertory.hpp"
 #include "utils/string_utils.hpp"
+#include "utils/unix/unix_utils.hpp"
+#include "utils/utils.hpp"
 
 namespace repertory {
-native_file_ptr native_file::clone(const native_file_ptr &nf) {
+auto native_file::get_handle() -> native_handle { return handle_; }
+
+native_file::~native_file() {
+  if (auto_close) {
+    close();
+  }
+}
+
+auto native_file::clone(const native_file_ptr &nf) -> native_file_ptr {
   std::string source_path;
 
 #ifdef _WIN32
@@ -31,53 +46,79 @@ native_file_ptr native_file::clone(const native_file_ptr &nf) {
   source_path.resize(PATH_MAX + 1);
 #ifdef __APPLE__
   fcntl(nf->get_handle(), F_GETPATH, &source_path[0u]);
-  a
 #else
-  readlink(("/proc/self/fd/" + utils::string::from_int64(nf->get_handle())).c_str(),
+  readlink(("/proc/self/fd/" + std::to_string(nf->get_handle())).c_str(),
            &source_path[0u], source_path.size());
 #endif
 #endif
   source_path = source_path.c_str();
 
   native_file_ptr clone;
-  if (native_file::open(source_path, clone) != api_error::success) {
-    throw std::runtime_error("unable to clone file " + source_path);
+  auto res = native_file::open(source_path, clone);
+  if (res != api_error::success) {
+    throw std::runtime_error("unable to open file|sp|" + source_path + "|err|" +
+                             api_error_to_string(res));
   }
 
   return clone;
 }
 
-api_error native_file::create_or_open(const std::string &source_path, native_file_ptr &nf) {
+auto native_file::create_or_open(const std::string &source_path,
+                                 [[maybe_unused]] bool should_chmod,
+                                 native_file_ptr &nf) -> api_error {
 #ifdef _WIN32
   auto handle = ::CreateFileA(source_path.c_str(), GENERIC_READ | GENERIC_WRITE,
-                              FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, OPEN_ALWAYS,
-                              FILE_FLAG_RANDOM_ACCESS, nullptr);
+                              FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr,
+                              OPEN_ALWAYS, FILE_FLAG_RANDOM_ACCESS, nullptr);
 #else
-  auto handle = ::open(source_path.c_str(), O_CREAT | O_RDWR | O_CLOEXEC, 0600u);
-  chmod(source_path.c_str(), 0600u);
+  auto handle = should_chmod ? ::open(source_path.c_str(),
+                                      O_CREAT | O_RDWR | O_CLOEXEC, 0600u)
+                             : ::open(source_path.c_str(), O_RDWR | O_CLOEXEC);
+  if (should_chmod) {
+    chmod(source_path.c_str(), 0600u);
+  }
 #endif
   nf = native_file::attach(handle);
-  return ((handle == REPERTORY_INVALID_HANDLE) ? api_error::os_error : api_error::success);
+  return ((handle == REPERTORY_INVALID_HANDLE) ? api_error::os_error
+                                               : api_error::success);
 }
 
-api_error native_file::open(const std::string &source_path, native_file_ptr &nf) {
+auto native_file::create_or_open(const std::string &source_path,
+                                 native_file_ptr &nf) -> api_error {
+  return create_or_open(source_path, true, nf);
+}
+
+auto native_file::open(const std::string &source_path, native_file_ptr &nf)
+    -> api_error {
+  return open(source_path, true, nf);
+}
+
+auto native_file::open(const std::string &source_path,
+                       [[maybe_unused]] bool should_chmod, native_file_ptr &nf)
+    -> api_error {
 #ifdef _WIN32
   auto handle = ::CreateFileA(source_path.c_str(), GENERIC_READ | GENERIC_WRITE,
-                              FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, OPEN_EXISTING,
-                              FILE_FLAG_RANDOM_ACCESS, nullptr);
+                              FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr,
+                              OPEN_EXISTING, FILE_FLAG_RANDOM_ACCESS, nullptr);
 #else
-  auto handle = ::open(source_path.c_str(), O_RDWR | O_CLOEXEC, 0600u);
-  chmod(source_path.c_str(), 0600u);
+  auto handle = should_chmod
+                    ? ::open(source_path.c_str(), O_RDWR | O_CLOEXEC, 0600u)
+                    : ::open(source_path.c_str(), O_RDONLY | O_CLOEXEC);
+  if (should_chmod) {
+    chmod(source_path.c_str(), 0600u);
+  }
 #endif
   nf = native_file::attach(handle);
-  return ((handle == REPERTORY_INVALID_HANDLE) ? api_error::os_error : api_error::success);
+  return ((handle == REPERTORY_INVALID_HANDLE) ? api_error::os_error
+                                               : api_error::success);
 }
 
-bool native_file::allocate(const std::uint64_t &file_size) {
+auto native_file::allocate(std::uint64_t file_size) -> bool {
 #ifdef _WIN32
   LARGE_INTEGER li{};
   li.QuadPart = file_size;
-  return (::SetFilePointerEx(handle_, li, nullptr, FILE_BEGIN) && ::SetEndOfFile(handle_));
+  return (::SetFilePointerEx(handle_, li, nullptr, FILE_BEGIN) &&
+          ::SetEndOfFile(handle_));
 #endif
 #ifdef __linux__
   return (fallocate(handle_, 0, 0, file_size) >= 0);
@@ -98,16 +139,17 @@ void native_file::close() {
   }
 }
 
-bool native_file::copy_from(const native_file_ptr &nf) {
+auto native_file::copy_from(const native_file_ptr &nf) -> bool {
   auto ret = false;
   std::uint64_t file_size = 0u;
   if ((ret = nf->get_file_size(file_size))) {
-    std::vector<char> buffer;
+    data_buffer buffer;
     buffer.resize(65536u * 2u);
     std::uint64_t offset = 0u;
     while (ret && (file_size > 0u)) {
       std::size_t bytes_read{};
-      if ((ret = nf->read_bytes(&buffer[0u], buffer.size(), offset, bytes_read))) {
+      if ((ret = nf->read_bytes(&buffer[0u], buffer.size(), offset,
+                                bytes_read))) {
         std::size_t bytes_written = 0u;
         ret = write_bytes(&buffer[0u], bytes_read, offset, bytes_written);
         file_size -= bytes_read;
@@ -120,7 +162,7 @@ bool native_file::copy_from(const native_file_ptr &nf) {
   return ret;
 }
 
-bool native_file::copy_from(const std::string &path) {
+auto native_file::copy_from(const std::string &path) -> bool {
   auto ret = false;
   native_file_ptr nf;
   if (native_file::create_or_open(path, nf) == api_error ::success) {
@@ -140,7 +182,7 @@ void native_file::flush() {
 #endif
 }
 
-bool native_file::get_file_size(std::uint64_t &file_size) {
+auto native_file::get_file_size(std::uint64_t &file_size) -> bool {
   auto ret = false;
 #ifdef _WIN32
   LARGE_INTEGER li{};
@@ -164,8 +206,9 @@ bool native_file::get_file_size(std::uint64_t &file_size) {
 }
 
 #ifdef _WIN32
-bool native_file::read_bytes(char *buffer, const std::size_t &read_size,
-                             const std::uint64_t &read_offset, std::size_t &bytes_read) {
+auto native_file::read_bytes(char *buffer, std::size_t read_size,
+                             std::uint64_t read_offset, std::size_t &bytes_read)
+    -> bool {
   recur_mutex_lock l(read_write_mutex_);
 
   auto ret = false;
@@ -176,7 +219,8 @@ bool native_file::read_bytes(char *buffer, const std::size_t &read_size,
     DWORD current_read = 0u;
     do {
       current_read = 0u;
-      ret = !!::ReadFile(handle_, &buffer[bytes_read], static_cast<DWORD>(read_size - bytes_read),
+      ret = !!::ReadFile(handle_, &buffer[bytes_read],
+                         static_cast<DWORD>(read_size - bytes_read),
                          &current_read, nullptr);
       bytes_read += current_read;
     } while (ret && (bytes_read < read_size) && (current_read != 0));
@@ -189,13 +233,14 @@ bool native_file::read_bytes(char *buffer, const std::size_t &read_size,
   return ret;
 }
 #else
-bool native_file::read_bytes(char *buffer, const std::size_t &read_size,
-                             const std::uint64_t &read_offset, std::size_t &bytes_read) {
+auto native_file::read_bytes(char *buffer, std::size_t read_size,
+                             std::uint64_t read_offset, std::size_t &bytes_read)
+    -> bool {
   bytes_read = 0u;
   ssize_t result = 0;
   do {
-    result =
-        pread64(handle_, &buffer[bytes_read], read_size - bytes_read, read_offset + bytes_read);
+    result = pread64(handle_, &buffer[bytes_read], read_size - bytes_read,
+                     read_offset + bytes_read);
     if (result > 0) {
       bytes_read += static_cast<size_t>(result);
     }
@@ -204,20 +249,22 @@ bool native_file::read_bytes(char *buffer, const std::size_t &read_size,
   return (result >= 0);
 }
 #endif
-bool native_file::truncate(const std::uint64_t &file_size) {
+auto native_file::truncate(std::uint64_t file_size) -> bool {
 #ifdef _WIN32
   recur_mutex_lock l(read_write_mutex_);
   LARGE_INTEGER li{};
   li.QuadPart = file_size;
-  return (::SetFilePointerEx(handle_, li, nullptr, FILE_BEGIN) && ::SetEndOfFile(handle_));
+  return (::SetFilePointerEx(handle_, li, nullptr, FILE_BEGIN) &&
+          ::SetEndOfFile(handle_));
 #else
   return (ftruncate(handle_, file_size) >= 0);
 #endif
 }
 
 #ifdef _WIN32
-bool native_file::write_bytes(const char *buffer, const std::size_t &write_size,
-                              const std::uint64_t &write_offset, std::size_t &bytes_written) {
+auto native_file::write_bytes(const char *buffer, std::size_t write_size,
+                              std::uint64_t write_offset,
+                              std::size_t &bytes_written) -> bool {
   recur_mutex_lock l(read_write_mutex_);
 
   bytes_written = 0u;
@@ -229,7 +276,8 @@ bool native_file::write_bytes(const char *buffer, const std::size_t &write_size,
     do {
       DWORD current_write = 0u;
       ret = !!::WriteFile(handle_, &buffer[bytes_written],
-                          static_cast<DWORD>(write_size - bytes_written), &current_write, nullptr);
+                          static_cast<DWORD>(write_size - bytes_written),
+                          &current_write, nullptr);
       bytes_written += current_write;
     } while (ret && (bytes_written < write_size));
   }
@@ -237,13 +285,14 @@ bool native_file::write_bytes(const char *buffer, const std::size_t &write_size,
   return ret;
 }
 #else
-bool native_file::write_bytes(const char *buffer, const std::size_t &write_size,
-                              const std::uint64_t &write_offset, std::size_t &bytes_written) {
+auto native_file::write_bytes(const char *buffer, std::size_t write_size,
+                              std::uint64_t write_offset,
+                              std::size_t &bytes_written) -> bool {
   bytes_written = 0;
   ssize_t result = 0;
   do {
-    result = pwrite64(handle_, &buffer[bytes_written], write_size - bytes_written,
-                      write_offset + bytes_written);
+    result = pwrite64(handle_, &buffer[bytes_written],
+                      write_size - bytes_written, write_offset + bytes_written);
     if (result > 0) {
       bytes_written += static_cast<size_t>(result);
     }