2.0.0-rc (#9)

### Issues

* \#1 \[bug\] Unable to mount S3 due to 'item_not_found' exception
* \#2 Require bucket name for S3 mounts
* \#3 \[bug\] File size is not being updated in S3 mount
* \#4 Upgrade to libfuse-3.x.x
* \#5 Switch to renterd for Sia support
* \#6 Switch to cpp-httplib to further reduce dependencies
* \#7 Remove global_data and calculate used disk space per provider
* \#8 Switch to libcurl for S3 mount support

### Changes from v1.x.x

* Added read-only encrypt provider
  * Pass-through mount point that transparently encrypts source data using `XChaCha20-Poly1305`
* Added S3 encryption support via `XChaCha20-Poly1305`
* Added replay protection to remote mounts
* Added support base64 writes in remote FUSE
* Created static linked Linux binaries for `amd64` and `aarch64` using `musl-libc`
* Removed legacy Sia renter support
* Removed Skynet support
* Fixed multiple remote mount WinFSP API issues on \*NIX servers
* Implemented chunked read and write
  * Writes for non-cached files are performed in chunks of 8Mib
* Removed `repertory-ui` support
* Removed `FreeBSD` support
* Switched to `libsodium` over `CryptoPP`
* Switched to `XChaCha20-Poly1305` for remote mounts
* Updated `GoogleTest` to v1.14.0
* Updated `JSON for Modern C++` to v3.11.2
* Updated `OpenSSL` to v1.1.1w
* Updated `RocksDB` to v8.5.3
* Updated `WinFSP` to 2023
* Updated `boost` to v1.78.0
* Updated `cURL` to v8.3.0
* Updated `zlib` to v1.3
* Use `upload_manager` for all providers
  * Adds a delay to uploads to prevent excessive API calls
  * Supports re-upload after mount restart for incomplete uploads
  * NOTE: Uploads for all providers are full file (no resume support)
    * Multipart upload support is planned for S3

Reviewed-on: #9

src/platform/unix_platform.cpp

@@ -1,36 +1,47 @@
 /*
-  Copyright <2018-2022> <scott.e.graves@protonmail.com>
+  Copyright <2018-2023> <scott.e.graves@protonmail.com>
 
-  Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
-  associated documentation files (the "Software"), to deal in the Software without restriction,
-  including without limitation the rights to use, copy, modify, merge, publish, distribute,
-  sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
   furnished to do so, subject to the following conditions:
 
-  The above copyright notice and this permission notice shall be included in all copies or
-  substantial portions of the Software.
+  The above copyright notice and this permission notice shall be included in all
+  copies or substantial portions of the Software.
 
-  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
-  NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
-  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
-  OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  SOFTWARE.
 */
 #ifndef _WIN32
+
 #include "platform/unix_platform.hpp"
+
 #include "app_config.hpp"
+#include "providers/i_provider.hpp"
+#include "types/startup_exception.hpp"
+#include "utils/error_utils.hpp"
 #include "utils/file_utils.hpp"
 #include "utils/path_utils.hpp"
+#include "utils/unix/unix_utils.hpp"
 
 namespace repertory {
 lock_data::lock_data(const provider_type &pt, std::string unique_id /*= ""*/)
     : pt_(pt),
       unique_id_(std::move(unique_id)),
-      mutex_id_("repertory2_" + app_config::get_provider_name(pt) + "_" + unique_id_) {
+      mutex_id_("repertory2_" + app_config::get_provider_name(pt) + "_" +
+                unique_id_) {
   lock_fd_ = open(get_lock_file().c_str(), O_CREAT | O_RDWR, S_IWUSR | S_IRUSR);
 }
 
-lock_data::lock_data() : pt_(provider_type::sia), unique_id_(""), mutex_id_(""), lock_fd_(-1) {}
+lock_data::lock_data()
+    : pt_(provider_type::sia), unique_id_(""), mutex_id_(""), lock_fd_(-1) {}
 
 lock_data::~lock_data() {
   if (lock_fd_ != -1) {
@@ -43,21 +54,31 @@ lock_data::~lock_data() {
   }
 }
 
-std::string lock_data::get_lock_data_file() {
+auto lock_data::get_lock_data_file() -> std::string {
   const auto dir = get_state_directory();
-  utils::file::create_full_directory_path(dir);
-  return utils::path::combine(dir, {"mountstate_" + std::to_string(getuid()) + ".json"});
+  if (not utils::file::create_full_directory_path(dir)) {
+    throw startup_exception("failed to create directory|sp|" + dir + "|err|" +
+                            std::to_string(utils::get_last_error_code()));
+  }
+  return utils::path::combine(
+      dir, {"mountstate_" + std::to_string(getuid()) + ".json"});
 }
 
-std::string lock_data::get_lock_file() {
+auto lock_data::get_lock_file() -> std::string {
   const auto dir = get_state_directory();
-  utils::file::create_full_directory_path(dir);
-  return utils::path::combine(dir, {mutex_id_ + "_" + std::to_string(getuid())});
+  if (not utils::file::create_full_directory_path(dir)) {
+    throw startup_exception("failed to create directory|sp|" + dir + "|err|" +
+                            std::to_string(utils::get_last_error_code()));
+  }
+
+  return utils::path::combine(dir,
+                              {mutex_id_ + "_" + std::to_string(getuid())});
 }
 
-bool lock_data::get_mount_state(json &mount_state) {
+auto lock_data::get_mount_state(json &mount_state) -> bool {
   auto ret = false;
-  auto fd = open(get_lock_data_file().c_str(), O_CREAT | O_RDWR, S_IWUSR | S_IRUSR);
+  auto fd =
+      open(get_lock_data_file().c_str(), O_CREAT | O_RDWR, S_IWUSR | S_IRUSR);
   if (fd != -1) {
     if (wait_for_lock(fd) == 0) {
       ret = utils::file::read_json_file(get_lock_data_file(), mount_state);
@@ -69,16 +90,17 @@ bool lock_data::get_mount_state(json &mount_state) {
   return ret;
 }
 
-std::string lock_data::get_state_directory() {
+auto lock_data::get_state_directory() -> std::string {
 #ifdef __APPLE__
-  return utils::path::resolve("~/Library/Application Support/" + std::string(REPERTORY_DATA_NAME) +
-                              "/state");
+  return utils::path::resolve("~/Library/Application Support/" +
+                              std::string(REPERTORY_DATA_NAME) + "/state");
 #else
-  return utils::path::resolve("~/.local/" + std::string(REPERTORY_DATA_NAME) + "/state");
+  return utils::path::resolve("~/.local/" + std::string(REPERTORY_DATA_NAME) +
+                              "/state");
 #endif
 }
 
-lock_result lock_data::grab_lock(const std::uint8_t &retry_count) {
+auto lock_data::grab_lock(std::uint8_t retry_count) -> lock_result {
   if (lock_fd_ == -1) {
     return lock_result::failure;
   }
@@ -86,7 +108,9 @@ lock_result lock_data::grab_lock(const std::uint8_t &retry_count) {
   lock_status_ = wait_for_lock(lock_fd_, retry_count);
   switch (lock_status_) {
   case 0:
-    set_mount_state(false, "", -1);
+    if (not set_mount_state(false, "", -1)) {
+      utils::error::raise_error(__FUNCTION__, "failed to set mount state");
+    }
     return lock_result::success;
   case EWOULDBLOCK:
     return lock_result::locked;
@@ -95,30 +119,44 @@ lock_result lock_data::grab_lock(const std::uint8_t &retry_count) {
   }
 }
 
-bool lock_data::set_mount_state(const bool &active, const std::string &mount_location,
-                                const int &pid) {
+auto lock_data::set_mount_state(bool active, const std::string &mount_location,
+                                int pid) -> bool {
   auto ret = false;
-  auto fd = open(get_lock_data_file().c_str(), O_CREAT | O_RDWR, S_IWUSR | S_IRUSR);
+  auto fd =
+      open(get_lock_data_file().c_str(), O_CREAT | O_RDWR, S_IWUSR | S_IRUSR);
   if (fd != -1) {
     if (wait_for_lock(fd) == 0) {
-      const auto mount_id = app_config::get_provider_display_name(pt_) + unique_id_;
+      const auto mount_id =
+          app_config::get_provider_display_name(pt_) + unique_id_;
       json mount_state;
-      utils::file::read_json_file(get_lock_data_file(), mount_state);
+      if (not utils::file::read_json_file(get_lock_data_file(), mount_state)) {
+        utils::error::raise_error(__FUNCTION__,
+                                  "failed to read mount state file|sp|" +
+                                      get_lock_file());
+      }
       if ((mount_state.find(mount_id) == mount_state.end()) ||
-          (mount_state[mount_id].find("Active") == mount_state[mount_id].end()) ||
+          (mount_state[mount_id].find("Active") ==
+           mount_state[mount_id].end()) ||
           (mount_state[mount_id]["Active"].get<bool>() != active) ||
-          (active && ((mount_state[mount_id].find("Location") == mount_state[mount_id].end()) ||
-                      (mount_state[mount_id]["Location"].get<std::string>() != mount_location)))) {
+          (active && ((mount_state[mount_id].find("Location") ==
+                       mount_state[mount_id].end()) ||
+                      (mount_state[mount_id]["Location"].get<std::string>() !=
+                       mount_location)))) {
         const auto lines = utils::file::read_file_lines(get_lock_data_file());
         const auto txt =
             std::accumulate(lines.begin(), lines.end(), std::string(),
-                            [](std::string s, const std::string &s2) { return std::move(s) + s2; });
+                            [](std::string s, const std::string &s2) {
+                              return std::move(s) + s2;
+                            });
         auto json = json::parse(txt.empty() ? "{}" : txt);
         json[mount_id] = {{"Active", active},
                           {"Location", active ? mount_location : ""},
                           {"PID", active ? pid : -1}};
         ret = utils::file::write_json_file(get_lock_data_file(), json);
+      } else {
+        ret = true;
       }
+
       flock(fd, LOCK_UN);
     }
 
@@ -127,14 +165,14 @@ bool lock_data::set_mount_state(const bool &active, const std::string &mount_loc
   return ret;
 }
 
-int lock_data::wait_for_lock(const int &fd, const std::uint8_t &retry_count) {
+auto lock_data::wait_for_lock(int fd, std::uint8_t retry_count) -> int {
   auto lock_status = EWOULDBLOCK;
   std::int16_t remain = retry_count * 100u;
   while ((remain > 0) && (lock_status == EWOULDBLOCK)) {
     if ((lock_status = flock(fd, LOCK_EX | LOCK_NB)) == -1) {
       if ((lock_status = errno) == EWOULDBLOCK) {
-        const auto sleep_ms =
-            utils::random_between(std::int16_t(1), std::min(remain, std::int16_t(100)));
+        const auto sleep_ms = utils::random_between(
+            std::int16_t(1), std::min(remain, std::int16_t(100)));
         std::this_thread::sleep_for(std::chrono::milliseconds(sleep_ms));
         remain -= sleep_ms;
       }
@@ -143,6 +181,56 @@ int lock_data::wait_for_lock(const int &fd, const std::uint8_t &retry_count) {
 
   return lock_status;
 }
+
+auto create_meta_attributes(
+    std::uint64_t accessed_date, std::uint32_t attributes,
+    std::uint64_t changed_date, std::uint64_t creation_date, bool directory,
+    const std::string &encryption_token, std::uint32_t gid,
+    const std::string &key, std::uint32_t mode, std::uint64_t modified_date,
+    std::uint32_t osx_backup, std::uint32_t osx_flags, std::uint64_t size,
+    const std::string &source_path, std::uint32_t uid,
+    std::uint64_t written_date) -> api_meta_map {
+  return {
+      {META_ACCESSED, std::to_string(accessed_date)},
+      {META_ATTRIBUTES, std::to_string(attributes)},
+      {META_BACKUP, std::to_string(osx_backup)},
+      {META_CHANGED, std::to_string(changed_date)},
+      {META_CREATION, std::to_string(creation_date)},
+      {META_DIRECTORY, utils::string::from_bool(directory)},
+      {META_ENCRYPTION_TOKEN, encryption_token},
+      {META_GID, std::to_string(gid)},
+      {META_KEY, key},
+      {META_MODE, std::to_string(mode)},
+      {META_MODIFIED, std::to_string(modified_date)},
+      {META_OSXFLAGS, std::to_string(osx_flags)},
+      {META_PINNED, "0"},
+      {META_SOURCE, source_path},
+      {META_SIZE, std::to_string(size)},
+      {META_UID, std::to_string(uid)},
+      {META_WRITTEN, std::to_string(written_date)},
+  };
+}
+
+auto provider_meta_handler(i_provider &provider, bool directory,
+                           const api_file &file) -> api_error {
+  const auto meta = create_meta_attributes(
+      file.accessed_date,
+      directory ? FILE_ATTRIBUTE_DIRECTORY
+                : FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_ARCHIVE,
+      file.changed_date, file.creation_date, directory, file.encryption_token,
+      getgid(), file.key,
+      directory ? S_IFDIR | S_IRUSR | S_IWUSR | S_IXUSR
+                : S_IFREG | S_IRUSR | S_IWUSR,
+      file.modified_date, 0u, 0u, file.file_size, file.source_path, getuid(),
+      file.modified_date);
+  auto res = provider.set_item_meta(file.api_path, meta);
+  if (res == api_error::success) {
+    event_system::instance().raise<filesystem_item_added>(
+        file.api_path, file.api_parent, directory);
+  }
+
+  return res;
+}
 } // namespace repertory
 
 #endif //_WIN32