2.0.0-rc (#9)

### Issues * \#1 \[bug\] Unable to mount S3 due to 'item_not_found' exception * \#2 Require bucket name for S3 mounts * \#3 \[bug\] File size is not being updated in S3 mount * \#4 Upgrade to libfuse-3.x.x * \#5 Switch to renterd for Sia support * \#6 Switch to cpp-httplib to further reduce dependencies * \#7 Remove global_data and calculate used disk space per provider * \#8 Switch to libcurl for S3 mount support ### Changes from v1.x.x * Added read-only encrypt provider * Pass-through mount point that transparently encrypts source data using `XChaCha20-Poly1305` * Added S3 encryption support via `XChaCha20-Poly1305` * Added replay protection to remote mounts * Added support base64 writes in remote FUSE * Created static linked Linux binaries for `amd64` and `aarch64` using `musl-libc` * Removed legacy Sia renter support * Removed Skynet support * Fixed multiple remote mount WinFSP API issues on \*NIX servers * Implemented chunked read and write * Writes for non-cached files are performed in chunks of 8Mib * Removed `repertory-ui` support * Removed `FreeBSD` support * Switched to `libsodium` over `CryptoPP` * Switched to `XChaCha20-Poly1305` for remote mounts * Updated `GoogleTest` to v1.14.0 * Updated `JSON for Modern C++` to v3.11.2 * Updated `OpenSSL` to v1.1.1w * Updated `RocksDB` to v8.5.3 * Updated `WinFSP` to 2023 * Updated `boost` to v1.78.0 * Updated `cURL` to v8.3.0 * Updated `zlib` to v1.3 * Use `upload_manager` for all providers * Adds a delay to uploads to prevent excessive API calls * Supports re-upload after mount restart for incomplete uploads * NOTE: Uploads for all providers are full file (no resume support) * Multipart upload support is planned for S3 Reviewed-on: #9
2023-10-29 06:55:59 +00:00
parent 3ff46723b8
commit f43c41f88a
839 changed files with 98214 additions and 92959 deletions
--- a/src/drives/directory_iterator.cpp
+++ b/src/drives/directory_iterator.cpp
@@ -1,53 +1,70 @@
 /*
-  Copyright <2018-2022> <scott.e.graves@protonmail.com>
+  Copyright <2018-2023> <scott.e.graves@protonmail.com>

-  Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
-  associated documentation files (the "Software"), to deal in the Software without restriction,
-  including without limitation the rights to use, copy, modify, merge, publish, distribute,
-  sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
  furnished to do so, subject to the following conditions:

-  The above copyright notice and this permission notice shall be included in all copies or
-  substantial portions of the Software.
+  The above copyright notice and this permission notice shall be included in all
+  copies or substantial portions of the Software.

-  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
-  NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
-  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
-  OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  SOFTWARE.
 */
 #include "drives/directory_iterator.hpp"
+
+#include "utils/error_utils.hpp"
 #include "utils/path_utils.hpp"

 namespace repertory {
 #ifndef _WIN32
-int directory_iterator::fill_buffer(const remote::file_offset &offset,
-                                    fuse_fill_dir_t filler_function, void *buffer,
-                                    populate_stat_callback populate_stat) {
+auto directory_iterator::fill_buffer(const remote::file_offset &offset,
+                                     fuse_fill_dir_t filler_function,
+                                     void *buffer,
+                                     populate_stat_callback populate_stat)
+    -> int {
  if (offset < items_.size()) {
-    std::string item_name;
-    struct stat st {};
-    struct stat *pst = nullptr;
-    switch (offset) {
-    case 0: {
-      item_name = ".";
-    } break;
+    try {
+      std::string item_name;
+      struct stat st {};
+      struct stat *pst = nullptr;
+      switch (offset) {
+      case 0: {
+        item_name = ".";
+      } break;

-    case 1: {
-      item_name = "..";
-    } break;
+      case 1: {
+        item_name = "..";
+      } break;

-    default: {
-      const auto &item = items_[offset];
-      item_name = utils::path::strip_to_file_name(item.api_path);
-      populate_stat(item.api_path, item.size, item.meta, item.directory, &st);
-      pst = &st;
-    } break;
-    }
+      default: {
+        const auto &item = items_[offset];
+        item_name = utils::path::strip_to_file_name(item.api_path);
+        populate_stat(item.api_path, item.size, item.meta, item.directory, &st);
+        pst = &st;
+      } break;
+      }

-    if (filler_function(buffer, &item_name[0], pst, offset + 1) != 0) {
-      errno = ENOMEM;
-      return -1;
+#if FUSE_USE_VERSION >= 30
+      if (filler_function(buffer, &item_name[0], pst, offset + 1,
+                          FUSE_FILL_DIR_PLUS) != 0) {
+#else
+      if (filler_function(buffer, &item_name[0], pst, offset + 1) != 0) {
+#endif
+        errno = ENOMEM;
+        return -1;
+      }
+    } catch (const std::exception &e) {
+      utils::error::raise_error(__FUNCTION__, e,
+                                "failed to fill fuse directory buffer");
    }

    return 0;
@@ -58,7 +75,7 @@ int directory_iterator::fill_buffer(const remote::file_offset &offset,
 }
 #endif // !_WIN32

-int directory_iterator::get(const std::size_t &offset, std::string &item) {
+auto directory_iterator::get(std::size_t offset, std::string &item) -> int {
  if (offset < items_.size()) {
    item = items_[offset].api_path;
    return 0;
@@ -68,7 +85,8 @@ int directory_iterator::get(const std::size_t &offset, std::string &item) {
  return -1;
 }

-api_error directory_iterator::get_directory_item(const std::size_t &offset, directory_item &di) {
+auto directory_iterator::get_directory_item(std::size_t offset,
+                                            directory_item &di) -> api_error {
  if (offset < items_.size()) {
    di = items_[offset];
    return api_error::success;
@@ -77,9 +95,12 @@ api_error directory_iterator::get_directory_item(const std::size_t &offset, dire
  return api_error::directory_end_of_files;
 }

-api_error directory_iterator::get_directory_item(const std::string &api_path, directory_item &di) {
-  auto iter = std::find_if(items_.begin(), items_.end(),
-                           [&](const auto &item) -> bool { return api_path == item.api_path; });
+auto directory_iterator::get_directory_item(const std::string &api_path,
+                                            directory_item &di) -> api_error {
+  auto iter =
+      std::find_if(items_.begin(), items_.end(), [&](const auto &item) -> bool {
+        return api_path == item.api_path;
+      });
  if (iter != items_.end()) {
    di = *iter;
    return api_error::success;
@@ -88,7 +109,7 @@ api_error directory_iterator::get_directory_item(const std::string &api_path, di
  return api_error::item_not_found;
 }

-int directory_iterator::get_json(const std::size_t &offset, json &item) {
+auto directory_iterator::get_json(std::size_t offset, json &item) -> int {
  if (offset < items_.size()) {
    item = items_[offset].to_json();
    return 0;
@@ -98,29 +119,35 @@ int directory_iterator::get_json(const std::size_t &offset, json &item) {
  return -1;
 }

-std::size_t directory_iterator::get_next_directory_offset(const std::string &api_path) const {
-  const auto it = std::find_if(items_.begin(), items_.end(), [&api_path](const auto &di) -> bool {
-    return api_path == di.api_path;
-  });
+auto directory_iterator::get_next_directory_offset(
+    const std::string &api_path) const -> std::size_t {
+  const auto it = std::find_if(
+      items_.begin(), items_.end(),
+      [&api_path](const auto &di) -> bool { return api_path == di.api_path; });

-  return (it == items_.end()) ? 0 : std::distance(items_.begin(), it) + std::size_t(1u);
+  return (it == items_.end())
+             ? 0
+             : std::distance(items_.begin(), it) + std::size_t(1u);
 }

-directory_iterator &directory_iterator::operator=(const directory_iterator &iterator) noexcept {
+auto directory_iterator::operator=(const directory_iterator &iterator) noexcept
+    -> directory_iterator & {
  if (this != &iterator) {
    items_ = iterator.items_;
  }
  return *this;
 }

-directory_iterator &directory_iterator::operator=(directory_iterator &&iterator) noexcept {
+auto directory_iterator::operator=(directory_iterator &&iterator) noexcept
+    -> directory_iterator & {
  if (this != &iterator) {
    items_ = std::move(iterator.items_);
  }
  return *this;
 }

-directory_iterator &directory_iterator::operator=(directory_item_list list) noexcept {
+auto directory_iterator::operator=(directory_item_list list) noexcept
+    -> directory_iterator & {
  if (&items_ != &list) {
    items_ = std::move(list);
  }