sia 2.0.0 support

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## v2.0.4-rc
 
+### BREAKING CHANGES
+
+* `renterd` v2.0.x is now required. Prior versions will fail to mount.
+
 ### Issues
 
 * ~~\#12 [Unit Test] Complete all providers unit tests~~
@@ -12,11 +16,13 @@
 
 ### Changes from v2.0.3-rc
 
+* Added Sia API version check prior to mounting
 * Continue documentation updates
 * Fixed setting `ApiAuth` via `set_value_by_name`
 * Fixed setting `HostConfig.ApiUser` via `set_value_by_name`
 * Fixed setting `HostConfig.Path` via `set_value_by_name`
 * Fixed setting `HostConfig.Protocol` via `set_value_by_name`
+* Integrated `renterd` version 2.x
 * Refactored `app_config` unit tests
 * Refactored polling to be more accurate on scheduling tasks
 

README.md

@@ -14,7 +14,7 @@ on Windows.
 
 ## Minimum Requirements
 
-* [Sia renterd](https://github.com/SiaFoundation/renterd/releases) v0.4.0+ for Sia support
+* [Sia renterd](https://github.com/SiaFoundation/renterd/releases) v2.0.0+ for Sia support
 * Only 64-bit operating systems are supported
   * By default, Linux requires `fusermount3`; otherwise, `repertory` must be manually compiled with `libfuse2` support
   * Windows requires the following dependencies to be installed:

repertory/librepertory/include/events/types/provider_invalid_version.hpp

@@ -0,0 +1,78 @@
+/*
+  Copyright <2018-2025> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all
+  copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  SOFTWARE.
+*/
+#ifndef REPERTORY_INCLUDE_EVENTS_TYPES_PRODIVER_INVALID_VERSION_HPP_
+#define REPERTORY_INCLUDE_EVENTS_TYPES_PRODIVER_INVALID_VERSION_HPP_
+
+#include "events/i_event.hpp"
+#include "types/repertory.hpp"
+
+namespace repertory {
+struct provider_invalid_version final : public i_event {
+  provider_invalid_version() = default;
+  provider_invalid_version(std::string_view function_name_,
+                           std::string required_version_,
+                           std::string returned_version_)
+      : function_name(std::string(function_name_)),
+        required_version(std::move(required_version_)),
+        returned_version(std::move(returned_version_)) {}
+
+  static constexpr const event_level level{event_level::error};
+  static constexpr const std::string_view name{"provider_invalid_version"};
+
+  std::string function_name;
+  std::string required_version;
+  std::string returned_version;
+
+  [[nodiscard]] auto get_event_level() const -> event_level override {
+    return level;
+  }
+
+  [[nodiscard]] auto get_name() const -> std::string_view override {
+    return name;
+  }
+
+  [[nodiscard]] auto get_single_line() const -> std::string override {
+    return fmt::format("{}|func|{}|required|{}|returned|{}", name,
+                       function_name, required_version, returned_version);
+  }
+};
+} // namespace repertory
+
+NLOHMANN_JSON_NAMESPACE_BEGIN
+template <> struct adl_serializer<repertory::provider_invalid_version> {
+  static void to_json(json &data,
+                      const repertory::provider_invalid_version &value) {
+    data["function_name"] = value.function_name;
+    data["required_version"] = value.required_version;
+    data["returned_version"] = value.returned_version;
+  }
+
+  static void from_json(const json &data,
+                        repertory::provider_invalid_version &value) {
+    data.at("function_name").get_to(value.function_name);
+    data.at("required_version").get_to(value.required_version);
+    data.at("returned_version").get_to(value.returned_version);
+  }
+};
+NLOHMANN_JSON_NAMESPACE_END
+
+#endif // REPERTORY_INCLUDE_EVENTS_TYPES_PRODIVER_INVALID_VERSION_HPP_

repertory/librepertory/include/providers/encrypt/encrypt_provider.hpp

@@ -85,6 +85,13 @@ private:
   void remove_deleted_files(stop_type &stop_requested);
 
 public:
+  [[nodiscard]] auto check_version(std::string &required_version,
+                                   std::string &returned_version) const
+      -> bool override {
+    required_version = returned_version = "";
+    return true;
+  }
+
   [[nodiscard]] auto create_directory(const std::string &api_path,
                                       api_meta_map &meta) -> api_error override;
 

repertory/librepertory/include/providers/i_provider.hpp

@@ -31,6 +31,10 @@ class i_provider {
   INTERFACE_SETUP(i_provider);
 
 public:
+  [[nodiscard]] virtual auto check_version(std::string &required_version,
+                                           std::string &returned_version) const
+      -> bool = 0;
+
   [[nodiscard]] virtual auto create_directory(const std::string &api_path,
                                               api_meta_map &meta)
       -> api_error = 0;

repertory/librepertory/include/providers/s3/s3_provider.hpp

@@ -113,6 +113,13 @@ protected:
       -> api_error override;
 
 public:
+  [[nodiscard]] auto check_version(std::string &required_version,
+                                   std::string &returned_version) const
+      -> bool override {
+    required_version = returned_version = "";
+    return true;
+  }
+
   [[nodiscard]] static auto convert_api_date(std::string_view date)
       -> std::uint64_t;
 

repertory/librepertory/include/providers/sia/sia_provider.hpp

@@ -80,6 +80,10 @@ protected:
       -> api_error override;
 
 public:
+  [[nodiscard]] auto check_version(std::string &required_version,
+                                   std::string &returned_version) const
+      -> bool override;
+
   [[nodiscard]] auto get_directory_item_count(const std::string &api_path) const
       -> std::uint64_t override;
 

repertory/librepertory/src/providers/base_provider.cpp

@@ -34,6 +34,7 @@
 #include "events/types/orphaned_file_processing_failed.hpp"
 #include "events/types/orphaned_source_file_detected.hpp"
 #include "events/types/orphaned_source_file_removed.hpp"
+#include "events/types/provider_invalid_version.hpp"
 #include "events/types/provider_offline.hpp"
 #include "events/types/provider_upload_begin.hpp"
 #include "events/types/provider_upload_end.hpp"
@@ -840,6 +841,14 @@ auto base_provider::start(api_item_added_callback api_item_added,
     return false;
   }
 
+  std::string returned_version;
+  std::string required_version;
+  if (not check_version(required_version, returned_version)) {
+    event_system::instance().raise<provider_invalid_version>(
+        function_name, required_version, returned_version);
+    return false;
+  }
+
   cache_size_mgr::instance().initialize(&config_);
 
   polling::instance().set_callback({

repertory/librepertory/src/providers/sia/sia_provider.cpp

@@ -32,6 +32,7 @@
 #include "providers/base_provider.hpp"
 #include "providers/s3/s3_provider.hpp"
 #include "types/repertory.hpp"
+#include "utils/common.hpp"
 #include "utils/error_utils.hpp"
 #include "utils/file_utils.hpp"
 #include "utils/path.hpp"
@@ -63,6 +64,52 @@ namespace repertory {
 sia_provider::sia_provider(app_config &config, i_http_comm &comm)
     : base_provider(config, comm) {}
 
+auto sia_provider::check_version(std::string &required_version,
+                                 std::string &returned_version) const -> bool {
+  REPERTORY_USES_FUNCTION_NAME();
+
+  required_version = "2.0.0";
+
+  try {
+    curl::requests::http_get get{};
+    get.allow_timeout = true;
+    get.path = "/api/bus/state";
+
+    nlohmann::json state_data;
+    std::string error_data;
+    get.response_handler = [&error_data, &state_data](auto &&data,
+                                                      long response_code) {
+      if (response_code == http_error_codes::ok) {
+        state_data = nlohmann::json::parse(data.begin(), data.end());
+        return;
+      }
+
+      error_data = std::string(data.begin(), data.end());
+    };
+
+    long response_code{};
+    stop_type stop_requested{};
+    if (not get_comm().make_request(get, response_code, stop_requested)) {
+      return false;
+    }
+
+    if (response_code != http_error_codes::ok) {
+      utils::error::raise_error(
+          function_name, response_code,
+          fmt::format("failed to check state|response|{}", error_data));
+      return false;
+    }
+
+    returned_version = state_data.at("version").get<std::string>().substr(1U);
+    return utils::compare_version_strings(returned_version, required_version) >=
+           0;
+  } catch (const std::exception &e) {
+    utils::error::raise_error(function_name, e, "failed to check version");
+  }
+
+  return false;
+}
+
 auto sia_provider::create_directory_impl(const std::string &api_path,
                                          api_meta_map & /* meta */)
     -> api_error {
@@ -70,7 +117,7 @@ auto sia_provider::create_directory_impl(const std::string &api_path,
 
   curl::requests::http_put_file put_file{};
   put_file.allow_timeout = true;
-  put_file.path = "/api/worker/objects" + api_path + "/";
+  put_file.path = "/api/worker/object" + api_path + "/";
   put_file.query["bucket"] = get_bucket(get_sia_config());
 
   std::string error_data;
@@ -112,10 +159,10 @@ auto sia_provider::get_directory_item_count(const std::string &api_path) const
     }
 
     std::uint64_t item_count{};
-    if (object_list.contains("entries")) {
-      for (const auto &entry : object_list.at("entries")) {
+    if (object_list.contains("objects")) {
+      for (const auto &entry : object_list.at("objects")) {
         try {
-          auto name{entry.at("name").get<std::string>()};
+          auto name{entry.at("key").get<std::string>()};
           auto entry_api_path{utils::path::create_api_path(name)};
           if (utils::string::ends_with(name, "/") &&
               (entry_api_path == api_path)) {
@@ -149,10 +196,10 @@ auto sia_provider::get_directory_items_impl(const std::string &api_path,
     return api_error::comm_error;
   }
 
-  if (object_list.contains("entries")) {
-    for (const auto &entry : object_list.at("entries")) {
+  if (object_list.contains("objects")) {
+    for (const auto &entry : object_list.at("objects")) {
       try {
-        auto name{entry.at("name").get<std::string>()};
+        auto name{entry.at("key").get<std::string>()};
         auto entry_api_path{utils::path::create_api_path(name)};
 
         auto directory{utils::string::ends_with(name, "/")};
@@ -222,20 +269,12 @@ auto sia_provider::get_file(const std::string &api_path, api_file &file) const
     }
 
     auto size{
-        file_data["object"].contains("Slabs")
-            ? std::accumulate(
-                  file_data["object"]["Slabs"].begin(),
-                  file_data["object"]["Slabs"].end(), std::uint64_t(0U),
-                  [](auto &&total_size, const json &slab) -> std::uint64_t {
-                    return total_size + slab["Length"].get<std::uint64_t>();
-                  })
-            : file_data["object"]["size"].get<std::uint64_t>(),
+        file_data.at("size").get<std::uint64_t>(),
     };
 
     api_meta_map meta{};
     if (get_item_meta(api_path, meta) == api_error::item_not_found) {
-      file = create_api_file(api_path, "", size,
-                             get_last_modified(file_data["object"]));
+      file = create_api_file(api_path, "", size, get_last_modified(file_data));
       get_api_item_added()(false, file);
     } else {
       file = create_api_file(api_path, size, meta);
@@ -263,9 +302,9 @@ auto sia_provider::get_file_list(api_file_list &list,
         return api_error::comm_error;
       }
 
-      if (object_list.contains("entries")) {
-        for (const auto &entry : object_list.at("entries")) {
-          auto name{entry.at("name").get<std::string>()};
+      if (object_list.contains("objects")) {
+        for (const auto &entry : object_list.at("objects")) {
+          auto name{entry.at("key").get<std::string>()};
           auto entry_api_path{utils::path::create_api_path(name)};
 
           if (utils::string::ends_with(name, "/")) {
@@ -326,8 +365,9 @@ auto sia_provider::get_object_info(const std::string &api_path,
   try {
     curl::requests::http_get get{};
     get.allow_timeout = true;
-    get.path = "/api/bus/objects" + api_path;
+    get.path = "/api/bus/object" + api_path;
     get.query["bucket"] = get_bucket(get_sia_config());
+    get.query["onlymetadata"] = "true";
 
     std::string error_data;
     get.response_handler = [&error_data, &object_info](auto &&data,
@@ -375,6 +415,7 @@ auto sia_provider::get_object_list(const std::string &api_path,
     get.allow_timeout = true;
     get.path = "/api/bus/objects" + api_path + "/";
     get.query["bucket"] = get_bucket(get_sia_config());
+    get.query["delimiter"] = "/";
 
     std::string error_data;
     get.response_handler = [&error_data, &object_list](auto &&data,
@@ -418,7 +459,7 @@ auto sia_provider::get_total_drive_space() const -> std::uint64_t {
   try {
     curl::requests::http_get get{};
     get.allow_timeout = true;
-    get.path = "/api/autopilot/config";
+    get.path = "/api/bus/autopilot";
     get.query["bucket"] = get_bucket(get_sia_config());
 
     json config_data;
@@ -468,17 +509,18 @@ auto sia_provider::is_directory(const std::string &api_path, bool &exists) const
 
     exists = false;
 
-    json object_list{};
-    if (not get_object_list(utils::path::get_parent_api_path(api_path),
-                            object_list)) {
-      return api_error::comm_error;
+    json file_data{};
+    auto res{get_object_info(api_path + '/', file_data)};
+    if (res == api_error::item_not_found) {
+      return api_error::success;
     }
 
-    exists = object_list.contains("entries") &&
-             std::ranges::find_if(object_list.at("entries"),
-                                  [&api_path](auto &&entry) -> bool {
-                                    return entry.at("name") == (api_path + "/");
-                                  }) != object_list.at("entries").end();
+    if (res != api_error::success) {
+      return res;
+    }
+
+    exists =
+        utils::string::ends_with(file_data.at("key").get<std::string>(), "/");
     return api_error::success;
   } catch (const std::exception &e) {
     utils::error::raise_api_path_error(
@@ -494,6 +536,7 @@ auto sia_provider::is_file(const std::string &api_path, bool &exists) const
 
   try {
     exists = false;
+
     if (api_path == "/") {
       return api_error::success;
     }
@@ -508,7 +551,8 @@ auto sia_provider::is_file(const std::string &api_path, bool &exists) const
       return res;
     }
 
-    exists = not file_data.contains("entries");
+    exists = not utils::string::ends_with(
+        file_data.at("key").get<std::string>(), "/");
     return api_error::success;
   } catch (const std::exception &e) {
     utils::error::raise_api_path_error(function_name, api_path, e,
@@ -572,8 +616,9 @@ auto sia_provider::read_file_bytes(const std::string &api_path,
 
   try {
     curl::requests::http_get get{};
-    get.path = "/api/worker/objects" + api_path;
+    get.path = "/api/worker/object" + api_path;
     get.query["bucket"] = get_bucket(get_sia_config());
+    get.headers["accept"] = "application/octet-stream";
     get.range = {{
         offset,
         offset + size - 1U,
@@ -590,6 +635,7 @@ auto sia_provider::read_file_bytes(const std::string &api_path,
          ++idx) {
       long response_code{};
       const auto notify_retry = [&]() {
+        fmt::println("{}", std::string(buffer.begin(), buffer.end()));
         if (response_code == 0) {
           utils::error::raise_api_path_error(
               function_name, api_path, api_error::comm_error,
@@ -635,7 +681,7 @@ auto sia_provider::remove_directory_impl(const std::string &api_path)
 
   curl::requests::http_delete del{};
   del.allow_timeout = true;
-  del.path = "/api/bus/objects" + api_path + "/";
+  del.path = "/api/bus/object" + api_path + "/";
   del.query["bucket"] = get_bucket(get_sia_config());
 
   std::string error_data;
@@ -671,7 +717,7 @@ auto sia_provider::remove_file_impl(const std::string &api_path) -> api_error {
 
   curl::requests::http_delete del{};
   del.allow_timeout = true;
-  del.path = "/api/bus/objects" + api_path;
+  del.path = "/api/bus/object" + api_path;
   del.query["bucket"] = get_bucket(get_sia_config());
 
   std::string error_data;
@@ -783,8 +829,9 @@ auto sia_provider::upload_file_impl(const std::string &api_path,
   REPERTORY_USES_FUNCTION_NAME();
 
   curl::requests::http_put_file put_file{};
-  put_file.path = "/api/worker/objects" + api_path;
+  put_file.path = "/api/worker/object" + api_path;
   put_file.query["bucket"] = get_bucket(get_sia_config());
+  put_file.headers["content-type"] = "application/octet-stream";
   put_file.source_path = source_path;
 
   std::string error_data;

repertory/repertory/include/cli/check_version.hpp

@@ -34,7 +34,7 @@ check_version(std::vector<const char *> /* args */,
   auto ret = exit_code::success;
 
   // TODO need to updated way to check version
-  // if (not((pt == provider_type::remote) || (pt == provider_type::s3))) {
+  // if (pt == provider_type::sia) {
   //   app_config config(pt, data_directory);
   //   curl_comm comm(config);
   //   json data, err;

repertory/repertory_test/include/mocks/mock_provider.hpp

@@ -36,6 +36,10 @@ private:
   const bool allow_rename_;
 
 public:
+  MOCK_METHOD(bool, check_version,
+              (std::string & required_version, std::string &returned_version),
+              (const, override));
+
   MOCK_METHOD(api_error, create_directory,
               (const std::string &api_path, api_meta_map &meta), (override));