From 66b6f581f94576fa98909ad57655709439d53734 Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Fri, 21 Feb 2025 13:58:30 -0600 Subject: [PATCH] sia 2.0.0 support --- CHANGELOG.md | 6 + README.md | 2 +- .../events/types/provider_invalid_version.hpp | 78 ++++++++++++ .../providers/encrypt/encrypt_provider.hpp | 7 ++ .../include/providers/i_provider.hpp | 4 + .../include/providers/s3/s3_provider.hpp | 7 ++ .../include/providers/sia/sia_provider.hpp | 4 + .../src/providers/base_provider.cpp | 9 ++ .../src/providers/sia/sia_provider.cpp | 119 ++++++++++++------ .../repertory/include/cli/check_version.hpp | 2 +- .../include/mocks/mock_provider.hpp | 4 + 11 files changed, 204 insertions(+), 38 deletions(-) create mode 100644 repertory/librepertory/include/events/types/provider_invalid_version.hpp diff --git a/CHANGELOG.md b/CHANGELOG.md index 0589e073..8b8677d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## v2.0.4-rc +### BREAKING CHANGES + +* `renterd` v2.0.x is now required. Prior versions will fail to mount. + ### Issues * ~~\#12 [Unit Test] Complete all providers unit tests~~ @@ -12,11 +16,13 @@ ### Changes from v2.0.3-rc +* Added Sia API version check prior to mounting * Continue documentation updates * Fixed setting `ApiAuth` via `set_value_by_name` * Fixed setting `HostConfig.ApiUser` via `set_value_by_name` * Fixed setting `HostConfig.Path` via `set_value_by_name` * Fixed setting `HostConfig.Protocol` via `set_value_by_name` +* Integrated `renterd` version 2.x * Refactored `app_config` unit tests * Refactored polling to be more accurate on scheduling tasks diff --git a/README.md b/README.md index e4f83697..b78006c3 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ on Windows. ## Minimum Requirements -* [Sia renterd](https://github.com/SiaFoundation/renterd/releases) v0.4.0+ for Sia support +* [Sia renterd](https://github.com/SiaFoundation/renterd/releases) v2.0.0+ for Sia support * Only 64-bit operating systems are supported * By default, Linux requires `fusermount3`; otherwise, `repertory` must be manually compiled with `libfuse2` support * Windows requires the following dependencies to be installed: diff --git a/repertory/librepertory/include/events/types/provider_invalid_version.hpp b/repertory/librepertory/include/events/types/provider_invalid_version.hpp new file mode 100644 index 00000000..4b535868 --- /dev/null +++ b/repertory/librepertory/include/events/types/provider_invalid_version.hpp @@ -0,0 +1,78 @@ +/* + Copyright <2018-2025> + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ +#ifndef REPERTORY_INCLUDE_EVENTS_TYPES_PRODIVER_INVALID_VERSION_HPP_ +#define REPERTORY_INCLUDE_EVENTS_TYPES_PRODIVER_INVALID_VERSION_HPP_ + +#include "events/i_event.hpp" +#include "types/repertory.hpp" + +namespace repertory { +struct provider_invalid_version final : public i_event { + provider_invalid_version() = default; + provider_invalid_version(std::string_view function_name_, + std::string required_version_, + std::string returned_version_) + : function_name(std::string(function_name_)), + required_version(std::move(required_version_)), + returned_version(std::move(returned_version_)) {} + + static constexpr const event_level level{event_level::error}; + static constexpr const std::string_view name{"provider_invalid_version"}; + + std::string function_name; + std::string required_version; + std::string returned_version; + + [[nodiscard]] auto get_event_level() const -> event_level override { + return level; + } + + [[nodiscard]] auto get_name() const -> std::string_view override { + return name; + } + + [[nodiscard]] auto get_single_line() const -> std::string override { + return fmt::format("{}|func|{}|required|{}|returned|{}", name, + function_name, required_version, returned_version); + } +}; +} // namespace repertory + +NLOHMANN_JSON_NAMESPACE_BEGIN +template <> struct adl_serializer { + static void to_json(json &data, + const repertory::provider_invalid_version &value) { + data["function_name"] = value.function_name; + data["required_version"] = value.required_version; + data["returned_version"] = value.returned_version; + } + + static void from_json(const json &data, + repertory::provider_invalid_version &value) { + data.at("function_name").get_to(value.function_name); + data.at("required_version").get_to(value.required_version); + data.at("returned_version").get_to(value.returned_version); + } +}; +NLOHMANN_JSON_NAMESPACE_END + +#endif // REPERTORY_INCLUDE_EVENTS_TYPES_PRODIVER_INVALID_VERSION_HPP_ diff --git a/repertory/librepertory/include/providers/encrypt/encrypt_provider.hpp b/repertory/librepertory/include/providers/encrypt/encrypt_provider.hpp index db4b4a3e..984f8500 100644 --- a/repertory/librepertory/include/providers/encrypt/encrypt_provider.hpp +++ b/repertory/librepertory/include/providers/encrypt/encrypt_provider.hpp @@ -85,6 +85,13 @@ private: void remove_deleted_files(stop_type &stop_requested); public: + [[nodiscard]] auto check_version(std::string &required_version, + std::string &returned_version) const + -> bool override { + required_version = returned_version = ""; + return true; + } + [[nodiscard]] auto create_directory(const std::string &api_path, api_meta_map &meta) -> api_error override; diff --git a/repertory/librepertory/include/providers/i_provider.hpp b/repertory/librepertory/include/providers/i_provider.hpp index 7e4836da..d1b5f616 100644 --- a/repertory/librepertory/include/providers/i_provider.hpp +++ b/repertory/librepertory/include/providers/i_provider.hpp @@ -31,6 +31,10 @@ class i_provider { INTERFACE_SETUP(i_provider); public: + [[nodiscard]] virtual auto check_version(std::string &required_version, + std::string &returned_version) const + -> bool = 0; + [[nodiscard]] virtual auto create_directory(const std::string &api_path, api_meta_map &meta) -> api_error = 0; diff --git a/repertory/librepertory/include/providers/s3/s3_provider.hpp b/repertory/librepertory/include/providers/s3/s3_provider.hpp index 20804914..4d9f1f04 100644 --- a/repertory/librepertory/include/providers/s3/s3_provider.hpp +++ b/repertory/librepertory/include/providers/s3/s3_provider.hpp @@ -113,6 +113,13 @@ protected: -> api_error override; public: + [[nodiscard]] auto check_version(std::string &required_version, + std::string &returned_version) const + -> bool override { + required_version = returned_version = ""; + return true; + } + [[nodiscard]] static auto convert_api_date(std::string_view date) -> std::uint64_t; diff --git a/repertory/librepertory/include/providers/sia/sia_provider.hpp b/repertory/librepertory/include/providers/sia/sia_provider.hpp index d4a07990..cdde708e 100644 --- a/repertory/librepertory/include/providers/sia/sia_provider.hpp +++ b/repertory/librepertory/include/providers/sia/sia_provider.hpp @@ -80,6 +80,10 @@ protected: -> api_error override; public: + [[nodiscard]] auto check_version(std::string &required_version, + std::string &returned_version) const + -> bool override; + [[nodiscard]] auto get_directory_item_count(const std::string &api_path) const -> std::uint64_t override; diff --git a/repertory/librepertory/src/providers/base_provider.cpp b/repertory/librepertory/src/providers/base_provider.cpp index 6fb37615..432f045d 100644 --- a/repertory/librepertory/src/providers/base_provider.cpp +++ b/repertory/librepertory/src/providers/base_provider.cpp @@ -34,6 +34,7 @@ #include "events/types/orphaned_file_processing_failed.hpp" #include "events/types/orphaned_source_file_detected.hpp" #include "events/types/orphaned_source_file_removed.hpp" +#include "events/types/provider_invalid_version.hpp" #include "events/types/provider_offline.hpp" #include "events/types/provider_upload_begin.hpp" #include "events/types/provider_upload_end.hpp" @@ -840,6 +841,14 @@ auto base_provider::start(api_item_added_callback api_item_added, return false; } + std::string returned_version; + std::string required_version; + if (not check_version(required_version, returned_version)) { + event_system::instance().raise( + function_name, required_version, returned_version); + return false; + } + cache_size_mgr::instance().initialize(&config_); polling::instance().set_callback({ diff --git a/repertory/librepertory/src/providers/sia/sia_provider.cpp b/repertory/librepertory/src/providers/sia/sia_provider.cpp index a3a6db1d..9b144694 100644 --- a/repertory/librepertory/src/providers/sia/sia_provider.cpp +++ b/repertory/librepertory/src/providers/sia/sia_provider.cpp @@ -32,6 +32,7 @@ #include "providers/base_provider.hpp" #include "providers/s3/s3_provider.hpp" #include "types/repertory.hpp" +#include "utils/common.hpp" #include "utils/error_utils.hpp" #include "utils/file_utils.hpp" #include "utils/path.hpp" @@ -63,6 +64,52 @@ namespace repertory { sia_provider::sia_provider(app_config &config, i_http_comm &comm) : base_provider(config, comm) {} +auto sia_provider::check_version(std::string &required_version, + std::string &returned_version) const -> bool { + REPERTORY_USES_FUNCTION_NAME(); + + required_version = "2.0.0"; + + try { + curl::requests::http_get get{}; + get.allow_timeout = true; + get.path = "/api/bus/state"; + + nlohmann::json state_data; + std::string error_data; + get.response_handler = [&error_data, &state_data](auto &&data, + long response_code) { + if (response_code == http_error_codes::ok) { + state_data = nlohmann::json::parse(data.begin(), data.end()); + return; + } + + error_data = std::string(data.begin(), data.end()); + }; + + long response_code{}; + stop_type stop_requested{}; + if (not get_comm().make_request(get, response_code, stop_requested)) { + return false; + } + + if (response_code != http_error_codes::ok) { + utils::error::raise_error( + function_name, response_code, + fmt::format("failed to check state|response|{}", error_data)); + return false; + } + + returned_version = state_data.at("version").get().substr(1U); + return utils::compare_version_strings(returned_version, required_version) >= + 0; + } catch (const std::exception &e) { + utils::error::raise_error(function_name, e, "failed to check version"); + } + + return false; +} + auto sia_provider::create_directory_impl(const std::string &api_path, api_meta_map & /* meta */) -> api_error { @@ -70,7 +117,7 @@ auto sia_provider::create_directory_impl(const std::string &api_path, curl::requests::http_put_file put_file{}; put_file.allow_timeout = true; - put_file.path = "/api/worker/objects" + api_path + "/"; + put_file.path = "/api/worker/object" + api_path + "/"; put_file.query["bucket"] = get_bucket(get_sia_config()); std::string error_data; @@ -112,10 +159,10 @@ auto sia_provider::get_directory_item_count(const std::string &api_path) const } std::uint64_t item_count{}; - if (object_list.contains("entries")) { - for (const auto &entry : object_list.at("entries")) { + if (object_list.contains("objects")) { + for (const auto &entry : object_list.at("objects")) { try { - auto name{entry.at("name").get()}; + auto name{entry.at("key").get()}; auto entry_api_path{utils::path::create_api_path(name)}; if (utils::string::ends_with(name, "/") && (entry_api_path == api_path)) { @@ -149,10 +196,10 @@ auto sia_provider::get_directory_items_impl(const std::string &api_path, return api_error::comm_error; } - if (object_list.contains("entries")) { - for (const auto &entry : object_list.at("entries")) { + if (object_list.contains("objects")) { + for (const auto &entry : object_list.at("objects")) { try { - auto name{entry.at("name").get()}; + auto name{entry.at("key").get()}; auto entry_api_path{utils::path::create_api_path(name)}; auto directory{utils::string::ends_with(name, "/")}; @@ -222,20 +269,12 @@ auto sia_provider::get_file(const std::string &api_path, api_file &file) const } auto size{ - file_data["object"].contains("Slabs") - ? std::accumulate( - file_data["object"]["Slabs"].begin(), - file_data["object"]["Slabs"].end(), std::uint64_t(0U), - [](auto &&total_size, const json &slab) -> std::uint64_t { - return total_size + slab["Length"].get(); - }) - : file_data["object"]["size"].get(), + file_data.at("size").get(), }; api_meta_map meta{}; if (get_item_meta(api_path, meta) == api_error::item_not_found) { - file = create_api_file(api_path, "", size, - get_last_modified(file_data["object"])); + file = create_api_file(api_path, "", size, get_last_modified(file_data)); get_api_item_added()(false, file); } else { file = create_api_file(api_path, size, meta); @@ -263,9 +302,9 @@ auto sia_provider::get_file_list(api_file_list &list, return api_error::comm_error; } - if (object_list.contains("entries")) { - for (const auto &entry : object_list.at("entries")) { - auto name{entry.at("name").get()}; + if (object_list.contains("objects")) { + for (const auto &entry : object_list.at("objects")) { + auto name{entry.at("key").get()}; auto entry_api_path{utils::path::create_api_path(name)}; if (utils::string::ends_with(name, "/")) { @@ -326,8 +365,9 @@ auto sia_provider::get_object_info(const std::string &api_path, try { curl::requests::http_get get{}; get.allow_timeout = true; - get.path = "/api/bus/objects" + api_path; + get.path = "/api/bus/object" + api_path; get.query["bucket"] = get_bucket(get_sia_config()); + get.query["onlymetadata"] = "true"; std::string error_data; get.response_handler = [&error_data, &object_info](auto &&data, @@ -375,6 +415,7 @@ auto sia_provider::get_object_list(const std::string &api_path, get.allow_timeout = true; get.path = "/api/bus/objects" + api_path + "/"; get.query["bucket"] = get_bucket(get_sia_config()); + get.query["delimiter"] = "/"; std::string error_data; get.response_handler = [&error_data, &object_list](auto &&data, @@ -418,7 +459,7 @@ auto sia_provider::get_total_drive_space() const -> std::uint64_t { try { curl::requests::http_get get{}; get.allow_timeout = true; - get.path = "/api/autopilot/config"; + get.path = "/api/bus/autopilot"; get.query["bucket"] = get_bucket(get_sia_config()); json config_data; @@ -468,17 +509,18 @@ auto sia_provider::is_directory(const std::string &api_path, bool &exists) const exists = false; - json object_list{}; - if (not get_object_list(utils::path::get_parent_api_path(api_path), - object_list)) { - return api_error::comm_error; + json file_data{}; + auto res{get_object_info(api_path + '/', file_data)}; + if (res == api_error::item_not_found) { + return api_error::success; } - exists = object_list.contains("entries") && - std::ranges::find_if(object_list.at("entries"), - [&api_path](auto &&entry) -> bool { - return entry.at("name") == (api_path + "/"); - }) != object_list.at("entries").end(); + if (res != api_error::success) { + return res; + } + + exists = + utils::string::ends_with(file_data.at("key").get(), "/"); return api_error::success; } catch (const std::exception &e) { utils::error::raise_api_path_error( @@ -494,6 +536,7 @@ auto sia_provider::is_file(const std::string &api_path, bool &exists) const try { exists = false; + if (api_path == "/") { return api_error::success; } @@ -508,7 +551,8 @@ auto sia_provider::is_file(const std::string &api_path, bool &exists) const return res; } - exists = not file_data.contains("entries"); + exists = not utils::string::ends_with( + file_data.at("key").get(), "/"); return api_error::success; } catch (const std::exception &e) { utils::error::raise_api_path_error(function_name, api_path, e, @@ -572,8 +616,9 @@ auto sia_provider::read_file_bytes(const std::string &api_path, try { curl::requests::http_get get{}; - get.path = "/api/worker/objects" + api_path; + get.path = "/api/worker/object" + api_path; get.query["bucket"] = get_bucket(get_sia_config()); + get.headers["accept"] = "application/octet-stream"; get.range = {{ offset, offset + size - 1U, @@ -590,6 +635,7 @@ auto sia_provider::read_file_bytes(const std::string &api_path, ++idx) { long response_code{}; const auto notify_retry = [&]() { + fmt::println("{}", std::string(buffer.begin(), buffer.end())); if (response_code == 0) { utils::error::raise_api_path_error( function_name, api_path, api_error::comm_error, @@ -635,7 +681,7 @@ auto sia_provider::remove_directory_impl(const std::string &api_path) curl::requests::http_delete del{}; del.allow_timeout = true; - del.path = "/api/bus/objects" + api_path + "/"; + del.path = "/api/bus/object" + api_path + "/"; del.query["bucket"] = get_bucket(get_sia_config()); std::string error_data; @@ -671,7 +717,7 @@ auto sia_provider::remove_file_impl(const std::string &api_path) -> api_error { curl::requests::http_delete del{}; del.allow_timeout = true; - del.path = "/api/bus/objects" + api_path; + del.path = "/api/bus/object" + api_path; del.query["bucket"] = get_bucket(get_sia_config()); std::string error_data; @@ -783,8 +829,9 @@ auto sia_provider::upload_file_impl(const std::string &api_path, REPERTORY_USES_FUNCTION_NAME(); curl::requests::http_put_file put_file{}; - put_file.path = "/api/worker/objects" + api_path; + put_file.path = "/api/worker/object" + api_path; put_file.query["bucket"] = get_bucket(get_sia_config()); + put_file.headers["content-type"] = "application/octet-stream"; put_file.source_path = source_path; std::string error_data; diff --git a/repertory/repertory/include/cli/check_version.hpp b/repertory/repertory/include/cli/check_version.hpp index cc2d0ab4..bb87edf9 100644 --- a/repertory/repertory/include/cli/check_version.hpp +++ b/repertory/repertory/include/cli/check_version.hpp @@ -34,7 +34,7 @@ check_version(std::vector /* args */, auto ret = exit_code::success; // TODO need to updated way to check version - // if (not((pt == provider_type::remote) || (pt == provider_type::s3))) { + // if (pt == provider_type::sia) { // app_config config(pt, data_directory); // curl_comm comm(config); // json data, err; diff --git a/repertory/repertory_test/include/mocks/mock_provider.hpp b/repertory/repertory_test/include/mocks/mock_provider.hpp index ca8e28ba..98b2a05a 100644 --- a/repertory/repertory_test/include/mocks/mock_provider.hpp +++ b/repertory/repertory_test/include/mocks/mock_provider.hpp @@ -36,6 +36,10 @@ private: const bool allow_rename_; public: + MOCK_METHOD(bool, check_version, + (std::string & required_version, std::string &returned_version), + (const, override)); + MOCK_METHOD(api_error, create_directory, (const std::string &api_path, api_meta_map &meta), (override));