BlockStorage/repertory
1
0
Fork 0
Code Issues 3 Pull Requests Releases 9 Wiki Activity

v2.0.4-rc (#37)
All checks were successful
BlockStorage/repertory/pipeline/head This commit looks good
Details

Browse Source 
# Changelog

## v2.0.4-rc

### BREAKING CHANGES

* `renterd` v2.0.0+ is now required. Prior versions will fail to mount.

### Issues

* \#35 [bug] Low frequency check is set to '0' instead of 1 hour by default
* \#36 [bug] Max cache size bytes is set to '0' by default

### Changes from v2.0.3-rc

* Added Sia API version check prior to mounting
* Added back `-cv` (check version) CLI option
* Continue documentation updates
* Fixed setting `ApiAuth` via `set_value_by_name`
* Fixed setting `HostConfig.ApiUser` via `set_value_by_name`
* Fixed setting `HostConfig.Path` via `set_value_by_name`
* Fixed setting `HostConfig.Protocol` via `set_value_by_name`
* Improved ring buffer read-ahead
* Integrated `renterd` version 2.0.0
* Prefer using local cache file when opening files
* Refactored `app_config` unit tests
* Refactored polling to be more accurate on scheduling tasks

Reviewed-on: #37
This commit is contained in:
sgraves76
2025-02-22 09:48:00 -06:00
parent fa439c634f
commit 24418ba03d
48 changed files with 2141 additions and 1200 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
repertory/librepertory/include/app_config.hpp
View File

@@ -31,26 +31,26 @@ private:
static stop_type stop_requested;
public:
[[nodiscard]] static auto
default_agent_name(const provider_type &prov) -> std::string;
[[nodiscard]] static auto default_agent_name(const provider_type &prov)
-> std::string;
[[nodiscard]] static auto
default_api_port(const provider_type &prov) -> std::uint16_t;
[[nodiscard]] static auto default_api_port(const provider_type &prov)
-> std::uint16_t;
[[nodiscard]] static auto
default_data_directory(const provider_type &prov) -> std::string;
[[nodiscard]] static auto default_data_directory(const provider_type &prov)
-> std::string;
[[nodiscard]] static auto
default_remote_api_port(const provider_type &prov) -> std::uint16_t;
[[nodiscard]] static auto default_remote_api_port(const provider_type &prov)
-> std::uint16_t;
[[nodiscard]] static auto
default_rpc_port(const provider_type &prov) -> std::uint16_t;
[[nodiscard]] static auto default_rpc_port(const provider_type &prov)
-> std::uint16_t;
[[nodiscard]] static auto
get_provider_display_name(const provider_type &prov) -> std::string;
[[nodiscard]] static auto get_provider_display_name(const provider_type &prov)
-> std::string;
[[nodiscard]] static auto
get_provider_name(const provider_type &prov) -> std::string;
[[nodiscard]] static auto get_provider_name(const provider_type &prov)
-> std::string;
public:
[[nodiscard]] static auto get_stop_requested() -> bool;
@@ -91,7 +91,6 @@ private:
std::atomic<std::uint8_t> max_upload_count_;
std::atomic<std::uint16_t> med_freq_interval_secs_;
std::atomic<std::uint16_t> online_check_retry_secs_;
std::atomic<std::uint16_t> orphaned_file_retention_days_;
std::atomic<download_type> preferred_download_type_;
std::atomic<std::uint16_t> retry_read_count_;
std::atomic<std::uint16_t> ring_buffer_file_size_;
@@ -172,8 +171,6 @@ public:
[[nodiscard]] auto get_online_check_retry_secs() const -> std::uint16_t;
[[nodiscard]] auto get_orphaned_file_retention_days() const -> std::uint16_t;
[[nodiscard]] auto get_preferred_download_type() const -> download_type;
[[nodiscard]] auto get_provider_type() const -> provider_type;
@@ -192,8 +189,11 @@ public:
[[nodiscard]] auto get_task_wait_ms() const -> std::uint16_t;
[[nodiscard]] auto
get_value_by_name(const std::string &name) const -> std::string;
[[nodiscard]] auto get_value_by_name(const std::string &name) const
-> std::string;
[[nodiscard]] auto get_raw_value_by_name(const std::string &name) const
-> std::string;
[[nodiscard]] auto get_version() const -> std::uint64_t;
@@ -239,8 +239,6 @@ public:
void set_online_check_retry_secs(std::uint16_t value);
void set_orphaned_file_retention_days(std::uint16_t value);
void set_preferred_download_type(const download_type &value);
void set_remote_config(remote::remote_config value);

59
repertory/librepertory/include/common.hpp
View File

@@ -57,7 +57,7 @@ using json = nlohmann::json;
inline constexpr const std::string_view REPERTORY = "repertory";
inline constexpr const std::wstring_view REPERTORY_W = L"repertory";
inline constexpr const std::uint64_t REPERTORY_CONFIG_VERSION = 0ULL;
inline constexpr const std::uint64_t REPERTORY_CONFIG_VERSION = 1ULL;
inline constexpr const std::string_view REPERTORY_DATA_NAME = "repertory2";
inline constexpr const std::string_view REPERTORY_MIN_REMOTE_VERSION = "2.0.0";
@@ -221,44 +221,25 @@ using WCHAR = wchar_t;
#define MAX_PATH 260
#define STATUS_SUCCESS \
std::uint32_t { 0U }
#define STATUS_ACCESS_DENIED \
std::uint32_t { 0xC0000022L }
#define STATUS_DEVICE_BUSY \
std::uint32_t { 0x80000011L }
#define STATUS_DEVICE_INSUFFICIENT_RESOURCES \
std::uint32_t { 0xC0000468L }
#define STATUS_DIRECTORY_NOT_EMPTY \
std::uint32_t { 0xC0000101L }
#define STATUS_FILE_IS_A_DIRECTORY \
std::uint32_t { 0xC00000BAL }
#define STATUS_FILE_TOO_LARGE \
std::uint32_t { 0xC0000904L }
#define STATUS_INSUFFICIENT_RESOURCES \
std::uint32_t { 0xC000009AL }
#define STATUS_INTERNAL_ERROR \
std::uint32_t { 0xC00000E5L }
#define STATUS_INVALID_ADDRESS \
std::uint32_t { 0xC0000141L }
#define STATUS_INVALID_HANDLE \
std::uint32_t { 0xC0000006L }
#define STATUS_INVALID_IMAGE_FORMAT \
std::uint32_t { 0xC000007BL }
#define STATUS_INVALID_PARAMETER \
std::uint32_t { 0xC000000DL }
#define STATUS_NO_MEMORY \
std::uint32_t { 0xC0000017L }
#define STATUS_NOT_IMPLEMENTED \
std::uint32_t { 0xC0000002L }
#define STATUS_OBJECT_NAME_EXISTS \
std::uint32_t { 0x40000000L }
#define STATUS_OBJECT_NAME_NOT_FOUND \
std::uint32_t { 0xC0000034L }
#define STATUS_OBJECT_PATH_INVALID \
std::uint32_t { 0xC0000039L }
#define STATUS_UNEXPECTED_IO_ERROR \
std::uint32_t { 0xC00000E9L }
#define STATUS_SUCCESS std::uint32_t{0U}
#define STATUS_ACCESS_DENIED std::uint32_t{0xC0000022L}
#define STATUS_DEVICE_BUSY std::uint32_t{0x80000011L}
#define STATUS_DEVICE_INSUFFICIENT_RESOURCES std::uint32_t{0xC0000468L}
#define STATUS_DIRECTORY_NOT_EMPTY std::uint32_t{0xC0000101L}
#define STATUS_FILE_IS_A_DIRECTORY std::uint32_t{0xC00000BAL}
#define STATUS_FILE_TOO_LARGE std::uint32_t{0xC0000904L}
#define STATUS_INSUFFICIENT_RESOURCES std::uint32_t{0xC000009AL}
#define STATUS_INTERNAL_ERROR std::uint32_t{0xC00000E5L}
#define STATUS_INVALID_ADDRESS std::uint32_t{0xC0000141L}
#define STATUS_INVALID_HANDLE std::uint32_t{0xC0000006L}
#define STATUS_INVALID_IMAGE_FORMAT std::uint32_t{0xC000007BL}
#define STATUS_INVALID_PARAMETER std::uint32_t{0xC000000DL}
#define STATUS_NO_MEMORY std::uint32_t{0xC0000017L}
#define STATUS_NOT_IMPLEMENTED std::uint32_t{0xC0000002L}
#define STATUS_OBJECT_NAME_EXISTS std::uint32_t{0x40000000L}
#define STATUS_OBJECT_NAME_NOT_FOUND std::uint32_t{0xC0000034L}
#define STATUS_OBJECT_PATH_INVALID std::uint32_t{0xC0000039L}
#define STATUS_UNEXPECTED_IO_ERROR std::uint32_t{0xC00000E9L}
#define CONVERT_STATUS_NOT_IMPLEMENTED(e) \
((std::uint32_t(e) == STATUS_NOT_IMPLEMENTED) ? -ENOTSUP : e)

1
repertory/librepertory/include/drives/winfsp/remotewinfsp/remote_client.hpp
View File

@@ -26,6 +26,7 @@
#include "comm/packet/packet_client.hpp"
#include "drives/remote/remote_open_file_table.hpp"
#include "drives/winfsp/remotewinfsp/i_remote_instance.hpp"
#include "types/remote.hpp"
namespace repertory {
class app_config;

78
repertory/librepertory/include/events/types/provider_invalid_version.hpp Normal file
View File

@@ -0,0 +1,78 @@
/*
Copyright <2018-2025> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#ifndef REPERTORY_INCLUDE_EVENTS_TYPES_PRODIVER_INVALID_VERSION_HPP_
#define REPERTORY_INCLUDE_EVENTS_TYPES_PRODIVER_INVALID_VERSION_HPP_
#include "events/i_event.hpp"
#include "types/repertory.hpp"
namespace repertory {
struct provider_invalid_version final : public i_event {
provider_invalid_version() = default;
provider_invalid_version(std::string_view function_name_,
std::string required_version_,
std::string returned_version_)
: function_name(std::string(function_name_)),
required_version(std::move(required_version_)),
returned_version(std::move(returned_version_)) {}
static constexpr const event_level level{event_level::error};
static constexpr const std::string_view name{"provider_invalid_version"};
std::string function_name;
std::string required_version;
std::string returned_version;
[[nodiscard]] auto get_event_level() const -> event_level override {
return level;
}
[[nodiscard]] auto get_name() const -> std::string_view override {
return name;
}
[[nodiscard]] auto get_single_line() const -> std::string override {
return fmt::format("{}|func|{}|required|{}|returned|{}", name,
function_name, required_version, returned_version);
}
};
} // namespace repertory
NLOHMANN_JSON_NAMESPACE_BEGIN
template <> struct adl_serializer<repertory::provider_invalid_version> {
static void to_json(json &data,
const repertory::provider_invalid_version &value) {
data["function_name"] = value.function_name;
data["required_version"] = value.required_version;
data["returned_version"] = value.returned_version;
}
static void from_json(const json &data,
repertory::provider_invalid_version &value) {
data.at("function_name").get_to(value.function_name);
data.at("required_version").get_to(value.required_version);
data.at("returned_version").get_to(value.returned_version);
}
};
NLOHMANN_JSON_NAMESPACE_END
#endif // REPERTORY_INCLUDE_EVENTS_TYPES_PRODIVER_INVALID_VERSION_HPP_

7
repertory/librepertory/include/providers/encrypt/encrypt_provider.hpp
View File

@@ -85,6 +85,13 @@ private:
void remove_deleted_files(stop_type &stop_requested);
public:
[[nodiscard]] auto check_version(std::string &required_version,
std::string &returned_version) const
-> bool override {
required_version = returned_version = "";
return true;
}
[[nodiscard]] auto create_directory(const std::string &api_path,
api_meta_map &meta) -> api_error override;

4
repertory/librepertory/include/providers/i_provider.hpp
View File

@@ -31,6 +31,10 @@ class i_provider {
INTERFACE_SETUP(i_provider);
public:
[[nodiscard]] virtual auto check_version(std::string &required_version,
std::string &returned_version) const
-> bool = 0;
[[nodiscard]] virtual auto create_directory(const std::string &api_path,
api_meta_map &meta)
-> api_error = 0;

7
repertory/librepertory/include/providers/s3/s3_provider.hpp
View File

@@ -113,6 +113,13 @@ protected:
-> api_error override;
public:
[[nodiscard]] auto check_version(std::string &required_version,
std::string &returned_version) const
-> bool override {
required_version = returned_version = "";
return true;
}
[[nodiscard]] static auto convert_api_date(std::string_view date)
-> std::uint64_t;

4
repertory/librepertory/include/providers/sia/sia_provider.hpp
View File

@@ -80,6 +80,10 @@ protected:
-> api_error override;
public:
[[nodiscard]] auto check_version(std::string &required_version,
std::string &returned_version) const
-> bool override;
[[nodiscard]] auto get_directory_item_count(const std::string &api_path) const
-> std::uint64_t override;

52
repertory/librepertory/include/types/repertory.hpp
View File

@@ -24,30 +24,27 @@
namespace repertory {
constexpr const auto default_api_auth_size{48U};
constexpr const auto default_download_timeout_ces{30U};
constexpr const auto default_download_timeout_secs{30U};
constexpr const auto default_eviction_delay_mins{1U};
constexpr const auto default_high_freq_interval_secs{30U};
constexpr const auto default_low_freq_interval_secs{0U * 60U};
constexpr const auto default_high_freq_interval_secs{std::uint16_t{30U}};
constexpr const auto default_low_freq_interval_secs{std::uint16_t(60U * 60U)};
constexpr const auto default_max_cache_size_bytes{
std::uint64_t(20UL * 1024UL * 1024UL * 1024UL),
std::uint64_t(20ULL * 1024ULL * 1024ULL * 1024ULL),
};
constexpr const auto default_max_upload_count{5U};
constexpr const auto default_med_freq_interval_secs{2U * 60U};
constexpr const auto default_med_freq_interval_secs{std::uint16_t{2U * 60U}};
constexpr const auto default_online_check_retry_secs{60U};
constexpr const auto default_orphaned_file_retention_days{15U};
constexpr const auto default_retry_read_count{6U};
constexpr const auto default_ring_buffer_file_size{512U};
constexpr const auto default_task_wait_ms{100U};
constexpr const auto default_timeout_ms{60000U};
constexpr const auto max_orphaned_file_retention_days{std::uint16_t(31U)};
constexpr const auto max_ring_buffer_file_size{std::uint16_t(1024U)};
constexpr const auto max_s3_object_name_length{1024U};
constexpr const auto min_cache_size_bytes{
std::uint64_t(100UL * 1024UL * 1024UL),
std::uint64_t(100ULL * 1024ULL * 1024ULL),
};
constexpr const auto min_download_timeout_secs{std::uint8_t(5U)};
constexpr const auto min_online_check_retry_secs{std::uint16_t(15U)};
constexpr const auto min_orphaned_file_retention_days{std::uint16_t(1U)};
constexpr const auto min_retry_read_count{std::uint16_t(2U)};
constexpr const auto min_ring_buffer_file_size{std::uint16_t(64U)};
constexpr const auto min_task_wait_ms{std::uint16_t(50U)};
@@ -333,7 +330,6 @@ struct directory_item final {
bool directory{false};
std::uint64_t size{};
api_meta_map meta;
bool resolved{false};
};
struct encrypt_config final {
@@ -493,8 +489,6 @@ inline constexpr const auto JSON_MED_FREQ_INTERVAL_SECS{
inline constexpr const auto JSON_META{"Meta"};
inline constexpr const auto JSON_ONLINE_CHECK_RETRY_SECS{
"OnlineCheckRetrySeconds"};
inline constexpr const auto JSON_ORPHANED_FILE_RETENTION_DAYS{
"OrphanedFileRetentionDays"};
inline constexpr const auto JSON_PATH{"Path"};
inline constexpr const auto JSON_PREFERRED_DOWNLOAD_TYPE{
"PreferredDownloadType"};
@@ -619,6 +613,16 @@ template <typename data_t> struct adl_serializer<repertory::atomic<data_t>> {
}
};
template <> struct adl_serializer<std::atomic<std::uint64_t>> {
static void to_json(json &data, const std::atomic<std::uint64_t> &value) {
data = value.load();
}
static void from_json(const json &data, std::atomic<std::uint64_t> &value) {
value.store(data.get<std::uint64_t>());
}
};
template <typename primitive_t>
struct adl_serializer<std::atomic<primitive_t>> {
static void to_json(json &data, const std::atomic<primitive_t> &value) {
@@ -642,6 +646,18 @@ template <> struct adl_serializer<std::atomic<repertory::database_type>> {
}
};
template <> struct adl_serializer<std::atomic<repertory::event_level>> {
static void to_json(json &data,
const std::atomic<repertory::event_level> &value) {
data = repertory::event_level_to_string(value.load());
}
static void from_json(const json &data,
std::atomic<repertory::event_level> &value) {
value.store(repertory::event_level_from_string(data.get<std::string>()));
}
};
template <> struct adl_serializer<std::atomic<repertory::download_type>> {
static void to_json(json &data,
const std::atomic<repertory::download_type> &value) {
@@ -674,15 +690,13 @@ template <> struct adl_serializer<repertory::download_type> {
}
};
template <> struct adl_serializer<std::atomic<repertory::event_level>> {
static void to_json(json &data,
const std::atomic<repertory::event_level> &value) {
data = repertory::event_level_to_string(value.load());
template <> struct adl_serializer<repertory::event_level> {
static void to_json(json &data, const repertory::event_level &value) {
data = repertory::event_level_to_string(value);
}
static void from_json(const json &data,
std::atomic<repertory::event_level> &value) {
value.store(repertory::event_level_from_string(data.get<std::string>()));
static void from_json(const json &data, repertory::event_level &value) {
value = repertory::event_level_from_string(data.get<std::string>());
}
};
NLOHMANN_JSON_NAMESPACE_END