BlockStorage/repertory
1
0
Fork 0
Code Issues 8 Pull Requests Releases 6 Wiki Activity

v2.0.4-rc (#37)
All checks were successful
BlockStorage/repertory/pipeline/head This commit looks good
Details

Browse Source 
# Changelog

## v2.0.4-rc

### BREAKING CHANGES

* `renterd` v2.0.0+ is now required. Prior versions will fail to mount.

### Issues

* \#35 [bug] Low frequency check is set to '0' instead of 1 hour by default
* \#36 [bug] Max cache size bytes is set to '0' by default

### Changes from v2.0.3-rc

* Added Sia API version check prior to mounting
* Added back `-cv` (check version) CLI option
* Continue documentation updates
* Fixed setting `ApiAuth` via `set_value_by_name`
* Fixed setting `HostConfig.ApiUser` via `set_value_by_name`
* Fixed setting `HostConfig.Path` via `set_value_by_name`
* Fixed setting `HostConfig.Protocol` via `set_value_by_name`
* Improved ring buffer read-ahead
* Integrated `renterd` version 2.0.0
* Prefer using local cache file when opening files
* Refactored `app_config` unit tests
* Refactored polling to be more accurate on scheduling tasks

Reviewed-on: #37
This commit is contained in:
Scott E. Graves
2025-02-22 09:48:00 -06:00
parent fa439c634f
commit 24418ba03d
48 changed files with 2141 additions and 1200 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
repertory/librepertory/include/app_config.hpp
View File

@ -31,26 +31,26 @@ private:
static stop_type stop_requested;
public:
[[nodiscard]] static auto
default_agent_name(const provider_type &prov) -> std::string;
[[nodiscard]] static auto default_agent_name(const provider_type &prov)
-> std::string;
[[nodiscard]] static auto
default_api_port(const provider_type &prov) -> std::uint16_t;
[[nodiscard]] static auto default_api_port(const provider_type &prov)
-> std::uint16_t;
[[nodiscard]] static auto
default_data_directory(const provider_type &prov) -> std::string;
[[nodiscard]] static auto default_data_directory(const provider_type &prov)
-> std::string;
[[nodiscard]] static auto
default_remote_api_port(const provider_type &prov) -> std::uint16_t;
[[nodiscard]] static auto default_remote_api_port(const provider_type &prov)
-> std::uint16_t;
[[nodiscard]] static auto
default_rpc_port(const provider_type &prov) -> std::uint16_t;
[[nodiscard]] static auto default_rpc_port(const provider_type &prov)
-> std::uint16_t;
[[nodiscard]] static auto
get_provider_display_name(const provider_type &prov) -> std::string;
[[nodiscard]] static auto get_provider_display_name(const provider_type &prov)
-> std::string;
[[nodiscard]] static auto
get_provider_name(const provider_type &prov) -> std::string;
[[nodiscard]] static auto get_provider_name(const provider_type &prov)
-> std::string;
public:
[[nodiscard]] static auto get_stop_requested() -> bool;
@ -91,7 +91,6 @@ private:
std::atomic<std::uint8_t> max_upload_count_;
std::atomic<std::uint16_t> med_freq_interval_secs_;
std::atomic<std::uint16_t> online_check_retry_secs_;
std::atomic<std::uint16_t> orphaned_file_retention_days_;
std::atomic<download_type> preferred_download_type_;
std::atomic<std::uint16_t> retry_read_count_;
std::atomic<std::uint16_t> ring_buffer_file_size_;
@ -172,8 +171,6 @@ public:
[[nodiscard]] auto get_online_check_retry_secs() const -> std::uint16_t;
[[nodiscard]] auto get_orphaned_file_retention_days() const -> std::uint16_t;
[[nodiscard]] auto get_preferred_download_type() const -> download_type;
[[nodiscard]] auto get_provider_type() const -> provider_type;
@ -192,8 +189,11 @@ public:
[[nodiscard]] auto get_task_wait_ms() const -> std::uint16_t;
[[nodiscard]] auto
get_value_by_name(const std::string &name) const -> std::string;
[[nodiscard]] auto get_value_by_name(const std::string &name) const
-> std::string;
[[nodiscard]] auto get_raw_value_by_name(const std::string &name) const
-> std::string;
[[nodiscard]] auto get_version() const -> std::uint64_t;
@ -239,8 +239,6 @@ public:
void set_online_check_retry_secs(std::uint16_t value);
void set_orphaned_file_retention_days(std::uint16_t value);
void set_preferred_download_type(const download_type &value);
void set_remote_config(remote::remote_config value);

59
repertory/librepertory/include/common.hpp
View File

@ -57,7 +57,7 @@ using json = nlohmann::json;
inline constexpr const std::string_view REPERTORY = "repertory";
inline constexpr const std::wstring_view REPERTORY_W = L"repertory";
inline constexpr const std::uint64_t REPERTORY_CONFIG_VERSION = 0ULL;
inline constexpr const std::uint64_t REPERTORY_CONFIG_VERSION = 1ULL;
inline constexpr const std::string_view REPERTORY_DATA_NAME = "repertory2";
inline constexpr const std::string_view REPERTORY_MIN_REMOTE_VERSION = "2.0.0";
@ -221,44 +221,25 @@ using WCHAR = wchar_t;
#define MAX_PATH 260
#define STATUS_SUCCESS \
std::uint32_t { 0U }
#define STATUS_ACCESS_DENIED \
std::uint32_t { 0xC0000022L }
#define STATUS_DEVICE_BUSY \
std::uint32_t { 0x80000011L }
#define STATUS_DEVICE_INSUFFICIENT_RESOURCES \
std::uint32_t { 0xC0000468L }
#define STATUS_DIRECTORY_NOT_EMPTY \
std::uint32_t { 0xC0000101L }
#define STATUS_FILE_IS_A_DIRECTORY \
std::uint32_t { 0xC00000BAL }
#define STATUS_FILE_TOO_LARGE \
std::uint32_t { 0xC0000904L }
#define STATUS_INSUFFICIENT_RESOURCES \
std::uint32_t { 0xC000009AL }
#define STATUS_INTERNAL_ERROR \
std::uint32_t { 0xC00000E5L }
#define STATUS_INVALID_ADDRESS \
std::uint32_t { 0xC0000141L }
#define STATUS_INVALID_HANDLE \
std::uint32_t { 0xC0000006L }
#define STATUS_INVALID_IMAGE_FORMAT \
std::uint32_t { 0xC000007BL }
#define STATUS_INVALID_PARAMETER \
std::uint32_t { 0xC000000DL }
#define STATUS_NO_MEMORY \
std::uint32_t { 0xC0000017L }
#define STATUS_NOT_IMPLEMENTED \
std::uint32_t { 0xC0000002L }
#define STATUS_OBJECT_NAME_EXISTS \
std::uint32_t { 0x40000000L }
#define STATUS_OBJECT_NAME_NOT_FOUND \
std::uint32_t { 0xC0000034L }
#define STATUS_OBJECT_PATH_INVALID \
std::uint32_t { 0xC0000039L }
#define STATUS_UNEXPECTED_IO_ERROR \
std::uint32_t { 0xC00000E9L }
#define STATUS_SUCCESS std::uint32_t{0U}
#define STATUS_ACCESS_DENIED std::uint32_t{0xC0000022L}
#define STATUS_DEVICE_BUSY std::uint32_t{0x80000011L}
#define STATUS_DEVICE_INSUFFICIENT_RESOURCES std::uint32_t{0xC0000468L}
#define STATUS_DIRECTORY_NOT_EMPTY std::uint32_t{0xC0000101L}
#define STATUS_FILE_IS_A_DIRECTORY std::uint32_t{0xC00000BAL}
#define STATUS_FILE_TOO_LARGE std::uint32_t{0xC0000904L}
#define STATUS_INSUFFICIENT_RESOURCES std::uint32_t{0xC000009AL}
#define STATUS_INTERNAL_ERROR std::uint32_t{0xC00000E5L}
#define STATUS_INVALID_ADDRESS std::uint32_t{0xC0000141L}
#define STATUS_INVALID_HANDLE std::uint32_t{0xC0000006L}
#define STATUS_INVALID_IMAGE_FORMAT std::uint32_t{0xC000007BL}
#define STATUS_INVALID_PARAMETER std::uint32_t{0xC000000DL}
#define STATUS_NO_MEMORY std::uint32_t{0xC0000017L}
#define STATUS_NOT_IMPLEMENTED std::uint32_t{0xC0000002L}
#define STATUS_OBJECT_NAME_EXISTS std::uint32_t{0x40000000L}
#define STATUS_OBJECT_NAME_NOT_FOUND std::uint32_t{0xC0000034L}
#define STATUS_OBJECT_PATH_INVALID std::uint32_t{0xC0000039L}
#define STATUS_UNEXPECTED_IO_ERROR std::uint32_t{0xC00000E9L}
#define CONVERT_STATUS_NOT_IMPLEMENTED(e) \
((std::uint32_t(e) == STATUS_NOT_IMPLEMENTED) ? -ENOTSUP : e)

1
repertory/librepertory/include/drives/winfsp/remotewinfsp/remote_client.hpp
View File

@ -26,6 +26,7 @@
#include "comm/packet/packet_client.hpp"
#include "drives/remote/remote_open_file_table.hpp"
#include "drives/winfsp/remotewinfsp/i_remote_instance.hpp"
#include "types/remote.hpp"
namespace repertory {
class app_config;

78
repertory/librepertory/include/events/types/provider_invalid_version.hpp Normal file
View File

@ -0,0 +1,78 @@
/*
Copyright <2018-2025> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#ifndef REPERTORY_INCLUDE_EVENTS_TYPES_PRODIVER_INVALID_VERSION_HPP_
#define REPERTORY_INCLUDE_EVENTS_TYPES_PRODIVER_INVALID_VERSION_HPP_
#include "events/i_event.hpp"
#include "types/repertory.hpp"
namespace repertory {
struct provider_invalid_version final : public i_event {
provider_invalid_version() = default;
provider_invalid_version(std::string_view function_name_,
std::string required_version_,
std::string returned_version_)
: function_name(std::string(function_name_)),
required_version(std::move(required_version_)),
returned_version(std::move(returned_version_)) {}
static constexpr const event_level level{event_level::error};
static constexpr const std::string_view name{"provider_invalid_version"};
std::string function_name;
std::string required_version;
std::string returned_version;
[[nodiscard]] auto get_event_level() const -> event_level override {
return level;
}
[[nodiscard]] auto get_name() const -> std::string_view override {
return name;
}
[[nodiscard]] auto get_single_line() const -> std::string override {
return fmt::format("{}|func|{}|required|{}|returned|{}", name,
function_name, required_version, returned_version);
}
};
} // namespace repertory
NLOHMANN_JSON_NAMESPACE_BEGIN
template <> struct adl_serializer<repertory::provider_invalid_version> {
static void to_json(json &data,
const repertory::provider_invalid_version &value) {
data["function_name"] = value.function_name;
data["required_version"] = value.required_version;
data["returned_version"] = value.returned_version;
}
static void from_json(const json &data,
repertory::provider_invalid_version &value) {
data.at("function_name").get_to(value.function_name);
data.at("required_version").get_to(value.required_version);
data.at("returned_version").get_to(value.returned_version);
}
};
NLOHMANN_JSON_NAMESPACE_END
#endif // REPERTORY_INCLUDE_EVENTS_TYPES_PRODIVER_INVALID_VERSION_HPP_

7
repertory/librepertory/include/providers/encrypt/encrypt_provider.hpp
View File

@ -85,6 +85,13 @@ private:
void remove_deleted_files(stop_type &stop_requested);
public:
[[nodiscard]] auto check_version(std::string &required_version,
std::string &returned_version) const
-> bool override {
required_version = returned_version = "";
return true;
}
[[nodiscard]] auto create_directory(const std::string &api_path,
api_meta_map &meta) -> api_error override;

4
repertory/librepertory/include/providers/i_provider.hpp
View File

@ -31,6 +31,10 @@ class i_provider {
INTERFACE_SETUP(i_provider);
public:
[[nodiscard]] virtual auto check_version(std::string &required_version,
std::string &returned_version) const
-> bool = 0;
[[nodiscard]] virtual auto create_directory(const std::string &api_path,
api_meta_map &meta)
-> api_error = 0;

7
repertory/librepertory/include/providers/s3/s3_provider.hpp
View File

@ -113,6 +113,13 @@ protected:
-> api_error override;
public:
[[nodiscard]] auto check_version(std::string &required_version,
std::string &returned_version) const
-> bool override {
required_version = returned_version = "";
return true;
}
[[nodiscard]] static auto convert_api_date(std::string_view date)
-> std::uint64_t;

4
repertory/librepertory/include/providers/sia/sia_provider.hpp
View File

@ -80,6 +80,10 @@ protected:
-> api_error override;
public:
[[nodiscard]] auto check_version(std::string &required_version,
std::string &returned_version) const
-> bool override;
[[nodiscard]] auto get_directory_item_count(const std::string &api_path) const
-> std::uint64_t override;

52
repertory/librepertory/include/types/repertory.hpp
View File

@ -24,30 +24,27 @@
namespace repertory {
constexpr const auto default_api_auth_size{48U};
constexpr const auto default_download_timeout_ces{30U};
constexpr const auto default_download_timeout_secs{30U};
constexpr const auto default_eviction_delay_mins{1U};
constexpr const auto default_high_freq_interval_secs{30U};
constexpr const auto default_low_freq_interval_secs{0U * 60U};
constexpr const auto default_high_freq_interval_secs{std::uint16_t{30U}};
constexpr const auto default_low_freq_interval_secs{std::uint16_t(60U * 60U)};
constexpr const auto default_max_cache_size_bytes{
std::uint64_t(20UL * 1024UL * 1024UL * 1024UL),
std::uint64_t(20ULL * 1024ULL * 1024ULL * 1024ULL),
};
constexpr const auto default_max_upload_count{5U};
constexpr const auto default_med_freq_interval_secs{2U * 60U};
constexpr const auto default_med_freq_interval_secs{std::uint16_t{2U * 60U}};
constexpr const auto default_online_check_retry_secs{60U};
constexpr const auto default_orphaned_file_retention_days{15U};
constexpr const auto default_retry_read_count{6U};
constexpr const auto default_ring_buffer_file_size{512U};
constexpr const auto default_task_wait_ms{100U};
constexpr const auto default_timeout_ms{60000U};
constexpr const auto max_orphaned_file_retention_days{std::uint16_t(31U)};
constexpr const auto max_ring_buffer_file_size{std::uint16_t(1024U)};
constexpr const auto max_s3_object_name_length{1024U};
constexpr const auto min_cache_size_bytes{
std::uint64_t(100UL * 1024UL * 1024UL),
std::uint64_t(100ULL * 1024ULL * 1024ULL),
};
constexpr const auto min_download_timeout_secs{std::uint8_t(5U)};
constexpr const auto min_online_check_retry_secs{std::uint16_t(15U)};
constexpr const auto min_orphaned_file_retention_days{std::uint16_t(1U)};
constexpr const auto min_retry_read_count{std::uint16_t(2U)};
constexpr const auto min_ring_buffer_file_size{std::uint16_t(64U)};
constexpr const auto min_task_wait_ms{std::uint16_t(50U)};
@ -333,7 +330,6 @@ struct directory_item final {
bool directory{false};
std::uint64_t size{};
api_meta_map meta;
bool resolved{false};
};
struct encrypt_config final {
@ -493,8 +489,6 @@ inline constexpr const auto JSON_MED_FREQ_INTERVAL_SECS{
inline constexpr const auto JSON_META{"Meta"};
inline constexpr const auto JSON_ONLINE_CHECK_RETRY_SECS{
"OnlineCheckRetrySeconds"};
inline constexpr const auto JSON_ORPHANED_FILE_RETENTION_DAYS{
"OrphanedFileRetentionDays"};
inline constexpr const auto JSON_PATH{"Path"};
inline constexpr const auto JSON_PREFERRED_DOWNLOAD_TYPE{
"PreferredDownloadType"};
@ -619,6 +613,16 @@ template <typename data_t> struct adl_serializer<repertory::atomic<data_t>> {
}
};
template <> struct adl_serializer<std::atomic<std::uint64_t>> {
static void to_json(json &data, const std::atomic<std::uint64_t> &value) {
data = value.load();
}
static void from_json(const json &data, std::atomic<std::uint64_t> &value) {
value.store(data.get<std::uint64_t>());
}
};
template <typename primitive_t>
struct adl_serializer<std::atomic<primitive_t>> {
static void to_json(json &data, const std::atomic<primitive_t> &value) {
@ -642,6 +646,18 @@ template <> struct adl_serializer<std::atomic<repertory::database_type>> {
}
};
template <> struct adl_serializer<std::atomic<repertory::event_level>> {
static void to_json(json &data,
const std::atomic<repertory::event_level> &value) {
data = repertory::event_level_to_string(value.load());
}
static void from_json(const json &data,
std::atomic<repertory::event_level> &value) {
value.store(repertory::event_level_from_string(data.get<std::string>()));
}
};
template <> struct adl_serializer<std::atomic<repertory::download_type>> {
static void to_json(json &data,
const std::atomic<repertory::download_type> &value) {
@ -674,15 +690,13 @@ template <> struct adl_serializer<repertory::download_type> {
}
};
template <> struct adl_serializer<std::atomic<repertory::event_level>> {
static void to_json(json &data,
const std::atomic<repertory::event_level> &value) {
data = repertory::event_level_to_string(value.load());
template <> struct adl_serializer<repertory::event_level> {
static void to_json(json &data, const repertory::event_level &value) {
data = repertory::event_level_to_string(value);
}
static void from_json(const json &data,
std::atomic<repertory::event_level> &value) {
value.store(repertory::event_level_from_string(data.get<std::string>()));
static void from_json(const json &data, repertory::event_level &value) {
value = repertory::event_level_from_string(data.get<std::string>());
}
};
NLOHMANN_JSON_NAMESPACE_END

79
repertory/librepertory/src/app_config.cpp
View File

@ -70,7 +70,7 @@ app_config::app_config(const provider_type &prov,
api_port_(default_rpc_port(prov)),
api_user_(std::string{REPERTORY}),
config_changed_(false),
download_timeout_secs_(default_download_timeout_ces),
download_timeout_secs_(default_download_timeout_secs),
enable_download_timeout_(true),
enable_drive_events_(false),
#if defined(_WIN32)
@ -85,7 +85,6 @@ app_config::app_config(const provider_type &prov,
max_upload_count_(default_max_upload_count),
med_freq_interval_secs_(default_med_freq_interval_secs),
online_check_retry_secs_(default_online_check_retry_secs),
orphaned_file_retention_days_(default_orphaned_file_retention_days),
preferred_download_type_(download_type::default_),
retry_read_count_(default_retry_read_count),
ring_buffer_file_size_(default_ring_buffer_file_size),
@ -166,8 +165,14 @@ app_config::app_config(const provider_type &prov,
[this]() { return get_host_config().api_password; }},
{fmt::format("{}.{}", JSON_HOST_CONFIG, JSON_API_PORT),
[this]() { return std::to_string(get_host_config().api_port); }},
{fmt::format("{}.{}", JSON_HOST_CONFIG, JSON_API_USER),
[this]() { return get_host_config().api_user; }},
{fmt::format("{}.{}", JSON_HOST_CONFIG, JSON_HOST_NAME_OR_IP),
[this]() { return get_host_config().host_name_or_ip; }},
{fmt::format("{}.{}", JSON_HOST_CONFIG, JSON_PATH),
[this]() { return get_host_config().path; }},
{fmt::format("{}.{}", JSON_HOST_CONFIG, JSON_PROTOCOL),
[this]() { return get_host_config().protocol; }},
{fmt::format("{}.{}", JSON_HOST_CONFIG, JSON_TIMEOUT_MS),
[this]() { return std::to_string(get_host_config().timeout_ms); }},
{JSON_LOW_FREQ_INTERVAL_SECS,
@ -180,8 +185,6 @@ app_config::app_config(const provider_type &prov,
[this]() { return std::to_string(get_med_frequency_interval_secs()); }},
{JSON_ONLINE_CHECK_RETRY_SECS,
[this]() { return std::to_string(get_online_check_retry_secs()); }},
{JSON_ORPHANED_FILE_RETENTION_DAYS,
[this]() { return std::to_string(get_orphaned_file_retention_days()); }},
{JSON_PREFERRED_DOWNLOAD_TYPE,
[this]() {
return download_type_to_string(get_preferred_download_type());
@ -250,7 +253,7 @@ app_config::app_config(const provider_type &prov,
value_set_lookup_ = {
{
JSON_API_PATH,
JSON_API_AUTH,
[this](const std::string &value) {
set_api_auth(value);
return get_api_auth();
@ -349,7 +352,7 @@ app_config::app_config(const provider_type &prov,
{
JSON_HIGH_FREQ_INTERVAL_SECS,
[this](const std::string &value) {
set_high_frequency_interval_secs(utils::string::to_uint8(value));
set_high_frequency_interval_secs(utils::string::to_uint16(value));
return std::to_string(get_high_frequency_interval_secs());
},
},
@ -380,6 +383,15 @@ app_config::app_config(const provider_type &prov,
return std::to_string(get_host_config().api_port);
},
},
{
fmt::format("{}.{}", JSON_HOST_CONFIG, JSON_API_USER),
[this](const std::string &value) {
auto cfg = get_host_config();
cfg.api_user = value;
set_host_config(cfg);
return get_host_config().api_user;
},
},
{
fmt::format("{}.{}", JSON_HOST_CONFIG, JSON_HOST_NAME_OR_IP),
[this](const std::string &value) {
@ -389,6 +401,24 @@ app_config::app_config(const provider_type &prov,
return get_host_config().host_name_or_ip;
},
},
{
fmt::format("{}.{}", JSON_HOST_CONFIG, JSON_PATH),
[this](const std::string &value) {
auto cfg = get_host_config();
cfg.path = value;
set_host_config(cfg);
return get_host_config().path;
},
},
{
fmt::format("{}.{}", JSON_HOST_CONFIG, JSON_PROTOCOL),
[this](const std::string &value) {
auto cfg = get_host_config();
cfg.protocol = value;
set_host_config(cfg);
return get_host_config().protocol;
},
},
{
fmt::format("{}.{}", JSON_HOST_CONFIG, JSON_TIMEOUT_MS),
[this](const std::string &value) {
@ -401,14 +431,14 @@ app_config::app_config(const provider_type &prov,
{
JSON_LOW_FREQ_INTERVAL_SECS,
[this](const std::string &value) {
set_low_frequency_interval_secs(utils::string::to_uint8(value));
set_low_frequency_interval_secs(utils::string::to_uint16(value));
return std::to_string(get_low_frequency_interval_secs());
},
},
{
JSON_MED_FREQ_INTERVAL_SECS,
[this](const std::string &value) {
set_med_frequency_interval_secs(utils::string::to_uint8(value));
set_med_frequency_interval_secs(utils::string::to_uint16(value));
return std::to_string(get_med_frequency_interval_secs());
},
},
@ -433,13 +463,6 @@ app_config::app_config(const provider_type &prov,
return std::to_string(get_online_check_retry_secs());
},
},
{
JSON_ORPHANED_FILE_RETENTION_DAYS,
[this](const std::string &value) {
set_orphaned_file_retention_days(utils::string::to_uint16(value));
return std::to_string(get_orphaned_file_retention_days());
},
},
{
JSON_PREFERRED_DOWNLOAD_TYPE,
[this](const std::string &value) {
@ -812,7 +835,6 @@ auto app_config::get_json() const -> json {
{JSON_MAX_UPLOAD_COUNT, max_upload_count_},
{JSON_MED_FREQ_INTERVAL_SECS, med_freq_interval_secs_},
{JSON_ONLINE_CHECK_RETRY_SECS, online_check_retry_secs_},
{JSON_ORPHANED_FILE_RETENTION_DAYS, orphaned_file_retention_days_},
{JSON_PREFERRED_DOWNLOAD_TYPE, preferred_download_type_},
{JSON_REMOTE_CONFIG, remote_config_},
{JSON_REMOTE_MOUNT, remote_mount_},
@ -834,7 +856,6 @@ auto app_config::get_json() const -> json {
ret.erase(JSON_MAX_CACHE_SIZE_BYTES);
ret.erase(JSON_MAX_UPLOAD_COUNT);
ret.erase(JSON_ONLINE_CHECK_RETRY_SECS);
ret.erase(JSON_ORPHANED_FILE_RETENTION_DAYS);
ret.erase(JSON_PREFERRED_DOWNLOAD_TYPE);
ret.erase(JSON_REMOTE_CONFIG);
ret.erase(JSON_RETRY_READ_COUNT);
@ -856,7 +877,6 @@ auto app_config::get_json() const -> json {
ret.erase(JSON_MAX_UPLOAD_COUNT);
ret.erase(JSON_MED_FREQ_INTERVAL_SECS);
ret.erase(JSON_ONLINE_CHECK_RETRY_SECS);
ret.erase(JSON_ORPHANED_FILE_RETENTION_DAYS);
ret.erase(JSON_PREFERRED_DOWNLOAD_TYPE);
ret.erase(JSON_REMOTE_MOUNT);
ret.erase(JSON_RETRY_READ_COUNT);
@ -912,12 +932,6 @@ auto app_config::get_online_check_retry_secs() const -> std::uint16_t {
return std::max(min_online_check_retry_secs, online_check_retry_secs_.load());
}
auto app_config::get_orphaned_file_retention_days() const -> std::uint16_t {
return std::min(max_orphaned_file_retention_days,
std::max(min_orphaned_file_retention_days,
orphaned_file_retention_days_.load()));
}
auto app_config::get_preferred_download_type() const -> download_type {
return preferred_download_type_;
}
@ -1053,8 +1067,6 @@ auto app_config::load() -> bool {
med_freq_interval_secs_, found);
get_value(json_document, JSON_ONLINE_CHECK_RETRY_SECS,
online_check_retry_secs_, found);
get_value(json_document, JSON_ORPHANED_FILE_RETENTION_DAYS,
orphaned_file_retention_days_, found);
get_value(json_document, JSON_PREFERRED_DOWNLOAD_TYPE,
preferred_download_type_, found);
get_value(json_document, JSON_REMOTE_CONFIG, remote_config_, found);
@ -1069,10 +1081,17 @@ auto app_config::load() -> bool {
std::uint64_t version{};
get_value(json_document, JSON_VERSION, version, found);
// Handle configuration defaults for new config versions
if (version != REPERTORY_CONFIG_VERSION) {
version_ = REPERTORY_CONFIG_VERSION;
// TODO upgrade future version
if (version_ == 1U) {
if (low_freq_interval_secs_ == 0UL) {
set_value(low_freq_interval_secs_, default_low_freq_interval_secs);
}
if (max_cache_size_bytes_ == 0UL) {
set_value(max_cache_size_bytes_, default_max_cache_size_bytes);
}
}
found = false;
}
@ -1199,10 +1218,6 @@ void app_config::set_online_check_retry_secs(std::uint16_t value) {
set_value(online_check_retry_secs_, value);
}
void app_config::set_orphaned_file_retention_days(std::uint16_t value) {
set_value(orphaned_file_retention_days_, value);
}
void app_config::set_preferred_download_type(const download_type &value) {
set_value(preferred_download_type_, value);
}

89
repertory/librepertory/src/drives/fuse/remotefuse/remote_fuse_drive.cpp
View File

@ -43,8 +43,8 @@
#include "utils/utils.hpp"
namespace repertory::remote_fuse {
auto remote_fuse_drive::access_impl(std::string api_path,
int mask) -> api_error {
auto remote_fuse_drive::access_impl(std::string api_path, int mask)
-> api_error {
return utils::to_api_error(
remote_instance_->fuse_access(api_path.c_str(), mask));
}
@ -62,8 +62,8 @@ auto remote_fuse_drive::chmod_impl(std::string api_path, mode_t mode,
struct fuse_file_info * /*f_info*/)
-> api_error {
#else
auto remote_fuse_drive::chmod_impl(std::string api_path,
mode_t mode) -> api_error {
auto remote_fuse_drive::chmod_impl(std::string api_path, mode_t mode)
-> api_error {
#endif
return utils::to_api_error(remote_instance_->fuse_chmod(
api_path.c_str(), static_cast<remote::file_mode>(mode)));
@ -74,8 +74,8 @@ auto remote_fuse_drive::chown_impl(std::string api_path, uid_t uid, gid_t gid,
struct fuse_file_info * /*f_info*/)
-> api_error {
#else
auto remote_fuse_drive::chown_impl(std::string api_path, uid_t uid,
gid_t gid) -> api_error {
auto remote_fuse_drive::chown_impl(std::string api_path, uid_t uid, gid_t gid)
-> api_error {
#endif
return utils::to_api_error(
remote_instance_->fuse_chown(api_path.c_str(), uid, gid));
@ -94,7 +94,7 @@ void remote_fuse_drive::destroy_impl(void *ptr) {
REPERTORY_USES_FUNCTION_NAME();
event_system::instance().raise<drive_unmount_pending>(function_name,
get_mount_location());
get_mount_location());
if (server_) {
server_->stop();
@ -116,14 +116,15 @@ void remote_fuse_drive::destroy_impl(void *ptr) {
}
event_system::instance().raise<drive_unmounted>(function_name,
get_mount_location());
get_mount_location());
fuse_base::destroy_impl(ptr);
}
auto remote_fuse_drive::fgetattr_impl(
std::string api_path, struct stat *unix_st,
struct fuse_file_info *f_info) -> api_error {
auto remote_fuse_drive::fgetattr_impl(std::string api_path,
struct stat *unix_st,
struct fuse_file_info *f_info)
-> api_error {
remote::stat r_stat{};
auto directory = false;
@ -184,8 +185,8 @@ auto remote_fuse_drive::getattr_impl(std::string api_path, struct stat *unix_st,
struct fuse_file_info * /*f_info*/)
-> api_error {
#else
auto remote_fuse_drive::getattr_impl(std::string api_path,
struct stat *unix_st) -> api_error {
auto remote_fuse_drive::getattr_impl(std::string api_path, struct stat *unix_st)
-> api_error {
#endif
bool directory = false;
remote::stat r_stat{};
@ -263,14 +264,14 @@ auto remote_fuse_drive::init_impl(struct fuse_conn_info *conn) -> void * {
server_ = std::make_shared<server>(config_);
server_->start();
event_system::instance().raise<drive_mounted>(function_name,
get_mount_location());
get_mount_location());
}
return ret;
}
auto remote_fuse_drive::mkdir_impl(std::string api_path,
mode_t mode) -> api_error {
auto remote_fuse_drive::mkdir_impl(std::string api_path, mode_t mode)
-> api_error {
return utils::to_api_error(remote_instance_->fuse_mkdir(
api_path.c_str(), static_cast<remote::file_mode>(mode)));
}
@ -295,8 +296,9 @@ auto remote_fuse_drive::open_impl(std::string api_path,
f_info->fh));
}
auto remote_fuse_drive::opendir_impl(
std::string api_path, struct fuse_file_info *f_info) -> api_error {
auto remote_fuse_drive::opendir_impl(std::string api_path,
struct fuse_file_info *f_info)
-> api_error {
if ((f_info->flags & O_APPEND) == O_APPEND ||
(f_info->flags & O_EXCL) == O_EXCL) {
return api_error::directory_exists;
@ -309,12 +311,14 @@ auto remote_fuse_drive::opendir_impl(
void remote_fuse_drive::populate_stat(const remote::stat &r_stat,
bool directory, struct stat &unix_st) {
std::memset(&unix_st, 0, sizeof(struct stat));
unix_st.st_blksize = r_stat.st_blksize;
unix_st.st_blocks = static_cast<blkcnt_t>(r_stat.st_blocks);
unix_st.st_blksize =
static_cast<decltype(unix_st.st_blksize)>(r_stat.st_blksize);
unix_st.st_blocks =
static_cast<decltype(unix_st.st_blocks)>(r_stat.st_blocks);
unix_st.st_gid = r_stat.st_gid;
unix_st.st_mode = (directory ? S_IFDIR : S_IFREG) | r_stat.st_mode;
unix_st.st_nlink = r_stat.st_nlink;
unix_st.st_size = static_cast<off_t>(r_stat.st_size);
unix_st.st_size = static_cast<decltype(unix_st.st_size)>(r_stat.st_size);
unix_st.st_uid = r_stat.st_uid;
#if defined(__APPLE__)
@ -373,14 +377,18 @@ auto remote_fuse_drive::read_impl(std::string api_path, char *buffer,
}
#if FUSE_USE_VERSION >= 30
auto remote_fuse_drive::readdir_impl(
std::string api_path, void *buf, fuse_fill_dir_t fuse_fill_dir,
off_t offset, struct fuse_file_info *f_info,
fuse_readdir_flags /*flags*/) -> api_error {
auto remote_fuse_drive::readdir_impl(std::string api_path, void *buf,
fuse_fill_dir_t fuse_fill_dir,
off_t offset,
struct fuse_file_info *f_info,
fuse_readdir_flags /*flags*/)
-> api_error {
#else
auto remote_fuse_drive::readdir_impl(
std::string api_path, void *buf, fuse_fill_dir_t fuse_fill_dir,
off_t offset, struct fuse_file_info *f_info) -> api_error {
auto remote_fuse_drive::readdir_impl(std::string api_path, void *buf,
fuse_fill_dir_t fuse_fill_dir,
off_t offset,
struct fuse_file_info *f_info)
-> api_error {
#endif
std::string item_path;
int res = 0;
@ -408,14 +416,16 @@ auto remote_fuse_drive::readdir_impl(
return utils::to_api_error(res);
}
auto remote_fuse_drive::release_impl(
std::string api_path, struct fuse_file_info *f_info) -> api_error {
auto remote_fuse_drive::release_impl(std::string api_path,
struct fuse_file_info *f_info)
-> api_error {
return utils::to_api_error(
remote_instance_->fuse_release(api_path.c_str(), f_info->fh));
}
auto remote_fuse_drive::releasedir_impl(
std::string api_path, struct fuse_file_info *f_info) -> api_error {
auto remote_fuse_drive::releasedir_impl(std::string api_path,
struct fuse_file_info *f_info)
-> api_error {
return utils::to_api_error(
remote_instance_->fuse_releasedir(api_path.c_str(), f_info->fh));
}
@ -512,8 +522,8 @@ api_error remote_fuse_drive::statfs_x_impl(std::string api_path,
return utils::to_api_error(res);
}
#else // __APPLE__
auto remote_fuse_drive::statfs_impl(std::string api_path,
struct statvfs *stbuf) -> api_error {
auto remote_fuse_drive::statfs_impl(std::string api_path, struct statvfs *stbuf)
-> api_error {
auto res = statvfs(config_.get_data_directory().c_str(), stbuf);
if (res == 0) {
remote::statfs r_stat{};
@ -540,8 +550,8 @@ auto remote_fuse_drive::truncate_impl(std::string api_path, off_t size,
struct fuse_file_info * /*f_info*/)
-> api_error {
#else
auto remote_fuse_drive::truncate_impl(std::string api_path,
off_t size) -> api_error {
auto remote_fuse_drive::truncate_impl(std::string api_path, off_t size)
-> api_error {
#endif
return utils::to_api_error(remote_instance_->fuse_truncate(
api_path.c_str(), static_cast<remote::file_offset>(size)));
@ -552,9 +562,10 @@ auto remote_fuse_drive::unlink_impl(std::string api_path) -> api_error {
}
#if FUSE_USE_VERSION >= 30
auto remote_fuse_drive::utimens_impl(
std::string api_path, const struct timespec tv[2],
struct fuse_file_info * /*f_info*/) -> api_error {
auto remote_fuse_drive::utimens_impl(std::string api_path,
const struct timespec tv[2],
struct fuse_file_info * /*f_info*/)
-> api_error {
#else
auto remote_fuse_drive::utimens_impl(std::string api_path,
const struct timespec tv[2]) -> api_error {

28
repertory/librepertory/src/drives/winfsp/remotewinfsp/remote_client.cpp
View File

@ -22,13 +22,11 @@
#include "drives/winfsp/remotewinfsp/remote_client.hpp"
#include "app_config.hpp"
#include "drives/winfsp/remotewinfsp/i_remote_instance.hpp"
#include "events/event_system.hpp"
#include "events/types/drive_mounted.hpp"
#include "events/types/drive_unmount_pending.hpp"
#include "events/types/drive_unmounted.hpp"
#include "types/repertory.hpp"
#include "utils/path.hpp"
#include "utils/string.hpp"
#include "version.hpp"
namespace repertory::remote_winfsp {
@ -44,11 +42,7 @@ auto remote_client::winfsp_can_delete(PVOID file_desc, PWSTR file_name)
request.encode(file_name);
std::uint32_t service_flags{};
auto ret{
packet_client_.send(function_name, request, service_flags),
};
return ret;
return packet_client_.send(function_name, request, service_flags);
}
auto remote_client::json_create_directory_snapshot(const std::string &path,
@ -103,11 +97,7 @@ auto remote_client::json_release_directory_snapshot(
request.encode(handle);
std::uint32_t service_flags{};
auto ret{
packet_client_.send(function_name, request, service_flags),
};
return ret;
return packet_client_.send(function_name, request, service_flags);
}
auto remote_client::winfsp_cleanup(PVOID file_desc, PWSTR file_name,
@ -273,8 +263,8 @@ auto remote_client::winfsp_get_security_by_name(PWSTR file_name,
packet request;
request.encode(file_name);
request.encode(static_cast<std::uint64_t>(
descriptor_size == nullptr ? 0 : *descriptor_size));
request.encode(descriptor_size == nullptr ? std::uint64_t(0U)
: *descriptor_size);
request.encode(static_cast<std::uint8_t>(attributes != nullptr));
packet response;
@ -417,7 +407,7 @@ auto remote_client::winfsp_read(PVOID file_desc, PVOID buffer, UINT64 offset,
ret = response.decode(buffer, *bytes_transferred);
#if defined(_WIN32)
if ((ret == STATUS_SUCCESS) &&
(not*bytes_transferred || (*bytes_transferred != length))) {
((*bytes_transferred == 0U) || (*bytes_transferred != length))) {
::SetLastError(ERROR_HANDLE_EOF);
}
#endif
@ -461,11 +451,7 @@ auto remote_client::winfsp_rename(PVOID file_desc, PWSTR file_name,
request.encode(replace_if_exists);
std::uint32_t service_flags{};
auto ret{
packet_client_.send(function_name, request, service_flags),
};
return ret;
return packet_client_.send(function_name, request, service_flags);
}
auto remote_client::winfsp_set_basic_info(

7
repertory/librepertory/src/file_manager/file_manager.cpp
View File

@ -466,6 +466,13 @@ auto file_manager::open(const std::string &api_path, bool directory,
return download_type::default_;
}
if (utils::file::file{fsi.source_path}.exists()) {
auto size = utils::file::file{fsi.source_path}.size();
if (size.has_value() && *size == fsi.size) {
return download_type::default_;
}
}
if (type == download_type::direct) {
return type;
}

106
repertory/librepertory/src/file_manager/ring_buffer_base.cpp
View File

@ -98,19 +98,23 @@ auto ring_buffer_base::close() -> bool {
return res;
}
auto ring_buffer_base::download_chunk(std::size_t chunk,
bool skip_active) -> api_error {
auto ring_buffer_base::download_chunk(std::size_t chunk, bool skip_active)
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
unique_mutex_lock chunk_lock(chunk_mtx_);
const auto unlock_and_notify = [this, &chunk_lock]() {
if (not skip_active) {
ring_pos_ = chunk;
}
const auto notify_and_unlock = [this, &chunk_lock]() {
chunk_notify_.notify_all();
chunk_lock.unlock();
};
const auto unlock_and_return =
[&unlock_and_notify](api_error res) -> api_error {
unlock_and_notify();
[&notify_and_unlock](api_error res) -> api_error {
notify_and_unlock();
return res;
};
@ -124,7 +128,7 @@ auto ring_buffer_base::download_chunk(std::size_t chunk,
}
auto active_download = get_active_downloads().at(chunk);
unlock_and_notify();
notify_and_unlock();
return active_download->wait();
}
@ -142,7 +146,7 @@ auto ring_buffer_base::download_chunk(std::size_t chunk,
chunk == (total_chunks_ - 1U) ? get_last_chunk_size()
: get_chunk_size(),
};
unlock_and_notify();
notify_and_unlock();
auto result{
get_provider().read_file_bytes(get_api_path(), data_size, data_offset,
@ -167,7 +171,7 @@ auto ring_buffer_base::download_chunk(std::size_t chunk,
}
get_active_downloads().erase(chunk);
unlock_and_notify();
notify_and_unlock();
active_download->notify(result);
return result;
@ -226,7 +230,6 @@ auto ring_buffer_base::read(std::size_t read_size, std::uint64_t read_offset,
} else if (chunk < ring_pos_) {
reverse(ring_pos_ - chunk);
}
res = download_chunk(chunk, false);
if (res != api_error::success) {
if (res == api_error::invalid_ring_buffer_position) {
@ -264,38 +267,46 @@ void ring_buffer_base::reader_thread() {
REPERTORY_USES_FUNCTION_NAME();
unique_mutex_lock chunk_lock(chunk_mtx_);
auto next_chunk{ring_pos_};
chunk_notify_.notify_all();
chunk_lock.unlock();
const auto notify_and_unlock = [this, &chunk_lock]() {
chunk_notify_.notify_all();
chunk_lock.unlock();
};
auto last_pos = ring_pos_;
auto next_chunk = ring_pos_;
notify_and_unlock();
while (not get_stop_requested()) {
chunk_lock.lock();
next_chunk = next_chunk + 1U > ring_end_ ? ring_begin_ : next_chunk + 1U;
const auto check_and_wait = [this, &chunk_lock, &next_chunk]() {
if (last_pos == ring_pos_) {
++next_chunk;
} else {
next_chunk = ring_pos_ + 1U;
last_pos = ring_pos_;
}
if (next_chunk > ring_end_) {
next_chunk = ring_begin_;
}
if (read_state_[next_chunk % read_state_.size()]) {
if (get_stop_requested()) {
chunk_notify_.notify_all();
chunk_lock.unlock();
notify_and_unlock();
return;
}
if (get_read_state().all()) {
chunk_notify_.wait(chunk_lock);
last_pos = ring_pos_;
next_chunk = ring_pos_;
}
chunk_notify_.notify_all();
chunk_lock.unlock();
};
if (read_state_[next_chunk % read_state_.size()]) {
check_and_wait();
notify_and_unlock();
continue;
}
chunk_notify_.notify_all();
chunk_lock.unlock();
notify_and_unlock();
download_chunk(next_chunk, true);
}
@ -351,30 +362,31 @@ void ring_buffer_base::update_position(std::size_t count, bool is_forward) {
if (is_forward ? (ring_pos_ + count) <= ring_end_
: (ring_pos_ - count) >= ring_begin_) {
ring_pos_ += is_forward ? count : -count;
} else {
auto delta = is_forward ? count - (ring_end_ - ring_pos_)
: count - (ring_pos_ - ring_begin_);
if (delta >= read_state_.size()) {
read_state_.set(0U, read_state_.size(), false);
ring_pos_ += is_forward ? count : -count;
ring_begin_ += is_forward ? delta : -delta;
} else {
for (std::size_t idx = 0U; idx < delta; ++idx) {
if (is_forward) {
read_state_[(ring_begin_ + idx) % read_state_.size()] = false;
} else {
read_state_[(ring_end_ - idx) % read_state_.size()] = false;
}
}
ring_begin_ += is_forward ? delta : -delta;
ring_pos_ += is_forward ? count : -count;
}
ring_end_ =
std::min(total_chunks_ - 1U, ring_begin_ + read_state_.size() - 1U);
chunk_notify_.notify_all();
return;
}
auto delta = is_forward ? count - (ring_end_ - ring_pos_)
: count - (ring_pos_ - ring_begin_);
if (delta >= read_state_.size()) {
read_state_.set(0U, read_state_.size(), false);
ring_pos_ += is_forward ? count : -count;
ring_begin_ += is_forward ? delta : -delta;
} else {
for (std::size_t idx = 0U; idx < delta; ++idx) {
if (is_forward) {
read_state_[(ring_begin_ + idx) % read_state_.size()] = false;
} else {
read_state_[(ring_end_ - idx) % read_state_.size()] = false;
}
}
ring_begin_ += is_forward ? delta : -delta;
ring_pos_ += is_forward ? count : -count;
}
ring_end_ =
std::min(total_chunks_ - 1U, ring_begin_ + read_state_.size() - 1U);
chunk_notify_.notify_all();
}
} // namespace repertory

12
repertory/librepertory/src/initialize.cpp
View File

@ -50,10 +50,6 @@
namespace repertory {
auto project_initialize() -> bool {
spdlog::drop_all();
spdlog::flush_every(std::chrono::seconds(10));
spdlog::set_pattern("%Y-%m-%d|%T.%e|%^%l%$|%v");
#if defined(PROJECT_REQUIRE_ALPINE) && !defined(PROJECT_IS_MINGW)
{
static constexpr const auto guard_size{4096U};
@ -68,6 +64,10 @@ auto project_initialize() -> bool {
}
#endif // defined(PROJECT_REQUIRE_ALPINE) && !defined (PROJECT_IS_MINGW)
spdlog::drop_all();
spdlog::flush_every(std::chrono::seconds(10));
spdlog::set_pattern("%Y-%m-%d|%T.%e|%^%l%$|%v");
#if defined(PROJECT_ENABLE_LIBSODIUM)
{
if (sodium_init() == -1) {
@ -77,7 +77,9 @@ auto project_initialize() -> bool {
#endif // defined(PROJECT_ENABLE_LIBSODIUM)
#if defined(PROJECT_ENABLE_OPENSSL)
{ SSL_library_init(); }
{
SSL_library_init();
}
#endif // defined(PROJECT_ENABLE_OPENSSL)
#if defined(PROJECT_ENABLE_CURL)

54
repertory/librepertory/src/providers/base_provider.cpp
View File

@ -34,6 +34,7 @@
#include "events/types/orphaned_file_processing_failed.hpp"
#include "events/types/orphaned_source_file_detected.hpp"
#include "events/types/orphaned_source_file_removed.hpp"
#include "events/types/provider_invalid_version.hpp"
#include "events/types/provider_offline.hpp"
#include "events/types/provider_upload_begin.hpp"
#include "events/types/provider_upload_end.hpp"
@ -69,8 +70,8 @@ void base_provider::add_all_items(stop_type &stop_requested) {
}
auto base_provider::create_api_file(std::string path, std::string key,
std::uint64_t size,
std::uint64_t file_time) -> api_file {
std::uint64_t size, std::uint64_t file_time)
-> api_file {
api_file file{};
file.api_path = utils::path::create_api_path(path);
file.api_parent = utils::path::get_parent_api_path(file.api_path);
@ -102,8 +103,8 @@ auto base_provider::create_api_file(std::string path, std::uint64_t size,
}
auto base_provider::create_directory_clone_source_meta(
const std::string &source_api_path,
const std::string &api_path) -> api_error {
const std::string &source_api_path, const std::string &api_path)
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
bool exists{};
@ -201,8 +202,8 @@ auto base_provider::create_directory(const std::string &api_path,
return api_error::error;
}
auto base_provider::create_file(const std::string &api_path,
api_meta_map &meta) -> api_error {
auto base_provider::create_file(const std::string &api_path, api_meta_map &meta)
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
try {
@ -259,8 +260,9 @@ auto base_provider::create_file(const std::string &api_path,
return api_error::error;
}
auto base_provider::get_api_path_from_source(
const std::string &source_path, std::string &api_path) const -> api_error {
auto base_provider::get_api_path_from_source(const std::string &source_path,
std::string &api_path) const
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
if (source_path.empty()) {
@ -273,8 +275,9 @@ auto base_provider::get_api_path_from_source(
return db3_->get_api_path(source_path, api_path);
}
auto base_provider::get_directory_items(
const std::string &api_path, directory_item_list &list) const -> api_error {
auto base_provider::get_directory_items(const std::string &api_path,
directory_item_list &list) const
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
try {
@ -283,8 +286,15 @@ auto base_provider::get_directory_items(
if (res != api_error::success) {
return res;
}
if (not exists) {
return api_error::directory_not_found;
res = is_file(api_path, exists);
if (res != api_error::success) {
utils::error::raise_api_path_error(
function_name, api_path, res, "failed to determine if file exists");
}
return exists ? api_error::item_exists : api_error::directory_not_found;
}
res = get_directory_items_impl(api_path, list);
@ -342,9 +352,10 @@ auto base_provider::get_file_size(const std::string &api_path,
return api_error::success;
}
auto base_provider::get_filesystem_item(
const std::string &api_path, bool directory,
filesystem_item &fsi) const -> api_error {
auto base_provider::get_filesystem_item(const std::string &api_path,
bool directory,
filesystem_item &fsi) const
-> api_error {
bool exists{};
auto res = is_directory(api_path, exists);
if (res != api_error::success) {
@ -377,9 +388,10 @@ auto base_provider::get_filesystem_item(
return api_error::success;
}
auto base_provider::get_filesystem_item_and_file(
const std::string &api_path, api_file &file,
filesystem_item &fsi) const -> api_error {
auto base_provider::get_filesystem_item_and_file(const std::string &api_path,
api_file &file,
filesystem_item &fsi) const
-> api_error {
auto res = get_file(api_path, file);
if (res != api_error::success) {
return res;
@ -829,6 +841,14 @@ auto base_provider::start(api_item_added_callback api_item_added,
return false;
}
std::string returned_version;
std::string required_version;
if (not check_version(required_version, returned_version)) {
event_system::instance().raise<provider_invalid_version>(
function_name, required_version, returned_version);
return false;
}
cache_size_mgr::instance().initialize(&config_);
polling::instance().set_callback({

24
repertory/librepertory/src/providers/encrypt/encrypt_provider.cpp
View File

@ -40,6 +40,7 @@
#include "utils/file_utils.hpp"
#include "utils/path.hpp"
#include "utils/polling.hpp"
#include <spdlog/fmt/bundled/base.h>
namespace repertory {
encrypt_provider::encrypt_provider(app_config &config)
@ -262,7 +263,6 @@ auto encrypt_provider::get_directory_items(const std::string &api_path,
dir_item.api_parent = file.api_parent;
dir_item.api_path = file.api_path;
dir_item.directory = dir_entry->is_directory_item();
dir_item.resolved = true;
dir_item.size = file.file_size;
create_item_meta(dir_item.meta, dir_item.directory, file);
@ -342,13 +342,23 @@ auto encrypt_provider::get_file_list(api_file_list &list,
const auto &cfg{get_encrypt_config()};
try {
for (const auto &dir_entry : utils::file::directory{cfg.path}.get_items()) {
std::string api_path{};
if (process_directory_entry(*dir_entry.get(), cfg, api_path)) {
list.emplace_back(create_api_file(
api_path, dir_entry->is_directory_item(), dir_entry->get_path()));
using func = std::function<void(std::string path)>;
const func process_directory = [&](std::string path) {
for (const auto &dir_entry : utils::file::directory{path}.get_items()) {
std::string api_path{};
if (dir_entry->is_directory_item()) {
process_directory_entry(*dir_entry.get(), cfg, api_path);
process_directory(dir_entry->get_path());
continue;
}
if (process_directory_entry(*dir_entry.get(), cfg, api_path)) {
list.emplace_back(create_api_file(
api_path, dir_entry->is_directory_item(), dir_entry->get_path()));
}
}
}
};
process_directory(cfg.path);
return api_error::success;
} catch (const std::exception &ex) {

79
repertory/librepertory/src/providers/s3/s3_provider.cpp
View File

@ -43,9 +43,9 @@
#include "utils/time.hpp"
namespace {
[[nodiscard]] auto
set_request_path(auto &request,
const std::string &object_name) -> repertory::api_error {
[[nodiscard]] auto set_request_path(auto &request,
const std::string &object_name)
-> repertory::api_error {
request.path = object_name;
if (request.path.substr(1U).size() > repertory::max_s3_object_name_length) {
return repertory::api_error::name_too_long;
@ -59,8 +59,9 @@ namespace repertory {
s3_provider::s3_provider(app_config &config, i_http_comm &comm)
: base_provider(config, comm) {}
auto s3_provider::add_if_not_found(
api_file &file, const std::string &object_name) const -> api_error {
auto s3_provider::add_if_not_found(api_file &file,
const std::string &object_name) const
-> api_error {
api_meta_map meta{};
auto res{get_item_meta(file.api_path, meta)};
if (res == api_error::item_not_found) {
@ -88,7 +89,7 @@ auto s3_provider::convert_api_date(std::string_view date) -> std::uint64_t {
1000000UL,
};
struct tm tm1 {};
struct tm tm1{};
#if defined(_WIN32)
utils::time::strptime(date_time.c_str(), "%Y-%m-%dT%T", &tm1);
return nanos + utils::time::windows_time_t_to_unix_time(_mkgmtime(&tm1));
@ -157,8 +158,9 @@ auto s3_provider::create_directory_impl(const std::string &api_path,
utils::path::create_api_path(is_encrypted ? meta[META_KEY] : api_path));
}
auto s3_provider::create_directory_paths(
const std::string &api_path, const std::string &key) const -> api_error {
auto s3_provider::create_directory_paths(const std::string &api_path,
const std::string &key) const
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
if (api_path == "/") {
@ -321,8 +323,9 @@ auto s3_provider::get_directory_item_count(const std::string &api_path) const
return 0U;
}
auto s3_provider::get_directory_items_impl(
const std::string &api_path, directory_item_list &list) const -> api_error {
auto s3_provider::get_directory_items_impl(const std::string &api_path,
directory_item_list &list) const
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
const auto &cfg{get_s3_config()};
@ -480,8 +483,8 @@ auto s3_provider::get_directory_items_impl(
return api_error::success;
}
auto s3_provider::get_file(const std::string &api_path,
api_file &file) const -> api_error {
auto s3_provider::get_file(const std::string &api_path, api_file &file) const
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
try {
@ -492,7 +495,19 @@ auto s3_provider::get_file(const std::string &api_path,
get_object_info(false, api_path, is_encrypted, object_name, result),
};
if (res != api_error::success) {
return res;
if (res != api_error::item_not_found) {
return res;
}
bool exists{};
res = is_directory(api_path, exists);
if (res != api_error::success) {
utils::error::raise_api_path_error(
function_name, api_path, res,
"failed to determine if directory exists");
}
return exists ? api_error::directory_exists : api_error::item_not_found;
}
file.api_path = api_path;
@ -521,8 +536,8 @@ auto s3_provider::get_file(const std::string &api_path,
return api_error::error;
}
auto s3_provider::get_file_list(api_file_list &list,
std::string &marker) const -> api_error {
auto s3_provider::get_file_list(api_file_list &list, std::string &marker) const
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
try {
@ -612,8 +627,9 @@ auto s3_provider::get_file_list(api_file_list &list,
return api_error::error;
}
auto s3_provider::get_last_modified(
bool directory, const std::string &api_path) const -> std::uint64_t {
auto s3_provider::get_last_modified(bool directory,
const std::string &api_path) const
-> std::uint64_t {
bool is_encrypted{};
std::string object_name;
head_object_result result{};
@ -623,9 +639,10 @@ auto s3_provider::get_last_modified(
: utils::time::get_time_now();
}
auto s3_provider::get_object_info(
bool directory, const std::string &api_path, bool &is_encrypted,
std::string &object_name, head_object_result &result) const -> api_error {
auto s3_provider::get_object_info(bool directory, const std::string &api_path,
bool &is_encrypted, std::string &object_name,
head_object_result &result) const
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
try {
@ -685,10 +702,12 @@ auto s3_provider::get_object_info(
return api_error::error;
}
auto s3_provider::get_object_list(
std::string &response_data, long &response_code,
std::optional<std::string> delimiter, std::optional<std::string> prefix,
std::optional<std::string> token) const -> bool {
auto s3_provider::get_object_list(std::string &response_data,
long &response_code,
std::optional<std::string> delimiter,
std::optional<std::string> prefix,
std::optional<std::string> token) const
-> bool {
curl::requests::http_get get{};
get.allow_timeout = true;
get.aws_service = "aws:amz:" + get_s3_config().region + ":s3";
@ -716,8 +735,8 @@ auto s3_provider::get_total_drive_space() const -> std::uint64_t {
return std::numeric_limits<std::int64_t>::max() / std::int64_t(2);
}
auto s3_provider::is_directory(const std::string &api_path,
bool &exists) const -> api_error {
auto s3_provider::is_directory(const std::string &api_path, bool &exists) const
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
try {
@ -745,8 +764,8 @@ auto s3_provider::is_directory(const std::string &api_path,
return api_error::error;
}
auto s3_provider::is_file(const std::string &api_path,
bool &exists) const -> api_error {
auto s3_provider::is_file(const std::string &api_path, bool &exists) const
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
try {
@ -1004,8 +1023,8 @@ auto s3_provider::rename_file(const std::string & /* from_api_path */,
return api_error::not_implemented;
}
auto s3_provider::set_meta_key(const std::string &api_path,
api_meta_map &meta) -> api_error {
auto s3_provider::set_meta_key(const std::string &api_path, api_meta_map &meta)
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
const auto &cfg{get_s3_config()};

137
repertory/librepertory/src/providers/sia/sia_provider.cpp
View File

@ -32,8 +32,8 @@
#include "providers/base_provider.hpp"
#include "providers/s3/s3_provider.hpp"
#include "types/repertory.hpp"
#include "utils/common.hpp"
#include "utils/error_utils.hpp"
#include "utils/file_utils.hpp"
#include "utils/path.hpp"
#include "utils/polling.hpp"
#include "utils/string.hpp"
@ -63,6 +63,54 @@ namespace repertory {
sia_provider::sia_provider(app_config &config, i_http_comm &comm)
: base_provider(config, comm) {}
auto sia_provider::check_version(std::string &required_version,
std::string &returned_version) const -> bool {
REPERTORY_USES_FUNCTION_NAME();
required_version = "2.0.0";
try {
curl::requests::http_get get{};
get.allow_timeout = true;
get.path = "/api/bus/state";
nlohmann::json state_data;
std::string error_data;
get.response_handler = [&error_data, &state_data](auto &&data,
long response_code) {
if (response_code == http_error_codes::ok) {
state_data = nlohmann::json::parse(data.begin(), data.end());
return;
}
error_data = std::string(data.begin(), data.end());
};
long response_code{};
stop_type stop_requested{};
if (not get_comm().make_request(get, response_code, stop_requested)) {
utils::error::raise_error(function_name, response_code,
"failed to check state");
return false;
}
if (response_code != http_error_codes::ok) {
utils::error::raise_error(
function_name, response_code,
fmt::format("failed to check state|response|{}", error_data));
return false;
}
returned_version = state_data.at("version").get<std::string>().substr(1U);
return utils::compare_version_strings(returned_version, required_version) >=
0;
} catch (const std::exception &e) {
utils::error::raise_error(function_name, e, "failed to check version");
}
return false;
}
auto sia_provider::create_directory_impl(const std::string &api_path,
api_meta_map & /* meta */)
-> api_error {
@ -70,7 +118,7 @@ auto sia_provider::create_directory_impl(const std::string &api_path,
curl::requests::http_put_file put_file{};
put_file.allow_timeout = true;
put_file.path = "/api/worker/objects" + api_path + "/";
put_file.path = "/api/worker/object" + api_path + "/";
put_file.query["bucket"] = get_bucket(get_sia_config());
std::string error_data;
@ -112,10 +160,10 @@ auto sia_provider::get_directory_item_count(const std::string &api_path) const
}
std::uint64_t item_count{};
if (object_list.contains("entries")) {
for (const auto &entry : object_list.at("entries")) {
if (object_list.contains("objects")) {
for (const auto &entry : object_list.at("objects")) {
try {
auto name{entry.at("name").get<std::string>()};
auto name{entry.at("key").get<std::string>()};
auto entry_api_path{utils::path::create_api_path(name)};
if (utils::string::ends_with(name, "/") &&
(entry_api_path == api_path)) {
@ -149,10 +197,10 @@ auto sia_provider::get_directory_items_impl(const std::string &api_path,
return api_error::comm_error;
}
if (object_list.contains("entries")) {
for (const auto &entry : object_list.at("entries")) {
if (object_list.contains("objects")) {
for (const auto &entry : object_list.at("objects")) {
try {
auto name{entry.at("name").get<std::string>()};
auto name{entry.at("key").get<std::string>()};
auto entry_api_path{utils::path::create_api_path(name)};
auto directory{utils::string::ends_with(name, "/")};
@ -185,7 +233,6 @@ auto sia_provider::get_directory_items_impl(const std::string &api_path,
dir_item.api_path = file.api_path;
dir_item.directory = directory;
dir_item.meta = meta;
dir_item.resolved = true;
dir_item.size = file.file_size;
list.emplace_back(std::move(dir_item));
} catch (const std::exception &e) {
@ -207,22 +254,28 @@ auto sia_provider::get_file(const std::string &api_path, api_file &file) const
json file_data{};
auto res{get_object_info(api_path, file_data)};
if (res != api_error::success) {
return res;
if (res != api_error::item_not_found) {
return res;
}
bool exists{};
res = is_directory(api_path, exists);
if (res != api_error::success) {
utils::error::raise_api_path_error(
function_name, api_path, res,
"failed to determine if directory exists");
}
return exists ? api_error::directory_exists : api_error::item_not_found;
}
auto slabs{file_data["object"]["Slabs"]};
auto size{
std::accumulate(
slabs.begin(), slabs.end(), std::uint64_t(0U),
[](auto &&total_size, const json &slab) -> std::uint64_t {
return total_size + slab["Length"].get<std::uint64_t>();
}),
file_data.at("size").get<std::uint64_t>(),
};
api_meta_map meta{};
if (get_item_meta(api_path, meta) == api_error::item_not_found) {
file = create_api_file(api_path, "", size,
get_last_modified(file_data["object"]));
file = create_api_file(api_path, "", size, get_last_modified(file_data));
get_api_item_added()(false, file);
} else {
file = create_api_file(api_path, size, meta);
@ -250,9 +303,9 @@ auto sia_provider::get_file_list(api_file_list &list,
return api_error::comm_error;
}
if (object_list.contains("entries")) {
for (const auto &entry : object_list.at("entries")) {
auto name{entry.at("name").get<std::string>()};
if (object_list.contains("objects")) {
for (const auto &entry : object_list.at("objects")) {
auto name{entry.at("key").get<std::string>()};
auto entry_api_path{utils::path::create_api_path(name)};
if (utils::string::ends_with(name, "/")) {
@ -313,8 +366,9 @@ auto sia_provider::get_object_info(const std::string &api_path,
try {
curl::requests::http_get get{};
get.allow_timeout = true;
get.path = "/api/bus/objects" + api_path;
get.path = "/api/bus/object" + api_path;
get.query["bucket"] = get_bucket(get_sia_config());
get.query["onlymetadata"] = "true";
std::string error_data;
get.response_handler = [&error_data, &object_info](auto &&data,
@ -362,6 +416,7 @@ auto sia_provider::get_object_list(const std::string &api_path,
get.allow_timeout = true;
get.path = "/api/bus/objects" + api_path + "/";
get.query["bucket"] = get_bucket(get_sia_config());
get.query["delimiter"] = "/";
std::string error_data;
get.response_handler = [&error_data, &object_list](auto &&data,
@ -405,7 +460,7 @@ auto sia_provider::get_total_drive_space() const -> std::uint64_t {
try {
curl::requests::http_get get{};
get.allow_timeout = true;
get.path = "/api/autopilot/config";
get.path = "/api/bus/autopilot";
get.query["bucket"] = get_bucket(get_sia_config());
json config_data;
@ -455,17 +510,18 @@ auto sia_provider::is_directory(const std::string &api_path, bool &exists) const
exists = false;
json object_list{};
if (not get_object_list(utils::path::get_parent_api_path(api_path),
object_list)) {
return api_error::comm_error;
json file_data{};
auto res{get_object_info(api_path + '/', file_data)};
if (res == api_error::item_not_found) {
return api_error::success;
}
exists = object_list.contains("entries") &&
std::ranges::find_if(object_list.at("entries"),
[&api_path](auto &&entry) -> bool {
return entry.at("name") == (api_path + "/");
}) != object_list.at("entries").end();
if (res != api_error::success) {
return res;
}
exists =
utils::string::ends_with(file_data.at("key").get<std::string>(), "/");
return api_error::success;
} catch (const std::exception &e) {
utils::error::raise_api_path_error(
@ -481,6 +537,7 @@ auto sia_provider::is_file(const std::string &api_path, bool &exists) const
try {
exists = false;
if (api_path == "/") {
return api_error::success;
}
@ -495,7 +552,8 @@ auto sia_provider::is_file(const std::string &api_path, bool &exists) const
return res;
}
exists = not file_data.contains("entries");
exists = not utils::string::ends_with(
file_data.at("key").get<std::string>(), "/");
return api_error::success;
} catch (const std::exception &e) {
utils::error::raise_api_path_error(function_name, api_path, e,
@ -559,8 +617,9 @@ auto sia_provider::read_file_bytes(const std::string &api_path,
try {
curl::requests::http_get get{};
get.path = "/api/worker/objects" + api_path;
get.path = "/api/worker/object" + api_path;
get.query["bucket"] = get_bucket(get_sia_config());
get.headers["accept"] = "application/octet-stream";
get.range = {{
offset,
offset + size - 1U,
@ -577,6 +636,7 @@ auto sia_provider::read_file_bytes(const std::string &api_path,
++idx) {
long response_code{};
const auto notify_retry = [&]() {
fmt::println("{}", std::string(buffer.begin(), buffer.end()));
if (response_code == 0) {
utils::error::raise_api_path_error(
function_name, api_path, api_error::comm_error,
@ -622,7 +682,7 @@ auto sia_provider::remove_directory_impl(const std::string &api_path)
curl::requests::http_delete del{};
del.allow_timeout = true;
del.path = "/api/bus/objects" + api_path + "/";
del.path = "/api/bus/object" + api_path + "/";
del.query["bucket"] = get_bucket(get_sia_config());
std::string error_data;
@ -658,7 +718,7 @@ auto sia_provider::remove_file_impl(const std::string &api_path) -> api_error {
curl::requests::http_delete del{};
del.allow_timeout = true;
del.path = "/api/bus/objects" + api_path;
del.path = "/api/bus/object" + api_path;
del.query["bucket"] = get_bucket(get_sia_config());
std::string error_data;
@ -697,12 +757,12 @@ auto sia_provider::rename_file(const std::string &from_api_path,
try {
curl::requests::http_post post{};
post.json = nlohmann::json({
{"bucket", get_bucket(get_sia_config())},
{"from", from_api_path},
{"to", to_api_path},
{"mode", "single"},
});
post.path = "/api/bus/objects/rename";
post.query["bucket"] = get_bucket(get_sia_config());
std::string error_data;
post.response_handler = [&error_data](auto &&data, long response_code) {
@ -770,8 +830,9 @@ auto sia_provider::upload_file_impl(const std::string &api_path,
REPERTORY_USES_FUNCTION_NAME();
curl::requests::http_put_file put_file{};
put_file.path = "/api/worker/objects" + api_path;
put_file.path = "/api/worker/object" + api_path;
put_file.query["bucket"] = get_bucket(get_sia_config());
put_file.headers["content-type"] = "application/octet-stream";
put_file.source_path = source_path;
std::string error_data;

71
repertory/librepertory/src/utils/polling.cpp
View File

@ -38,47 +38,58 @@ void polling::frequency_thread(
std::function<std::uint32_t()> get_frequency_seconds, frequency freq) {
REPERTORY_USES_FUNCTION_NAME();
auto last_run = std::chrono::system_clock::time_point::min();
while (not get_stop_requested()) {
unique_mutex_lock lock(mutex_);
auto futures = std::accumulate(
items_.begin(), items_.end(), std::deque<tasks::task_ptr>{},
[this, &freq](auto &&list, auto &&item) -> auto {
if (item.second.freq != freq) {
auto elapsed = std::chrono::duration_cast<std::chrono::seconds>(
std::chrono::system_clock::now() - last_run);
auto max_elapsed = std::chrono::seconds(get_frequency_seconds());
if (last_run == std::chrono::system_clock::time_point::min() ||
elapsed >= max_elapsed) {
unique_mutex_lock lock(mutex_);
auto futures = std::accumulate(
items_.begin(), items_.end(), std::deque<tasks::task_ptr>{},
[this, &freq](auto &&list, auto &&item) -> auto {
if (item.second.freq != freq) {
return list;
}
auto future = tasks::instance().schedule({
[this, &freq, item](auto &&task_stopped) {
if (config_->get_event_level() == event_level::trace ||
freq != frequency::second) {
event_system::instance().raise<polling_item_begin>(
function_name, item.first);
}
item.second.action(task_stopped);
if (config_->get_event_level() == event_level::trace ||
freq != frequency::second) {
event_system::instance().raise<polling_item_end>(
function_name, item.first);
}
},
});
list.emplace_back(future);
return list;
}
auto future = tasks::instance().schedule({
[this, &freq, item](auto &&task_stopped) {
if (config_->get_event_level() == event_level::trace ||
freq != frequency::second) {
event_system::instance().raise<polling_item_begin>(
function_name, item.first);
}
item.second.action(task_stopped);
if (config_->get_event_level() == event_level::trace ||
freq != frequency::second) {
event_system::instance().raise<polling_item_end>(
function_name, item.first);
}
},
});
lock.unlock();
list.emplace_back(future);
return list;
});
lock.unlock();
while (not futures.empty()) {
futures.front()->wait();
futures.pop_front();
}
while (not futures.empty()) {
futures.front()->wait();
futures.pop_front();
last_run = std::chrono::system_clock::now();
elapsed = std::chrono::seconds(0U);
}
unique_mutex_lock lock(mutex_);
if (get_stop_requested()) {
return;
}
lock.lock();
notify_.wait_for(lock, std::chrono::seconds(get_frequency_seconds()));
notify_.wait_for(lock, max_elapsed - elapsed);
}
}