BlockStorage/repertory
1
0
Fork 0
Code Issues 7 Pull Requests Releases 6 Wiki Activity

refactor

Browse Source
This commit is contained in:
Scott E. Graves 2024-12-19 08:33:51 -06:00
parent 60b06790a3
commit e4289295e8
7 changed files with 105 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
repertory/librepertory/include/providers/encrypt/encrypt_provider.hpp
View File

@ -54,6 +54,7 @@ private:
private:
app_config &config_;
encrypt_config encrypt_config_;
private:
std::unique_ptr<i_file_db> db_{nullptr};
@ -73,6 +74,10 @@ private:
const std::string &source_path)>
callback) const -> api_error;
[[nodiscard]] auto get_encrypt_config() const -> const encrypt_config & {
return encrypt_config_;
}
auto process_directory_entry(const utils::file::i_fs_item &dir_entry,
const encrypt_config &cfg,
std::string &api_path) const -> bool;

7
repertory/librepertory/include/providers/s3/s3_provider.hpp
View File

@ -46,6 +46,9 @@ public:
auto operator=(const s3_provider &) -> s3_provider & = delete;
auto operator=(s3_provider &&) -> s3_provider & = delete;
private:
s3_config s3_config_;
private:
[[nodiscard]] auto add_if_not_found(api_file &file,
const std::string &object_name) const
@ -78,6 +81,10 @@ private:
std::optional<std::string> token = std::nullopt) const
-> bool;
[[nodiscard]] auto get_s3_config() const -> const s3_config & {
return s3_config_;
}
protected:
[[nodiscard]] auto create_directory_impl(const std::string &api_path,
api_meta_map &meta)

7
repertory/librepertory/include/providers/sia/sia_provider.hpp
View File

@ -45,6 +45,9 @@ public:
auto operator=(const sia_provider &) -> sia_provider & = delete;
auto operator=(sia_provider &&) -> sia_provider & = delete;
private:
sia_config sia_config_;
private:
[[nodiscard]] auto get_object_info(const std::string &api_path,
json &object_info) const -> api_error;
@ -52,6 +55,10 @@ private:
[[nodiscard]] auto get_object_list(const std::string &api_path,
nlohmann::json &object_list) const -> bool;
[[nodiscard]] auto get_sia_config() const -> const auto & {
return sia_config_;
}
protected:
[[nodiscard]] auto create_directory_impl(const std::string &api_path,
api_meta_map &meta)

20
repertory/librepertory/src/providers/base_provider.cpp
View File

@ -456,7 +456,7 @@ void base_provider::process_removed_files(std::deque<removed_item> removed_list,
REPERTORY_USES_FUNCTION_NAME();
auto orphaned_directory =
utils::path::combine(config_.get_data_directory(), {"orphaned"});
utils::path::combine(get_config().get_data_directory(), {"orphaned"});
for (const auto &item : removed_list) {
if (stop_requested) {
return;
@ -670,8 +670,10 @@ void base_provider::remove_unmatched_source_files(
return;
}
const auto &cfg = get_config();
auto source_list =
utils::file::directory{config_.get_cache_directory()}.get_files();
utils::file::directory{cfg.get_cache_directory()}.get_files();
for (const auto &source_file : source_list) {
if (stop_requested) {
return;
@ -684,15 +686,15 @@ void base_provider::remove_unmatched_source_files(
}
auto reference_time =
source_file->get_time(config_.get_eviction_uses_accessed_time()
source_file->get_time(cfg.get_eviction_uses_accessed_time()
? utils::file::time_type::accessed
: utils::file::time_type::modified);
if (not reference_time.has_value()) {
continue;
}
auto delay = (config_.get_eviction_delay_mins() * 60UL) *
utils::time::NANOS_PER_SECOND;
auto delay =
(cfg.get_eviction_delay_mins() * 60UL) * utils::time::NANOS_PER_SECOND;
if ((reference_time.value() + static_cast<std::uint64_t>(delay)) >=
utils::time::get_time_now()) {
continue;
@ -736,17 +738,19 @@ auto base_provider::start(api_item_added_callback api_item_added,
auto online{false};
auto unmount_requested{false};
{
const auto &cfg = get_config();
repertory::event_consumer consumer(
"unmount_requested",
[&unmount_requested](const event &) { unmount_requested = true; });
for (std::uint16_t idx = 0U; not online && not unmount_requested &&
(idx < config_.get_online_check_retry_secs());
(idx < cfg.get_online_check_retry_secs());
++idx) {
online = is_online();
if (not online) {
event_system::instance().raise<provider_offline>(
config_.get_host_config().host_name_or_ip,
config_.get_host_config().api_port);
cfg.get_host_config().host_name_or_ip,
cfg.get_host_config().api_port);
std::this_thread::sleep_for(1s);
}
}

26
repertory/librepertory/src/providers/encrypt/encrypt_provider.cpp
View File

@ -38,7 +38,8 @@ const std::string file_table = "file";
} // namespace
namespace repertory {
encrypt_provider::encrypt_provider(app_config &config) : config_(config) {}
encrypt_provider::encrypt_provider(app_config &config)
: config_(config), encrypt_config_(config.get_encrypt_config()) {}
auto encrypt_provider::create_api_file(const std::string &api_path,
bool directory,
@ -115,7 +116,8 @@ auto encrypt_provider::do_fs_operation(
std::function<api_error(const encrypt_config &cfg,
const std::string &source_path)>
callback) const -> api_error {
auto cfg = config_.get_encrypt_config();
const auto &cfg = get_encrypt_config();
std::string source_path{api_path};
if (api_path != "/" && not utils::encryption::decrypt_file_path(
cfg.encryption_token, source_path)) {
@ -327,9 +329,9 @@ auto encrypt_provider::get_file_list(api_file_list &list,
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
auto cfg = config_.get_encrypt_config();
try {
const auto &cfg = get_encrypt_config();
for (const auto &dir_entry : utils::file::directory{cfg.path}.get_items()) {
std::string api_path{};
if (process_directory_entry(*dir_entry.get(), cfg, api_path)) {
@ -501,9 +503,8 @@ auto encrypt_provider::get_item_meta(const std::string &api_path,
}
auto encrypt_provider::get_total_drive_space() const -> std::uint64_t {
auto total_space =
utils::file::get_total_drive_space(config_.get_encrypt_config().path);
return total_space.value_or(0U);
return utils::file::get_total_drive_space(get_encrypt_config().path)
.value_or(0U);
}
auto encrypt_provider::get_total_item_count() const -> std::uint64_t {
@ -521,7 +522,7 @@ auto encrypt_provider::get_total_item_count() const -> std::uint64_t {
auto encrypt_provider::get_used_drive_space() const -> std::uint64_t {
auto free_space =
utils::file::get_free_drive_space(config_.get_encrypt_config().path);
utils::file::get_free_drive_space(get_encrypt_config().path);
return free_space.has_value() ? get_total_drive_space() - free_space.value()
: 0U;
}
@ -585,10 +586,7 @@ auto encrypt_provider::is_file_writeable(const std::string & /*api_path*/) const
}
auto encrypt_provider::is_online() const -> bool {
return utils::file::directory{
config_.get_encrypt_config().path,
}
.exists();
return utils::file::directory{get_encrypt_config().path}.exists();
}
auto encrypt_provider::process_directory_entry(
@ -744,7 +742,7 @@ auto encrypt_provider::read_file_bytes(const std::string &api_path,
auto file_size{opt_size.value()};
auto cfg = config_.get_encrypt_config();
const auto &cfg = get_encrypt_config();
unique_recur_mutex_lock reader_lookup_lock(reader_lookup_mtx_);
@ -848,7 +846,7 @@ auto encrypt_provider::start(api_item_added_callback /*api_item_added*/,
fmt::format("failed to get root|{}", api_error_to_string(result)));
}
auto cfg_path = utils::path::absolute(config_.get_encrypt_config().path);
auto cfg_path = utils::path::absolute(get_encrypt_config().path);
if (result == api_error::success) {
auto cur_path = utils::path::absolute(source_path);
#if defined(_WIN32)

94
repertory/librepertory/src/providers/s3/s3_provider.cpp
View File

@ -39,12 +39,11 @@
namespace repertory {
s3_provider::s3_provider(app_config &config, i_http_comm &comm)
: base_provider(config, comm) {
get_comm().enable_s3_path_style(config.get_s3_config().use_path_style);
}
: base_provider(config, comm) {}
auto s3_provider::add_if_not_found(
api_file &file, const std::string &object_name) const -> api_error {
auto s3_provider::add_if_not_found(api_file &file,
const std::string &object_name) const
-> api_error {
api_meta_map meta{};
if (get_item_meta(file.api_path, meta) == api_error::item_not_found) {
auto err = create_path_directories(
@ -70,7 +69,7 @@ auto s3_provider::convert_api_date(std::string_view date) -> std::uint64_t {
utils::string::split(date_parts.at(1U), 'Z', true).at(0U)) *
1000000UL;
struct tm tm1 {};
struct tm tm1{};
#if defined(_WIN32)
utils::time::strptime(date_time.c_str(), "%Y-%m-%dT%T", &tm1);
return nanos + utils::time::windows_time_t_to_unix_time(_mkgmtime(&tm1));
@ -85,7 +84,7 @@ auto s3_provider::create_directory_impl(const std::string &api_path,
api_meta_map &meta) -> api_error {
REPERTORY_USES_FUNCTION_NAME();
auto cfg = get_config().get_s3_config();
const auto &cfg = get_s3_config();
auto is_encrypted = not cfg.encryption_token.empty();
stop_type stop_requested{false};
@ -138,7 +137,8 @@ auto s3_provider::create_file_extra(const std::string &api_path,
api_meta_map &meta) -> api_error {
REPERTORY_USES_FUNCTION_NAME();
if (not get_config().get_s3_config().encryption_token.empty()) {
const auto &cfg = get_s3_config();
if (not cfg.encryption_token.empty()) {
std::string encrypted_file_path;
auto res = get_item_meta(utils::path::get_parent_api_path(api_path),
META_KEY, encrypted_file_path);
@ -150,7 +150,7 @@ auto s3_provider::create_file_extra(const std::string &api_path,
data_buffer result;
utils::encryption::encrypt_data(
get_config().get_s3_config().encryption_token,
cfg.encryption_token,
*(utils::string::split(api_path, '/', false).end() - 1U), result);
meta[META_KEY] = utils::path::create_api_path(
@ -161,15 +161,17 @@ auto s3_provider::create_file_extra(const std::string &api_path,
return api_error::success;
}
auto s3_provider::create_path_directories(
const std::string &api_path, const std::string &key) const -> api_error {
auto s3_provider::create_path_directories(const std::string &api_path,
const std::string &key) const
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
if (api_path == "/") {
return api_error::success;
}
auto encryption_token = get_config().get_s3_config().encryption_token;
const auto &cfg = get_s3_config();
auto encryption_token = cfg.encryption_token;
auto is_encrypted = not encryption_token.empty();
auto path_parts = utils::string::split(api_path, '/', false);
@ -179,8 +181,6 @@ auto s3_provider::create_path_directories(
return api_error::error;
}
auto cfg = get_config().get_s3_config();
std::string cur_key{'/'};
std::string cur_path{'/'};
for (std::size_t idx = 0U; idx < path_parts.size(); ++idx) {
@ -244,7 +244,7 @@ auto s3_provider::decrypt_object_name(std::string &object_name) const
auto parts = utils::string::split(object_name, '/', false);
for (auto &&part : parts) {
if (not utils::encryption::decrypt_file_name(
get_config().get_s3_config().encryption_token, part)) {
get_s3_config().encryption_token, part)) {
return api_error::decryption_error;
}
}
@ -258,7 +258,7 @@ auto s3_provider::get_directory_item_count(const std::string &api_path) const
REPERTORY_USES_FUNCTION_NAME();
try {
auto cfg = get_config().get_s3_config();
const auto &cfg = get_s3_config();
auto is_encrypted = not cfg.encryption_token.empty();
std::string key;
if (is_encrypted) {
@ -330,11 +330,12 @@ auto s3_provider::get_directory_item_count(const std::string &api_path) const
return 0U;
}
auto s3_provider::get_directory_items_impl(
const std::string &api_path, directory_item_list &list) const -> api_error {
auto s3_provider::get_directory_items_impl(const std::string &api_path,
directory_item_list &list) const
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
auto cfg = get_config().get_s3_config();
const auto &cfg = get_s3_config();
auto is_encrypted = not cfg.encryption_token.empty();
auto ret = api_error::success;
@ -476,8 +477,8 @@ auto s3_provider::get_directory_items_impl(
return ret;
}
auto s3_provider::get_file(const std::string &api_path,
api_file &file) const -> api_error {
auto s3_provider::get_file(const std::string &api_path, api_file &file) const
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
try {
@ -516,8 +517,8 @@ auto s3_provider::get_file(const std::string &api_path,
return api_error::error;
}
auto s3_provider::get_file_list(api_file_list &list,
std::string &marker) const -> api_error {
auto s3_provider::get_file_list(api_file_list &list, std::string &marker) const
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
std::string response_data;
@ -559,8 +560,7 @@ auto s3_provider::get_file_list(api_file_list &list,
continue;
}
auto is_encrypted =
not get_config().get_s3_config().encryption_token.empty();
auto is_encrypted = not get_s3_config().encryption_token.empty();
if (is_encrypted) {
auto err = decrypt_object_name(api_path);
if (err != api_error::success) {
@ -593,8 +593,9 @@ auto s3_provider::get_file_list(api_file_list &list,
return grab_more ? api_error::more_data : api_error::success;
}
auto s3_provider::get_last_modified(
bool directory, const std::string &api_path) const -> std::uint64_t {
auto s3_provider::get_last_modified(bool directory,
const std::string &api_path) const
-> std::uint64_t {
bool is_encrypted{};
std::string object_name;
head_object_result result{};
@ -604,13 +605,14 @@ auto s3_provider::get_last_modified(
: utils::time::get_time_now();
}
auto s3_provider::get_object_info(
bool directory, const std::string &api_path, bool &is_encrypted,
std::string &object_name, head_object_result &result) const -> api_error {
auto s3_provider::get_object_info(bool directory, const std::string &api_path,
bool &is_encrypted, std::string &object_name,
head_object_result &result) const
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
try {
auto cfg = get_config().get_s3_config();
const auto &cfg = get_s3_config();
is_encrypted = not cfg.encryption_token.empty();
std::string key;
@ -655,13 +657,15 @@ auto s3_provider::get_object_info(
return api_error::error;
}
auto s3_provider::get_object_list(
std::string &response_data, long &response_code,
std::optional<std::string> delimiter, std::optional<std::string> prefix,
std::optional<std::string> token) const -> bool {
auto s3_provider::get_object_list(std::string &response_data,
long &response_code,
std::optional<std::string> delimiter,
std::optional<std::string> prefix,
std::optional<std::string> token) const
-> bool {
curl::requests::http_get get{};
get.allow_timeout = true;
get.aws_service = "aws:amz:" + get_config().get_s3_config().region + ":s3";
get.aws_service = "aws:amz:" + get_s3_config().region + ":s3";
get.path = '/';
get.query["list-type"] = "2";
if (delimiter.has_value() && not delimiter.value().empty()) {
@ -687,8 +691,8 @@ auto s3_provider::get_total_drive_space() const -> std::uint64_t {
return std::numeric_limits<std::int64_t>::max() / std::int64_t(2);
}
auto s3_provider::is_directory(const std::string &api_path,
bool &exists) const -> api_error {
auto s3_provider::is_directory(const std::string &api_path, bool &exists) const
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
exists = false;
@ -715,8 +719,8 @@ auto s3_provider::is_directory(const std::string &api_path,
return api_error::error;
}
auto s3_provider::is_file(const std::string &api_path,
bool &exists) const -> api_error {
auto s3_provider::is_file(const std::string &api_path, bool &exists) const
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
exists = false;
@ -753,7 +757,7 @@ auto s3_provider::read_file_bytes(const std::string &api_path, std::size_t size,
REPERTORY_USES_FUNCTION_NAME();
try {
auto cfg = get_config().get_s3_config();
const auto &cfg = get_s3_config();
auto is_encrypted = not cfg.encryption_token.empty();
std::string key;
if (is_encrypted) {
@ -859,7 +863,7 @@ auto s3_provider::remove_directory_impl(const std::string &api_path)
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
auto cfg = get_config().get_s3_config();
const auto &cfg = get_s3_config();
auto is_encrypted = not cfg.encryption_token.empty();
std::string key;
@ -901,7 +905,7 @@ auto s3_provider::remove_directory_impl(const std::string &api_path)
auto s3_provider::remove_file_impl(const std::string &api_path) -> api_error {
REPERTORY_USES_FUNCTION_NAME();
auto cfg = get_config().get_s3_config();
const auto &cfg = get_s3_config();
auto is_encrypted = not cfg.encryption_token.empty();
std::string key;
@ -949,6 +953,8 @@ auto s3_provider::rename_file(const std::string & /* from_api_path */,
auto s3_provider::start(api_item_added_callback api_item_added,
i_file_manager *mgr) -> bool {
event_system::instance().raise<service_started>("s3_provider");
s3_config_ = get_config().get_s3_config();
get_comm().enable_s3_path_style(s3_config_.use_path_style);
return base_provider::start(api_item_added, mgr);
}
@ -972,7 +978,7 @@ auto s3_provider::upload_file_impl(const std::string &api_path,
file_size = opt_size.value();
}
auto cfg = get_config().get_s3_config();
const auto &cfg = get_s3_config();
auto is_encrypted = not cfg.encryption_token.empty();
std::string key;

23
repertory/librepertory/src/providers/sia/sia_provider.cpp
View File

@ -37,7 +37,7 @@
#include "utils/utils.hpp"
namespace {
[[nodiscard]] auto get_bucket(repertory::sia_config cfg) -> std::string {
[[nodiscard]] auto get_bucket(const repertory::sia_config &cfg) -> std::string {
repertory::utils::string::trim(cfg.bucket);
if (cfg.bucket.empty()) {
return "default";
@ -68,7 +68,7 @@ auto sia_provider::create_directory_impl(const std::string &api_path,
curl::requests::http_put_file put_file{};
put_file.allow_timeout = true;
put_file.path = "/api/worker/objects" + api_path + "/";
put_file.query["bucket"] = get_bucket(get_config().get_sia_config());
put_file.query["bucket"] = get_bucket(get_sia_config());
long response_code{};
stop_type stop_requested{};
@ -289,7 +289,7 @@ auto sia_provider::get_object_info(const std::string &api_path,
curl::requests::http_get get{};
get.allow_timeout = true;
get.path = "/api/bus/objects" + api_path;
get.query["bucket"] = get_bucket(get_config().get_sia_config());
get.query["bucket"] = get_bucket(get_sia_config());
get.response_handler = [&object_info](const data_buffer &data,
long response_code) {
@ -330,7 +330,7 @@ auto sia_provider::get_object_list(const std::string &api_path,
curl::requests::http_get get{};
get.allow_timeout = true;
get.path = "/api/bus/objects" + api_path + "/";
get.query["bucket"] = get_bucket(get_config().get_sia_config());
get.query["bucket"] = get_bucket(get_sia_config());
get.response_handler = [&object_list](const data_buffer &data,
long response_code) {
@ -364,7 +364,7 @@ auto sia_provider::get_total_drive_space() const -> std::uint64_t {
curl::requests::http_get get{};
get.allow_timeout = true;
get.path = "/api/autopilot/config";
get.query["bucket"] = get_bucket(get_config().get_sia_config());
get.query["bucket"] = get_bucket(get_sia_config());
json config_data{};
get.response_handler = [&config_data](const data_buffer &data,
@ -465,7 +465,7 @@ auto sia_provider::is_online() const -> bool {
curl::requests::http_get get{};
get.allow_timeout = true;
get.path = "/api/bus/consensus/state";
get.query["bucket"] = get_bucket(get_config().get_sia_config());
get.query["bucket"] = get_bucket(get_sia_config());
json state_data{};
get.response_handler = [&state_data](const data_buffer &data,
@ -506,7 +506,7 @@ auto sia_provider::read_file_bytes(const std::string &api_path,
curl::requests::http_get get{};
get.path = "/api/worker/objects" + api_path;
get.query["bucket"] = get_bucket(get_config().get_sia_config());
get.query["bucket"] = get_bucket(get_sia_config());
get.range = {{
offset,
offset + size - 1U,
@ -561,7 +561,7 @@ auto sia_provider::remove_directory_impl(const std::string &api_path)
curl::requests::http_delete del{};
del.allow_timeout = true;
del.path = "/api/bus/objects" + api_path + "/";
del.query["bucket"] = get_bucket(get_config().get_sia_config());
del.query["bucket"] = get_bucket(get_sia_config());
long response_code{};
stop_type stop_requested{};
@ -587,7 +587,7 @@ auto sia_provider::remove_file_impl(const std::string &api_path) -> api_error {
curl::requests::http_delete del{};
del.allow_timeout = true;
del.path = "/api/bus/objects" + api_path;
del.query["bucket"] = get_bucket(get_config().get_sia_config());
del.query["bucket"] = get_bucket(get_sia_config());
long response_code{};
stop_type stop_requested{};
@ -619,7 +619,7 @@ auto sia_provider::rename_file(const std::string &from_api_path,
{"mode", "single"},
});
post.path = "/api/bus/objects/rename";
post.query["bucket"] = get_bucket(get_config().get_sia_config());
post.query["bucket"] = get_bucket(get_sia_config());
long response_code{};
stop_type stop_requested{};
@ -644,6 +644,7 @@ auto sia_provider::rename_file(const std::string &from_api_path,
auto sia_provider::start(api_item_added_callback api_item_added,
i_file_manager *mgr) -> bool {
event_system::instance().raise<service_started>("sia_provider");
sia_config_ = get_config().get_sia_config();
return base_provider::start(api_item_added, mgr);
}
@ -660,7 +661,7 @@ auto sia_provider::upload_file_impl(const std::string &api_path,
curl::requests::http_put_file put_file{};
put_file.path = "/api/worker/objects" + api_path;
put_file.query["bucket"] = get_bucket(get_config().get_sia_config());
put_file.query["bucket"] = get_bucket(get_sia_config());
put_file.source_path = source_path;
long response_code{};