BlockStorage/repertory
1
0
Fork 0
Code Issues 11 Pull Requests Releases 7 Wiki Activity

revert
All checks were successful
BlockStorage/repertory_linux_builds/pipeline/head This commit looks good
Details
BlockStorage/repertory_osx_builds/pipeline/head This commit looks good
Details

Browse Source
This commit is contained in:
Scott E. Graves
2024-01-29 11:36:26 -06:00
parent d175a38ad1
commit 99533a9687
213 changed files with 43429 additions and 41103 deletions
Download Patch File Download Diff File Expand all files Collapse all files

126
src/providers/sia/sia_provider.cpp
View File

@@ -1,5 +1,5 @@
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Copyright <2018-2024> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
@@ -31,7 +31,6 @@
#include "utils/file_utils.hpp"
#include "utils/path_utils.hpp"
#include "utils/polling.hpp"
#include "utils/rocksdb_utils.hpp"
#include "utils/string_utils.hpp"
#include "utils/utils.hpp"
@@ -42,6 +41,8 @@ sia_provider::sia_provider(app_config &config, i_http_comm &comm)
auto sia_provider::create_directory_impl(const std::string &api_path,
api_meta_map & /* meta */)
-> api_error {
constexpr const auto *function_name = static_cast<const char *>(__FUNCTION__);
curl::requests::http_put_file put_file{};
put_file.allow_timeout = true;
put_file.path = "/api/worker/objects" + api_path + "/";
@@ -49,14 +50,14 @@ auto sia_provider::create_directory_impl(const std::string &api_path,
long response_code{};
stop_type stop_requested{};
if (not get_comm().make_request(put_file, response_code, stop_requested)) {
utils::error::raise_api_path_error(__FUNCTION__, api_path,
utils::error::raise_api_path_error(function_name, api_path,
api_error::comm_error,
"failed to create directory");
return api_error::comm_error;
}
if (response_code != http_error_codes::ok) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, response_code,
utils::error::raise_api_path_error(function_name, api_path, response_code,
"failed to create directory");
return api_error::comm_error;
}
@@ -66,6 +67,8 @@ auto sia_provider::create_directory_impl(const std::string &api_path,
auto sia_provider::get_directory_item_count(const std::string &api_path) const
-> std::uint64_t {
constexpr const auto *function_name = static_cast<const char *>(__FUNCTION__);
try {
json object_list{};
if (not get_object_list(api_path, object_list)) {
@@ -84,7 +87,7 @@ auto sia_provider::get_directory_item_count(const std::string &api_path) const
}
++item_count;
} catch (const std::exception &e) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, e,
utils::error::raise_api_path_error(function_name, api_path, e,
"failed to process entry|" +
entry.dump());
}
@@ -93,7 +96,7 @@ auto sia_provider::get_directory_item_count(const std::string &api_path) const
return item_count;
} catch (const std::exception &e) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, e,
utils::error::raise_api_path_error(function_name, api_path, e,
"failed to get directory item count");
}
@@ -103,6 +106,8 @@ auto sia_provider::get_directory_item_count(const std::string &api_path) const
auto sia_provider::get_directory_items_impl(const std::string &api_path,
directory_item_list &list) const
-> api_error {
constexpr const auto *function_name = static_cast<const char *>(__FUNCTION__);
json object_list{};
if (not get_object_list(api_path, object_list)) {
return api_error::comm_error;
@@ -129,7 +134,7 @@ auto sia_provider::get_directory_items_impl(const std::string &api_path,
get_api_item_added()(directory, file);
auto res = get_item_meta(entry_api_path, meta);
if (res != api_error::success) {
utils::error::raise_error(__FUNCTION__, res,
utils::error::raise_error(function_name, res,
"failed to get item meta");
continue;
}
@@ -148,7 +153,7 @@ auto sia_provider::get_directory_items_impl(const std::string &api_path,
dir_item.size = file.file_size;
list.emplace_back(std::move(dir_item));
} catch (const std::exception &e) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, e,
utils::error::raise_api_path_error(function_name, api_path, e,
"failed to process entry|" +
entry.dump());
}
@@ -186,6 +191,8 @@ auto sia_provider::get_file(const std::string &api_path, api_file &file) const
}
auto sia_provider::get_file_list(api_file_list &list) const -> api_error {
constexpr const auto *function_name = static_cast<const char *>(__FUNCTION__);
using dir_func = std::function<api_error(std::string api_path)>;
const dir_func get_files_in_dir = [&](std::string api_path) -> api_error {
try {
@@ -236,7 +243,7 @@ auto sia_provider::get_file_list(api_file_list &list) const -> api_error {
return api_error::success;
} catch (const std::exception &e) {
utils::error::raise_error(__FUNCTION__, e,
utils::error::raise_error(function_name, e,
"failed to process directory|" + api_path);
}
@@ -248,6 +255,8 @@ auto sia_provider::get_file_list(api_file_list &list) const -> api_error {
auto sia_provider::get_object_info(const std::string &api_path,
json &object_info) const -> api_error {
constexpr const auto *function_name = static_cast<const char *>(__FUNCTION__);
try {
curl::requests::http_get get{};
get.allow_timeout = true;
@@ -271,14 +280,14 @@ auto sia_provider::get_object_info(const std::string &api_path,
}
if (response_code != http_error_codes::ok) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, response_code,
utils::error::raise_api_path_error(function_name, api_path, response_code,
"failed to get object info");
return api_error::comm_error;
}
return api_error::success;
} catch (const std::exception &e) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, e,
utils::error::raise_api_path_error(function_name, api_path, e,
"failed to get object info");
}
@@ -287,6 +296,8 @@ auto sia_provider::get_object_info(const std::string &api_path,
auto sia_provider::get_object_list(const std::string &api_path,
nlohmann::json &object_list) const -> bool {
constexpr const auto *function_name = static_cast<const char *>(__FUNCTION__);
curl::requests::http_get get{};
get.allow_timeout = true;
get.path = "/api/bus/objects" + api_path + "/";
@@ -301,14 +312,14 @@ auto sia_provider::get_object_list(const std::string &api_path,
long response_code{};
stop_type stop_requested{};
if (not get_comm().make_request(get, response_code, stop_requested)) {
utils::error::raise_api_path_error(__FUNCTION__, api_path,
utils::error::raise_api_path_error(function_name, api_path,
api_error::comm_error,
"failed to get object list");
return false;
}
if (response_code != http_error_codes::ok) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, response_code,
utils::error::raise_api_path_error(function_name, api_path, response_code,
"failed to get object list");
return false;
}
@@ -317,6 +328,8 @@ auto sia_provider::get_object_list(const std::string &api_path,
}
auto sia_provider::get_total_drive_space() const -> std::uint64_t {
constexpr const auto *function_name = static_cast<const char *>(__FUNCTION__);
try {
curl::requests::http_get get{};
get.allow_timeout = true;
@@ -337,14 +350,14 @@ auto sia_provider::get_total_drive_space() const -> std::uint64_t {
}
if (response_code != http_error_codes::ok) {
utils::error::raise_error(__FUNCTION__, response_code,
utils::error::raise_error(function_name, response_code,
"failed to get total drive space");
return 0U;
}
return config_data["contracts"]["storage"].get<std::uint64_t>();
} catch (const std::exception &e) {
utils::error::raise_error(__FUNCTION__, e,
utils::error::raise_error(function_name, e,
"failed to get total drive space");
}
@@ -352,6 +365,8 @@ auto sia_provider::get_total_drive_space() const -> std::uint64_t {
}
auto sia_provider::get_used_drive_space_impl() const -> std::uint64_t {
constexpr const auto *function_name = static_cast<const char *>(__FUNCTION__);
curl::requests::http_get get{};
get.allow_timeout = true;
get.path = "/api/bus/stats/objects";
@@ -371,7 +386,7 @@ auto sia_provider::get_used_drive_space_impl() const -> std::uint64_t {
}
if (response_code != http_error_codes::ok) {
utils::error::raise_error(__FUNCTION__, response_code,
utils::error::raise_error(function_name, response_code,
"failed to get used drive space");
return 0U;
}
@@ -381,6 +396,8 @@ auto sia_provider::get_used_drive_space_impl() const -> std::uint64_t {
auto sia_provider::is_directory(const std::string &api_path, bool &exists) const
-> api_error {
constexpr const auto *function_name = static_cast<const char *>(__FUNCTION__);
if (api_path == "/") {
exists = true;
return api_error::success;
@@ -403,7 +420,7 @@ auto sia_provider::is_directory(const std::string &api_path, bool &exists) const
}) != object_list.at("entries").end();
return api_error::success;
} catch (const std::exception &e) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, e,
utils::error::raise_api_path_error(function_name, api_path, e,
"failed to determine path is directory");
}
@@ -412,6 +429,8 @@ auto sia_provider::is_directory(const std::string &api_path, bool &exists) const
auto sia_provider::is_file(const std::string &api_path, bool &exists) const
-> api_error {
constexpr const auto *function_name = static_cast<const char *>(__FUNCTION__);
exists = false;
if (api_path == "/") {
return api_error::success;
@@ -431,7 +450,7 @@ auto sia_provider::is_file(const std::string &api_path, bool &exists) const
exists = not file_data.contains("entries");
return api_error::success;
} catch (const std::exception &e) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, e,
utils::error::raise_api_path_error(function_name, api_path, e,
"failed to determine path is directory");
}
@@ -439,6 +458,8 @@ auto sia_provider::is_file(const std::string &api_path, bool &exists) const
}
auto sia_provider::is_online() const -> bool {
constexpr const auto *function_name = static_cast<const char *>(__FUNCTION__);
try {
curl::requests::http_get get{};
get.allow_timeout = true;
@@ -455,22 +476,22 @@ auto sia_provider::is_online() const -> bool {
long response_code{};
stop_type stop_requested{};
if (not get_comm().make_request(get, response_code, stop_requested)) {
utils::error::raise_error(__FUNCTION__, api_error::comm_error,
utils::error::raise_error(function_name, api_error::comm_error,
"failed to determine if provider is online");
return false;
}
if (response_code != http_error_codes::ok) {
utils::error::raise_error(__FUNCTION__, response_code,
utils::error::raise_error(function_name, response_code,
"failed to determine if provider is online");
return false;
}
event_system::instance().raise<debug_log>(__FUNCTION__, "",
event_system::instance().raise<debug_log>(function_name, "",
state_data.dump());
return true;
} catch (const std::exception &e) {
utils::error::raise_error(__FUNCTION__, e,
utils::error::raise_error(function_name, e,
"failed to determine if provider is online");
}
@@ -481,6 +502,8 @@ auto sia_provider::read_file_bytes(const std::string &api_path,
std::size_t size, std::uint64_t offset,
data_buffer &buffer,
stop_type &stop_requested) -> api_error {
constexpr const auto *function_name = static_cast<const char *>(__FUNCTION__);
curl::requests::http_get get{};
get.path = "/api/worker/objects" + api_path;
get.range = {{
@@ -498,13 +521,13 @@ auto sia_provider::read_file_bytes(const std::string &api_path,
const auto notify_retry = [&]() {
if (response_code == 0) {
utils::error::raise_api_path_error(
__FUNCTION__, api_path, api_error::comm_error,
function_name, api_path, api_error::comm_error,
"read file bytes failed|offset|" + std::to_string(offset) +
"|size|" + std::to_string(size) + "|retry|" +
std::to_string(i + 1U));
} else {
utils::error::raise_api_path_error(
__FUNCTION__, api_path, response_code,
function_name, api_path, response_code,
"read file bytes failed|offset|" + std::to_string(offset) +
"|size|" + std::to_string(size) + "|retry|" +
std::to_string(i + 1U));
@@ -531,6 +554,8 @@ auto sia_provider::read_file_bytes(const std::string &api_path,
auto sia_provider::remove_directory_impl(const std::string &api_path)
-> api_error {
constexpr const auto *function_name = static_cast<const char *>(__FUNCTION__);
curl::requests::http_delete del{};
del.allow_timeout = true;
del.path = "/api/bus/objects" + api_path + "/";
@@ -538,14 +563,14 @@ auto sia_provider::remove_directory_impl(const std::string &api_path)
long response_code{};
stop_type stop_requested{};
if (not get_comm().make_request(del, response_code, stop_requested)) {
utils::error::raise_api_path_error(__FUNCTION__, api_path,
utils::error::raise_api_path_error(function_name, api_path,
api_error::comm_error,
"failed to remove directory");
return api_error::comm_error;
}
if (response_code != http_error_codes::ok) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, response_code,
utils::error::raise_api_path_error(function_name, api_path, response_code,
"failed to remove directory");
return api_error::comm_error;
}
@@ -554,6 +579,8 @@ auto sia_provider::remove_directory_impl(const std::string &api_path)
}
auto sia_provider::remove_file_impl(const std::string &api_path) -> api_error {
constexpr const auto *function_name = static_cast<const char *>(__FUNCTION__);
curl::requests::http_delete del{};
del.allow_timeout = true;
del.path = "/api/bus/objects" + api_path;
@@ -561,14 +588,15 @@ auto sia_provider::remove_file_impl(const std::string &api_path) -> api_error {
long response_code{};
stop_type stop_requested{};
if (not get_comm().make_request(del, response_code, stop_requested)) {
utils::error::raise_api_path_error(
__FUNCTION__, api_path, api_error::comm_error, "failed to remove file");
utils::error::raise_api_path_error(function_name, api_path,
api_error::comm_error,
"failed to remove file");
return api_error::comm_error;
}
if (response_code != http_error_codes::ok &&
response_code != http_error_codes::not_found) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, response_code,
utils::error::raise_api_path_error(function_name, api_path, response_code,
"failed to remove file");
return api_error::comm_error;
}
@@ -576,10 +604,36 @@ auto sia_provider::remove_file_impl(const std::string &api_path) -> api_error {
return api_error::success;
}
auto sia_provider::rename_file(const std::string & /*from_api_path*/,
const std::string & /*to_api_path*/)
-> api_error {
return api_error::not_implemented;
auto sia_provider::rename_file(const std::string &from_api_path,
const std::string &to_api_path) -> api_error {
constexpr const auto *function_name = static_cast<const char *>(__FUNCTION__);
curl::requests::http_post post{};
post.path = "/api/bus/objects/rename";
post.json = nlohmann::json({
{"from", from_api_path},
{"to", to_api_path},
{"mode", "single"},
});
long response_code{};
stop_type stop_requested{};
if (not get_comm().make_request(post, response_code, stop_requested)) {
utils::error::raise_api_path_error(
function_name, from_api_path + '|' + to_api_path, api_error::comm_error,
"failed to rename file");
return api_error::comm_error;
}
if (response_code < http_error_codes::ok ||
response_code >= http_error_codes::multiple_choices) {
utils::error::raise_api_path_error(
function_name, from_api_path + '|' + to_api_path, response_code,
"failed to rename file file");
return api_error::comm_error;
}
return get_db().rename_item_meta(from_api_path, to_api_path);
}
auto sia_provider::start(api_item_added_callback api_item_added,
@@ -597,20 +651,22 @@ void sia_provider::stop() {
auto sia_provider::upload_file_impl(const std::string &api_path,
const std::string &source_path,
stop_type &stop_requested) -> api_error {
constexpr const auto *function_name = static_cast<const char *>(__FUNCTION__);
curl::requests::http_put_file put_file{};
put_file.path = "/api/worker/objects" + api_path;
put_file.source_path = source_path;
long response_code{};
if (not get_comm().make_request(put_file, response_code, stop_requested)) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, source_path,
utils::error::raise_api_path_error(function_name, api_path, source_path,
api_error::comm_error,
"failed to upload file");
return api_error::comm_error;
}
if (response_code != http_error_codes::ok) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, source_path,
utils::error::raise_api_path_error(function_name, api_path, source_path,
response_code, "failed to upload file");
return api_error::comm_error;
}