BlockStorage/repertory
1
0
Fork 0
Code Issues 11 Pull Requests Releases 7 Wiki Activity

refactor s3 provider

Browse Source
This commit is contained in:
Scott E. Graves
2023-11-11 11:32:14 -06:00
parent f2c1f64f02
commit 68476cbc00
55 changed files with 1504 additions and 5050 deletions
Download Patch File Download Diff File Expand all files Collapse all files

83
src/providers/sia/sia_provider.cpp
View File

@@ -23,7 +23,6 @@
#include "app_config.hpp"
#include "comm/i_http_comm.hpp"
#include "db/meta_db.hpp"
#include "events/events.hpp"
#include "file_manager/i_file_manager.hpp"
#include "types/repertory.hpp"
@@ -31,6 +30,7 @@
#include "utils/file_utils.hpp"
#include "utils/path_utils.hpp"
#include "utils/polling.hpp"
#include "utils/rocksdb_utils.hpp"
#include "utils/string_utils.hpp"
#include "utils/utils.hpp"
@@ -47,7 +47,7 @@ auto sia_provider::get_object_info(const std::string &api_path,
get.response_handler = [&object_info](const data_buffer &data,
long response_code) {
if (response_code == 200) {
if (response_code == http_error_codes::ok) {
object_info = nlohmann::json::parse(data.begin(), data.end());
}
};
@@ -58,11 +58,11 @@ auto sia_provider::get_object_info(const std::string &api_path,
return api_error::comm_error;
}
if (response_code == 404) {
if (response_code == http_error_codes::not_found) {
return api_error::item_not_found;
}
if (response_code != 200) {
if (response_code != http_error_codes::ok) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, response_code,
"failed to get object info");
return api_error::comm_error;
@@ -77,7 +77,7 @@ auto sia_provider::get_object_info(const std::string &api_path,
return api_error::error;
}
auto sia_provider::get_object_list(const std::string api_path,
auto sia_provider::get_object_list(const std::string &api_path,
nlohmann::json &object_list) const -> bool {
curl::requests::http_get get{};
get.allow_timeout = true;
@@ -85,7 +85,7 @@ auto sia_provider::get_object_list(const std::string api_path,
get.response_handler = [&object_list](const data_buffer &data,
long response_code) {
if (response_code == 200) {
if (response_code == http_error_codes::ok) {
object_list = nlohmann::json::parse(data.begin(), data.end());
}
};
@@ -99,7 +99,7 @@ auto sia_provider::get_object_list(const std::string api_path,
return false;
}
if (response_code != 200) {
if (response_code != http_error_codes::ok) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, response_code,
"failed to get object list");
return false;
@@ -166,8 +166,6 @@ auto sia_provider::create_directory(const std::string &api_path,
try {
curl::requests::http_put_file put_file{};
put_file.file_name =
*(utils::string::split(api_path, '/', false).end() - 1u);
put_file.path = "/api/worker/objects" + api_path + "/";
long response_code{};
@@ -179,7 +177,7 @@ auto sia_provider::create_directory(const std::string &api_path,
return api_error::comm_error;
}
if (response_code != 200) {
if (response_code != http_error_codes::ok) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, response_code,
"failed to create directory");
return api_error::comm_error;
@@ -272,8 +270,6 @@ auto sia_provider::create_file(const std::string &api_path, api_meta_map &meta)
try {
curl::requests::http_put_file put_file{};
put_file.file_name =
*(utils::string::split(api_path, '/', false).end() - 1u);
put_file.path = "/api/worker/objects" + api_path;
long response_code{};
@@ -285,7 +281,7 @@ auto sia_provider::create_file(const std::string &api_path, api_meta_map &meta)
return api_error::comm_error;
}
if (response_code != 200) {
if (response_code != http_error_codes::ok) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, response_code,
"failed to create file");
return api_error::comm_error;
@@ -716,7 +712,7 @@ auto sia_provider::get_total_drive_space() const -> std::uint64_t {
json config_data{};
get.response_handler = [&config_data](const data_buffer &data,
long response_code) {
if (response_code == 200) {
if (response_code == http_error_codes::ok) {
config_data = nlohmann::json::parse(data.begin(), data.end());
}
};
@@ -727,7 +723,7 @@ auto sia_provider::get_total_drive_space() const -> std::uint64_t {
return 0U;
}
if (response_code != 200) {
if (response_code != http_error_codes::ok) {
utils::error::raise_error(__FUNCTION__, response_code,
"failed to get total drive space");
return 0U;
@@ -762,7 +758,7 @@ auto sia_provider::get_used_drive_space() const -> std::uint64_t {
json object_data{};
get.response_handler = [&object_data](const data_buffer &data,
long response_code) {
if (response_code == 200) {
if (response_code == http_error_codes::ok) {
object_data = nlohmann::json::parse(data.begin(), data.end());
}
};
@@ -773,7 +769,7 @@ auto sia_provider::get_used_drive_space() const -> std::uint64_t {
return 0U;
}
if (response_code != 200) {
if (response_code != http_error_codes::ok) {
utils::error::raise_error(__FUNCTION__, response_code,
"failed to get used drive space");
return 0U;
@@ -797,6 +793,8 @@ auto sia_provider::is_directory(const std::string &api_path, bool &exists) const
return api_error::success;
}
exists = false;
try {
json object_list{};
if (not get_object_list(utils::path::get_parent_api_path(api_path),
@@ -821,24 +819,31 @@ auto sia_provider::is_directory(const std::string &api_path, bool &exists) const
auto sia_provider::is_file(const std::string &api_path, bool &exists) const
-> api_error {
exists = false;
if (api_path == "/") {
return api_error::success;
}
json file_data{};
auto res = get_object_info(api_path, file_data);
if (res == api_error::item_not_found) {
exists = false;
try {
json file_data{};
auto res = get_object_info(api_path, file_data);
if (res == api_error::item_not_found) {
return api_error::success;
}
if (res != api_error::success) {
return res;
}
exists = not file_data.contains("entries");
return api_error::success;
} catch (const std::exception &e) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, e,
"failed to determine path is directory");
}
if (res != api_error::success) {
return res;
}
exists = not file_data.contains("entries");
return api_error::success;
return api_error::error;
}
auto sia_provider::is_file_writeable(const std::string &api_path) const
@@ -861,7 +866,7 @@ auto sia_provider::is_online() const -> bool {
json state_data{};
get.response_handler = [&state_data](const data_buffer &data,
long response_code) {
if (response_code == 200) {
if (response_code == http_error_codes::ok) {
state_data = nlohmann::json::parse(data.begin(), data.end());
}
};
@@ -874,7 +879,7 @@ auto sia_provider::is_online() const -> bool {
return false;
}
if (response_code != 200) {
if (response_code != http_error_codes::ok) {
utils::error::raise_error(__FUNCTION__, response_code,
"failed to determine if provider is online");
return false;
@@ -897,7 +902,10 @@ auto sia_provider::read_file_bytes(const std::string &api_path,
stop_type &stop_requested) -> api_error {
curl::requests::http_get get{};
get.path = "/api/worker/objects" + api_path;
get.range = {{offset, offset + size - 1U}};
get.range = {{
offset,
offset + size - 1U,
}};
get.response_handler = [&buffer](const data_buffer &data,
long /*response_code*/) { buffer = data; };
@@ -928,7 +936,8 @@ auto sia_provider::read_file_bytes(const std::string &api_path,
continue;
}
if (response_code < 200 || response_code >= 300) {
if (response_code < http_error_codes::ok ||
response_code >= http_error_codes::multiple_choices) {
notify_retry();
continue;
}
@@ -1064,7 +1073,7 @@ auto sia_provider::remove_directory(const std::string &api_path) -> api_error {
return notify_end(api_error::comm_error);
}
if (response_code != 200) {
if (response_code != http_error_codes::ok) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, response_code,
"failed to remove directory");
return notify_end(api_error::comm_error);
@@ -1138,7 +1147,8 @@ auto sia_provider::remove_file(const std::string &api_path) -> api_error {
return notify_end(api_error::comm_error);
}
if (response_code != 200 && response_code != 404) {
if (response_code != http_error_codes::ok &&
response_code != http_error_codes::not_found) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, response_code,
"failed to remove file");
return notify_end(api_error::comm_error);
@@ -1283,7 +1293,6 @@ void sia_provider::stop() {
auto sia_provider::upload_file(const std::string &api_path,
const std::string &source_path,
const std::string & /* encryption_token */,
stop_type &stop_requested) -> api_error {
event_system::instance().raise<provider_upload_begin>(api_path, source_path);
@@ -1296,8 +1305,6 @@ auto sia_provider::upload_file(const std::string &api_path,
try {
curl::requests::http_put_file put_file{};
put_file.file_name =
*(utils::string::split(api_path, '/', false).end() - 1u);
put_file.path = "/api/worker/objects" + api_path;
put_file.source_path = source_path;
@@ -1309,7 +1316,7 @@ auto sia_provider::upload_file(const std::string &api_path,
return notify_end(api_error::comm_error);
}
if (response_code != 200) {
if (response_code != http_error_codes::ok) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, source_path,
response_code,
"failed to upload file");