BlockStorage/repertory
1
0
Fork 0
Code Issues 3 Pull Requests Releases 9 Wiki Activity

[bug] Sia provider error responses are not being logged #30

Browse Source
This commit is contained in:
sgraves76
2024-12-31 13:34:08 -06:00
parent 1a11d55782
commit 36908f7da9
1 changed files with 174 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files

317
repertory/librepertory/src/providers/sia/sia_provider.cpp
View File

@@ -197,30 +197,39 @@ auto sia_provider::get_directory_items_impl(const std::string &api_path,
auto sia_provider::get_file(const std::string &api_path, api_file &file) const
-> api_error {
json file_data{};
auto res = get_object_info(api_path, file_data);
if (res != api_error::success) {
return res;
REPERTORY_USES_FUNCTION_NAME();
try {
json file_data{};
auto res = get_object_info(api_path, file_data);
if (res != api_error::success) {
return res;
}
auto slabs = file_data["object"]["Slabs"];
auto size = std::accumulate(
slabs.begin(), slabs.end(), std::uint64_t(0U),
[](std::uint64_t total_size,
const nlohmann::json &slab) -> std::uint64_t {
return total_size + slab["Length"].get<std::uint64_t>();
});
api_meta_map meta{};
if (get_item_meta(api_path, meta) == api_error::item_not_found) {
file = create_api_file(api_path, "", size,
get_last_modified(file_data["object"]));
get_api_item_added()(false, file);
} else {
file = create_api_file(api_path, size, meta);
}
return api_error::success;
} catch (const std::exception &e) {
utils::error::raise_api_path_error(function_name, api_path, e,
"failed to get file");
}
auto slabs = file_data["object"]["Slabs"];
auto size =
std::accumulate(slabs.begin(), slabs.end(), std::uint64_t(0U),
[](std::uint64_t total_size,
const nlohmann::json &slab) -> std::uint64_t {
return total_size + slab["Length"].get<std::uint64_t>();
});
api_meta_map meta{};
if (get_item_meta(api_path, meta) == api_error::item_not_found) {
file = create_api_file(api_path, "", size,
get_last_modified(file_data["object"]));
get_api_item_added()(false, file);
} else {
file = create_api_file(api_path, size, meta);
}
return api_error::success;
return api_error::error;
}
auto sia_provider::get_file_list(api_file_list &list,
@@ -341,39 +350,46 @@ auto sia_provider::get_object_list(const std::string &api_path,
nlohmann::json &object_list) const -> bool {
REPERTORY_USES_FUNCTION_NAME();
curl::requests::http_get get{};
get.allow_timeout = true;
get.path = "/api/bus/objects" + api_path + "/";
get.query["bucket"] = get_bucket(get_sia_config());
try {
curl::requests::http_get get{};
get.allow_timeout = true;
get.path = "/api/bus/objects" + api_path + "/";
get.query["bucket"] = get_bucket(get_sia_config());
std::string error_data;
get.response_handler = [&error_data, &object_list](auto &&data,
long response_code) {
if (response_code == http_error_codes::ok) {
object_list = nlohmann::json::parse(data.begin(), data.end());
return;
std::string error_data;
get.response_handler = [&error_data, &object_list](auto &&data,
long response_code) {
if (response_code == http_error_codes::ok) {
object_list = nlohmann::json::parse(data.begin(), data.end());
return;
}
error_data = std::string(data.begin(), data.end());
};
long response_code{};
stop_type stop_requested{};
if (not get_comm().make_request(get, response_code, stop_requested)) {
utils::error::raise_api_path_error(function_name, api_path,
api_error::comm_error,
"failed to get object list");
return false;
}
error_data = std::string(data.begin(), data.end());
};
if (response_code != http_error_codes::ok) {
utils::error::raise_api_path_error(
function_name, api_path, response_code,
fmt::format("failed to get object list|response|{}", error_data));
return false;
}
long response_code{};
stop_type stop_requested{};
if (not get_comm().make_request(get, response_code, stop_requested)) {
utils::error::raise_api_path_error(function_name, api_path,
api_error::comm_error,
return true;
} catch (const std::exception &e) {
utils::error::raise_api_path_error(function_name, api_path, e,
"failed to get object list");
return false;
}
if (response_code != http_error_codes::ok) {
utils::error::raise_api_path_error(
function_name, api_path, response_code,
fmt::format("failed to get object list|response|{}", error_data));
return false;
}
return true;
return false;
}
auto sia_provider::get_total_drive_space() const -> std::uint64_t {
@@ -404,8 +420,8 @@ auto sia_provider::get_total_drive_space() const -> std::uint64_t {
}
if (response_code != http_error_codes::ok) {
utils::error::raise_api_path_error(
function_name, api_path, response_code,
utils::error::raise_error(
function_name, response_code,
fmt::format("failed to get total drive space|response|{}",
error_data));
return 0U;
@@ -424,14 +440,14 @@ auto sia_provider::is_directory(const std::string &api_path, bool &exists) const
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
if (api_path == "/") {
exists = true;
return api_error::success;
}
exists = false;
try {
if (api_path == "/") {
exists = true;
return api_error::success;
}
exists = false;
json object_list{};
if (not get_object_list(utils::path::get_parent_api_path(api_path),
object_list)) {
@@ -446,8 +462,8 @@ auto sia_provider::is_directory(const std::string &api_path, bool &exists) const
}) != object_list.at("entries").end();
return api_error::success;
} catch (const std::exception &e) {
utils::error::raise_api_path_error(function_name, api_path, e,
"failed to determine path is directory");
utils::error::raise_api_path_error(
function_name, api_path, e, "failed to determine if path is directory");
}
return api_error::error;
@@ -457,12 +473,12 @@ auto sia_provider::is_file(const std::string &api_path, bool &exists) const
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
exists = false;
if (api_path == "/") {
return api_error::success;
}
try {
exists = false;
if (api_path == "/") {
return api_error::success;
}
json file_data{};
auto res = get_object_info(api_path, file_data);
if (res == api_error::item_not_found) {
@@ -477,7 +493,7 @@ auto sia_provider::is_file(const std::string &api_path, bool &exists) const
return api_error::success;
} catch (const std::exception &e) {
utils::error::raise_api_path_error(function_name, api_path, e,
"failed to determine path is directory");
"failed to determine if path is file");
}
return api_error::error;
@@ -513,8 +529,8 @@ auto sia_provider::is_online() const -> bool {
}
if (response_code != http_error_codes::ok) {
utils::error::raise_api_path_error(
function_name, api_path, response_code,
utils::error::raise_error(
function_name, response_code,
fmt::format("failed to determine if provider is online|response|{}",
error_data));
return false;
@@ -535,55 +551,62 @@ auto sia_provider::read_file_bytes(const std::string &api_path,
stop_type &stop_requested) -> api_error {
REPERTORY_USES_FUNCTION_NAME();
curl::requests::http_get get{};
get.path = "/api/worker/objects" + api_path;
get.query["bucket"] = get_bucket(get_sia_config());
get.range = {{
offset,
offset + size - 1U,
}};
get.response_handler = [&buffer](auto &&data, long /* response_code */) {
buffer = data;
};
auto res = api_error::comm_error;
for (std::uint32_t idx = 0U;
not stop_requested && res != api_error::success &&
idx < get_config().get_retry_read_count() + 1U;
++idx) {
long response_code{};
const auto notify_retry = [&]() {
if (response_code == 0) {
utils::error::raise_api_path_error(
function_name, api_path, api_error::comm_error,
"read file bytes failed|offset|" + std::to_string(offset) +
"|size|" + std::to_string(size) + "|retry|" +
std::to_string(idx + 1U));
} else {
utils::error::raise_api_path_error(
function_name, api_path, response_code,
"read file bytes failed|offset|" + std::to_string(offset) +
"|size|" + std::to_string(size) + "|retry|" +
std::to_string(idx + 1U));
}
std::this_thread::sleep_for(1s);
try {
curl::requests::http_get get{};
get.path = "/api/worker/objects" + api_path;
get.query["bucket"] = get_bucket(get_sia_config());
get.range = {{
offset,
offset + size - 1U,
}};
get.response_handler = [&buffer](auto &&data, long /* response_code */) {
buffer = data;
};
if (not get_comm().make_request(get, response_code, stop_requested)) {
notify_retry();
continue;
auto res{api_error::comm_error};
for (std::uint32_t idx = 0U;
not stop_requested && res != api_error::success &&
idx < get_config().get_retry_read_count() + 1U;
++idx) {
long response_code{};
const auto notify_retry = [&]() {
if (response_code == 0) {
utils::error::raise_api_path_error(
function_name, api_path, api_error::comm_error,
"read file bytes failed|offset|" + std::to_string(offset) +
"|size|" + std::to_string(size) + "|retry|" +
std::to_string(idx + 1U));
} else {
utils::error::raise_api_path_error(
function_name, api_path, response_code,
"read file bytes failed|offset|" + std::to_string(offset) +
"|size|" + std::to_string(size) + "|retry|" +
std::to_string(idx + 1U));
}
std::this_thread::sleep_for(1s);
};
if (not get_comm().make_request(get, response_code, stop_requested)) {
notify_retry();
continue;
}
if (response_code < http_error_codes::ok ||
response_code >= http_error_codes::multiple_choices) {
notify_retry();
continue;
}
res = api_error::success;
}
if (response_code < http_error_codes::ok ||
response_code >= http_error_codes::multiple_choices) {
notify_retry();
continue;
}
res = api_error::success;
return res;
} catch (const std::exception &e) {
utils::error::raise_api_path_error(function_name, e, api_path,
"failed to read file bytes");
}
return res;
return api_error::error;
}
auto sia_provider::remove_directory_impl(const std::string &api_path)
@@ -614,8 +637,9 @@ auto sia_provider::remove_directory_impl(const std::string &api_path)
}
if (response_code != http_error_codes::ok) {
utils::error::raise_api_path_error(function_name, api_path, response_code,
fmt::format("failed to remove directory|response|", error_data);
utils::error::raise_api_path_error(
function_name, api_path, response_code,
fmt::format("failed to remove directory|response|{}", error_data));
return api_error::comm_error;
}
@@ -663,43 +687,50 @@ auto sia_provider::rename_file(const std::string &from_api_path,
const std::string &to_api_path) -> api_error {
REPERTORY_USES_FUNCTION_NAME();
curl::requests::http_post post{};
post.json = nlohmann::json({
{"from", from_api_path},
{"to", to_api_path},
{"mode", "single"},
});
post.path = "/api/bus/objects/rename";
post.query["bucket"] = get_bucket(get_sia_config());
try {
curl::requests::http_post post{};
post.json = nlohmann::json({
{"from", from_api_path},
{"to", to_api_path},
{"mode", "single"},
});
post.path = "/api/bus/objects/rename";
post.query["bucket"] = get_bucket(get_sia_config());
std::string error_data;
post.response_handler = [&error_data](auto &&data, long response_code) {
if (response_code == http_error_codes::ok) {
return;
std::string error_data;
post.response_handler = [&error_data](auto &&data, long response_code) {
if (response_code == http_error_codes::ok) {
return;
}
error_data = std::string(data.begin(), data.end());
};
long response_code{};
stop_type stop_requested{};
if (not get_comm().make_request(post, response_code, stop_requested)) {
utils::error::raise_api_path_error(
function_name, fmt::format("{}|{}", from_api_path, to_api_path),
api_error::comm_error, "failed to rename file");
return api_error::comm_error;
}
error_data = std::string(data.begin(), data.end());
};
if (response_code < http_error_codes::ok ||
response_code >= http_error_codes::multiple_choices) {
utils::error::raise_api_path_error(
function_name, fmt::format("{}|{}", from_api_path, to_api_path),
response_code,
fmt::format("failed to rename file file|response|{}", error_data));
return api_error::comm_error;
}
long response_code{};
stop_type stop_requested{};
if (not get_comm().make_request(post, response_code, stop_requested)) {
utils::error::raise_api_path_error(
function_name, fmt::format("{}|{}", from_api_path, to_api_path), api_error::comm_error,
"failed to rename file");
return api_error::comm_error;
return get_db().rename_item_meta(from_api_path, to_api_path);
} catch (const std::exception &e) {
utils::error::raise_api_path_error(function_name, e, api_path,
"failed to rename file");
}
if (response_code < http_error_codes::ok ||
response_code >= http_error_codes::multiple_choices) {
utils::error::raise_api_path_error(
function_name, fmt::format("{}|{}", from_api_path, to_api_path),
response_code,
fmt::format("failed to rename file file|response|{}", error_data));
return api_error::comm_error;
}
return get_db().rename_item_meta(from_api_path, to_api_path);
return api_error::error;
}
auto sia_provider::start(api_item_added_callback api_item_added,