BlockStorage/repertory
1
0
Fork 0
Code Issues 7 Pull Requests Releases 6 Wiki Activity

[bug] Sia provider error responses are not being logged #30

Browse Source
This commit is contained in:
Scott E. Graves 2024-12-31 13:34:08 -06:00
parent 1a11d55782
commit 36908f7da9
1 changed files with 174 additions and 143 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

317
repertory/librepertory/src/providers/sia/sia_provider.cpp
View File

@ -197,30 +197,39 @@ auto sia_provider::get_directory_items_impl(const std::string &api_path,
auto sia_provider::get_file(const std::string &api_path, api_file &file) const auto sia_provider::get_file(const std::string &api_path, api_file &file) const
-> api_error { -> api_error {
json file_data{}; REPERTORY_USES_FUNCTION_NAME();
auto res = get_object_info(api_path, file_data);
if (res != api_error::success) { try {
return res; json file_data{};
auto res = get_object_info(api_path, file_data);
if (res != api_error::success) {
return res;
}
auto slabs = file_data["object"]["Slabs"];
auto size = std::accumulate(
slabs.begin(), slabs.end(), std::uint64_t(0U),
[](std::uint64_t total_size,
const nlohmann::json &slab) -> std::uint64_t {
return total_size + slab["Length"].get<std::uint64_t>();
});
api_meta_map meta{};
if (get_item_meta(api_path, meta) == api_error::item_not_found) {
file = create_api_file(api_path, "", size,
get_last_modified(file_data["object"]));
get_api_item_added()(false, file);
} else {
file = create_api_file(api_path, size, meta);
}
return api_error::success;
} catch (const std::exception &e) {
utils::error::raise_api_path_error(function_name, api_path, e,
"failed to get file");
} }
auto slabs = file_data["object"]["Slabs"]; return api_error::error;
auto size =
std::accumulate(slabs.begin(), slabs.end(), std::uint64_t(0U),
[](std::uint64_t total_size,
const nlohmann::json &slab) -> std::uint64_t {
return total_size + slab["Length"].get<std::uint64_t>();
});
api_meta_map meta{};
if (get_item_meta(api_path, meta) == api_error::item_not_found) {
file = create_api_file(api_path, "", size,
get_last_modified(file_data["object"]));
get_api_item_added()(false, file);
} else {
file = create_api_file(api_path, size, meta);
}
return api_error::success;
} }
auto sia_provider::get_file_list(api_file_list &list, auto sia_provider::get_file_list(api_file_list &list,
@ -341,39 +350,46 @@ auto sia_provider::get_object_list(const std::string &api_path,
nlohmann::json &object_list) const -> bool { nlohmann::json &object_list) const -> bool {
REPERTORY_USES_FUNCTION_NAME(); REPERTORY_USES_FUNCTION_NAME();
curl::requests::http_get get{}; try {
get.allow_timeout = true; curl::requests::http_get get{};
get.path = "/api/bus/objects" + api_path + "/"; get.allow_timeout = true;
get.query["bucket"] = get_bucket(get_sia_config()); get.path = "/api/bus/objects" + api_path + "/";
get.query["bucket"] = get_bucket(get_sia_config());
std::string error_data; std::string error_data;
get.response_handler = [&error_data, &object_list](auto &&data, get.response_handler = [&error_data, &object_list](auto &&data,
long response_code) { long response_code) {
if (response_code == http_error_codes::ok) { if (response_code == http_error_codes::ok) {
object_list = nlohmann::json::parse(data.begin(), data.end()); object_list = nlohmann::json::parse(data.begin(), data.end());
return; return;
}
error_data = std::string(data.begin(), data.end());
};
long response_code{};
stop_type stop_requested{};
if (not get_comm().make_request(get, response_code, stop_requested)) {
utils::error::raise_api_path_error(function_name, api_path,
api_error::comm_error,
"failed to get object list");
return false;
} }
error_data = std::string(data.begin(), data.end()); if (response_code != http_error_codes::ok) {
}; utils::error::raise_api_path_error(
function_name, api_path, response_code,
fmt::format("failed to get object list|response|{}", error_data));
return false;
}
long response_code{}; return true;
stop_type stop_requested{}; } catch (const std::exception &e) {
if (not get_comm().make_request(get, response_code, stop_requested)) { utils::error::raise_api_path_error(function_name, api_path, e,
utils::error::raise_api_path_error(function_name, api_path,
api_error::comm_error,
"failed to get object list"); "failed to get object list");
return false;
} }
if (response_code != http_error_codes::ok) { return false;
utils::error::raise_api_path_error(
function_name, api_path, response_code,
fmt::format("failed to get object list|response|{}", error_data));
return false;
}
return true;
} }
auto sia_provider::get_total_drive_space() const -> std::uint64_t { auto sia_provider::get_total_drive_space() const -> std::uint64_t {
@ -404,8 +420,8 @@ auto sia_provider::get_total_drive_space() const -> std::uint64_t {
} }
if (response_code != http_error_codes::ok) { if (response_code != http_error_codes::ok) {
utils::error::raise_api_path_error( utils::error::raise_error(
function_name, api_path, response_code, function_name, response_code,
fmt::format("failed to get total drive space|response|{}", fmt::format("failed to get total drive space|response|{}",
error_data)); error_data));
return 0U; return 0U;
@ -424,14 +440,14 @@ auto sia_provider::is_directory(const std::string &api_path, bool &exists) const
-> api_error { -> api_error {
REPERTORY_USES_FUNCTION_NAME(); REPERTORY_USES_FUNCTION_NAME();
if (api_path == "/") {
exists = true;
return api_error::success;
}
exists = false;
try { try {
if (api_path == "/") {
exists = true;
return api_error::success;
}
exists = false;
json object_list{}; json object_list{};
if (not get_object_list(utils::path::get_parent_api_path(api_path), if (not get_object_list(utils::path::get_parent_api_path(api_path),
object_list)) { object_list)) {
@ -446,8 +462,8 @@ auto sia_provider::is_directory(const std::string &api_path, bool &exists) const
}) != object_list.at("entries").end(); }) != object_list.at("entries").end();
return api_error::success; return api_error::success;
} catch (const std::exception &e) { } catch (const std::exception &e) {
utils::error::raise_api_path_error(function_name, api_path, e, utils::error::raise_api_path_error(
"failed to determine path is directory"); function_name, api_path, e, "failed to determine if path is directory");
} }
return api_error::error; return api_error::error;
@ -457,12 +473,12 @@ auto sia_provider::is_file(const std::string &api_path, bool &exists) const
-> api_error { -> api_error {
REPERTORY_USES_FUNCTION_NAME(); REPERTORY_USES_FUNCTION_NAME();
exists = false;
if (api_path == "/") {
return api_error::success;
}
try { try {
exists = false;
if (api_path == "/") {
return api_error::success;
}
json file_data{}; json file_data{};
auto res = get_object_info(api_path, file_data); auto res = get_object_info(api_path, file_data);
if (res == api_error::item_not_found) { if (res == api_error::item_not_found) {
@ -477,7 +493,7 @@ auto sia_provider::is_file(const std::string &api_path, bool &exists) const
return api_error::success; return api_error::success;
} catch (const std::exception &e) { } catch (const std::exception &e) {
utils::error::raise_api_path_error(function_name, api_path, e, utils::error::raise_api_path_error(function_name, api_path, e,
"failed to determine path is directory"); "failed to determine if path is file");
} }
return api_error::error; return api_error::error;
@ -513,8 +529,8 @@ auto sia_provider::is_online() const -> bool {
} }
if (response_code != http_error_codes::ok) { if (response_code != http_error_codes::ok) {
utils::error::raise_api_path_error( utils::error::raise_error(
function_name, api_path, response_code, function_name, response_code,
fmt::format("failed to determine if provider is online|response|{}", fmt::format("failed to determine if provider is online|response|{}",
error_data)); error_data));
return false; return false;
@ -535,55 +551,62 @@ auto sia_provider::read_file_bytes(const std::string &api_path,
stop_type &stop_requested) -> api_error { stop_type &stop_requested) -> api_error {
REPERTORY_USES_FUNCTION_NAME(); REPERTORY_USES_FUNCTION_NAME();
curl::requests::http_get get{}; try {
get.path = "/api/worker/objects" + api_path; curl::requests::http_get get{};
get.query["bucket"] = get_bucket(get_sia_config()); get.path = "/api/worker/objects" + api_path;
get.range = {{ get.query["bucket"] = get_bucket(get_sia_config());
offset, get.range = {{
offset + size - 1U, offset,
}}; offset + size - 1U,
get.response_handler = [&buffer](auto &&data, long /* response_code */) { }};
buffer = data; get.response_handler = [&buffer](auto &&data, long /* response_code */) {
}; buffer = data;
auto res = api_error::comm_error;
for (std::uint32_t idx = 0U;
not stop_requested && res != api_error::success &&
idx < get_config().get_retry_read_count() + 1U;
++idx) {
long response_code{};
const auto notify_retry = [&]() {
if (response_code == 0) {
utils::error::raise_api_path_error(
function_name, api_path, api_error::comm_error,
"read file bytes failed|offset|" + std::to_string(offset) +
"|size|" + std::to_string(size) + "|retry|" +
std::to_string(idx + 1U));
} else {
utils::error::raise_api_path_error(
function_name, api_path, response_code,
"read file bytes failed|offset|" + std::to_string(offset) +
"|size|" + std::to_string(size) + "|retry|" +
std::to_string(idx + 1U));
}
std::this_thread::sleep_for(1s);
}; };
if (not get_comm().make_request(get, response_code, stop_requested)) { auto res{api_error::comm_error};
notify_retry(); for (std::uint32_t idx = 0U;
continue; not stop_requested && res != api_error::success &&
idx < get_config().get_retry_read_count() + 1U;
++idx) {
long response_code{};
const auto notify_retry = [&]() {
if (response_code == 0) {
utils::error::raise_api_path_error(
function_name, api_path, api_error::comm_error,
"read file bytes failed|offset|" + std::to_string(offset) +
"|size|" + std::to_string(size) + "|retry|" +
std::to_string(idx + 1U));
} else {
utils::error::raise_api_path_error(
function_name, api_path, response_code,
"read file bytes failed|offset|" + std::to_string(offset) +
"|size|" + std::to_string(size) + "|retry|" +
std::to_string(idx + 1U));
}
std::this_thread::sleep_for(1s);
};
if (not get_comm().make_request(get, response_code, stop_requested)) {
notify_retry();
continue;
}
if (response_code < http_error_codes::ok ||
response_code >= http_error_codes::multiple_choices) {
notify_retry();
continue;
}
res = api_error::success;
} }
if (response_code < http_error_codes::ok || return res;
response_code >= http_error_codes::multiple_choices) { } catch (const std::exception &e) {
notify_retry(); utils::error::raise_api_path_error(function_name, e, api_path,
continue; "failed to read file bytes");
}
res = api_error::success;
} }
return res; return api_error::error;
} }
auto sia_provider::remove_directory_impl(const std::string &api_path) auto sia_provider::remove_directory_impl(const std::string &api_path)
@ -614,8 +637,9 @@ auto sia_provider::remove_directory_impl(const std::string &api_path)
} }
if (response_code != http_error_codes::ok) { if (response_code != http_error_codes::ok) {
utils::error::raise_api_path_error(function_name, api_path, response_code, utils::error::raise_api_path_error(
fmt::format("failed to remove directory|response|", error_data); function_name, api_path, response_code,
fmt::format("failed to remove directory|response|{}", error_data));
return api_error::comm_error; return api_error::comm_error;
} }
@ -663,43 +687,50 @@ auto sia_provider::rename_file(const std::string &from_api_path,
const std::string &to_api_path) -> api_error { const std::string &to_api_path) -> api_error {
REPERTORY_USES_FUNCTION_NAME(); REPERTORY_USES_FUNCTION_NAME();
curl::requests::http_post post{}; try {
post.json = nlohmann::json({ curl::requests::http_post post{};
{"from", from_api_path}, post.json = nlohmann::json({
{"to", to_api_path}, {"from", from_api_path},
{"mode", "single"}, {"to", to_api_path},
}); {"mode", "single"},
post.path = "/api/bus/objects/rename"; });
post.query["bucket"] = get_bucket(get_sia_config()); post.path = "/api/bus/objects/rename";
post.query["bucket"] = get_bucket(get_sia_config());
std::string error_data; std::string error_data;
post.response_handler = [&error_data](auto &&data, long response_code) { post.response_handler = [&error_data](auto &&data, long response_code) {
if (response_code == http_error_codes::ok) { if (response_code == http_error_codes::ok) {
return; return;
}
error_data = std::string(data.begin(), data.end());
};
long response_code{};
stop_type stop_requested{};
if (not get_comm().make_request(post, response_code, stop_requested)) {
utils::error::raise_api_path_error(
function_name, fmt::format("{}|{}", from_api_path, to_api_path),
api_error::comm_error, "failed to rename file");
return api_error::comm_error;
} }
error_data = std::string(data.begin(), data.end()); if (response_code < http_error_codes::ok ||
}; response_code >= http_error_codes::multiple_choices) {
utils::error::raise_api_path_error(
function_name, fmt::format("{}|{}", from_api_path, to_api_path),
response_code,
fmt::format("failed to rename file file|response|{}", error_data));
return api_error::comm_error;
}
long response_code{}; return get_db().rename_item_meta(from_api_path, to_api_path);
stop_type stop_requested{}; } catch (const std::exception &e) {
if (not get_comm().make_request(post, response_code, stop_requested)) { utils::error::raise_api_path_error(function_name, e, api_path,
utils::error::raise_api_path_error( "failed to rename file");
function_name, fmt::format("{}|{}", from_api_path, to_api_path), api_error::comm_error,
"failed to rename file");
return api_error::comm_error;
} }
if (response_code < http_error_codes::ok || return api_error::error;
response_code >= http_error_codes::multiple_choices) {
utils::error::raise_api_path_error(
function_name, fmt::format("{}|{}", from_api_path, to_api_path),
response_code,
fmt::format("failed to rename file file|response|{}", error_data));
return api_error::comm_error;
}
return get_db().rename_item_meta(from_api_path, to_api_path);
} }
auto sia_provider::start(api_item_added_callback api_item_added, auto sia_provider::start(api_item_added_callback api_item_added,