From 1766f916979851929cca7b798402aa53a4c193e8 Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Tue, 14 Nov 2023 18:35:18 -0600 Subject: [PATCH] extract common behavior --- include/providers/base_provider.hpp | 8 + include/providers/s3/s3_provider.hpp | 17 +- include/providers/sia/sia_provider.hpp | 8 +- src/providers/base_provider.cpp | 43 +++++ src/providers/s3/s3_provider.cpp | 251 +++++++++++-------------- src/providers/sia/sia_provider.cpp | 119 +++++------- 6 files changed, 217 insertions(+), 229 deletions(-) diff --git a/include/providers/base_provider.hpp b/include/providers/base_provider.hpp index 8c5220d8..86616f4b 100644 --- a/include/providers/base_provider.hpp +++ b/include/providers/base_provider.hpp @@ -83,6 +83,10 @@ protected: return config_; } + [[nodiscard]] virtual auto + get_directory_items_impl(const std::string &api_path, + directory_item_list &list) const -> api_error = 0; + [[nodiscard]] auto get_db() const -> rocksdb::DB * { return db_.get(); } [[nodiscard]] auto get_file_mgr() -> i_file_manager * { return fm_; } @@ -107,6 +111,10 @@ public: std::string &api_path) const -> api_error override; + [[nodiscard]] auto get_directory_items(const std::string &api_path, + directory_item_list &list) const + -> api_error override; + [[nodiscard]] auto get_file_size(const std::string &api_path, std::uint64_t &file_size) const -> api_error override; diff --git a/include/providers/s3/s3_provider.hpp b/include/providers/s3/s3_provider.hpp index 51f085a9..a8cc425e 100644 --- a/include/providers/s3/s3_provider.hpp +++ b/include/providers/s3/s3_provider.hpp @@ -49,10 +49,6 @@ private: const std::string &object_name) const -> api_error; - [[nodiscard]] auto create_directory_impl(const std::string &api_path, - api_meta_map &meta) - -> api_error override; - [[nodiscard]] auto create_file_extra(const std::string &api_path, api_meta_map &meta) -> api_error override; @@ -75,14 +71,19 @@ private: std::optional prefix = std::nullopt) const -> bool; +protected: + [[nodiscard]] auto create_directory_impl(const std::string &api_path, + api_meta_map &meta) + -> api_error override; + + [[nodiscard]] auto get_directory_items_impl(const std::string &api_path, + directory_item_list &list) const + -> api_error override; + public: [[nodiscard]] auto get_directory_item_count(const std::string &api_path) const -> std::uint64_t override; - [[nodiscard]] auto get_directory_items(const std::string &api_path, - directory_item_list &list) const - -> api_error override; - [[nodiscard]] auto get_file(const std::string &api_path, api_file &file) const -> api_error override; diff --git a/include/providers/sia/sia_provider.hpp b/include/providers/sia/sia_provider.hpp index 178bec4b..8f3fe510 100644 --- a/include/providers/sia/sia_provider.hpp +++ b/include/providers/sia/sia_provider.hpp @@ -54,14 +54,14 @@ protected: api_meta_map &meta) -> api_error override; + [[nodiscard]] auto get_directory_items_impl(const std::string &api_path, + directory_item_list &list) const + -> api_error override; + public: [[nodiscard]] auto get_directory_item_count(const std::string &api_path) const -> std::uint64_t override; - [[nodiscard]] auto get_directory_items(const std::string &api_path, - directory_item_list &list) const - -> api_error override; - [[nodiscard]] auto get_file(const std::string &api_path, api_file &file) const -> api_error override; diff --git a/src/providers/base_provider.cpp b/src/providers/base_provider.cpp index b7b61bcc..7b5603bc 100644 --- a/src/providers/base_provider.cpp +++ b/src/providers/base_provider.cpp @@ -246,6 +246,49 @@ auto base_provider::get_api_path_from_source(const std::string &source_path, return api_error::item_not_found; } +auto base_provider::get_directory_items(const std::string &api_path, + directory_item_list &list) const + -> api_error { + bool exists{}; + auto res = is_directory(api_path, exists); + if (res != api_error::success) { + return res; + } + if (not exists) { + return api_error::directory_not_found; + } + + try { + res = get_directory_items_impl(api_path, list); + if (res != api_error::success) { + return res; + } + } catch (const std::exception &e) { + utils::error::raise_api_path_error(__FUNCTION__, api_path, e, + "failed to get directory items"); + return api_error::error; + } + + std::sort(list.begin(), list.end(), + [](const auto &item1, const auto &item2) -> bool { + return (item1.directory && not item2.directory) || + (not(item2.directory && not item1.directory) && + (item1.api_path.compare(item2.api_path) < 0)); + }); + + list.insert(list.begin(), directory_item{ + "..", + "", + true, + }); + list.insert(list.begin(), directory_item{ + ".", + "", + true, + }); + return api_error::success; +} + auto base_provider::get_file_size(const std::string &api_path, std::uint64_t &file_size) const -> api_error { bool exists{}; diff --git a/src/providers/s3/s3_provider.cpp b/src/providers/s3/s3_provider.cpp index 26a8bc16..8c1acc2c 100644 --- a/src/providers/s3/s3_provider.cpp +++ b/src/providers/s3/s3_provider.cpp @@ -254,151 +254,120 @@ auto s3_provider::get_directory_item_count(const std::string &api_path) const return 0U; } -auto s3_provider::get_directory_items(const std::string &api_path, - directory_item_list &list) const +auto s3_provider::get_directory_items_impl(const std::string &api_path, + directory_item_list &list) const -> api_error { - try { - bool exists{}; - auto res = is_directory(api_path, exists); - if (res != api_error::success) { - return res; + const auto cfg = get_config().get_s3_config(); + const auto is_encrypted = not cfg.encryption_token.empty(); + + auto ret = api_error::success; + std::string key; + if (is_encrypted) { + ret = get_item_meta(api_path, META_KEY, key); + if (ret != api_error::success) { + return ret; } - if (not exists) { - return api_error::directory_not_found; - } - - const auto cfg = get_config().get_s3_config(); - const auto is_encrypted = not cfg.encryption_token.empty(); - std::string key; - if (is_encrypted) { - res = get_item_meta(api_path, META_KEY, key); - if (res != api_error::success) { - return res; - } - } - - const auto object_name = - api_path == "/" - ? "" - : utils::path::create_api_path(is_encrypted ? key : api_path); - - std::string response_data{}; - long response_code{}; - auto prefix = object_name.empty() ? object_name : object_name + "/"; - - if (not get_object_list(response_data, response_code, "/", prefix)) { - return api_error::comm_error; - } - - if (response_code == http_error_codes::not_found) { - return api_error::directory_not_found; - } - - if (response_code != http_error_codes::ok) { - return api_error::comm_error; - } - - pugi::xml_document doc; - auto parse_res = doc.load_string(response_data.c_str()); - if (parse_res.status != pugi::xml_parse_status::status_ok) { - return api_error::error; - } - - const auto add_directory_item = - [&](bool directory, const std::string &name, - std::function get_size) - -> api_error { - auto child_api_path = - utils::path::create_api_path(utils::path::combine("/", {name})); - std::string child_object_name; - if (is_encrypted) { - child_object_name = child_api_path; - res = utils::encryption::decrypt_file_path(cfg.encryption_token, - child_api_path); - if (res != api_error::success) { - return res; - } - } - - directory_item dir_item{}; - dir_item.api_path = child_api_path; - dir_item.api_parent = utils::path::get_parent_api_path(dir_item.api_path); - dir_item.directory = directory; - dir_item.size = get_size(dir_item); - res = get_item_meta(child_api_path, dir_item.meta); - if (res == api_error::item_not_found) { - if (directory) { - res = create_path_directories(child_api_path, child_object_name); - if (res != api_error::success) { - return res; - } - } else { - auto file = - create_api_file(child_api_path, child_object_name, dir_item.size); - res = add_if_not_found(file, child_object_name); - if (res != api_error::success) { - return res; - } - } - - res = get_item_meta(child_api_path, dir_item.meta); - } - - if (res != api_error::success) { - return res; - } - - list.push_back(std::move(dir_item)); - return api_error::success; - }; - - auto node_list = - doc.select_nodes("/ListBucketResult/CommonPrefixes/Prefix"); - for (const auto &node : node_list) { - add_directory_item( - true, node.node().text().as_string(), - [](const directory_item &) -> std::uint64_t { return 0U; }); - } - - node_list = doc.select_nodes("/ListBucketResult/Contents"); - for (const auto &node : node_list) { - auto child_object_name = utils::path::create_api_path( - node.node().select_node("Key").node().text().as_string()); - if (child_object_name != utils::path::create_api_path(prefix)) { - auto size = node.node().select_node("Size").node().text().as_ullong(); - add_directory_item( - false, child_object_name, - [&is_encrypted, &size](const directory_item &) -> std::uint64_t { - return is_encrypted ? utils::encryption::encrypting_reader:: - calculate_decrypted_size(size) - : size; - }); - } - } - - std::sort(list.begin(), list.end(), - [](const auto &item1, const auto &item2) -> bool { - return (item1.directory && not item2.directory) || - (not(item2.directory && not item1.directory) && - (item1.api_path.compare(item2.api_path) < 0)); - }); - - list.insert(list.begin(), directory_item{ - "..", - "", - true, - }); - list.insert(list.begin(), directory_item{ - ".", - "", - true, - }); - return api_error::success; - } catch (const std::exception &e) { - utils::error::raise_error(__FUNCTION__, e, "exception occurred"); } - return api_error::error; + const auto object_name = + api_path == "/" + ? "" + : utils::path::create_api_path(is_encrypted ? key : api_path); + + std::string response_data{}; + long response_code{}; + auto prefix = object_name.empty() ? object_name : object_name + "/"; + + if (not get_object_list(response_data, response_code, "/", prefix)) { + return api_error::comm_error; + } + + if (response_code == http_error_codes::not_found) { + return api_error::directory_not_found; + } + + if (response_code != http_error_codes::ok) { + return api_error::comm_error; + } + + pugi::xml_document doc; + auto parse_res = doc.load_string(response_data.c_str()); + if (parse_res.status != pugi::xml_parse_status::status_ok) { + return api_error::error; + } + + const auto add_directory_item = + [&](bool directory, const std::string &name, + std::function get_size) + -> api_error { + auto child_api_path = + utils::path::create_api_path(utils::path::combine("/", {name})); + std::string child_object_name; + if (is_encrypted) { + child_object_name = child_api_path; + ret = utils::encryption::decrypt_file_path(cfg.encryption_token, + child_api_path); + if (ret != api_error::success) { + return ret; + } + } + + directory_item dir_item{}; + dir_item.api_path = child_api_path; + dir_item.api_parent = utils::path::get_parent_api_path(dir_item.api_path); + dir_item.directory = directory; + dir_item.size = get_size(dir_item); + ret = get_item_meta(child_api_path, dir_item.meta); + if (ret == api_error::item_not_found) { + if (directory) { + ret = create_path_directories(child_api_path, child_object_name); + if (ret != api_error::success) { + return ret; + } + } else { + auto file = + create_api_file(child_api_path, child_object_name, dir_item.size); + ret = add_if_not_found(file, child_object_name); + if (ret != api_error::success) { + return ret; + } + } + + ret = get_item_meta(child_api_path, dir_item.meta); + } + + if (ret != api_error::success) { + return ret; + } + + list.push_back(std::move(dir_item)); + return api_error::success; + }; + + auto node_list = doc.select_nodes("/ListBucketResult/CommonPrefixes/Prefix"); + for (const auto &node : node_list) { + add_directory_item( + true, node.node().text().as_string(), + [](const directory_item &) -> std::uint64_t { return 0U; }); + } + + node_list = doc.select_nodes("/ListBucketResult/Contents"); + for (const auto &node : node_list) { + auto child_object_name = utils::path::create_api_path( + node.node().select_node("Key").node().text().as_string()); + if (child_object_name != utils::path::create_api_path(prefix)) { + auto size = node.node().select_node("Size").node().text().as_ullong(); + add_directory_item( + false, child_object_name, + [&is_encrypted, &size](const directory_item &) -> std::uint64_t { + return is_encrypted ? utils::encryption::encrypting_reader:: + calculate_decrypted_size(size) + : size; + }); + } + } + + return ret; } auto s3_provider::get_file(const std::string &api_path, api_file &file) const diff --git a/src/providers/sia/sia_provider.cpp b/src/providers/sia/sia_provider.cpp index 4fab8792..c4bd05cd 100644 --- a/src/providers/sia/sia_provider.cpp +++ b/src/providers/sia/sia_provider.cpp @@ -99,94 +99,61 @@ auto sia_provider::get_directory_item_count(const std::string &api_path) const return 0U; } -auto sia_provider::get_directory_items(const std::string &api_path, - directory_item_list &list) const +auto sia_provider::get_directory_items_impl(const std::string &api_path, + directory_item_list &list) const -> api_error { - bool exists{}; - auto res = is_directory(api_path, exists); - if (res != api_error::success) { - return res; - } - if (not exists) { - return api_error::directory_not_found; + json object_list{}; + if (not get_object_list(api_path, object_list)) { + return api_error::comm_error; } - try { - json object_list{}; - if (not get_object_list(api_path, object_list)) { - return api_error::comm_error; - } + if (object_list.contains("entries")) { + for (const auto &entry : object_list.at("entries")) { + try { + auto name = entry.at("name").get(); + auto entry_api_path = utils::path::create_api_path(name); - if (object_list.contains("entries")) { - for (const auto &entry : object_list.at("entries")) { - try { - auto name = entry.at("name").get(); - auto entry_api_path = utils::path::create_api_path(name); + auto directory = utils::string::ends_with(name, "/"); + if (directory && (entry_api_path == api_path)) { + continue; + } - auto directory = utils::string::ends_with(name, "/"); - if (directory && (entry_api_path == api_path)) { + api_file file{}; + + api_meta_map meta{}; + if (get_item_meta(entry_api_path, meta) == api_error::item_not_found) { + file = create_api_file( + entry_api_path, "", + directory ? 0U : entry["size"].get()); + get_api_item_added()(directory, file); + auto res = get_item_meta(entry_api_path, meta); + if (res != api_error::success) { + utils::error::raise_error(__FUNCTION__, res, + "failed to get item meta"); continue; } - - api_file file{}; - - api_meta_map meta{}; - if (get_item_meta(entry_api_path, meta) == - api_error::item_not_found) { - file = create_api_file( - entry_api_path, "", - directory ? 0U : entry["size"].get()); - get_api_item_added()(directory, file); - res = get_item_meta(entry_api_path, meta); - if (res != api_error::success) { - utils::error::raise_error(__FUNCTION__, res, - "failed to get item meta"); - continue; - } - } else { - file = create_api_file( - entry_api_path, - directory ? 0U : entry["size"].get(), meta); - } - - directory_item dir_item{}; - dir_item.api_parent = file.api_parent; - dir_item.api_path = file.api_path; - dir_item.directory = directory; - dir_item.meta = meta; - dir_item.resolved = true; - dir_item.size = file.file_size; - list.emplace_back(std::move(dir_item)); - } catch (const std::exception &e) { - utils::error::raise_api_path_error(__FUNCTION__, api_path, e, - "failed to process entry|" + - entry.dump()); + } else { + file = create_api_file( + entry_api_path, + directory ? 0U : entry["size"].get(), meta); } + + directory_item dir_item{}; + dir_item.api_parent = file.api_parent; + dir_item.api_path = file.api_path; + dir_item.directory = directory; + dir_item.meta = meta; + dir_item.resolved = true; + dir_item.size = file.file_size; + list.emplace_back(std::move(dir_item)); + } catch (const std::exception &e) { + utils::error::raise_api_path_error(__FUNCTION__, api_path, e, + "failed to process entry|" + + entry.dump()); } } - } catch (const std::exception &e) { - utils::error::raise_api_path_error(__FUNCTION__, api_path, e, - "failed to get directory items"); - return api_error::error; } - std::sort(list.begin(), list.end(), - [](const auto &item1, const auto &item2) -> bool { - return (item1.directory && not item2.directory) || - (not(item2.directory && not item1.directory) && - (item1.api_path.compare(item2.api_path) < 0)); - }); - - list.insert(list.begin(), directory_item{ - "..", - "", - true, - }); - list.insert(list.begin(), directory_item{ - ".", - "", - true, - }); return api_error::success; }