refactor
This commit is contained in:
parent
4fe9d09f0a
commit
b9fe2746e3
@ -43,20 +43,22 @@ namespace repertory {
|
||||
s3_provider::s3_provider(app_config &config, i_http_comm &comm)
|
||||
: base_provider(config, comm) {}
|
||||
|
||||
auto s3_provider::add_if_not_found(
|
||||
api_file &file, const std::string &object_name) const -> api_error {
|
||||
auto s3_provider::add_if_not_found(api_file &file,
|
||||
const std::string &object_name) const
|
||||
-> api_error {
|
||||
api_meta_map meta{};
|
||||
if (get_item_meta(file.api_path, meta) == api_error::item_not_found) {
|
||||
auto err = create_path_directories(
|
||||
auto res = get_item_meta(file.api_path, meta);
|
||||
if (res == api_error::item_not_found) {
|
||||
res = create_path_directories(
|
||||
file.api_parent, utils::path::get_parent_api_path(object_name));
|
||||
if (err != api_error::success) {
|
||||
return err;
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
|
||||
get_api_item_added()(false, file);
|
||||
}
|
||||
|
||||
return api_error::success;
|
||||
return res;
|
||||
}
|
||||
|
||||
auto s3_provider::convert_api_date(std::string_view date) -> std::uint64_t {
|
||||
@ -70,7 +72,7 @@ auto s3_provider::convert_api_date(std::string_view date) -> std::uint64_t {
|
||||
utils::string::split(date_parts.at(1U), 'Z', true).at(0U)) *
|
||||
1000000UL;
|
||||
|
||||
struct tm tm1 {};
|
||||
struct tm tm1{};
|
||||
#if defined(_WIN32)
|
||||
utils::time::strptime(date_time.c_str(), "%Y-%m-%dT%T", &tm1);
|
||||
return nanos + utils::time::windows_time_t_to_unix_time(_mkgmtime(&tm1));
|
||||
@ -168,8 +170,9 @@ auto s3_provider::create_file_extra(const std::string &api_path,
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
auto s3_provider::create_path_directories(
|
||||
const std::string &api_path, const std::string &key) const -> api_error {
|
||||
auto s3_provider::create_path_directories(const std::string &api_path,
|
||||
const std::string &key) const
|
||||
-> api_error {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
if (api_path == "/") {
|
||||
@ -347,34 +350,99 @@ auto s3_provider::get_directory_item_count(const std::string &api_path) const
|
||||
return 0U;
|
||||
}
|
||||
|
||||
auto s3_provider::get_directory_items_impl(
|
||||
const std::string &api_path, directory_item_list &list) const -> api_error {
|
||||
auto s3_provider::get_directory_items_impl(const std::string &api_path,
|
||||
directory_item_list &list) const
|
||||
-> api_error {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
const auto &cfg = get_s3_config();
|
||||
auto is_encrypted = not cfg.encryption_token.empty();
|
||||
const auto &cfg{get_s3_config()};
|
||||
auto is_encrypted{not cfg.encryption_token.empty()};
|
||||
|
||||
const auto add_diretory_item = [this, &cfg, &is_encrypted, &list](
|
||||
const std::string &child_object_name,
|
||||
bool directory, auto node) -> api_error {
|
||||
auto res{api_error::success};
|
||||
|
||||
directory_item dir_item{};
|
||||
dir_item.api_path = child_object_name;
|
||||
dir_item.directory = directory;
|
||||
if (is_encrypted) {
|
||||
if (not utils::encryption::decrypt_file_path(cfg.encryption_token,
|
||||
dir_item.api_path)) {
|
||||
return api_error::decryption_error;
|
||||
}
|
||||
}
|
||||
dir_item.api_path = utils::path::create_api_path(dir_item.api_path);
|
||||
dir_item.api_parent = utils::path::get_parent_api_path(dir_item.api_path);
|
||||
|
||||
if (directory) {
|
||||
res = get_item_meta(dir_item.api_path, dir_item.meta);
|
||||
if (res == api_error::item_not_found) {
|
||||
res = create_path_directories(dir_item.api_path, child_object_name);
|
||||
}
|
||||
} else {
|
||||
std::string size_str;
|
||||
if (get_item_meta(dir_item.api_path, META_SIZE, size_str) ==
|
||||
api_error::success) {
|
||||
dir_item.size = utils::string::to_uint64(size_str);
|
||||
} else {
|
||||
auto size = node.select_node("Size").node().text().as_ullong();
|
||||
|
||||
dir_item.size = is_encrypted ? utils::encryption::encrypting_reader::
|
||||
calculate_decrypted_size(size)
|
||||
: size;
|
||||
}
|
||||
|
||||
res = get_item_meta(dir_item.api_path, dir_item.meta);
|
||||
if (res == api_error::item_not_found) {
|
||||
auto last_modified = convert_api_date(
|
||||
node.select_node("LastModified").node().text().as_string());
|
||||
|
||||
api_file file{};
|
||||
file.api_path = dir_item.api_path;
|
||||
file.api_parent = dir_item.api_parent;
|
||||
file.accessed_date = file.changed_date = file.creation_date =
|
||||
file.modified_date = last_modified;
|
||||
file.file_size = dir_item.size;
|
||||
if (is_encrypted) {
|
||||
file.key = child_object_name;
|
||||
}
|
||||
|
||||
res = add_if_not_found(file, child_object_name);
|
||||
}
|
||||
}
|
||||
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
|
||||
list.push_back(std::move(dir_item));
|
||||
return api_error::success;
|
||||
};
|
||||
|
||||
auto ret = api_error::success;
|
||||
std::string key;
|
||||
if (is_encrypted) {
|
||||
ret = get_item_meta(api_path, META_KEY, key);
|
||||
if (ret != api_error::success) {
|
||||
return ret;
|
||||
auto res = get_item_meta(api_path, META_KEY, key);
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
}
|
||||
|
||||
auto object_name =
|
||||
auto object_name{
|
||||
api_path == "/"
|
||||
? ""
|
||||
: utils::path::create_api_path(is_encrypted ? key : api_path);
|
||||
: utils::path::create_api_path(is_encrypted ? key : api_path),
|
||||
};
|
||||
|
||||
std::string response_data{};
|
||||
long response_code{};
|
||||
auto prefix = object_name.empty() ? object_name : object_name + "/";
|
||||
auto prefix{
|
||||
object_name.empty() ? object_name : object_name + "/",
|
||||
};
|
||||
|
||||
auto grab_more{true};
|
||||
std::string token{};
|
||||
while (grab_more) {
|
||||
std::string response_data{};
|
||||
long response_code{};
|
||||
if (not get_object_list(response_data, response_code, "/", prefix, token)) {
|
||||
return api_error::comm_error;
|
||||
}
|
||||
@ -411,34 +479,10 @@ auto s3_provider::get_directory_items_impl(
|
||||
for (const auto &node : node_list) {
|
||||
auto child_object_name = utils::path::create_api_path(
|
||||
utils::path::combine("/", {node.node().text().as_string()}));
|
||||
directory_item dir_item{};
|
||||
dir_item.api_path = child_object_name;
|
||||
if (is_encrypted) {
|
||||
if (not utils::encryption::decrypt_file_path(cfg.encryption_token,
|
||||
dir_item.api_path)) {
|
||||
return api_error::decryption_error;
|
||||
}
|
||||
}
|
||||
dir_item.api_path = utils::path::create_api_path(dir_item.api_path);
|
||||
dir_item.api_parent = utils::path::get_parent_api_path(dir_item.api_path);
|
||||
dir_item.directory = true;
|
||||
dir_item.size = 0U;
|
||||
|
||||
auto res = get_item_meta(dir_item.api_path, dir_item.meta);
|
||||
if (res == api_error::item_not_found) {
|
||||
res = create_path_directories(dir_item.api_path, child_object_name);
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
|
||||
res = get_item_meta(dir_item.api_path, dir_item.meta);
|
||||
}
|
||||
|
||||
auto res = add_diretory_item(child_object_name, true, node.node());
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
|
||||
list.push_back(std::move(dir_item));
|
||||
}
|
||||
|
||||
node_list = doc.select_nodes("/ListBucketResult/Contents");
|
||||
@ -449,66 +493,18 @@ auto s3_provider::get_directory_items_impl(
|
||||
continue;
|
||||
}
|
||||
|
||||
directory_item dir_item{};
|
||||
dir_item.api_path = child_object_name;
|
||||
if (is_encrypted) {
|
||||
if (not utils::encryption::decrypt_file_path(cfg.encryption_token,
|
||||
dir_item.api_path)) {
|
||||
return api_error::decryption_error;
|
||||
}
|
||||
}
|
||||
dir_item.api_path = utils::path::create_api_path(dir_item.api_path);
|
||||
dir_item.api_parent = utils::path::get_parent_api_path(dir_item.api_path);
|
||||
dir_item.directory = false;
|
||||
|
||||
std::string size_str;
|
||||
if (get_item_meta(dir_item.api_path, META_SIZE, size_str) ==
|
||||
api_error::success) {
|
||||
dir_item.size = utils::string::to_uint64(size_str);
|
||||
} else {
|
||||
auto size = node.node().select_node("Size").node().text().as_ullong();
|
||||
|
||||
dir_item.size = is_encrypted ? utils::encryption::encrypting_reader::
|
||||
calculate_decrypted_size(size)
|
||||
: size;
|
||||
}
|
||||
|
||||
auto res = get_item_meta(dir_item.api_path, dir_item.meta);
|
||||
if (res == api_error::item_not_found) {
|
||||
auto last_modified = convert_api_date(
|
||||
node.node().select_node("LastModified").node().text().as_string());
|
||||
|
||||
api_file file{};
|
||||
file.api_path = dir_item.api_path;
|
||||
file.api_parent = dir_item.api_parent;
|
||||
file.accessed_date = file.changed_date = file.creation_date =
|
||||
file.modified_date = last_modified;
|
||||
file.file_size = dir_item.size;
|
||||
if (is_encrypted) {
|
||||
file.key = child_object_name;
|
||||
}
|
||||
|
||||
res = add_if_not_found(file, child_object_name);
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
|
||||
res = get_item_meta(dir_item.api_path, dir_item.meta);
|
||||
}
|
||||
|
||||
auto res = add_diretory_item(child_object_name, false, node.node());
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
|
||||
list.push_back(std::move(dir_item));
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
auto s3_provider::get_file(const std::string &api_path,
|
||||
api_file &file) const -> api_error {
|
||||
auto s3_provider::get_file(const std::string &api_path, api_file &file) const
|
||||
-> api_error {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
try {
|
||||
@ -547,8 +543,8 @@ auto s3_provider::get_file(const std::string &api_path,
|
||||
return api_error::error;
|
||||
}
|
||||
|
||||
auto s3_provider::get_file_list(api_file_list &list,
|
||||
std::string &marker) const -> api_error {
|
||||
auto s3_provider::get_file_list(api_file_list &list, std::string &marker) const
|
||||
-> api_error {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
std::string response_data;
|
||||
@ -623,8 +619,9 @@ auto s3_provider::get_file_list(api_file_list &list,
|
||||
return grab_more ? api_error::more_data : api_error::success;
|
||||
}
|
||||
|
||||
auto s3_provider::get_last_modified(
|
||||
bool directory, const std::string &api_path) const -> std::uint64_t {
|
||||
auto s3_provider::get_last_modified(bool directory,
|
||||
const std::string &api_path) const
|
||||
-> std::uint64_t {
|
||||
bool is_encrypted{};
|
||||
std::string object_name;
|
||||
head_object_result result{};
|
||||
@ -634,9 +631,10 @@ auto s3_provider::get_last_modified(
|
||||
: utils::time::get_time_now();
|
||||
}
|
||||
|
||||
auto s3_provider::get_object_info(
|
||||
bool directory, const std::string &api_path, bool &is_encrypted,
|
||||
std::string &object_name, head_object_result &result) const -> api_error {
|
||||
auto s3_provider::get_object_info(bool directory, const std::string &api_path,
|
||||
bool &is_encrypted, std::string &object_name,
|
||||
head_object_result &result) const
|
||||
-> api_error {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
try {
|
||||
@ -691,10 +689,12 @@ auto s3_provider::get_object_info(
|
||||
return api_error::error;
|
||||
}
|
||||
|
||||
auto s3_provider::get_object_list(
|
||||
std::string &response_data, long &response_code,
|
||||
std::optional<std::string> delimiter, std::optional<std::string> prefix,
|
||||
std::optional<std::string> token) const -> bool {
|
||||
auto s3_provider::get_object_list(std::string &response_data,
|
||||
long &response_code,
|
||||
std::optional<std::string> delimiter,
|
||||
std::optional<std::string> prefix,
|
||||
std::optional<std::string> token) const
|
||||
-> bool {
|
||||
curl::requests::http_get get{};
|
||||
get.allow_timeout = true;
|
||||
get.aws_service = "aws:amz:" + get_s3_config().region + ":s3";
|
||||
@ -722,8 +722,8 @@ auto s3_provider::get_total_drive_space() const -> std::uint64_t {
|
||||
return std::numeric_limits<std::int64_t>::max() / std::int64_t(2);
|
||||
}
|
||||
|
||||
auto s3_provider::is_directory(const std::string &api_path,
|
||||
bool &exists) const -> api_error {
|
||||
auto s3_provider::is_directory(const std::string &api_path, bool &exists) const
|
||||
-> api_error {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
exists = false;
|
||||
@ -750,8 +750,8 @@ auto s3_provider::is_directory(const std::string &api_path,
|
||||
return api_error::error;
|
||||
}
|
||||
|
||||
auto s3_provider::is_file(const std::string &api_path,
|
||||
bool &exists) const -> api_error {
|
||||
auto s3_provider::is_file(const std::string &api_path, bool &exists) const
|
||||
-> api_error {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
exists = false;
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user