BlockStorage/repertory
1
0
Fork 0
Code Issues 7 Pull Requests Releases 6 Wiki Activity

fix drive space

Browse Source
This commit is contained in:
Scott E. Graves 2024-11-30 16:58:26 -06:00
parent b1735ab0af
commit 8af198f7a4
4 changed files with 223 additions and 211 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

146
repertory/librepertory/include/providers/base_provider.hpp
View File

@ -44,23 +44,25 @@ private:
api_item_added_callback api_item_added_; api_item_added_callback api_item_added_;
std::unique_ptr<meta_db> db3_; std::unique_ptr<meta_db> db3_;
i_file_manager *fm_{}; i_file_manager *fm_{};
mutable std::atomic<std::uint64_t> used_space;
private: private:
void remove_deleted_files(bool source_only); void remove_deleted_files(bool source_only);
void update_used_drive_space() const;
protected: protected:
[[nodiscard]] static auto create_api_file(std::string path, std::string key, [[nodiscard]] static auto
std::uint64_t size, create_api_file(std::string path, std::string key, std::uint64_t size,
std::uint64_t file_time) std::uint64_t file_time) -> api_file;
-> api_file;
[[nodiscard]] static auto create_api_file(std::string path, [[nodiscard]] static auto create_api_file(std::string path,
std::uint64_t size, std::uint64_t size,
api_meta_map &meta) -> api_file; api_meta_map &meta) -> api_file;
[[nodiscard]] virtual auto create_directory_impl(const std::string &api_path, [[nodiscard]] virtual auto
api_meta_map &meta) create_directory_impl(const std::string &api_path,
-> api_error = 0; api_meta_map &meta) -> api_error = 0;
[[nodiscard]] virtual auto [[nodiscard]] virtual auto
create_file_extra(const std::string & /* api_path */, create_file_extra(const std::string & /* api_path */,
@ -72,8 +74,8 @@ protected:
return api_item_added_; return api_item_added_;
} }
[[nodiscard]] auto get_api_item_added() const [[nodiscard]] auto
-> const api_item_added_callback & { get_api_item_added() const -> const api_item_added_callback & {
return api_item_added_; return api_item_added_;
} }
@ -97,25 +99,23 @@ protected:
return fm_; return fm_;
} }
[[nodiscard]] virtual auto get_used_drive_space_impl() const [[nodiscard]] virtual auto
-> std::uint64_t = 0; get_used_drive_space_impl() const -> std::uint64_t = 0;
[[nodiscard]] virtual auto remove_directory_impl(const std::string &api_path) [[nodiscard]] virtual auto
-> api_error = 0; remove_directory_impl(const std::string &api_path) -> api_error = 0;
[[nodiscard]] virtual auto remove_file_impl(const std::string &api_path) [[nodiscard]] virtual auto
-> api_error = 0; remove_file_impl(const std::string &api_path) -> api_error = 0;
[[nodiscard]] virtual auto upload_file_impl(const std::string &api_path, [[nodiscard]] virtual auto
const std::string &source_path, upload_file_impl(const std::string &api_path, const std::string &source_path,
stop_type &stop_requested) stop_type &stop_requested) -> api_error = 0;
-> api_error = 0;
public: public:
[[nodiscard]] auto [[nodiscard]] auto create_directory_clone_source_meta(
create_directory_clone_source_meta(const std::string &source_api_path, const std::string &source_api_path,
const std::string &api_path) const std::string &api_path) -> api_error override;
-> api_error override;
[[nodiscard]] auto create_directory(const std::string &api_path, [[nodiscard]] auto create_directory(const std::string &api_path,
api_meta_map &meta) -> api_error override; api_meta_map &meta) -> api_error override;
@ -123,82 +123,76 @@ public:
[[nodiscard]] auto create_file(const std::string &api_path, [[nodiscard]] auto create_file(const std::string &api_path,
api_meta_map &meta) -> api_error override; api_meta_map &meta) -> api_error override;
[[nodiscard]] auto get_api_path_from_source(const std::string &source_path, [[nodiscard]] auto
std::string &api_path) const get_api_path_from_source(const std::string &source_path,
-> api_error override; std::string &api_path) const -> api_error override;
[[nodiscard]] auto get_directory_items(const std::string &api_path,
directory_item_list &list) const
-> api_error override;
[[nodiscard]] auto get_file_size(const std::string &api_path,
std::uint64_t &file_size) const
-> api_error override;
[[nodiscard]] auto get_filesystem_item(const std::string &api_path,
bool directory,
filesystem_item &fsi) const
-> api_error override;
[[nodiscard]] auto get_filesystem_item_and_file(const std::string &api_path,
api_file &f,
filesystem_item &fsi) const
-> api_error override;
[[nodiscard]] auto [[nodiscard]] auto
get_filesystem_item_from_source_path(const std::string &source_path, get_directory_items(const std::string &api_path,
filesystem_item &fsi) const directory_item_list &list) const -> api_error override;
-> api_error override;
[[nodiscard]] auto get_item_meta(const std::string &api_path, [[nodiscard]] auto
api_meta_map &meta) const get_file_size(const std::string &api_path,
-> api_error override; std::uint64_t &file_size) const -> api_error override;
[[nodiscard]] auto get_item_meta(const std::string &api_path, [[nodiscard]] auto
const std::string &key, get_filesystem_item(const std::string &api_path, bool directory,
std::string &value) const filesystem_item &fsi) const -> api_error override;
-> api_error override;
[[nodiscard]] auto get_pinned_files() const [[nodiscard]] auto get_filesystem_item_and_file(
-> std::vector<std::string> override; const std::string &api_path, api_file &f,
filesystem_item &fsi) const -> api_error override;
[[nodiscard]] auto get_filesystem_item_from_source_path(
const std::string &source_path,
filesystem_item &fsi) const -> api_error override;
[[nodiscard]] auto
get_item_meta(const std::string &api_path,
api_meta_map &meta) const -> api_error override;
[[nodiscard]] auto
get_item_meta(const std::string &api_path, const std::string &key,
std::string &value) const -> api_error override;
[[nodiscard]] auto
get_pinned_files() const -> std::vector<std::string> override;
[[nodiscard]] auto get_total_item_count() const -> std::uint64_t override; [[nodiscard]] auto get_total_item_count() const -> std::uint64_t override;
[[nodiscard]] auto get_used_drive_space() const -> std::uint64_t override; [[nodiscard]] auto get_used_drive_space() const -> std::uint64_t override;
[[nodiscard]] auto is_file_writeable(const std::string &api_path) const [[nodiscard]] auto
-> bool override; is_file_writeable(const std::string &api_path) const -> bool override;
[[nodiscard]] auto is_read_only() const -> bool override { return false; } [[nodiscard]] auto is_read_only() const -> bool override { return false; }
[[nodiscard]] auto remove_directory(const std::string &api_path) [[nodiscard]] auto
-> api_error override; remove_directory(const std::string &api_path) -> api_error override;
[[nodiscard]] auto remove_file(const std::string &api_path) [[nodiscard]] auto
-> api_error override; remove_file(const std::string &api_path) -> api_error override;
[[nodiscard]] auto remove_item_meta(const std::string &api_path, [[nodiscard]] auto
const std::string &key) remove_item_meta(const std::string &api_path,
-> api_error override; const std::string &key) -> api_error override;
[[nodiscard]] auto set_item_meta(const std::string &api_path, [[nodiscard]] auto
const std::string &key, set_item_meta(const std::string &api_path, const std::string &key,
const std::string &value) const std::string &value) -> api_error override;
-> api_error override;
[[nodiscard]] auto set_item_meta(const std::string &api_path, [[nodiscard]] auto
const api_meta_map &meta) set_item_meta(const std::string &api_path,
-> api_error override; const api_meta_map &meta) -> api_error override;
[[nodiscard]] auto start(api_item_added_callback api_item_added, [[nodiscard]] auto start(api_item_added_callback api_item_added,
i_file_manager *mgr) -> bool override; i_file_manager *mgr) -> bool override;
void stop() override; void stop() override;
[[nodiscard]] auto upload_file(const std::string &api_path, [[nodiscard]] auto
const std::string &source_path, upload_file(const std::string &api_path, const std::string &source_path,
stop_type &stop_requested) stop_type &stop_requested) -> api_error override;
-> api_error override;
}; };
} // namespace repertory } // namespace repertory

92
repertory/librepertory/include/providers/s3/s3_provider.hpp
View File

@ -47,71 +47,70 @@ public:
auto operator=(s3_provider &&) -> s3_provider & = delete; auto operator=(s3_provider &&) -> s3_provider & = delete;
private: private:
[[nodiscard]] auto add_if_not_found(api_file &file, [[nodiscard]] auto
const std::string &object_name) const add_if_not_found(api_file &file,
-> api_error; const std::string &object_name) const -> api_error;
[[nodiscard]] auto create_file_extra(const std::string &api_path, [[nodiscard]] auto
api_meta_map &meta) create_file_extra(const std::string &api_path,
-> api_error override; api_meta_map &meta) -> api_error override;
[[nodiscard]] auto create_path_directories(const std::string &api_path, [[nodiscard]] auto
const std::string &key) const create_path_directories(const std::string &api_path,
-> api_error; const std::string &key) const -> api_error;
[[nodiscard]] auto decrypt_object_name(std::string &object_name) const [[nodiscard]] auto
-> api_error; decrypt_object_name(std::string &object_name) const -> api_error;
[[nodiscard]] auto get_last_modified(bool directory, [[nodiscard]] auto
const std::string &api_path) const get_last_modified(bool directory,
-> std::uint64_t; const std::string &api_path) const -> std::uint64_t;
[[nodiscard]] auto [[nodiscard]] auto
get_object_info(bool directory, const std::string &api_path, get_object_info(bool directory, const std::string &api_path,
bool &is_encrypted, std::string &object_name, bool &is_encrypted, std::string &object_name,
head_object_result &result) const -> api_error; head_object_result &result) const -> api_error;
[[nodiscard]] auto [[nodiscard]] auto get_object_list(
get_object_list(std::string &response_data, long &response_code, std::string &response_data, long &response_code,
std::optional<std::string> delimiter = std::nullopt, std::optional<std::string> delimiter = std::nullopt,
std::optional<std::string> prefix = std::nullopt) const std::optional<std::string> prefix = std::nullopt,
-> bool; std::optional<std::string> token = std::nullopt) const -> bool;
protected: protected:
[[nodiscard]] auto create_directory_impl(const std::string &api_path, [[nodiscard]] auto
api_meta_map &meta) create_directory_impl(const std::string &api_path,
-> api_error override; api_meta_map &meta) -> api_error override;
[[nodiscard]] auto get_directory_items_impl(const std::string &api_path, [[nodiscard]] auto get_directory_items_impl(const std::string &api_path,
directory_item_list &list) const directory_item_list &list) const
-> api_error override; -> api_error override;
[[nodiscard]] auto get_used_drive_space_impl() const [[nodiscard]] auto
-> std::uint64_t override; get_used_drive_space_impl() const -> std::uint64_t override;
[[nodiscard]] auto remove_directory_impl(const std::string &api_path) [[nodiscard]] auto
-> api_error override; remove_directory_impl(const std::string &api_path) -> api_error override;
[[nodiscard]] auto remove_file_impl(const std::string &api_path) [[nodiscard]] auto
-> api_error override; remove_file_impl(const std::string &api_path) -> api_error override;
[[nodiscard]] auto upload_file_impl(const std::string &api_path, [[nodiscard]] auto
const std::string &source_path, upload_file_impl(const std::string &api_path, const std::string &source_path,
stop_type &stop_requested) stop_type &stop_requested) -> api_error override;
-> api_error override;
public: public:
[[nodiscard]] static auto convert_api_date(std::string_view date) [[nodiscard]] static auto
-> std::uint64_t; convert_api_date(std::string_view date) -> std::uint64_t;
[[nodiscard]] auto get_directory_item_count(const std::string &api_path) const [[nodiscard]] auto get_directory_item_count(const std::string &api_path) const
-> std::uint64_t override; -> std::uint64_t override;
[[nodiscard]] auto get_file(const std::string &api_path, api_file &file) const [[nodiscard]] auto get_file(const std::string &api_path,
-> api_error override; api_file &file) const -> api_error override;
[[nodiscard]] auto get_file_list(api_file_list &list) const [[nodiscard]] auto
-> api_error override; get_file_list(api_file_list &list) const -> api_error override;
[[nodiscard]] auto get_total_drive_space() const -> std::uint64_t override; [[nodiscard]] auto get_total_drive_space() const -> std::uint64_t override;
@ -122,8 +121,8 @@ public:
[[nodiscard]] auto is_directory(const std::string &api_path, [[nodiscard]] auto is_directory(const std::string &api_path,
bool &exists) const -> api_error override; bool &exists) const -> api_error override;
[[nodiscard]] auto is_file(const std::string &api_path, bool &exists) const [[nodiscard]] auto is_file(const std::string &api_path,
-> api_error override; bool &exists) const -> api_error override;
[[nodiscard]] auto is_online() const -> bool override; [[nodiscard]] auto is_online() const -> bool override;
@ -131,15 +130,14 @@ public:
return false; return false;
}; };
[[nodiscard]] auto read_file_bytes(const std::string &api_path, [[nodiscard]] auto
std::size_t size, std::uint64_t offset, read_file_bytes(const std::string &api_path, std::size_t size,
data_buffer &data, std::uint64_t offset, data_buffer &data,
stop_type &stop_requested) stop_type &stop_requested) -> api_error override;
-> api_error override;
[[nodiscard]] auto rename_file(const std::string &from_api_path, [[nodiscard]] auto
const std::string &to_api_path) rename_file(const std::string &from_api_path,
-> api_error override; const std::string &to_api_path) -> api_error override;
[[nodiscard]] auto start(api_item_added_callback api_item_added, [[nodiscard]] auto start(api_item_added_callback api_item_added,
i_file_manager *mgr) -> bool override; i_file_manager *mgr) -> bool override;

70
repertory/librepertory/src/providers/base_provider.cpp
View File

@ -33,8 +33,8 @@
namespace repertory { namespace repertory {
auto base_provider::create_api_file(std::string path, std::string key, auto base_provider::create_api_file(std::string path, std::string key,
std::uint64_t size, std::uint64_t file_time) std::uint64_t size,
-> api_file { std::uint64_t file_time) -> api_file {
api_file file{}; api_file file{};
file.api_path = utils::path::create_api_path(path); file.api_path = utils::path::create_api_path(path);
file.api_parent = utils::path::get_parent_api_path(file.api_path); file.api_parent = utils::path::get_parent_api_path(file.api_path);
@ -66,8 +66,8 @@ auto base_provider::create_api_file(std::string path, std::uint64_t size,
} }
auto base_provider::create_directory_clone_source_meta( auto base_provider::create_directory_clone_source_meta(
const std::string &source_api_path, const std::string &api_path) const std::string &source_api_path,
-> api_error { const std::string &api_path) -> api_error {
REPERTORY_USES_FUNCTION_NAME(); REPERTORY_USES_FUNCTION_NAME();
bool exists{}; bool exists{};
@ -164,8 +164,8 @@ auto base_provider::create_directory(const std::string &api_path,
return set_item_meta(api_path, meta); return set_item_meta(api_path, meta);
} }
auto base_provider::create_file(const std::string &api_path, api_meta_map &meta) auto base_provider::create_file(const std::string &api_path,
-> api_error { api_meta_map &meta) -> api_error {
REPERTORY_USES_FUNCTION_NAME(); REPERTORY_USES_FUNCTION_NAME();
bool exists{}; bool exists{};
@ -222,9 +222,8 @@ auto base_provider::create_file(const std::string &api_path, api_meta_map &meta)
return api_error::error; return api_error::error;
} }
auto base_provider::get_api_path_from_source(const std::string &source_path, auto base_provider::get_api_path_from_source(
std::string &api_path) const const std::string &source_path, std::string &api_path) const -> api_error {
-> api_error {
REPERTORY_USES_FUNCTION_NAME(); REPERTORY_USES_FUNCTION_NAME();
if (source_path.empty()) { if (source_path.empty()) {
@ -237,9 +236,8 @@ auto base_provider::get_api_path_from_source(const std::string &source_path,
return db3_->get_api_path(source_path, api_path); return db3_->get_api_path(source_path, api_path);
} }
auto base_provider::get_directory_items(const std::string &api_path, auto base_provider::get_directory_items(
directory_item_list &list) const const std::string &api_path, directory_item_list &list) const -> api_error {
-> api_error {
REPERTORY_USES_FUNCTION_NAME(); REPERTORY_USES_FUNCTION_NAME();
bool exists{}; bool exists{};
@ -303,10 +301,9 @@ auto base_provider::get_file_size(const std::string &api_path,
return api_error::success; return api_error::success;
} }
auto base_provider::get_filesystem_item(const std::string &api_path, auto base_provider::get_filesystem_item(
bool directory, const std::string &api_path, bool directory,
filesystem_item &fsi) const filesystem_item &fsi) const -> api_error {
-> api_error {
bool exists{}; bool exists{};
auto res = is_directory(api_path, exists); auto res = is_directory(api_path, exists);
if (res != api_error::success) { if (res != api_error::success) {
@ -339,10 +336,9 @@ auto base_provider::get_filesystem_item(const std::string &api_path,
return api_error::success; return api_error::success;
} }
auto base_provider::get_filesystem_item_and_file(const std::string &api_path, auto base_provider::get_filesystem_item_and_file(
api_file &file, const std::string &api_path, api_file &file,
filesystem_item &fsi) const filesystem_item &fsi) const -> api_error {
-> api_error {
auto res = get_file(api_path, file); auto res = get_file(api_path, file);
if (res != api_error::success) { if (res != api_error::success) {
return res; return res;
@ -403,18 +399,11 @@ auto base_provider::get_total_item_count() const -> std::uint64_t {
} }
auto base_provider::get_used_drive_space() const -> std::uint64_t { auto base_provider::get_used_drive_space() const -> std::uint64_t {
REPERTORY_USES_FUNCTION_NAME(); if (used_space == 0U) {
update_used_drive_space();
try {
auto used_space = get_used_drive_space_impl();
get_file_mgr()->update_used_space(used_space);
return used_space;
} catch (const std::exception &ex) {
utils::error::raise_error(function_name, ex,
"failed to get used drive space");
} }
return 0U; return used_space;
} }
auto base_provider::is_file_writeable(const std::string &api_path) const auto base_provider::is_file_writeable(const std::string &api_path) const
@ -471,10 +460,10 @@ void base_provider::remove_deleted_files(bool source_only) {
return; return;
} }
struct removed_item { struct removed_item final {
std::string api_path{}; std::string api_path;
bool directory{}; bool directory{};
std::string source_path{}; std::string source_path;
}; };
api_file_list list{}; api_file_list list{};
@ -713,6 +702,8 @@ auto base_provider::start(api_item_added_callback api_item_added,
polling::instance().set_callback({"check_deleted", polling::frequency::low, polling::instance().set_callback({"check_deleted", polling::frequency::low,
[this]() { remove_deleted_files(false); }}); [this]() { remove_deleted_files(false); }});
polling::instance().set_callback({"drive_space", polling::frequency::high,
[this]() { update_used_drive_space(); }});
return true; return true;
} }
@ -721,6 +712,19 @@ void base_provider::stop() {
db3_.reset(); db3_.reset();
} }
void base_provider::update_used_drive_space() const {
REPERTORY_USES_FUNCTION_NAME();
try {
auto next_used_space = get_used_drive_space_impl();
get_file_mgr()->update_used_space(next_used_space);
used_space = next_used_space;
} catch (const std::exception &ex) {
utils::error::raise_error(function_name, ex,
"failed to get used drive space");
}
}
auto base_provider::upload_file(const std::string &api_path, auto base_provider::upload_file(const std::string &api_path,
const std::string &source_path, const std::string &source_path,
stop_type &stop_requested) -> api_error { stop_type &stop_requested) -> api_error {

84
repertory/librepertory/src/providers/s3/s3_provider.cpp
View File

@ -44,9 +44,8 @@ s3_provider::s3_provider(app_config &config, i_http_comm &comm)
get_comm().enable_s3_path_style(config.get_s3_config().use_path_style); get_comm().enable_s3_path_style(config.get_s3_config().use_path_style);
} }
auto s3_provider::add_if_not_found(api_file &file, auto s3_provider::add_if_not_found(
const std::string &object_name) const api_file &file, const std::string &object_name) const -> api_error {
-> api_error {
api_meta_map meta{}; api_meta_map meta{};
if (get_item_meta(file.api_path, meta) == api_error::item_not_found) { if (get_item_meta(file.api_path, meta) == api_error::item_not_found) {
auto err = create_path_directories( auto err = create_path_directories(
@ -72,7 +71,7 @@ auto s3_provider::convert_api_date(std::string_view date) -> std::uint64_t {
utils::string::split(date_parts.at(1U), 'Z', true).at(0U)) * utils::string::split(date_parts.at(1U), 'Z', true).at(0U)) *
1000000UL; 1000000UL;
struct tm tm1{}; struct tm tm1 {};
#if defined(_WIN32) #if defined(_WIN32)
utils::time::strptime(date_time.c_str(), "%Y-%m-%dT%T", &tm1); utils::time::strptime(date_time.c_str(), "%Y-%m-%dT%T", &tm1);
return nanos + utils::time::windows_time_t_to_unix_time(_mkgmtime(&tm1)); return nanos + utils::time::windows_time_t_to_unix_time(_mkgmtime(&tm1));
@ -163,9 +162,8 @@ auto s3_provider::create_file_extra(const std::string &api_path,
return api_error::success; return api_error::success;
} }
auto s3_provider::create_path_directories(const std::string &api_path, auto s3_provider::create_path_directories(
const std::string &key) const const std::string &api_path, const std::string &key) const -> api_error {
-> api_error {
if (api_path == "/") { if (api_path == "/") {
return api_error::success; return api_error::success;
} }
@ -281,9 +279,8 @@ auto s3_provider::get_directory_item_count(const std::string &api_path) const
return 0U; return 0U;
} }
auto s3_provider::get_directory_items_impl(const std::string &api_path, auto s3_provider::get_directory_items_impl(
directory_item_list &list) const const std::string &api_path, directory_item_list &list) const -> api_error {
-> api_error {
REPERTORY_USES_FUNCTION_NAME(); REPERTORY_USES_FUNCTION_NAME();
const auto cfg = get_config().get_s3_config(); const auto cfg = get_config().get_s3_config();
@ -402,8 +399,8 @@ auto s3_provider::get_directory_items_impl(const std::string &api_path,
return ret; return ret;
} }
auto s3_provider::get_file(const std::string &api_path, api_file &file) const auto s3_provider::get_file(const std::string &api_path,
-> api_error { api_file &file) const -> api_error {
REPERTORY_USES_FUNCTION_NAME(); REPERTORY_USES_FUNCTION_NAME();
try { try {
@ -500,9 +497,8 @@ auto s3_provider::get_file_list(api_file_list &list) const -> api_error {
return api_error::success; return api_error::success;
} }
auto s3_provider::get_last_modified(bool directory, auto s3_provider::get_last_modified(
const std::string &api_path) const bool directory, const std::string &api_path) const -> std::uint64_t {
-> std::uint64_t {
bool is_encrypted{}; bool is_encrypted{};
std::string object_name; std::string object_name;
head_object_result result{}; head_object_result result{};
@ -512,10 +508,9 @@ auto s3_provider::get_last_modified(bool directory,
: utils::time::get_time_now(); : utils::time::get_time_now();
} }
auto s3_provider::get_object_info(bool directory, const std::string &api_path, auto s3_provider::get_object_info(
bool &is_encrypted, std::string &object_name, bool directory, const std::string &api_path, bool &is_encrypted,
head_object_result &result) const std::string &object_name, head_object_result &result) const -> api_error {
-> api_error {
REPERTORY_USES_FUNCTION_NAME(); REPERTORY_USES_FUNCTION_NAME();
try { try {
@ -564,11 +559,10 @@ auto s3_provider::get_object_info(bool directory, const std::string &api_path,
return api_error::error; return api_error::error;
} }
auto s3_provider::get_object_list(std::string &response_data, auto s3_provider::get_object_list(
long &response_code, std::string &response_data, long &response_code,
std::optional<std::string> delimiter, std::optional<std::string> delimiter, std::optional<std::string> prefix,
std::optional<std::string> prefix) const std::optional<std::string> token) const -> bool {
-> bool {
curl::requests::http_get get{}; curl::requests::http_get get{};
get.allow_timeout = true; get.allow_timeout = true;
get.aws_service = "aws:amz:" + get_config().get_s3_config().region + ":s3"; get.aws_service = "aws:amz:" + get_config().get_s3_config().region + ":s3";
@ -581,6 +575,9 @@ auto s3_provider::get_object_list(std::string &response_data,
get.query["prefix"] = prefix.value(); get.query["prefix"] = prefix.value();
utils::string::left_trim(get.query["prefix"], '/'); utils::string::left_trim(get.query["prefix"], '/');
} }
if (token.has_value() && not token.value().empty()) {
get.query["continuation-token"] = token.value();
}
get.response_handler = [&response_data](const data_buffer &data, get.response_handler = [&response_data](const data_buffer &data,
long /*response_code*/) { long /*response_code*/) {
response_data = std::string(data.begin(), data.end()); response_data = std::string(data.begin(), data.end());
@ -595,38 +592,57 @@ auto s3_provider::get_total_drive_space() const -> std::uint64_t {
} }
auto s3_provider::get_used_drive_space_impl() const -> std::uint64_t { auto s3_provider::get_used_drive_space_impl() const -> std::uint64_t {
auto grab_more{true};
std::string token{};
std::uint64_t total_size{};
while (grab_more) {
std::string response_data; std::string response_data;
long response_code{}; long response_code{};
if (not get_object_list(response_data, response_code)) { if (not get_object_list(response_data, response_code, std::nullopt,
return 0U; std::nullopt, token)) {
return total_size;
} }
if (response_code != http_error_codes::ok) { if (response_code != http_error_codes::ok) {
return 0U; return total_size;
} }
pugi::xml_document doc; pugi::xml_document doc;
auto res = doc.load_string(response_data.c_str()); auto res = doc.load_string(response_data.c_str());
if (res.status != pugi::xml_parse_status::status_ok) { if (res.status != pugi::xml_parse_status::status_ok) {
return 0U; return total_size;
} }
const auto cfg = get_config().get_s3_config(); const auto cfg = get_config().get_s3_config();
const auto is_encrypted = not cfg.encryption_token.empty(); const auto is_encrypted = not cfg.encryption_token.empty();
auto node_list = doc.select_nodes("/ListBucketResult/Contents"); auto node_list = doc.select_nodes("/ListBucketResult/Contents");
return std::accumulate( grab_more = doc.select_node("/ListBucketResult/IsTruncated")
node_list.begin(), node_list.end(), std::uint64_t(0U), .node()
.text()
.as_bool();
if (grab_more) {
token = doc.select_node("/ListBucketResult/NextContinuationToken")
.node()
.text()
.as_string();
}
total_size = std::accumulate(
node_list.begin(), node_list.end(), total_size,
[&is_encrypted](std::uint64_t total, auto node) -> std::uint64_t { [&is_encrypted](std::uint64_t total, auto node) -> std::uint64_t {
auto size = node.node().select_node("Size").node().text().as_ullong(); auto size = node.node().select_node("Size").node().text().as_ullong();
return total + (is_encrypted ? utils::encryption::encrypting_reader:: return total + (is_encrypted ? utils::encryption::encrypting_reader::
calculate_decrypted_size(size) calculate_decrypted_size(size)
: size); : size);
}); });
}
return total_size;
} }
auto s3_provider::is_directory(const std::string &api_path, bool &exists) const auto s3_provider::is_directory(const std::string &api_path,
-> api_error { bool &exists) const -> api_error {
REPERTORY_USES_FUNCTION_NAME(); REPERTORY_USES_FUNCTION_NAME();
exists = false; exists = false;
@ -653,8 +669,8 @@ auto s3_provider::is_directory(const std::string &api_path, bool &exists) const
return api_error::error; return api_error::error;
} }
auto s3_provider::is_file(const std::string &api_path, bool &exists) const auto s3_provider::is_file(const std::string &api_path,
-> api_error { bool &exists) const -> api_error {
REPERTORY_USES_FUNCTION_NAME(); REPERTORY_USES_FUNCTION_NAME();
exists = false; exists = false;