BlockStorage/repertory
1
0
Fork 0
Code Issues 7 Pull Requests Releases 6 Wiki Activity

fix drive space

Browse Source
This commit is contained in:
Scott E. Graves 2024-11-30 16:58:26 -06:00
parent b1735ab0af
commit 8af198f7a4
4 changed files with 223 additions and 211 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

146
repertory/librepertory/include/providers/base_provider.hpp
View File

@ -44,23 +44,25 @@ private:
api_item_added_callback api_item_added_;
std::unique_ptr<meta_db> db3_;
i_file_manager *fm_{};
mutable std::atomic<std::uint64_t> used_space;
private:
void remove_deleted_files(bool source_only);
void update_used_drive_space() const;
protected:
[[nodiscard]] static auto create_api_file(std::string path, std::string key,
std::uint64_t size,
std::uint64_t file_time)
-> api_file;
[[nodiscard]] static auto
create_api_file(std::string path, std::string key, std::uint64_t size,
std::uint64_t file_time) -> api_file;
[[nodiscard]] static auto create_api_file(std::string path,
std::uint64_t size,
api_meta_map &meta) -> api_file;
[[nodiscard]] virtual auto create_directory_impl(const std::string &api_path,
api_meta_map &meta)
-> api_error = 0;
[[nodiscard]] virtual auto
create_directory_impl(const std::string &api_path,
api_meta_map &meta) -> api_error = 0;
[[nodiscard]] virtual auto
create_file_extra(const std::string & /* api_path */,
@ -72,8 +74,8 @@ protected:
return api_item_added_;
}
[[nodiscard]] auto get_api_item_added() const
-> const api_item_added_callback & {
[[nodiscard]] auto
get_api_item_added() const -> const api_item_added_callback & {
return api_item_added_;
}
@ -97,25 +99,23 @@ protected:
return fm_;
}
[[nodiscard]] virtual auto get_used_drive_space_impl() const
-> std::uint64_t = 0;
[[nodiscard]] virtual auto
get_used_drive_space_impl() const -> std::uint64_t = 0;
[[nodiscard]] virtual auto remove_directory_impl(const std::string &api_path)
-> api_error = 0;
[[nodiscard]] virtual auto
remove_directory_impl(const std::string &api_path) -> api_error = 0;
[[nodiscard]] virtual auto remove_file_impl(const std::string &api_path)
-> api_error = 0;
[[nodiscard]] virtual auto
remove_file_impl(const std::string &api_path) -> api_error = 0;
[[nodiscard]] virtual auto upload_file_impl(const std::string &api_path,
const std::string &source_path,
stop_type &stop_requested)
-> api_error = 0;
[[nodiscard]] virtual auto
upload_file_impl(const std::string &api_path, const std::string &source_path,
stop_type &stop_requested) -> api_error = 0;
public:
[[nodiscard]] auto
create_directory_clone_source_meta(const std::string &source_api_path,
const std::string &api_path)
-> api_error override;
[[nodiscard]] auto create_directory_clone_source_meta(
const std::string &source_api_path,
const std::string &api_path) -> api_error override;
[[nodiscard]] auto create_directory(const std::string &api_path,
api_meta_map &meta) -> api_error override;
@ -123,82 +123,76 @@ public:
[[nodiscard]] auto create_file(const std::string &api_path,
api_meta_map &meta) -> api_error override;
[[nodiscard]] auto get_api_path_from_source(const std::string &source_path,
std::string &api_path) const
-> api_error override;
[[nodiscard]] auto get_directory_items(const std::string &api_path,
directory_item_list &list) const
-> api_error override;
[[nodiscard]] auto get_file_size(const std::string &api_path,
std::uint64_t &file_size) const
-> api_error override;
[[nodiscard]] auto get_filesystem_item(const std::string &api_path,
bool directory,
filesystem_item &fsi) const
-> api_error override;
[[nodiscard]] auto get_filesystem_item_and_file(const std::string &api_path,
api_file &f,
filesystem_item &fsi) const
-> api_error override;
[[nodiscard]] auto
get_api_path_from_source(const std::string &source_path,
std::string &api_path) const -> api_error override;
[[nodiscard]] auto
get_filesystem_item_from_source_path(const std::string &source_path,
filesystem_item &fsi) const
-> api_error override;
get_directory_items(const std::string &api_path,
directory_item_list &list) const -> api_error override;
[[nodiscard]] auto get_item_meta(const std::string &api_path,
api_meta_map &meta) const
-> api_error override;
[[nodiscard]] auto
get_file_size(const std::string &api_path,
std::uint64_t &file_size) const -> api_error override;
[[nodiscard]] auto get_item_meta(const std::string &api_path,
const std::string &key,
std::string &value) const
-> api_error override;
[[nodiscard]] auto
get_filesystem_item(const std::string &api_path, bool directory,
filesystem_item &fsi) const -> api_error override;
[[nodiscard]] auto get_pinned_files() const
-> std::vector<std::string> override;
[[nodiscard]] auto get_filesystem_item_and_file(
const std::string &api_path, api_file &f,
filesystem_item &fsi) const -> api_error override;
[[nodiscard]] auto get_filesystem_item_from_source_path(
const std::string &source_path,
filesystem_item &fsi) const -> api_error override;
[[nodiscard]] auto
get_item_meta(const std::string &api_path,
api_meta_map &meta) const -> api_error override;
[[nodiscard]] auto
get_item_meta(const std::string &api_path, const std::string &key,
std::string &value) const -> api_error override;
[[nodiscard]] auto
get_pinned_files() const -> std::vector<std::string> override;
[[nodiscard]] auto get_total_item_count() const -> std::uint64_t override;
[[nodiscard]] auto get_used_drive_space() const -> std::uint64_t override;
[[nodiscard]] auto is_file_writeable(const std::string &api_path) const
-> bool override;
[[nodiscard]] auto
is_file_writeable(const std::string &api_path) const -> bool override;
[[nodiscard]] auto is_read_only() const -> bool override { return false; }
[[nodiscard]] auto remove_directory(const std::string &api_path)
-> api_error override;
[[nodiscard]] auto
remove_directory(const std::string &api_path) -> api_error override;
[[nodiscard]] auto remove_file(const std::string &api_path)
-> api_error override;
[[nodiscard]] auto
remove_file(const std::string &api_path) -> api_error override;
[[nodiscard]] auto remove_item_meta(const std::string &api_path,
const std::string &key)
-> api_error override;
[[nodiscard]] auto
remove_item_meta(const std::string &api_path,
const std::string &key) -> api_error override;
[[nodiscard]] auto set_item_meta(const std::string &api_path,
const std::string &key,
const std::string &value)
-> api_error override;
[[nodiscard]] auto
set_item_meta(const std::string &api_path, const std::string &key,
const std::string &value) -> api_error override;
[[nodiscard]] auto set_item_meta(const std::string &api_path,
const api_meta_map &meta)
-> api_error override;
[[nodiscard]] auto
set_item_meta(const std::string &api_path,
const api_meta_map &meta) -> api_error override;
[[nodiscard]] auto start(api_item_added_callback api_item_added,
i_file_manager *mgr) -> bool override;
void stop() override;
[[nodiscard]] auto upload_file(const std::string &api_path,
const std::string &source_path,
stop_type &stop_requested)
-> api_error override;
[[nodiscard]] auto
upload_file(const std::string &api_path, const std::string &source_path,
stop_type &stop_requested) -> api_error override;
};
} // namespace repertory

94
repertory/librepertory/include/providers/s3/s3_provider.hpp
View File

@ -47,71 +47,70 @@ public:
auto operator=(s3_provider &&) -> s3_provider & = delete;
private:
[[nodiscard]] auto add_if_not_found(api_file &file,
const std::string &object_name) const
-> api_error;
[[nodiscard]] auto
add_if_not_found(api_file &file,
const std::string &object_name) const -> api_error;
[[nodiscard]] auto create_file_extra(const std::string &api_path,
api_meta_map &meta)
-> api_error override;
[[nodiscard]] auto
create_file_extra(const std::string &api_path,
api_meta_map &meta) -> api_error override;
[[nodiscard]] auto create_path_directories(const std::string &api_path,
const std::string &key) const
-> api_error;
[[nodiscard]] auto
create_path_directories(const std::string &api_path,
const std::string &key) const -> api_error;
[[nodiscard]] auto decrypt_object_name(std::string &object_name) const
-> api_error;
[[nodiscard]] auto
decrypt_object_name(std::string &object_name) const -> api_error;
[[nodiscard]] auto get_last_modified(bool directory,
const std::string &api_path) const
-> std::uint64_t;
[[nodiscard]] auto
get_last_modified(bool directory,
const std::string &api_path) const -> std::uint64_t;
[[nodiscard]] auto
get_object_info(bool directory, const std::string &api_path,
bool &is_encrypted, std::string &object_name,
head_object_result &result) const -> api_error;
[[nodiscard]] auto
get_object_list(std::string &response_data, long &response_code,
std::optional<std::string> delimiter = std::nullopt,
std::optional<std::string> prefix = std::nullopt) const
-> bool;
[[nodiscard]] auto get_object_list(
std::string &response_data, long &response_code,
std::optional<std::string> delimiter = std::nullopt,
std::optional<std::string> prefix = std::nullopt,
std::optional<std::string> token = std::nullopt) const -> bool;
protected:
[[nodiscard]] auto create_directory_impl(const std::string &api_path,
api_meta_map &meta)
-> api_error override;
[[nodiscard]] auto
create_directory_impl(const std::string &api_path,
api_meta_map &meta) -> api_error override;
[[nodiscard]] auto get_directory_items_impl(const std::string &api_path,
directory_item_list &list) const
-> api_error override;
[[nodiscard]] auto get_used_drive_space_impl() const
-> std::uint64_t override;
[[nodiscard]] auto
get_used_drive_space_impl() const -> std::uint64_t override;
[[nodiscard]] auto remove_directory_impl(const std::string &api_path)
-> api_error override;
[[nodiscard]] auto
remove_directory_impl(const std::string &api_path) -> api_error override;
[[nodiscard]] auto remove_file_impl(const std::string &api_path)
-> api_error override;
[[nodiscard]] auto
remove_file_impl(const std::string &api_path) -> api_error override;
[[nodiscard]] auto upload_file_impl(const std::string &api_path,
const std::string &source_path,
stop_type &stop_requested)
-> api_error override;
[[nodiscard]] auto
upload_file_impl(const std::string &api_path, const std::string &source_path,
stop_type &stop_requested) -> api_error override;
public:
[[nodiscard]] static auto convert_api_date(std::string_view date)
-> std::uint64_t;
[[nodiscard]] static auto
convert_api_date(std::string_view date) -> std::uint64_t;
[[nodiscard]] auto get_directory_item_count(const std::string &api_path) const
-> std::uint64_t override;
[[nodiscard]] auto get_file(const std::string &api_path, api_file &file) const
-> api_error override;
[[nodiscard]] auto get_file(const std::string &api_path,
api_file &file) const -> api_error override;
[[nodiscard]] auto get_file_list(api_file_list &list) const
-> api_error override;
[[nodiscard]] auto
get_file_list(api_file_list &list) const -> api_error override;
[[nodiscard]] auto get_total_drive_space() const -> std::uint64_t override;
@ -122,8 +121,8 @@ public:
[[nodiscard]] auto is_directory(const std::string &api_path,
bool &exists) const -> api_error override;
[[nodiscard]] auto is_file(const std::string &api_path, bool &exists) const
-> api_error override;
[[nodiscard]] auto is_file(const std::string &api_path,
bool &exists) const -> api_error override;
[[nodiscard]] auto is_online() const -> bool override;
@ -131,15 +130,14 @@ public:
return false;
};
[[nodiscard]] auto read_file_bytes(const std::string &api_path,
std::size_t size, std::uint64_t offset,
data_buffer &data,
stop_type &stop_requested)
-> api_error override;
[[nodiscard]] auto
read_file_bytes(const std::string &api_path, std::size_t size,
std::uint64_t offset, data_buffer &data,
stop_type &stop_requested) -> api_error override;
[[nodiscard]] auto rename_file(const std::string &from_api_path,
const std::string &to_api_path)
-> api_error override;
[[nodiscard]] auto
rename_file(const std::string &from_api_path,
const std::string &to_api_path) -> api_error override;
[[nodiscard]] auto start(api_item_added_callback api_item_added,
i_file_manager *mgr) -> bool override;

70
repertory/librepertory/src/providers/base_provider.cpp
View File

@ -33,8 +33,8 @@
namespace repertory {
auto base_provider::create_api_file(std::string path, std::string key,
std::uint64_t size, std::uint64_t file_time)
-> api_file {
std::uint64_t size,
std::uint64_t file_time) -> api_file {
api_file file{};
file.api_path = utils::path::create_api_path(path);
file.api_parent = utils::path::get_parent_api_path(file.api_path);
@ -66,8 +66,8 @@ auto base_provider::create_api_file(std::string path, std::uint64_t size,
}
auto base_provider::create_directory_clone_source_meta(
const std::string &source_api_path, const std::string &api_path)
-> api_error {
const std::string &source_api_path,
const std::string &api_path) -> api_error {
REPERTORY_USES_FUNCTION_NAME();
bool exists{};
@ -164,8 +164,8 @@ auto base_provider::create_directory(const std::string &api_path,
return set_item_meta(api_path, meta);
}
auto base_provider::create_file(const std::string &api_path, api_meta_map &meta)
-> api_error {
auto base_provider::create_file(const std::string &api_path,
api_meta_map &meta) -> api_error {
REPERTORY_USES_FUNCTION_NAME();
bool exists{};
@ -222,9 +222,8 @@ auto base_provider::create_file(const std::string &api_path, api_meta_map &meta)
return api_error::error;
}
auto base_provider::get_api_path_from_source(const std::string &source_path,
std::string &api_path) const
-> api_error {
auto base_provider::get_api_path_from_source(
const std::string &source_path, std::string &api_path) const -> api_error {
REPERTORY_USES_FUNCTION_NAME();
if (source_path.empty()) {
@ -237,9 +236,8 @@ auto base_provider::get_api_path_from_source(const std::string &source_path,
return db3_->get_api_path(source_path, api_path);
}
auto base_provider::get_directory_items(const std::string &api_path,
directory_item_list &list) const
-> api_error {
auto base_provider::get_directory_items(
const std::string &api_path, directory_item_list &list) const -> api_error {
REPERTORY_USES_FUNCTION_NAME();
bool exists{};
@ -303,10 +301,9 @@ auto base_provider::get_file_size(const std::string &api_path,
return api_error::success;
}
auto base_provider::get_filesystem_item(const std::string &api_path,
bool directory,
filesystem_item &fsi) const
-> api_error {
auto base_provider::get_filesystem_item(
const std::string &api_path, bool directory,
filesystem_item &fsi) const -> api_error {
bool exists{};
auto res = is_directory(api_path, exists);
if (res != api_error::success) {
@ -339,10 +336,9 @@ auto base_provider::get_filesystem_item(const std::string &api_path,
return api_error::success;
}
auto base_provider::get_filesystem_item_and_file(const std::string &api_path,
api_file &file,
filesystem_item &fsi) const
-> api_error {
auto base_provider::get_filesystem_item_and_file(
const std::string &api_path, api_file &file,
filesystem_item &fsi) const -> api_error {
auto res = get_file(api_path, file);
if (res != api_error::success) {
return res;
@ -403,18 +399,11 @@ auto base_provider::get_total_item_count() const -> std::uint64_t {
}
auto base_provider::get_used_drive_space() const -> std::uint64_t {
REPERTORY_USES_FUNCTION_NAME();
try {
auto used_space = get_used_drive_space_impl();
get_file_mgr()->update_used_space(used_space);
return used_space;
} catch (const std::exception &ex) {
utils::error::raise_error(function_name, ex,
"failed to get used drive space");
if (used_space == 0U) {
update_used_drive_space();
}
return 0U;
return used_space;
}
auto base_provider::is_file_writeable(const std::string &api_path) const
@ -471,10 +460,10 @@ void base_provider::remove_deleted_files(bool source_only) {
return;
}
struct removed_item {
std::string api_path{};
struct removed_item final {
std::string api_path;
bool directory{};
std::string source_path{};
std::string source_path;
};
api_file_list list{};
@ -713,6 +702,8 @@ auto base_provider::start(api_item_added_callback api_item_added,
polling::instance().set_callback({"check_deleted", polling::frequency::low,
[this]() { remove_deleted_files(false); }});
polling::instance().set_callback({"drive_space", polling::frequency::high,
[this]() { update_used_drive_space(); }});
return true;
}
@ -721,6 +712,19 @@ void base_provider::stop() {
db3_.reset();
}
void base_provider::update_used_drive_space() const {
REPERTORY_USES_FUNCTION_NAME();
try {
auto next_used_space = get_used_drive_space_impl();
get_file_mgr()->update_used_space(next_used_space);
used_space = next_used_space;
} catch (const std::exception &ex) {
utils::error::raise_error(function_name, ex,
"failed to get used drive space");
}
}
auto base_provider::upload_file(const std::string &api_path,
const std::string &source_path,
stop_type &stop_requested) -> api_error {

124
repertory/librepertory/src/providers/s3/s3_provider.cpp
View File

@ -44,9 +44,8 @@ s3_provider::s3_provider(app_config &config, i_http_comm &comm)
get_comm().enable_s3_path_style(config.get_s3_config().use_path_style);
}
auto s3_provider::add_if_not_found(api_file &file,
const std::string &object_name) const
-> api_error {
auto s3_provider::add_if_not_found(
api_file &file, const std::string &object_name) const -> api_error {
api_meta_map meta{};
if (get_item_meta(file.api_path, meta) == api_error::item_not_found) {
auto err = create_path_directories(
@ -72,7 +71,7 @@ auto s3_provider::convert_api_date(std::string_view date) -> std::uint64_t {
utils::string::split(date_parts.at(1U), 'Z', true).at(0U)) *
1000000UL;
struct tm tm1{};
struct tm tm1 {};
#if defined(_WIN32)
utils::time::strptime(date_time.c_str(), "%Y-%m-%dT%T", &tm1);
return nanos + utils::time::windows_time_t_to_unix_time(_mkgmtime(&tm1));
@ -163,9 +162,8 @@ auto s3_provider::create_file_extra(const std::string &api_path,
return api_error::success;
}
auto s3_provider::create_path_directories(const std::string &api_path,
const std::string &key) const
-> api_error {
auto s3_provider::create_path_directories(
const std::string &api_path, const std::string &key) const -> api_error {
if (api_path == "/") {
return api_error::success;
}
@ -281,9 +279,8 @@ auto s3_provider::get_directory_item_count(const std::string &api_path) const
return 0U;
}
auto s3_provider::get_directory_items_impl(const std::string &api_path,
directory_item_list &list) const
-> api_error {
auto s3_provider::get_directory_items_impl(
const std::string &api_path, directory_item_list &list) const -> api_error {
REPERTORY_USES_FUNCTION_NAME();
const auto cfg = get_config().get_s3_config();
@ -402,8 +399,8 @@ auto s3_provider::get_directory_items_impl(const std::string &api_path,
return ret;
}
auto s3_provider::get_file(const std::string &api_path, api_file &file) const
-> api_error {
auto s3_provider::get_file(const std::string &api_path,
api_file &file) const -> api_error {
REPERTORY_USES_FUNCTION_NAME();
try {
@ -500,9 +497,8 @@ auto s3_provider::get_file_list(api_file_list &list) const -> api_error {
return api_error::success;
}
auto s3_provider::get_last_modified(bool directory,
const std::string &api_path) const
-> std::uint64_t {
auto s3_provider::get_last_modified(
bool directory, const std::string &api_path) const -> std::uint64_t {
bool is_encrypted{};
std::string object_name;
head_object_result result{};
@ -512,10 +508,9 @@ auto s3_provider::get_last_modified(bool directory,
: utils::time::get_time_now();
}
auto s3_provider::get_object_info(bool directory, const std::string &api_path,
bool &is_encrypted, std::string &object_name,
head_object_result &result) const
-> api_error {
auto s3_provider::get_object_info(
bool directory, const std::string &api_path, bool &is_encrypted,
std::string &object_name, head_object_result &result) const -> api_error {
REPERTORY_USES_FUNCTION_NAME();
try {
@ -564,11 +559,10 @@ auto s3_provider::get_object_info(bool directory, const std::string &api_path,
return api_error::error;
}
auto s3_provider::get_object_list(std::string &response_data,
long &response_code,
std::optional<std::string> delimiter,
std::optional<std::string> prefix) const
-> bool {
auto s3_provider::get_object_list(
std::string &response_data, long &response_code,
std::optional<std::string> delimiter, std::optional<std::string> prefix,
std::optional<std::string> token) const -> bool {
curl::requests::http_get get{};
get.allow_timeout = true;
get.aws_service = "aws:amz:" + get_config().get_s3_config().region + ":s3";
@ -581,6 +575,9 @@ auto s3_provider::get_object_list(std::string &response_data,
get.query["prefix"] = prefix.value();
utils::string::left_trim(get.query["prefix"], '/');
}
if (token.has_value() && not token.value().empty()) {
get.query["continuation-token"] = token.value();
}
get.response_handler = [&response_data](const data_buffer &data,
long /*response_code*/) {
response_data = std::string(data.begin(), data.end());
@ -595,38 +592,57 @@ auto s3_provider::get_total_drive_space() const -> std::uint64_t {
}
auto s3_provider::get_used_drive_space_impl() const -> std::uint64_t {
std::string response_data;
long response_code{};
if (not get_object_list(response_data, response_code)) {
return 0U;
auto grab_more{true};
std::string token{};
std::uint64_t total_size{};
while (grab_more) {
std::string response_data;
long response_code{};
if (not get_object_list(response_data, response_code, std::nullopt,
std::nullopt, token)) {
return total_size;
}
if (response_code != http_error_codes::ok) {
return total_size;
}
pugi::xml_document doc;
auto res = doc.load_string(response_data.c_str());
if (res.status != pugi::xml_parse_status::status_ok) {
return total_size;
}
const auto cfg = get_config().get_s3_config();
const auto is_encrypted = not cfg.encryption_token.empty();
auto node_list = doc.select_nodes("/ListBucketResult/Contents");
grab_more = doc.select_node("/ListBucketResult/IsTruncated")
.node()
.text()
.as_bool();
if (grab_more) {
token = doc.select_node("/ListBucketResult/NextContinuationToken")
.node()
.text()
.as_string();
}
total_size = std::accumulate(
node_list.begin(), node_list.end(), total_size,
[&is_encrypted](std::uint64_t total, auto node) -> std::uint64_t {
auto size = node.node().select_node("Size").node().text().as_ullong();
return total + (is_encrypted ? utils::encryption::encrypting_reader::
calculate_decrypted_size(size)
: size);
});
}
if (response_code != http_error_codes::ok) {
return 0U;
}
pugi::xml_document doc;
auto res = doc.load_string(response_data.c_str());
if (res.status != pugi::xml_parse_status::status_ok) {
return 0U;
}
const auto cfg = get_config().get_s3_config();
const auto is_encrypted = not cfg.encryption_token.empty();
auto node_list = doc.select_nodes("/ListBucketResult/Contents");
return std::accumulate(
node_list.begin(), node_list.end(), std::uint64_t(0U),
[&is_encrypted](std::uint64_t total, auto node) -> std::uint64_t {
auto size = node.node().select_node("Size").node().text().as_ullong();
return total + (is_encrypted ? utils::encryption::encrypting_reader::
calculate_decrypted_size(size)
: size);
});
return total_size;
}
auto s3_provider::is_directory(const std::string &api_path, bool &exists) const
-> api_error {
auto s3_provider::is_directory(const std::string &api_path,
bool &exists) const -> api_error {
REPERTORY_USES_FUNCTION_NAME();
exists = false;
@ -653,8 +669,8 @@ auto s3_provider::is_directory(const std::string &api_path, bool &exists) const
return api_error::error;
}
auto s3_provider::is_file(const std::string &api_path, bool &exists) const
-> api_error {
auto s3_provider::is_file(const std::string &api_path,
bool &exists) const -> api_error {
REPERTORY_USES_FUNCTION_NAME();
exists = false;