BlockStorage/repertory
1
0
Fork 0
Code Issues 8 Pull Requests Releases 6 Wiki Activity

fixes
All checks were successful
BlockStorage/repertory/pipeline/head This commit looks good
Details

Browse Source
This commit is contained in:
Scott E. Graves 2024-12-02 18:26:14 -06:00
parent 412b807d3f
commit ea8eb53a34
3 changed files with 113 additions and 139 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

141
repertory/librepertory/include/providers/base_provider.hpp
View File

@ -47,21 +47,20 @@ private:
std::atomic<bool> stop_requested_{false}; std::atomic<bool> stop_requested_{false};
private: private:
void remove_deleted_files(bool source_only); void remove_deleted_files();
protected: protected:
[[nodiscard]] static auto create_api_file(std::string path, std::string key, [[nodiscard]] static auto
std::uint64_t size, create_api_file(std::string path, std::string key, std::uint64_t size,
std::uint64_t file_time) std::uint64_t file_time) -> api_file;
-> api_file;
[[nodiscard]] static auto create_api_file(std::string path, [[nodiscard]] static auto create_api_file(std::string path,
std::uint64_t size, std::uint64_t size,
api_meta_map &meta) -> api_file; api_meta_map &meta) -> api_file;
[[nodiscard]] virtual auto create_directory_impl(const std::string &api_path, [[nodiscard]] virtual auto
api_meta_map &meta) create_directory_impl(const std::string &api_path,
-> api_error = 0; api_meta_map &meta) -> api_error = 0;
[[nodiscard]] virtual auto [[nodiscard]] virtual auto
create_file_extra(const std::string & /* api_path */, create_file_extra(const std::string & /* api_path */,
@ -73,8 +72,8 @@ protected:
return api_item_added_; return api_item_added_;
} }
[[nodiscard]] auto get_api_item_added() const [[nodiscard]] auto
-> const api_item_added_callback & { get_api_item_added() const -> const api_item_added_callback & {
return api_item_added_; return api_item_added_;
} }
@ -100,22 +99,20 @@ protected:
return fm_; return fm_;
} }
[[nodiscard]] virtual auto remove_directory_impl(const std::string &api_path) [[nodiscard]] virtual auto
-> api_error = 0; remove_directory_impl(const std::string &api_path) -> api_error = 0;
[[nodiscard]] virtual auto remove_file_impl(const std::string &api_path) [[nodiscard]] virtual auto
-> api_error = 0; remove_file_impl(const std::string &api_path) -> api_error = 0;
[[nodiscard]] virtual auto upload_file_impl(const std::string &api_path, [[nodiscard]] virtual auto
const std::string &source_path, upload_file_impl(const std::string &api_path, const std::string &source_path,
stop_type &stop_requested) stop_type &stop_requested) -> api_error = 0;
-> api_error = 0;
public: public:
[[nodiscard]] auto [[nodiscard]] auto create_directory_clone_source_meta(
create_directory_clone_source_meta(const std::string &source_api_path, const std::string &source_api_path,
const std::string &api_path) const std::string &api_path) -> api_error override;
-> api_error override;
[[nodiscard]] auto create_directory(const std::string &api_path, [[nodiscard]] auto create_directory(const std::string &api_path,
api_meta_map &meta) -> api_error override; api_meta_map &meta) -> api_error override;
@ -123,82 +120,76 @@ public:
[[nodiscard]] auto create_file(const std::string &api_path, [[nodiscard]] auto create_file(const std::string &api_path,
api_meta_map &meta) -> api_error override; api_meta_map &meta) -> api_error override;
[[nodiscard]] auto get_api_path_from_source(const std::string &source_path, [[nodiscard]] auto
std::string &api_path) const get_api_path_from_source(const std::string &source_path,
-> api_error override; std::string &api_path) const -> api_error override;
[[nodiscard]] auto get_directory_items(const std::string &api_path,
directory_item_list &list) const
-> api_error override;
[[nodiscard]] auto get_file_size(const std::string &api_path,
std::uint64_t &file_size) const
-> api_error override;
[[nodiscard]] auto get_filesystem_item(const std::string &api_path,
bool directory,
filesystem_item &fsi) const
-> api_error override;
[[nodiscard]] auto get_filesystem_item_and_file(const std::string &api_path,
api_file &f,
filesystem_item &fsi) const
-> api_error override;
[[nodiscard]] auto [[nodiscard]] auto
get_filesystem_item_from_source_path(const std::string &source_path, get_directory_items(const std::string &api_path,
filesystem_item &fsi) const directory_item_list &list) const -> api_error override;
-> api_error override;
[[nodiscard]] auto get_item_meta(const std::string &api_path, [[nodiscard]] auto
api_meta_map &meta) const get_file_size(const std::string &api_path,
-> api_error override; std::uint64_t &file_size) const -> api_error override;
[[nodiscard]] auto get_item_meta(const std::string &api_path, [[nodiscard]] auto
const std::string &key, get_filesystem_item(const std::string &api_path, bool directory,
std::string &value) const filesystem_item &fsi) const -> api_error override;
-> api_error override;
[[nodiscard]] auto get_pinned_files() const [[nodiscard]] auto get_filesystem_item_and_file(
-> std::vector<std::string> override; const std::string &api_path, api_file &f,
filesystem_item &fsi) const -> api_error override;
[[nodiscard]] auto get_filesystem_item_from_source_path(
const std::string &source_path,
filesystem_item &fsi) const -> api_error override;
[[nodiscard]] auto
get_item_meta(const std::string &api_path,
api_meta_map &meta) const -> api_error override;
[[nodiscard]] auto
get_item_meta(const std::string &api_path, const std::string &key,
std::string &value) const -> api_error override;
[[nodiscard]] auto
get_pinned_files() const -> std::vector<std::string> override;
[[nodiscard]] auto get_total_item_count() const -> std::uint64_t override; [[nodiscard]] auto get_total_item_count() const -> std::uint64_t override;
[[nodiscard]] auto get_used_drive_space() const -> std::uint64_t override; [[nodiscard]] auto get_used_drive_space() const -> std::uint64_t override;
[[nodiscard]] auto is_file_writeable(const std::string &api_path) const [[nodiscard]] auto
-> bool override; is_file_writeable(const std::string &api_path) const -> bool override;
[[nodiscard]] auto is_read_only() const -> bool override { return false; } [[nodiscard]] auto is_read_only() const -> bool override { return false; }
[[nodiscard]] auto remove_directory(const std::string &api_path) [[nodiscard]] auto
-> api_error override; remove_directory(const std::string &api_path) -> api_error override;
[[nodiscard]] auto remove_file(const std::string &api_path) [[nodiscard]] auto
-> api_error override; remove_file(const std::string &api_path) -> api_error override;
[[nodiscard]] auto remove_item_meta(const std::string &api_path, [[nodiscard]] auto
const std::string &key) remove_item_meta(const std::string &api_path,
-> api_error override; const std::string &key) -> api_error override;
[[nodiscard]] auto set_item_meta(const std::string &api_path, [[nodiscard]] auto
const std::string &key, set_item_meta(const std::string &api_path, const std::string &key,
const std::string &value) const std::string &value) -> api_error override;
-> api_error override;
[[nodiscard]] auto set_item_meta(const std::string &api_path, [[nodiscard]] auto
const api_meta_map &meta) set_item_meta(const std::string &api_path,
-> api_error override; const api_meta_map &meta) -> api_error override;
[[nodiscard]] auto start(api_item_added_callback api_item_added, [[nodiscard]] auto start(api_item_added_callback api_item_added,
i_file_manager *mgr) -> bool override; i_file_manager *mgr) -> bool override;
void stop() override; void stop() override;
[[nodiscard]] auto upload_file(const std::string &api_path, [[nodiscard]] auto
const std::string &source_path, upload_file(const std::string &api_path, const std::string &source_path,
stop_type &stop_requested) stop_type &stop_requested) -> api_error override;
-> api_error override;
}; };
} // namespace repertory } // namespace repertory

46
repertory/librepertory/src/providers/base_provider.cpp
View File

@ -33,8 +33,8 @@
namespace repertory { namespace repertory {
auto base_provider::create_api_file(std::string path, std::string key, auto base_provider::create_api_file(std::string path, std::string key,
std::uint64_t size, std::uint64_t file_time) std::uint64_t size,
-> api_file { std::uint64_t file_time) -> api_file {
api_file file{}; api_file file{};
file.api_path = utils::path::create_api_path(path); file.api_path = utils::path::create_api_path(path);
file.api_parent = utils::path::get_parent_api_path(file.api_path); file.api_parent = utils::path::get_parent_api_path(file.api_path);
@ -66,8 +66,8 @@ auto base_provider::create_api_file(std::string path, std::uint64_t size,
} }
auto base_provider::create_directory_clone_source_meta( auto base_provider::create_directory_clone_source_meta(
const std::string &source_api_path, const std::string &api_path) const std::string &source_api_path,
-> api_error { const std::string &api_path) -> api_error {
REPERTORY_USES_FUNCTION_NAME(); REPERTORY_USES_FUNCTION_NAME();
bool exists{}; bool exists{};
@ -164,8 +164,8 @@ auto base_provider::create_directory(const std::string &api_path,
return set_item_meta(api_path, meta); return set_item_meta(api_path, meta);
} }
auto base_provider::create_file(const std::string &api_path, api_meta_map &meta) auto base_provider::create_file(const std::string &api_path,
-> api_error { api_meta_map &meta) -> api_error {
REPERTORY_USES_FUNCTION_NAME(); REPERTORY_USES_FUNCTION_NAME();
bool exists{}; bool exists{};
@ -222,9 +222,8 @@ auto base_provider::create_file(const std::string &api_path, api_meta_map &meta)
return api_error::error; return api_error::error;
} }
auto base_provider::get_api_path_from_source(const std::string &source_path, auto base_provider::get_api_path_from_source(
std::string &api_path) const const std::string &source_path, std::string &api_path) const -> api_error {
-> api_error {
REPERTORY_USES_FUNCTION_NAME(); REPERTORY_USES_FUNCTION_NAME();
if (source_path.empty()) { if (source_path.empty()) {
@ -237,9 +236,8 @@ auto base_provider::get_api_path_from_source(const std::string &source_path,
return db3_->get_api_path(source_path, api_path); return db3_->get_api_path(source_path, api_path);
} }
auto base_provider::get_directory_items(const std::string &api_path, auto base_provider::get_directory_items(
directory_item_list &list) const const std::string &api_path, directory_item_list &list) const -> api_error {
-> api_error {
REPERTORY_USES_FUNCTION_NAME(); REPERTORY_USES_FUNCTION_NAME();
bool exists{}; bool exists{};
@ -303,10 +301,9 @@ auto base_provider::get_file_size(const std::string &api_path,
return api_error::success; return api_error::success;
} }
auto base_provider::get_filesystem_item(const std::string &api_path, auto base_provider::get_filesystem_item(
bool directory, const std::string &api_path, bool directory,
filesystem_item &fsi) const filesystem_item &fsi) const -> api_error {
-> api_error {
bool exists{}; bool exists{};
auto res = is_directory(api_path, exists); auto res = is_directory(api_path, exists);
if (res != api_error::success) { if (res != api_error::success) {
@ -339,10 +336,9 @@ auto base_provider::get_filesystem_item(const std::string &api_path,
return api_error::success; return api_error::success;
} }
auto base_provider::get_filesystem_item_and_file(const std::string &api_path, auto base_provider::get_filesystem_item_and_file(
api_file &file, const std::string &api_path, api_file &file,
filesystem_item &fsi) const filesystem_item &fsi) const -> api_error {
-> api_error {
auto res = get_file(api_path, file); auto res = get_file(api_path, file);
if (res != api_error::success) { if (res != api_error::success) {
return res; return res;
@ -417,7 +413,7 @@ auto base_provider::is_file_writeable(const std::string &api_path) const
return not exists; return not exists;
} }
void base_provider::remove_deleted_files(bool source_only) { void base_provider::remove_deleted_files() {
REPERTORY_USES_FUNCTION_NAME(); REPERTORY_USES_FUNCTION_NAME();
if (not is_read_only()) { if (not is_read_only()) {
@ -460,10 +456,6 @@ void base_provider::remove_deleted_files(bool source_only) {
} }
} }
if (source_only) {
return;
}
struct removed_item final { struct removed_item final {
std::string api_path; std::string api_path;
bool directory{}; bool directory{};
@ -722,10 +714,8 @@ auto base_provider::start(api_item_added_callback api_item_added,
return false; return false;
} }
remove_deleted_files(true);
polling::instance().set_callback({"check_deleted", polling::frequency::low, polling::instance().set_callback({"check_deleted", polling::frequency::low,
[this]() { remove_deleted_files(false); }}); [this]() { remove_deleted_files(); }});
return true; return true;
} }

65
repertory/librepertory/src/providers/s3/s3_provider.cpp
View File

@ -43,9 +43,8 @@ s3_provider::s3_provider(app_config &config, i_http_comm &comm)
get_comm().enable_s3_path_style(config.get_s3_config().use_path_style); get_comm().enable_s3_path_style(config.get_s3_config().use_path_style);
} }
auto s3_provider::add_if_not_found(api_file &file, auto s3_provider::add_if_not_found(
const std::string &object_name) const api_file &file, const std::string &object_name) const -> api_error {
-> api_error {
api_meta_map meta{}; api_meta_map meta{};
if (get_item_meta(file.api_path, meta) == api_error::item_not_found) { if (get_item_meta(file.api_path, meta) == api_error::item_not_found) {
auto err = create_path_directories( auto err = create_path_directories(
@ -71,7 +70,7 @@ auto s3_provider::convert_api_date(std::string_view date) -> std::uint64_t {
utils::string::split(date_parts.at(1U), 'Z', true).at(0U)) * utils::string::split(date_parts.at(1U), 'Z', true).at(0U)) *
1000000UL; 1000000UL;
struct tm tm1{}; struct tm tm1 {};
#if defined(_WIN32) #if defined(_WIN32)
utils::time::strptime(date_time.c_str(), "%Y-%m-%dT%T", &tm1); utils::time::strptime(date_time.c_str(), "%Y-%m-%dT%T", &tm1);
return nanos + utils::time::windows_time_t_to_unix_time(_mkgmtime(&tm1)); return nanos + utils::time::windows_time_t_to_unix_time(_mkgmtime(&tm1));
@ -162,9 +161,8 @@ auto s3_provider::create_file_extra(const std::string &api_path,
return api_error::success; return api_error::success;
} }
auto s3_provider::create_path_directories(const std::string &api_path, auto s3_provider::create_path_directories(
const std::string &key) const const std::string &api_path, const std::string &key) const -> api_error {
-> api_error {
if (api_path == "/") { if (api_path == "/") {
return api_error::success; return api_error::success;
} }
@ -297,9 +295,8 @@ auto s3_provider::get_directory_item_count(const std::string &api_path) const
return 0U; return 0U;
} }
auto s3_provider::get_directory_items_impl(const std::string &api_path, auto s3_provider::get_directory_items_impl(
directory_item_list &list) const const std::string &api_path, directory_item_list &list) const -> api_error {
-> api_error {
REPERTORY_USES_FUNCTION_NAME(); REPERTORY_USES_FUNCTION_NAME();
auto cfg = get_config().get_s3_config(); auto cfg = get_config().get_s3_config();
@ -444,8 +441,8 @@ auto s3_provider::get_directory_items_impl(const std::string &api_path,
return ret; return ret;
} }
auto s3_provider::get_file(const std::string &api_path, api_file &file) const auto s3_provider::get_file(const std::string &api_path,
-> api_error { api_file &file) const -> api_error {
REPERTORY_USES_FUNCTION_NAME(); REPERTORY_USES_FUNCTION_NAME();
try { try {
@ -462,7 +459,7 @@ auto s3_provider::get_file(const std::string &api_path, api_file &file) const
file.api_parent = utils::path::get_parent_api_path(api_path); file.api_parent = utils::path::get_parent_api_path(api_path);
file.accessed_date = file.changed_date = file.creation_date = file.accessed_date = file.changed_date = file.creation_date =
file.modified_date = result.last_modified; file.modified_date = result.last_modified;
file.key = is_encrypted ? utils::path::create_api_path(api_path) : ""; file.key = is_encrypted ? utils::path::create_api_path(object_name) : "";
std::string size_str; std::string size_str;
if (get_item_meta(file.api_path, META_SIZE, size_str) == if (get_item_meta(file.api_path, META_SIZE, size_str) ==
@ -484,8 +481,8 @@ auto s3_provider::get_file(const std::string &api_path, api_file &file) const
return api_error::error; return api_error::error;
} }
auto s3_provider::get_file_list(api_file_list &list, std::string &marker) const auto s3_provider::get_file_list(api_file_list &list,
-> api_error { std::string &marker) const -> api_error {
REPERTORY_USES_FUNCTION_NAME(); REPERTORY_USES_FUNCTION_NAME();
std::string response_data; std::string response_data;
@ -494,7 +491,6 @@ auto s3_provider::get_file_list(api_file_list &list, std::string &marker) const
std::nullopt, marker)) { std::nullopt, marker)) {
return api_error::comm_error; return api_error::comm_error;
} }
marker.clear();
if (response_code != http_error_codes::ok) { if (response_code != http_error_codes::ok) {
utils::error::raise_error(function_name, response_code, utils::error::raise_error(function_name, response_code,
@ -521,8 +517,9 @@ auto s3_provider::get_file_list(api_file_list &list, std::string &marker) const
auto node_list = doc.select_nodes("/ListBucketResult/Contents"); auto node_list = doc.select_nodes("/ListBucketResult/Contents");
for (auto &&node : node_list) { for (auto &&node : node_list) {
auto api_path = auto object_name =
std::string{node.node().select_node("Key").node().text().as_string()}; std::string{node.node().select_node("Key").node().text().as_string()};
auto api_path{object_name};
if (utils::string::ends_with(api_path, "/")) { if (utils::string::ends_with(api_path, "/")) {
continue; continue;
} }
@ -549,8 +546,8 @@ auto s3_provider::get_file_list(api_file_list &list, std::string &marker) const
? utils::encryption::encrypting_reader::calculate_decrypted_size( ? utils::encryption::encrypting_reader::calculate_decrypted_size(
size) size)
: size; : size;
file.key = is_encrypted ? utils::path::create_api_path(api_path) : ""; file.key = is_encrypted ? utils::path::create_api_path(object_name) : "";
auto err = add_if_not_found(file, api_path); auto err = add_if_not_found(file, file.key);
if (err != api_error::success) { if (err != api_error::success) {
return err; return err;
} }
@ -561,9 +558,8 @@ auto s3_provider::get_file_list(api_file_list &list, std::string &marker) const
return grab_more ? api_error::more_data : api_error::success; return grab_more ? api_error::more_data : api_error::success;
} }
auto s3_provider::get_last_modified(bool directory, auto s3_provider::get_last_modified(
const std::string &api_path) const bool directory, const std::string &api_path) const -> std::uint64_t {
-> std::uint64_t {
bool is_encrypted{}; bool is_encrypted{};
std::string object_name; std::string object_name;
head_object_result result{}; head_object_result result{};
@ -573,10 +569,9 @@ auto s3_provider::get_last_modified(bool directory,
: utils::time::get_time_now(); : utils::time::get_time_now();
} }
auto s3_provider::get_object_info(bool directory, const std::string &api_path, auto s3_provider::get_object_info(
bool &is_encrypted, std::string &object_name, bool directory, const std::string &api_path, bool &is_encrypted,
head_object_result &result) const std::string &object_name, head_object_result &result) const -> api_error {
-> api_error {
REPERTORY_USES_FUNCTION_NAME(); REPERTORY_USES_FUNCTION_NAME();
try { try {
@ -625,12 +620,10 @@ auto s3_provider::get_object_info(bool directory, const std::string &api_path,
return api_error::error; return api_error::error;
} }
auto s3_provider::get_object_list(std::string &response_data, auto s3_provider::get_object_list(
long &response_code, std::string &response_data, long &response_code,
std::optional<std::string> delimiter, std::optional<std::string> delimiter, std::optional<std::string> prefix,
std::optional<std::string> prefix, std::optional<std::string> token) const -> bool {
std::optional<std::string> token) const
-> bool {
curl::requests::http_get get{}; curl::requests::http_get get{};
get.allow_timeout = true; get.allow_timeout = true;
get.aws_service = "aws:amz:" + get_config().get_s3_config().region + ":s3"; get.aws_service = "aws:amz:" + get_config().get_s3_config().region + ":s3";
@ -659,8 +652,8 @@ auto s3_provider::get_total_drive_space() const -> std::uint64_t {
return std::numeric_limits<std::int64_t>::max() / std::int64_t(2); return std::numeric_limits<std::int64_t>::max() / std::int64_t(2);
} }
auto s3_provider::is_directory(const std::string &api_path, bool &exists) const auto s3_provider::is_directory(const std::string &api_path,
-> api_error { bool &exists) const -> api_error {
REPERTORY_USES_FUNCTION_NAME(); REPERTORY_USES_FUNCTION_NAME();
exists = false; exists = false;
@ -687,8 +680,8 @@ auto s3_provider::is_directory(const std::string &api_path, bool &exists) const
return api_error::error; return api_error::error;
} }
auto s3_provider::is_file(const std::string &api_path, bool &exists) const auto s3_provider::is_file(const std::string &api_path,
-> api_error { bool &exists) const -> api_error {
REPERTORY_USES_FUNCTION_NAME(); REPERTORY_USES_FUNCTION_NAME();
exists = false; exists = false;