BlockStorage/repertory
1
0
Fork 0
Code Issues 8 Pull Requests Releases 6 Wiki Activity

refactor used drive space and get_file_list

Browse Source
This commit is contained in:
Scott E. Graves
2024-12-02 13:44:04 -06:00
parent 587527e418
commit cfba3d3082
21 changed files with 442 additions and 447 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
repertory/librepertory/include/file_manager/file_manager.hpp
View File

@ -161,8 +161,6 @@ public:
void start();
void stop();
void update_used_space(std::uint64_t &used_space) const override;
};
} // namespace repertory

19
repertory/librepertory/include/file_manager/i_file_manager.hpp
View File

@ -31,21 +31,20 @@ class i_file_manager {
INTERFACE_SETUP(i_file_manager);
public:
[[nodiscard]] virtual auto
evict_file(const std::string &api_path) -> bool = 0;
[[nodiscard]] virtual auto get_directory_items(
const std::string &api_path) const -> directory_item_list = 0;
[[nodiscard]] virtual auto evict_file(const std::string &api_path)
-> bool = 0;
[[nodiscard]] virtual auto
get_open_files() const -> std::unordered_map<std::string, std::size_t> = 0;
get_directory_items(const std::string &api_path) const
-> directory_item_list = 0;
[[nodiscard]] virtual auto get_open_files() const
-> std::unordered_map<std::string, std::size_t> = 0;
[[nodiscard]] virtual auto has_no_open_file_handles() const -> bool = 0;
[[nodiscard]] virtual auto
is_processing(const std::string &api_path) const -> bool = 0;
virtual void update_used_space(std::uint64_t &used_space) const = 0;
[[nodiscard]] virtual auto is_processing(const std::string &api_path) const
-> bool = 0;
};
} // namespace repertory

149
repertory/librepertory/include/providers/base_provider.hpp
View File

@ -44,25 +44,24 @@ private:
api_item_added_callback api_item_added_;
std::unique_ptr<meta_db> db3_;
i_file_manager *fm_{};
mutable std::atomic<std::uint64_t> used_space;
std::atomic<bool> stop_requested_{false};
private:
void remove_deleted_files(bool source_only);
void update_used_drive_space() const;
protected:
[[nodiscard]] static auto
create_api_file(std::string path, std::string key, std::uint64_t size,
std::uint64_t file_time) -> api_file;
[[nodiscard]] static auto create_api_file(std::string path, std::string key,
std::uint64_t size,
std::uint64_t file_time)
-> api_file;
[[nodiscard]] static auto create_api_file(std::string path,
std::uint64_t size,
api_meta_map &meta) -> api_file;
[[nodiscard]] virtual auto
create_directory_impl(const std::string &api_path,
api_meta_map &meta) -> api_error = 0;
[[nodiscard]] virtual auto create_directory_impl(const std::string &api_path,
api_meta_map &meta)
-> api_error = 0;
[[nodiscard]] virtual auto
create_file_extra(const std::string & /* api_path */,
@ -74,8 +73,8 @@ protected:
return api_item_added_;
}
[[nodiscard]] auto
get_api_item_added() const -> const api_item_added_callback & {
[[nodiscard]] auto get_api_item_added() const
-> const api_item_added_callback & {
return api_item_added_;
}
@ -89,6 +88,8 @@ protected:
[[nodiscard]] auto get_db() -> meta_db & { return *db3_; }
[[nodiscard]] auto get_db() const -> const meta_db & { return *db3_; }
[[nodiscard]] virtual auto
get_directory_items_impl(const std::string &api_path,
directory_item_list &list) const -> api_error = 0;
@ -99,23 +100,25 @@ protected:
return fm_;
}
[[nodiscard]] virtual auto
get_used_drive_space_impl() const -> std::uint64_t = 0;
[[nodiscard]] virtual auto get_used_drive_space_impl() const
-> std::uint64_t = 0;
[[nodiscard]] virtual auto
remove_directory_impl(const std::string &api_path) -> api_error = 0;
[[nodiscard]] virtual auto remove_directory_impl(const std::string &api_path)
-> api_error = 0;
[[nodiscard]] virtual auto
remove_file_impl(const std::string &api_path) -> api_error = 0;
[[nodiscard]] virtual auto remove_file_impl(const std::string &api_path)
-> api_error = 0;
[[nodiscard]] virtual auto
upload_file_impl(const std::string &api_path, const std::string &source_path,
stop_type &stop_requested) -> api_error = 0;
[[nodiscard]] virtual auto upload_file_impl(const std::string &api_path,
const std::string &source_path,
stop_type &stop_requested)
-> api_error = 0;
public:
[[nodiscard]] auto create_directory_clone_source_meta(
const std::string &source_api_path,
const std::string &api_path) -> api_error override;
[[nodiscard]] auto
create_directory_clone_source_meta(const std::string &source_api_path,
const std::string &api_path)
-> api_error override;
[[nodiscard]] auto create_directory(const std::string &api_path,
api_meta_map &meta) -> api_error override;
@ -123,76 +126,82 @@ public:
[[nodiscard]] auto create_file(const std::string &api_path,
api_meta_map &meta) -> api_error override;
[[nodiscard]] auto
get_api_path_from_source(const std::string &source_path,
std::string &api_path) const -> api_error override;
[[nodiscard]] auto get_api_path_from_source(const std::string &source_path,
std::string &api_path) const
-> api_error override;
[[nodiscard]] auto get_directory_items(const std::string &api_path,
directory_item_list &list) const
-> api_error override;
[[nodiscard]] auto get_file_size(const std::string &api_path,
std::uint64_t &file_size) const
-> api_error override;
[[nodiscard]] auto get_filesystem_item(const std::string &api_path,
bool directory,
filesystem_item &fsi) const
-> api_error override;
[[nodiscard]] auto get_filesystem_item_and_file(const std::string &api_path,
api_file &f,
filesystem_item &fsi) const
-> api_error override;
[[nodiscard]] auto
get_directory_items(const std::string &api_path,
directory_item_list &list) const -> api_error override;
get_filesystem_item_from_source_path(const std::string &source_path,
filesystem_item &fsi) const
-> api_error override;
[[nodiscard]] auto
get_file_size(const std::string &api_path,
std::uint64_t &file_size) const -> api_error override;
[[nodiscard]] auto get_item_meta(const std::string &api_path,
api_meta_map &meta) const
-> api_error override;
[[nodiscard]] auto
get_filesystem_item(const std::string &api_path, bool directory,
filesystem_item &fsi) const -> api_error override;
[[nodiscard]] auto get_item_meta(const std::string &api_path,
const std::string &key,
std::string &value) const
-> api_error override;
[[nodiscard]] auto get_filesystem_item_and_file(
const std::string &api_path, api_file &f,
filesystem_item &fsi) const -> api_error override;
[[nodiscard]] auto get_filesystem_item_from_source_path(
const std::string &source_path,
filesystem_item &fsi) const -> api_error override;
[[nodiscard]] auto
get_item_meta(const std::string &api_path,
api_meta_map &meta) const -> api_error override;
[[nodiscard]] auto
get_item_meta(const std::string &api_path, const std::string &key,
std::string &value) const -> api_error override;
[[nodiscard]] auto
get_pinned_files() const -> std::vector<std::string> override;
[[nodiscard]] auto get_pinned_files() const
-> std::vector<std::string> override;
[[nodiscard]] auto get_total_item_count() const -> std::uint64_t override;
[[nodiscard]] auto get_used_drive_space() const -> std::uint64_t override;
[[nodiscard]] auto
is_file_writeable(const std::string &api_path) const -> bool override;
[[nodiscard]] auto is_file_writeable(const std::string &api_path) const
-> bool override;
[[nodiscard]] auto is_read_only() const -> bool override { return false; }
[[nodiscard]] auto
remove_directory(const std::string &api_path) -> api_error override;
[[nodiscard]] auto remove_directory(const std::string &api_path)
-> api_error override;
[[nodiscard]] auto
remove_file(const std::string &api_path) -> api_error override;
[[nodiscard]] auto remove_file(const std::string &api_path)
-> api_error override;
[[nodiscard]] auto
remove_item_meta(const std::string &api_path,
const std::string &key) -> api_error override;
[[nodiscard]] auto remove_item_meta(const std::string &api_path,
const std::string &key)
-> api_error override;
[[nodiscard]] auto
set_item_meta(const std::string &api_path, const std::string &key,
const std::string &value) -> api_error override;
[[nodiscard]] auto set_item_meta(const std::string &api_path,
const std::string &key,
const std::string &value)
-> api_error override;
[[nodiscard]] auto
set_item_meta(const std::string &api_path,
const api_meta_map &meta) -> api_error override;
[[nodiscard]] auto set_item_meta(const std::string &api_path,
const api_meta_map &meta)
-> api_error override;
[[nodiscard]] auto start(api_item_added_callback api_item_added,
i_file_manager *mgr) -> bool override;
void stop() override;
[[nodiscard]] auto
upload_file(const std::string &api_path, const std::string &source_path,
stop_type &stop_requested) -> api_error override;
[[nodiscard]] auto upload_file(const std::string &api_path,
const std::string &source_path,
stop_type &stop_requested)
-> api_error override;
};
} // namespace repertory

3
repertory/librepertory/include/providers/encrypt/encrypt_provider.hpp
View File

@ -111,7 +111,8 @@ public:
[[nodiscard]] auto get_file(const std::string &api_path, api_file &file) const
-> api_error override;
[[nodiscard]] auto get_file_list(api_file_list &list) const
[[nodiscard]] auto get_file_list(api_file_list &list,
std::string &marker) const
-> api_error override;
[[nodiscard]] auto get_file_size(const std::string &api_path,

3
repertory/librepertory/include/providers/i_provider.hpp
View File

@ -58,7 +58,8 @@ public:
[[nodiscard]] virtual auto get_file(const std::string &api_path,
api_file &file) const -> api_error = 0;
[[nodiscard]] virtual auto get_file_list(api_file_list &list) const
[[nodiscard]] virtual auto get_file_list(api_file_list &list,
std::string &marker) const
-> api_error = 0;
[[nodiscard]] virtual auto get_file_size(const std::string &api_path,

8
repertory/librepertory/include/providers/meta_db.hpp
View File

@ -63,14 +63,16 @@ public:
[[nodiscard]] auto get_total_item_count() const -> std::uint64_t;
[[nodiscard]] auto get_total_size() const -> std::uint64_t;
void remove_api_path(const std::string &api_path);
[[nodiscard]] auto remove_item_meta(const std::string &api_path,
const std::string &key) -> api_error;
[[nodiscard]] auto
rename_item_meta(const std::string &from_api_path,
const std::string &to_api_path) -> api_error;
[[nodiscard]] auto rename_item_meta(const std::string &from_api_path,
const std::string &to_api_path)
-> api_error;
[[nodiscard]] auto set_item_meta(const std::string &api_path,
const std::string &key,

96
repertory/librepertory/include/providers/s3/s3_provider.hpp
View File

@ -47,70 +47,73 @@ public:
auto operator=(s3_provider &&) -> s3_provider & = delete;
private:
[[nodiscard]] auto
add_if_not_found(api_file &file,
const std::string &object_name) const -> api_error;
[[nodiscard]] auto add_if_not_found(api_file &file,
const std::string &object_name) const
-> api_error;
[[nodiscard]] auto
create_file_extra(const std::string &api_path,
api_meta_map &meta) -> api_error override;
[[nodiscard]] auto create_file_extra(const std::string &api_path,
api_meta_map &meta)
-> api_error override;
[[nodiscard]] auto
create_path_directories(const std::string &api_path,
const std::string &key) const -> api_error;
[[nodiscard]] auto create_path_directories(const std::string &api_path,
const std::string &key) const
-> api_error;
[[nodiscard]] auto
decrypt_object_name(std::string &object_name) const -> api_error;
[[nodiscard]] auto decrypt_object_name(std::string &object_name) const
-> api_error;
[[nodiscard]] auto
get_last_modified(bool directory,
const std::string &api_path) const -> std::uint64_t;
[[nodiscard]] auto get_last_modified(bool directory,
const std::string &api_path) const
-> std::uint64_t;
[[nodiscard]] auto
get_object_info(bool directory, const std::string &api_path,
bool &is_encrypted, std::string &object_name,
head_object_result &result) const -> api_error;
[[nodiscard]] auto get_object_list(
std::string &response_data, long &response_code,
std::optional<std::string> delimiter = std::nullopt,
std::optional<std::string> prefix = std::nullopt,
std::optional<std::string> token = std::nullopt) const -> bool;
[[nodiscard]] auto
get_object_list(std::string &response_data, long &response_code,
std::optional<std::string> delimiter = std::nullopt,
std::optional<std::string> prefix = std::nullopt,
std::optional<std::string> token = std::nullopt) const
-> bool;
protected:
[[nodiscard]] auto
create_directory_impl(const std::string &api_path,
api_meta_map &meta) -> api_error override;
[[nodiscard]] auto create_directory_impl(const std::string &api_path,
api_meta_map &meta)
-> api_error override;
[[nodiscard]] auto get_directory_items_impl(const std::string &api_path,
directory_item_list &list) const
-> api_error override;
[[nodiscard]] auto
get_used_drive_space_impl() const -> std::uint64_t override;
[[nodiscard]] auto get_used_drive_space_impl() const
-> std::uint64_t override;
[[nodiscard]] auto
remove_directory_impl(const std::string &api_path) -> api_error override;
[[nodiscard]] auto remove_directory_impl(const std::string &api_path)
-> api_error override;
[[nodiscard]] auto
remove_file_impl(const std::string &api_path) -> api_error override;
[[nodiscard]] auto remove_file_impl(const std::string &api_path)
-> api_error override;
[[nodiscard]] auto
upload_file_impl(const std::string &api_path, const std::string &source_path,
stop_type &stop_requested) -> api_error override;
[[nodiscard]] auto upload_file_impl(const std::string &api_path,
const std::string &source_path,
stop_type &stop_requested)
-> api_error override;
public:
[[nodiscard]] static auto
convert_api_date(std::string_view date) -> std::uint64_t;
[[nodiscard]] static auto convert_api_date(std::string_view date)
-> std::uint64_t;
[[nodiscard]] auto get_directory_item_count(const std::string &api_path) const
-> std::uint64_t override;
[[nodiscard]] auto get_file(const std::string &api_path,
api_file &file) const -> api_error override;
[[nodiscard]] auto get_file(const std::string &api_path, api_file &file) const
-> api_error override;
[[nodiscard]] auto
get_file_list(api_file_list &list) const -> api_error override;
[[nodiscard]] auto get_file_list(api_file_list &list,
std::string &marker) const
-> api_error override;
[[nodiscard]] auto get_total_drive_space() const -> std::uint64_t override;
@ -121,8 +124,8 @@ public:
[[nodiscard]] auto is_directory(const std::string &api_path,
bool &exists) const -> api_error override;
[[nodiscard]] auto is_file(const std::string &api_path,
bool &exists) const -> api_error override;
[[nodiscard]] auto is_file(const std::string &api_path, bool &exists) const
-> api_error override;
[[nodiscard]] auto is_online() const -> bool override;
@ -130,14 +133,15 @@ public:
return false;
};
[[nodiscard]] auto
read_file_bytes(const std::string &api_path, std::size_t size,
std::uint64_t offset, data_buffer &data,
stop_type &stop_requested) -> api_error override;
[[nodiscard]] auto read_file_bytes(const std::string &api_path,
std::size_t size, std::uint64_t offset,
data_buffer &data,
stop_type &stop_requested)
-> api_error override;
[[nodiscard]] auto
rename_file(const std::string &from_api_path,
const std::string &to_api_path) -> api_error override;
[[nodiscard]] auto rename_file(const std::string &from_api_path,
const std::string &to_api_path)
-> api_error override;
[[nodiscard]] auto start(api_item_added_callback api_item_added,
i_file_manager *mgr) -> bool override;

3
repertory/librepertory/include/providers/sia/sia_provider.hpp
View File

@ -82,7 +82,8 @@ public:
[[nodiscard]] auto get_file(const std::string &api_path, api_file &file) const
-> api_error override;
[[nodiscard]] auto get_file_list(api_file_list &list) const
[[nodiscard]] auto get_file_list(api_file_list &list,
std::string &marker) const
-> api_error override;
[[nodiscard]] auto get_provider_type() const -> provider_type override {

15
repertory/librepertory/include/types/repertory.hpp
View File

@ -79,6 +79,7 @@ enum class api_error {
invalid_version,
item_exists,
item_not_found,
more_data,
no_disk_space,
not_implemented,
not_supported,
@ -95,16 +96,16 @@ enum class api_error {
[[nodiscard]] auto api_error_from_string(std::string_view str) -> api_error;
[[nodiscard]] auto
api_error_to_string(const api_error &error) -> const std::string &;
[[nodiscard]] auto api_error_to_string(const api_error &error)
-> const std::string &;
enum class download_type { direct, fallback, ring_buffer };
[[nodiscard]] auto
download_type_from_string(std::string type,
const download_type &default_type) -> download_type;
[[nodiscard]] auto download_type_from_string(std::string type,
const download_type &default_type)
-> download_type;
[[nodiscard]] auto
download_type_to_string(const download_type &type) -> std::string;
[[nodiscard]] auto download_type_to_string(const download_type &type)
-> std::string;
enum class exit_code : std::int32_t {
success,