Compare commits
28 Commits
75657bec58
...
0bbf6b5942
Author | SHA1 | Date | |
---|---|---|---|
0bbf6b5942 | |||
74ef36956d | |||
bbe065819c | |||
3493054f1a | |||
743281497c | |||
eaa6a6f92e | |||
a25dd8178e | |||
71463bfee4 | |||
270df62dbe | |||
5bd7ded7d9 | |||
b5048a422f | |||
c27528fe73 | |||
c3978562d5 | |||
637892cff1 | |||
998a3d9d83 | |||
8ed8822625 | |||
196abaebcc | |||
653e8000ff | |||
ea362211aa | |||
a4a9d069d3 | |||
9cb7f9accf | |||
4e5454efb5 | |||
a8c3f31298 | |||
f3093c27cd | |||
4526a9d860 | |||
a1ada5aa8c | |||
1ae8b8b793 | |||
f44cc99a82 |
@@ -15,6 +15,7 @@
|
||||
|
||||
### Changes from v2.0.2-rc
|
||||
|
||||
* Always use direct for read-only providers
|
||||
* Updated build system to Alpine 3.21.0
|
||||
* Updated build system to MinGW-w64 12.0.0
|
||||
* Updated copyright to 2018-2025
|
||||
|
@@ -31,7 +31,7 @@ class i_file_db {
|
||||
public:
|
||||
struct file_info final {
|
||||
std::string api_path;
|
||||
bool directory;
|
||||
bool directory{};
|
||||
std::string source_path;
|
||||
};
|
||||
|
||||
@@ -56,6 +56,10 @@ public:
|
||||
|
||||
[[nodiscard]] virtual auto count() const -> std::uint64_t = 0;
|
||||
|
||||
virtual void enumerate_item_list(
|
||||
std::function<void(const std::vector<i_file_db::file_info> &)> callback,
|
||||
stop_type &stop_requested) const = 0;
|
||||
|
||||
[[nodiscard]] virtual auto get_api_path(const std::string &source_path,
|
||||
std::string &api_path) const
|
||||
-> api_error = 0;
|
||||
@@ -80,7 +84,7 @@ public:
|
||||
get_file_source_path(const std::string &api_path,
|
||||
std::string &source_path) const -> api_error = 0;
|
||||
|
||||
[[nodiscard]] virtual auto get_item_list() const
|
||||
[[nodiscard]] virtual auto get_item_list(stop_type &stop_requested) const
|
||||
-> std::vector<file_info> = 0;
|
||||
|
||||
[[nodiscard]] virtual auto get_source_path(const std::string &api_path,
|
||||
|
@@ -41,45 +41,42 @@ public:
|
||||
std::string source_path;
|
||||
};
|
||||
|
||||
struct upload_entry final {
|
||||
std::string api_path;
|
||||
std::string source_path;
|
||||
};
|
||||
using upload_entry = upload_active_entry;
|
||||
|
||||
public:
|
||||
[[nodiscard]] virtual auto add_resume(const resume_entry &entry) -> bool = 0;
|
||||
|
||||
[[nodiscard]] virtual auto add_upload(const upload_entry &entry) -> bool = 0;
|
||||
|
||||
[[nodiscard]] virtual auto
|
||||
add_upload_active(const upload_active_entry &entry) -> bool = 0;
|
||||
[[nodiscard]] virtual auto add_upload_active(const upload_active_entry &entry)
|
||||
-> bool = 0;
|
||||
|
||||
virtual void clear() = 0;
|
||||
|
||||
[[nodiscard]] virtual auto
|
||||
get_next_upload() const -> std::optional<upload_entry> = 0;
|
||||
[[nodiscard]] virtual auto get_next_upload() const
|
||||
-> std::optional<upload_entry> = 0;
|
||||
|
||||
[[nodiscard]] virtual auto
|
||||
get_resume_list() const -> std::vector<resume_entry> = 0;
|
||||
[[nodiscard]] virtual auto get_resume_list() const
|
||||
-> std::vector<resume_entry> = 0;
|
||||
|
||||
[[nodiscard]] virtual auto get_upload(const std::string &api_path) const
|
||||
-> std::optional<upload_entry> = 0;
|
||||
|
||||
[[nodiscard]] virtual auto
|
||||
get_upload_active_list() const -> std::vector<upload_active_entry> = 0;
|
||||
[[nodiscard]] virtual auto get_upload_active_list() const
|
||||
-> std::vector<upload_active_entry> = 0;
|
||||
|
||||
[[nodiscard]] virtual auto
|
||||
remove_resume(const std::string &api_path) -> bool = 0;
|
||||
[[nodiscard]] virtual auto remove_resume(const std::string &api_path)
|
||||
-> bool = 0;
|
||||
|
||||
[[nodiscard]] virtual auto
|
||||
remove_upload(const std::string &api_path) -> bool = 0;
|
||||
[[nodiscard]] virtual auto remove_upload(const std::string &api_path)
|
||||
-> bool = 0;
|
||||
|
||||
[[nodiscard]] virtual auto
|
||||
remove_upload_active(const std::string &api_path) -> bool = 0;
|
||||
[[nodiscard]] virtual auto remove_upload_active(const std::string &api_path)
|
||||
-> bool = 0;
|
||||
|
||||
[[nodiscard]] virtual auto
|
||||
rename_resume(const std::string &from_api_path,
|
||||
const std::string &to_api_path) -> bool = 0;
|
||||
[[nodiscard]] virtual auto rename_resume(const std::string &from_api_path,
|
||||
const std::string &to_api_path)
|
||||
-> bool = 0;
|
||||
};
|
||||
} // namespace repertory
|
||||
|
||||
|
@@ -31,6 +31,10 @@ class i_meta_db {
|
||||
public:
|
||||
virtual void clear() = 0;
|
||||
|
||||
virtual void enumerate_api_path_list(
|
||||
std::function<void(const std::vector<std::string> &)> callback,
|
||||
stop_type &stop_requested) const = 0;
|
||||
|
||||
[[nodiscard]] virtual auto get_api_path(const std::string &source_path,
|
||||
std::string &api_path) const
|
||||
-> api_error = 0;
|
||||
|
@@ -67,50 +67,54 @@ private:
|
||||
rocksdb::Transaction *txn) -> rocksdb::Status;
|
||||
|
||||
public:
|
||||
[[nodiscard]] auto
|
||||
add_directory(const std::string &api_path,
|
||||
const std::string &source_path) -> api_error override;
|
||||
[[nodiscard]] auto add_directory(const std::string &api_path,
|
||||
const std::string &source_path)
|
||||
-> api_error override;
|
||||
|
||||
[[nodiscard]] auto
|
||||
add_or_update_file(const i_file_db::file_data &data) -> api_error override;
|
||||
[[nodiscard]] auto add_or_update_file(const i_file_db::file_data &data)
|
||||
-> api_error override;
|
||||
|
||||
void clear() override;
|
||||
|
||||
[[nodiscard]] auto count() const -> std::uint64_t override;
|
||||
|
||||
[[nodiscard]] auto
|
||||
get_api_path(const std::string &source_path,
|
||||
std::string &api_path) const -> api_error override;
|
||||
void enumerate_item_list(
|
||||
std::function<void(const std::vector<i_file_db::file_info> &)> callback,
|
||||
stop_type &stop_requested) const override;
|
||||
|
||||
[[nodiscard]] auto
|
||||
get_directory_api_path(const std::string &source_path,
|
||||
std::string &api_path) const -> api_error override;
|
||||
[[nodiscard]] auto get_api_path(const std::string &source_path,
|
||||
std::string &api_path) const
|
||||
-> api_error override;
|
||||
|
||||
[[nodiscard]] auto get_directory_api_path(const std::string &source_path,
|
||||
std::string &api_path) const
|
||||
-> api_error override;
|
||||
|
||||
[[nodiscard]] auto get_directory_source_path(const std::string &api_path,
|
||||
std::string &source_path) const
|
||||
-> api_error override;
|
||||
|
||||
[[nodiscard]] auto
|
||||
get_file_api_path(const std::string &source_path,
|
||||
std::string &api_path) const -> api_error override;
|
||||
[[nodiscard]] auto get_file_api_path(const std::string &source_path,
|
||||
std::string &api_path) const
|
||||
-> api_error override;
|
||||
|
||||
[[nodiscard]] auto
|
||||
get_file_data(const std::string &api_path,
|
||||
i_file_db::file_data &data) const -> api_error override;
|
||||
[[nodiscard]] auto get_file_data(const std::string &api_path,
|
||||
i_file_db::file_data &data) const
|
||||
-> api_error override;
|
||||
|
||||
[[nodiscard]] auto
|
||||
get_file_source_path(const std::string &api_path,
|
||||
std::string &source_path) const -> api_error override;
|
||||
[[nodiscard]] auto get_file_source_path(const std::string &api_path,
|
||||
std::string &source_path) const
|
||||
-> api_error override;
|
||||
|
||||
[[nodiscard]] auto
|
||||
get_item_list() const -> std::vector<i_file_db::file_info> override;
|
||||
[[nodiscard]] auto get_item_list(stop_type &stop_requested) const
|
||||
-> std::vector<i_file_db::file_info> override;
|
||||
|
||||
[[nodiscard]] auto
|
||||
get_source_path(const std::string &api_path,
|
||||
std::string &source_path) const -> api_error override;
|
||||
[[nodiscard]] auto get_source_path(const std::string &api_path,
|
||||
std::string &source_path) const
|
||||
-> api_error override;
|
||||
|
||||
[[nodiscard]] auto
|
||||
remove_item(const std::string &api_path) -> api_error override;
|
||||
[[nodiscard]] auto remove_item(const std::string &api_path)
|
||||
-> api_error override;
|
||||
};
|
||||
} // namespace repertory
|
||||
|
||||
|
@@ -80,6 +80,10 @@ private:
|
||||
public:
|
||||
void clear() override;
|
||||
|
||||
void enumerate_api_path_list(
|
||||
std::function<void(const std::vector<std::string> &)> callback,
|
||||
stop_type &stop_requested) const override;
|
||||
|
||||
[[nodiscard]] auto get_api_path(const std::string &source_path,
|
||||
std::string &api_path) const
|
||||
-> api_error override;
|
||||
|
@@ -53,6 +53,10 @@ public:
|
||||
|
||||
[[nodiscard]] auto count() const -> std::uint64_t override;
|
||||
|
||||
void enumerate_item_list(
|
||||
std::function<void(const std::vector<i_file_db::file_info> &)> callback,
|
||||
stop_type &stop_requested) const override;
|
||||
|
||||
[[nodiscard]] auto get_api_path(const std::string &source_path,
|
||||
std::string &api_path) const
|
||||
-> api_error override;
|
||||
@@ -77,7 +81,7 @@ public:
|
||||
std::string &source_path) const
|
||||
-> api_error override;
|
||||
|
||||
[[nodiscard]] auto get_item_list() const
|
||||
[[nodiscard]] auto get_item_list(stop_type &stop_requested) const
|
||||
-> std::vector<i_file_db::file_info> override;
|
||||
|
||||
[[nodiscard]] auto get_source_path(const std::string &api_path,
|
||||
|
@@ -50,6 +50,10 @@ private:
|
||||
public:
|
||||
void clear() override;
|
||||
|
||||
void enumerate_api_path_list(
|
||||
std::function<void(const std::vector<std::string> &)> callback,
|
||||
stop_type &stop_requested) const override;
|
||||
|
||||
[[nodiscard]] auto get_api_path(const std::string &source_path,
|
||||
std::string &api_path) const
|
||||
-> api_error override;
|
||||
|
@@ -42,8 +42,8 @@ public:
|
||||
direct_open_file(const direct_open_file &) noexcept = delete;
|
||||
direct_open_file(direct_open_file &&) noexcept = delete;
|
||||
auto operator=(direct_open_file &&) noexcept -> direct_open_file & = delete;
|
||||
auto
|
||||
operator=(const direct_open_file &) noexcept -> direct_open_file & = delete;
|
||||
auto operator=(const direct_open_file &) noexcept
|
||||
-> direct_open_file & = delete;
|
||||
|
||||
private:
|
||||
std::array<data_buffer, min_ring_size> ring_data_;
|
||||
@@ -51,22 +51,26 @@ private:
|
||||
protected:
|
||||
[[nodiscard]] auto on_check_start() -> bool override;
|
||||
|
||||
[[nodiscard]] auto
|
||||
on_chunk_downloaded(std::size_t /* chunk */,
|
||||
const data_buffer & /* buffer */) -> api_error override {
|
||||
[[nodiscard]] auto on_chunk_downloaded(std::size_t /* chunk */,
|
||||
const data_buffer & /* buffer */)
|
||||
-> api_error override {
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
[[nodiscard]] auto
|
||||
on_read_chunk(std::size_t chunk, std::size_t read_size,
|
||||
std::uint64_t read_offset, data_buffer &data,
|
||||
std::size_t &bytes_read) -> api_error override;
|
||||
[[nodiscard]] auto on_read_chunk(std::size_t chunk, std::size_t read_size,
|
||||
std::uint64_t read_offset, data_buffer &data,
|
||||
std::size_t &bytes_read)
|
||||
-> api_error override;
|
||||
|
||||
[[nodiscard]] auto use_buffer(std::size_t chunk,
|
||||
std::function<api_error(data_buffer &)> func)
|
||||
-> api_error override;
|
||||
|
||||
public:
|
||||
[[nodiscard]] auto get_source_path() const -> std::string override {
|
||||
return "direct";
|
||||
}
|
||||
|
||||
[[nodiscard]] auto native_operation(native_operation_callback /* callback */)
|
||||
-> api_error override {
|
||||
return api_error::not_supported;
|
||||
|
@@ -53,19 +53,19 @@ private:
|
||||
i_file_manager *fm_{};
|
||||
|
||||
private:
|
||||
void add_all_items(const stop_type &stop_requested);
|
||||
void add_all_items(stop_type &stop_requested);
|
||||
|
||||
void process_removed_directories(std::deque<removed_item> removed_list,
|
||||
const stop_type &stop_requested);
|
||||
stop_type &stop_requested);
|
||||
|
||||
void process_removed_files(std::deque<removed_item> removed_list,
|
||||
const stop_type &stop_requested);
|
||||
stop_type &stop_requested);
|
||||
|
||||
void process_removed_items(const stop_type &stop_requested);
|
||||
void process_removed_items(stop_type &stop_requested);
|
||||
|
||||
void remove_deleted_items(const stop_type &stop_requested);
|
||||
void remove_deleted_items(stop_type &stop_requested);
|
||||
|
||||
void remove_unmatched_source_files(const stop_type &stop_requested);
|
||||
void remove_unmatched_source_files(stop_type &stop_requested);
|
||||
|
||||
protected:
|
||||
[[nodiscard]] static auto create_api_file(std::string path, std::string key,
|
||||
|
@@ -82,7 +82,7 @@ private:
|
||||
const encrypt_config &cfg,
|
||||
std::string &api_path) const -> bool;
|
||||
|
||||
void remove_deleted_files(const stop_type &stop_requested);
|
||||
void remove_deleted_files(stop_type &stop_requested);
|
||||
|
||||
public:
|
||||
[[nodiscard]] auto create_directory(const std::string &api_path,
|
||||
|
@@ -54,6 +54,10 @@ private:
|
||||
const std::string &object_name) const
|
||||
-> api_error;
|
||||
|
||||
[[nodiscard]] auto
|
||||
create_directory_object(const std::string &api_path,
|
||||
const std::string &object_name) const -> api_error;
|
||||
|
||||
[[nodiscard]] auto create_directory_paths(const std::string &api_path,
|
||||
const std::string &key) const
|
||||
-> api_error;
|
||||
@@ -85,6 +89,9 @@ private:
|
||||
return s3_config_;
|
||||
}
|
||||
|
||||
[[nodiscard]] auto set_meta_key(const std::string &api_path,
|
||||
api_meta_map &meta) -> api_error;
|
||||
|
||||
protected:
|
||||
[[nodiscard]] auto create_directory_impl(const std::string &api_path,
|
||||
api_meta_map &meta)
|
||||
|
@@ -40,7 +40,7 @@ public:
|
||||
struct polling_item final {
|
||||
std::string name;
|
||||
frequency freq;
|
||||
std::function<void(const stop_type &stop_requested)> action;
|
||||
std::function<void(stop_type &stop_requested)> action;
|
||||
};
|
||||
|
||||
public:
|
||||
|
@@ -30,7 +30,7 @@ class app_config;
|
||||
class tasks final {
|
||||
public:
|
||||
struct task final {
|
||||
std::function<void(const stop_type &task_stopped)> action;
|
||||
std::function<void(stop_type &task_stopped)> action;
|
||||
};
|
||||
|
||||
class i_task {
|
||||
|
@@ -149,6 +149,50 @@ auto rdb_file_db::count() const -> std::uint64_t {
|
||||
return ret;
|
||||
}
|
||||
|
||||
void rdb_file_db::enumerate_item_list(
|
||||
std::function<void(const std::vector<i_file_db::file_info> &)> callback,
|
||||
stop_type &stop_requested) const {
|
||||
std::vector<i_file_db::file_info> list;
|
||||
{
|
||||
auto iter = create_iterator(file_family_);
|
||||
for (iter->SeekToFirst(); not stop_requested && iter->Valid();
|
||||
iter->Next()) {
|
||||
auto json_data = json::parse(iter->value().ToString());
|
||||
list.emplace_back(i_file_db::file_info{
|
||||
iter->key().ToString(),
|
||||
false,
|
||||
json_data.at("source_path").get<std::string>(),
|
||||
});
|
||||
|
||||
if (list.size() < 100U) {
|
||||
continue;
|
||||
}
|
||||
|
||||
callback(list);
|
||||
list.clear();
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
auto iter = create_iterator(directory_family_);
|
||||
for (iter->SeekToFirst(); not stop_requested && iter->Valid();
|
||||
iter->Next()) {
|
||||
list.emplace_back(i_file_db::file_info{
|
||||
iter->key().ToString(),
|
||||
true,
|
||||
iter->value().ToString(),
|
||||
});
|
||||
|
||||
if (list.size() < 100U) {
|
||||
continue;
|
||||
}
|
||||
|
||||
callback(list);
|
||||
list.clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
auto rdb_file_db::get_api_path(const std::string &source_path,
|
||||
std::string &api_path) const -> api_error {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
@@ -159,8 +203,9 @@ auto rdb_file_db::get_api_path(const std::string &source_path,
|
||||
});
|
||||
}
|
||||
|
||||
auto rdb_file_db::get_directory_api_path(
|
||||
const std::string &source_path, std::string &api_path) const -> api_error {
|
||||
auto rdb_file_db::get_directory_api_path(const std::string &source_path,
|
||||
std::string &api_path) const
|
||||
-> api_error {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
auto result = perform_action(function_name, [&]() -> rocksdb::Status {
|
||||
@@ -183,8 +228,9 @@ auto rdb_file_db::get_directory_api_path(
|
||||
: result;
|
||||
}
|
||||
|
||||
auto rdb_file_db::get_directory_source_path(
|
||||
const std::string &api_path, std::string &source_path) const -> api_error {
|
||||
auto rdb_file_db::get_directory_source_path(const std::string &api_path,
|
||||
std::string &source_path) const
|
||||
-> api_error {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
auto result = perform_action(function_name, [&]() -> rocksdb::Status {
|
||||
@@ -245,8 +291,9 @@ auto rdb_file_db::get_file_data(const std::string &api_path,
|
||||
return result;
|
||||
}
|
||||
|
||||
auto rdb_file_db::get_file_source_path(
|
||||
const std::string &api_path, std::string &source_path) const -> api_error {
|
||||
auto rdb_file_db::get_file_source_path(const std::string &api_path,
|
||||
std::string &source_path) const
|
||||
-> api_error {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
auto result = perform_action(function_name, [&]() -> rocksdb::Status {
|
||||
@@ -264,11 +311,13 @@ auto rdb_file_db::get_file_source_path(
|
||||
return result;
|
||||
}
|
||||
|
||||
auto rdb_file_db::get_item_list() const -> std::vector<i_file_db::file_info> {
|
||||
auto rdb_file_db::get_item_list(stop_type &stop_requested) const
|
||||
-> std::vector<i_file_db::file_info> {
|
||||
std::vector<i_file_db::file_info> ret{};
|
||||
{
|
||||
auto iter = create_iterator(directory_family_);
|
||||
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
|
||||
for (iter->SeekToFirst(); not stop_requested && iter->Valid();
|
||||
iter->Next()) {
|
||||
ret.emplace_back(i_file_db::file_info{
|
||||
iter->key().ToString(),
|
||||
true,
|
||||
@@ -279,11 +328,12 @@ auto rdb_file_db::get_item_list() const -> std::vector<i_file_db::file_info> {
|
||||
|
||||
{
|
||||
auto iter = create_iterator(file_family_);
|
||||
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
|
||||
for (iter->SeekToFirst(); not stop_requested && iter->Valid();
|
||||
iter->Next()) {
|
||||
auto json_data = json::parse(iter->value().ToString());
|
||||
ret.emplace_back(i_file_db::file_info{
|
||||
iter->key().ToString(),
|
||||
true,
|
||||
false,
|
||||
json_data.at("source_path").get<std::string>(),
|
||||
});
|
||||
}
|
||||
|
@@ -64,6 +64,24 @@ auto rdb_meta_db::create_iterator(rocksdb::ColumnFamilyHandle *family) const
|
||||
db_->NewIterator(rocksdb::ReadOptions{}, family));
|
||||
}
|
||||
|
||||
void rdb_meta_db::enumerate_api_path_list(
|
||||
std::function<void(const std::vector<std::string> &)> callback,
|
||||
stop_type &stop_requested) const {
|
||||
std::vector<std::string> list{};
|
||||
|
||||
auto iter = create_iterator(meta_family_);
|
||||
for (iter->SeekToFirst(); not stop_requested && iter->Valid(); iter->Next()) {
|
||||
list.push_back(iter->key().ToString());
|
||||
|
||||
if (list.size() < 100U) {
|
||||
continue;
|
||||
}
|
||||
|
||||
callback(list);
|
||||
list.clear();
|
||||
}
|
||||
}
|
||||
|
||||
auto rdb_meta_db::get_api_path(const std::string &source_path,
|
||||
std::string &api_path) const -> api_error {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
@@ -133,6 +133,33 @@ auto sqlite_file_db::count() const -> std::uint64_t {
|
||||
return 0U;
|
||||
}
|
||||
|
||||
void sqlite_file_db::enumerate_item_list(
|
||||
std::function<void(const std::vector<i_file_db::file_info> &)> callback,
|
||||
stop_type &stop_requested) const {
|
||||
std::vector<i_file_db::file_info> list;
|
||||
|
||||
auto result = utils::db::sqlite::db_select{*db_, file_table}.go();
|
||||
while (not stop_requested && result.has_row()) {
|
||||
std::optional<utils::db::sqlite::db_result::row> row;
|
||||
if (result.get_row(row) && row.has_value()) {
|
||||
list.emplace_back(i_file_db::file_info{
|
||||
.api_path = row->get_column("api_path").get_value<std::string>(),
|
||||
.directory =
|
||||
row->get_column("directory").get_value<std::int64_t>() == 1,
|
||||
.source_path =
|
||||
row->get_column("source_path").get_value<std::string>(),
|
||||
});
|
||||
|
||||
if (list.size() < 100U) {
|
||||
continue;
|
||||
}
|
||||
|
||||
callback(list);
|
||||
list.clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
auto sqlite_file_db::get_api_path(const std::string &source_path,
|
||||
std::string &api_path) const -> api_error {
|
||||
auto result = utils::db::sqlite::db_select{*db_, file_table}
|
||||
@@ -282,22 +309,22 @@ auto sqlite_file_db::get_file_source_path(const std::string &api_path,
|
||||
return api_error::item_not_found;
|
||||
}
|
||||
|
||||
auto sqlite_file_db::get_item_list() const
|
||||
auto sqlite_file_db::get_item_list(stop_type &stop_requested) const
|
||||
-> std::vector<i_file_db::file_info> {
|
||||
std::vector<i_file_db::file_info> ret;
|
||||
|
||||
auto result = utils::db::sqlite::db_select{*db_, file_table}.go();
|
||||
while (result.has_row()) {
|
||||
while (not stop_requested && result.has_row()) {
|
||||
std::optional<utils::db::sqlite::db_result::row> row;
|
||||
if (result.get_row(row) && row.has_value()) {
|
||||
ret.emplace_back(i_file_db::file_info{
|
||||
row->get_column("api_path").get_value<std::string>(),
|
||||
row->get_column("directory").get_value<std::int64_t>() == 1,
|
||||
row->get_column("source_path").get_value<std::string>(),
|
||||
.api_path = row->get_column("api_path").get_value<std::string>(),
|
||||
.directory =
|
||||
row->get_column("directory").get_value<std::int64_t>() == 1,
|
||||
.source_path =
|
||||
row->get_column("source_path").get_value<std::string>(),
|
||||
});
|
||||
}
|
||||
|
||||
result.next_row();
|
||||
}
|
||||
|
||||
return ret;
|
||||
|
@@ -75,6 +75,28 @@ void sqlite_meta_db::clear() {
|
||||
std::to_string(result.get_error()));
|
||||
}
|
||||
|
||||
void sqlite_meta_db::enumerate_api_path_list(
|
||||
std::function<void(const std::vector<std::string> &)> callback,
|
||||
stop_type &stop_requested) const {
|
||||
auto result =
|
||||
utils::db::sqlite::db_select{*db_, table_name}.column("api_path").go();
|
||||
|
||||
std::vector<std::string> list{};
|
||||
while (not stop_requested && result.has_row()) {
|
||||
std::optional<utils::db::sqlite::db_result::row> row;
|
||||
if (result.get_row(row) && row.has_value()) {
|
||||
list.push_back(row->get_column("api_path").get_value<std::string>());
|
||||
|
||||
if (list.size() < 100U) {
|
||||
continue;
|
||||
}
|
||||
|
||||
callback(list);
|
||||
list.clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
auto sqlite_meta_db::get_api_path(const std::string &source_path,
|
||||
std::string &api_path) const -> api_error {
|
||||
auto result = utils::db::sqlite::db_select{*db_, table_name}
|
||||
|
@@ -30,7 +30,6 @@
|
||||
#include "utils/file_utils.hpp"
|
||||
#include "utils/time.hpp"
|
||||
#include "utils/utils.hpp"
|
||||
#include <spdlog/fmt/bundled/base.h>
|
||||
|
||||
namespace repertory {
|
||||
auto eviction::check_minimum_requirements(const std::string &file_path)
|
||||
|
@@ -118,8 +118,8 @@ auto fuse_base::chflags_(const char *path, uint32_t flags) -> int {
|
||||
#endif // __APPLE__
|
||||
|
||||
#if FUSE_USE_VERSION >= 30
|
||||
auto fuse_base::chmod_(const char *path, mode_t mode,
|
||||
struct fuse_file_info *fi) -> int {
|
||||
auto fuse_base::chmod_(const char *path, mode_t mode, struct fuse_file_info *fi)
|
||||
-> int {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
return instance().execute_callback(
|
||||
@@ -186,7 +186,7 @@ void fuse_base::display_options(
|
||||
#if FUSE_USE_VERSION >= 30
|
||||
fuse_cmdline_help();
|
||||
#else
|
||||
struct fuse_operations fuse_ops {};
|
||||
struct fuse_operations fuse_ops{};
|
||||
fuse_main(args.size(),
|
||||
reinterpret_cast<char **>(const_cast<char **>(args.data())),
|
||||
&fuse_ops, nullptr);
|
||||
@@ -196,7 +196,7 @@ void fuse_base::display_options(
|
||||
}
|
||||
|
||||
void fuse_base::display_version_information(std::vector<const char *> args) {
|
||||
struct fuse_operations fuse_ops {};
|
||||
struct fuse_operations fuse_ops{};
|
||||
fuse_main(static_cast<int>(args.size()),
|
||||
reinterpret_cast<char **>(const_cast<char **>(args.data())),
|
||||
&fuse_ops, nullptr);
|
||||
@@ -334,8 +334,8 @@ auto fuse_base::getxtimes_(const char *path, struct timespec *bkuptime,
|
||||
#endif // __APPLE__
|
||||
|
||||
#if FUSE_USE_VERSION >= 30
|
||||
auto fuse_base::init_(struct fuse_conn_info *conn,
|
||||
struct fuse_config *cfg) -> void * {
|
||||
auto fuse_base::init_(struct fuse_conn_info *conn, struct fuse_config *cfg)
|
||||
-> void * {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
return execute_void_pointer_callback(function_name, [&]() -> void * {
|
||||
@@ -408,7 +408,7 @@ auto fuse_base::mount(std::vector<std::string> args) -> int {
|
||||
|
||||
char *mount_location{nullptr};
|
||||
#if FUSE_USE_VERSION >= 30
|
||||
struct fuse_cmdline_opts opts {};
|
||||
struct fuse_cmdline_opts opts{};
|
||||
fuse_parse_cmdline(&fa, &opts);
|
||||
mount_location = opts.mountpoint;
|
||||
#else
|
||||
@@ -477,8 +477,8 @@ auto fuse_base::read_(const char *path, char *buffer, size_t read_size,
|
||||
#if FUSE_USE_VERSION >= 30
|
||||
auto fuse_base::readdir_(const char *path, void *buf,
|
||||
fuse_fill_dir_t fuse_fill_dir, off_t offset,
|
||||
struct fuse_file_info *fi,
|
||||
fuse_readdir_flags flags) -> int {
|
||||
struct fuse_file_info *fi, fuse_readdir_flags flags)
|
||||
-> int {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
return instance().execute_callback(
|
||||
@@ -510,8 +510,8 @@ auto fuse_base::release_(const char *path, struct fuse_file_info *fi) -> int {
|
||||
});
|
||||
}
|
||||
|
||||
auto fuse_base::releasedir_(const char *path,
|
||||
struct fuse_file_info *fi) -> int {
|
||||
auto fuse_base::releasedir_(const char *path, struct fuse_file_info *fi)
|
||||
-> int {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
return instance().execute_callback(
|
||||
@@ -521,8 +521,8 @@ auto fuse_base::releasedir_(const char *path,
|
||||
}
|
||||
|
||||
#if FUSE_USE_VERSION >= 30
|
||||
auto fuse_base::rename_(const char *from, const char *to,
|
||||
unsigned int flags) -> int {
|
||||
auto fuse_base::rename_(const char *from, const char *to, unsigned int flags)
|
||||
-> int {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
return instance().execute_callback(
|
||||
@@ -603,7 +603,7 @@ auto fuse_base::listxattr_(const char *path, char *buffer, size_t size) -> int {
|
||||
void fuse_base::notify_fuse_args_parsed(const std::vector<std::string> &args) {
|
||||
event_system::instance().raise<fuse_args_parsed>(std::accumulate(
|
||||
args.begin(), args.end(), std::string(),
|
||||
[](std::string command_line, const auto &arg) {
|
||||
[](auto &&command_line, auto &&arg) -> auto {
|
||||
command_line += (command_line.empty() ? arg : (" " + std::string(arg)));
|
||||
return command_line;
|
||||
}));
|
||||
@@ -765,8 +765,8 @@ auto fuse_base::setattr_x_(const char *path, struct setattr_x *attr) -> int {
|
||||
});
|
||||
}
|
||||
|
||||
auto fuse_base::setbkuptime_(const char *path,
|
||||
const struct timespec *bkuptime) -> int {
|
||||
auto fuse_base::setbkuptime_(const char *path, const struct timespec *bkuptime)
|
||||
-> int {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
return instance().execute_callback(
|
||||
@@ -775,8 +775,8 @@ auto fuse_base::setbkuptime_(const char *path,
|
||||
});
|
||||
}
|
||||
|
||||
auto fuse_base::setchgtime_(const char *path,
|
||||
const struct timespec *chgtime) -> int {
|
||||
auto fuse_base::setchgtime_(const char *path, const struct timespec *chgtime)
|
||||
-> int {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
return instance().execute_callback(
|
||||
@@ -785,8 +785,8 @@ auto fuse_base::setchgtime_(const char *path,
|
||||
});
|
||||
}
|
||||
|
||||
auto fuse_base::setcrtime_(const char *path,
|
||||
const struct timespec *crtime) -> int {
|
||||
auto fuse_base::setcrtime_(const char *path, const struct timespec *crtime)
|
||||
-> int {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
return instance().execute_callback(
|
||||
|
@@ -42,7 +42,7 @@ void remote_open_file_table::close_all(const std::string &client_id) {
|
||||
auto compat_handles =
|
||||
std::accumulate(compat_file_lookup_.begin(), compat_file_lookup_.end(),
|
||||
std::vector<remote::file_handle>(),
|
||||
[&client_id](auto &&list, auto &&value) {
|
||||
[&client_id](auto &&list, auto &&value) -> auto {
|
||||
auto &&op_info = value.second;
|
||||
if (op_info->client_id == client_id) {
|
||||
list.insert(list.end(), op_info->handles.begin(),
|
||||
@@ -54,7 +54,7 @@ void remote_open_file_table::close_all(const std::string &client_id) {
|
||||
|
||||
auto handles = std::accumulate(
|
||||
file_lookup_.begin(), file_lookup_.end(), std::vector<native_handle>(),
|
||||
[&client_id](auto &&list, auto &&value) {
|
||||
[&client_id](auto &&list, auto &&value) -> auto {
|
||||
auto &&op_info = value.second;
|
||||
if (op_info->client_id == client_id) {
|
||||
list.insert(list.end(), op_info->handles.begin(),
|
||||
@@ -161,21 +161,21 @@ auto remote_open_file_table::has_compat_open_info(
|
||||
|
||||
void remote_open_file_table::remove_all(const std::string &file_path) {
|
||||
unique_recur_mutex_lock lock(file_mutex_);
|
||||
auto compat_open_list = std::accumulate(
|
||||
compat_file_lookup_.begin(), compat_file_lookup_.end(),
|
||||
std::vector<remote::file_handle>(),
|
||||
[&file_path](auto &&list, auto &&kv) -> std::vector<remote::file_handle> {
|
||||
if (kv.first == file_path) {
|
||||
auto *op_info = kv.second.get();
|
||||
list.insert(list.end(), op_info->handles.begin(),
|
||||
op_info->handles.end());
|
||||
}
|
||||
return list;
|
||||
});
|
||||
auto compat_open_list =
|
||||
std::accumulate(compat_file_lookup_.begin(), compat_file_lookup_.end(),
|
||||
std::vector<remote::file_handle>(),
|
||||
[&file_path](auto &&list, auto &&kv) -> auto {
|
||||
if (kv.first == file_path) {
|
||||
auto *op_info = kv.second.get();
|
||||
list.insert(list.end(), op_info->handles.begin(),
|
||||
op_info->handles.end());
|
||||
}
|
||||
return list;
|
||||
});
|
||||
|
||||
auto open_list = std::accumulate(
|
||||
file_lookup_.begin(), file_lookup_.end(), std::vector<native_handle>(),
|
||||
[&file_path](auto &&list, auto &&kv) -> std::vector<native_handle> {
|
||||
[&file_path](auto &&list, auto &&kv) -> auto {
|
||||
if (kv.first == file_path) {
|
||||
auto *op_info = kv.second.get();
|
||||
list.insert(list.end(), op_info->handles.begin(),
|
||||
|
@@ -97,7 +97,7 @@ void file_manager::close_timed_out_files() {
|
||||
auto closeable_list =
|
||||
std::accumulate(open_file_lookup_.begin(), open_file_lookup_.end(),
|
||||
std::vector<std::shared_ptr<i_closeable_open_file>>{},
|
||||
[](auto items, const auto &item) -> auto {
|
||||
[](auto &&items, auto &&item) -> auto {
|
||||
if (item.second->get_open_file_count() == 0U &&
|
||||
item.second->can_close()) {
|
||||
items.push_back(item.second);
|
||||
@@ -235,6 +235,10 @@ auto file_manager::get_open_file_count(const std::string &api_path) const
|
||||
|
||||
auto file_manager::get_open_file(std::uint64_t handle, bool write_supported,
|
||||
std::shared_ptr<i_open_file> &file) -> bool {
|
||||
if (write_supported && provider_.is_read_only()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
unique_recur_mutex_lock open_lock(open_file_mtx_);
|
||||
auto file_ptr = get_open_file_by_handle(handle);
|
||||
if (not file_ptr) {
|
||||
@@ -265,23 +269,22 @@ auto file_manager::get_open_file_count() const -> std::size_t {
|
||||
|
||||
auto file_manager::get_open_files() const
|
||||
-> std::unordered_map<std::string, std::size_t> {
|
||||
std::unordered_map<std::string, std::size_t> ret;
|
||||
|
||||
recur_mutex_lock open_lock(open_file_mtx_);
|
||||
for (const auto &item : open_file_lookup_) {
|
||||
ret[item.first] = item.second->get_open_file_count();
|
||||
}
|
||||
|
||||
return ret;
|
||||
return std::accumulate(open_file_lookup_.begin(), open_file_lookup_.end(),
|
||||
std::unordered_map<std::string, std::size_t>{},
|
||||
[](auto &&map, auto &&item) -> auto {
|
||||
map[item.first] = item.second->get_open_file_count();
|
||||
return map;
|
||||
});
|
||||
}
|
||||
|
||||
auto file_manager::get_open_handle_count() const -> std::size_t {
|
||||
recur_mutex_lock open_lock(open_file_mtx_);
|
||||
return std::accumulate(
|
||||
open_file_lookup_.begin(), open_file_lookup_.end(), std::size_t(0U),
|
||||
[](std::size_t count, const auto &item) -> std::size_t {
|
||||
return count + item.second->get_open_file_count();
|
||||
});
|
||||
return std::accumulate(open_file_lookup_.begin(), open_file_lookup_.end(),
|
||||
std::size_t(0U),
|
||||
[](auto &&count, auto &&item) -> auto {
|
||||
return count + item.second->get_open_file_count();
|
||||
});
|
||||
}
|
||||
|
||||
auto file_manager::get_stored_downloads() const
|
||||
@@ -380,10 +383,11 @@ auto file_manager::open(const std::string &api_path, bool directory,
|
||||
return open(api_path, directory, ofd, handle, file, nullptr);
|
||||
}
|
||||
|
||||
auto file_manager::open(
|
||||
const std::string &api_path, bool directory, const open_file_data &ofd,
|
||||
std::uint64_t &handle, std::shared_ptr<i_open_file> &file,
|
||||
std::shared_ptr<i_closeable_open_file> closeable_file) -> api_error {
|
||||
auto file_manager::open(const std::string &api_path, bool directory,
|
||||
const open_file_data &ofd, std::uint64_t &handle,
|
||||
std::shared_ptr<i_open_file> &file,
|
||||
std::shared_ptr<i_closeable_open_file> closeable_file)
|
||||
-> api_error {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
const auto create_and_add_handle =
|
||||
@@ -435,6 +439,10 @@ auto file_manager::open(
|
||||
auto ring_size{ring_buffer_file_size / chunk_size};
|
||||
|
||||
const auto get_download_type = [&](download_type type) -> download_type {
|
||||
if (not directory && provider_.is_read_only()) {
|
||||
return download_type::direct;
|
||||
}
|
||||
|
||||
if (directory || fsi.size == 0U || is_processing(api_path)) {
|
||||
return download_type::default_;
|
||||
}
|
||||
@@ -747,8 +755,8 @@ auto file_manager::rename_directory(const std::string &from_api_path,
|
||||
}
|
||||
|
||||
auto file_manager::rename_file(const std::string &from_api_path,
|
||||
const std::string &to_api_path,
|
||||
bool overwrite) -> api_error {
|
||||
const std::string &to_api_path, bool overwrite)
|
||||
-> api_error {
|
||||
if (not provider_.is_rename_supported()) {
|
||||
return api_error::not_implemented;
|
||||
}
|
||||
|
@@ -150,16 +150,16 @@ auto lock_data::set_mount_state(bool active, const std::string &mount_location,
|
||||
(mount_state[mount_id]["Location"].get<std::string>() !=
|
||||
mount_location)))) {
|
||||
const auto lines = utils::file::read_file_lines(get_lock_data_file());
|
||||
const auto txt =
|
||||
std::accumulate(lines.begin(), lines.end(), std::string(),
|
||||
[](std::string s, const std::string &s2) {
|
||||
return std::move(s) + s2;
|
||||
});
|
||||
auto json = json::parse(txt.empty() ? "{}" : txt);
|
||||
json[mount_id] = {{"Active", active},
|
||||
{"Location", active ? mount_location : ""},
|
||||
{"PID", active ? pid : -1}};
|
||||
ret = utils::file::write_json_file(get_lock_data_file(), json);
|
||||
const auto txt = std::accumulate(
|
||||
lines.begin(), lines.end(), std::string(),
|
||||
[](auto &&lines, auto &&line) -> auto { return lines + line; });
|
||||
auto json_data = json::parse(txt.empty() ? "{}" : txt);
|
||||
json_data[mount_id] = {
|
||||
{"Active", active},
|
||||
{"Location", active ? mount_location : ""},
|
||||
{"PID", active ? pid : -1},
|
||||
};
|
||||
ret = utils::file::write_json_file(get_lock_data_file(), json_data);
|
||||
} else {
|
||||
ret = true;
|
||||
}
|
||||
|
@@ -36,7 +36,7 @@
|
||||
#include "utils/time.hpp"
|
||||
|
||||
namespace repertory {
|
||||
void base_provider::add_all_items(const stop_type &stop_requested) {
|
||||
void base_provider::add_all_items(stop_type &stop_requested) {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
api_file_list list{};
|
||||
@@ -140,76 +140,77 @@ auto base_provider::create_directory(const std::string &api_path,
|
||||
api_meta_map &meta) -> api_error {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
bool exists{};
|
||||
auto res = is_directory(api_path, exists);
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
if (exists) {
|
||||
utils::error::raise_api_path_error(function_name, api_path,
|
||||
api_error::directory_exists,
|
||||
"failed to create directory");
|
||||
return api_error::directory_exists;
|
||||
}
|
||||
|
||||
res = is_file(api_path, exists);
|
||||
if (res != api_error::success) {
|
||||
utils::error::raise_api_path_error(function_name, api_path, res,
|
||||
"failed to create directory");
|
||||
return res;
|
||||
}
|
||||
if (exists) {
|
||||
utils::error::raise_api_path_error(function_name, api_path,
|
||||
api_error::item_exists,
|
||||
"failed to create directory");
|
||||
return api_error::item_exists;
|
||||
}
|
||||
|
||||
try {
|
||||
bool exists{};
|
||||
auto res = is_directory(api_path, exists);
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
if (exists) {
|
||||
utils::error::raise_api_path_error(function_name, api_path,
|
||||
api_error::directory_exists,
|
||||
"failed to create directory");
|
||||
return api_error::directory_exists;
|
||||
}
|
||||
|
||||
res = is_file(api_path, exists);
|
||||
if (res != api_error::success) {
|
||||
utils::error::raise_api_path_error(function_name, api_path, res,
|
||||
"failed to create directory");
|
||||
return res;
|
||||
}
|
||||
if (exists) {
|
||||
utils::error::raise_api_path_error(function_name, api_path,
|
||||
api_error::item_exists,
|
||||
"failed to create directory");
|
||||
return api_error::item_exists;
|
||||
}
|
||||
|
||||
res = create_directory_impl(api_path, meta);
|
||||
if (res != api_error::success) {
|
||||
utils::error::raise_api_path_error(function_name, api_path, res,
|
||||
"failed to create directory");
|
||||
return res;
|
||||
}
|
||||
|
||||
meta[META_DIRECTORY] = utils::string::from_bool(true);
|
||||
return set_item_meta(api_path, meta);
|
||||
} catch (const std::exception &e) {
|
||||
utils::error::raise_api_path_error(function_name, api_path, e,
|
||||
"failed to create directory");
|
||||
return api_error::error;
|
||||
}
|
||||
|
||||
meta[META_DIRECTORY] = utils::string::from_bool(true);
|
||||
return set_item_meta(api_path, meta);
|
||||
return api_error::error;
|
||||
}
|
||||
|
||||
auto base_provider::create_file(const std::string &api_path, api_meta_map &meta)
|
||||
-> api_error {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
bool exists{};
|
||||
auto res = is_directory(api_path, exists);
|
||||
if (res != api_error::success && res != api_error::item_not_found) {
|
||||
return res;
|
||||
}
|
||||
if (exists) {
|
||||
utils::error::raise_api_path_error(function_name, api_path,
|
||||
api_error::directory_exists,
|
||||
"failed to create file");
|
||||
return api_error::directory_exists;
|
||||
}
|
||||
|
||||
res = is_file(api_path, exists);
|
||||
if (res != api_error::success && res != api_error::item_not_found) {
|
||||
return res;
|
||||
}
|
||||
if (exists) {
|
||||
utils::error::raise_api_path_error(function_name, api_path,
|
||||
api_error::item_exists,
|
||||
"failed to create file");
|
||||
return api_error::item_exists;
|
||||
}
|
||||
|
||||
try {
|
||||
bool exists{};
|
||||
auto res = is_directory(api_path, exists);
|
||||
if (res != api_error::success && res != api_error::item_not_found) {
|
||||
return res;
|
||||
}
|
||||
if (exists) {
|
||||
utils::error::raise_api_path_error(function_name, api_path,
|
||||
api_error::directory_exists,
|
||||
"failed to create file");
|
||||
return api_error::directory_exists;
|
||||
}
|
||||
|
||||
res = is_file(api_path, exists);
|
||||
if (res != api_error::success && res != api_error::item_not_found) {
|
||||
return res;
|
||||
}
|
||||
if (exists) {
|
||||
utils::error::raise_api_path_error(function_name, api_path,
|
||||
api_error::item_exists,
|
||||
"failed to create file");
|
||||
return api_error::item_exists;
|
||||
}
|
||||
|
||||
res = create_file_extra(api_path, meta);
|
||||
if (res != api_error::success) {
|
||||
utils::error::raise_api_path_error(function_name, api_path, res,
|
||||
@@ -260,16 +261,16 @@ auto base_provider::get_directory_items(const std::string &api_path,
|
||||
-> api_error {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
bool exists{};
|
||||
auto res = is_directory(api_path, exists);
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
if (not exists) {
|
||||
return api_error::directory_not_found;
|
||||
}
|
||||
|
||||
try {
|
||||
bool exists{};
|
||||
auto res = is_directory(api_path, exists);
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
if (not exists) {
|
||||
return api_error::directory_not_found;
|
||||
}
|
||||
|
||||
res = get_directory_items_impl(api_path, list);
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
@@ -436,7 +437,7 @@ auto base_provider::is_file_writeable(const std::string &api_path) const
|
||||
}
|
||||
|
||||
void base_provider::process_removed_directories(
|
||||
std::deque<removed_item> removed_list, const stop_type &stop_requested) {
|
||||
std::deque<removed_item> removed_list, stop_type &stop_requested) {
|
||||
for (const auto &item : removed_list) {
|
||||
if (stop_requested) {
|
||||
return;
|
||||
@@ -453,7 +454,7 @@ void base_provider::process_removed_directories(
|
||||
}
|
||||
|
||||
void base_provider::process_removed_files(std::deque<removed_item> removed_list,
|
||||
const stop_type &stop_requested) {
|
||||
stop_type &stop_requested) {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
auto orphaned_directory =
|
||||
@@ -512,64 +513,66 @@ void base_provider::process_removed_files(std::deque<removed_item> removed_list,
|
||||
}
|
||||
}
|
||||
|
||||
void base_provider::process_removed_items(const stop_type &stop_requested) {
|
||||
auto list = db3_->get_api_path_list();
|
||||
[[maybe_unused]] auto res =
|
||||
std::all_of(list.begin(), list.end(), [&](auto &&api_path) -> bool {
|
||||
if (stop_requested) {
|
||||
return false;
|
||||
}
|
||||
|
||||
tasks::instance().schedule({
|
||||
[this, api_path](auto &&task_stopped) {
|
||||
api_meta_map meta{};
|
||||
if (get_item_meta(api_path, meta) != api_error::success) {
|
||||
return;
|
||||
void base_provider::process_removed_items(stop_type &stop_requested) {
|
||||
db3_->enumerate_api_path_list(
|
||||
[this, &stop_requested](auto &&list) {
|
||||
[[maybe_unused]] auto res =
|
||||
std::all_of(list.begin(), list.end(), [&](auto &&api_path) -> bool {
|
||||
if (stop_requested) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (utils::string::to_bool(meta[META_DIRECTORY])) {
|
||||
return;
|
||||
}
|
||||
// bool exists{};
|
||||
// if (is_directory(api_path, exists) != api_error::success) {
|
||||
// return;
|
||||
// }
|
||||
//
|
||||
// if (exists) {
|
||||
// return;
|
||||
// }
|
||||
//
|
||||
// // process_removed_directories(
|
||||
// // {
|
||||
// // removed_item{api_path, true, ""},
|
||||
// // },
|
||||
// // stop_requested2);
|
||||
//
|
||||
// return;
|
||||
// }
|
||||
tasks::instance().schedule({
|
||||
[this, api_path](auto &&task_stopped) {
|
||||
api_meta_map meta{};
|
||||
if (get_item_meta(api_path, meta) != api_error::success) {
|
||||
return;
|
||||
}
|
||||
|
||||
bool exists{};
|
||||
if (is_file(api_path, exists) != api_error::success) {
|
||||
return;
|
||||
}
|
||||
if (utils::string::to_bool(meta[META_DIRECTORY])) {
|
||||
bool exists{};
|
||||
if (is_directory(api_path, exists) !=
|
||||
api_error::success) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (exists) {
|
||||
return;
|
||||
}
|
||||
if (exists) {
|
||||
return;
|
||||
}
|
||||
|
||||
process_removed_files(
|
||||
{
|
||||
removed_item{api_path, false, meta[META_SOURCE]},
|
||||
process_removed_directories(
|
||||
{
|
||||
removed_item{api_path, true, ""},
|
||||
},
|
||||
task_stopped);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
bool exists{};
|
||||
if (is_file(api_path, exists) != api_error::success) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (exists) {
|
||||
return;
|
||||
}
|
||||
|
||||
process_removed_files(
|
||||
{
|
||||
removed_item{api_path, false, meta[META_SOURCE]},
|
||||
},
|
||||
task_stopped);
|
||||
},
|
||||
task_stopped);
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
return not stop_requested;
|
||||
});
|
||||
return not stop_requested;
|
||||
});
|
||||
},
|
||||
stop_requested);
|
||||
}
|
||||
|
||||
void base_provider::remove_deleted_items(const stop_type &stop_requested) {
|
||||
void base_provider::remove_deleted_items(stop_type &stop_requested) {
|
||||
add_all_items(stop_requested);
|
||||
if (stop_requested) {
|
||||
return;
|
||||
@@ -665,8 +668,7 @@ auto base_provider::remove_item_meta(const std::string &api_path,
|
||||
return db3_->remove_item_meta(api_path, key);
|
||||
}
|
||||
|
||||
void base_provider::remove_unmatched_source_files(
|
||||
const stop_type &stop_requested) {
|
||||
void base_provider::remove_unmatched_source_files(stop_type &stop_requested) {
|
||||
if (is_read_only()) {
|
||||
return;
|
||||
}
|
||||
@@ -783,8 +785,6 @@ auto base_provider::upload_file(const std::string &api_path,
|
||||
stop_type &stop_requested) -> api_error {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
event_system::instance().raise<provider_upload_begin>(api_path, source_path);
|
||||
|
||||
const auto notify_end = [&api_path,
|
||||
&source_path](api_error error) -> api_error {
|
||||
event_system::instance().raise<provider_upload_end>(api_path, source_path,
|
||||
@@ -793,6 +793,9 @@ auto base_provider::upload_file(const std::string &api_path,
|
||||
};
|
||||
|
||||
try {
|
||||
event_system::instance().raise<provider_upload_begin>(api_path,
|
||||
source_path);
|
||||
|
||||
return notify_end(upload_file_impl(api_path, source_path, stop_requested));
|
||||
} catch (const std::exception &e) {
|
||||
utils::error::raise_error(function_name, e, "exception occurred");
|
||||
|
@@ -33,15 +33,17 @@
|
||||
#include "utils/file_utils.hpp"
|
||||
#include "utils/path.hpp"
|
||||
#include "utils/polling.hpp"
|
||||
#include <utils/config.hpp>
|
||||
|
||||
namespace repertory {
|
||||
encrypt_provider::encrypt_provider(app_config &config)
|
||||
: config_(config), encrypt_config_(config.get_encrypt_config()) {}
|
||||
|
||||
auto encrypt_provider::create_api_file(
|
||||
const std::string &api_path, bool directory,
|
||||
const std::string &source_path) -> api_file {
|
||||
auto times = utils::file::get_times(source_path);
|
||||
auto encrypt_provider::create_api_file(const std::string &api_path,
|
||||
bool directory,
|
||||
const std::string &source_path)
|
||||
-> api_file {
|
||||
auto times{utils::file::get_times(source_path)};
|
||||
if (not times.has_value()) {
|
||||
throw std::runtime_error("failed to get file times");
|
||||
}
|
||||
@@ -66,10 +68,10 @@ auto encrypt_provider::create_api_file(
|
||||
void encrypt_provider::create_item_meta(api_meta_map &meta, bool directory,
|
||||
const api_file &file) {
|
||||
#if defined(_WIN32)
|
||||
struct _stat64 buf {};
|
||||
struct _stat64 buf{};
|
||||
_stat64(file.source_path.c_str(), &buf);
|
||||
#else // !defined(_WIN32)
|
||||
struct stat buf {};
|
||||
struct stat buf{};
|
||||
stat(file.source_path.c_str(), &buf);
|
||||
#endif // defined(_WIN32)
|
||||
|
||||
@@ -112,7 +114,7 @@ auto encrypt_provider::do_fs_operation(
|
||||
std::function<api_error(const encrypt_config &cfg,
|
||||
const std::string &source_path)>
|
||||
callback) const -> api_error {
|
||||
const auto &cfg = get_encrypt_config();
|
||||
const auto &cfg{get_encrypt_config()};
|
||||
|
||||
std::string source_path{api_path};
|
||||
if (api_path != "/" && not utils::encryption::decrypt_file_path(
|
||||
@@ -129,7 +131,7 @@ auto encrypt_provider::do_fs_operation(
|
||||
: api_error::item_not_found;
|
||||
}
|
||||
|
||||
auto exists = utils::file::file{source_path}.exists();
|
||||
auto exists{utils::file::file{source_path}.exists()};
|
||||
if (exists && directory) {
|
||||
return api_error::item_exists;
|
||||
}
|
||||
@@ -149,8 +151,9 @@ auto encrypt_provider::do_fs_operation(
|
||||
return callback(cfg, source_path);
|
||||
}
|
||||
|
||||
auto encrypt_provider::get_api_path_from_source(
|
||||
const std::string &source_path, std::string &api_path) const -> api_error {
|
||||
auto encrypt_provider::get_api_path_from_source(const std::string &source_path,
|
||||
std::string &api_path) const
|
||||
-> api_error {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
try {
|
||||
@@ -168,19 +171,20 @@ auto encrypt_provider::get_directory_item_count(
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
std::uint64_t count{};
|
||||
auto res = do_fs_operation(
|
||||
api_path, true,
|
||||
[&api_path, &count](const encrypt_config & /* cfg */,
|
||||
const std::string &source_path) -> api_error {
|
||||
try {
|
||||
count = utils::file::directory{source_path}.count();
|
||||
} catch (const std::exception &ex) {
|
||||
utils::error::raise_api_path_error(
|
||||
function_name, api_path, source_path, ex,
|
||||
"failed to get directory item count");
|
||||
}
|
||||
return api_error::success;
|
||||
});
|
||||
auto res{
|
||||
do_fs_operation(api_path, true,
|
||||
[&api_path, &count](auto && /* cfg */,
|
||||
auto &&source_path) -> api_error {
|
||||
try {
|
||||
count = utils::file::directory{source_path}.count();
|
||||
} catch (const std::exception &ex) {
|
||||
utils::error::raise_api_path_error(
|
||||
function_name, api_path, source_path, ex,
|
||||
"failed to get directory item count");
|
||||
}
|
||||
return api_error::success;
|
||||
}),
|
||||
};
|
||||
if (res != api_error::success) {
|
||||
utils::error::raise_api_path_error(function_name, api_path, res,
|
||||
"failed to get directory item count");
|
||||
@@ -189,22 +193,24 @@ auto encrypt_provider::get_directory_item_count(
|
||||
return count;
|
||||
}
|
||||
|
||||
auto encrypt_provider::get_directory_items(
|
||||
const std::string &api_path, directory_item_list &list) const -> api_error {
|
||||
auto encrypt_provider::get_directory_items(const std::string &api_path,
|
||||
directory_item_list &list) const
|
||||
-> api_error {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
return do_fs_operation(
|
||||
api_path, true,
|
||||
[this, &list](const encrypt_config &cfg,
|
||||
const std::string &source_path) -> api_error {
|
||||
[this, &list](auto &&cfg, auto &&source_path) -> api_error {
|
||||
try {
|
||||
for (const auto &dir_entry :
|
||||
utils::file::directory{source_path}.get_items()) {
|
||||
try {
|
||||
std::string current_api_path;
|
||||
if (dir_entry->is_directory_item()) {
|
||||
auto result = db_->get_directory_api_path(dir_entry->get_path(),
|
||||
current_api_path);
|
||||
auto result{
|
||||
db_->get_directory_api_path(dir_entry->get_path(),
|
||||
current_api_path),
|
||||
};
|
||||
if (result != api_error::success &&
|
||||
result != api_error::directory_not_found) {
|
||||
// TODO raise error
|
||||
@@ -224,8 +230,10 @@ auto encrypt_provider::get_directory_items(
|
||||
}
|
||||
}
|
||||
} else {
|
||||
auto result = db_->get_file_api_path(dir_entry->get_path(),
|
||||
current_api_path);
|
||||
auto result{
|
||||
db_->get_file_api_path(dir_entry->get_path(),
|
||||
current_api_path),
|
||||
};
|
||||
if (result != api_error::success &&
|
||||
result != api_error::item_not_found) {
|
||||
// TODO raise error
|
||||
@@ -238,9 +246,11 @@ auto encrypt_provider::get_directory_items(
|
||||
}
|
||||
}
|
||||
|
||||
auto file = create_api_file(current_api_path,
|
||||
dir_entry->is_directory_item(),
|
||||
dir_entry->get_path());
|
||||
auto file{
|
||||
create_api_file(current_api_path,
|
||||
dir_entry->is_directory_item(),
|
||||
dir_entry->get_path()),
|
||||
};
|
||||
|
||||
directory_item dir_item{};
|
||||
dir_item.api_parent = file.api_parent;
|
||||
@@ -294,7 +304,7 @@ auto encrypt_provider::get_file(const std::string &api_path,
|
||||
|
||||
try {
|
||||
bool exists{};
|
||||
auto res = is_directory(api_path, exists);
|
||||
auto res{is_directory(api_path, exists)};
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
@@ -303,7 +313,7 @@ auto encrypt_provider::get_file(const std::string &api_path,
|
||||
}
|
||||
|
||||
std::string source_path;
|
||||
auto result = db_->get_file_source_path(api_path, source_path);
|
||||
auto result{db_->get_file_source_path(api_path, source_path)};
|
||||
if (result != api_error::success) {
|
||||
return result;
|
||||
}
|
||||
@@ -318,11 +328,12 @@ auto encrypt_provider::get_file(const std::string &api_path,
|
||||
return api_error::error;
|
||||
}
|
||||
|
||||
auto encrypt_provider::get_file_list(
|
||||
api_file_list &list, std::string & /* marker */) const -> api_error {
|
||||
auto encrypt_provider::get_file_list(api_file_list &list,
|
||||
std::string & /* marker */) const
|
||||
-> api_error {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
const auto &cfg = get_encrypt_config();
|
||||
const auto &cfg{get_encrypt_config()};
|
||||
|
||||
try {
|
||||
for (const auto &dir_entry : utils::file::directory{cfg.path}.get_items()) {
|
||||
@@ -342,13 +353,14 @@ auto encrypt_provider::get_file_list(
|
||||
return api_error::error;
|
||||
}
|
||||
|
||||
auto encrypt_provider::get_file_size(
|
||||
const std::string &api_path, std::uint64_t &file_size) const -> api_error {
|
||||
auto encrypt_provider::get_file_size(const std::string &api_path,
|
||||
std::uint64_t &file_size) const
|
||||
-> api_error {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
try {
|
||||
std::string source_path;
|
||||
auto result = db_->get_file_source_path(api_path, source_path);
|
||||
auto result{db_->get_file_source_path(api_path, source_path)};
|
||||
if (result != api_error::success) {
|
||||
return result;
|
||||
}
|
||||
@@ -364,12 +376,13 @@ auto encrypt_provider::get_file_size(
|
||||
return api_error::error;
|
||||
}
|
||||
|
||||
auto encrypt_provider::get_filesystem_item(
|
||||
const std::string &api_path, bool directory,
|
||||
filesystem_item &fsi) const -> api_error {
|
||||
auto encrypt_provider::get_filesystem_item(const std::string &api_path,
|
||||
bool directory,
|
||||
filesystem_item &fsi) const
|
||||
-> api_error {
|
||||
std::string source_path;
|
||||
if (directory) {
|
||||
auto result = db_->get_directory_source_path(api_path, source_path);
|
||||
auto result{db_->get_directory_source_path(api_path, source_path)};
|
||||
if (result != api_error::success) {
|
||||
return result;
|
||||
}
|
||||
@@ -382,7 +395,7 @@ auto encrypt_provider::get_filesystem_item(
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
auto result = db_->get_file_source_path(api_path, source_path);
|
||||
auto result{db_->get_file_source_path(api_path, source_path)};
|
||||
if (result != api_error::success) {
|
||||
return result;
|
||||
}
|
||||
@@ -400,7 +413,7 @@ auto encrypt_provider::get_filesystem_item(
|
||||
auto encrypt_provider::get_filesystem_item_from_source_path(
|
||||
const std::string &source_path, filesystem_item &fsi) const -> api_error {
|
||||
std::string api_path{};
|
||||
auto res = get_api_path_from_source(source_path, api_path);
|
||||
auto res{get_api_path_from_source(source_path, api_path)};
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
@@ -417,14 +430,15 @@ auto encrypt_provider::get_filesystem_item_from_source_path(
|
||||
return get_filesystem_item(api_path, false, fsi);
|
||||
}
|
||||
|
||||
auto encrypt_provider::get_filesystem_item_and_file(
|
||||
const std::string &api_path, api_file &file,
|
||||
filesystem_item &fsi) const -> api_error {
|
||||
auto encrypt_provider::get_filesystem_item_and_file(const std::string &api_path,
|
||||
api_file &file,
|
||||
filesystem_item &fsi) const
|
||||
-> api_error {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
try {
|
||||
bool exists{};
|
||||
auto res = is_directory(api_path, exists);
|
||||
auto res{is_directory(api_path, exists)};
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
@@ -432,7 +446,7 @@ auto encrypt_provider::get_filesystem_item_and_file(
|
||||
return api_error::directory_exists;
|
||||
}
|
||||
|
||||
auto ret = get_filesystem_item(api_path, exists, fsi);
|
||||
auto ret{get_filesystem_item(api_path, exists, fsi)};
|
||||
if (ret != api_error::success) {
|
||||
return ret;
|
||||
}
|
||||
@@ -457,7 +471,7 @@ auto encrypt_provider::get_item_meta(const std::string &api_path,
|
||||
|
||||
try {
|
||||
std::string source_path;
|
||||
auto result = db_->get_source_path(api_path, source_path);
|
||||
auto result{db_->get_source_path(api_path, source_path)};
|
||||
if (result != api_error::success) {
|
||||
return result;
|
||||
}
|
||||
@@ -468,7 +482,7 @@ auto encrypt_provider::get_item_meta(const std::string &api_path,
|
||||
return result;
|
||||
}
|
||||
|
||||
auto file = create_api_file(api_path, is_dir, source_path);
|
||||
auto file{create_api_file(api_path, is_dir, source_path)};
|
||||
create_item_meta(meta, is_dir, file);
|
||||
return api_error::success;
|
||||
} catch (const std::exception &ex) {
|
||||
@@ -483,7 +497,7 @@ auto encrypt_provider::get_item_meta(const std::string &api_path,
|
||||
const std::string &key,
|
||||
std::string &value) const -> api_error {
|
||||
api_meta_map meta{};
|
||||
auto ret = get_item_meta(api_path, meta);
|
||||
auto ret{get_item_meta(api_path, meta)};
|
||||
if (ret != api_error::success) {
|
||||
return ret;
|
||||
}
|
||||
@@ -511,8 +525,9 @@ auto encrypt_provider::get_total_item_count() const -> std::uint64_t {
|
||||
}
|
||||
|
||||
auto encrypt_provider::get_used_drive_space() const -> std::uint64_t {
|
||||
auto free_space =
|
||||
utils::file::get_free_drive_space(get_encrypt_config().path);
|
||||
auto free_space{
|
||||
utils::file::get_free_drive_space(get_encrypt_config().path),
|
||||
};
|
||||
return free_space.has_value() ? get_total_drive_space() - free_space.value()
|
||||
: 0U;
|
||||
}
|
||||
@@ -523,7 +538,7 @@ auto encrypt_provider::is_directory(const std::string &api_path,
|
||||
|
||||
try {
|
||||
std::string source_path;
|
||||
auto result = db_->get_directory_source_path(api_path, source_path);
|
||||
auto result{db_->get_directory_source_path(api_path, source_path)};
|
||||
|
||||
if (result != api_error::success) {
|
||||
if (result != api_error::directory_not_found) {
|
||||
@@ -544,13 +559,13 @@ auto encrypt_provider::is_directory(const std::string &api_path,
|
||||
return api_error::error;
|
||||
}
|
||||
|
||||
auto encrypt_provider::is_file(const std::string &api_path,
|
||||
bool &exists) const -> api_error {
|
||||
auto encrypt_provider::is_file(const std::string &api_path, bool &exists) const
|
||||
-> api_error {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
try {
|
||||
std::string source_path;
|
||||
auto result = db_->get_file_source_path(api_path, source_path);
|
||||
auto result{db_->get_file_source_path(api_path, source_path)};
|
||||
if (result != api_error::success) {
|
||||
if (result != api_error::item_not_found) {
|
||||
return result;
|
||||
@@ -587,8 +602,10 @@ auto encrypt_provider::process_directory_entry(
|
||||
try {
|
||||
const auto do_add_directory =
|
||||
[this, &cfg](std::string_view dir_path) -> std::string {
|
||||
auto encrypted_parts = utils::string::split(
|
||||
utils::path::create_api_path(dir_path), '/', false);
|
||||
auto encrypted_parts{
|
||||
utils::string::split(utils::path::create_api_path(dir_path), '/',
|
||||
false),
|
||||
};
|
||||
|
||||
for (std::size_t part_idx = 1U; part_idx < encrypted_parts.size();
|
||||
++part_idx) {
|
||||
@@ -611,8 +628,9 @@ auto encrypt_provider::process_directory_entry(
|
||||
current_source_path = utils::path::combine(current_source_path, {part});
|
||||
|
||||
std::string current_api_path{};
|
||||
auto result =
|
||||
db_->get_directory_api_path(current_source_path, current_api_path);
|
||||
auto result{
|
||||
db_->get_directory_api_path(current_source_path, current_api_path),
|
||||
};
|
||||
if (result == api_error::directory_not_found) {
|
||||
current_api_path = utils::path::create_api_path(
|
||||
current_encrypted_path + '/' + encrypted_parts.at(current_idx));
|
||||
@@ -652,11 +670,12 @@ auto encrypt_provider::process_directory_entry(
|
||||
}
|
||||
|
||||
if (dir_entry.is_file_item() && not dir_entry.is_symlink()) {
|
||||
auto relative_path =
|
||||
utils::path::get_relative_path(dir_entry.get_path(), cfg.path);
|
||||
auto relative_path{
|
||||
utils::path::get_relative_path(dir_entry.get_path(), cfg.path),
|
||||
};
|
||||
|
||||
i_file_db::file_data data;
|
||||
auto file_res = db_->get_file_data(dir_entry.get_path(), data);
|
||||
auto file_res{db_->get_file_data(dir_entry.get_path(), data)};
|
||||
if (file_res != api_error::success &&
|
||||
file_res != api_error::item_not_found) {
|
||||
// TODO raise error
|
||||
@@ -664,8 +683,10 @@ auto encrypt_provider::process_directory_entry(
|
||||
}
|
||||
|
||||
std::string api_parent{};
|
||||
auto parent_res = db_->get_directory_api_path(
|
||||
utils::path::get_parent_path(dir_entry.get_path()), api_parent);
|
||||
auto parent_res{
|
||||
db_->get_directory_api_path(
|
||||
utils::path::get_parent_path(dir_entry.get_path()), api_parent),
|
||||
};
|
||||
if (parent_res != api_error::success &&
|
||||
parent_res != api_error::directory_not_found) {
|
||||
// TODO raise error
|
||||
@@ -720,27 +741,28 @@ auto encrypt_provider::read_file_bytes(const std::string &api_path,
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
i_file_db::file_data file_data{};
|
||||
auto result = db_->get_file_data(api_path, file_data);
|
||||
auto result{db_->get_file_data(api_path, file_data)};
|
||||
if (result != api_error::success) {
|
||||
return result;
|
||||
}
|
||||
|
||||
auto opt_size = utils::file::file{file_data.source_path}.size();
|
||||
auto opt_size{utils::file::file{file_data.source_path}.size()};
|
||||
if (not opt_size.has_value()) {
|
||||
return api_error::os_error;
|
||||
}
|
||||
|
||||
auto file_size{opt_size.value()};
|
||||
|
||||
const auto &cfg = get_encrypt_config();
|
||||
const auto &cfg{get_encrypt_config()};
|
||||
|
||||
unique_recur_mutex_lock reader_lookup_lock(reader_lookup_mtx_);
|
||||
|
||||
if (file_data.file_size != file_size) {
|
||||
auto relative_path =
|
||||
utils::path::get_relative_path(file_data.source_path, cfg.path);
|
||||
auto relative_path{
|
||||
utils::path::get_relative_path(file_data.source_path, cfg.path),
|
||||
};
|
||||
|
||||
auto info = std::make_shared<reader_info>();
|
||||
auto info{std::make_shared<reader_info>()};
|
||||
info->reader = std::make_unique<utils::encryption::encrypting_reader>(
|
||||
relative_path, file_data.source_path, stop_requested,
|
||||
cfg.encryption_token, utils::path::get_parent_path(relative_path));
|
||||
@@ -758,7 +780,7 @@ auto encrypt_provider::read_file_bytes(const std::string &api_path,
|
||||
}
|
||||
} else if (reader_lookup_.find(file_data.source_path) ==
|
||||
reader_lookup_.end()) {
|
||||
auto info = std::make_shared<reader_info>();
|
||||
auto info{std::make_shared<reader_info>()};
|
||||
info->reader = std::make_unique<utils::encryption::encrypting_reader>(
|
||||
api_path, file_data.source_path, stop_requested, cfg.encryption_token,
|
||||
std::move(file_data.iv_list));
|
||||
@@ -769,7 +791,7 @@ auto encrypt_provider::read_file_bytes(const std::string &api_path,
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
auto info = reader_lookup_.at(file_data.source_path);
|
||||
auto info{reader_lookup_.at(file_data.source_path)};
|
||||
info->last_access_time = std::chrono::system_clock::now();
|
||||
reader_lookup_lock.unlock();
|
||||
|
||||
@@ -777,45 +799,56 @@ auto encrypt_provider::read_file_bytes(const std::string &api_path,
|
||||
info->reader->set_read_position(offset);
|
||||
data.resize(size);
|
||||
|
||||
auto res =
|
||||
auto res{
|
||||
info->reader->reader_function(reinterpret_cast<char *>(data.data()), 1U,
|
||||
data.size(), info->reader.get());
|
||||
if (res == 0) {
|
||||
return api_error::os_error;
|
||||
}
|
||||
|
||||
return api_error::success;
|
||||
data.size(), info->reader.get()),
|
||||
};
|
||||
return res == 0 ? api_error::os_error : api_error::success;
|
||||
}
|
||||
|
||||
void encrypt_provider::remove_deleted_files(const stop_type &stop_requested) {
|
||||
std::vector<i_file_db::file_info> removed_list{};
|
||||
void encrypt_provider::remove_deleted_files(stop_type &stop_requested) {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
for (const auto &item : db_->get_item_list()) {
|
||||
if (stop_requested) {
|
||||
return;
|
||||
}
|
||||
db_->enumerate_item_list(
|
||||
[this, &stop_requested](auto &&list) {
|
||||
std::vector<i_file_db::file_info> removed_list{};
|
||||
for (const auto &item : list) {
|
||||
if (stop_requested) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (not utils::path::exists(item.source_path)) {
|
||||
removed_list.emplace_back(item);
|
||||
}
|
||||
}
|
||||
if (utils::path::exists(item.source_path)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const auto &item : removed_list) {
|
||||
if (stop_requested) {
|
||||
return;
|
||||
}
|
||||
removed_list.emplace_back(item);
|
||||
}
|
||||
|
||||
// TODO handle error
|
||||
auto del_res = db_->remove_item(item.api_path);
|
||||
if (item.directory) {
|
||||
event_system::instance().raise<directory_removed_externally>(
|
||||
item.api_path, item.source_path);
|
||||
continue;
|
||||
}
|
||||
for (const auto &item : removed_list) {
|
||||
if (stop_requested) {
|
||||
return;
|
||||
}
|
||||
|
||||
event_system::instance().raise<file_removed_externally>(item.api_path,
|
||||
item.source_path);
|
||||
}
|
||||
auto res{db_->remove_item(item.api_path)};
|
||||
if (res != api_error::success) {
|
||||
utils::error::raise_api_path_error(
|
||||
function_name, item.api_path, item.source_path, res,
|
||||
fmt::format("failed to process externally removed item|dir|{}",
|
||||
utils::string::from_bool(item.directory)));
|
||||
continue;
|
||||
}
|
||||
|
||||
if (item.directory) {
|
||||
event_system::instance().raise<directory_removed_externally>(
|
||||
item.api_path, item.source_path);
|
||||
continue;
|
||||
}
|
||||
|
||||
event_system::instance().raise<file_removed_externally>(
|
||||
item.api_path, item.source_path);
|
||||
}
|
||||
},
|
||||
stop_requested);
|
||||
}
|
||||
|
||||
auto encrypt_provider::start(api_item_added_callback /*api_item_added*/,
|
||||
@@ -829,16 +862,16 @@ auto encrypt_provider::start(api_item_added_callback /*api_item_added*/,
|
||||
db_ = create_file_db(config_);
|
||||
|
||||
std::string source_path;
|
||||
auto result = db_->get_directory_source_path("/", source_path);
|
||||
auto result{db_->get_directory_source_path("/", source_path)};
|
||||
if (result != api_error::success &&
|
||||
result != api_error::directory_not_found) {
|
||||
throw startup_exception(
|
||||
fmt::format("failed to get root|{}", api_error_to_string(result)));
|
||||
}
|
||||
|
||||
auto cfg_path = utils::path::absolute(get_encrypt_config().path);
|
||||
auto cfg_path{utils::path::absolute(get_encrypt_config().path)};
|
||||
if (result == api_error::success) {
|
||||
auto cur_path = utils::path::absolute(source_path);
|
||||
auto cur_path{utils::path::absolute(source_path)};
|
||||
#if defined(_WIN32)
|
||||
if (utils::string::to_lower(cur_path) !=
|
||||
utils::string::to_lower(cfg_path)) {
|
||||
|
@@ -40,9 +40,9 @@
|
||||
#include "utils/time.hpp"
|
||||
|
||||
namespace {
|
||||
[[nodiscard]] auto
|
||||
set_request_path(auto &request,
|
||||
const std::string &object_name) -> repertory::api_error {
|
||||
[[nodiscard]] auto set_request_path(auto &request,
|
||||
const std::string &object_name)
|
||||
-> repertory::api_error {
|
||||
request.path = object_name;
|
||||
if (request.path.substr(1U).size() > repertory::max_s3_object_name_length) {
|
||||
return repertory::api_error::name_too_long;
|
||||
@@ -56,10 +56,11 @@ namespace repertory {
|
||||
s3_provider::s3_provider(app_config &config, i_http_comm &comm)
|
||||
: base_provider(config, comm) {}
|
||||
|
||||
auto s3_provider::add_if_not_found(
|
||||
api_file &file, const std::string &object_name) const -> api_error {
|
||||
auto s3_provider::add_if_not_found(api_file &file,
|
||||
const std::string &object_name) const
|
||||
-> api_error {
|
||||
api_meta_map meta{};
|
||||
auto res = get_item_meta(file.api_path, meta);
|
||||
auto res{get_item_meta(file.api_path, meta)};
|
||||
if (res == api_error::item_not_found) {
|
||||
res = create_directory_paths(file.api_parent,
|
||||
utils::path::get_parent_api_path(object_name));
|
||||
@@ -75,16 +76,17 @@ auto s3_provider::add_if_not_found(
|
||||
|
||||
auto s3_provider::convert_api_date(std::string_view date) -> std::uint64_t {
|
||||
// 2009-10-12T17:50:30.000Z
|
||||
auto date_parts = utils::string::split(date, '.', true);
|
||||
auto date_time = date_parts.at(0U);
|
||||
auto nanos =
|
||||
auto date_parts{utils::string::split(date, '.', true)};
|
||||
auto date_time{date_parts.at(0U)};
|
||||
auto nanos{
|
||||
(date_parts.size() <= 1U)
|
||||
? 0U
|
||||
: utils::string::to_uint64(
|
||||
utils::string::split(date_parts.at(1U), 'Z', true).at(0U)) *
|
||||
1000000UL;
|
||||
1000000UL,
|
||||
};
|
||||
|
||||
struct tm tm1 {};
|
||||
struct tm tm1{};
|
||||
#if defined(_WIN32)
|
||||
utils::time::strptime(date_time.c_str(), "%Y-%m-%dT%T", &tm1);
|
||||
return nanos + utils::time::windows_time_t_to_unix_time(_mkgmtime(&tm1));
|
||||
@@ -95,36 +97,12 @@ auto s3_provider::convert_api_date(std::string_view date) -> std::uint64_t {
|
||||
#endif // defined(_WIN32)
|
||||
}
|
||||
|
||||
auto s3_provider::create_directory_impl(const std::string &api_path,
|
||||
api_meta_map &meta) -> api_error {
|
||||
auto s3_provider::create_directory_object(const std::string &api_path,
|
||||
const std::string &object_name) const
|
||||
-> api_error {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
const auto &cfg = get_s3_config();
|
||||
auto is_encrypted = not cfg.encryption_token.empty();
|
||||
stop_type stop_requested{false};
|
||||
|
||||
if (is_encrypted) {
|
||||
std::string encrypted_file_path;
|
||||
auto res = get_item_meta(utils::path::get_parent_api_path(api_path),
|
||||
META_KEY, encrypted_file_path);
|
||||
if (res != api_error::success) {
|
||||
utils::error::raise_api_path_error(function_name, api_path, res,
|
||||
"failed to create file");
|
||||
return res;
|
||||
}
|
||||
|
||||
data_buffer result;
|
||||
utils::encryption::encrypt_data(
|
||||
cfg.encryption_token,
|
||||
*(utils::string::split(api_path, '/', false).end() - 1U), result);
|
||||
|
||||
meta[META_KEY] = utils::path::create_api_path(
|
||||
utils::path::combine(utils::path::create_api_path(encrypted_file_path),
|
||||
{utils::collection::to_hex_string(result)}));
|
||||
}
|
||||
|
||||
auto object_name =
|
||||
utils::path::create_api_path(is_encrypted ? meta[META_KEY] : api_path);
|
||||
const auto &cfg{get_s3_config()};
|
||||
|
||||
std::string response_data;
|
||||
curl::requests::http_put_file put_dir{};
|
||||
@@ -136,11 +114,12 @@ auto s3_provider::create_directory_impl(const std::string &api_path,
|
||||
response_data = std::string(data.begin(), data.end());
|
||||
};
|
||||
|
||||
auto res = set_request_path(put_dir, object_name + '/');
|
||||
auto res{set_request_path(put_dir, object_name + '/')};
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
|
||||
stop_type stop_requested{false};
|
||||
long response_code{};
|
||||
if (not get_comm().make_request(put_dir, response_code, stop_requested)) {
|
||||
utils::error::raise_api_path_error(function_name, api_path,
|
||||
@@ -159,20 +138,38 @@ auto s3_provider::create_directory_impl(const std::string &api_path,
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
auto s3_provider::create_directory_paths(
|
||||
const std::string &api_path, const std::string &key) const -> api_error {
|
||||
auto s3_provider::create_directory_impl(const std::string &api_path,
|
||||
api_meta_map &meta) -> api_error {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
auto res{set_meta_key(api_path, meta)};
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
|
||||
const auto &cfg{get_s3_config()};
|
||||
auto is_encrypted{not cfg.encryption_token.empty()};
|
||||
|
||||
return create_directory_object(
|
||||
api_path,
|
||||
utils::path::create_api_path(is_encrypted ? meta[META_KEY] : api_path));
|
||||
}
|
||||
|
||||
auto s3_provider::create_directory_paths(const std::string &api_path,
|
||||
const std::string &key) const
|
||||
-> api_error {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
if (api_path == "/") {
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
const auto &cfg = get_s3_config();
|
||||
auto encryption_token = cfg.encryption_token;
|
||||
auto is_encrypted = not encryption_token.empty();
|
||||
const auto &cfg{get_s3_config()};
|
||||
auto encryption_token{cfg.encryption_token};
|
||||
auto is_encrypted{not encryption_token.empty()};
|
||||
|
||||
auto path_parts = utils::string::split(api_path, '/', false);
|
||||
auto key_parts = utils::string::split(key, '/', false);
|
||||
auto path_parts{utils::string::split(api_path, '/', false)};
|
||||
auto key_parts{utils::string::split(key, '/', false)};
|
||||
|
||||
if (is_encrypted && key_parts.size() != path_parts.size()) {
|
||||
return api_error::error;
|
||||
@@ -189,7 +186,7 @@ auto s3_provider::create_directory_paths(
|
||||
utils::path::combine(cur_path, {path_parts.at(idx)}));
|
||||
|
||||
std::string value;
|
||||
auto res = get_item_meta(cur_path, META_DIRECTORY, value);
|
||||
auto res{get_item_meta(cur_path, META_DIRECTORY, value)};
|
||||
if (res == api_error::success) {
|
||||
if (not utils::string::to_bool(value)) {
|
||||
return api_error::item_exists;
|
||||
@@ -205,42 +202,18 @@ auto s3_provider::create_directory_paths(
|
||||
}
|
||||
|
||||
if (not exists) {
|
||||
std::string response_data;
|
||||
curl::requests::http_put_file put_dir{};
|
||||
put_dir.allow_timeout = true;
|
||||
put_dir.aws_service = "aws:amz:" + cfg.region + ":s3";
|
||||
put_dir.response_handler = [&response_data](auto &&data,
|
||||
long /*response_code*/) {
|
||||
response_data = std::string(data.begin(), data.end());
|
||||
};
|
||||
|
||||
res = set_request_path(put_dir,
|
||||
(is_encrypted ? cur_key : cur_path) + '/');
|
||||
res = create_directory_object(api_path,
|
||||
(is_encrypted ? cur_key : cur_path));
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
|
||||
stop_type stop_requested{false};
|
||||
long response_code{};
|
||||
if (not get_comm().make_request(put_dir, response_code,
|
||||
stop_requested)) {
|
||||
utils::error::raise_api_path_error(
|
||||
function_name, cur_path, api_error::comm_error,
|
||||
"failed to create directory object");
|
||||
return api_error::comm_error;
|
||||
}
|
||||
|
||||
if (response_code != http_error_codes::ok) {
|
||||
utils::error::raise_api_path_error(
|
||||
function_name, api_path, response_code,
|
||||
fmt::format("failed to create directory|response|{}",
|
||||
response_data));
|
||||
return api_error::comm_error;
|
||||
}
|
||||
}
|
||||
|
||||
auto dir = create_api_file(cur_path, cur_key, 0U,
|
||||
get_last_modified(true, cur_path));
|
||||
auto dir{
|
||||
create_api_file(cur_path, cur_key, 0U,
|
||||
exists ? get_last_modified(true, cur_path)
|
||||
: utils::time::get_time_now()),
|
||||
};
|
||||
get_api_item_added()(true, dir);
|
||||
}
|
||||
|
||||
@@ -254,30 +227,7 @@ auto s3_provider::create_directory_paths(
|
||||
|
||||
auto s3_provider::create_file_extra(const std::string &api_path,
|
||||
api_meta_map &meta) -> api_error {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
const auto &cfg = get_s3_config();
|
||||
if (not cfg.encryption_token.empty()) {
|
||||
std::string encrypted_file_path;
|
||||
auto res = get_item_meta(utils::path::get_parent_api_path(api_path),
|
||||
META_KEY, encrypted_file_path);
|
||||
if (res != api_error::success) {
|
||||
utils::error::raise_api_path_error(function_name, api_path, res,
|
||||
"failed to create file");
|
||||
return res;
|
||||
}
|
||||
|
||||
data_buffer result;
|
||||
utils::encryption::encrypt_data(
|
||||
cfg.encryption_token,
|
||||
*(utils::string::split(api_path, '/', false).end() - 1U), result);
|
||||
|
||||
meta[META_KEY] = utils::path::create_api_path(
|
||||
utils::path::combine(utils::path::create_api_path(encrypted_file_path),
|
||||
{utils::collection::to_hex_string(result)}));
|
||||
}
|
||||
|
||||
return api_error::success;
|
||||
return set_meta_key(api_path, meta);
|
||||
}
|
||||
|
||||
auto s3_provider::decrypt_object_name(std::string &object_name) const
|
||||
@@ -295,24 +245,26 @@ auto s3_provider::get_directory_item_count(const std::string &api_path) const
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
try {
|
||||
const auto &cfg = get_s3_config();
|
||||
auto is_encrypted = not cfg.encryption_token.empty();
|
||||
const auto &cfg{get_s3_config()};
|
||||
auto is_encrypted{not cfg.encryption_token.empty()};
|
||||
|
||||
std::string key;
|
||||
if (is_encrypted) {
|
||||
auto res = get_item_meta(api_path, META_KEY, key);
|
||||
auto res{get_item_meta(api_path, META_KEY, key)};
|
||||
if (res != api_error::success) {
|
||||
return 0U;
|
||||
}
|
||||
}
|
||||
|
||||
auto object_name =
|
||||
auto object_name{
|
||||
api_path == "/"
|
||||
? ""
|
||||
: utils::path::create_api_path(is_encrypted ? key : api_path);
|
||||
: utils::path::create_api_path(is_encrypted ? key : api_path),
|
||||
};
|
||||
|
||||
std::string response_data{};
|
||||
long response_code{};
|
||||
auto prefix = object_name.empty() ? object_name : object_name + "/";
|
||||
auto prefix{object_name.empty() ? object_name : object_name + "/"};
|
||||
|
||||
auto grab_more{true};
|
||||
std::string token{};
|
||||
@@ -332,7 +284,7 @@ auto s3_provider::get_directory_item_count(const std::string &api_path) const
|
||||
}
|
||||
|
||||
pugi::xml_document doc;
|
||||
auto res = doc.load_string(response_data.c_str());
|
||||
auto res{doc.load_string(response_data.c_str())};
|
||||
if (res.status != pugi::xml_parse_status::status_ok) {
|
||||
return total_count;
|
||||
}
|
||||
@@ -348,8 +300,9 @@ auto s3_provider::get_directory_item_count(const std::string &api_path) const
|
||||
.as_string();
|
||||
}
|
||||
|
||||
auto node_list =
|
||||
doc.select_nodes("/ListBucketResult/CommonPrefixes/Prefix");
|
||||
auto node_list{
|
||||
doc.select_nodes("/ListBucketResult/CommonPrefixes/Prefix"),
|
||||
};
|
||||
total_count += node_list.size();
|
||||
|
||||
node_list = doc.select_nodes("/ListBucketResult/Contents");
|
||||
@@ -367,8 +320,9 @@ auto s3_provider::get_directory_item_count(const std::string &api_path) const
|
||||
return 0U;
|
||||
}
|
||||
|
||||
auto s3_provider::get_directory_items_impl(
|
||||
const std::string &api_path, directory_item_list &list) const -> api_error {
|
||||
auto s3_provider::get_directory_items_impl(const std::string &api_path,
|
||||
directory_item_list &list) const
|
||||
-> api_error {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
const auto &cfg{get_s3_config()};
|
||||
@@ -402,7 +356,7 @@ auto s3_provider::get_directory_items_impl(
|
||||
api_error::success) {
|
||||
dir_item.size = utils::string::to_uint64(size_str);
|
||||
} else {
|
||||
auto size = node.select_node("Size").node().text().as_ullong();
|
||||
auto size{node.select_node("Size").node().text().as_ullong()};
|
||||
|
||||
dir_item.size = is_encrypted ? utils::encryption::encrypting_reader::
|
||||
calculate_decrypted_size(size)
|
||||
@@ -411,8 +365,10 @@ auto s3_provider::get_directory_items_impl(
|
||||
|
||||
res = get_item_meta(dir_item.api_path, dir_item.meta);
|
||||
if (res == api_error::item_not_found) {
|
||||
auto last_modified = convert_api_date(
|
||||
node.select_node("LastModified").node().text().as_string());
|
||||
auto last_modified{
|
||||
convert_api_date(
|
||||
node.select_node("LastModified").node().text().as_string()),
|
||||
};
|
||||
|
||||
api_file file{};
|
||||
file.api_path = dir_item.api_path;
|
||||
@@ -438,7 +394,7 @@ auto s3_provider::get_directory_items_impl(
|
||||
|
||||
std::string key;
|
||||
if (is_encrypted) {
|
||||
auto res = get_item_meta(api_path, META_KEY, key);
|
||||
auto res{get_item_meta(api_path, META_KEY, key)};
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
@@ -474,7 +430,7 @@ auto s3_provider::get_directory_items_impl(
|
||||
}
|
||||
|
||||
pugi::xml_document doc;
|
||||
auto parse_res = doc.load_string(response_data.c_str());
|
||||
auto parse_res{doc.load_string(response_data.c_str())};
|
||||
if (parse_res.status != pugi::xml_parse_status::status_ok) {
|
||||
return api_error::error;
|
||||
}
|
||||
@@ -490,12 +446,15 @@ auto s3_provider::get_directory_items_impl(
|
||||
.as_string();
|
||||
}
|
||||
|
||||
auto node_list =
|
||||
doc.select_nodes("/ListBucketResult/CommonPrefixes/Prefix");
|
||||
auto node_list{
|
||||
doc.select_nodes("/ListBucketResult/CommonPrefixes/Prefix"),
|
||||
};
|
||||
for (const auto &node : node_list) {
|
||||
auto child_object_name = utils::path::create_api_path(
|
||||
utils::path::combine("/", {node.node().text().as_string()}));
|
||||
auto res = add_diretory_item(child_object_name, true, node.node());
|
||||
auto child_object_name{
|
||||
utils::path::create_api_path(
|
||||
utils::path::combine("/", {node.node().text().as_string()})),
|
||||
};
|
||||
auto res{add_diretory_item(child_object_name, true, node.node())};
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
@@ -503,13 +462,15 @@ auto s3_provider::get_directory_items_impl(
|
||||
|
||||
node_list = doc.select_nodes("/ListBucketResult/Contents");
|
||||
for (const auto &node : node_list) {
|
||||
auto child_object_name = utils::path::create_api_path(
|
||||
node.node().select_node("Key").node().text().as_string());
|
||||
auto child_object_name{
|
||||
utils::path::create_api_path(
|
||||
node.node().select_node("Key").node().text().as_string()),
|
||||
};
|
||||
if (child_object_name == utils::path::create_api_path(prefix)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
auto res = add_diretory_item(child_object_name, false, node.node());
|
||||
auto res{add_diretory_item(child_object_name, false, node.node())};
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
@@ -519,16 +480,17 @@ auto s3_provider::get_directory_items_impl(
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
auto s3_provider::get_file(const std::string &api_path,
|
||||
api_file &file) const -> api_error {
|
||||
auto s3_provider::get_file(const std::string &api_path, api_file &file) const
|
||||
-> api_error {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
try {
|
||||
bool is_encrypted{};
|
||||
std::string object_name;
|
||||
head_object_result result{};
|
||||
auto res =
|
||||
get_object_info(false, api_path, is_encrypted, object_name, result);
|
||||
auto res{
|
||||
get_object_info(false, api_path, is_encrypted, object_name, result),
|
||||
};
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
@@ -559,8 +521,8 @@ auto s3_provider::get_file(const std::string &api_path,
|
||||
return api_error::error;
|
||||
}
|
||||
|
||||
auto s3_provider::get_file_list(api_file_list &list,
|
||||
std::string &marker) const -> api_error {
|
||||
auto s3_provider::get_file_list(api_file_list &list, std::string &marker) const
|
||||
-> api_error {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
try {
|
||||
@@ -578,17 +540,19 @@ auto s3_provider::get_file_list(api_file_list &list,
|
||||
}
|
||||
|
||||
pugi::xml_document doc;
|
||||
auto result = doc.load_string(response_data.c_str());
|
||||
auto result{doc.load_string(response_data.c_str())};
|
||||
if (result.status != pugi::xml_parse_status::status_ok) {
|
||||
utils::error::raise_error(function_name, result.status,
|
||||
"failed to parse xml document");
|
||||
return api_error::comm_error;
|
||||
}
|
||||
|
||||
auto grab_more = doc.select_node("/ListBucketResult/IsTruncated")
|
||||
.node()
|
||||
.text()
|
||||
.as_bool();
|
||||
auto grab_more{
|
||||
doc.select_node("/ListBucketResult/IsTruncated")
|
||||
.node()
|
||||
.text()
|
||||
.as_bool(),
|
||||
};
|
||||
if (grab_more) {
|
||||
marker = doc.select_node("/ListBucketResult/NextContinuationToken")
|
||||
.node()
|
||||
@@ -596,24 +560,26 @@ auto s3_provider::get_file_list(api_file_list &list,
|
||||
.as_string();
|
||||
}
|
||||
|
||||
auto node_list = doc.select_nodes("/ListBucketResult/Contents");
|
||||
auto node_list{doc.select_nodes("/ListBucketResult/Contents")};
|
||||
for (const auto &node : node_list) {
|
||||
auto object_name =
|
||||
std::string{node.node().select_node("Key").node().text().as_string()};
|
||||
auto object_name{
|
||||
std::string(node.node().select_node("Key").node().text().as_string()),
|
||||
};
|
||||
if (utils::string::ends_with(object_name, "/")) {
|
||||
continue;
|
||||
}
|
||||
|
||||
auto api_path{object_name};
|
||||
auto is_encrypted = not get_s3_config().encryption_token.empty();
|
||||
auto is_encrypted{not get_s3_config().encryption_token.empty()};
|
||||
|
||||
if (is_encrypted) {
|
||||
auto res = decrypt_object_name(api_path);
|
||||
auto res{decrypt_object_name(api_path)};
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
}
|
||||
|
||||
auto size = node.node().select_node("Size").node().text().as_ullong();
|
||||
auto size{node.node().select_node("Size").node().text().as_ullong()};
|
||||
|
||||
api_file file{};
|
||||
file.api_path = utils::path::create_api_path(api_path);
|
||||
@@ -630,7 +596,7 @@ auto s3_provider::get_file_list(api_file_list &list,
|
||||
size)
|
||||
: size;
|
||||
file.key = is_encrypted ? utils::path::create_api_path(object_name) : "";
|
||||
auto res = add_if_not_found(file, file.key);
|
||||
auto res{add_if_not_found(file, file.key)};
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
@@ -646,8 +612,9 @@ auto s3_provider::get_file_list(api_file_list &list,
|
||||
return api_error::error;
|
||||
}
|
||||
|
||||
auto s3_provider::get_last_modified(
|
||||
bool directory, const std::string &api_path) const -> std::uint64_t {
|
||||
auto s3_provider::get_last_modified(bool directory,
|
||||
const std::string &api_path) const
|
||||
-> std::uint64_t {
|
||||
bool is_encrypted{};
|
||||
std::string object_name;
|
||||
head_object_result result{};
|
||||
@@ -657,18 +624,19 @@ auto s3_provider::get_last_modified(
|
||||
: utils::time::get_time_now();
|
||||
}
|
||||
|
||||
auto s3_provider::get_object_info(
|
||||
bool directory, const std::string &api_path, bool &is_encrypted,
|
||||
std::string &object_name, head_object_result &result) const -> api_error {
|
||||
auto s3_provider::get_object_info(bool directory, const std::string &api_path,
|
||||
bool &is_encrypted, std::string &object_name,
|
||||
head_object_result &result) const
|
||||
-> api_error {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
try {
|
||||
const auto &cfg = get_s3_config();
|
||||
const auto &cfg{get_s3_config()};
|
||||
is_encrypted = not cfg.encryption_token.empty();
|
||||
|
||||
std::string key;
|
||||
if (is_encrypted) {
|
||||
auto res = get_item_meta(api_path, META_KEY, key);
|
||||
auto res{get_item_meta(api_path, META_KEY, key)};
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
@@ -685,8 +653,9 @@ auto s3_provider::get_object_info(
|
||||
long /*response_code*/) {
|
||||
response_data = std::string(data.begin(), data.end());
|
||||
};
|
||||
auto res =
|
||||
set_request_path(head, directory ? object_name + '/' : object_name);
|
||||
auto res{
|
||||
set_request_path(head, directory ? object_name + '/' : object_name),
|
||||
};
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
@@ -718,10 +687,12 @@ auto s3_provider::get_object_info(
|
||||
return api_error::error;
|
||||
}
|
||||
|
||||
auto s3_provider::get_object_list(
|
||||
std::string &response_data, long &response_code,
|
||||
std::optional<std::string> delimiter, std::optional<std::string> prefix,
|
||||
std::optional<std::string> token) const -> bool {
|
||||
auto s3_provider::get_object_list(std::string &response_data,
|
||||
long &response_code,
|
||||
std::optional<std::string> delimiter,
|
||||
std::optional<std::string> prefix,
|
||||
std::optional<std::string> token) const
|
||||
-> bool {
|
||||
curl::requests::http_get get{};
|
||||
get.allow_timeout = true;
|
||||
get.aws_service = "aws:amz:" + get_s3_config().region + ":s3";
|
||||
@@ -749,8 +720,8 @@ auto s3_provider::get_total_drive_space() const -> std::uint64_t {
|
||||
return std::numeric_limits<std::int64_t>::max() / std::int64_t(2);
|
||||
}
|
||||
|
||||
auto s3_provider::is_directory(const std::string &api_path,
|
||||
bool &exists) const -> api_error {
|
||||
auto s3_provider::is_directory(const std::string &api_path, bool &exists) const
|
||||
-> api_error {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
try {
|
||||
@@ -763,8 +734,9 @@ auto s3_provider::is_directory(const std::string &api_path,
|
||||
bool is_encrypted{};
|
||||
std::string object_name;
|
||||
head_object_result result{};
|
||||
auto res =
|
||||
get_object_info(true, api_path, is_encrypted, object_name, result);
|
||||
auto res{
|
||||
get_object_info(true, api_path, is_encrypted, object_name, result),
|
||||
};
|
||||
if (res != api_error::item_not_found && res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
@@ -777,8 +749,8 @@ auto s3_provider::is_directory(const std::string &api_path,
|
||||
return api_error::error;
|
||||
}
|
||||
|
||||
auto s3_provider::is_file(const std::string &api_path,
|
||||
bool &exists) const -> api_error {
|
||||
auto s3_provider::is_file(const std::string &api_path, bool &exists) const
|
||||
-> api_error {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
try {
|
||||
@@ -790,8 +762,9 @@ auto s3_provider::is_file(const std::string &api_path,
|
||||
bool is_encrypted{};
|
||||
std::string object_name;
|
||||
head_object_result result{};
|
||||
auto res =
|
||||
get_object_info(false, api_path, is_encrypted, object_name, result);
|
||||
auto res{
|
||||
get_object_info(false, api_path, is_encrypted, object_name, result),
|
||||
};
|
||||
if (res != api_error::item_not_found && res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
@@ -815,24 +788,26 @@ auto s3_provider::read_file_bytes(const std::string &api_path, std::size_t size,
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
try {
|
||||
const auto &cfg = get_s3_config();
|
||||
auto is_encrypted = not cfg.encryption_token.empty();
|
||||
const auto &cfg{get_s3_config()};
|
||||
auto is_encrypted{not cfg.encryption_token.empty()};
|
||||
|
||||
std::string key;
|
||||
if (is_encrypted) {
|
||||
auto res = get_item_meta(api_path, META_KEY, key);
|
||||
auto res{get_item_meta(api_path, META_KEY, key)};
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
}
|
||||
|
||||
auto object_name =
|
||||
utils::path::create_api_path(is_encrypted ? key : api_path);
|
||||
auto object_name{
|
||||
utils::path::create_api_path(is_encrypted ? key : api_path),
|
||||
};
|
||||
|
||||
const auto read_bytes =
|
||||
[this, &api_path, &cfg, &object_name,
|
||||
&stop_requested](std::size_t read_size, std::size_t read_offset,
|
||||
data_buffer &read_buffer) -> api_error {
|
||||
auto res = api_error::error;
|
||||
auto res{api_error::error};
|
||||
for (std::uint32_t idx = 0U;
|
||||
not stop_requested && res != api_error::success &&
|
||||
idx < get_config().get_retry_read_count() + 1U;
|
||||
@@ -893,12 +868,12 @@ auto s3_provider::read_file_bytes(const std::string &api_path, std::size_t size,
|
||||
}
|
||||
|
||||
std::string temp;
|
||||
auto res = get_item_meta(api_path, META_SIZE, temp);
|
||||
auto res{get_item_meta(api_path, META_SIZE, temp)};
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
|
||||
auto total_size = utils::string::to_uint64(temp);
|
||||
auto total_size{utils::string::to_uint64(temp)};
|
||||
return utils::encryption::read_encrypted_range(
|
||||
{offset, offset + size - 1U},
|
||||
utils::encryption::generate_key<utils::encryption::hash_256_t>(
|
||||
@@ -924,19 +899,20 @@ auto s3_provider::remove_directory_impl(const std::string &api_path)
|
||||
-> api_error {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
const auto &cfg = get_s3_config();
|
||||
auto is_encrypted = not cfg.encryption_token.empty();
|
||||
const auto &cfg{get_s3_config()};
|
||||
auto is_encrypted{not cfg.encryption_token.empty()};
|
||||
|
||||
std::string key;
|
||||
if (is_encrypted) {
|
||||
auto res = get_item_meta(api_path, META_KEY, key);
|
||||
auto res{get_item_meta(api_path, META_KEY, key)};
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
}
|
||||
|
||||
auto object_name =
|
||||
utils::path::create_api_path(is_encrypted ? key : api_path);
|
||||
auto object_name{
|
||||
utils::path::create_api_path(is_encrypted ? key : api_path),
|
||||
};
|
||||
|
||||
std::string response_data;
|
||||
curl::requests::http_delete del_dir{};
|
||||
@@ -947,7 +923,7 @@ auto s3_provider::remove_directory_impl(const std::string &api_path)
|
||||
response_data = std::string(data.begin(), data.end());
|
||||
};
|
||||
|
||||
auto res = set_request_path(del_dir, object_name + '/');
|
||||
auto res{set_request_path(del_dir, object_name + '/')};
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
@@ -976,19 +952,20 @@ auto s3_provider::remove_directory_impl(const std::string &api_path)
|
||||
auto s3_provider::remove_file_impl(const std::string &api_path) -> api_error {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
const auto &cfg = get_s3_config();
|
||||
auto is_encrypted = not cfg.encryption_token.empty();
|
||||
const auto &cfg{get_s3_config()};
|
||||
auto is_encrypted{not cfg.encryption_token.empty()};
|
||||
|
||||
std::string key;
|
||||
if (is_encrypted) {
|
||||
auto res = get_item_meta(api_path, META_KEY, key);
|
||||
auto res{get_item_meta(api_path, META_KEY, key)};
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
}
|
||||
|
||||
auto object_name =
|
||||
utils::path::create_api_path(is_encrypted ? key : api_path);
|
||||
auto object_name{
|
||||
utils::path::create_api_path(is_encrypted ? key : api_path),
|
||||
};
|
||||
|
||||
std::string response_data;
|
||||
curl::requests::http_delete del_file{};
|
||||
@@ -998,7 +975,7 @@ auto s3_provider::remove_file_impl(const std::string &api_path) -> api_error {
|
||||
long /*response_code*/) {
|
||||
response_data = std::string(data.begin(), data.end());
|
||||
};
|
||||
auto res = set_request_path(del_file, object_name);
|
||||
auto res{set_request_path(del_file, object_name)};
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
@@ -1030,6 +1007,40 @@ auto s3_provider::rename_file(const std::string & /* from_api_path */,
|
||||
return api_error::not_implemented;
|
||||
}
|
||||
|
||||
auto s3_provider::set_meta_key(const std::string &api_path, api_meta_map &meta)
|
||||
-> api_error {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
const auto &cfg{get_s3_config()};
|
||||
auto is_encrypted{not cfg.encryption_token.empty()};
|
||||
if (not is_encrypted) {
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
std::string encrypted_parent_path;
|
||||
auto res{
|
||||
get_item_meta(utils::path::get_parent_api_path(api_path), META_KEY,
|
||||
encrypted_parent_path),
|
||||
};
|
||||
if (res != api_error::success) {
|
||||
utils::error::raise_api_path_error(function_name, api_path, res,
|
||||
"failed to create file");
|
||||
return res;
|
||||
}
|
||||
|
||||
data_buffer result;
|
||||
utils::encryption::encrypt_data(
|
||||
cfg.encryption_token,
|
||||
*(utils::string::split(api_path, '/', false).end() - 1U), result);
|
||||
|
||||
meta[META_KEY] = utils::path::create_api_path(
|
||||
utils::path::combine(utils::path::create_api_path(encrypted_parent_path),
|
||||
{
|
||||
utils::collection::to_hex_string(result),
|
||||
}));
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
auto s3_provider::start(api_item_added_callback api_item_added,
|
||||
i_file_manager *mgr) -> bool {
|
||||
event_system::instance().raise<service_started>("s3_provider");
|
||||
@@ -1051,26 +1062,27 @@ auto s3_provider::upload_file_impl(const std::string &api_path,
|
||||
|
||||
std::uint64_t file_size{};
|
||||
if (utils::file::file{source_path}.exists()) {
|
||||
auto opt_size = utils::file::file{source_path}.size();
|
||||
auto opt_size{utils::file::file{source_path}.size()};
|
||||
if (not opt_size.has_value()) {
|
||||
return api_error::comm_error;
|
||||
}
|
||||
file_size = opt_size.value();
|
||||
}
|
||||
|
||||
const auto &cfg = get_s3_config();
|
||||
auto is_encrypted = not cfg.encryption_token.empty();
|
||||
const auto &cfg{get_s3_config()};
|
||||
auto is_encrypted{not cfg.encryption_token.empty()};
|
||||
|
||||
std::string key;
|
||||
if (is_encrypted) {
|
||||
auto res = get_item_meta(api_path, META_KEY, key);
|
||||
auto res{get_item_meta(api_path, META_KEY, key)};
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
}
|
||||
|
||||
auto object_name =
|
||||
utils::path::create_api_path(is_encrypted ? key : api_path);
|
||||
auto object_name{
|
||||
utils::path::create_api_path(is_encrypted ? key : api_path),
|
||||
};
|
||||
|
||||
std::string response_data;
|
||||
curl::requests::http_put_file put_file{};
|
||||
@@ -1081,7 +1093,7 @@ auto s3_provider::upload_file_impl(const std::string &api_path,
|
||||
};
|
||||
put_file.source_path = source_path;
|
||||
|
||||
auto res = set_request_path(put_file, object_name);
|
||||
auto res{set_request_path(put_file, object_name)};
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
|
@@ -111,8 +111,8 @@ auto sia_provider::get_directory_item_count(const std::string &api_path) const
|
||||
if (object_list.contains("entries")) {
|
||||
for (const auto &entry : object_list.at("entries")) {
|
||||
try {
|
||||
auto name = entry.at("name").get<std::string>();
|
||||
auto entry_api_path = utils::path::create_api_path(name);
|
||||
auto name{entry.at("name").get<std::string>()};
|
||||
auto entry_api_path{utils::path::create_api_path(name)};
|
||||
if (utils::string::ends_with(name, "/") &&
|
||||
(entry_api_path == api_path)) {
|
||||
continue;
|
||||
@@ -148,10 +148,10 @@ auto sia_provider::get_directory_items_impl(const std::string &api_path,
|
||||
if (object_list.contains("entries")) {
|
||||
for (const auto &entry : object_list.at("entries")) {
|
||||
try {
|
||||
auto name = entry.at("name").get<std::string>();
|
||||
auto entry_api_path = utils::path::create_api_path(name);
|
||||
auto name{entry.at("name").get<std::string>()};
|
||||
auto entry_api_path{utils::path::create_api_path(name)};
|
||||
|
||||
auto directory = utils::string::ends_with(name, "/");
|
||||
auto directory{utils::string::ends_with(name, "/")};
|
||||
if (directory && (entry_api_path == api_path)) {
|
||||
continue;
|
||||
}
|
||||
@@ -164,7 +164,7 @@ auto sia_provider::get_directory_items_impl(const std::string &api_path,
|
||||
: entry["size"].get<std::uint64_t>(),
|
||||
get_last_modified(entry));
|
||||
get_api_item_added()(directory, file);
|
||||
auto res = get_item_meta(entry_api_path, meta);
|
||||
auto res{get_item_meta(entry_api_path, meta)};
|
||||
if (res != api_error::success) {
|
||||
utils::error::raise_api_path_error(function_name, entry_api_path,
|
||||
res, "failed to get item meta");
|
||||
@@ -201,18 +201,19 @@ auto sia_provider::get_file(const std::string &api_path, api_file &file) const
|
||||
|
||||
try {
|
||||
json file_data{};
|
||||
auto res = get_object_info(api_path, file_data);
|
||||
auto res{get_object_info(api_path, file_data)};
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
|
||||
auto slabs = file_data["object"]["Slabs"];
|
||||
auto size = std::accumulate(
|
||||
slabs.begin(), slabs.end(), std::uint64_t(0U),
|
||||
[](std::uint64_t total_size,
|
||||
const nlohmann::json &slab) -> std::uint64_t {
|
||||
return total_size + slab["Length"].get<std::uint64_t>();
|
||||
});
|
||||
auto slabs{file_data["object"]["Slabs"]};
|
||||
auto size{
|
||||
std::accumulate(
|
||||
slabs.begin(), slabs.end(), std::uint64_t(0U),
|
||||
[](auto &&total_size, const json &slab) -> std::uint64_t {
|
||||
return total_size + slab["Length"].get<std::uint64_t>();
|
||||
}),
|
||||
};
|
||||
|
||||
api_meta_map meta{};
|
||||
if (get_item_meta(api_path, meta) == api_error::item_not_found) {
|
||||
@@ -247,8 +248,8 @@ auto sia_provider::get_file_list(api_file_list &list,
|
||||
|
||||
if (object_list.contains("entries")) {
|
||||
for (const auto &entry : object_list.at("entries")) {
|
||||
auto name = entry.at("name").get<std::string>();
|
||||
auto entry_api_path = utils::path::create_api_path(name);
|
||||
auto name{entry.at("name").get<std::string>()};
|
||||
auto entry_api_path{utils::path::create_api_path(name)};
|
||||
|
||||
if (utils::string::ends_with(name, "/")) {
|
||||
if (entry_api_path == utils::path::create_api_path(api_path)) {
|
||||
@@ -258,12 +259,14 @@ auto sia_provider::get_file_list(api_file_list &list,
|
||||
api_meta_map meta{};
|
||||
if (get_item_meta(entry_api_path, meta) ==
|
||||
api_error::item_not_found) {
|
||||
auto dir = create_api_file(entry_api_path, "", 0U,
|
||||
get_last_modified(entry));
|
||||
auto dir{
|
||||
create_api_file(entry_api_path, "", 0U,
|
||||
get_last_modified(entry)),
|
||||
};
|
||||
get_api_item_added()(true, dir);
|
||||
}
|
||||
|
||||
auto res = get_files_in_dir(entry_api_path);
|
||||
auto res{get_files_in_dir(entry_api_path)};
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
@@ -480,7 +483,7 @@ auto sia_provider::is_file(const std::string &api_path, bool &exists) const
|
||||
}
|
||||
|
||||
json file_data{};
|
||||
auto res = get_object_info(api_path, file_data);
|
||||
auto res{get_object_info(api_path, file_data)};
|
||||
if (res == api_error::item_not_found) {
|
||||
return api_error::success;
|
||||
}
|
||||
|
@@ -35,7 +35,7 @@ void polling::frequency_thread(
|
||||
unique_mutex_lock lock(mutex_);
|
||||
auto futures = std::accumulate(
|
||||
items_.begin(), items_.end(), std::deque<tasks::task_ptr>{},
|
||||
[this, &freq](auto &&list, auto &&item) {
|
||||
[this, &freq](auto &&list, auto &&item) -> auto {
|
||||
if (item.second.freq != freq) {
|
||||
return list;
|
||||
}
|
||||
|
@@ -30,12 +30,16 @@ TYPED_TEST(file_db_test, can_add_and_remove_directory) {
|
||||
|
||||
EXPECT_EQ(api_error::success, this->file_db->add_directory("/", "c:\\test"));
|
||||
|
||||
auto list = this->file_db->get_item_list();
|
||||
stop_type stop_requested{false};
|
||||
auto list = this->file_db->get_item_list(stop_requested);
|
||||
EXPECT_EQ(1U, list.size());
|
||||
EXPECT_STREQ("/", list.at(0U).api_path.c_str());
|
||||
EXPECT_TRUE(list.at(0U).directory);
|
||||
EXPECT_STREQ("c:\\test", list.at(0U).source_path.c_str());
|
||||
|
||||
EXPECT_EQ(api_error::success, this->file_db->remove_item("/"));
|
||||
|
||||
list = this->file_db->get_item_list();
|
||||
list = this->file_db->get_item_list(stop_requested);
|
||||
EXPECT_EQ(0U, list.size());
|
||||
}
|
||||
|
||||
@@ -49,12 +53,16 @@ TYPED_TEST(file_db_test, can_add_and_remove_file) {
|
||||
"c:\\test\\file.txt",
|
||||
}));
|
||||
|
||||
auto list = this->file_db->get_item_list();
|
||||
stop_type stop_requested{false};
|
||||
auto list = this->file_db->get_item_list(stop_requested);
|
||||
EXPECT_EQ(1U, list.size());
|
||||
EXPECT_STREQ("/file", list.at(0U).api_path.c_str());
|
||||
EXPECT_FALSE(list.at(0U).directory);
|
||||
EXPECT_STREQ("c:\\test\\file.txt", list.at(0U).source_path.c_str());
|
||||
|
||||
EXPECT_EQ(api_error::success, this->file_db->remove_item("/file"));
|
||||
|
||||
list = this->file_db->get_item_list();
|
||||
list = this->file_db->get_item_list(stop_requested);
|
||||
EXPECT_EQ(0U, list.size());
|
||||
}
|
||||
|
||||
|
Reference in New Issue
Block a user