diff --git a/repertory/librepertory/include/file_manager/file_manager.hpp b/repertory/librepertory/include/file_manager/file_manager.hpp index 7469d57f..9258bc85 100644 --- a/repertory/librepertory/include/file_manager/file_manager.hpp +++ b/repertory/librepertory/include/file_manager/file_manager.hpp @@ -57,8 +57,8 @@ public: open_file_base(const open_file_base &) noexcept = delete; open_file_base(open_file_base &&) noexcept = delete; auto operator=(open_file_base &&) noexcept -> open_file_base & = delete; - auto - operator=(const open_file_base &) noexcept -> open_file_base & = delete; + auto operator=(const open_file_base &) noexcept + -> open_file_base & = delete; public: class download final { @@ -168,17 +168,17 @@ public: [[nodiscard]] auto get_filesystem_item() const -> filesystem_item override; - [[nodiscard]] auto - get_handles() const -> std::vector override; + [[nodiscard]] auto get_handles() const + -> std::vector override; - [[nodiscard]] auto - get_open_data() -> std::map & override; + [[nodiscard]] auto get_open_data() + -> std::map & override; [[nodiscard]] auto get_open_data() const -> const std::map & override; - [[nodiscard]] auto - get_open_data(std::uint64_t handle) -> open_file_data & override; + [[nodiscard]] auto get_open_data(std::uint64_t handle) + -> open_file_data & override; [[nodiscard]] auto get_open_data(std::uint64_t handle) const -> const open_file_data & override; @@ -267,8 +267,8 @@ public: public: auto close() -> bool override; - [[nodiscard]] auto - get_read_state() const -> boost::dynamic_bitset<> override; + [[nodiscard]] auto get_read_state() const + -> boost::dynamic_bitset<> override; [[nodiscard]] auto get_read_state(std::size_t chunk) const -> bool override; @@ -276,20 +276,20 @@ public: auto is_write_supported() const -> bool override { return true; } - [[nodiscard]] auto - native_operation(native_operation_callback callback) -> api_error override; + [[nodiscard]] auto native_operation(native_operation_callback callback) + -> api_error override; - [[nodiscard]] auto - native_operation(std::uint64_t new_file_size, - native_operation_callback callback) -> api_error override; + [[nodiscard]] auto native_operation(std::uint64_t new_file_size, + native_operation_callback callback) + -> api_error override; void remove(std::uint64_t handle) override; [[nodiscard]] auto read(std::size_t read_size, std::uint64_t read_offset, data_buffer &data) -> api_error override; - [[nodiscard]] auto - resize(std::uint64_t new_file_size) -> api_error override; + [[nodiscard]] auto resize(std::uint64_t new_file_size) + -> api_error override; [[nodiscard]] auto write(std::uint64_t write_offset, const data_buffer &data, @@ -356,8 +356,8 @@ public: return last_chunk_; } - [[nodiscard]] auto - get_read_state() const -> boost::dynamic_bitset<> override; + [[nodiscard]] auto get_read_state() const + -> boost::dynamic_bitset<> override; [[nodiscard]] auto get_read_state(std::size_t chunk) const -> bool override; @@ -369,12 +369,12 @@ public: auto is_write_supported() const -> bool override { return false; } - [[nodiscard]] auto - native_operation(native_operation_callback callback) -> api_error override; + [[nodiscard]] auto native_operation(native_operation_callback callback) + -> api_error override; - [[nodiscard]] auto - native_operation(std::uint64_t, - native_operation_callback) -> api_error override { + [[nodiscard]] auto native_operation(std::uint64_t, + native_operation_callback) + -> api_error override { return api_error::not_supported; } @@ -391,8 +391,8 @@ public: void set_api_path(const std::string &api_path) override; - [[nodiscard]] auto write(std::uint64_t, const data_buffer &, - std::size_t &) -> api_error override { + [[nodiscard]] auto write(std::uint64_t, const data_buffer &, std::size_t &) + -> api_error override { return api_error::not_supported; } }; @@ -458,8 +458,7 @@ private: i_provider &provider_; private: - utils::db::sqlite::db3_t db_; - std::uint64_t next_handle_{0U}; + std::atomic next_handle_{0U}; mutable std::recursive_mutex open_file_mtx_; std::unordered_map> open_file_lookup_; @@ -470,6 +469,8 @@ private: std::unique_ptr upload_thread_; private: + [[nodiscard]] auto create_db() const -> utils::db::sqlite::db3_t; + void close_timed_out_files(); auto get_open_file_by_handle(std::uint64_t handle) const @@ -483,12 +484,22 @@ private: std::shared_ptr closeable_file) -> api_error; void queue_upload(const std::string &api_path, const std::string &source_path, - bool no_lock); + bool no_lock, sqlite3 *db); - void remove_upload(const std::string &api_path, bool no_lock); + void remove_resume(const std::string &api_path, + const std::string &source_path, sqlite3 *db); + void remove_upload(const std::string &api_path, bool no_lock, sqlite3 *db); + + [[nodiscard]] auto rename_directory(const std::string &from_api_path, + const std::string &to_api_path, + sqlite3 *db) -> api_error; + + [[nodiscard]] auto rename_file(const std::string &from_api_path, + const std::string &to_api_path, bool overwrite, + sqlite3 *db) -> api_error; void swap_renamed_items(std::string from_api_path, std::string to_api_path, - bool directory); + bool directory, sqlite3 *db); void upload_completed(const file_upload_completed &evt); @@ -498,7 +509,8 @@ public: [[nodiscard]] auto get_next_handle() -> std::uint64_t; auto handle_file_rename(const std::string &from_api_path, - const std::string &to_api_path) -> api_error; + const std::string &to_api_path, sqlite3 *db) + -> api_error; void queue_upload(const i_open_file &file) override; @@ -537,8 +549,8 @@ public: [[nodiscard]] auto has_no_open_file_handles() const -> bool override; - [[nodiscard]] auto - is_processing(const std::string &api_path) const -> bool override; + [[nodiscard]] auto is_processing(const std::string &api_path) const + -> bool override; #if defined(PROJECT_TESTING) [[nodiscard]] auto open(std::shared_ptr of, @@ -551,13 +563,13 @@ public: [[nodiscard]] auto remove_file(const std::string &api_path) -> api_error; - [[nodiscard]] auto - rename_directory(const std::string &from_api_path, - const std::string &to_api_path) -> api_error; + [[nodiscard]] auto rename_directory(const std::string &from_api_path, + const std::string &to_api_path) + -> api_error; [[nodiscard]] auto rename_file(const std::string &from_api_path, - const std::string &to_api_path, - bool overwrite) -> api_error; + const std::string &to_api_path, bool overwrite) + -> api_error; void start(); diff --git a/repertory/librepertory/src/file_manager/file_manager.cpp b/repertory/librepertory/src/file_manager/file_manager.cpp index e204de1e..f2667140 100644 --- a/repertory/librepertory/src/file_manager/file_manager.cpp +++ b/repertory/librepertory/src/file_manager/file_manager.cpp @@ -25,7 +25,6 @@ #include "file_manager/events.hpp" #include "providers/i_provider.hpp" #include "types/repertory.hpp" -#include "types/startup_exception.hpp" #include "utils/common.hpp" #include "utils/db/sqlite/db_common.hpp" #include "utils/db/sqlite/db_delete.hpp" @@ -40,8 +39,8 @@ #include "utils/time.hpp" namespace { -[[nodiscard]] auto -create_resume_entry(const repertory::i_open_file &file) -> json { +[[nodiscard]] auto create_resume_entry(const repertory::i_open_file &file) + -> json { return { {"chunk_size", file.get_chunk_size()}, {"path", file.get_api_path()}, @@ -104,32 +103,6 @@ namespace repertory { file_manager::file_manager(app_config &config, i_provider &provider) : config_(config), provider_(provider) { if (not provider_.is_direct_only()) { - auto db_path = - utils::path::combine(config.get_data_directory(), {"file_manager.db"}); - - sqlite3 *db3{nullptr}; - auto res = - sqlite3_open_v2(db_path.c_str(), &db3, - SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, nullptr); - if (res != SQLITE_OK) { - throw startup_exception("failed to open db|" + db_path + '|' + - std::to_string(res) + '|' + sqlite3_errstr(res)); - } - db_ = utils::db::sqlite::db3_t{ - db3, - utils::db::sqlite::sqlite3_deleter(), - }; - - for (auto &&create_item : sql_create_tables) { - std::string err; - if (not utils::db::sqlite::execute_sql(*db_, create_item.second, err)) { - db_.reset(); - throw startup_exception(err); - } - } - - utils::db::sqlite::set_journal_mode(*db_); - E_SUBSCRIBE_EXACT(file_upload_completed, [this](const file_upload_completed &completed) { this->upload_completed(completed); @@ -139,7 +112,6 @@ file_manager::file_manager(app_config &config, i_provider &provider) file_manager::~file_manager() { stop(); - db_.reset(); E_CONSUMER_RELEASE(); } @@ -194,6 +166,38 @@ void file_manager::close_timed_out_files() { closeable_list.clear(); } +auto file_manager::create_db() const -> utils::db::sqlite::db3_t { + auto db_path = + utils::path::combine(config_.get_data_directory(), {"file_manager.db"}); + + sqlite3 *db3{nullptr}; + auto db_res = + sqlite3_open_v2(db_path.c_str(), &db3, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, nullptr); + if (db_res != SQLITE_OK) { + throw std::runtime_error("failed to open db|" + db_path + '|' + + std::to_string(db_res) + '|' + + sqlite3_errstr(db_res)); + } + + auto db = utils::db::sqlite::db3_t{ + db3, + utils::db::sqlite::sqlite3_deleter(), + }; + + for (auto &&create_item : sql_create_tables) { + std::string err; + if (not utils::db::sqlite::execute_sql(*db, create_item.second, err)) { + db.reset(); + throw std::runtime_error(err); + } + } + + utils::db::sqlite::set_journal_mode(*db); + + return db; +} + auto file_manager::create(const std::string &api_path, api_meta_map &meta, open_file_data ofd, std::uint64_t &handle, std::shared_ptr &file) -> api_error { @@ -271,7 +275,7 @@ auto file_manager::get_directory_items(const std::string &api_path) const auto file_manager::get_next_handle() -> std::uint64_t { if (++next_handle_ == 0U) { - next_handle_++; + ++next_handle_; } return next_handle_; @@ -349,23 +353,27 @@ auto file_manager::get_open_handle_count() const -> std::size_t { auto file_manager::get_stored_downloads() const -> std::vector { REPERTORY_USES_FUNCTION_NAME(); - std::vector ret; - if (not provider_.is_direct_only()) { - auto result = utils::db::sqlite::db_select{*db_.get(), resume_table}.go(); - while (result.has_row()) { - try { - std::optional row; - if (not result.get_row(row)) { - continue; - } - if (not row.has_value()) { - continue; - } + if (provider_.is_direct_only()) { + return {}; + } - ret.push_back(row.value().get_column("data").get_value_as_json()); - } catch (const std::exception &ex) { - utils::error::raise_error(function_name, ex, "query error"); + auto db = create_db(); + + std::vector ret; + auto result = utils::db::sqlite::db_select{*db.get(), resume_table}.go(); + while (result.has_row()) { + try { + std::optional row; + if (not result.get_row(row)) { + continue; } + if (not row.has_value()) { + continue; + } + + ret.push_back(row.value().get_column("data").get_value_as_json()); + } catch (const std::exception &ex) { + utils::error::raise_error(function_name, ex, "query error"); } } @@ -373,8 +381,8 @@ auto file_manager::get_stored_downloads() const -> std::vector { } auto file_manager::handle_file_rename(const std::string &from_api_path, - const std::string &to_api_path) - -> api_error { + const std::string &to_api_path, + sqlite3 *db) -> api_error { std::string source_path{}; auto file_iter = open_file_lookup_.find(from_api_path); if (file_iter != open_file_lookup_.end()) { @@ -387,7 +395,7 @@ auto file_manager::handle_file_rename(const std::string &from_api_path, source_path = upload_lookup_.at(from_api_path)->get_source_path(); } } else { - auto result = utils::db::sqlite::db_select{*db_.get(), upload_table} + auto result = utils::db::sqlite::db_select{*db, upload_table} .column("source_path") .where("api_path") .equals(from_api_path) @@ -399,22 +407,22 @@ auto file_manager::handle_file_rename(const std::string &from_api_path, } } - remove_upload(from_api_path); + remove_upload(from_api_path, true, db); auto ret = provider_.rename_file(from_api_path, to_api_path); if (ret != api_error::success) { - queue_upload(from_api_path, source_path, false); + queue_upload(from_api_path, source_path, false, db); return ret; } - swap_renamed_items(from_api_path, to_api_path, false); + swap_renamed_items(from_api_path, to_api_path, false, db); ret = source_path.empty() ? api_error::success : provider_.set_item_meta(to_api_path, META_SOURCE, source_path); if (should_upload) { - queue_upload(to_api_path, source_path, false); + queue_upload(to_api_path, source_path, false, db); } return ret; @@ -435,7 +443,9 @@ auto file_manager::is_processing(const std::string &api_path) const -> bool { } upload_lock.unlock(); - utils::db::sqlite::db_select query{*db_.get(), upload_table}; + auto db = create_db(); + + utils::db::sqlite::db_select query{*db.get(), upload_table}; if (query.where("api_path").equals(api_path).go().has_row()) { return true; }; @@ -455,10 +465,11 @@ auto file_manager::open(const std::string &api_path, bool directory, return open(api_path, directory, ofd, handle, file, nullptr); } -auto file_manager::open( - const std::string &api_path, bool directory, const open_file_data &ofd, - std::uint64_t &handle, std::shared_ptr &file, - std::shared_ptr closeable_file) -> api_error { +auto file_manager::open(const std::string &api_path, bool directory, + const open_file_data &ofd, std::uint64_t &handle, + std::shared_ptr &file, + std::shared_ptr closeable_file) + -> api_error { const auto create_and_add_handle = [&](std::shared_ptr cur_file) { handle = get_next_handle(); @@ -502,11 +513,14 @@ auto file_manager::open( } void file_manager::queue_upload(const i_open_file &file) { - return queue_upload(file.get_api_path(), file.get_source_path(), false); + auto db = create_db(); + return queue_upload(file.get_api_path(), file.get_source_path(), false, + db.get()); } void file_manager::queue_upload(const std::string &api_path, - const std::string &source_path, bool no_lock) { + const std::string &source_path, bool no_lock, + sqlite3 *db) { if (provider_.is_direct_only()) { return; } @@ -515,10 +529,10 @@ void file_manager::queue_upload(const std::string &api_path, if (not no_lock) { lock = std::make_unique(upload_mtx_); } - remove_upload(api_path, true); + remove_upload(api_path, true, db); auto result = - utils::db::sqlite::db_insert{*db_.get(), upload_table} + utils::db::sqlite::db_insert{*db, upload_table} .or_replace() .column_value("api_path", api_path) .column_value("date_time", @@ -526,7 +540,7 @@ void file_manager::queue_upload(const std::string &api_path, .column_value("source_path", source_path) .go(); if (result.ok()) { - remove_resume(api_path, source_path); + remove_resume(api_path, source_path, db); event_system::instance().raise(api_path, source_path); } else { event_system::instance().raise( @@ -552,9 +566,10 @@ auto file_manager::remove_file(const std::string &api_path) -> api_error { close_all(api_path); - remove_upload(api_path); + auto db = create_db(); + remove_upload(api_path, true, db.get()); - auto result = utils::db::sqlite::db_delete{*db_.get(), resume_table} + auto result = utils::db::sqlite::db_delete{*db.get(), resume_table} .where("api_path") .equals(api_path) .go(); @@ -580,7 +595,13 @@ auto file_manager::remove_file(const std::string &api_path) -> api_error { void file_manager::remove_resume(const std::string &api_path, const std::string &source_path) { - auto result = utils::db::sqlite::db_delete{*db_.get(), resume_table} + auto db = create_db(); + return remove_resume(api_path, source_path, db.get()); +} + +void file_manager::remove_resume(const std::string &api_path, + const std::string &source_path, sqlite3 *db) { + auto result = utils::db::sqlite::db_delete{*db, resume_table} .where("api_path") .equals(api_path) .go(); @@ -591,10 +612,12 @@ void file_manager::remove_resume(const std::string &api_path, } void file_manager::remove_upload(const std::string &api_path) { - remove_upload(api_path, false); + auto db = create_db(); + remove_upload(api_path, false, db.get()); } -void file_manager::remove_upload(const std::string &api_path, bool no_lock) { +void file_manager::remove_upload(const std::string &api_path, bool no_lock, + sqlite3 *db) { REPERTORY_USES_FUNCTION_NAME(); if (provider_.is_direct_only()) { @@ -606,7 +629,7 @@ void file_manager::remove_upload(const std::string &api_path, bool no_lock) { lock = std::make_unique(upload_mtx_); } - auto result = utils::db::sqlite::db_delete{*db_.get(), upload_table} + auto result = utils::db::sqlite::db_delete{*db, upload_table} .where("api_path") .equals(api_path) .go(); @@ -616,7 +639,7 @@ void file_manager::remove_upload(const std::string &api_path, bool no_lock) { "failed to remove from upload table"); } - result = utils::db::sqlite::db_delete{*db_.get(), upload_active_table} + result = utils::db::sqlite::db_delete{*db, upload_active_table} .where("api_path") .equals(api_path) .go(); @@ -643,6 +666,13 @@ void file_manager::remove_upload(const std::string &api_path, bool no_lock) { auto file_manager::rename_directory(const std::string &from_api_path, const std::string &to_api_path) -> api_error { + auto db = create_db(); + return rename_directory(from_api_path, to_api_path, db.get()); +} + +auto file_manager::rename_directory(const std::string &from_api_path, + const std::string &to_api_path, sqlite3 *db) + -> api_error { if (not provider_.is_rename_supported()) { return api_error::not_implemented; } @@ -698,8 +728,9 @@ auto file_manager::rename_directory(const std::string &from_api_path, auto old_api_path = api_path; auto new_api_path = utils::path::create_api_path(utils::path::combine( to_api_path, {old_api_path.substr(from_api_path.size())})); - res = list[i].directory ? rename_directory(old_api_path, new_api_path) - : rename_file(old_api_path, new_api_path, false); + res = list[i].directory + ? rename_directory(old_api_path, new_api_path, db) + : rename_file(old_api_path, new_api_path, false, db); } } @@ -712,13 +743,20 @@ auto file_manager::rename_directory(const std::string &from_api_path, return res; } - swap_renamed_items(from_api_path, to_api_path, true); + swap_renamed_items(from_api_path, to_api_path, true, db); return api_error::success; } auto file_manager::rename_file(const std::string &from_api_path, - const std::string &to_api_path, - bool overwrite) -> api_error { + const std::string &to_api_path, bool overwrite) + -> api_error { + auto db = create_db(); + return rename_file(from_api_path, to_api_path, overwrite, db.get()); +} + +auto file_manager::rename_file(const std::string &from_api_path, + const std::string &to_api_path, bool overwrite, + sqlite3 *db) -> api_error { if (not provider_.is_rename_supported()) { return api_error::not_implemented; } @@ -786,12 +824,18 @@ auto file_manager::rename_file(const std::string &from_api_path, } } - return handle_file_rename(from_api_path, to_api_path); + return handle_file_rename(from_api_path, to_api_path, db); } void file_manager::start() { REPERTORY_USES_FUNCTION_NAME(); + if (upload_thread_) { + return; + } + + stop_requested_ = false; + polling::instance().set_callback( {"timed_out_close", polling::frequency::second, [this]() { this->close_timed_out_files(); }}); @@ -801,106 +845,104 @@ void file_manager::start() { return; } - if (not upload_thread_) { - stop_requested_ = false; + auto db = create_db(); + + struct active_item final { + std::string api_path; + std::string source_path; + }; + + std::vector active_items{}; + + auto result = + utils::db::sqlite::db_select{*db.get(), upload_active_table}.go(); + while (result.has_row()) { + try { + std::optional row; + if (result.get_row(row) && row.has_value()) { + active_items.emplace_back(active_item{ + row->get_column("api_path").get_value(), + row->get_column("source_path").get_value(), + }); + } + } catch (const std::exception &ex) { + utils::error::raise_error(function_name, ex, "query error"); + } + } + + for (auto &&active_item : active_items) { + queue_upload(active_item.api_path, active_item.source_path, false, + db.get()); + } + active_items.clear(); + + result = utils::db::sqlite::db_select{*db.get(), resume_table}.go(); + if (not result.ok()) { + return; + } + + while (result.has_row()) { + try { + std::optional row; + if (not(result.get_row(row) && row.has_value())) { + return; + } + + auto resume_entry = row.value().get_column("data").get_value_as_json(); - struct active_item final { std::string api_path; std::string source_path; - }; + std::size_t chunk_size{}; + boost::dynamic_bitset<> read_state; + restore_resume_entry(resume_entry, api_path, chunk_size, read_state, + source_path); - std::vector active_items{}; - - auto result = - utils::db::sqlite::db_select{*db_.get(), upload_active_table}.go(); - while (result.has_row()) { - try { - std::optional row; - if (result.get_row(row) && row.has_value()) { - active_items.emplace_back(active_item{ - row->get_column("api_path").get_value(), - row->get_column("source_path").get_value(), - }); - } - } catch (const std::exception &ex) { - utils::error::raise_error(function_name, ex, "query error"); - std::abort(); + filesystem_item fsi{}; + auto res = provider_.get_filesystem_item(api_path, false, fsi); + if (res != api_error::success) { + event_system::instance().raise( + api_path, source_path, + "failed to get filesystem item|" + api_error_to_string(res)); + continue; } - } - for (auto &&active_item : active_items) { - queue_upload(active_item.api_path, active_item.source_path, false); - } - active_items.clear(); - - result = utils::db::sqlite::db_select{*db_.get(), resume_table}.go(); - if (not result.ok()) { - return; - } - - while (result.has_row()) { - try { - std::optional row; - if (not(result.get_row(row) && row.has_value())) { - return; - } - - auto resume_entry = row.value().get_column("data").get_value_as_json(); - - std::string api_path; - std::string source_path; - std::size_t chunk_size{}; - boost::dynamic_bitset<> read_state; - restore_resume_entry(resume_entry, api_path, chunk_size, read_state, - source_path); - - filesystem_item fsi{}; - auto res = provider_.get_filesystem_item(api_path, false, fsi); - if (res != api_error::success) { - event_system::instance().raise( - api_path, source_path, - "failed to get filesystem item|" + api_error_to_string(res)); - continue; - } - - if (source_path != fsi.source_path) { - event_system::instance().raise( - fsi.api_path, fsi.source_path, - "source path mismatch|expected|" + source_path + "|actual|" + - fsi.source_path); - continue; - } - - auto opt_size = utils::file::file{fsi.source_path}.size(); - if (not opt_size.has_value()) { - event_system::instance().raise( - fsi.api_path, fsi.source_path, - "failed to get file size: " + - std::to_string(utils::get_last_error_code())); - continue; - } - - auto file_size{opt_size.value()}; - if (file_size != fsi.size) { - event_system::instance().raise( - fsi.api_path, fsi.source_path, - "file size mismatch|expected|" + std::to_string(fsi.size) + - "|actual|" + std::to_string(file_size)); - continue; - } - - auto closeable_file = std::make_shared( - chunk_size, - config_.get_enable_chunk_download_timeout() - ? config_.get_chunk_downloader_timeout_secs() - : 0U, - fsi, provider_, read_state, *this); - open_file_lookup_[api_path] = closeable_file; - event_system::instance().raise(fsi.api_path, - fsi.source_path); - } catch (const std::exception &ex) { - utils::error::raise_error(function_name, ex, "query error"); + if (source_path != fsi.source_path) { + event_system::instance().raise( + fsi.api_path, fsi.source_path, + "source path mismatch|expected|" + source_path + "|actual|" + + fsi.source_path); + continue; } + + auto opt_size = utils::file::file{fsi.source_path}.size(); + if (not opt_size.has_value()) { + event_system::instance().raise( + fsi.api_path, fsi.source_path, + "failed to get file size: " + + std::to_string(utils::get_last_error_code())); + continue; + } + + auto file_size{opt_size.value()}; + if (file_size != fsi.size) { + event_system::instance().raise( + fsi.api_path, fsi.source_path, + "file size mismatch|expected|" + std::to_string(fsi.size) + + "|actual|" + std::to_string(file_size)); + continue; + } + + auto closeable_file = std::make_shared( + chunk_size, + config_.get_enable_chunk_download_timeout() + ? config_.get_chunk_downloader_timeout_secs() + : 0U, + fsi, provider_, read_state, *this); + open_file_lookup_[api_path] = closeable_file; + event_system::instance().raise(fsi.api_path, + fsi.source_path); + } catch (const std::exception &ex) { + utils::error::raise_error(function_name, ex, "query error"); } } @@ -909,40 +951,43 @@ void file_manager::start() { } void file_manager::stop() { - if (not stop_requested_) { - event_system::instance().raise("file_manager"); - polling::instance().remove_callback("timed_out_close"); - stop_requested_ = true; - - unique_mutex_lock upload_lock(upload_mtx_); - upload_notify_.notify_all(); - upload_lock.unlock(); - - if (upload_thread_) { - upload_thread_->join(); - upload_thread_.reset(); - } - - open_file_lookup_.clear(); - - upload_lock.lock(); - for (auto &&item : upload_lookup_) { - item.second->stop(); - } - upload_notify_.notify_all(); - upload_lock.unlock(); - - while (not upload_lookup_.empty()) { - upload_lock.lock(); - if (not upload_lookup_.empty()) { - upload_notify_.wait_for(upload_lock, 1ms); - } - upload_notify_.notify_all(); - upload_lock.unlock(); - } - - event_system::instance().raise("file_manager"); + if (stop_requested_) { + return; } + + event_system::instance().raise("file_manager"); + polling::instance().remove_callback("timed_out_close"); + stop_requested_ = true; + + unique_mutex_lock upload_lock(upload_mtx_); + upload_notify_.notify_all(); + upload_lock.unlock(); + + if (upload_thread_) { + upload_thread_->join(); + } + + open_file_lookup_.clear(); + + upload_lock.lock(); + for (auto &&item : upload_lookup_) { + item.second->stop(); + } + upload_notify_.notify_all(); + upload_lock.unlock(); + + while (not upload_lookup_.empty()) { + upload_lock.lock(); + if (not upload_lookup_.empty()) { + upload_notify_.wait_for(upload_lock, 1ms); + } + upload_notify_.notify_all(); + upload_lock.unlock(); + } + + upload_thread_.reset(); + + event_system::instance().raise("file_manager"); } void file_manager::store_resume(const i_open_file &file) { @@ -950,7 +995,8 @@ void file_manager::store_resume(const i_open_file &file) { return; } - auto result = utils::db::sqlite::db_insert{*db_.get(), resume_table} + auto db = create_db(); + auto result = utils::db::sqlite::db_insert{*db.get(), resume_table} .or_replace() .column_value("api_path", file.get_api_path()) .column_value("data", create_resume_entry(file).dump()) @@ -968,7 +1014,8 @@ void file_manager::store_resume(const i_open_file &file) { } void file_manager::swap_renamed_items(std::string from_api_path, - std::string to_api_path, bool directory) { + std::string to_api_path, bool directory, + sqlite3 *db) { REPERTORY_USES_FUNCTION_NAME(); auto file_iter = open_file_lookup_.find(from_api_path); @@ -983,7 +1030,7 @@ void file_manager::swap_renamed_items(std::string from_api_path, return; } - auto result = utils::db::sqlite::db_update{*db_.get(), resume_table} + auto result = utils::db::sqlite::db_update{*db, resume_table} .column_value("api_path", to_api_path) .where("api_path") .equals(from_api_path) @@ -1001,13 +1048,14 @@ void file_manager::upload_completed(const file_upload_completed &evt) { unique_mutex_lock upload_lock(upload_mtx_); if (not utils::string::to_bool(evt.get_cancelled().get())) { + auto db = create_db(); + auto err = api_error_from_string(evt.get_result().get()); if (err == api_error::success) { - auto result = - utils::db::sqlite::db_delete{*db_.get(), upload_active_table} - .where("api_path") - .equals(evt.get_api_path().get()) - .go(); + auto result = utils::db::sqlite::db_delete{*db.get(), upload_active_table} + .where("api_path") + .equals(evt.get_api_path().get()) + .go(); if (not result.ok()) { utils::error::raise_api_path_error( function_name, evt.get_api_path().get(), @@ -1023,12 +1071,12 @@ void file_manager::upload_completed(const file_upload_completed &evt) { not utils::file::file(evt.get_source().get()).exists()) { event_system::instance().raise( evt.get_api_path(), evt.get_source()); - remove_upload(evt.get_api_path(), true); + remove_upload(evt.get_api_path(), true, db.get()); } else { event_system::instance().raise( evt.get_api_path(), evt.get_source(), err); - queue_upload(evt.get_api_path(), evt.get_source(), true); + queue_upload(evt.get_api_path(), evt.get_source(), true, db.get()); upload_notify_.wait_for(upload_lock, 5s); } } @@ -1040,6 +1088,8 @@ void file_manager::upload_completed(const file_upload_completed &evt) { void file_manager::upload_handler() { REPERTORY_USES_FUNCTION_NAME(); + auto db = create_db(); + while (not stop_requested_) { auto should_wait{true}; unique_mutex_lock upload_lock(upload_mtx_); @@ -1049,7 +1099,7 @@ void file_manager::upload_handler() { } if (upload_lookup_.size() < config_.get_max_upload_count()) { - auto result = utils::db::sqlite::db_select{*db_.get(), upload_table} + auto result = utils::db::sqlite::db_select{*db.get(), upload_table} .order_by("api_path", true) .limit(1) .go(); @@ -1067,7 +1117,7 @@ void file_manager::upload_handler() { should_wait = false; event_system::instance().raise(api_path, source_path); - remove_upload(api_path, true); + remove_upload(api_path, true, db.get()); } break; case api_error::success: { @@ -1075,14 +1125,13 @@ void file_manager::upload_handler() { upload_lookup_[fsi.api_path] = std::make_unique(fsi, provider_); - auto del_res = - utils::db::sqlite::db_delete{*db_.get(), upload_table} - .where("api_path") - .equals(api_path) - .go(); + auto del_res = utils::db::sqlite::db_delete{*db.get(), upload_table} + .where("api_path") + .equals(api_path) + .go(); if (del_res.ok()) { auto ins_res = - utils::db::sqlite::db_insert{*db_.get(), upload_active_table} + utils::db::sqlite::db_insert{*db.get(), upload_active_table} .column_value("api_path", api_path) .column_value("source_path", source_path) .go(); @@ -1097,7 +1146,7 @@ void file_manager::upload_handler() { default: { event_system::instance().raise(api_path, source_path, res); - queue_upload(api_path, source_path, true); + queue_upload(api_path, source_path, true, db.get()); } break; } } diff --git a/repertory/repertory_test/src/file_manager_test.cpp b/repertory/repertory_test/src/file_manager_test.cpp index 882e6acf..20a0dc6c 100644 --- a/repertory/repertory_test/src/file_manager_test.cpp +++ b/repertory/repertory_test/src/file_manager_test.cpp @@ -40,11 +40,6 @@ #include "utils/time.hpp" #include "utils/utils.hpp" -namespace { -const auto file_manager_dir = repertory::utils::path::combine( - repertory::test::get_test_output_dir(), {"file_manager_test"}); -} - namespace repertory { auto file_manager::open(std::shared_ptr of, const open_file_data &ofd, std::uint64_t &handle, @@ -52,802 +47,225 @@ auto file_manager::open(std::shared_ptr of, return open(of->get_api_path(), of->is_directory(), ofd, handle, f, of); } -TEST(file_manager, can_start_and_stop) { - { - console_consumer c; +class file_manager_test : public ::testing::Test { +public: + console_consumer c; + std::unique_ptr cfg; + mock_provider mp; + static std::atomic inst; + std::string file_manager_dir; + +protected: + void SetUp() override { event_system::instance().start(); - app_config cfg(provider_type::sia, file_manager_dir); - mock_provider mp; + file_manager_dir = repertory::utils::path::combine( + repertory::test::get_test_output_dir(), + {"file_manager_test" + std::to_string(++inst)}); - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - - event_consumer es("service_started", [](const event &e) { - const auto &ee = dynamic_cast(e); - EXPECT_STREQ("file_manager", ee.get_service().get().c_str()); - }); - event_consumer es2("service_shutdown_begin", [](const event &e) { - const auto &ee = dynamic_cast(e); - EXPECT_STREQ("file_manager", ee.get_service().get().c_str()); - }); - event_consumer es3("service_shutdown_end", [](const event &e) { - const auto &ee = dynamic_cast(e); - EXPECT_STREQ("file_manager", ee.get_service().get().c_str()); - }); - - event_capture ec( - {"service_started", "service_shutdown_begin", "service_shutdown_end"}); - - file_manager fm(cfg, mp); - fm.start(); - fm.stop(); - - ec.wait_for_empty(); - - event_system::instance().stop(); + cfg = std::make_unique(provider_type::sia, file_manager_dir); + cfg->set_enable_chunk_downloader_timeout(false); } - EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); -} - -TEST(file_manager, can_create_and_close_file) { - { - console_consumer c; - event_system::instance().start(); - - app_config cfg(provider_type::sia, file_manager_dir); - cfg.set_enable_chunk_downloader_timeout(false); - mock_provider mp; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - polling::instance().start(&cfg); - - file_manager fm(cfg, mp); - fm.start(); - - event_capture capture({ - "item_timeout", - "filesystem_item_opened", - "filesystem_item_handle_opened", - "filesystem_item_handle_closed", - "filesystem_item_closed", - }); - - const auto source_path = utils::path::combine( - cfg.get_cache_directory(), {utils::create_uuid_string()}); - - std::uint64_t handle{}; - { - std::shared_ptr f; - - const auto now = utils::time::get_time_now(); - auto meta = create_meta_attributes( - now, FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_ARCHIVE, now + 1u, - now + 2u, false, 1, "key", 2, now + 3u, 3u, 4u, 0u, source_path, 10, - now + 4u); - - EXPECT_CALL(mp, create_file("/test_create.txt", meta)) - .WillOnce(Return(api_error::success)); - EXPECT_CALL(mp, get_filesystem_item) - .WillOnce([&meta](const std::string &api_path, bool directory, - filesystem_item &fsi) -> api_error { - EXPECT_STREQ("/test_create.txt", api_path.c_str()); - EXPECT_FALSE(directory); - fsi.api_path = api_path; - fsi.api_parent = utils::path::get_parent_api_path(api_path); - fsi.directory = directory; - fsi.size = utils::string::to_uint64(meta[META_SIZE]); - fsi.source_path = meta[META_SOURCE]; - return api_error::success; - }); - - event_consumer ec("filesystem_item_opened", [&](const event &e) { - const auto &ee = dynamic_cast(e); - EXPECT_STREQ("/test_create.txt", - ee.get_api_path().get().c_str()); - EXPECT_STREQ(source_path.c_str(), - ee.get_source().get().c_str()); - EXPECT_STREQ("0", ee.get_directory().get().c_str()); - }); - - event_consumer ec2("filesystem_item_handle_opened", [&](const event &e) { - const auto &ee = dynamic_cast(e); - EXPECT_STREQ("/test_create.txt", - ee.get_api_path().get().c_str()); - EXPECT_STREQ(source_path.c_str(), - ee.get_source().get().c_str()); - EXPECT_STREQ("0", ee.get_directory().get().c_str()); - EXPECT_STREQ("1", ee.get_handle().get().c_str()); - }); - -#if defined(_WIN32) - EXPECT_EQ(api_error::success, - fm.create("/test_create.txt", meta, {}, handle, f)); -#else - EXPECT_EQ(api_error::success, - fm.create("/test_create.txt", meta, O_RDWR, handle, f)); -#endif - EXPECT_EQ(std::size_t(1u), fm.get_open_file_count()); - EXPECT_EQ(std::size_t(1u), fm.get_open_handle_count()); - EXPECT_EQ(std::uint64_t(1u), handle); - } - - event_consumer ec3("filesystem_item_closed", [&](const event &e) { - const auto &ee = dynamic_cast(e); - EXPECT_STREQ("/test_create.txt", - ee.get_api_path().get().c_str()); - EXPECT_STREQ(source_path.c_str(), - ee.get_source().get().c_str()); - EXPECT_STREQ("0", ee.get_directory().get().c_str()); - }); - - event_consumer ec4("filesystem_item_handle_closed", [&](const event &e) { - const auto &ee = dynamic_cast(e); - EXPECT_STREQ("/test_create.txt", - ee.get_api_path().get().c_str()); - EXPECT_STREQ(source_path.c_str(), - ee.get_source().get().c_str()); - EXPECT_STREQ("0", ee.get_directory().get().c_str()); - EXPECT_STREQ("1", ee.get_handle().get().c_str()); - }); - - fm.close(handle); - - EXPECT_EQ(std::size_t(1u), fm.get_open_file_count()); - EXPECT_EQ(std::size_t(0u), fm.get_open_handle_count()); - - capture.wait_for_empty(); - EXPECT_EQ(std::size_t(0u), fm.get_open_file_count()); - - fm.stop(); - - polling::instance().stop(); + void TearDown() override { + // EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); event_system::instance().stop(); } +}; - EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); +std::atomic file_manager_test::inst{0U}; + +TEST_F(file_manager_test, can_start_and_stop) { + EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); + + event_consumer es("service_started", [](const event &e) { + const auto &ee = dynamic_cast(e); + EXPECT_STREQ("file_manager", ee.get_service().get().c_str()); + }); + event_consumer es2("service_shutdown_begin", [](const event &e) { + const auto &ee = dynamic_cast(e); + EXPECT_STREQ("file_manager", ee.get_service().get().c_str()); + }); + event_consumer es3("service_shutdown_end", [](const event &e) { + const auto &ee = dynamic_cast(e); + EXPECT_STREQ("file_manager", ee.get_service().get().c_str()); + }); + + event_capture ec( + {"service_started", "service_shutdown_begin", "service_shutdown_end"}); + + file_manager fm(*cfg, mp); + fm.start(); + fm.stop(); + + ec.wait_for_empty(); } -TEST(file_manager, can_open_and_close_file) { +TEST_F(file_manager_test, can_create_and_close_file) { + EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); + polling::instance().start(cfg.get()); + + file_manager fm(*cfg, mp); + fm.start(); + + event_capture capture({ + "item_timeout", + "filesystem_item_opened", + "filesystem_item_handle_opened", + "filesystem_item_handle_closed", + "filesystem_item_closed", + }); + + const auto source_path = utils::path::combine(cfg->get_cache_directory(), + {utils::create_uuid_string()}); + + std::uint64_t handle{}; { - console_consumer c; - event_system::instance().start(); - - app_config cfg(provider_type::sia, file_manager_dir); - cfg.set_enable_chunk_downloader_timeout(false); - mock_provider mp; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - - polling::instance().start(&cfg); - file_manager fm(cfg, mp); - fm.start(); - - event_capture capture({ - "item_timeout", - "filesystem_item_opened", - "filesystem_item_handle_opened", - "filesystem_item_handle_closed", - "filesystem_item_closed", - }); - const auto source_path = utils::path::combine( - cfg.get_cache_directory(), {utils::create_uuid_string()}); - - std::uint64_t handle{}; - { - - const auto now = utils::time::get_time_now(); - auto meta = create_meta_attributes( - now, FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_ARCHIVE, now + 1u, - now + 2u, false, 1, "key", 2, now + 3u, 3u, 4u, 0u, source_path, 10, - now + 4u); - - EXPECT_CALL(mp, create_file).Times(0u); - - EXPECT_CALL(mp, get_filesystem_item) - .WillOnce([&meta](const std::string &api_path, bool directory, - filesystem_item &fsi) -> api_error { - EXPECT_STREQ("/test_open.txt", api_path.c_str()); - EXPECT_FALSE(directory); - fsi.api_path = api_path; - fsi.api_parent = utils::path::get_parent_api_path(api_path); - fsi.directory = directory; - fsi.size = utils::string::to_uint64(meta[META_SIZE]); - fsi.source_path = meta[META_SOURCE]; - return api_error::success; - }); - - event_consumer ec("filesystem_item_opened", [&](const event &e) { - const auto &ee = dynamic_cast(e); - EXPECT_STREQ("/test_open.txt", - ee.get_api_path().get().c_str()); - EXPECT_STREQ(source_path.c_str(), - ee.get_source().get().c_str()); - EXPECT_STREQ("0", ee.get_directory().get().c_str()); - }); - - event_consumer ec2("filesystem_item_handle_opened", [&](const event &e) { - const auto &ee = dynamic_cast(e); - EXPECT_STREQ("/test_open.txt", - ee.get_api_path().get().c_str()); - EXPECT_STREQ(source_path.c_str(), - ee.get_source().get().c_str()); - EXPECT_STREQ("0", ee.get_directory().get().c_str()); - EXPECT_STREQ("1", ee.get_handle().get().c_str()); - }); - - std::shared_ptr f; -#if defined(_WIN32) - EXPECT_EQ(api_error::success, - fm.open("/test_open.txt", false, {}, handle, f)); -#else - EXPECT_EQ(api_error::success, - fm.open("/test_open.txt", false, O_RDWR, handle, f)); -#endif - EXPECT_EQ(std::size_t(1u), fm.get_open_file_count()); - EXPECT_EQ(std::size_t(1u), fm.get_open_handle_count()); - EXPECT_EQ(std::uint64_t(1u), handle); - } - - event_consumer ec3("filesystem_item_closed", [&](const event &e) { - const auto &ee = dynamic_cast(e); - EXPECT_STREQ("/test_open.txt", - ee.get_api_path().get().c_str()); - EXPECT_STREQ(source_path.c_str(), - ee.get_source().get().c_str()); - EXPECT_STREQ("0", ee.get_directory().get().c_str()); - }); - - event_consumer ec4("filesystem_item_handle_closed", [&](const event &e) { - const auto &ee = dynamic_cast(e); - EXPECT_STREQ("/test_open.txt", - ee.get_api_path().get().c_str()); - EXPECT_STREQ(source_path.c_str(), - ee.get_source().get().c_str()); - EXPECT_STREQ("0", ee.get_directory().get().c_str()); - EXPECT_STREQ("1", ee.get_handle().get().c_str()); - }); - - fm.close(handle); - - EXPECT_EQ(std::size_t(1u), fm.get_open_file_count()); - EXPECT_EQ(std::size_t(0u), fm.get_open_handle_count()); - - capture.wait_for_empty(); - EXPECT_EQ(std::size_t(0u), fm.get_open_file_count()); - - fm.stop(); - - polling::instance().stop(); - event_system::instance().stop(); - } - - EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); -} - -TEST(file_manager, can_open_and_close_multiple_handles_for_same_file) { - { - console_consumer c; - event_system::instance().start(); - - app_config cfg(provider_type::sia, file_manager_dir); - cfg.set_enable_chunk_downloader_timeout(false); - mock_provider mp; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - - polling::instance().start(&cfg); - file_manager fm(cfg, mp); - fm.start(); - - { - const auto source_path = utils::path::combine( - cfg.get_cache_directory(), {utils::create_uuid_string()}); - - const auto now = utils::time::get_time_now(); - auto meta = create_meta_attributes( - now, FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_ARCHIVE, now + 1u, - now + 2u, false, 1, "key", 2, now + 3u, 3u, 4u, 0u, source_path, 10, - now + 4u); - - EXPECT_CALL(mp, create_file).Times(0u); - - EXPECT_CALL(mp, get_filesystem_item) - .WillOnce([&meta](const std::string &api_path, bool directory, - filesystem_item &fsi) -> api_error { - EXPECT_STREQ("/test_open.txt", api_path.c_str()); - EXPECT_FALSE(directory); - fsi.api_path = api_path; - fsi.api_parent = utils::path::get_parent_api_path(api_path); - fsi.directory = directory; - fsi.size = utils::string::to_uint64(meta[META_SIZE]); - fsi.source_path = meta[META_SOURCE]; - return api_error::success; - }); - - std::array handles; - for (std::uint8_t i = 0u; i < handles.size(); i++) { - std::shared_ptr f; -#if defined(_WIN32) - EXPECT_EQ(api_error::success, - fm.open("/test_open.txt", false, {}, handles[i], f)); -#else - EXPECT_EQ(api_error::success, - fm.open("/test_open.txt", false, O_RDWR, handles[i], f)); -#endif - - EXPECT_EQ(std::size_t(1u), fm.get_open_file_count()); - EXPECT_EQ(std::size_t(i + 1u), fm.get_open_handle_count()); - EXPECT_EQ(std::uint64_t(i + 1u), handles[i]); - } - - for (std::uint8_t i = 0u; i < handles.size(); i++) { - EXPECT_EQ(std::size_t(1u), fm.get_open_file_count()); - EXPECT_EQ(std::size_t(handles.size() - i), fm.get_open_handle_count()); - fm.close(handles[i]); - } - fm.stop(); - } - - EXPECT_EQ(std::size_t(0u), fm.get_open_file_count()); - EXPECT_EQ(std::size_t(0u), fm.get_open_handle_count()); - - polling::instance().stop(); - event_system::instance().stop(); - } - - EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); -} - -TEST(file_manager, download_is_stored_after_write_if_partially_downloaded) { - { - console_consumer c; - event_system::instance().start(); - - app_config cfg(provider_type::sia, file_manager_dir); - mock_provider mp; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - - file_manager fm(cfg, mp); - fm.start(); - - const auto source_path = utils::path::combine( - cfg.get_cache_directory(), {utils::create_uuid_string()}); - - event_consumer es("download_resume_added", [&source_path](const event &e) { - const auto &ee = dynamic_cast(e); - EXPECT_STREQ("/test_write_partial_download.txt", - ee.get_api_path().get().c_str()); - EXPECT_STREQ(source_path.c_str(), - ee.get_dest_path().get().c_str()); - }); - event_capture ec({"download_resume_added"}, - {"file_upload_completed", "file_upload_queued"}); - - const auto now = utils::time::get_time_now(); - auto meta = create_meta_attributes( - now, FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_ARCHIVE, now + 1u, now + 2u, - false, 1, "key", 2, now + 3u, 3u, 4u, - utils::encryption::encrypting_reader::get_data_chunk_size() * 4u, - source_path, 10, now + 4u); - auto &nf = - test::create_random_file(utils::string::to_uint64(meta[META_SIZE])); - - EXPECT_CALL(mp, get_filesystem_item) - .WillRepeatedly([&meta](const std::string &api_path, bool directory, - filesystem_item &fsi) -> api_error { - EXPECT_STREQ("/test_write_partial_download.txt", api_path.c_str()); - EXPECT_FALSE(directory); - fsi.api_path = api_path; - fsi.api_parent = utils::path::get_parent_api_path(api_path); - fsi.directory = directory; - fsi.size = utils::string::to_uint64(meta[META_SIZE]); - fsi.source_path = meta[META_SOURCE]; - return api_error::success; - }); - - std::uint64_t handle{}; std::shared_ptr f; -#if defined(_WIN32) - EXPECT_EQ(api_error::success, fm.open("/test_write_partial_download.txt", - false, {}, handle, f)); -#else - EXPECT_EQ(api_error::success, fm.open("/test_write_partial_download.txt", - false, O_RDWR, handle, f)); -#endif - - EXPECT_CALL(mp, read_file_bytes) - .WillRepeatedly([&nf](const std::string & /* api_path */, - std::size_t size, std::uint64_t offset, - data_buffer &data, - stop_type &stop_requested) -> api_error { - if (stop_requested) { - return api_error::download_stopped; - } - - if (offset == 0u) { - std::size_t bytes_read{}; - data.resize(size); - auto ret = nf.read(data, offset, &bytes_read) ? api_error::success - : api_error::os_error; - EXPECT_EQ(bytes_read, data.size()); - return ret; - } - - while (not stop_requested) { - std::this_thread::sleep_for(100ms); - } - - return api_error::download_stopped; - }); - EXPECT_CALL(mp, set_item_meta("/test_write_partial_download.txt", _)) - .WillOnce( - [](const std::string &, const api_meta_map &meta2) -> api_error { - EXPECT_NO_THROW(EXPECT_FALSE(meta2.at(META_CHANGED).empty())); - EXPECT_NO_THROW(EXPECT_FALSE(meta2.at(META_MODIFIED).empty())); - EXPECT_NO_THROW(EXPECT_FALSE(meta2.at(META_WRITTEN).empty())); - return api_error::success; - }); - EXPECT_CALL(mp, upload_file).Times(0u); - - std::size_t bytes_written{}; - data_buffer data = {0, 1, 2}; - EXPECT_EQ(api_error::success, f->write(0u, data, bytes_written)); - EXPECT_EQ(std::size_t(3u), bytes_written); - f.reset(); - - fm.close(handle); - - EXPECT_EQ(std::size_t(1u), fm.get_open_file_count()); - EXPECT_EQ(std::size_t(0u), fm.get_open_handle_count()); - - fm.stop(); - ec.wait_for_empty(); - - event_capture ec2({"download_restored", "download_resume_added"}, - {"file_upload_completed", "file_upload_queued"}); - EXPECT_EQ(std::size_t(0u), fm.get_open_file_count()); - EXPECT_EQ(std::size_t(0u), fm.get_open_handle_count()); - - auto stored_downloads = fm.get_stored_downloads(); - EXPECT_EQ(std::size_t(1u), stored_downloads.size()); - std::cout << stored_downloads[0u].dump(2) << std::endl; - - EXPECT_STREQ("/test_write_partial_download.txt", - stored_downloads[0u]["path"].get().c_str()); - EXPECT_EQ(utils::encryption::encrypting_reader::get_data_chunk_size(), - stored_downloads[0u]["chunk_size"].get()); - auto read_state = utils::string::to_dynamic_bitset( - stored_downloads[0u]["read_state"].get()); - EXPECT_TRUE(read_state[0u]); - for (std::size_t i = 1u; i < read_state.size(); i++) { - EXPECT_FALSE(read_state[i]); - } - EXPECT_STREQ(source_path.c_str(), - stored_downloads[0u]["source"].get().c_str()); - - fm.start(); - - event_consumer es2("download_restored", [&source_path](const event &e) { - const auto &ee = dynamic_cast(e); - EXPECT_STREQ("/test_write_partial_download.txt", - ee.get_api_path().get().c_str()); - EXPECT_STREQ(source_path.c_str(), - ee.get_dest_path().get().c_str()); - }); - - EXPECT_EQ(std::size_t(1u), fm.get_open_file_count()); - EXPECT_EQ(std::size_t(0u), fm.get_open_handle_count()); - fm.stop(); - ec2.wait_for_empty(); - - EXPECT_EQ(std::size_t(0u), fm.get_open_file_count()); - EXPECT_EQ(std::size_t(0u), fm.get_open_handle_count()); - - nf.close(); - - event_system::instance().stop(); - } - - EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); -} - -TEST(file_manager, upload_occurs_after_write_if_fully_downloaded) { - { - console_consumer c; - event_system::instance().start(); - - app_config cfg(provider_type::sia, file_manager_dir); - cfg.set_enable_chunk_downloader_timeout(false); - mock_provider mp; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - - polling::instance().start(&cfg); - file_manager fm(cfg, mp); - fm.start(); - - const auto source_path = utils::path::combine( - cfg.get_cache_directory(), {utils::create_uuid_string()}); - - event_consumer es("file_upload_queued", [&source_path](const event &e) { - const auto &ee = dynamic_cast(e); - EXPECT_STREQ("/test_write_full_download.txt", - ee.get_api_path().get().c_str()); - EXPECT_STREQ(source_path.c_str(), - ee.get_source().get().c_str()); - }); - event_consumer es2("file_upload_completed", [&source_path](const event &e) { - const auto &ee = dynamic_cast(e); - EXPECT_STREQ("/test_write_full_download.txt", - ee.get_api_path().get().c_str()); - EXPECT_STREQ(source_path.c_str(), - ee.get_source().get().c_str()); - }); - event_capture ec({"download_end"}); const auto now = utils::time::get_time_now(); - auto meta = create_meta_attributes( - now, FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_ARCHIVE, now + 1u, now + 2u, - false, 1, "key", 2, now + 3u, 3u, 4u, - utils::encryption::encrypting_reader::get_data_chunk_size() * 4u, - source_path, 10, now + 4u); - auto &nf = - test::create_random_file(utils::string::to_uint64(meta[META_SIZE])); - - EXPECT_CALL(mp, get_filesystem_item) - .WillRepeatedly([&meta](const std::string &api_path, bool directory, - filesystem_item &fsi) -> api_error { - EXPECT_STREQ("/test_write_full_download.txt", api_path.c_str()); - EXPECT_FALSE(directory); - fsi.api_path = api_path; - fsi.api_parent = utils::path::get_parent_api_path(api_path); - fsi.directory = directory; - fsi.size = utils::string::to_uint64(meta[META_SIZE]); - fsi.source_path = meta[META_SOURCE]; - return api_error::success; - }); - - std::uint64_t handle{}; - std::shared_ptr f; -#if defined(_WIN32) - EXPECT_EQ(api_error::success, - fm.open("/test_write_full_download.txt", false, {}, handle, f)); -#else - EXPECT_EQ(api_error::success, fm.open("/test_write_full_download.txt", - false, O_RDWR, handle, f)); -#endif - - EXPECT_CALL(mp, read_file_bytes) - .WillRepeatedly([&nf](const std::string & /* api_path */, - std::size_t size, std::uint64_t offset, - data_buffer &data, - stop_type & /* stop_requested */) -> api_error { - std::size_t bytes_read{}; - data.resize(size); - auto ret = nf.read(data, offset, &bytes_read) ? api_error::success - : api_error::os_error; - EXPECT_EQ(bytes_read, data.size()); - return ret; - }); - EXPECT_CALL(mp, set_item_meta("/test_write_full_download.txt", _)) - .WillOnce( - [](const std::string &, const api_meta_map &meta2) -> api_error { - EXPECT_NO_THROW(EXPECT_FALSE(meta2.at(META_CHANGED).empty())); - EXPECT_NO_THROW(EXPECT_FALSE(meta2.at(META_MODIFIED).empty())); - EXPECT_NO_THROW(EXPECT_FALSE(meta2.at(META_WRITTEN).empty())); - return api_error::success; - }); - std::size_t bytes_written{}; - data_buffer data = {0, 1, 2}; - EXPECT_EQ(api_error::success, f->write(0u, data, bytes_written)); - EXPECT_EQ(std::size_t(3u), bytes_written); - f.reset(); - - ec.wait_for_empty(); - - EXPECT_CALL(mp, - upload_file("/test_write_full_download.txt", source_path, _)) - .WillOnce(Return(api_error::success)); - - event_capture ec2( - {"item_timeout", "file_upload_queued", "file_upload_completed"}); - fm.close(handle); - - ec2.wait_for_empty(); - - EXPECT_EQ(std::size_t(0U), fm.get_open_file_count()); - EXPECT_EQ(std::size_t(0U), fm.get_open_handle_count()); - - fm.stop(); - - nf.close(); - - polling::instance().stop(); - event_system::instance().stop(); - } - - EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); -} - -TEST(file_manager, can_evict_file) { - { - console_consumer c; - event_system::instance().start(); - - app_config cfg(provider_type::sia, file_manager_dir); - cfg.set_enable_chunk_downloader_timeout(false); - mock_provider mp; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - - file_manager fm(cfg, mp); - fm.start(); - - event_capture capture({ - "filesystem_item_opened", - "filesystem_item_handle_opened", - "filesystem_item_handle_closed", - "filesystem_item_closed", - "file_upload_completed", - }); - - const auto source_path = utils::path::combine( - cfg.get_cache_directory(), {utils::create_uuid_string()}); - - const auto now = utils::time::get_time_now(); - auto meta = create_meta_attributes( now, FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_ARCHIVE, now + 1u, now + 2u, false, 1, "key", 2, now + 3u, 3u, 4u, 0u, source_path, 10, now + 4u); - std::uint64_t handle{}; - { - std::shared_ptr f; - EXPECT_CALL(mp, create_file("/test_evict.txt", meta)) - .WillOnce(Return(api_error::success)); - EXPECT_CALL(mp, get_filesystem_item) - .WillRepeatedly([&meta](const std::string &api_path, bool directory, - filesystem_item &fsi) -> api_error { - EXPECT_STREQ("/test_evict.txt", api_path.c_str()); - EXPECT_FALSE(directory); - fsi.api_path = api_path; - fsi.api_parent = utils::path::get_parent_api_path(api_path); - fsi.directory = directory; - fsi.size = utils::string::to_uint64(meta[META_SIZE]); - fsi.source_path = meta[META_SOURCE]; - return api_error::success; - }); + EXPECT_CALL(mp, create_file("/test_create.txt", meta)) + .WillOnce(Return(api_error::success)); + EXPECT_CALL(mp, get_filesystem_item) + .WillOnce([&meta](const std::string &api_path, bool directory, + filesystem_item &fsi) -> api_error { + EXPECT_STREQ("/test_create.txt", api_path.c_str()); + EXPECT_FALSE(directory); + fsi.api_path = api_path; + fsi.api_parent = utils::path::get_parent_api_path(api_path); + fsi.directory = directory; + fsi.size = utils::string::to_uint64(meta[META_SIZE]); + fsi.source_path = meta[META_SOURCE]; + return api_error::success; + }); + + event_consumer ec("filesystem_item_opened", [&](const event &e) { + const auto &ee = dynamic_cast(e); + EXPECT_STREQ("/test_create.txt", + ee.get_api_path().get().c_str()); + EXPECT_STREQ(source_path.c_str(), + ee.get_source().get().c_str()); + EXPECT_STREQ("0", ee.get_directory().get().c_str()); + }); + + event_consumer ec2("filesystem_item_handle_opened", [&](const event &e) { + const auto &ee = dynamic_cast(e); + EXPECT_STREQ("/test_create.txt", + ee.get_api_path().get().c_str()); + EXPECT_STREQ(source_path.c_str(), + ee.get_source().get().c_str()); + EXPECT_STREQ("0", ee.get_directory().get().c_str()); + EXPECT_STREQ("1", ee.get_handle().get().c_str()); + }); #if defined(_WIN32) - EXPECT_EQ(api_error::success, - fm.create("/test_evict.txt", meta, {}, handle, f)); + EXPECT_EQ(api_error::success, + fm.create("/test_create.txt", meta, {}, handle, f)); #else - EXPECT_EQ(api_error::success, - fm.create("/test_evict.txt", meta, O_RDWR, handle, f)); + EXPECT_EQ(api_error::success, + fm.create("/test_create.txt", meta, O_RDWR, handle, f)); #endif - EXPECT_CALL(mp, set_item_meta("/test_evict.txt", _)) - .Times(2) - .WillRepeatedly(Return(api_error::success)); - EXPECT_CALL(mp, upload_file(_, _, _)) - .WillOnce(Return(api_error::success)); - - data_buffer data{{0, 1, 1}}; - std::size_t bytes_written{}; - EXPECT_EQ(api_error::success, f->write(0U, data, bytes_written)); - - auto opt_size = utils::file::file{source_path}.size(); - EXPECT_TRUE(opt_size.has_value()); - EXPECT_EQ(static_cast(data.size()), opt_size.value()); - } - - fm.close(handle); - capture.wait_for_empty(); - - EXPECT_TRUE(utils::retry_action( - [&fm]() -> bool { return not fm.is_processing("/test_evict.txt"); })); - - EXPECT_CALL(mp, get_item_meta(_, META_SOURCE, _)) - .WillOnce([&source_path](const std::string &api_path, - const std::string &key, - std::string &value) -> api_error { - EXPECT_STREQ("/test_evict.txt", api_path.c_str()); - EXPECT_STREQ(META_SOURCE.c_str(), key.c_str()); - value = source_path; - return api_error::success; - }); - EXPECT_CALL(mp, get_item_meta(_, META_PINNED, _)) - .WillOnce([](const std::string &api_path, const std::string &key, - std::string &value) -> api_error { - EXPECT_STREQ("/test_evict.txt", api_path.c_str()); - EXPECT_STREQ(META_PINNED.c_str(), key.c_str()); - value = "0"; - return api_error::success; - }); - EXPECT_TRUE(fm.evict_file("/test_evict.txt")); - EXPECT_FALSE(utils::file::file(source_path).exists()); - - fm.stop(); - - event_system::instance().stop(); + EXPECT_EQ(std::size_t(1u), fm.get_open_file_count()); + EXPECT_EQ(std::size_t(1u), fm.get_open_handle_count()); + EXPECT_EQ(std::uint64_t(1u), handle); } - EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); + event_consumer ec3("filesystem_item_closed", [&](const event &e) { + const auto &ee = dynamic_cast(e); + EXPECT_STREQ("/test_create.txt", + ee.get_api_path().get().c_str()); + EXPECT_STREQ(source_path.c_str(), + ee.get_source().get().c_str()); + EXPECT_STREQ("0", ee.get_directory().get().c_str()); + }); + + event_consumer ec4("filesystem_item_handle_closed", [&](const event &e) { + const auto &ee = dynamic_cast(e); + EXPECT_STREQ("/test_create.txt", + ee.get_api_path().get().c_str()); + EXPECT_STREQ(source_path.c_str(), + ee.get_source().get().c_str()); + EXPECT_STREQ("0", ee.get_directory().get().c_str()); + EXPECT_STREQ("1", ee.get_handle().get().c_str()); + }); + + fm.close(handle); + + EXPECT_EQ(std::size_t(1u), fm.get_open_file_count()); + EXPECT_EQ(std::size_t(0u), fm.get_open_handle_count()); + + capture.wait_for_empty(); + EXPECT_EQ(std::size_t(0u), fm.get_open_file_count()); + + fm.stop(); + + polling::instance().stop(); } -TEST(file_manager, evict_file_fails_if_file_is_pinned) { +TEST_F(file_manager_test, can_open_and_close_file) { + EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); + + polling::instance().start(cfg.get()); + file_manager fm(*cfg, mp); + fm.start(); + + event_capture capture({ + "item_timeout", + "filesystem_item_opened", + "filesystem_item_handle_opened", + "filesystem_item_handle_closed", + "filesystem_item_closed", + }); + const auto source_path = utils::path::combine(cfg->get_cache_directory(), + {utils::create_uuid_string()}); + + std::uint64_t handle{}; { - app_config cfg(provider_type::sia, file_manager_dir); - cfg.set_enable_chunk_downloader_timeout(false); - mock_provider mp; - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - file_manager fm(cfg, mp); + const auto now = utils::time::get_time_now(); + auto meta = create_meta_attributes( + now, FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_ARCHIVE, now + 1u, now + 2u, + false, 1, "key", 2, now + 3u, 3u, 4u, 0u, source_path, 10, now + 4u); - EXPECT_CALL(mp, get_item_meta(_, META_PINNED, _)) - .WillOnce([](const std::string &api_path, const std::string &key, - std::string &value) -> api_error { - EXPECT_STREQ("/test_open.txt", api_path.c_str()); - EXPECT_STREQ(META_PINNED.c_str(), key.c_str()); - value = "1"; - return api_error::success; - }); - - EXPECT_FALSE(fm.evict_file("/test_open.txt")); - - event_system::instance().stop(); - } - - EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); -} - -TEST(file_manager, evict_file_fails_if_provider_is_direct_only) { - { - app_config cfg(provider_type::sia, file_manager_dir); - cfg.set_enable_chunk_downloader_timeout(false); - mock_provider mp; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(true)); - file_manager fm(cfg, mp); - EXPECT_FALSE(fm.evict_file("/test.txt")); - } - - EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); -} - -TEST(file_manager, evict_file_fails_if_file_is_open) { - { - app_config cfg(provider_type::sia, file_manager_dir); - cfg.set_enable_chunk_downloader_timeout(false); - mock_provider mp; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - file_manager fm(cfg, mp); + EXPECT_CALL(mp, create_file).Times(0u); EXPECT_CALL(mp, get_filesystem_item) - .WillOnce([](const std::string &api_path, bool directory, - filesystem_item &fsi) -> api_error { + .WillOnce([&meta](const std::string &api_path, bool directory, + filesystem_item &fsi) -> api_error { EXPECT_STREQ("/test_open.txt", api_path.c_str()); EXPECT_FALSE(directory); fsi.api_path = api_path; fsi.api_parent = utils::path::get_parent_api_path(api_path); fsi.directory = directory; - fsi.size = 0U; + fsi.size = utils::string::to_uint64(meta[META_SIZE]); + fsi.source_path = meta[META_SOURCE]; return api_error::success; }); - EXPECT_CALL(mp, set_item_meta(_, _, _)) - .WillOnce([](const std::string &api_path, const std::string &key, - const std::string &value) -> api_error { - EXPECT_STREQ("/test_open.txt", api_path.c_str()); - EXPECT_STREQ(META_SOURCE.c_str(), key.c_str()); - EXPECT_FALSE(value.empty()); - return api_error::success; - }); + event_consumer ec("filesystem_item_opened", [&](const event &e) { + const auto &ee = dynamic_cast(e); + EXPECT_STREQ("/test_open.txt", + ee.get_api_path().get().c_str()); + EXPECT_STREQ(source_path.c_str(), + ee.get_source().get().c_str()); + EXPECT_STREQ("0", ee.get_directory().get().c_str()); + }); - std::uint64_t handle{}; - std::shared_ptr f{}; + event_consumer ec2("filesystem_item_handle_opened", [&](const event &e) { + const auto &ee = dynamic_cast(e); + EXPECT_STREQ("/test_open.txt", + ee.get_api_path().get().c_str()); + EXPECT_STREQ(source_path.c_str(), + ee.get_source().get().c_str()); + EXPECT_STREQ("0", ee.get_directory().get().c_str()); + EXPECT_STREQ("1", ee.get_handle().get().c_str()); + }); + + std::shared_ptr f; #if defined(_WIN32) EXPECT_EQ(api_error::success, fm.open("/test_open.txt", false, {}, handle, f)); @@ -855,934 +273,1435 @@ TEST(file_manager, evict_file_fails_if_file_is_open) { EXPECT_EQ(api_error::success, fm.open("/test_open.txt", false, O_RDWR, handle, f)); #endif - - EXPECT_FALSE(fm.evict_file("/test_open.txt")); - - fm.close(handle); + EXPECT_EQ(std::size_t(1u), fm.get_open_file_count()); + EXPECT_EQ(std::size_t(1u), fm.get_open_handle_count()); + EXPECT_EQ(std::uint64_t(1u), handle); } - EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); + event_consumer ec3("filesystem_item_closed", [&](const event &e) { + const auto &ee = dynamic_cast(e); + EXPECT_STREQ("/test_open.txt", + ee.get_api_path().get().c_str()); + EXPECT_STREQ(source_path.c_str(), + ee.get_source().get().c_str()); + EXPECT_STREQ("0", ee.get_directory().get().c_str()); + }); + + event_consumer ec4("filesystem_item_handle_closed", [&](const event &e) { + const auto &ee = dynamic_cast(e); + EXPECT_STREQ("/test_open.txt", + ee.get_api_path().get().c_str()); + EXPECT_STREQ(source_path.c_str(), + ee.get_source().get().c_str()); + EXPECT_STREQ("0", ee.get_directory().get().c_str()); + EXPECT_STREQ("1", ee.get_handle().get().c_str()); + }); + + fm.close(handle); + + EXPECT_EQ(std::size_t(1u), fm.get_open_file_count()); + EXPECT_EQ(std::size_t(0u), fm.get_open_handle_count()); + + capture.wait_for_empty(); + EXPECT_EQ(std::size_t(0u), fm.get_open_file_count()); + + fm.stop(); + + polling::instance().stop(); } -TEST(file_manager, - evict_file_fails_if_unable_to_get_source_path_from_item_meta) { +TEST_F(file_manager_test, can_open_and_close_multiple_handles_for_same_file) { + EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); + + polling::instance().start(cfg.get()); + file_manager fm(*cfg, mp); + fm.start(); + { - app_config cfg(provider_type::sia, file_manager_dir); - cfg.set_enable_chunk_downloader_timeout(false); - mock_provider mp; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - file_manager fm(cfg, mp); - - EXPECT_CALL(mp, get_item_meta(_, META_SOURCE, _)) - .WillOnce([](const std::string &api_path, const std::string &key, - std::string & /*value*/) -> api_error { - EXPECT_STREQ("/test_open.txt", api_path.c_str()); - EXPECT_STREQ(META_SOURCE.c_str(), key.c_str()); - return api_error::error; - }); - - EXPECT_CALL(mp, get_item_meta(_, META_PINNED, _)) - .WillOnce([](const std::string &api_path, const std::string &key, - std::string &value) -> api_error { - EXPECT_STREQ("/test_open.txt", api_path.c_str()); - EXPECT_STREQ(META_PINNED.c_str(), key.c_str()); - value = "0"; - return api_error::success; - }); - - EXPECT_FALSE(fm.evict_file("/test_open.txt")); - } - - EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); -} - -TEST(file_manager, evict_file_fails_if_source_path_is_empty) { - { - app_config cfg(provider_type::sia, file_manager_dir); - cfg.set_enable_chunk_downloader_timeout(false); - mock_provider mp; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - file_manager fm(cfg, mp); - - EXPECT_CALL(mp, get_item_meta(_, META_SOURCE, _)) - .WillOnce([](const std::string &api_path, const std::string &key, - std::string &value) -> api_error { - EXPECT_STREQ("/test_open.txt", api_path.c_str()); - EXPECT_STREQ(META_SOURCE.c_str(), key.c_str()); - value = ""; - return api_error::success; - }); - EXPECT_CALL(mp, get_item_meta(_, META_PINNED, _)) - .WillOnce([](const std::string &api_path, const std::string &key, - std::string &value) -> api_error { - EXPECT_STREQ("/test_open.txt", api_path.c_str()); - EXPECT_STREQ(META_PINNED.c_str(), key.c_str()); - value = "0"; - return api_error::success; - }); - - EXPECT_FALSE(fm.evict_file("/test_open.txt")); - } - - EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); -} - -TEST(file_manager, evict_file_fails_if_file_is_uploading) { - { - console_consumer c; - event_system::instance().start(); - - app_config cfg(provider_type::sia, file_manager_dir); - cfg.set_enable_chunk_downloader_timeout(false); - mock_provider mp; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - - file_manager fm(cfg, mp); - fm.start(); - - event_capture capture({ - "filesystem_item_opened", - "filesystem_item_handle_opened", - "filesystem_item_handle_closed", - "filesystem_item_closed", - "file_upload_completed", - }); - const auto source_path = utils::path::combine( - cfg.get_cache_directory(), {utils::create_uuid_string()}); - - const auto now = utils::time::get_time_now(); - - auto meta = create_meta_attributes( - now, FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_ARCHIVE, now + 1u, now + 2u, - false, 1, "", 2, now + 3u, 3u, 4u, 0u, source_path, 10, now + 4u); - std::uint64_t handle{}; - { - std::shared_ptr f; - - EXPECT_CALL(mp, create_file("/test_evict.txt", meta)) - .WillOnce(Return(api_error::success)); - EXPECT_CALL(mp, get_filesystem_item) - .WillRepeatedly([&meta](const std::string &api_path, bool directory, - filesystem_item &fsi) -> api_error { - EXPECT_STREQ("/test_evict.txt", api_path.c_str()); - EXPECT_FALSE(directory); - fsi.api_path = api_path; - fsi.api_parent = utils::path::get_parent_api_path(api_path); - fsi.directory = directory; - fsi.size = utils::string::to_uint64(meta[META_SIZE]); - fsi.source_path = meta[META_SOURCE]; - return api_error::success; - }); - -#if defined(_WIN32) - EXPECT_EQ(api_error::success, - fm.create("/test_evict.txt", meta, {}, handle, f)); -#else - EXPECT_EQ(api_error::success, - fm.create("/test_evict.txt", meta, O_RDWR, handle, f)); -#endif - EXPECT_CALL(mp, set_item_meta("/test_evict.txt", _)) - .Times(2) - .WillRepeatedly(Return(api_error::success)); - EXPECT_CALL(mp, upload_file) - .WillOnce([](const std::string &api_path, - const std::string &source_path2, - stop_type & /*stop_requested*/) -> api_error { - EXPECT_STREQ("/test_evict.txt", api_path.c_str()); - EXPECT_FALSE(source_path2.empty()); - std::this_thread::sleep_for(3s); - return api_error::success; - }); - - data_buffer data{{0, 1, 1}}; - std::size_t bytes_written{}; - EXPECT_EQ(api_error::success, f->write(0U, data, bytes_written)); - - auto opt_size = utils::file::file{source_path}.size(); - EXPECT_TRUE(opt_size.has_value()); - EXPECT_EQ(static_cast(data.size()), opt_size.value()); - - fm.close(handle); - - EXPECT_TRUE(utils::retry_action( - [&fm]() -> bool { return fm.is_processing("/test_evict.txt"); })); - EXPECT_FALSE(fm.evict_file("/test_evict.txt")); - } - - capture.wait_for_empty(); - - EXPECT_TRUE(utils::file::file(source_path).exists()); - - fm.stop(); - - event_system::instance().stop(); - } - - EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); -} - -TEST(file_manager, evict_file_fails_if_file_is_in_upload_queue) { - { - app_config cfg(provider_type::sia, file_manager_dir); - cfg.set_enable_chunk_downloader_timeout(false); - mock_provider mp; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - - file_manager fm(cfg, mp); - - mock_open_file of{}; - EXPECT_CALL(of, is_directory).WillRepeatedly(Return(false)); - EXPECT_CALL(of, get_api_path).WillRepeatedly(Return("/test_evict.txt")); - EXPECT_CALL(of, get_source_path).WillRepeatedly(Return("/test_evict.src")); - fm.queue_upload(of); - - EXPECT_TRUE(fm.is_processing("/test_evict.txt")); - EXPECT_FALSE(fm.evict_file("/test_evict.txt")); - } - - EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); -} - -TEST(file_manager, evict_file_fails_if_file_is_modified) { - { - app_config cfg(provider_type::sia, file_manager_dir); - cfg.set_enable_chunk_downloader_timeout(false); - mock_provider mp; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - - file_manager fm(cfg, mp); - EXPECT_CALL(mp, get_filesystem_item) - .WillOnce([](const std::string &api_path, bool directory, - filesystem_item &fsi) -> api_error { - EXPECT_STREQ("/test_evict.txt", api_path.c_str()); - EXPECT_FALSE(directory); - fsi.api_path = api_path; - fsi.api_parent = utils::path::get_parent_api_path(api_path); - fsi.directory = directory; - fsi.size = 1U; - fsi.source_path = "/test_evict.src"; - return api_error::success; - }); - - auto of = std::make_shared(); - EXPECT_CALL(*of, is_directory).WillOnce(Return(false)); - EXPECT_CALL(*of, add).WillOnce(Return()); - EXPECT_CALL(*of, get_api_path).WillRepeatedly(Return("/test_evict.txt")); - EXPECT_CALL(*of, get_source_path).WillRepeatedly(Return("/test_evict.src")); - EXPECT_CALL(*of, is_modified).Times(2).WillRepeatedly(Return(true)); - - std::uint64_t handle{}; - std::shared_ptr f; -#if defined(_WIN32) - EXPECT_EQ(api_error::success, fm.open(of, {}, handle, f)); -#else - EXPECT_EQ(api_error::success, fm.open(of, O_RDWR, handle, f)); -#endif - - EXPECT_TRUE(fm.is_processing("/test_evict.txt")); - EXPECT_FALSE(fm.evict_file("/test_evict.txt")); - } - - EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); -} - -TEST(file_manager, evict_file_fails_if_file_is_not_complete) { - { - app_config cfg(provider_type::sia, file_manager_dir); - cfg.set_enable_chunk_downloader_timeout(false); - mock_provider mp; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - - file_manager fm(cfg, mp); - EXPECT_CALL(mp, get_filesystem_item) - .WillOnce([](const std::string &api_path, bool directory, - filesystem_item &fsi) -> api_error { - EXPECT_STREQ("/test_evict.txt", api_path.c_str()); - EXPECT_FALSE(directory); - fsi.api_path = api_path; - fsi.api_parent = utils::path::get_parent_api_path(api_path); - fsi.directory = directory; - fsi.size = 1U; - return api_error::success; - }); - - auto of = std::make_shared(); - EXPECT_CALL(*of, is_directory).WillOnce(Return(false)); - EXPECT_CALL(*of, add).WillOnce(Return()); - EXPECT_CALL(*of, get_api_path).WillRepeatedly(Return("/test_evict.txt")); - EXPECT_CALL(*of, get_source_path).WillRepeatedly(Return("/test_evict.src")); - EXPECT_CALL(*of, is_modified).Times(2).WillRepeatedly(Return(false)); - EXPECT_CALL(*of, is_complete).Times(2).WillRepeatedly(Return(false)); - EXPECT_CALL(mp, set_item_meta("/test_evict.txt", META_SOURCE, _)) - .WillOnce(Return(api_error::success)); - - std::uint64_t handle{}; - std::shared_ptr f; -#if defined(_WIN32) - EXPECT_EQ(api_error::success, fm.open(of, {}, handle, f)); -#else - EXPECT_EQ(api_error::success, fm.open(of, O_RDWR, handle, f)); -#endif - - EXPECT_TRUE(fm.is_processing("/test_evict.txt")); - EXPECT_FALSE(fm.evict_file("/test_evict.txt")); - } - - EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); -} - -TEST(file_manager, can_get_directory_items) { - { - app_config cfg(provider_type::sia, file_manager_dir); - cfg.set_enable_chunk_downloader_timeout(false); - mock_provider mp; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - - file_manager fm(cfg, mp); - EXPECT_CALL(mp, get_directory_items) - .WillOnce([](const std::string &api_path, - directory_item_list &list) -> api_error { - EXPECT_STREQ("/", api_path.c_str()); - list.insert(list.begin(), directory_item{ - "..", - "", - true, - }); - list.insert(list.begin(), directory_item{ - ".", - "", - true, - }); - return api_error::success; - }); - auto list = fm.get_directory_items("/"); - EXPECT_EQ(std::size_t(2U), list.size()); - } - - EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); -} - -TEST(file_manager, file_is_not_opened_if_provider_create_file_fails) { - { - app_config cfg(provider_type::sia, file_manager_dir); - cfg.set_enable_chunk_downloader_timeout(false); - mock_provider mp; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); + cfg->get_cache_directory(), {utils::create_uuid_string()}); const auto now = utils::time::get_time_now(); auto meta = create_meta_attributes( now, FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_ARCHIVE, now + 1u, now + 2u, - false, 1, "", 2, now + 3u, 3u, 4u, 0u, "/test_create.src", 10, - now + 4u); - file_manager fm(cfg, mp); + false, 1, "key", 2, now + 3u, 3u, 4u, 0u, source_path, 10, now + 4u); - EXPECT_CALL(mp, create_file("/test_create.txt", meta)) - .WillOnce(Return(api_error::error)); - - std::uint64_t handle{}; - std::shared_ptr f; -#if defined(_WIN32) - EXPECT_EQ(api_error::error, - fm.create("/test_create.txt", meta, {}, handle, f)); -#else - EXPECT_EQ(api_error::error, - fm.create("/test_create.txt", meta, O_RDWR, handle, f)); -#endif - EXPECT_FALSE(f); - EXPECT_EQ(std::size_t(0U), fm.get_open_file_count()); - } - - EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); -} - -TEST(file_manager, create_fails_if_provider_create_is_unsuccessful) { - { - app_config cfg(provider_type::sia, file_manager_dir); - cfg.set_enable_chunk_downloader_timeout(false); - mock_provider mp; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - - file_manager fm(cfg, mp); - - EXPECT_CALL(mp, create_file("/test_create.txt", _)) - .WillOnce(Return(api_error::error)); - - std::uint64_t handle{}; - std::shared_ptr f{}; - api_meta_map meta{}; -#if defined(_WIN32) - EXPECT_EQ(api_error::error, - fm.create("/test_create.txt", meta, {}, handle, f)); -#else - EXPECT_EQ(api_error::error, - fm.create("/test_create.txt", meta, O_RDWR, handle, f)); -#endif - EXPECT_EQ(std::size_t(0U), fm.get_open_file_count()); - } - - EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); -} - -TEST(file_manager, get_open_file_fails_if_file_is_not_open) { - { - app_config cfg(provider_type::sia, file_manager_dir); - cfg.set_enable_chunk_downloader_timeout(false); - mock_provider mp; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - - file_manager fm(cfg, mp); - - std::shared_ptr f{}; - EXPECT_FALSE(fm.get_open_file(0U, true, f)); - EXPECT_FALSE(f); - - EXPECT_FALSE(fm.get_open_file(0U, false, f)); - EXPECT_FALSE(f); - } - - EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); -} - -TEST(file_manager, - get_open_file_promotes_non_writeable_file_if_writeable_is_specified) { - { - app_config cfg(provider_type::sia, file_manager_dir); - cfg.set_enable_chunk_downloader_timeout(false); - mock_provider mp; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - - file_manager fm(cfg, mp); - - auto non_writeable = std::make_shared(); - EXPECT_CALL(*non_writeable, is_directory).WillOnce(Return(false)); - EXPECT_CALL(*non_writeable, add).WillOnce(Return()); - EXPECT_CALL(*non_writeable, get_api_path) - .WillRepeatedly(Return("/test_open.txt")); - EXPECT_CALL(*non_writeable, get_source_path) - .WillRepeatedly(Return("/test_open.src")); - EXPECT_CALL(*non_writeable, is_modified).WillRepeatedly(Return(true)); - EXPECT_CALL(*non_writeable, is_write_supported) - .WillRepeatedly(Return(false)); - EXPECT_CALL(*non_writeable, is_write_supported) - .WillRepeatedly(Return(false)); - - EXPECT_CALL(*non_writeable, get_filesystem_item) - .WillOnce([api_path = "/test_open.txt"]() -> filesystem_item { - filesystem_item fsi{}; - fsi.api_path = api_path; - fsi.api_parent = utils::path::get_parent_api_path(api_path); - fsi.directory = false; - fsi.size = 10U; - fsi.source_path = "/test_open.src"; - return fsi; - }); + EXPECT_CALL(mp, create_file).Times(0u); EXPECT_CALL(mp, get_filesystem_item) - .WillOnce([](const std::string &api_path, bool directory, - filesystem_item &fsi) -> api_error { + .WillOnce([&meta](const std::string &api_path, bool directory, + filesystem_item &fsi) -> api_error { EXPECT_STREQ("/test_open.txt", api_path.c_str()); EXPECT_FALSE(directory); fsi.api_path = api_path; fsi.api_parent = utils::path::get_parent_api_path(api_path); fsi.directory = directory; - fsi.size = 10U; - fsi.source_path = "/test_open.src"; - return api_error::success; - }); - - std::uint64_t handle{}; - std::shared_ptr f{}; -#if defined(_WIN32) - EXPECT_EQ(api_error::success, fm.open(non_writeable, {}, handle, f)); - EXPECT_CALL(*non_writeable, get_open_data()) - .WillOnce([&handle]() -> std::map & { - static std::map map; - map[handle] = {}; - return map; - }); -#else - EXPECT_EQ(api_error::success, fm.open(non_writeable, O_RDWR, handle, f)); - EXPECT_CALL(*non_writeable, get_open_data()) - .WillOnce([&handle]() -> std::map & { - static std::map map; - map[handle] = O_RDWR; - return map; - }); -#endif - - EXPECT_CALL(mp, set_item_meta("/test_open.txt", META_SOURCE, _)) - .WillOnce(Return(api_error::success)); - - EXPECT_CALL(*non_writeable, has_handle(1)).WillOnce([]() -> bool { - return true; - }); - EXPECT_TRUE(fm.get_open_file(handle, true, f)); - EXPECT_NE(non_writeable.get(), f.get()); - EXPECT_EQ(std::size_t(1U), fm.get_open_file_count()); - - std::shared_ptr f2{}; - EXPECT_TRUE(fm.get_open_file(handle, false, f2)); - EXPECT_EQ(f.get(), f2.get()); - EXPECT_EQ(std::size_t(1U), fm.get_open_file_count()); - } - - EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); -} - -TEST(file_manager, open_file_fails_if_file_is_not_found) { - { - app_config cfg(provider_type::sia, file_manager_dir); - cfg.set_enable_chunk_downloader_timeout(false); - mock_provider mp; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - - file_manager fm(cfg, mp); - - std::shared_ptr f{}; - EXPECT_FALSE(fm.get_open_file(1U, true, f)); - EXPECT_EQ(std::size_t(0U), fm.get_open_file_count()); - EXPECT_FALSE(f); - - EXPECT_FALSE(fm.get_open_file(1U, false, f)); - EXPECT_FALSE(f); - EXPECT_EQ(std::size_t(0U), fm.get_open_file_count()); - } - - EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); -} - -TEST(file_manager, open_file_fails_if_provider_get_filesystem_item_fails) { - { - app_config cfg(provider_type::sia, file_manager_dir); - cfg.set_enable_chunk_downloader_timeout(false); - mock_provider mp; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - - file_manager fm(cfg, mp); - - auto of = std::make_shared(); - EXPECT_CALL(*of, is_directory).WillRepeatedly(Return(false)); - EXPECT_CALL(*of, get_api_path).WillRepeatedly(Return("/test_open.txt")); - EXPECT_CALL(*of, get_source_path).WillRepeatedly(Return("/test_open.src")); - - EXPECT_CALL(mp, get_filesystem_item) - .WillOnce([](const std::string &api_path, bool directory, - filesystem_item & /*fsi*/) -> api_error { - EXPECT_STREQ("/test_open.txt", api_path.c_str()); - EXPECT_FALSE(directory); - return api_error::error; - }); - - std::uint64_t handle{}; - std::shared_ptr f{}; -#if defined(_WIN32) - EXPECT_EQ(api_error::error, fm.open(of, {}, handle, f)); -#else - EXPECT_EQ(api_error::error, fm.open(of, O_RDWR, handle, f)); -#endif - EXPECT_FALSE(fm.get_open_file(1U, true, f)); - EXPECT_EQ(std::size_t(0U), fm.get_open_file_count()); - EXPECT_FALSE(f); - - EXPECT_FALSE(fm.get_open_file(1U, false, f)); - EXPECT_FALSE(f); - EXPECT_EQ(std::size_t(0U), fm.get_open_file_count()); - } - - EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); -} - -TEST(file_manager, open_file_fails_if_provider_set_item_meta_fails) { - { - app_config cfg(provider_type::sia, file_manager_dir); - cfg.set_enable_chunk_downloader_timeout(false); - mock_provider mp; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - - file_manager fm(cfg, mp); - - auto of = std::make_shared(); - EXPECT_CALL(*of, is_directory).WillRepeatedly(Return(false)); - EXPECT_CALL(*of, get_api_path).WillRepeatedly(Return("/test_open.txt")); - EXPECT_CALL(*of, get_source_path).WillRepeatedly(Return("/test_open.src")); - - EXPECT_CALL(mp, get_filesystem_item) - .WillOnce([](const std::string &api_path, bool directory, - filesystem_item &fsi) -> api_error { - EXPECT_STREQ("/test_open.txt", api_path.c_str()); - EXPECT_FALSE(directory); - fsi.api_path = api_path; - fsi.api_parent = utils::path::get_parent_api_path(api_path); - fsi.directory = directory; - fsi.size = 0U; - return api_error::success; - }); - - EXPECT_CALL(mp, set_item_meta("/test_open.txt", META_SOURCE, _)) - .WillOnce(Return(api_error::error)); - - std::uint64_t handle{}; - std::shared_ptr f{}; -#if defined(_WIN32) - EXPECT_EQ(api_error::error, fm.open(of, {}, handle, f)); -#else - EXPECT_EQ(api_error::error, fm.open(of, O_RDWR, handle, f)); -#endif - EXPECT_FALSE(fm.get_open_file(1U, true, f)); - EXPECT_EQ(std::size_t(0U), fm.get_open_file_count()); - EXPECT_FALSE(f); - - EXPECT_FALSE(fm.get_open_file(1U, false, f)); - EXPECT_FALSE(f); - EXPECT_EQ(std::size_t(0U), fm.get_open_file_count()); - } - - EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); -} - -TEST(file_manager, open_file_creates_source_path_if_empty) { - { - app_config cfg(provider_type::sia, file_manager_dir); - cfg.set_enable_chunk_downloader_timeout(false); - mock_provider mp; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - - file_manager fm(cfg, mp); - - auto of = std::make_shared(); - EXPECT_CALL(*of, add).WillOnce(Return()); - EXPECT_CALL(*of, is_directory).WillRepeatedly(Return(false)); - EXPECT_CALL(*of, is_write_supported).WillRepeatedly(Return(true)); - EXPECT_CALL(*of, get_api_path).WillRepeatedly(Return("/test_open.txt")); - EXPECT_CALL(*of, get_source_path).WillRepeatedly(Return("")); - - EXPECT_CALL(mp, get_filesystem_item) - .WillOnce([](const std::string &api_path, bool directory, - filesystem_item &fsi) -> api_error { - EXPECT_STREQ("/test_open.txt", api_path.c_str()); - EXPECT_FALSE(directory); - fsi.api_path = api_path; - fsi.api_parent = utils::path::get_parent_api_path(api_path); - fsi.directory = directory; - fsi.size = 0U; - return api_error::success; - }); - - EXPECT_CALL(mp, set_item_meta("/test_open.txt", _, _)) - .WillOnce([](const std::string &api_path, const std::string &key, - const std::string &value) -> api_error { - EXPECT_STREQ("/test_open.txt", api_path.c_str()); - EXPECT_STREQ(META_SOURCE.c_str(), key.c_str()); - EXPECT_FALSE(value.empty()); - return api_error::success; - }); - - std::uint64_t handle{}; - std::shared_ptr f{}; -#if defined(_WIN32) - EXPECT_EQ(api_error::success, fm.open(of, {}, handle, f)); -#else - EXPECT_EQ(api_error::success, fm.open(of, O_RDWR, handle, f)); -#endif - EXPECT_CALL(*of, has_handle(1)).Times(2).WillRepeatedly([]() -> bool { - return true; - }); - - EXPECT_TRUE(fm.get_open_file(1U, true, f)); - EXPECT_EQ(std::size_t(1U), fm.get_open_file_count()); - EXPECT_TRUE(f); - - EXPECT_TRUE(fm.get_open_file(1U, false, f)); - EXPECT_TRUE(f); - EXPECT_EQ(std::size_t(1U), fm.get_open_file_count()); - } - - EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); -} - -TEST(file_manager, open_file_first_file_handle_is_not_zero) { - { - app_config cfg(provider_type::sia, file_manager_dir); - cfg.set_enable_chunk_downloader_timeout(false); - mock_provider mp; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - - file_manager fm(cfg, mp); - - auto of = std::make_shared(); - EXPECT_CALL(*of, add).WillOnce(Return()); - EXPECT_CALL(*of, is_directory).WillRepeatedly(Return(false)); - EXPECT_CALL(*of, is_write_supported).WillRepeatedly(Return(true)); - EXPECT_CALL(*of, get_api_path).WillRepeatedly(Return("/test_open.txt")); - EXPECT_CALL(*of, get_source_path).WillRepeatedly(Return("/test_open.src")); - - EXPECT_CALL(mp, get_filesystem_item) - .WillOnce([](const std::string &api_path, bool directory, - filesystem_item &fsi) -> api_error { - EXPECT_STREQ("/test_open.txt", api_path.c_str()); - EXPECT_FALSE(directory); - fsi.api_path = api_path; - fsi.api_parent = utils::path::get_parent_api_path(api_path); - fsi.directory = directory; - fsi.size = 0U; - fsi.source_path = "/test_open.src"; - return api_error::success; - }); - - std::uint64_t handle{}; - std::shared_ptr f{}; -#if defined(_WIN32) - EXPECT_EQ(api_error::success, fm.open(of, {}, handle, f)); -#else - EXPECT_EQ(api_error::success, fm.open(of, O_RDWR, handle, f)); -#endif - EXPECT_CALL(*of, has_handle(1)).WillOnce([]() -> bool { return true; }); - - EXPECT_TRUE(fm.get_open_file(1U, true, f)); - EXPECT_GT(handle, std::uint64_t(0U)); - } - - EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); -} - -TEST(file_manager, can_remove_file) { - { - app_config cfg(provider_type::sia, file_manager_dir); - cfg.set_enable_chunk_downloader_timeout(false); - mock_provider mp; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - - file_manager fm(cfg, mp); - - { - auto file = utils::file::file::open_or_create_file("./test_remove.txt"); - EXPECT_TRUE(*file); - } - EXPECT_TRUE(utils::file::file("./test_remove.txt").exists()); - - EXPECT_CALL(mp, get_filesystem_item) - .WillOnce([](const std::string &api_path, bool directory, - filesystem_item &fsi) -> api_error { - EXPECT_STREQ("/test_remove.txt", api_path.c_str()); - EXPECT_FALSE(directory); - fsi.api_path = api_path; - fsi.api_parent = utils::path::get_parent_api_path(api_path); - fsi.directory = directory; - fsi.size = 0U; - fsi.source_path = "./test_remove.txt"; - return api_error::success; - }); - EXPECT_CALL(mp, remove_file("/test_remove.txt")) - .WillOnce(Return(api_error::success)); - - EXPECT_EQ(api_error::success, fm.remove_file("/test_remove.txt")); - - EXPECT_FALSE(utils::file::file("./test_remove.txt").exists()); - } - - EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); -} - -TEST(file_manager, can_queue_and_remove_upload) { - { - console_consumer c; - event_system::instance().start(); - event_capture ec({"file_upload_queued", "download_resume_removed"}); - - app_config cfg(provider_type::sia, file_manager_dir); - cfg.set_enable_chunk_downloader_timeout(false); - mock_provider mp; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - - file_manager fm(cfg, mp); - - mock_open_file of{}; - EXPECT_CALL(of, get_api_path).WillOnce(Return("/test_queue.txt")); - EXPECT_CALL(of, get_source_path).WillOnce(Return("/test_queue.src")); - - EXPECT_FALSE(fm.is_processing("/test_queue.txt")); - fm.queue_upload(of); - EXPECT_TRUE(fm.is_processing("/test_queue.txt")); - - fm.remove_upload("/test_queue.txt"); - EXPECT_FALSE(fm.is_processing("/test_queue.txt")); - - ec.wait_for_empty(); - - event_system::instance().stop(); - } - - EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); -} - -TEST(file_manager, file_is_closed_after_download_timeout) { - { - console_consumer c; - event_system::instance().start(); - - app_config cfg(provider_type::sia, file_manager_dir); - cfg.set_chunk_downloader_timeout_secs(3U); - polling::instance().start(&cfg); - - mock_provider mp; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - - file_manager fm(cfg, mp); - fm.start(); - - const auto source_path = utils::path::combine( - cfg.get_cache_directory(), {utils::create_uuid_string()}); - - event_consumer es("item_timeout", [](const event &e) { - const auto &ee = dynamic_cast(e); - EXPECT_STREQ("/test_download_timeout.txt", - ee.get_api_path().get().c_str()); - }); - - const auto now = utils::time::get_time_now(); - auto meta = create_meta_attributes( - now, FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_ARCHIVE, now + 1u, now + 2u, - false, 1, "key", 2, now + 3u, 3u, 4u, - utils::encryption::encrypting_reader::get_data_chunk_size() * 4u, - source_path, 10, now + 4u); - - EXPECT_CALL(mp, get_filesystem_item) - .WillRepeatedly([&meta](const std::string &api_path, bool directory, - filesystem_item &fsi) -> api_error { - EXPECT_STREQ("/test_download_timeout.txt", api_path.c_str()); - EXPECT_FALSE(directory); - fsi.api_path = api_path; - fsi.api_parent = utils::path::get_parent_api_path(api_path); - fsi.directory = directory; fsi.size = utils::string::to_uint64(meta[META_SIZE]); fsi.source_path = meta[META_SOURCE]; return api_error::success; }); - event_capture ec({"item_timeout"}); - - std::uint64_t handle{}; - std::shared_ptr f; + std::array handles; + for (std::uint8_t i = 0u; i < handles.size(); i++) { + std::shared_ptr f; #if defined(_WIN32) - EXPECT_EQ(api_error::success, - fm.open("/test_download_timeout.txt", false, {}, handle, f)); + EXPECT_EQ(api_error::success, + fm.open("/test_open.txt", false, {}, handles[i], f)); #else - EXPECT_EQ(api_error::success, - fm.open("/test_download_timeout.txt", false, O_RDWR, handle, f)); + EXPECT_EQ(api_error::success, + fm.open("/test_open.txt", false, O_RDWR, handles[i], f)); #endif - EXPECT_CALL(mp, read_file_bytes) - .WillRepeatedly([](const std::string & /* api_path */, - std::size_t /*size*/, std::uint64_t offset, - data_buffer & /*data*/, - stop_type &stop_requested) -> api_error { - if (stop_requested) { - return api_error::download_stopped; - } + EXPECT_EQ(std::size_t(1u), fm.get_open_file_count()); + EXPECT_EQ(std::size_t(i + 1u), fm.get_open_handle_count()); + EXPECT_EQ(std::uint64_t(i + 1u), handles[i]); + } - if (offset == 0u) { - return api_error::success; - } - - while (not stop_requested) { - std::this_thread::sleep_for(100ms); - } - - return api_error::download_stopped; - }); - - data_buffer data{}; - EXPECT_EQ(api_error::success, f->read(1U, 0U, data)); - - fm.close(handle); - - EXPECT_CALL(mp, set_item_meta("/test_download_timeout.txt", META_SOURCE, _)) - .WillOnce(Return(api_error::success)); - - EXPECT_EQ(std::size_t(1U), fm.get_open_file_count()); - ec.wait_for_empty(); - - EXPECT_EQ(std::size_t(0U), fm.get_open_file_count()); + for (std::uint8_t i = 0u; i < handles.size(); i++) { + EXPECT_EQ(std::size_t(1u), fm.get_open_file_count()); + EXPECT_EQ(std::size_t(handles.size() - i), fm.get_open_handle_count()); + fm.close(handles[i]); + } fm.stop(); - - polling::instance().stop(); - event_system::instance().stop(); } - EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); + EXPECT_EQ(std::size_t(0u), fm.get_open_file_count()); + EXPECT_EQ(std::size_t(0u), fm.get_open_handle_count()); + + polling::instance().stop(); } -TEST(file_manager, remove_file_fails_if_file_does_not_exist) { - { - app_config cfg(provider_type::sia, file_manager_dir); - cfg.set_enable_chunk_downloader_timeout(false); - mock_provider mp; +TEST_F(file_manager_test, + download_is_stored_after_write_if_partially_downloaded) { + EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); + file_manager fm(*cfg, mp); + fm.start(); - file_manager fm(cfg, mp); + const auto source_path = utils::path::combine(cfg->get_cache_directory(), + {utils::create_uuid_string()}); - EXPECT_CALL(mp, get_filesystem_item) - .WillOnce([](const std::string &api_path, const bool &directory, - filesystem_item & /*fsi*/) -> api_error { - EXPECT_STREQ("/test_remove.txt", api_path.c_str()); - EXPECT_FALSE(directory); - return api_error::item_not_found; - }); + event_consumer es("download_resume_added", [&source_path](const event &e) { + const auto &ee = dynamic_cast(e); + EXPECT_STREQ("/test_write_partial_download.txt", + ee.get_api_path().get().c_str()); + EXPECT_STREQ(source_path.c_str(), + ee.get_dest_path().get().c_str()); + }); + event_capture ec({"download_resume_added"}, + {"file_upload_completed", "file_upload_queued"}); - EXPECT_EQ(api_error::item_not_found, fm.remove_file("/test_remove.txt")); + const auto now = utils::time::get_time_now(); + auto meta = create_meta_attributes( + now, FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_ARCHIVE, now + 1u, now + 2u, + false, 1, "key", 2, now + 3u, 3u, 4u, + utils::encryption::encrypting_reader::get_data_chunk_size() * 4u, + source_path, 10, now + 4u); + auto &nf = + test::create_random_file(utils::string::to_uint64(meta[META_SIZE])); + + EXPECT_CALL(mp, get_filesystem_item) + .WillRepeatedly([&meta](const std::string &api_path, bool directory, + filesystem_item &fsi) -> api_error { + EXPECT_STREQ("/test_write_partial_download.txt", api_path.c_str()); + EXPECT_FALSE(directory); + fsi.api_path = api_path; + fsi.api_parent = utils::path::get_parent_api_path(api_path); + fsi.directory = directory; + fsi.size = utils::string::to_uint64(meta[META_SIZE]); + fsi.source_path = meta[META_SOURCE]; + return api_error::success; + }); + + std::uint64_t handle{}; + std::shared_ptr f; +#if defined(_WIN32) + EXPECT_EQ(api_error::success, + fm.open("/test_write_partial_download.txt", false, {}, handle, f)); +#else + EXPECT_EQ(api_error::success, fm.open("/test_write_partial_download.txt", + false, O_RDWR, handle, f)); +#endif + + EXPECT_CALL(mp, read_file_bytes) + .WillRepeatedly([&nf](const std::string & /* api_path */, + std::size_t size, std::uint64_t offset, + data_buffer &data, + stop_type &stop_requested) -> api_error { + if (stop_requested) { + return api_error::download_stopped; + } + + if (offset == 0u) { + std::size_t bytes_read{}; + data.resize(size); + auto ret = nf.read(data, offset, &bytes_read) ? api_error::success + : api_error::os_error; + EXPECT_EQ(bytes_read, data.size()); + return ret; + } + + while (not stop_requested) { + std::this_thread::sleep_for(100ms); + } + + return api_error::download_stopped; + }); + EXPECT_CALL(mp, set_item_meta("/test_write_partial_download.txt", _)) + .WillOnce( + [](const std::string &, const api_meta_map &meta2) -> api_error { + EXPECT_NO_THROW(EXPECT_FALSE(meta2.at(META_CHANGED).empty())); + EXPECT_NO_THROW(EXPECT_FALSE(meta2.at(META_MODIFIED).empty())); + EXPECT_NO_THROW(EXPECT_FALSE(meta2.at(META_WRITTEN).empty())); + return api_error::success; + }); + EXPECT_CALL(mp, upload_file).Times(0u); + + std::size_t bytes_written{}; + data_buffer data = {0, 1, 2}; + EXPECT_EQ(api_error::success, f->write(0u, data, bytes_written)); + EXPECT_EQ(std::size_t(3u), bytes_written); + f.reset(); + + fm.close(handle); + + EXPECT_EQ(std::size_t(1u), fm.get_open_file_count()); + EXPECT_EQ(std::size_t(0u), fm.get_open_handle_count()); + + fm.stop(); + ec.wait_for_empty(); + + event_capture ec2({"download_restored", "download_resume_added"}, + {"file_upload_completed", "file_upload_queued"}); + EXPECT_EQ(std::size_t(0u), fm.get_open_file_count()); + EXPECT_EQ(std::size_t(0u), fm.get_open_handle_count()); + + auto stored_downloads = fm.get_stored_downloads(); + EXPECT_EQ(std::size_t(1u), stored_downloads.size()); + std::cout << stored_downloads[0u].dump(2) << std::endl; + + EXPECT_STREQ("/test_write_partial_download.txt", + stored_downloads[0u]["path"].get().c_str()); + EXPECT_EQ(utils::encryption::encrypting_reader::get_data_chunk_size(), + stored_downloads[0u]["chunk_size"].get()); + auto read_state = utils::string::to_dynamic_bitset( + stored_downloads[0u]["read_state"].get()); + EXPECT_TRUE(read_state[0u]); + for (std::size_t i = 1u; i < read_state.size(); i++) { + EXPECT_FALSE(read_state[i]); } + EXPECT_STREQ(source_path.c_str(), + stored_downloads[0u]["source"].get().c_str()); - EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); + fm.start(); + + event_consumer es2("download_restored", [&source_path](const event &e) { + const auto &ee = dynamic_cast(e); + EXPECT_STREQ("/test_write_partial_download.txt", + ee.get_api_path().get().c_str()); + EXPECT_STREQ(source_path.c_str(), + ee.get_dest_path().get().c_str()); + }); + + EXPECT_EQ(std::size_t(1u), fm.get_open_file_count()); + EXPECT_EQ(std::size_t(0u), fm.get_open_handle_count()); + fm.stop(); + ec2.wait_for_empty(); + + EXPECT_EQ(std::size_t(0u), fm.get_open_file_count()); + EXPECT_EQ(std::size_t(0u), fm.get_open_handle_count()); + + nf.close(); } -TEST(file_manager, remove_file_fails_if_provider_remove_file_fails) { +TEST_F(file_manager_test, upload_occurs_after_write_if_fully_downloaded) { + EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); + + polling::instance().start(cfg.get()); + file_manager fm(*cfg, mp); + fm.start(); + + const auto source_path = utils::path::combine(cfg->get_cache_directory(), + {utils::create_uuid_string()}); + + event_consumer es("file_upload_queued", [&source_path](const event &e) { + const auto &ee = dynamic_cast(e); + EXPECT_STREQ("/test_write_full_download.txt", + ee.get_api_path().get().c_str()); + EXPECT_STREQ(source_path.c_str(), + ee.get_source().get().c_str()); + }); + event_consumer es2("file_upload_completed", [&source_path](const event &e) { + const auto &ee = dynamic_cast(e); + EXPECT_STREQ("/test_write_full_download.txt", + ee.get_api_path().get().c_str()); + EXPECT_STREQ(source_path.c_str(), + ee.get_source().get().c_str()); + }); + event_capture ec({"download_end"}); + + const auto now = utils::time::get_time_now(); + auto meta = create_meta_attributes( + now, FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_ARCHIVE, now + 1u, now + 2u, + false, 1, "key", 2, now + 3u, 3u, 4u, + utils::encryption::encrypting_reader::get_data_chunk_size() * 4u, + source_path, 10, now + 4u); + auto &nf = + test::create_random_file(utils::string::to_uint64(meta[META_SIZE])); + + EXPECT_CALL(mp, get_filesystem_item) + .WillRepeatedly([&meta](const std::string &api_path, bool directory, + filesystem_item &fsi) -> api_error { + EXPECT_STREQ("/test_write_full_download.txt", api_path.c_str()); + EXPECT_FALSE(directory); + fsi.api_path = api_path; + fsi.api_parent = utils::path::get_parent_api_path(api_path); + fsi.directory = directory; + fsi.size = utils::string::to_uint64(meta[META_SIZE]); + fsi.source_path = meta[META_SOURCE]; + return api_error::success; + }); + + std::uint64_t handle{}; + std::shared_ptr f; +#if defined(_WIN32) + EXPECT_EQ(api_error::success, + fm.open("/test_write_full_download.txt", false, {}, handle, f)); +#else + EXPECT_EQ(api_error::success, + fm.open("/test_write_full_download.txt", false, O_RDWR, handle, f)); +#endif + + EXPECT_CALL(mp, read_file_bytes) + .WillRepeatedly([&nf](const std::string & /* api_path */, + std::size_t size, std::uint64_t offset, + data_buffer &data, + stop_type & /* stop_requested */) -> api_error { + std::size_t bytes_read{}; + data.resize(size); + auto ret = nf.read(data, offset, &bytes_read) ? api_error::success + : api_error::os_error; + EXPECT_EQ(bytes_read, data.size()); + return ret; + }); + EXPECT_CALL(mp, set_item_meta("/test_write_full_download.txt", _)) + .WillOnce( + [](const std::string &, const api_meta_map &meta2) -> api_error { + EXPECT_NO_THROW(EXPECT_FALSE(meta2.at(META_CHANGED).empty())); + EXPECT_NO_THROW(EXPECT_FALSE(meta2.at(META_MODIFIED).empty())); + EXPECT_NO_THROW(EXPECT_FALSE(meta2.at(META_WRITTEN).empty())); + return api_error::success; + }); + std::size_t bytes_written{}; + data_buffer data = {0, 1, 2}; + EXPECT_EQ(api_error::success, f->write(0u, data, bytes_written)); + EXPECT_EQ(std::size_t(3u), bytes_written); + f.reset(); + + ec.wait_for_empty(); + + EXPECT_CALL(mp, upload_file("/test_write_full_download.txt", source_path, _)) + .WillOnce(Return(api_error::success)); + + event_capture ec2( + {"item_timeout", "file_upload_queued", "file_upload_completed"}); + fm.close(handle); + + ec2.wait_for_empty(); + + EXPECT_EQ(std::size_t(0U), fm.get_open_file_count()); + EXPECT_EQ(std::size_t(0U), fm.get_open_handle_count()); + + fm.stop(); + + nf.close(); + + polling::instance().stop(); +} + +TEST_F(file_manager_test, can_evict_file) { + EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); + + file_manager fm(*cfg, mp); + fm.start(); + + event_capture capture({ + "filesystem_item_opened", + "filesystem_item_handle_opened", + "filesystem_item_handle_closed", + "filesystem_item_closed", + "file_upload_completed", + }); + + const auto source_path = utils::path::combine(cfg->get_cache_directory(), + {utils::create_uuid_string()}); + + const auto now = utils::time::get_time_now(); + + auto meta = create_meta_attributes( + now, FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_ARCHIVE, now + 1u, now + 2u, + false, 1, "key", 2, now + 3u, 3u, 4u, 0u, source_path, 10, now + 4u); + std::uint64_t handle{}; { - app_config cfg(provider_type::sia, file_manager_dir); - cfg.set_enable_chunk_downloader_timeout(false); - mock_provider mp; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - - file_manager fm(cfg, mp); + std::shared_ptr f; + EXPECT_CALL(mp, create_file("/test_evict.txt", meta)) + .WillOnce(Return(api_error::success)); EXPECT_CALL(mp, get_filesystem_item) - .WillOnce([](const std::string &api_path, const bool &directory, - filesystem_item &fsi) -> api_error { - EXPECT_STREQ("/test_remove.txt", api_path.c_str()); + .WillRepeatedly([&meta](const std::string &api_path, bool directory, + filesystem_item &fsi) -> api_error { + EXPECT_STREQ("/test_evict.txt", api_path.c_str()); EXPECT_FALSE(directory); fsi.api_path = api_path; fsi.api_parent = utils::path::get_parent_api_path(api_path); fsi.directory = directory; - fsi.size = 0U; + fsi.size = utils::string::to_uint64(meta[META_SIZE]); + fsi.source_path = meta[META_SOURCE]; return api_error::success; }); - EXPECT_CALL(mp, remove_file("/test_remove.txt")) - .WillOnce(Return(api_error::item_not_found)); - EXPECT_EQ(api_error::item_not_found, fm.remove_file("/test_remove.txt")); +#if defined(_WIN32) + EXPECT_EQ(api_error::success, + fm.create("/test_evict.txt", meta, {}, handle, f)); +#else + EXPECT_EQ(api_error::success, + fm.create("/test_evict.txt", meta, O_RDWR, handle, f)); +#endif + EXPECT_CALL(mp, set_item_meta("/test_evict.txt", _)) + .Times(2) + .WillRepeatedly(Return(api_error::success)); + EXPECT_CALL(mp, upload_file(_, _, _)).WillOnce(Return(api_error::success)); + + data_buffer data{{0, 1, 1}}; + std::size_t bytes_written{}; + EXPECT_EQ(api_error::success, f->write(0U, data, bytes_written)); + + auto opt_size = utils::file::file{source_path}.size(); + EXPECT_TRUE(opt_size.has_value()); + EXPECT_EQ(static_cast(data.size()), opt_size.value()); } - EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); + fm.close(handle); + capture.wait_for_empty(); + + EXPECT_TRUE(utils::retry_action( + [&fm]() -> bool { return not fm.is_processing("/test_evict.txt"); })); + + EXPECT_CALL(mp, get_item_meta(_, META_SOURCE, _)) + .WillOnce([&source_path](const std::string &api_path, + const std::string &key, + std::string &value) -> api_error { + EXPECT_STREQ("/test_evict.txt", api_path.c_str()); + EXPECT_STREQ(META_SOURCE.c_str(), key.c_str()); + value = source_path; + return api_error::success; + }); + EXPECT_CALL(mp, get_item_meta(_, META_PINNED, _)) + .WillOnce([](const std::string &api_path, const std::string &key, + std::string &value) -> api_error { + EXPECT_STREQ("/test_evict.txt", api_path.c_str()); + EXPECT_STREQ(META_PINNED.c_str(), key.c_str()); + value = "0"; + return api_error::success; + }); + EXPECT_TRUE(fm.evict_file("/test_evict.txt")); + EXPECT_FALSE(utils::file::file(source_path).exists()); + + fm.stop(); } + +TEST_F(file_manager_test, evict_file_fails_if_file_is_pinned) { + EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); + file_manager fm(*cfg, mp); + + EXPECT_CALL(mp, get_item_meta(_, META_PINNED, _)) + .WillOnce([](const std::string &api_path, const std::string &key, + std::string &value) -> api_error { + EXPECT_STREQ("/test_open.txt", api_path.c_str()); + EXPECT_STREQ(META_PINNED.c_str(), key.c_str()); + value = "1"; + return api_error::success; + }); + + EXPECT_FALSE(fm.evict_file("/test_open.txt")); +} + +TEST_F(file_manager_test, evict_file_fails_if_provider_is_direct_only) { + EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(true)); + file_manager fm(*cfg, mp); + EXPECT_FALSE(fm.evict_file("/test.txt")); +} + +TEST_F(file_manager_test, evict_file_fails_if_file_is_open) { + EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); + file_manager fm(*cfg, mp); + + EXPECT_CALL(mp, get_filesystem_item) + .WillOnce([](const std::string &api_path, bool directory, + filesystem_item &fsi) -> api_error { + EXPECT_STREQ("/test_open.txt", api_path.c_str()); + EXPECT_FALSE(directory); + fsi.api_path = api_path; + fsi.api_parent = utils::path::get_parent_api_path(api_path); + fsi.directory = directory; + fsi.size = 0U; + return api_error::success; + }); + + EXPECT_CALL(mp, set_item_meta(_, _, _)) + .WillOnce([](const std::string &api_path, const std::string &key, + const std::string &value) -> api_error { + EXPECT_STREQ("/test_open.txt", api_path.c_str()); + EXPECT_STREQ(META_SOURCE.c_str(), key.c_str()); + EXPECT_FALSE(value.empty()); + return api_error::success; + }); + + std::uint64_t handle{}; + std::shared_ptr f{}; +#if defined(_WIN32) + EXPECT_EQ(api_error::success, + fm.open("/test_open.txt", false, {}, handle, f)); +#else + EXPECT_EQ(api_error::success, + fm.open("/test_open.txt", false, O_RDWR, handle, f)); +#endif + + EXPECT_FALSE(fm.evict_file("/test_open.txt")); + + fm.close(handle); +} + +// TEST(file_manager, +// evict_file_fails_if_unable_to_get_source_path_from_item_meta) { +// { +// app_config cfg(provider_type::sia, file_manager_dir); +// cfg.set_enable_chunk_downloader_timeout(false); +// mock_provider mp; +// +// EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); +// file_manager fm(cfg, mp); +// +// EXPECT_CALL(mp, get_item_meta(_, META_SOURCE, _)) +// .WillOnce([](const std::string &api_path, const std::string &key, +// std::string & /*value*/) -> api_error { +// EXPECT_STREQ("/test_open.txt", api_path.c_str()); +// EXPECT_STREQ(META_SOURCE.c_str(), key.c_str()); +// return api_error::error; +// }); +// +// EXPECT_CALL(mp, get_item_meta(_, META_PINNED, _)) +// .WillOnce([](const std::string &api_path, const std::string &key, +// std::string &value) -> api_error { +// EXPECT_STREQ("/test_open.txt", api_path.c_str()); +// EXPECT_STREQ(META_PINNED.c_str(), key.c_str()); +// value = "0"; +// return api_error::success; +// }); +// +// EXPECT_FALSE(fm.evict_file("/test_open.txt")); +// } +// +// EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); +// } +// +// TEST(file_manager, evict_file_fails_if_source_path_is_empty) { +// { +// app_config cfg(provider_type::sia, file_manager_dir); +// cfg.set_enable_chunk_downloader_timeout(false); +// mock_provider mp; +// +// EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); +// file_manager fm(cfg, mp); +// +// EXPECT_CALL(mp, get_item_meta(_, META_SOURCE, _)) +// .WillOnce([](const std::string &api_path, const std::string &key, +// std::string &value) -> api_error { +// EXPECT_STREQ("/test_open.txt", api_path.c_str()); +// EXPECT_STREQ(META_SOURCE.c_str(), key.c_str()); +// value = ""; +// return api_error::success; +// }); +// EXPECT_CALL(mp, get_item_meta(_, META_PINNED, _)) +// .WillOnce([](const std::string &api_path, const std::string &key, +// std::string &value) -> api_error { +// EXPECT_STREQ("/test_open.txt", api_path.c_str()); +// EXPECT_STREQ(META_PINNED.c_str(), key.c_str()); +// value = "0"; +// return api_error::success; +// }); +// +// EXPECT_FALSE(fm.evict_file("/test_open.txt")); +// } +// +// EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); +// } +// +// TEST(file_manager, evict_file_fails_if_file_is_uploading) { +// { +// console_consumer c; +// event_system::instance().start(); +// +// app_config cfg(provider_type::sia, file_manager_dir); +// cfg.set_enable_chunk_downloader_timeout(false); +// mock_provider mp; +// +// EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); +// +// file_manager fm(cfg, mp); +// fm.start(); +// +// event_capture capture({ +// "filesystem_item_opened", +// "filesystem_item_handle_opened", +// "filesystem_item_handle_closed", +// "filesystem_item_closed", +// "file_upload_completed", +// }); +// +// const auto source_path = utils::path::combine( +// cfg.get_cache_directory(), {utils::create_uuid_string()}); +// +// const auto now = utils::time::get_time_now(); +// +// auto meta = create_meta_attributes( +// now, FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_ARCHIVE, now + 1u, now + +// 2u, false, 1, "", 2, now + 3u, 3u, 4u, 0u, source_path, 10, now + +// 4u); +// std::uint64_t handle{}; +// { +// std::shared_ptr f; +// +// EXPECT_CALL(mp, create_file("/test_evict.txt", meta)) +// .WillOnce(Return(api_error::success)); +// EXPECT_CALL(mp, get_filesystem_item) +// .WillRepeatedly([&meta](const std::string &api_path, bool +// directory, +// filesystem_item &fsi) -> api_error { +// EXPECT_STREQ("/test_evict.txt", api_path.c_str()); +// EXPECT_FALSE(directory); +// fsi.api_path = api_path; +// fsi.api_parent = utils::path::get_parent_api_path(api_path); +// fsi.directory = directory; +// fsi.size = utils::string::to_uint64(meta[META_SIZE]); +// fsi.source_path = meta[META_SOURCE]; +// return api_error::success; +// }); +// +// #if defined(_WIN32) +// EXPECT_EQ(api_error::success, +// fm.create("/test_evict.txt", meta, {}, handle, f)); +// #else +// EXPECT_EQ(api_error::success, +// fm.create("/test_evict.txt", meta, O_RDWR, handle, f)); +// #endif +// EXPECT_CALL(mp, set_item_meta("/test_evict.txt", _)) +// .Times(2) +// .WillRepeatedly(Return(api_error::success)); +// EXPECT_CALL(mp, upload_file) +// .WillOnce([](const std::string &api_path, +// const std::string &source_path2, +// stop_type & /*stop_requested*/) -> api_error { +// EXPECT_STREQ("/test_evict.txt", api_path.c_str()); +// EXPECT_FALSE(source_path2.empty()); +// std::this_thread::sleep_for(3s); +// return api_error::success; +// }); +// +// data_buffer data{{0, 1, 1}}; +// std::size_t bytes_written{}; +// EXPECT_EQ(api_error::success, f->write(0U, data, bytes_written)); +// +// auto opt_size = utils::file::file{source_path}.size(); +// EXPECT_TRUE(opt_size.has_value()); +// EXPECT_EQ(static_cast(data.size()), opt_size.value()); +// +// fm.close(handle); +// +// EXPECT_TRUE(utils::retry_action( +// [&fm]() -> bool { return fm.is_processing("/test_evict.txt"); })); +// EXPECT_FALSE(fm.evict_file("/test_evict.txt")); +// } +// +// capture.wait_for_empty(); +// +// EXPECT_TRUE(utils::file::file(source_path).exists()); +// +// fm.stop(); +// +// event_system::instance().stop(); +// } +// +// EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); +// } +// +// TEST(file_manager, evict_file_fails_if_file_is_in_upload_queue) { +// { +// app_config cfg(provider_type::sia, file_manager_dir); +// cfg.set_enable_chunk_downloader_timeout(false); +// mock_provider mp; +// +// EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); +// +// file_manager fm(cfg, mp); +// +// mock_open_file of{}; +// EXPECT_CALL(of, is_directory).WillRepeatedly(Return(false)); +// EXPECT_CALL(of, get_api_path).WillRepeatedly(Return("/test_evict.txt")); +// EXPECT_CALL(of, +// get_source_path).WillRepeatedly(Return("/test_evict.src")); +// fm.queue_upload(of); +// +// EXPECT_TRUE(fm.is_processing("/test_evict.txt")); +// EXPECT_FALSE(fm.evict_file("/test_evict.txt")); +// } +// +// EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); +// } +// +// TEST(file_manager, evict_file_fails_if_file_is_modified) { +// { +// app_config cfg(provider_type::sia, file_manager_dir); +// cfg.set_enable_chunk_downloader_timeout(false); +// mock_provider mp; +// +// EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); +// +// file_manager fm(cfg, mp); +// EXPECT_CALL(mp, get_filesystem_item) +// .WillOnce([](const std::string &api_path, bool directory, +// filesystem_item &fsi) -> api_error { +// EXPECT_STREQ("/test_evict.txt", api_path.c_str()); +// EXPECT_FALSE(directory); +// fsi.api_path = api_path; +// fsi.api_parent = utils::path::get_parent_api_path(api_path); +// fsi.directory = directory; +// fsi.size = 1U; +// fsi.source_path = "/test_evict.src"; +// return api_error::success; +// }); +// +// auto of = std::make_shared(); +// EXPECT_CALL(*of, is_directory).WillOnce(Return(false)); +// EXPECT_CALL(*of, add).WillOnce(Return()); +// EXPECT_CALL(*of, get_api_path).WillRepeatedly(Return("/test_evict.txt")); +// EXPECT_CALL(*of, +// get_source_path).WillRepeatedly(Return("/test_evict.src")); +// EXPECT_CALL(*of, is_modified).Times(2).WillRepeatedly(Return(true)); +// +// std::uint64_t handle{}; +// std::shared_ptr f; +// #if defined(_WIN32) +// EXPECT_EQ(api_error::success, fm.open(of, {}, handle, f)); +// #else +// EXPECT_EQ(api_error::success, fm.open(of, O_RDWR, handle, f)); +// #endif +// +// EXPECT_TRUE(fm.is_processing("/test_evict.txt")); +// EXPECT_FALSE(fm.evict_file("/test_evict.txt")); +// } +// +// EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); +// } +// +// TEST(file_manager, evict_file_fails_if_file_is_not_complete) { +// { +// app_config cfg(provider_type::sia, file_manager_dir); +// cfg.set_enable_chunk_downloader_timeout(false); +// mock_provider mp; +// +// EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); +// +// file_manager fm(cfg, mp); +// EXPECT_CALL(mp, get_filesystem_item) +// .WillOnce([](const std::string &api_path, bool directory, +// filesystem_item &fsi) -> api_error { +// EXPECT_STREQ("/test_evict.txt", api_path.c_str()); +// EXPECT_FALSE(directory); +// fsi.api_path = api_path; +// fsi.api_parent = utils::path::get_parent_api_path(api_path); +// fsi.directory = directory; +// fsi.size = 1U; +// return api_error::success; +// }); +// +// auto of = std::make_shared(); +// EXPECT_CALL(*of, is_directory).WillOnce(Return(false)); +// EXPECT_CALL(*of, add).WillOnce(Return()); +// EXPECT_CALL(*of, get_api_path).WillRepeatedly(Return("/test_evict.txt")); +// EXPECT_CALL(*of, +// get_source_path).WillRepeatedly(Return("/test_evict.src")); +// EXPECT_CALL(*of, is_modified).Times(2).WillRepeatedly(Return(false)); +// EXPECT_CALL(*of, is_complete).Times(2).WillRepeatedly(Return(false)); +// EXPECT_CALL(mp, set_item_meta("/test_evict.txt", META_SOURCE, _)) +// .WillOnce(Return(api_error::success)); +// +// std::uint64_t handle{}; +// std::shared_ptr f; +// #if defined(_WIN32) +// EXPECT_EQ(api_error::success, fm.open(of, {}, handle, f)); +// #else +// EXPECT_EQ(api_error::success, fm.open(of, O_RDWR, handle, f)); +// #endif +// +// EXPECT_TRUE(fm.is_processing("/test_evict.txt")); +// EXPECT_FALSE(fm.evict_file("/test_evict.txt")); +// } +// +// EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); +// } +// +// TEST(file_manager, can_get_directory_items) { +// { +// app_config cfg(provider_type::sia, file_manager_dir); +// cfg.set_enable_chunk_downloader_timeout(false); +// mock_provider mp; +// +// EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); +// +// file_manager fm(cfg, mp); +// EXPECT_CALL(mp, get_directory_items) +// .WillOnce([](const std::string &api_path, +// directory_item_list &list) -> api_error { +// EXPECT_STREQ("/", api_path.c_str()); +// list.insert(list.begin(), directory_item{ +// "..", +// "", +// true, +// }); +// list.insert(list.begin(), directory_item{ +// ".", +// "", +// true, +// }); +// return api_error::success; +// }); +// auto list = fm.get_directory_items("/"); +// EXPECT_EQ(std::size_t(2U), list.size()); +// } +// +// EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); +// } +// +// TEST(file_manager, file_is_not_opened_if_provider_create_file_fails) { +// { +// app_config cfg(provider_type::sia, file_manager_dir); +// cfg.set_enable_chunk_downloader_timeout(false); +// mock_provider mp; +// +// EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); +// +// const auto now = utils::time::get_time_now(); +// auto meta = create_meta_attributes( +// now, FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_ARCHIVE, now + 1u, now + +// 2u, false, 1, "", 2, now + 3u, 3u, 4u, 0u, "/test_create.src", 10, +// now + 4u); +// file_manager fm(cfg, mp); +// +// EXPECT_CALL(mp, create_file("/test_create.txt", meta)) +// .WillOnce(Return(api_error::error)); +// +// std::uint64_t handle{}; +// std::shared_ptr f; +// #if defined(_WIN32) +// EXPECT_EQ(api_error::error, +// fm.create("/test_create.txt", meta, {}, handle, f)); +// #else +// EXPECT_EQ(api_error::error, +// fm.create("/test_create.txt", meta, O_RDWR, handle, f)); +// #endif +// EXPECT_FALSE(f); +// EXPECT_EQ(std::size_t(0U), fm.get_open_file_count()); +// } +// +// EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); +// } +// +// TEST(file_manager, create_fails_if_provider_create_is_unsuccessful) { +// { +// app_config cfg(provider_type::sia, file_manager_dir); +// cfg.set_enable_chunk_downloader_timeout(false); +// mock_provider mp; +// +// EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); +// +// file_manager fm(cfg, mp); +// +// EXPECT_CALL(mp, create_file("/test_create.txt", _)) +// .WillOnce(Return(api_error::error)); +// +// std::uint64_t handle{}; +// std::shared_ptr f{}; +// api_meta_map meta{}; +// #if defined(_WIN32) +// EXPECT_EQ(api_error::error, +// fm.create("/test_create.txt", meta, {}, handle, f)); +// #else +// EXPECT_EQ(api_error::error, +// fm.create("/test_create.txt", meta, O_RDWR, handle, f)); +// #endif +// EXPECT_EQ(std::size_t(0U), fm.get_open_file_count()); +// } +// +// EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); +// } +// +// TEST(file_manager, get_open_file_fails_if_file_is_not_open) { +// { +// app_config cfg(provider_type::sia, file_manager_dir); +// cfg.set_enable_chunk_downloader_timeout(false); +// mock_provider mp; +// +// EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); +// +// file_manager fm(cfg, mp); +// +// std::shared_ptr f{}; +// EXPECT_FALSE(fm.get_open_file(0U, true, f)); +// EXPECT_FALSE(f); +// +// EXPECT_FALSE(fm.get_open_file(0U, false, f)); +// EXPECT_FALSE(f); +// } +// +// EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); +// } +// +// TEST(file_manager, +// get_open_file_promotes_non_writeable_file_if_writeable_is_specified) { +// { +// app_config cfg(provider_type::sia, file_manager_dir); +// cfg.set_enable_chunk_downloader_timeout(false); +// mock_provider mp; +// +// EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); +// +// file_manager fm(cfg, mp); +// +// auto non_writeable = std::make_shared(); +// EXPECT_CALL(*non_writeable, is_directory).WillOnce(Return(false)); +// EXPECT_CALL(*non_writeable, add).WillOnce(Return()); +// EXPECT_CALL(*non_writeable, get_api_path) +// .WillRepeatedly(Return("/test_open.txt")); +// EXPECT_CALL(*non_writeable, get_source_path) +// .WillRepeatedly(Return("/test_open.src")); +// EXPECT_CALL(*non_writeable, is_modified).WillRepeatedly(Return(true)); +// EXPECT_CALL(*non_writeable, is_write_supported) +// .WillRepeatedly(Return(false)); +// EXPECT_CALL(*non_writeable, is_write_supported) +// .WillRepeatedly(Return(false)); +// +// EXPECT_CALL(*non_writeable, get_filesystem_item) +// .WillOnce([api_path = "/test_open.txt"]() -> filesystem_item { +// filesystem_item fsi{}; +// fsi.api_path = api_path; +// fsi.api_parent = utils::path::get_parent_api_path(api_path); +// fsi.directory = false; +// fsi.size = 10U; +// fsi.source_path = "/test_open.src"; +// return fsi; +// }); +// +// EXPECT_CALL(mp, get_filesystem_item) +// .WillOnce([](const std::string &api_path, bool directory, +// filesystem_item &fsi) -> api_error { +// EXPECT_STREQ("/test_open.txt", api_path.c_str()); +// EXPECT_FALSE(directory); +// fsi.api_path = api_path; +// fsi.api_parent = utils::path::get_parent_api_path(api_path); +// fsi.directory = directory; +// fsi.size = 10U; +// fsi.source_path = "/test_open.src"; +// return api_error::success; +// }); +// +// std::uint64_t handle{}; +// std::shared_ptr f{}; +// #if defined(_WIN32) +// EXPECT_EQ(api_error::success, fm.open(non_writeable, {}, handle, f)); +// EXPECT_CALL(*non_writeable, get_open_data()) +// .WillOnce([&handle]() -> std::map & { +// static std::map map; +// map[handle] = {}; +// return map; +// }); +// #else +// EXPECT_EQ(api_error::success, fm.open(non_writeable, O_RDWR, handle, f)); +// EXPECT_CALL(*non_writeable, get_open_data()) +// .WillOnce([&handle]() -> std::map & { +// static std::map map; +// map[handle] = O_RDWR; +// return map; +// }); +// #endif +// +// EXPECT_CALL(mp, set_item_meta("/test_open.txt", META_SOURCE, _)) +// .WillOnce(Return(api_error::success)); +// +// EXPECT_CALL(*non_writeable, has_handle(1)).WillOnce([]() -> bool { +// return true; +// }); +// EXPECT_TRUE(fm.get_open_file(handle, true, f)); +// EXPECT_NE(non_writeable.get(), f.get()); +// EXPECT_EQ(std::size_t(1U), fm.get_open_file_count()); +// +// std::shared_ptr f2{}; +// EXPECT_TRUE(fm.get_open_file(handle, false, f2)); +// EXPECT_EQ(f.get(), f2.get()); +// EXPECT_EQ(std::size_t(1U), fm.get_open_file_count()); +// } +// +// EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); +// } +// +// TEST(file_manager, open_file_fails_if_file_is_not_found) { +// { +// app_config cfg(provider_type::sia, file_manager_dir); +// cfg.set_enable_chunk_downloader_timeout(false); +// mock_provider mp; +// +// EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); +// +// file_manager fm(cfg, mp); +// +// std::shared_ptr f{}; +// EXPECT_FALSE(fm.get_open_file(1U, true, f)); +// EXPECT_EQ(std::size_t(0U), fm.get_open_file_count()); +// EXPECT_FALSE(f); +// +// EXPECT_FALSE(fm.get_open_file(1U, false, f)); +// EXPECT_FALSE(f); +// EXPECT_EQ(std::size_t(0U), fm.get_open_file_count()); +// } +// +// EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); +// } +// +// TEST(file_manager, open_file_fails_if_provider_get_filesystem_item_fails) { +// { +// app_config cfg(provider_type::sia, file_manager_dir); +// cfg.set_enable_chunk_downloader_timeout(false); +// mock_provider mp; +// +// EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); +// +// file_manager fm(cfg, mp); +// +// auto of = std::make_shared(); +// EXPECT_CALL(*of, is_directory).WillRepeatedly(Return(false)); +// EXPECT_CALL(*of, get_api_path).WillRepeatedly(Return("/test_open.txt")); +// EXPECT_CALL(*of, +// get_source_path).WillRepeatedly(Return("/test_open.src")); +// +// EXPECT_CALL(mp, get_filesystem_item) +// .WillOnce([](const std::string &api_path, bool directory, +// filesystem_item & /*fsi*/) -> api_error { +// EXPECT_STREQ("/test_open.txt", api_path.c_str()); +// EXPECT_FALSE(directory); +// return api_error::error; +// }); +// +// std::uint64_t handle{}; +// std::shared_ptr f{}; +// #if defined(_WIN32) +// EXPECT_EQ(api_error::error, fm.open(of, {}, handle, f)); +// #else +// EXPECT_EQ(api_error::error, fm.open(of, O_RDWR, handle, f)); +// #endif +// EXPECT_FALSE(fm.get_open_file(1U, true, f)); +// EXPECT_EQ(std::size_t(0U), fm.get_open_file_count()); +// EXPECT_FALSE(f); +// +// EXPECT_FALSE(fm.get_open_file(1U, false, f)); +// EXPECT_FALSE(f); +// EXPECT_EQ(std::size_t(0U), fm.get_open_file_count()); +// } +// +// EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); +// } +// +// TEST(file_manager, open_file_fails_if_provider_set_item_meta_fails) { +// { +// app_config cfg(provider_type::sia, file_manager_dir); +// cfg.set_enable_chunk_downloader_timeout(false); +// mock_provider mp; +// +// EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); +// +// file_manager fm(cfg, mp); +// +// auto of = std::make_shared(); +// EXPECT_CALL(*of, is_directory).WillRepeatedly(Return(false)); +// EXPECT_CALL(*of, get_api_path).WillRepeatedly(Return("/test_open.txt")); +// EXPECT_CALL(*of, +// get_source_path).WillRepeatedly(Return("/test_open.src")); +// +// EXPECT_CALL(mp, get_filesystem_item) +// .WillOnce([](const std::string &api_path, bool directory, +// filesystem_item &fsi) -> api_error { +// EXPECT_STREQ("/test_open.txt", api_path.c_str()); +// EXPECT_FALSE(directory); +// fsi.api_path = api_path; +// fsi.api_parent = utils::path::get_parent_api_path(api_path); +// fsi.directory = directory; +// fsi.size = 0U; +// return api_error::success; +// }); +// +// EXPECT_CALL(mp, set_item_meta("/test_open.txt", META_SOURCE, _)) +// .WillOnce(Return(api_error::error)); +// +// std::uint64_t handle{}; +// std::shared_ptr f{}; +// #if defined(_WIN32) +// EXPECT_EQ(api_error::error, fm.open(of, {}, handle, f)); +// #else +// EXPECT_EQ(api_error::error, fm.open(of, O_RDWR, handle, f)); +// #endif +// EXPECT_FALSE(fm.get_open_file(1U, true, f)); +// EXPECT_EQ(std::size_t(0U), fm.get_open_file_count()); +// EXPECT_FALSE(f); +// +// EXPECT_FALSE(fm.get_open_file(1U, false, f)); +// EXPECT_FALSE(f); +// EXPECT_EQ(std::size_t(0U), fm.get_open_file_count()); +// } +// +// EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); +// } +// +// TEST(file_manager, open_file_creates_source_path_if_empty) { +// { +// app_config cfg(provider_type::sia, file_manager_dir); +// cfg.set_enable_chunk_downloader_timeout(false); +// mock_provider mp; +// +// EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); +// +// file_manager fm(cfg, mp); +// +// auto of = std::make_shared(); +// EXPECT_CALL(*of, add).WillOnce(Return()); +// EXPECT_CALL(*of, is_directory).WillRepeatedly(Return(false)); +// EXPECT_CALL(*of, is_write_supported).WillRepeatedly(Return(true)); +// EXPECT_CALL(*of, get_api_path).WillRepeatedly(Return("/test_open.txt")); +// EXPECT_CALL(*of, get_source_path).WillRepeatedly(Return("")); +// +// EXPECT_CALL(mp, get_filesystem_item) +// .WillOnce([](const std::string &api_path, bool directory, +// filesystem_item &fsi) -> api_error { +// EXPECT_STREQ("/test_open.txt", api_path.c_str()); +// EXPECT_FALSE(directory); +// fsi.api_path = api_path; +// fsi.api_parent = utils::path::get_parent_api_path(api_path); +// fsi.directory = directory; +// fsi.size = 0U; +// return api_error::success; +// }); +// +// EXPECT_CALL(mp, set_item_meta("/test_open.txt", _, _)) +// .WillOnce([](const std::string &api_path, const std::string &key, +// const std::string &value) -> api_error { +// EXPECT_STREQ("/test_open.txt", api_path.c_str()); +// EXPECT_STREQ(META_SOURCE.c_str(), key.c_str()); +// EXPECT_FALSE(value.empty()); +// return api_error::success; +// }); +// +// std::uint64_t handle{}; +// std::shared_ptr f{}; +// #if defined(_WIN32) +// EXPECT_EQ(api_error::success, fm.open(of, {}, handle, f)); +// #else +// EXPECT_EQ(api_error::success, fm.open(of, O_RDWR, handle, f)); +// #endif +// EXPECT_CALL(*of, has_handle(1)).Times(2).WillRepeatedly([]() -> bool { +// return true; +// }); +// +// EXPECT_TRUE(fm.get_open_file(1U, true, f)); +// EXPECT_EQ(std::size_t(1U), fm.get_open_file_count()); +// EXPECT_TRUE(f); +// +// EXPECT_TRUE(fm.get_open_file(1U, false, f)); +// EXPECT_TRUE(f); +// EXPECT_EQ(std::size_t(1U), fm.get_open_file_count()); +// } +// +// EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); +// } +// +// TEST(file_manager, open_file_first_file_handle_is_not_zero) { +// { +// app_config cfg(provider_type::sia, file_manager_dir); +// cfg.set_enable_chunk_downloader_timeout(false); +// mock_provider mp; +// +// EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); +// +// file_manager fm(cfg, mp); +// +// auto of = std::make_shared(); +// EXPECT_CALL(*of, add).WillOnce(Return()); +// EXPECT_CALL(*of, is_directory).WillRepeatedly(Return(false)); +// EXPECT_CALL(*of, is_write_supported).WillRepeatedly(Return(true)); +// EXPECT_CALL(*of, get_api_path).WillRepeatedly(Return("/test_open.txt")); +// EXPECT_CALL(*of, +// get_source_path).WillRepeatedly(Return("/test_open.src")); +// +// EXPECT_CALL(mp, get_filesystem_item) +// .WillOnce([](const std::string &api_path, bool directory, +// filesystem_item &fsi) -> api_error { +// EXPECT_STREQ("/test_open.txt", api_path.c_str()); +// EXPECT_FALSE(directory); +// fsi.api_path = api_path; +// fsi.api_parent = utils::path::get_parent_api_path(api_path); +// fsi.directory = directory; +// fsi.size = 0U; +// fsi.source_path = "/test_open.src"; +// return api_error::success; +// }); +// +// std::uint64_t handle{}; +// std::shared_ptr f{}; +// #if defined(_WIN32) +// EXPECT_EQ(api_error::success, fm.open(of, {}, handle, f)); +// #else +// EXPECT_EQ(api_error::success, fm.open(of, O_RDWR, handle, f)); +// #endif +// EXPECT_CALL(*of, has_handle(1)).WillOnce([]() -> bool { return true; }); +// +// EXPECT_TRUE(fm.get_open_file(1U, true, f)); +// EXPECT_GT(handle, std::uint64_t(0U)); +// } +// +// EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); +// } +// +// TEST(file_manager, can_remove_file) { +// { +// app_config cfg(provider_type::sia, file_manager_dir); +// cfg.set_enable_chunk_downloader_timeout(false); +// mock_provider mp; +// +// EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); +// +// file_manager fm(cfg, mp); +// +// { +// auto file = +// utils::file::file::open_or_create_file("./test_remove.txt"); +// EXPECT_TRUE(*file); +// } +// EXPECT_TRUE(utils::file::file("./test_remove.txt").exists()); +// +// EXPECT_CALL(mp, get_filesystem_item) +// .WillOnce([](const std::string &api_path, bool directory, +// filesystem_item &fsi) -> api_error { +// EXPECT_STREQ("/test_remove.txt", api_path.c_str()); +// EXPECT_FALSE(directory); +// fsi.api_path = api_path; +// fsi.api_parent = utils::path::get_parent_api_path(api_path); +// fsi.directory = directory; +// fsi.size = 0U; +// fsi.source_path = "./test_remove.txt"; +// return api_error::success; +// }); +// EXPECT_CALL(mp, remove_file("/test_remove.txt")) +// .WillOnce(Return(api_error::success)); +// +// EXPECT_EQ(api_error::success, fm.remove_file("/test_remove.txt")); +// +// EXPECT_FALSE(utils::file::file("./test_remove.txt").exists()); +// } +// +// EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); +// } +// +// TEST(file_manager, can_queue_and_remove_upload) { +// { +// console_consumer c; +// event_system::instance().start(); +// event_capture ec({"file_upload_queued", "download_resume_removed"}); +// +// app_config cfg(provider_type::sia, file_manager_dir); +// cfg.set_enable_chunk_downloader_timeout(false); +// mock_provider mp; +// +// EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); +// +// file_manager fm(cfg, mp); +// +// mock_open_file of{}; +// EXPECT_CALL(of, get_api_path).WillOnce(Return("/test_queue.txt")); +// EXPECT_CALL(of, get_source_path).WillOnce(Return("/test_queue.src")); +// +// EXPECT_FALSE(fm.is_processing("/test_queue.txt")); +// fm.queue_upload(of); +// EXPECT_TRUE(fm.is_processing("/test_queue.txt")); +// +// fm.remove_upload("/test_queue.txt"); +// EXPECT_FALSE(fm.is_processing("/test_queue.txt")); +// +// ec.wait_for_empty(); +// +// event_system::instance().stop(); +// } +// +// EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); +// } +// +// TEST(file_manager, file_is_closed_after_download_timeout) { +// { +// console_consumer c; +// event_system::instance().start(); +// +// app_config cfg(provider_type::sia, file_manager_dir); +// cfg.set_chunk_downloader_timeout_secs(3U); +// polling::instance().start(&cfg); +// +// mock_provider mp; +// +// EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); +// +// file_manager fm(cfg, mp); +// fm.start(); +// +// const auto source_path = utils::path::combine( +// cfg.get_cache_directory(), {utils::create_uuid_string()}); +// +// event_consumer es("item_timeout", [](const event &e) { +// const auto &ee = dynamic_cast(e); +// EXPECT_STREQ("/test_download_timeout.txt", +// ee.get_api_path().get().c_str()); +// }); +// +// const auto now = utils::time::get_time_now(); +// auto meta = create_meta_attributes( +// now, FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_ARCHIVE, now + 1u, now + +// 2u, false, 1, "key", 2, now + 3u, 3u, 4u, +// utils::encryption::encrypting_reader::get_data_chunk_size() * 4u, +// source_path, 10, now + 4u); +// +// EXPECT_CALL(mp, get_filesystem_item) +// .WillRepeatedly([&meta](const std::string &api_path, bool directory, +// filesystem_item &fsi) -> api_error { +// EXPECT_STREQ("/test_download_timeout.txt", api_path.c_str()); +// EXPECT_FALSE(directory); +// fsi.api_path = api_path; +// fsi.api_parent = utils::path::get_parent_api_path(api_path); +// fsi.directory = directory; +// fsi.size = utils::string::to_uint64(meta[META_SIZE]); +// fsi.source_path = meta[META_SOURCE]; +// return api_error::success; +// }); +// +// event_capture ec({"item_timeout"}); +// +// std::uint64_t handle{}; +// std::shared_ptr f; +// #if defined(_WIN32) +// EXPECT_EQ(api_error::success, +// fm.open("/test_download_timeout.txt", false, {}, handle, f)); +// #else +// EXPECT_EQ(api_error::success, +// fm.open("/test_download_timeout.txt", false, O_RDWR, handle, +// f)); +// #endif +// +// EXPECT_CALL(mp, read_file_bytes) +// .WillRepeatedly([](const std::string & /* api_path */, +// std::size_t /*size*/, std::uint64_t offset, +// data_buffer & /*data*/, +// stop_type &stop_requested) -> api_error { +// if (stop_requested) { +// return api_error::download_stopped; +// } +// +// if (offset == 0u) { +// return api_error::success; +// } +// +// while (not stop_requested) { +// std::this_thread::sleep_for(100ms); +// } +// +// return api_error::download_stopped; +// }); +// +// data_buffer data{}; +// EXPECT_EQ(api_error::success, f->read(1U, 0U, data)); +// +// fm.close(handle); +// +// EXPECT_CALL(mp, set_item_meta("/test_download_timeout.txt", META_SOURCE, +// _)) +// .WillOnce(Return(api_error::success)); +// +// EXPECT_EQ(std::size_t(1U), fm.get_open_file_count()); +// ec.wait_for_empty(); +// +// EXPECT_EQ(std::size_t(0U), fm.get_open_file_count()); +// fm.stop(); +// +// polling::instance().stop(); +// event_system::instance().stop(); +// } +// +// EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); +// } +// +// TEST(file_manager, remove_file_fails_if_file_does_not_exist) { +// { +// app_config cfg(provider_type::sia, file_manager_dir); +// cfg.set_enable_chunk_downloader_timeout(false); +// mock_provider mp; +// +// EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); +// +// file_manager fm(cfg, mp); +// +// EXPECT_CALL(mp, get_filesystem_item) +// .WillOnce([](const std::string &api_path, const bool &directory, +// filesystem_item & /*fsi*/) -> api_error { +// EXPECT_STREQ("/test_remove.txt", api_path.c_str()); +// EXPECT_FALSE(directory); +// return api_error::item_not_found; +// }); +// +// EXPECT_EQ(api_error::item_not_found, fm.remove_file("/test_remove.txt")); +// } +// +// EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); +// } +// +// TEST(file_manager, remove_file_fails_if_provider_remove_file_fails) { +// { +// app_config cfg(provider_type::sia, file_manager_dir); +// cfg.set_enable_chunk_downloader_timeout(false); +// mock_provider mp; +// +// EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); +// +// file_manager fm(cfg, mp); +// +// EXPECT_CALL(mp, get_filesystem_item) +// .WillOnce([](const std::string &api_path, const bool &directory, +// filesystem_item &fsi) -> api_error { +// EXPECT_STREQ("/test_remove.txt", api_path.c_str()); +// EXPECT_FALSE(directory); +// fsi.api_path = api_path; +// fsi.api_parent = utils::path::get_parent_api_path(api_path); +// fsi.directory = directory; +// fsi.size = 0U; +// return api_error::success; +// }); +// EXPECT_CALL(mp, remove_file("/test_remove.txt")) +// .WillOnce(Return(api_error::item_not_found)); +// +// EXPECT_EQ(api_error::item_not_found, fm.remove_file("/test_remove.txt")); +// } +// +// EXPECT_TRUE(utils::file::directory(file_manager_dir).remove_recursively()); +// } } // namespace repertory diff --git a/support/include/utils/db/sqlite/db_common.hpp b/support/include/utils/db/sqlite/db_common.hpp index 666559d0..b379dd67 100644 --- a/support/include/utils/db/sqlite/db_common.hpp +++ b/support/include/utils/db/sqlite/db_common.hpp @@ -28,17 +28,13 @@ namespace repertory::utils::db::sqlite { using db_types_t = std::variant; -struct sqlite3_deleter { - void operator()(sqlite3 *db3) const { - if (db3 != nullptr) { - sqlite3_close_v2(db3); - } - } +struct sqlite3_deleter final { + void operator()(sqlite3 *db3) const; }; using db3_t = std::unique_ptr; -struct sqlite3_statement_deleter { +struct sqlite3_statement_deleter final { void operator()(sqlite3_stmt *stmt) const { if (stmt != nullptr) { sqlite3_finalize(stmt); @@ -104,16 +100,7 @@ public: } #if defined(PROJECT_ENABLE_JSON) - [[nodiscard]] auto get_value_as_json() const -> nlohmann::json { - return std::visit( - overloaded{ - [this](std::int64_t value) -> auto { - return nlohmann::json({{name_, value}}); - }, - [](auto &&value) -> auto { return nlohmann::json::parse(value); }, - }, - value_); - } + [[nodiscard]] auto get_value_as_json() const -> nlohmann::json; #endif // defined(PROJECT_ENABLE_JSON) }; @@ -211,7 +198,8 @@ public: [[nodiscard]] auto get_error() const -> std::int32_t { return res_; } [[nodiscard]] auto get_error_str() const -> std::string { - return sqlite3_errstr(res_); + auto &&err_msg = sqlite3_errstr(res_); + return err_msg == nullptr ? std::to_string(res_) : err_msg; } [[nodiscard]] auto get_row(std::optional> &row) const -> bool { diff --git a/support/include/utils/error.hpp b/support/include/utils/error.hpp index 9d809b43..13ff7693 100644 --- a/support/include/utils/error.hpp +++ b/support/include/utils/error.hpp @@ -72,8 +72,7 @@ extern std::atomic exception_handler; } #endif // defined(PROJECT_ENABLE_TESTING) -[[nodiscard]] auto create_error_message(std::string_view function_name, - std::vector items) +[[nodiscard]] auto create_error_message(std::vector items) -> std::string; void handle_error(std::string_view function_name, std::string_view msg); diff --git a/support/src/utils/db/sqlite/db_common.cpp b/support/src/utils/db/sqlite/db_common.cpp index 8fc9257a..5caf4872 100644 --- a/support/src/utils/db/sqlite/db_common.cpp +++ b/support/src/utils/db/sqlite/db_common.cpp @@ -19,13 +19,63 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -#include "utils/db/sqlite/db_common.hpp" - #if defined(PROJECT_ENABLE_SQLITE) +#include "utils/db/sqlite/db_common.hpp" + +#include "utils/common.hpp" +#include "utils/error.hpp" + namespace repertory::utils::db::sqlite { -auto execute_sql(sqlite3 &db3, const std::string &sql, - std::string &err) -> bool { +void sqlite3_deleter::operator()(sqlite3 *db3) const { + REPERTORY_USES_FUNCTION_NAME(); + + if (db3 == nullptr) { + return; + } + + utils::error::handle_error(function_name, "closing database handle"); + sqlite3_db_cacheflush(db3); + + std::string err_msg; + execute_sql(*db3, "PRAGMA wal_checkpoint(full);", err_msg); + + if (not utils::retry_action( + [&db3]() -> bool { + auto res = sqlite3_close_v2(db3); + if (res == SQLITE_OK) { + return true; + } + + auto &&err_str = sqlite3_errstr(res); + utils::error::handle_error( + function_name, + utils::error::create_error_message({ + "failed to close database", + (err_str == nullptr ? std::to_string(res) : err_str), + })); + return false; + }, + 60U)) { + utils::error::handle_error(function_name, "failed to close database"); + } +} + +#if defined(PROJECT_ENABLE_JSON) +auto db_column::get_value_as_json() const -> nlohmann::json { + return std::visit( + overloaded{ + [this](std::int64_t value) -> auto { + return nlohmann::json({{name_, value}}); + }, + [](auto &&value) -> auto { return nlohmann::json::parse(value); }, + }, + value_); +} +#endif // defined(PROJECT_ENABLE_JSON) + +auto execute_sql(sqlite3 &db3, const std::string &sql, std::string &err) + -> bool { char *err_msg{nullptr}; auto res = sqlite3_exec(&db3, sql.c_str(), nullptr, nullptr, &err_msg); if (err_msg != nullptr) { diff --git a/support/src/utils/db/sqlite/db_delete.cpp b/support/src/utils/db/sqlite/db_delete.cpp index 0ef0ad3c..e6c85205 100644 --- a/support/src/utils/db/sqlite/db_delete.cpp +++ b/support/src/utils/db/sqlite/db_delete.cpp @@ -24,7 +24,10 @@ #if defined(PROJECT_ENABLE_SQLITE) namespace repertory::utils::db::sqlite { -void db_delete::context::clear() { where_data.reset(); } +void db_delete::context::clear() { + // stmt.reset(); + where_data.reset(); +} auto db_delete::context::db_delete_op_t::dump() const -> std::string { return db_delete{ctx}.dump(); diff --git a/support/src/utils/db/sqlite/db_insert.cpp b/support/src/utils/db/sqlite/db_insert.cpp index 47cd62db..602e74c8 100644 --- a/support/src/utils/db/sqlite/db_insert.cpp +++ b/support/src/utils/db/sqlite/db_insert.cpp @@ -24,10 +24,13 @@ #if defined(PROJECT_ENABLE_SQLITE) namespace repertory::utils::db::sqlite { -void db_insert::context::clear() { values.clear(); } +void db_insert::context::clear() { + // stmt.reset(); + values.clear(); +} -auto db_insert::column_value(std::string column_name, - db_types_t value) -> db_insert & { +auto db_insert::column_value(std::string column_name, db_types_t value) + -> db_insert & { ctx_->values[column_name] = value; return *this; } diff --git a/support/src/utils/db/sqlite/db_select.cpp b/support/src/utils/db/sqlite/db_select.cpp index ec6650a2..8c1098ae 100644 --- a/support/src/utils/db/sqlite/db_select.cpp +++ b/support/src/utils/db/sqlite/db_select.cpp @@ -31,6 +31,7 @@ void db_select::context::clear() { limit.reset(); offset.reset(); order_by.reset(); + // stmt.reset(); where_data.reset(); } @@ -72,8 +73,8 @@ auto db_select::column(std::string column_name) -> db_select & { return *this; } -auto db_select::count(std::string column_name, - std::string as_column_name) -> db_select & { +auto db_select::count(std::string column_name, std::string as_column_name) + -> db_select & { ctx_->count_columns[column_name] = as_column_name; return *this; } @@ -209,8 +210,8 @@ auto db_select::offset(std::int32_t value) -> db_select & { return *this; } -auto db_select::order_by(std::string column_name, - bool ascending) -> db_select & { +auto db_select::order_by(std::string column_name, bool ascending) + -> db_select & { ctx_->order_by = {column_name, ascending}; return *this; } diff --git a/support/src/utils/db/sqlite/db_update.cpp b/support/src/utils/db/sqlite/db_update.cpp index 23360487..94c6a06e 100644 --- a/support/src/utils/db/sqlite/db_update.cpp +++ b/support/src/utils/db/sqlite/db_update.cpp @@ -28,6 +28,7 @@ void db_update::context::clear() { column_values.clear(); limit.reset(); order_by.reset(); + // stmt.reset(); where_data.reset(); } @@ -52,8 +53,8 @@ auto db_update::context::db_update_op_t::order_by(std::string column_name, return *this; } -auto db_update::column_value(std::string column_name, - db_types_t value) -> db_update & { +auto db_update::column_value(std::string column_name, db_types_t value) + -> db_update & { ctx_->column_values[column_name] = value; return *this; } @@ -173,8 +174,8 @@ auto db_update::limit(std::int32_t value) -> db_update & { return *this; } -auto db_update::order_by(std::string column_name, - bool ascending) -> db_update & { +auto db_update::order_by(std::string column_name, bool ascending) + -> db_update & { ctx_->order_by = {column_name, ascending}; return *this; } diff --git a/support/src/utils/error.cpp b/support/src/utils/error.cpp index 4392dd07..a9f71e4b 100644 --- a/support/src/utils/error.cpp +++ b/support/src/utils/error.cpp @@ -25,10 +25,8 @@ namespace repertory::utils::error { std::atomic exception_handler{ &default_exception_handler}; -auto create_error_message(std::string_view function_name, - std::vector items) -> std::string { +auto create_error_message(std::vector items) -> std::string { std::stringstream stream{}; - stream << function_name; for (auto &&item : items) { stream << '|' << item; } diff --git a/support/src/utils/file_file.cpp b/support/src/utils/file_file.cpp index 6cc5199f..34c54062 100644 --- a/support/src/utils/file_file.cpp +++ b/support/src/utils/file_file.cpp @@ -486,7 +486,7 @@ auto file::write(const unsigned char *data, std::size_t to_write, break; } - bytes_written += res; + bytes_written += written; } flush();