From ad79c5daf5120b81b5d80b324d4a0d0a91c68391 Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Fri, 18 Oct 2024 14:22:24 -0500 Subject: [PATCH] refactor --- .../include/file_manager/file_manager.hpp | 403 +---- .../include/file_manager/open_file.hpp | 122 ++ .../include/file_manager/open_file_base.hpp | 194 +++ .../file_manager/ring_buffer_open_file.hpp | 132 ++ .../file_manager/upload.hpp} | 116 +- .../src/file_manager/file_manager.cpp | 4 + .../file_manager_open_file_base_download.cpp | 43 - ...le_manager_open_file.cpp => open_file.cpp} | 1194 +++++++------- ..._open_file_base.cpp => open_file_base.cpp} | 552 +++---- ...pen_file.cpp => ring_buffer_open_file.cpp} | 645 ++++---- .../{file_manager_upload.cpp => upload.cpp} | 130 +- ..._open_file_test.cpp => open_file_test.cpp} | 1370 ++++++++--------- ...est.cpp => ring_buffer_open_file_test.cpp} | 1176 +++++++------- ...anager_upload_test.cpp => upload_test.cpp} | 364 ++--- .../include/utils/db/sqlite/db_where_t.hpp | 4 +- support/src/utils/error.cpp | 3 +- support/src/utils/file_file.cpp | 2 + 17 files changed, 3263 insertions(+), 3191 deletions(-) create mode 100644 repertory/librepertory/include/file_manager/open_file.hpp create mode 100644 repertory/librepertory/include/file_manager/open_file_base.hpp create mode 100644 repertory/librepertory/include/file_manager/ring_buffer_open_file.hpp rename repertory/librepertory/{src/file_manager/file_manager_open_file_base_io_item.cpp => include/file_manager/upload.hpp} (50%) delete mode 100644 repertory/librepertory/src/file_manager/file_manager_open_file_base_download.cpp rename repertory/librepertory/src/file_manager/{file_manager_open_file.cpp => open_file.cpp} (85%) rename repertory/librepertory/src/file_manager/{file_manager_open_file_base.cpp => open_file_base.cpp} (73%) rename repertory/librepertory/src/file_manager/{file_manager_ring_buffer_open_file.cpp => ring_buffer_open_file.cpp} (83%) rename repertory/librepertory/src/file_manager/{file_manager_upload.cpp => upload.cpp} (95%) rename repertory/repertory_test/src/{file_manager_open_file_test.cpp => open_file_test.cpp} (97%) rename repertory/repertory_test/src/{file_manager_ring_buffer_open_file_test.cpp => ring_buffer_open_file_test.cpp} (97%) rename repertory/repertory_test/src/{file_manager_upload_test.cpp => upload_test.cpp} (97%) diff --git a/repertory/librepertory/include/file_manager/file_manager.hpp b/repertory/librepertory/include/file_manager/file_manager.hpp index 9258bc85..87332df7 100644 --- a/repertory/librepertory/include/file_manager/file_manager.hpp +++ b/repertory/librepertory/include/file_manager/file_manager.hpp @@ -27,6 +27,7 @@ #include "file_manager/i_file_manager.hpp" #include "file_manager/i_open_file.hpp" #include "file_manager/i_upload_manager.hpp" +#include "file_manager/upload.hpp" #include "platform/platform.hpp" #include "types/repertory.hpp" #include "utils/db/sqlite/db_common.hpp" @@ -39,408 +40,6 @@ class i_provider; class file_manager final : public i_file_manager, public i_upload_manager { E_CONSUMER(); -public: - class open_file_base : public i_closeable_open_file { - public: - open_file_base(std::uint64_t chunk_size, std::uint8_t chunk_timeout, - filesystem_item fsi, i_provider &provider); - - open_file_base(std::uint64_t chunk_size, std::uint8_t chunk_timeout, - filesystem_item fsi, - std::map open_data, - i_provider &provider); - - ~open_file_base() override = default; - - public: - open_file_base() = delete; - open_file_base(const open_file_base &) noexcept = delete; - open_file_base(open_file_base &&) noexcept = delete; - auto operator=(open_file_base &&) noexcept -> open_file_base & = delete; - auto operator=(const open_file_base &) noexcept - -> open_file_base & = delete; - - public: - class download final { - public: - download() = default; - - ~download() = default; - - public: - download(const download &) noexcept = delete; - download(download &&) noexcept = delete; - auto operator=(download &&) noexcept -> download & = delete; - auto operator=(const download &) noexcept -> download & = delete; - - private: - bool complete_{false}; - api_error error_{api_error::success}; - std::mutex mtx_; - std::condition_variable notify_; - - public: - void notify(const api_error &err); - - auto wait() -> api_error; - }; - - class io_item final { - public: - io_item(std::function action) : action_(std::move(action)) {} - - ~io_item() = default; - - public: - io_item() = delete; - io_item(const io_item &) noexcept = delete; - io_item(io_item &&) noexcept = delete; - auto operator=(io_item &&) noexcept -> io_item & = delete; - auto operator=(const io_item &) noexcept -> io_item & = delete; - - private: - std::function action_; - std::mutex mtx_; - std::condition_variable notify_; - std::optional result_; - - public: - void action(); - - [[nodiscard]] auto get_result() -> api_error; - }; - - protected: - std::uint64_t chunk_size_; - std::uint8_t chunk_timeout_; - filesystem_item fsi_; - std::size_t last_chunk_size_; - std::map open_data_; - i_provider &provider_; - - private: - api_error error_{api_error::success}; - mutable std::mutex error_mtx_; - stop_type io_stop_requested_{false}; - std::unique_ptr io_thread_; - - protected: - std::unordered_map> - active_downloads_; - mutable std::recursive_mutex file_mtx_; - std::atomic last_access_{ - std::chrono::system_clock::now()}; - bool modified_{false}; - std::unique_ptr nf_; - mutable std::mutex io_thread_mtx_; - std::condition_variable io_thread_notify_; - std::deque> io_thread_queue_; - bool removed_{false}; - - private: - void file_io_thread(); - - protected: - [[nodiscard]] auto do_io(std::function action) -> api_error; - - virtual auto is_download_complete() const -> bool = 0; - - void reset_timeout(); - - auto set_api_error(const api_error &e) -> api_error; - - public: - void add(std::uint64_t handle, open_file_data ofd) override; - - [[nodiscard]] auto can_close() const -> bool override; - - auto close() -> bool override; - - [[nodiscard]] auto get_api_error() const -> api_error; - - [[nodiscard]] auto get_api_path() const -> std::string override; - - [[nodiscard]] auto get_chunk_size() const -> std::size_t override { - return chunk_size_; - } - - [[nodiscard]] auto get_file_size() const -> std::uint64_t override; - - [[nodiscard]] auto get_filesystem_item() const -> filesystem_item override; - - [[nodiscard]] auto get_handles() const - -> std::vector override; - - [[nodiscard]] auto get_open_data() - -> std::map & override; - - [[nodiscard]] auto get_open_data() const - -> const std::map & override; - - [[nodiscard]] auto get_open_data(std::uint64_t handle) - -> open_file_data & override; - - [[nodiscard]] auto get_open_data(std::uint64_t handle) const - -> const open_file_data & override; - - [[nodiscard]] auto get_open_file_count() const -> std::size_t override; - - [[nodiscard]] auto get_source_path() const -> std::string override { - return fsi_.source_path; - } - - [[nodiscard]] auto has_handle(std::uint64_t handle) const -> bool override { - return open_data_.find(handle) != open_data_.end(); - } - - [[nodiscard]] auto is_directory() const -> bool override { - return fsi_.directory; - } - - [[nodiscard]] auto is_modified() const -> bool override; - - void remove(std::uint64_t handle) override; - - void set_api_path(const std::string &api_path) override; - }; - - class open_file final : public open_file_base { - public: - open_file(std::uint64_t chunk_size, std::uint8_t chunk_timeout, - filesystem_item fsi, i_provider &provider, i_upload_manager &mgr); - - open_file(std::uint64_t chunk_size, std::uint8_t chunk_timeout, - filesystem_item fsi, - std::map open_data, - i_provider &provider, i_upload_manager &mgr); - - open_file(std::uint64_t chunk_size, std::uint8_t chunk_timeout, - filesystem_item fsi, i_provider &provider, - std::optional> read_state, - i_upload_manager &mgr); - - private: - open_file(std::uint64_t chunk_size, std::uint8_t chunk_timeout, - filesystem_item fsi, - std::map open_data, - i_provider &provider, - std::optional> read_state, - i_upload_manager &mgr); - - public: - open_file() = delete; - open_file(const open_file &) noexcept = delete; - open_file(open_file &&) noexcept = delete; - auto operator=(open_file &&) noexcept -> open_file & = delete; - auto operator=(const open_file &) noexcept -> open_file & = delete; - - public: - ~open_file() override; - - private: - i_upload_manager &mgr_; - - private: - bool notified_ = false; - std::size_t read_chunk_index_{}; - boost::dynamic_bitset<> read_state_; - std::unique_ptr reader_thread_; - std::unique_ptr download_thread_; - stop_type stop_requested_ = false; - - private: - void download_chunk(std::size_t chunk, bool skip_active, bool should_reset); - - void download_range(std::size_t start_chunk_index, - std::size_t end_chunk_index_inclusive, - bool should_reset); - - void set_modified(); - - void update_background_reader(std::size_t read_chunk); - - protected: - auto is_download_complete() const -> bool override { - return read_state_.all(); - } - - public: - auto close() -> bool override; - - [[nodiscard]] auto get_read_state() const - -> boost::dynamic_bitset<> override; - - [[nodiscard]] auto get_read_state(std::size_t chunk) const -> bool override; - - [[nodiscard]] auto is_complete() const -> bool override; - - auto is_write_supported() const -> bool override { return true; } - - [[nodiscard]] auto native_operation(native_operation_callback callback) - -> api_error override; - - [[nodiscard]] auto native_operation(std::uint64_t new_file_size, - native_operation_callback callback) - -> api_error override; - - void remove(std::uint64_t handle) override; - - [[nodiscard]] auto read(std::size_t read_size, std::uint64_t read_offset, - data_buffer &data) -> api_error override; - - [[nodiscard]] auto resize(std::uint64_t new_file_size) - -> api_error override; - - [[nodiscard]] auto write(std::uint64_t write_offset, - const data_buffer &data, - std::size_t &bytes_written) -> api_error override; - }; - - class ring_buffer_open_file final : public open_file_base { - public: - ring_buffer_open_file(std::string buffer_directory, - std::uint64_t chunk_size, std::uint8_t chunk_timeout, - filesystem_item fsi, i_provider &provider); - - ring_buffer_open_file(std::string buffer_directory, - std::uint64_t chunk_size, std::uint8_t chunk_timeout, - filesystem_item fsi, i_provider &provider, - std::size_t ring_size); - - ~ring_buffer_open_file() override; - - public: - ring_buffer_open_file() = delete; - ring_buffer_open_file(const ring_buffer_open_file &) noexcept = delete; - ring_buffer_open_file(ring_buffer_open_file &&) noexcept = delete; - auto operator=(ring_buffer_open_file &&) noexcept - -> ring_buffer_open_file & = delete; - auto operator=(const ring_buffer_open_file &) noexcept - -> ring_buffer_open_file & = delete; - - private: - boost::dynamic_bitset<> ring_state_; - std::size_t total_chunks_; - - private: - std::unique_ptr chunk_forward_thread_; - std::unique_ptr chunk_reverse_thread_; - std::condition_variable chunk_notify_; - mutable std::mutex chunk_mtx_; - std::size_t current_chunk_{}; - std::size_t first_chunk_{}; - std::size_t last_chunk_; - - private: - auto download_chunk(std::size_t chunk) -> api_error; - - void forward_reader_thread(std::size_t count); - - void reverse_reader_thread(std::size_t count); - - protected: - auto is_download_complete() const -> bool override; - - public: - void forward(std::size_t count); - - [[nodiscard]] auto get_current_chunk() const -> std::size_t { - return current_chunk_; - } - - [[nodiscard]] auto get_first_chunk() const -> std::size_t { - return first_chunk_; - } - - [[nodiscard]] auto get_last_chunk() const -> std::size_t { - return last_chunk_; - } - - [[nodiscard]] auto get_read_state() const - -> boost::dynamic_bitset<> override; - - [[nodiscard]] auto get_read_state(std::size_t chunk) const -> bool override; - - [[nodiscard]] auto get_total_chunks() const -> std::size_t { - return total_chunks_; - } - - [[nodiscard]] auto is_complete() const -> bool override { return true; } - - auto is_write_supported() const -> bool override { return false; } - - [[nodiscard]] auto native_operation(native_operation_callback callback) - -> api_error override; - - [[nodiscard]] auto native_operation(std::uint64_t, - native_operation_callback) - -> api_error override { - return api_error::not_supported; - } - - [[nodiscard]] auto read(std::size_t read_size, std::uint64_t read_offset, - data_buffer &data) -> api_error override; - - [[nodiscard]] auto resize(std::uint64_t) -> api_error override { - return api_error::not_supported; - } - - void reverse(std::size_t count); - - void set(std::size_t first_chunk, std::size_t current_chunk); - - void set_api_path(const std::string &api_path) override; - - [[nodiscard]] auto write(std::uint64_t, const data_buffer &, std::size_t &) - -> api_error override { - return api_error::not_supported; - } - }; - - class upload final { - public: - upload(filesystem_item fsi, i_provider &provider); - - ~upload(); - - public: - upload() = delete; - upload(const upload &) noexcept = delete; - upload(upload &&) noexcept = delete; - auto operator=(upload &&) noexcept -> upload & = delete; - auto operator=(const upload &) noexcept -> upload & = delete; - - private: - filesystem_item fsi_; - i_provider &provider_; - - private: - bool cancelled_{false}; - api_error error_{api_error::success}; - std::unique_ptr thread_; - stop_type stop_requested_{false}; - - private: - void upload_thread(); - - public: - void cancel(); - - [[nodiscard]] auto get_api_error() const -> api_error { return error_; } - - [[nodiscard]] auto get_api_path() const -> std::string { - return fsi_.api_path; - } - - [[nodiscard]] auto get_source_path() const -> std::string { - return fsi_.source_path; - } - - [[nodiscard]] auto is_cancelled() const -> bool { return cancelled_; } - - void stop(); - }; - public: file_manager(app_config &config, i_provider &provider); diff --git a/repertory/librepertory/include/file_manager/open_file.hpp b/repertory/librepertory/include/file_manager/open_file.hpp new file mode 100644 index 00000000..15722c6f --- /dev/null +++ b/repertory/librepertory/include/file_manager/open_file.hpp @@ -0,0 +1,122 @@ +/* + Copyright <2018-2024> + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ +#ifndef REPERTORY_INCLUDE_FILE_MANAGER_OPEN_FILE_HPP_ +#define REPERTORY_INCLUDE_FILE_MANAGER_OPEN_FILE_HPP_ + +#include "file_manager/open_file_base.hpp" + +#include "types/repertory.hpp" + +namespace repertory { +class i_provider; +class i_upload_manager; + +class open_file final : public open_file_base { +public: + open_file(std::uint64_t chunk_size, std::uint8_t chunk_timeout, + filesystem_item fsi, i_provider &provider, i_upload_manager &mgr); + + open_file(std::uint64_t chunk_size, std::uint8_t chunk_timeout, + filesystem_item fsi, + std::map open_data, + i_provider &provider, i_upload_manager &mgr); + + open_file(std::uint64_t chunk_size, std::uint8_t chunk_timeout, + filesystem_item fsi, i_provider &provider, + std::optional> read_state, + i_upload_manager &mgr); + +private: + open_file(std::uint64_t chunk_size, std::uint8_t chunk_timeout, + filesystem_item fsi, + std::map open_data, + i_provider &provider, + std::optional> read_state, + i_upload_manager &mgr); + +public: + open_file() = delete; + open_file(const open_file &) noexcept = delete; + open_file(open_file &&) noexcept = delete; + auto operator=(open_file &&) noexcept -> open_file & = delete; + auto operator=(const open_file &) noexcept -> open_file & = delete; + +public: + ~open_file() override; + +private: + i_upload_manager &mgr_; + +private: + bool notified_ = false; + std::size_t read_chunk_index_{}; + boost::dynamic_bitset<> read_state_; + std::unique_ptr reader_thread_; + std::unique_ptr download_thread_; + stop_type stop_requested_ = false; + +private: + void download_chunk(std::size_t chunk, bool skip_active, bool should_reset); + + void download_range(std::size_t start_chunk_index, + std::size_t end_chunk_index_inclusive, bool should_reset); + + void set_modified(); + + void update_background_reader(std::size_t read_chunk); + +protected: + auto is_download_complete() const -> bool override { + return read_state_.all(); + } + +public: + auto close() -> bool override; + + [[nodiscard]] auto get_read_state() const -> boost::dynamic_bitset<> override; + + [[nodiscard]] auto get_read_state(std::size_t chunk) const -> bool override; + + [[nodiscard]] auto is_complete() const -> bool override; + + auto is_write_supported() const -> bool override { return true; } + + [[nodiscard]] auto native_operation(native_operation_callback callback) + -> api_error override; + + [[nodiscard]] auto native_operation(std::uint64_t new_file_size, + native_operation_callback callback) + -> api_error override; + + void remove(std::uint64_t handle) override; + + [[nodiscard]] auto read(std::size_t read_size, std::uint64_t read_offset, + data_buffer &data) -> api_error override; + + [[nodiscard]] auto resize(std::uint64_t new_file_size) -> api_error override; + + [[nodiscard]] auto write(std::uint64_t write_offset, const data_buffer &data, + std::size_t &bytes_written) -> api_error override; +}; +} // namespace repertory + +#endif // REPERTORY_INCLUDE_FILE_MANAGER_OPEN_FILE_HPP_ diff --git a/repertory/librepertory/include/file_manager/open_file_base.hpp b/repertory/librepertory/include/file_manager/open_file_base.hpp new file mode 100644 index 00000000..707b9637 --- /dev/null +++ b/repertory/librepertory/include/file_manager/open_file_base.hpp @@ -0,0 +1,194 @@ +/* + Copyright <2018-2024> + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ +#ifndef REPERTORY_INCLUDE_FILE_MANAGER_OPEN_FILE_BASE_HPP_ +#define REPERTORY_INCLUDE_FILE_MANAGER_OPEN_FILE_BASE_HPP_ + +#include "file_manager/i_open_file.hpp" + +#include "utils/types/file/i_file.hpp" + +namespace repertory { +class i_provider; + +class open_file_base : public i_closeable_open_file { +public: + open_file_base(std::uint64_t chunk_size, std::uint8_t chunk_timeout, + filesystem_item fsi, i_provider &provider); + + open_file_base(std::uint64_t chunk_size, std::uint8_t chunk_timeout, + filesystem_item fsi, + std::map open_data, + i_provider &provider); + + ~open_file_base() override = default; + +public: + open_file_base() = delete; + open_file_base(const open_file_base &) noexcept = delete; + open_file_base(open_file_base &&) noexcept = delete; + auto operator=(open_file_base &&) noexcept -> open_file_base & = delete; + auto operator=(const open_file_base &) noexcept -> open_file_base & = delete; + +public: + class download final { + public: + download() = default; + + ~download() = default; + + public: + download(const download &) noexcept = delete; + download(download &&) noexcept = delete; + auto operator=(download &&) noexcept -> download & = delete; + auto operator=(const download &) noexcept -> download & = delete; + + private: + bool complete_{false}; + api_error error_{api_error::success}; + std::mutex mtx_; + std::condition_variable notify_; + + public: + void notify(const api_error &err); + + auto wait() -> api_error; + }; + + class io_item final { + public: + io_item(std::function action) : action_(std::move(action)) {} + + ~io_item() = default; + + public: + io_item() = delete; + io_item(const io_item &) noexcept = delete; + io_item(io_item &&) noexcept = delete; + auto operator=(io_item &&) noexcept -> io_item & = delete; + auto operator=(const io_item &) noexcept -> io_item & = delete; + + private: + std::function action_; + std::mutex mtx_; + std::condition_variable notify_; + std::optional result_; + + public: + void action(); + + [[nodiscard]] auto get_result() -> api_error; + }; + +protected: + std::uint64_t chunk_size_; + std::uint8_t chunk_timeout_; + filesystem_item fsi_; + std::size_t last_chunk_size_; + std::map open_data_; + i_provider &provider_; + +private: + api_error error_{api_error::success}; + mutable std::mutex error_mtx_; + stop_type io_stop_requested_{false}; + std::unique_ptr io_thread_; + +protected: + std::unordered_map> active_downloads_; + mutable std::recursive_mutex file_mtx_; + std::atomic last_access_{ + std::chrono::system_clock::now()}; + bool modified_{false}; + std::unique_ptr nf_; + mutable std::mutex io_thread_mtx_; + std::condition_variable io_thread_notify_; + std::deque> io_thread_queue_; + bool removed_{false}; + +private: + void file_io_thread(); + +protected: + [[nodiscard]] auto do_io(std::function action) -> api_error; + + virtual auto is_download_complete() const -> bool = 0; + + void reset_timeout(); + + auto set_api_error(const api_error &e) -> api_error; + +public: + void add(std::uint64_t handle, open_file_data ofd) override; + + [[nodiscard]] auto can_close() const -> bool override; + + auto close() -> bool override; + + [[nodiscard]] auto get_api_error() const -> api_error; + + [[nodiscard]] auto get_api_path() const -> std::string override; + + [[nodiscard]] auto get_chunk_size() const -> std::size_t override { + return chunk_size_; + } + + [[nodiscard]] auto get_file_size() const -> std::uint64_t override; + + [[nodiscard]] auto get_filesystem_item() const -> filesystem_item override; + + [[nodiscard]] auto get_handles() const -> std::vector override; + + [[nodiscard]] auto get_open_data() + -> std::map & override; + + [[nodiscard]] auto get_open_data() const + -> const std::map & override; + + [[nodiscard]] auto get_open_data(std::uint64_t handle) + -> open_file_data & override; + + [[nodiscard]] auto get_open_data(std::uint64_t handle) const + -> const open_file_data & override; + + [[nodiscard]] auto get_open_file_count() const -> std::size_t override; + + [[nodiscard]] auto get_source_path() const -> std::string override { + return fsi_.source_path; + } + + [[nodiscard]] auto has_handle(std::uint64_t handle) const -> bool override { + return open_data_.find(handle) != open_data_.end(); + } + + [[nodiscard]] auto is_directory() const -> bool override { + return fsi_.directory; + } + + [[nodiscard]] auto is_modified() const -> bool override; + + void remove(std::uint64_t handle) override; + + void set_api_path(const std::string &api_path) override; +}; +} // namespace repertory + +#endif // REPERTORY_INCLUDE_FILE_MANAGER_OPEN_FILE_BASE_HPP_ diff --git a/repertory/librepertory/include/file_manager/ring_buffer_open_file.hpp b/repertory/librepertory/include/file_manager/ring_buffer_open_file.hpp new file mode 100644 index 00000000..5c5a35f1 --- /dev/null +++ b/repertory/librepertory/include/file_manager/ring_buffer_open_file.hpp @@ -0,0 +1,132 @@ +/* + Copyright <2018-2024> + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ +#ifndef REPERTORY_INCLUDE_FILE_MANAGER_RING_BUFFER_OPEN_FILE_HPP_ +#define REPERTORY_INCLUDE_FILE_MANAGER_RING_BUFFER_OPEN_FILE_HPP_ + +#include "file_manager/open_file_base.hpp" + +#include "types/repertory.hpp" + +namespace repertory { +class i_provider; +class i_upload_manager; + +class ring_buffer_open_file final : public open_file_base { +public: + ring_buffer_open_file(std::string buffer_directory, std::uint64_t chunk_size, + std::uint8_t chunk_timeout, filesystem_item fsi, + i_provider &provider); + + ring_buffer_open_file(std::string buffer_directory, std::uint64_t chunk_size, + std::uint8_t chunk_timeout, filesystem_item fsi, + i_provider &provider, std::size_t ring_size); + + ~ring_buffer_open_file() override; + +public: + ring_buffer_open_file() = delete; + ring_buffer_open_file(const ring_buffer_open_file &) noexcept = delete; + ring_buffer_open_file(ring_buffer_open_file &&) noexcept = delete; + auto operator=(ring_buffer_open_file &&) noexcept + -> ring_buffer_open_file & = delete; + auto operator=(const ring_buffer_open_file &) noexcept + -> ring_buffer_open_file & = delete; + +private: + boost::dynamic_bitset<> ring_state_; + std::size_t total_chunks_; + +private: + std::unique_ptr chunk_forward_thread_; + std::unique_ptr chunk_reverse_thread_; + std::condition_variable chunk_notify_; + mutable std::mutex chunk_mtx_; + std::size_t current_chunk_{}; + std::size_t first_chunk_{}; + std::size_t last_chunk_; + +private: + auto download_chunk(std::size_t chunk) -> api_error; + + void forward_reader_thread(std::size_t count); + + void reverse_reader_thread(std::size_t count); + +protected: + auto is_download_complete() const -> bool override; + +public: + void forward(std::size_t count); + + [[nodiscard]] auto get_current_chunk() const -> std::size_t { + return current_chunk_; + } + + [[nodiscard]] auto get_first_chunk() const -> std::size_t { + return first_chunk_; + } + + [[nodiscard]] auto get_last_chunk() const -> std::size_t { + return last_chunk_; + } + + [[nodiscard]] auto get_read_state() const -> boost::dynamic_bitset<> override; + + [[nodiscard]] auto get_read_state(std::size_t chunk) const -> bool override; + + [[nodiscard]] auto get_total_chunks() const -> std::size_t { + return total_chunks_; + } + + [[nodiscard]] auto is_complete() const -> bool override { return true; } + + auto is_write_supported() const -> bool override { return false; } + + [[nodiscard]] auto native_operation(native_operation_callback callback) + -> api_error override; + + [[nodiscard]] auto native_operation(std::uint64_t, native_operation_callback) + -> api_error override { + return api_error::not_supported; + } + + [[nodiscard]] auto read(std::size_t read_size, std::uint64_t read_offset, + data_buffer &data) -> api_error override; + + [[nodiscard]] auto resize(std::uint64_t) -> api_error override { + return api_error::not_supported; + } + + void reverse(std::size_t count); + + void set(std::size_t first_chunk, std::size_t current_chunk); + + void set_api_path(const std::string &api_path) override; + + [[nodiscard]] auto write(std::uint64_t, const data_buffer &, std::size_t &) + -> api_error override { + return api_error::not_supported; + } +}; +} // namespace repertory + +#endif // REPERTORY_INCLUDE_FILE_MANAGER_RING_BUFFER_OPEN_FILE_HPP_ diff --git a/repertory/librepertory/src/file_manager/file_manager_open_file_base_io_item.cpp b/repertory/librepertory/include/file_manager/upload.hpp similarity index 50% rename from repertory/librepertory/src/file_manager/file_manager_open_file_base_io_item.cpp rename to repertory/librepertory/include/file_manager/upload.hpp index 7741fd18..8e64c5da 100644 --- a/repertory/librepertory/src/file_manager/file_manager_open_file_base_io_item.cpp +++ b/repertory/librepertory/include/file_manager/upload.hpp @@ -1,41 +1,75 @@ -/* - Copyright <2018-2024> - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. -*/ -#include "file_manager/file_manager.hpp" - -namespace repertory { -void file_manager::open_file_base::io_item::action() { - result_ = action_(); - - mutex_lock lock(mtx_); - notify_.notify_all(); -} - -auto file_manager::open_file_base::io_item::get_result() -> api_error { - unique_mutex_lock lock(mtx_); - if (result_.has_value()) { - return result_.value(); - } - - notify_.wait(lock); - return result_.value_or(api_error::error); -} -} // namespace repertory +/* + Copyright <2018-2024> + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ +#ifndef REPERTORY_INCLUDE_FILE_MANAGER_UPLOAD_HPP_ +#define REPERTORY_INCLUDE_FILE_MANAGER_UPLOAD_HPP_ + +#include "types/repertory.hpp" + +namespace repertory { +class i_provider; + +class upload final { +public: + upload(filesystem_item fsi, i_provider &provider); + + ~upload(); + +public: + upload() = delete; + upload(const upload &) noexcept = delete; + upload(upload &&) noexcept = delete; + auto operator=(upload &&) noexcept -> upload & = delete; + auto operator=(const upload &) noexcept -> upload & = delete; + +private: + filesystem_item fsi_; + i_provider &provider_; + +private: + bool cancelled_{false}; + api_error error_{api_error::success}; + std::unique_ptr thread_; + stop_type stop_requested_{false}; + +private: + void upload_thread(); + +public: + void cancel(); + + [[nodiscard]] auto get_api_error() const -> api_error { return error_; } + + [[nodiscard]] auto get_api_path() const -> std::string { + return fsi_.api_path; + } + + [[nodiscard]] auto get_source_path() const -> std::string { + return fsi_.source_path; + } + + [[nodiscard]] auto is_cancelled() const -> bool { return cancelled_; } + + void stop(); +}; +} // namespace repertory + +#endif // REPERTORY_INCLUDE_FILE_MANAGER_UPLOAD_HPP_ diff --git a/repertory/librepertory/src/file_manager/file_manager.cpp b/repertory/librepertory/src/file_manager/file_manager.cpp index f2667140..7e9e8880 100644 --- a/repertory/librepertory/src/file_manager/file_manager.cpp +++ b/repertory/librepertory/src/file_manager/file_manager.cpp @@ -23,6 +23,10 @@ #include "app_config.hpp" #include "file_manager/events.hpp" +#include "file_manager/open_file.hpp" +#include "file_manager/open_file_base.hpp" +#include "file_manager/ring_buffer_open_file.hpp" +#include "file_manager/upload.hpp" #include "providers/i_provider.hpp" #include "types/repertory.hpp" #include "utils/common.hpp" diff --git a/repertory/librepertory/src/file_manager/file_manager_open_file_base_download.cpp b/repertory/librepertory/src/file_manager/file_manager_open_file_base_download.cpp deleted file mode 100644 index 1c26b666..00000000 --- a/repertory/librepertory/src/file_manager/file_manager_open_file_base_download.cpp +++ /dev/null @@ -1,43 +0,0 @@ -/* - Copyright <2018-2024> - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. -*/ -#include "file_manager/file_manager.hpp" - -namespace repertory { -void file_manager::open_file_base::download::notify(const api_error &err) { - complete_ = true; - error_ = err; - unique_mutex_lock lock(mtx_); - notify_.notify_all(); -} - -auto file_manager::open_file_base::download::wait() -> api_error { - if (not complete_) { - unique_mutex_lock lock(mtx_); - if (not complete_) { - notify_.wait(lock); - } - notify_.notify_all(); - } - - return error_; -} -} // namespace repertory diff --git a/repertory/librepertory/src/file_manager/file_manager_open_file.cpp b/repertory/librepertory/src/file_manager/open_file.cpp similarity index 85% rename from repertory/librepertory/src/file_manager/file_manager_open_file.cpp rename to repertory/librepertory/src/file_manager/open_file.cpp index 80e632dd..fd5f5aa9 100644 --- a/repertory/librepertory/src/file_manager/file_manager_open_file.cpp +++ b/repertory/librepertory/src/file_manager/open_file.cpp @@ -1,598 +1,596 @@ -/* - Copyright <2018-2024> - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. -*/ -#include "file_manager/file_manager.hpp" - -#include "file_manager/events.hpp" -#include "platform/platform.hpp" -#include "providers/i_provider.hpp" -#include "types/repertory.hpp" -#include "types/startup_exception.hpp" -#include "utils/common.hpp" -#include "utils/error_utils.hpp" -#include "utils/file_utils.hpp" -#include "utils/path.hpp" -#include "utils/time.hpp" -#include "utils/utils.hpp" - -namespace repertory { -file_manager::open_file::open_file(std::uint64_t chunk_size, - std::uint8_t chunk_timeout, - filesystem_item fsi, i_provider &provider, - i_upload_manager &mgr) - : open_file(chunk_size, chunk_timeout, fsi, {}, provider, std::nullopt, - mgr) {} - -file_manager::open_file::open_file( - std::uint64_t chunk_size, std::uint8_t chunk_timeout, filesystem_item fsi, - std::map open_data, i_provider &provider, - i_upload_manager &mgr) - : open_file(chunk_size, chunk_timeout, fsi, open_data, provider, - std::nullopt, mgr) {} - -file_manager::open_file::open_file( - std::uint64_t chunk_size, std::uint8_t chunk_timeout, filesystem_item fsi, - i_provider &provider, std::optional> read_state, - i_upload_manager &mgr) - : open_file(chunk_size, chunk_timeout, fsi, {}, provider, read_state, mgr) { -} - -file_manager::open_file::open_file( - std::uint64_t chunk_size, std::uint8_t chunk_timeout, filesystem_item fsi, - std::map open_data, i_provider &provider, - std::optional> read_state, i_upload_manager &mgr) - : open_file_base(chunk_size, chunk_timeout, fsi, open_data, provider), - mgr_(mgr) { - if (fsi_.directory && read_state.has_value()) { - throw startup_exception("cannot resume a directory|" + fsi.api_path); - } - - if (not fsi.directory) { - nf_ = utils::file::file::open_or_create_file(fsi.source_path, - provider_.is_direct_only()); - set_api_error(*nf_ ? api_error::success : api_error::os_error); - if (get_api_error() == api_error::success) { - if (read_state.has_value()) { - read_state_ = read_state.value(); - set_modified(); - } else if (fsi_.size > 0U) { - read_state_.resize(static_cast(utils::divide_with_ceiling( - fsi_.size, chunk_size)), - false); - - auto file_size = nf_->size(); - if (provider_.is_direct_only() || file_size == fsi.size) { - read_state_.set(0U, read_state_.size(), true); - } else if (not nf_->truncate(fsi.size)) { - set_api_error(api_error::os_error); - } - } - - if (get_api_error() != api_error::success && *nf_) { - nf_->close(); - } - } - } -} - -file_manager::open_file::~open_file() { close(); } - -void file_manager::open_file::download_chunk(std::size_t chunk, - bool skip_active, - bool should_reset) { - if (should_reset) { - reset_timeout(); - } - - unique_recur_mutex_lock download_lock(file_mtx_); - if ((get_api_error() == api_error::success) && (chunk < read_state_.size()) && - not read_state_[chunk]) { - if (active_downloads_.find(chunk) != active_downloads_.end()) { - if (not skip_active) { - auto active_download = active_downloads_.at(chunk); - download_lock.unlock(); - - active_download->wait(); - } - - return; - } - - const auto data_offset = chunk * chunk_size_; - const auto data_size = - (chunk == read_state_.size() - 1U) ? last_chunk_size_ : chunk_size_; - if (active_downloads_.empty() && (read_state_.count() == 0U)) { - event_system::instance().raise(fsi_.api_path, - fsi_.source_path); - } - event_system::instance().raise( - fsi_.api_path, fsi_.source_path, chunk, read_state_.size(), - read_state_.count()); - - active_downloads_[chunk] = std::make_shared(); - download_lock.unlock(); - - if (should_reset) { - reset_timeout(); - } - - std::async(std::launch::async, [this, chunk, data_size, data_offset, - should_reset]() { - const auto notify_complete = [this, chunk, should_reset]() { - unique_recur_mutex_lock file_lock(file_mtx_); - auto active_download = active_downloads_.at(chunk); - active_downloads_.erase(chunk); - event_system::instance().raise( - fsi_.api_path, fsi_.source_path, chunk, read_state_.size(), - read_state_.count(), get_api_error()); - if (get_api_error() == api_error::success) { - auto progress = (static_cast(read_state_.count()) / - static_cast(read_state_.size()) * 100.0); - event_system::instance().raise( - fsi_.api_path, fsi_.source_path, progress); - if (read_state_.all() && not notified_) { - notified_ = true; - event_system::instance().raise( - fsi_.api_path, fsi_.source_path, get_api_error()); - } - } else if (not notified_) { - notified_ = true; - event_system::instance().raise( - fsi_.api_path, fsi_.source_path, get_api_error()); - } - file_lock.unlock(); - - active_download->notify(get_api_error()); - - if (should_reset) { - reset_timeout(); - } - }; - - data_buffer data; - auto res = provider_.read_file_bytes(get_api_path(), data_size, - data_offset, data, stop_requested_); - if (res != api_error::success) { - set_api_error(res); - notify_complete(); - return; - } - - if (should_reset) { - reset_timeout(); - } - - res = do_io([&]() -> api_error { - std::size_t bytes_written{}; - if (not nf_->write(data, data_offset, &bytes_written)) { - return api_error::os_error; - } - - if (should_reset) { - reset_timeout(); - } - return api_error::success; - }); - if (res != api_error::success) { - set_api_error(res); - notify_complete(); - return; - } - - unique_recur_mutex_lock file_lock(file_mtx_); - read_state_.set(chunk); - file_lock.unlock(); - - notify_complete(); - }).wait(); - } -} - -void file_manager::open_file::download_range( - std::size_t start_chunk_index, std::size_t end_chunk_index_inclusive, - bool should_reset) { - for (std::size_t chunk = start_chunk_index; - chunk <= end_chunk_index_inclusive; chunk++) { - download_chunk(chunk, false, should_reset); - if (get_api_error() != api_error::success) { - return; - } - } -} - -auto file_manager::open_file::get_read_state() const - -> boost::dynamic_bitset<> { - recur_mutex_lock file_lock(file_mtx_); - return read_state_; -} - -auto file_manager::open_file::get_read_state(std::size_t chunk) const -> bool { - recur_mutex_lock file_lock(file_mtx_); - return read_state_[chunk]; -} - -auto file_manager::open_file::is_complete() const -> bool { - recur_mutex_lock file_lock(file_mtx_); - return read_state_.all(); -} - -auto file_manager::open_file::native_operation( - i_open_file::native_operation_callback callback) -> api_error { - unique_recur_mutex_lock file_lock(file_mtx_); - if (stop_requested_) { - return api_error::download_stopped; - } - file_lock.unlock(); - - return do_io([&]() -> api_error { return callback(nf_->get_handle()); }); -} - -auto file_manager::open_file::native_operation( - std::uint64_t new_file_size, - i_open_file::native_operation_callback callback) -> api_error { - REPERTORY_USES_FUNCTION_NAME(); - - if (fsi_.directory) { - return api_error::invalid_operation; - } - - unique_recur_mutex_lock file_lock(file_mtx_); - if (stop_requested_) { - return api_error::download_stopped; - } - file_lock.unlock(); - - const auto is_empty_file = new_file_size == 0U; - const auto last_chunk = - is_empty_file ? std::size_t(0U) - : static_cast(utils::divide_with_ceiling( - new_file_size, chunk_size_)) - - 1U; - - file_lock.lock(); - if (not is_empty_file && (last_chunk < read_state_.size())) { - file_lock.unlock(); - update_background_reader(0U); - - download_chunk(last_chunk, false, true); - if (get_api_error() != api_error::success) { - return get_api_error(); - } - file_lock.lock(); - } - - const auto original_file_size = get_file_size(); - - auto res = do_io([&]() -> api_error { return callback(nf_->get_handle()); }); - if (res != api_error::success) { - utils::error::raise_api_path_error(function_name, get_api_path(), - utils::get_last_error_code(), - "failed to allocate file"); - return res; - } - - { - auto file_size = nf_->size().value_or(0U); - if (file_size != new_file_size) { - utils::error::raise_api_path_error( - function_name, get_api_path(), api_error::file_size_mismatch, - "allocated file size mismatch|expected|" + - std::to_string(new_file_size) + "|actual|" + - std::to_string(file_size)); - return set_api_error(api_error::error); - } - } - - if (is_empty_file || (read_state_.size() != (last_chunk + 1U))) { - read_state_.resize(is_empty_file ? 0U : last_chunk + 1U); - - if (not is_empty_file) { - read_state_[last_chunk] = true; - } - - last_chunk_size_ = static_cast( - new_file_size <= chunk_size_ ? new_file_size - : (new_file_size % chunk_size_) == 0U ? chunk_size_ - : new_file_size % chunk_size_); - } - - if (original_file_size != new_file_size) { - set_modified(); - - fsi_.size = new_file_size; - const auto now = std::to_string(utils::time::get_time_now()); - res = provider_.set_item_meta( - fsi_.api_path, { - {META_CHANGED, now}, - {META_MODIFIED, now}, - {META_SIZE, std::to_string(new_file_size)}, - {META_WRITTEN, now}, - }); - if (res != api_error::success) { - utils::error::raise_api_path_error(function_name, get_api_path(), res, - "failed to set file meta"); - return set_api_error(res); - } - } - - return res; -} - -auto file_manager::open_file::read(std::size_t read_size, - std::uint64_t read_offset, data_buffer &data) - -> api_error { - if (fsi_.directory) { - return api_error::invalid_operation; - } - - read_size = - utils::calculate_read_size(get_file_size(), read_size, read_offset); - if (read_size == 0U) { - return api_error::success; - } - - const auto read_from_source = [this, &data, &read_offset, - &read_size]() -> api_error { - return do_io([this, &data, &read_offset, &read_size]() -> api_error { - if (provider_.is_direct_only()) { - return provider_.read_file_bytes(fsi_.api_path, read_size, read_offset, - data, stop_requested_); - } - - data.resize(read_size); - std::size_t bytes_read{}; - return nf_->read(data.data(), read_size, read_offset, &bytes_read) - ? api_error::success - : api_error::os_error; - }); - }; - - unique_recur_mutex_lock file_lock(file_mtx_); - if (read_state_.all()) { - reset_timeout(); - return read_from_source(); - } - file_lock.unlock(); - - const auto start_chunk_index = - static_cast(read_offset / chunk_size_); - const auto end_chunk_index = - static_cast((read_size + read_offset) / chunk_size_); - - update_background_reader(start_chunk_index); - - download_range(start_chunk_index, end_chunk_index, true); - if (get_api_error() != api_error::success) { - return get_api_error(); - } - - file_lock.lock(); - return get_api_error() == api_error::success ? read_from_source() - : get_api_error(); -} - -void file_manager::open_file::remove(std::uint64_t handle) { - recur_mutex_lock file_lock(file_mtx_); - open_file_base::remove(handle); - if (modified_ && read_state_.all() && - (get_api_error() == api_error::success)) { - mgr_.queue_upload(*this); - modified_ = false; - } - - if (removed_ && (get_open_file_count() == 0U)) { - removed_ = false; - } -} - -auto file_manager::open_file::resize(std::uint64_t new_file_size) -> api_error { - if (fsi_.directory) { - return api_error::invalid_operation; - } - - return native_operation( - new_file_size, [this, &new_file_size](native_handle) -> api_error { - return nf_->truncate(new_file_size) ? api_error::success - : api_error::os_error; - }); -} - -auto file_manager::open_file::close() -> bool { - REPERTORY_USES_FUNCTION_NAME(); - - if (not fsi_.directory && not stop_requested_) { - stop_requested_ = true; - - unique_mutex_lock reader_lock(io_thread_mtx_); - io_thread_notify_.notify_all(); - reader_lock.unlock(); - - if (reader_thread_) { - reader_thread_->join(); - reader_thread_.reset(); - } - - if (open_file_base::close()) { - { - const auto err = get_api_error(); - if (err == api_error::success || - err == api_error::download_incomplete || - err == api_error::download_stopped) { - if (modified_ && not read_state_.all()) { - set_api_error(api_error::download_incomplete); - } else if (not modified_ && (fsi_.size > 0U) && - not read_state_.all()) { - set_api_error(api_error::download_stopped); - } - } - } - - nf_->close(); - - if (modified_ && (get_api_error() == api_error::success)) { - mgr_.queue_upload(*this); - } else if (modified_ && - (get_api_error() == api_error::download_incomplete)) { - mgr_.store_resume(*this); - } else if (get_api_error() != api_error::success) { - mgr_.remove_resume(get_api_path(), get_source_path()); - if (not utils::file::file(fsi_.source_path).remove()) { - utils::error::raise_api_path_error( - function_name, get_api_path(), fsi_.source_path, - utils::get_last_error_code(), "failed to delete file"); - } - - auto parent = utils::path::get_parent_path(fsi_.source_path); - fsi_.source_path = - utils::path::combine(parent, {utils::create_uuid_string()}); - const auto res = provider_.set_item_meta(fsi_.api_path, META_SOURCE, - fsi_.source_path); - if (res != api_error::success) { - utils::error::raise_api_path_error(function_name, get_api_path(), - fsi_.source_path, res, - "failed to set file meta"); - } - } - } - - return true; - } - - return false; -} - -void file_manager::open_file::set_modified() { - if (not modified_) { - modified_ = true; - mgr_.store_resume(*this); - } - - if (not removed_) { - removed_ = true; - mgr_.remove_upload(get_api_path()); - } -} - -void file_manager::open_file::update_background_reader(std::size_t read_chunk) { - recur_mutex_lock reader_lock(file_mtx_); - read_chunk_index_ = read_chunk; - - if (not reader_thread_ && not stop_requested_) { - reader_thread_ = std::make_unique([this]() { - std::size_t next_chunk{}; - while (not stop_requested_) { - unique_recur_mutex_lock file_lock(file_mtx_); - if ((fsi_.size == 0U) || read_state_.all()) { - file_lock.unlock(); - - unique_mutex_lock io_lock(io_thread_mtx_); - if (not stop_requested_ && io_thread_queue_.empty()) { - io_thread_notify_.wait(io_lock); - } - io_thread_notify_.notify_all(); - io_lock.unlock(); - } else { - do { - next_chunk = read_chunk_index_ = - ((read_chunk_index_ + 1U) >= read_state_.size()) - ? 0U - : read_chunk_index_ + 1U; - } while ((next_chunk != 0U) && (active_downloads_.find(next_chunk) != - active_downloads_.end())); - - file_lock.unlock(); - download_chunk(next_chunk, true, false); - } - } - }); - } -} - -auto file_manager::open_file::write(std::uint64_t write_offset, - const data_buffer &data, - std::size_t &bytes_written) -> api_error { - REPERTORY_USES_FUNCTION_NAME(); - - bytes_written = 0U; - - if (fsi_.directory || provider_.is_direct_only()) { - return api_error::invalid_operation; - } - - if (data.empty()) { - return api_error::success; - } - - unique_recur_mutex_lock write_lock(file_mtx_); - if (stop_requested_) { - return api_error::download_stopped; - } - write_lock.unlock(); - - const auto start_chunk_index = - static_cast(write_offset / chunk_size_); - const auto end_chunk_index = - static_cast((write_offset + data.size()) / chunk_size_); - - update_background_reader(start_chunk_index); - - download_range(start_chunk_index, - std::min(read_state_.size() - 1U, end_chunk_index), true); - if (get_api_error() != api_error::success) { - return get_api_error(); - } - - write_lock.lock(); - if ((write_offset + data.size()) > fsi_.size) { - auto res = resize(write_offset + data.size()); - if (res != api_error::success) { - return res; - } - } - - auto res = do_io([&]() -> api_error { - if (not nf_->write(data, write_offset, &bytes_written)) { - return api_error::os_error; - } - - reset_timeout(); - return api_error::success; - }); - if (res != api_error::success) { - return set_api_error(res); - } - - const auto now = std::to_string(utils::time::get_time_now()); - res = provider_.set_item_meta(fsi_.api_path, { - {META_CHANGED, now}, - {META_MODIFIED, now}, - {META_WRITTEN, now}, - }); - if (res != api_error::success) { - utils::error::raise_api_path_error(function_name, get_api_path(), res, - "failed to set file meta"); - return set_api_error(res); - } - - set_modified(); - return api_error::success; -} -} // namespace repertory +/* + Copyright <2018-2024> + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ +#include "file_manager/open_file.hpp" + +#include "file_manager/events.hpp" +#include "file_manager/i_upload_manager.hpp" +#include "platform/platform.hpp" +#include "providers/i_provider.hpp" +#include "types/repertory.hpp" +#include "types/startup_exception.hpp" +#include "utils/common.hpp" +#include "utils/error_utils.hpp" +#include "utils/file_utils.hpp" +#include "utils/path.hpp" +#include "utils/time.hpp" +#include "utils/utils.hpp" + +namespace repertory { +open_file::open_file(std::uint64_t chunk_size, std::uint8_t chunk_timeout, + filesystem_item fsi, i_provider &provider, + i_upload_manager &mgr) + : open_file(chunk_size, chunk_timeout, fsi, {}, provider, std::nullopt, + mgr) {} + +open_file::open_file(std::uint64_t chunk_size, std::uint8_t chunk_timeout, + filesystem_item fsi, + std::map open_data, + i_provider &provider, i_upload_manager &mgr) + : open_file(chunk_size, chunk_timeout, fsi, open_data, provider, + std::nullopt, mgr) {} + +open_file::open_file(std::uint64_t chunk_size, std::uint8_t chunk_timeout, + filesystem_item fsi, i_provider &provider, + std::optional> read_state, + i_upload_manager &mgr) + : open_file(chunk_size, chunk_timeout, fsi, {}, provider, read_state, mgr) { +} + +open_file::open_file(std::uint64_t chunk_size, std::uint8_t chunk_timeout, + filesystem_item fsi, + std::map open_data, + i_provider &provider, + std::optional> read_state, + i_upload_manager &mgr) + : open_file_base(chunk_size, chunk_timeout, fsi, open_data, provider), + mgr_(mgr) { + if (fsi_.directory && read_state.has_value()) { + throw startup_exception("cannot resume a directory|" + fsi.api_path); + } + + if (not fsi.directory) { + nf_ = utils::file::file::open_or_create_file(fsi.source_path, + provider_.is_direct_only()); + set_api_error(*nf_ ? api_error::success : api_error::os_error); + if (get_api_error() == api_error::success) { + if (read_state.has_value()) { + read_state_ = read_state.value(); + set_modified(); + } else if (fsi_.size > 0U) { + read_state_.resize(static_cast(utils::divide_with_ceiling( + fsi_.size, chunk_size)), + false); + + auto file_size = nf_->size(); + if (provider_.is_direct_only() || file_size == fsi.size) { + read_state_.set(0U, read_state_.size(), true); + } else if (not nf_->truncate(fsi.size)) { + set_api_error(api_error::os_error); + } + } + + if (get_api_error() != api_error::success && *nf_) { + nf_->close(); + } + } + } +} + +open_file::~open_file() { close(); } + +void open_file::download_chunk(std::size_t chunk, bool skip_active, + bool should_reset) { + if (should_reset) { + reset_timeout(); + } + + unique_recur_mutex_lock download_lock(file_mtx_); + if ((get_api_error() == api_error::success) && (chunk < read_state_.size()) && + not read_state_[chunk]) { + if (active_downloads_.find(chunk) != active_downloads_.end()) { + if (not skip_active) { + auto active_download = active_downloads_.at(chunk); + download_lock.unlock(); + + active_download->wait(); + } + + return; + } + + const auto data_offset = chunk * chunk_size_; + const auto data_size = + (chunk == read_state_.size() - 1U) ? last_chunk_size_ : chunk_size_; + if (active_downloads_.empty() && (read_state_.count() == 0U)) { + event_system::instance().raise(fsi_.api_path, + fsi_.source_path); + } + event_system::instance().raise( + fsi_.api_path, fsi_.source_path, chunk, read_state_.size(), + read_state_.count()); + + active_downloads_[chunk] = std::make_shared(); + download_lock.unlock(); + + if (should_reset) { + reset_timeout(); + } + + std::async(std::launch::async, [this, chunk, data_size, data_offset, + should_reset]() { + const auto notify_complete = [this, chunk, should_reset]() { + unique_recur_mutex_lock file_lock(file_mtx_); + auto active_download = active_downloads_.at(chunk); + active_downloads_.erase(chunk); + event_system::instance().raise( + fsi_.api_path, fsi_.source_path, chunk, read_state_.size(), + read_state_.count(), get_api_error()); + if (get_api_error() == api_error::success) { + auto progress = (static_cast(read_state_.count()) / + static_cast(read_state_.size()) * 100.0); + event_system::instance().raise( + fsi_.api_path, fsi_.source_path, progress); + if (read_state_.all() && not notified_) { + notified_ = true; + event_system::instance().raise( + fsi_.api_path, fsi_.source_path, get_api_error()); + } + } else if (not notified_) { + notified_ = true; + event_system::instance().raise( + fsi_.api_path, fsi_.source_path, get_api_error()); + } + file_lock.unlock(); + + active_download->notify(get_api_error()); + + if (should_reset) { + reset_timeout(); + } + }; + + data_buffer data; + auto res = provider_.read_file_bytes(get_api_path(), data_size, + data_offset, data, stop_requested_); + if (res != api_error::success) { + set_api_error(res); + notify_complete(); + return; + } + + if (should_reset) { + reset_timeout(); + } + + res = do_io([&]() -> api_error { + std::size_t bytes_written{}; + if (not nf_->write(data, data_offset, &bytes_written)) { + return api_error::os_error; + } + + if (should_reset) { + reset_timeout(); + } + return api_error::success; + }); + if (res != api_error::success) { + set_api_error(res); + notify_complete(); + return; + } + + unique_recur_mutex_lock file_lock(file_mtx_); + read_state_.set(chunk); + file_lock.unlock(); + + notify_complete(); + }).wait(); + } +} + +void open_file::download_range(std::size_t start_chunk_index, + std::size_t end_chunk_index_inclusive, + bool should_reset) { + for (std::size_t chunk = start_chunk_index; + chunk <= end_chunk_index_inclusive; chunk++) { + download_chunk(chunk, false, should_reset); + if (get_api_error() != api_error::success) { + return; + } + } +} + +auto open_file::get_read_state() const -> boost::dynamic_bitset<> { + recur_mutex_lock file_lock(file_mtx_); + return read_state_; +} + +auto open_file::get_read_state(std::size_t chunk) const -> bool { + recur_mutex_lock file_lock(file_mtx_); + return read_state_[chunk]; +} + +auto open_file::is_complete() const -> bool { + recur_mutex_lock file_lock(file_mtx_); + return read_state_.all(); +} + +auto open_file::native_operation( + i_open_file::native_operation_callback callback) -> api_error { + unique_recur_mutex_lock file_lock(file_mtx_); + if (stop_requested_) { + return api_error::download_stopped; + } + file_lock.unlock(); + + return do_io([&]() -> api_error { return callback(nf_->get_handle()); }); +} + +auto open_file::native_operation( + std::uint64_t new_file_size, + i_open_file::native_operation_callback callback) -> api_error { + REPERTORY_USES_FUNCTION_NAME(); + + if (fsi_.directory) { + return api_error::invalid_operation; + } + + unique_recur_mutex_lock file_lock(file_mtx_); + if (stop_requested_) { + return api_error::download_stopped; + } + file_lock.unlock(); + + const auto is_empty_file = new_file_size == 0U; + const auto last_chunk = + is_empty_file ? std::size_t(0U) + : static_cast(utils::divide_with_ceiling( + new_file_size, chunk_size_)) - + 1U; + + file_lock.lock(); + if (not is_empty_file && (last_chunk < read_state_.size())) { + file_lock.unlock(); + update_background_reader(0U); + + download_chunk(last_chunk, false, true); + if (get_api_error() != api_error::success) { + return get_api_error(); + } + file_lock.lock(); + } + + const auto original_file_size = get_file_size(); + + auto res = do_io([&]() -> api_error { return callback(nf_->get_handle()); }); + if (res != api_error::success) { + utils::error::raise_api_path_error(function_name, get_api_path(), + utils::get_last_error_code(), + "failed to allocate file"); + return res; + } + + { + auto file_size = nf_->size().value_or(0U); + if (file_size != new_file_size) { + utils::error::raise_api_path_error( + function_name, get_api_path(), api_error::file_size_mismatch, + "allocated file size mismatch|expected|" + + std::to_string(new_file_size) + "|actual|" + + std::to_string(file_size)); + return set_api_error(api_error::error); + } + } + + if (is_empty_file || (read_state_.size() != (last_chunk + 1U))) { + read_state_.resize(is_empty_file ? 0U : last_chunk + 1U); + + if (not is_empty_file) { + read_state_[last_chunk] = true; + } + + last_chunk_size_ = static_cast( + new_file_size <= chunk_size_ ? new_file_size + : (new_file_size % chunk_size_) == 0U ? chunk_size_ + : new_file_size % chunk_size_); + } + + if (original_file_size != new_file_size) { + set_modified(); + + fsi_.size = new_file_size; + const auto now = std::to_string(utils::time::get_time_now()); + res = provider_.set_item_meta( + fsi_.api_path, { + {META_CHANGED, now}, + {META_MODIFIED, now}, + {META_SIZE, std::to_string(new_file_size)}, + {META_WRITTEN, now}, + }); + if (res != api_error::success) { + utils::error::raise_api_path_error(function_name, get_api_path(), res, + "failed to set file meta"); + return set_api_error(res); + } + } + + return res; +} + +auto open_file::read(std::size_t read_size, std::uint64_t read_offset, + data_buffer &data) -> api_error { + if (fsi_.directory) { + return api_error::invalid_operation; + } + + read_size = + utils::calculate_read_size(get_file_size(), read_size, read_offset); + if (read_size == 0U) { + return api_error::success; + } + + const auto read_from_source = [this, &data, &read_offset, + &read_size]() -> api_error { + return do_io([this, &data, &read_offset, &read_size]() -> api_error { + if (provider_.is_direct_only()) { + return provider_.read_file_bytes(fsi_.api_path, read_size, read_offset, + data, stop_requested_); + } + + data.resize(read_size); + std::size_t bytes_read{}; + return nf_->read(data.data(), read_size, read_offset, &bytes_read) + ? api_error::success + : api_error::os_error; + }); + }; + + unique_recur_mutex_lock file_lock(file_mtx_); + if (read_state_.all()) { + reset_timeout(); + return read_from_source(); + } + file_lock.unlock(); + + const auto start_chunk_index = + static_cast(read_offset / chunk_size_); + const auto end_chunk_index = + static_cast((read_size + read_offset) / chunk_size_); + + update_background_reader(start_chunk_index); + + download_range(start_chunk_index, end_chunk_index, true); + if (get_api_error() != api_error::success) { + return get_api_error(); + } + + file_lock.lock(); + return get_api_error() == api_error::success ? read_from_source() + : get_api_error(); +} + +void open_file::remove(std::uint64_t handle) { + recur_mutex_lock file_lock(file_mtx_); + open_file_base::remove(handle); + if (modified_ && read_state_.all() && + (get_api_error() == api_error::success)) { + mgr_.queue_upload(*this); + modified_ = false; + } + + if (removed_ && (get_open_file_count() == 0U)) { + removed_ = false; + } +} + +auto open_file::resize(std::uint64_t new_file_size) -> api_error { + if (fsi_.directory) { + return api_error::invalid_operation; + } + + return native_operation( + new_file_size, [this, &new_file_size](native_handle) -> api_error { + return nf_->truncate(new_file_size) ? api_error::success + : api_error::os_error; + }); +} + +auto open_file::close() -> bool { + REPERTORY_USES_FUNCTION_NAME(); + + if (not fsi_.directory && not stop_requested_) { + stop_requested_ = true; + + unique_mutex_lock reader_lock(io_thread_mtx_); + io_thread_notify_.notify_all(); + reader_lock.unlock(); + + if (reader_thread_) { + reader_thread_->join(); + reader_thread_.reset(); + } + + if (open_file_base::close()) { + { + const auto err = get_api_error(); + if (err == api_error::success || + err == api_error::download_incomplete || + err == api_error::download_stopped) { + if (modified_ && not read_state_.all()) { + set_api_error(api_error::download_incomplete); + } else if (not modified_ && (fsi_.size > 0U) && + not read_state_.all()) { + set_api_error(api_error::download_stopped); + } + } + } + + nf_->close(); + + if (modified_ && (get_api_error() == api_error::success)) { + mgr_.queue_upload(*this); + } else if (modified_ && + (get_api_error() == api_error::download_incomplete)) { + mgr_.store_resume(*this); + } else if (get_api_error() != api_error::success) { + mgr_.remove_resume(get_api_path(), get_source_path()); + if (not utils::file::file(fsi_.source_path).remove()) { + utils::error::raise_api_path_error( + function_name, get_api_path(), fsi_.source_path, + utils::get_last_error_code(), "failed to delete file"); + } + + auto parent = utils::path::get_parent_path(fsi_.source_path); + fsi_.source_path = + utils::path::combine(parent, {utils::create_uuid_string()}); + const auto res = provider_.set_item_meta(fsi_.api_path, META_SOURCE, + fsi_.source_path); + if (res != api_error::success) { + utils::error::raise_api_path_error(function_name, get_api_path(), + fsi_.source_path, res, + "failed to set file meta"); + } + } + } + + return true; + } + + return false; +} + +void open_file::set_modified() { + if (not modified_) { + modified_ = true; + mgr_.store_resume(*this); + } + + if (not removed_) { + removed_ = true; + mgr_.remove_upload(get_api_path()); + } +} + +void open_file::update_background_reader(std::size_t read_chunk) { + recur_mutex_lock reader_lock(file_mtx_); + read_chunk_index_ = read_chunk; + + if (not reader_thread_ && not stop_requested_) { + reader_thread_ = std::make_unique([this]() { + std::size_t next_chunk{}; + while (not stop_requested_) { + unique_recur_mutex_lock file_lock(file_mtx_); + if ((fsi_.size == 0U) || read_state_.all()) { + file_lock.unlock(); + + unique_mutex_lock io_lock(io_thread_mtx_); + if (not stop_requested_ && io_thread_queue_.empty()) { + io_thread_notify_.wait(io_lock); + } + io_thread_notify_.notify_all(); + io_lock.unlock(); + } else { + do { + next_chunk = read_chunk_index_ = + ((read_chunk_index_ + 1U) >= read_state_.size()) + ? 0U + : read_chunk_index_ + 1U; + } while ((next_chunk != 0U) && (active_downloads_.find(next_chunk) != + active_downloads_.end())); + + file_lock.unlock(); + download_chunk(next_chunk, true, false); + } + } + }); + } +} + +auto open_file::write(std::uint64_t write_offset, const data_buffer &data, + std::size_t &bytes_written) -> api_error { + REPERTORY_USES_FUNCTION_NAME(); + + bytes_written = 0U; + + if (fsi_.directory || provider_.is_direct_only()) { + return api_error::invalid_operation; + } + + if (data.empty()) { + return api_error::success; + } + + unique_recur_mutex_lock write_lock(file_mtx_); + if (stop_requested_) { + return api_error::download_stopped; + } + write_lock.unlock(); + + const auto start_chunk_index = + static_cast(write_offset / chunk_size_); + const auto end_chunk_index = + static_cast((write_offset + data.size()) / chunk_size_); + + update_background_reader(start_chunk_index); + + download_range(start_chunk_index, + std::min(read_state_.size() - 1U, end_chunk_index), true); + if (get_api_error() != api_error::success) { + return get_api_error(); + } + + write_lock.lock(); + if ((write_offset + data.size()) > fsi_.size) { + auto res = resize(write_offset + data.size()); + if (res != api_error::success) { + return res; + } + } + + auto res = do_io([&]() -> api_error { + if (not nf_->write(data, write_offset, &bytes_written)) { + return api_error::os_error; + } + + reset_timeout(); + return api_error::success; + }); + if (res != api_error::success) { + return set_api_error(res); + } + + const auto now = std::to_string(utils::time::get_time_now()); + res = provider_.set_item_meta(fsi_.api_path, { + {META_CHANGED, now}, + {META_MODIFIED, now}, + {META_WRITTEN, now}, + }); + if (res != api_error::success) { + utils::error::raise_api_path_error(function_name, get_api_path(), res, + "failed to set file meta"); + return set_api_error(res); + } + + set_modified(); + return api_error::success; +} +} // namespace repertory diff --git a/repertory/librepertory/src/file_manager/file_manager_open_file_base.cpp b/repertory/librepertory/src/file_manager/open_file_base.cpp similarity index 73% rename from repertory/librepertory/src/file_manager/file_manager_open_file_base.cpp rename to repertory/librepertory/src/file_manager/open_file_base.cpp index 2afdc2c5..12b2b500 100644 --- a/repertory/librepertory/src/file_manager/file_manager_open_file_base.cpp +++ b/repertory/librepertory/src/file_manager/open_file_base.cpp @@ -1,261 +1,291 @@ -/* - Copyright <2018-2024> - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. -*/ -#include "file_manager/file_manager.hpp" - -#include "providers/i_provider.hpp" -#include "utils/path.hpp" - -namespace repertory { -file_manager::open_file_base::open_file_base(std::uint64_t chunk_size, - std::uint8_t chunk_timeout, - filesystem_item fsi, - i_provider &provider) - : open_file_base(chunk_size, chunk_timeout, fsi, {}, provider) {} - -file_manager::open_file_base::open_file_base( - std::uint64_t chunk_size, std::uint8_t chunk_timeout, filesystem_item fsi, - std::map open_data, i_provider &provider) - : chunk_size_(chunk_size), - chunk_timeout_(chunk_timeout), - fsi_(std::move(fsi)), - last_chunk_size_(static_cast( - fsi.size <= chunk_size ? fsi.size - : (fsi.size % chunk_size) == 0U ? chunk_size - : fsi.size % chunk_size)), - open_data_(std::move(open_data)), - provider_(provider) { - if (not fsi.directory) { - io_thread_ = std::make_unique([this] { file_io_thread(); }); - } -} - -void file_manager::open_file_base::add(std::uint64_t handle, - open_file_data ofd) { - recur_mutex_lock file_lock(file_mtx_); - open_data_[handle] = ofd; - if (open_data_.size() == 1U) { - event_system::instance().raise( - fsi_.api_path, fsi_.source_path, fsi_.directory); - } - - event_system::instance().raise( - fsi_.api_path, handle, fsi_.source_path, fsi_.directory); -} - -auto file_manager::open_file_base::can_close() const -> bool { - recur_mutex_lock file_lock(file_mtx_); - if (fsi_.directory) { - return true; - } - - if (not open_data_.empty()) { - return false; - } - - if (modified_) { - return false; - } - - if (get_api_error() != api_error::success) { - return true; - } - - if (is_download_complete()) { - return true; - } - - if (provider_.is_direct_only()) { - return true; - } - - const std::chrono::system_clock::time_point last_access = last_access_; - const auto duration = std::chrono::duration_cast( - std::chrono::system_clock::now() - last_access); - return (duration.count() >= chunk_timeout_); -} - -auto file_manager::open_file_base::do_io(std::function action) - -> api_error { - unique_mutex_lock io_lock(io_thread_mtx_); - auto item = std::make_shared(action); - io_thread_queue_.emplace_back(item); - io_thread_notify_.notify_all(); - io_lock.unlock(); - - return item->get_result(); -} - -void file_manager::open_file_base::file_io_thread() { - unique_mutex_lock io_lock(io_thread_mtx_); - io_thread_notify_.notify_all(); - io_lock.unlock(); - - const auto process_queue = [&]() { - io_lock.lock(); - if (not io_stop_requested_ && io_thread_queue_.empty()) { - io_thread_notify_.wait(io_lock); - } - - while (not io_thread_queue_.empty()) { - auto *item = io_thread_queue_.front().get(); - io_thread_notify_.notify_all(); - io_lock.unlock(); - - item->action(); - - io_lock.lock(); - io_thread_queue_.pop_front(); - } - - io_thread_notify_.notify_all(); - io_lock.unlock(); - }; - - while (not io_stop_requested_) { - process_queue(); - } - - process_queue(); -} - -auto file_manager::open_file_base::get_api_error() const -> api_error { - mutex_lock error_lock(error_mtx_); - return error_; -} - -auto file_manager::open_file_base::get_api_path() const -> std::string { - recur_mutex_lock file_lock(file_mtx_); - return fsi_.api_path; -} - -auto file_manager::open_file_base::get_file_size() const -> std::uint64_t { - recur_mutex_lock file_lock(file_mtx_); - return fsi_.size; -} - -auto file_manager::open_file_base::get_filesystem_item() const - -> filesystem_item { - recur_mutex_lock file_lock(file_mtx_); - return fsi_; -} - -auto file_manager::open_file_base::get_handles() const - -> std::vector { - recur_mutex_lock file_lock(file_mtx_); - std::vector ret; - for (auto &&item : open_data_) { - ret.emplace_back(item.first); - } - - return ret; -} - -auto file_manager::open_file_base::get_open_data() - -> std::map & { - recur_mutex_lock file_lock(file_mtx_); - return open_data_; -} - -auto file_manager::open_file_base::get_open_data() const - -> const std::map & { - recur_mutex_lock file_lock(file_mtx_); - return open_data_; -} - -auto file_manager::open_file_base::get_open_data(std::uint64_t handle) - -> open_file_data & { - recur_mutex_lock file_lock(file_mtx_); - return open_data_.at(handle); -} - -auto file_manager::open_file_base::get_open_data(std::uint64_t handle) const - -> const open_file_data & { - recur_mutex_lock file_lock(file_mtx_); - return open_data_.at(handle); -} - -auto file_manager::open_file_base::get_open_file_count() const -> std::size_t { - recur_mutex_lock file_lock(file_mtx_); - return open_data_.size(); -} - -auto file_manager::open_file_base::is_modified() const -> bool { - recur_mutex_lock file_lock(file_mtx_); - return modified_; -} - -void file_manager::open_file_base::remove(std::uint64_t handle) { - recur_mutex_lock file_lock(file_mtx_); - open_data_.erase(handle); - event_system::instance().raise( - fsi_.api_path, handle, fsi_.source_path, fsi_.directory, modified_); - if (open_data_.empty()) { - event_system::instance().raise( - fsi_.api_path, fsi_.source_path, fsi_.directory, modified_); - } -} - -void file_manager::open_file_base::reset_timeout() { - last_access_ = std::chrono::system_clock::now(); -} - -auto file_manager::open_file_base::set_api_error(const api_error &err) - -> api_error { - mutex_lock error_lock(error_mtx_); - if (error_ != err) { - return ((error_ = (error_ == api_error::success || - error_ == api_error::download_incomplete || - error_ == api_error::download_stopped - ? err - : error_))); - } - - return error_; -} - -void file_manager::open_file_base::set_api_path(const std::string &api_path) { - recur_mutex_lock file_lock(file_mtx_); - fsi_.api_path = api_path; - fsi_.api_parent = utils::path::get_parent_api_path(api_path); -} - -auto file_manager::open_file_base::close() -> bool { - unique_mutex_lock io_lock(io_thread_mtx_); - if (not fsi_.directory && not io_stop_requested_) { - io_stop_requested_ = true; - io_thread_notify_.notify_all(); - io_lock.unlock(); - - if (io_thread_) { - io_thread_->join(); - io_thread_.reset(); - return true; - } - - return false; - } - - io_thread_notify_.notify_all(); - io_lock.unlock(); - return false; -} -} // namespace repertory +/* + Copyright <2018-2024> + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ +#include "file_manager/open_file_base.hpp" + +#include "file_manager/events.hpp" +#include "providers/i_provider.hpp" +#include "utils/path.hpp" + +namespace repertory { +void open_file_base::download::notify(const api_error &err) { + complete_ = true; + error_ = err; + unique_mutex_lock lock(mtx_); + notify_.notify_all(); +} + +auto open_file_base::download::wait() -> api_error { + if (not complete_) { + unique_mutex_lock lock(mtx_); + if (not complete_) { + notify_.wait(lock); + } + notify_.notify_all(); + } + + return error_; +} + +void open_file_base::io_item::action() { + result_ = action_(); + + mutex_lock lock(mtx_); + notify_.notify_all(); +} + +auto open_file_base::io_item::get_result() -> api_error { + unique_mutex_lock lock(mtx_); + if (result_.has_value()) { + return result_.value(); + } + + notify_.wait(lock); + return result_.value_or(api_error::error); +} + +open_file_base::open_file_base(std::uint64_t chunk_size, + std::uint8_t chunk_timeout, filesystem_item fsi, + i_provider &provider) + : open_file_base(chunk_size, chunk_timeout, fsi, {}, provider) {} + +open_file_base::open_file_base( + std::uint64_t chunk_size, std::uint8_t chunk_timeout, filesystem_item fsi, + std::map open_data, i_provider &provider) + : chunk_size_(chunk_size), + chunk_timeout_(chunk_timeout), + fsi_(std::move(fsi)), + last_chunk_size_(static_cast( + fsi.size <= chunk_size ? fsi.size + : (fsi.size % chunk_size) == 0U ? chunk_size + : fsi.size % chunk_size)), + open_data_(std::move(open_data)), + provider_(provider) { + if (not fsi.directory) { + io_thread_ = std::make_unique([this] { file_io_thread(); }); + } +} + +void open_file_base::add(std::uint64_t handle, open_file_data ofd) { + recur_mutex_lock file_lock(file_mtx_); + open_data_[handle] = ofd; + if (open_data_.size() == 1U) { + event_system::instance().raise( + fsi_.api_path, fsi_.source_path, fsi_.directory); + } + + event_system::instance().raise( + fsi_.api_path, handle, fsi_.source_path, fsi_.directory); +} + +auto open_file_base::can_close() const -> bool { + recur_mutex_lock file_lock(file_mtx_); + if (fsi_.directory) { + return true; + } + + if (not open_data_.empty()) { + return false; + } + + if (modified_) { + return false; + } + + if (get_api_error() != api_error::success) { + return true; + } + + if (is_download_complete()) { + return true; + } + + if (provider_.is_direct_only()) { + return true; + } + + const std::chrono::system_clock::time_point last_access = last_access_; + const auto duration = std::chrono::duration_cast( + std::chrono::system_clock::now() - last_access); + return (duration.count() >= chunk_timeout_); +} + +auto open_file_base::do_io(std::function action) -> api_error { + unique_mutex_lock io_lock(io_thread_mtx_); + auto item = std::make_shared(action); + io_thread_queue_.emplace_back(item); + io_thread_notify_.notify_all(); + io_lock.unlock(); + + return item->get_result(); +} + +void open_file_base::file_io_thread() { + unique_mutex_lock io_lock(io_thread_mtx_); + io_thread_notify_.notify_all(); + io_lock.unlock(); + + const auto process_queue = [&]() { + io_lock.lock(); + if (not io_stop_requested_ && io_thread_queue_.empty()) { + io_thread_notify_.wait(io_lock); + } + + while (not io_thread_queue_.empty()) { + auto *item = io_thread_queue_.front().get(); + io_thread_notify_.notify_all(); + io_lock.unlock(); + + item->action(); + + io_lock.lock(); + io_thread_queue_.pop_front(); + } + + io_thread_notify_.notify_all(); + io_lock.unlock(); + }; + + while (not io_stop_requested_) { + process_queue(); + } + + process_queue(); +} + +auto open_file_base::get_api_error() const -> api_error { + mutex_lock error_lock(error_mtx_); + return error_; +} + +auto open_file_base::get_api_path() const -> std::string { + recur_mutex_lock file_lock(file_mtx_); + return fsi_.api_path; +} + +auto open_file_base::get_file_size() const -> std::uint64_t { + recur_mutex_lock file_lock(file_mtx_); + return fsi_.size; +} + +auto open_file_base::get_filesystem_item() const -> filesystem_item { + recur_mutex_lock file_lock(file_mtx_); + return fsi_; +} + +auto open_file_base::get_handles() const -> std::vector { + recur_mutex_lock file_lock(file_mtx_); + std::vector ret; + for (auto &&item : open_data_) { + ret.emplace_back(item.first); + } + + return ret; +} + +auto open_file_base::get_open_data() + -> std::map & { + recur_mutex_lock file_lock(file_mtx_); + return open_data_; +} + +auto open_file_base::get_open_data() const + -> const std::map & { + recur_mutex_lock file_lock(file_mtx_); + return open_data_; +} + +auto open_file_base::get_open_data(std::uint64_t handle) -> open_file_data & { + recur_mutex_lock file_lock(file_mtx_); + return open_data_.at(handle); +} + +auto open_file_base::get_open_data(std::uint64_t handle) const + -> const open_file_data & { + recur_mutex_lock file_lock(file_mtx_); + return open_data_.at(handle); +} + +auto open_file_base::get_open_file_count() const -> std::size_t { + recur_mutex_lock file_lock(file_mtx_); + return open_data_.size(); +} + +auto open_file_base::is_modified() const -> bool { + recur_mutex_lock file_lock(file_mtx_); + return modified_; +} + +void open_file_base::remove(std::uint64_t handle) { + recur_mutex_lock file_lock(file_mtx_); + open_data_.erase(handle); + event_system::instance().raise( + fsi_.api_path, handle, fsi_.source_path, fsi_.directory, modified_); + if (open_data_.empty()) { + event_system::instance().raise( + fsi_.api_path, fsi_.source_path, fsi_.directory, modified_); + } +} + +void open_file_base::reset_timeout() { + last_access_ = std::chrono::system_clock::now(); +} + +auto open_file_base::set_api_error(const api_error &err) -> api_error { + mutex_lock error_lock(error_mtx_); + if (error_ != err) { + return ((error_ = (error_ == api_error::success || + error_ == api_error::download_incomplete || + error_ == api_error::download_stopped + ? err + : error_))); + } + + return error_; +} + +void open_file_base::set_api_path(const std::string &api_path) { + recur_mutex_lock file_lock(file_mtx_); + fsi_.api_path = api_path; + fsi_.api_parent = utils::path::get_parent_api_path(api_path); +} + +auto open_file_base::close() -> bool { + unique_mutex_lock io_lock(io_thread_mtx_); + if (not fsi_.directory && not io_stop_requested_) { + io_stop_requested_ = true; + io_thread_notify_.notify_all(); + io_lock.unlock(); + + if (io_thread_) { + io_thread_->join(); + io_thread_.reset(); + return true; + } + + return false; + } + + io_thread_notify_.notify_all(); + io_lock.unlock(); + return false; +} +} // namespace repertory diff --git a/repertory/librepertory/src/file_manager/file_manager_ring_buffer_open_file.cpp b/repertory/librepertory/src/file_manager/ring_buffer_open_file.cpp similarity index 83% rename from repertory/librepertory/src/file_manager/file_manager_ring_buffer_open_file.cpp rename to repertory/librepertory/src/file_manager/ring_buffer_open_file.cpp index 172993fe..12a082bf 100644 --- a/repertory/librepertory/src/file_manager/file_manager_ring_buffer_open_file.cpp +++ b/repertory/librepertory/src/file_manager/ring_buffer_open_file.cpp @@ -1,322 +1,323 @@ -/* - Copyright <2018-2024> - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. -*/ -#include "file_manager/file_manager.hpp" - -#include "app_config.hpp" -#include "file_manager/events.hpp" -#include "platform/platform.hpp" -#include "providers/i_provider.hpp" -#include "types/repertory.hpp" -#include "utils/common.hpp" -#include "utils/encrypting_reader.hpp" -#include "utils/file_utils.hpp" -#include "utils/path.hpp" -#include "utils/utils.hpp" - -namespace repertory { -file_manager::ring_buffer_open_file::ring_buffer_open_file( - std::string buffer_directory, std::uint64_t chunk_size, - std::uint8_t chunk_timeout, filesystem_item fsi, i_provider &provider) - : ring_buffer_open_file(std::move(buffer_directory), chunk_size, - chunk_timeout, std::move(fsi), provider, - (1024ULL * 1024ULL * 1024ULL) / chunk_size) {} - -file_manager::ring_buffer_open_file::ring_buffer_open_file( - std::string buffer_directory, std::uint64_t chunk_size, - std::uint8_t chunk_timeout, filesystem_item fsi, i_provider &provider, - std::size_t ring_size) - : open_file_base(chunk_size, chunk_timeout, fsi, provider), - ring_state_(ring_size), - total_chunks_(static_cast( - utils::divide_with_ceiling(fsi.size, chunk_size_))) { - if ((ring_size % 2U) != 0U) { - throw std::runtime_error("ring size must be a multiple of 2"); - } - - if (ring_size < 4U) { - throw std::runtime_error("ring size must be greater than or equal to 4"); - } - - if (fsi.size < (ring_state_.size() * chunk_size)) { - throw std::runtime_error("file size is less than ring buffer size"); - } - - last_chunk_ = ring_state_.size() - 1U; - ring_state_.set(0U, ring_state_.size(), true); - - buffer_directory = utils::path::absolute(buffer_directory); - if (not utils::file::directory(buffer_directory).create_directory()) { - throw std::runtime_error("failed to create buffer directory|path|" + - buffer_directory + "|err|" + - std::to_string(utils::get_last_error_code())); - } - - fsi_.source_path = - utils::path::combine(buffer_directory, {utils::create_uuid_string()}); - nf_ = utils::file::file::open_or_create_file(fsi_.source_path); - if (not*nf_) { - throw std::runtime_error("failed to create buffer file|err|" + - std::to_string(utils::get_last_error_code())); - } - - if (not nf_->truncate(ring_state_.size() * chunk_size)) { - nf_->close(); - throw std::runtime_error("failed to resize buffer file|err|" + - std::to_string(utils::get_last_error_code())); - } -} - -file_manager::ring_buffer_open_file::~ring_buffer_open_file() { - REPERTORY_USES_FUNCTION_NAME(); - - close(); - - nf_->close(); - if (not utils::file::file(fsi_.source_path).remove()) { - utils::error::raise_api_path_error( - function_name, fsi_.api_path, fsi_.source_path, - utils::get_last_error_code(), "failed to delete file"); - } -} - -auto file_manager::file_manager::ring_buffer_open_file::download_chunk( - std::size_t chunk) -> api_error { - unique_mutex_lock chunk_lock(chunk_mtx_); - if (active_downloads_.find(chunk) != active_downloads_.end()) { - auto active_download = active_downloads_.at(chunk); - chunk_notify_.notify_all(); - chunk_lock.unlock(); - - return active_download->wait(); - } - - if (ring_state_[chunk % ring_state_.size()]) { - auto active_download = std::make_shared(); - active_downloads_[chunk] = active_download; - ring_state_[chunk % ring_state_.size()] = false; - chunk_notify_.notify_all(); - chunk_lock.unlock(); - - data_buffer buffer((chunk == (total_chunks_ - 1U)) ? last_chunk_size_ - : chunk_size_); - - stop_type stop_requested = !!ring_state_[chunk % ring_state_.size()]; - auto res = - provider_.read_file_bytes(fsi_.api_path, buffer.size(), - chunk * chunk_size_, buffer, stop_requested); - if (res == api_error::success) { - res = do_io([&]() -> api_error { - std::size_t bytes_written{}; - if (not nf_->write(buffer, (chunk % ring_state_.size()) * chunk_size_, - &bytes_written)) { - return api_error::os_error; - } - - return api_error::success; - }); - } - - active_download->notify(res); - - chunk_lock.lock(); - active_downloads_.erase(chunk); - chunk_notify_.notify_all(); - return res; - } - - chunk_notify_.notify_all(); - chunk_lock.unlock(); - - return api_error::success; -} - -void file_manager::ring_buffer_open_file::forward(std::size_t count) { - mutex_lock chunk_lock(chunk_mtx_); - if ((current_chunk_ + count) > (total_chunks_ - 1U)) { - count = (total_chunks_ - 1U) - current_chunk_; - } - - if ((current_chunk_ + count) <= last_chunk_) { - current_chunk_ += count; - } else { - const auto added = count - (last_chunk_ - current_chunk_); - if (added >= ring_state_.size()) { - ring_state_.set(0U, ring_state_.size(), true); - current_chunk_ += count; - first_chunk_ += added; - last_chunk_ = - std::min(total_chunks_ - 1U, first_chunk_ + ring_state_.size() - 1U); - } else { - for (std::size_t idx = 0U; idx < added; ++idx) { - ring_state_[(first_chunk_ + idx) % ring_state_.size()] = true; - } - first_chunk_ += added; - current_chunk_ += count; - last_chunk_ = - std::min(total_chunks_ - 1U, first_chunk_ + ring_state_.size() - 1U); - } - } - - chunk_notify_.notify_all(); -} - -auto file_manager::ring_buffer_open_file::get_read_state() const - -> boost::dynamic_bitset<> { - recur_mutex_lock file_lock(file_mtx_); - auto read_state = ring_state_; - return read_state.flip(); -} - -auto file_manager::ring_buffer_open_file::get_read_state( - std::size_t chunk) const -> bool { - recur_mutex_lock file_lock(file_mtx_); - return not ring_state_[chunk % ring_state_.size()]; -} - -auto file_manager::ring_buffer_open_file::is_download_complete() const -> bool { - return false; -} - -auto file_manager::ring_buffer_open_file::native_operation( - i_open_file::native_operation_callback callback) -> api_error { - return do_io([&]() -> api_error { return callback(nf_->get_handle()); }); -} - -void file_manager::ring_buffer_open_file::reverse(std::size_t count) { - mutex_lock chunk_lock(chunk_mtx_); - if (current_chunk_ < count) { - count = current_chunk_; - } - - if ((current_chunk_ - count) >= first_chunk_) { - current_chunk_ -= count; - } else { - const auto removed = count - (current_chunk_ - first_chunk_); - if (removed >= ring_state_.size()) { - ring_state_.set(0U, ring_state_.size(), true); - current_chunk_ -= count; - first_chunk_ = current_chunk_; - last_chunk_ = - std::min(total_chunks_ - 1U, first_chunk_ + ring_state_.size() - 1U); - } else { - for (std::size_t idx = 0U; idx < removed; ++idx) { - ring_state_[(last_chunk_ - idx) % ring_state_.size()] = true; - } - first_chunk_ -= removed; - current_chunk_ -= count; - last_chunk_ = - std::min(total_chunks_ - 1U, first_chunk_ + ring_state_.size() - 1U); - } - } - - chunk_notify_.notify_all(); -} - -auto file_manager::ring_buffer_open_file::read(std::size_t read_size, - std::uint64_t read_offset, - data_buffer &data) -> api_error { - if (fsi_.directory) { - return api_error::invalid_operation; - } - - reset_timeout(); - - read_size = utils::calculate_read_size(fsi_.size, read_size, read_offset); - if (read_size == 0U) { - return api_error::success; - } - - const auto start_chunk_index = - static_cast(read_offset / chunk_size_); - read_offset = read_offset - (start_chunk_index * chunk_size_); - data_buffer buffer(chunk_size_); - - auto res = api_error::success; - for (std::size_t chunk = start_chunk_index; - (res == api_error::success) && (read_size > 0U); ++chunk) { - if (chunk > current_chunk_) { - forward(chunk - current_chunk_); - } else if (chunk < current_chunk_) { - reverse(current_chunk_ - chunk); - } - - reset_timeout(); - res = download_chunk(chunk); - if (res == api_error::success) { - const auto to_read = std::min( - static_cast(chunk_size_ - read_offset), read_size); - res = do_io([this, &buffer, &chunk, &data, read_offset, - &to_read]() -> api_error { - std::size_t bytes_read{}; - auto ret = - nf_->read(buffer, ((chunk % ring_state_.size()) * chunk_size_), - &bytes_read) - ? api_error::success - : api_error::os_error; - if (ret == api_error::success) { - data.insert(data.end(), - buffer.begin() + static_cast(read_offset), - buffer.begin() + - static_cast(read_offset + to_read)); - reset_timeout(); - } - - return ret; - }); - read_offset = 0U; - read_size -= to_read; - } - } - - return res; -} - -void file_manager::ring_buffer_open_file::set(std::size_t first_chunk, - std::size_t current_chunk) { - mutex_lock chunk_lock(chunk_mtx_); - if (first_chunk >= total_chunks_) { - chunk_notify_.notify_all(); - throw std::runtime_error("first chunk must be less than total chunks"); - } - - first_chunk_ = first_chunk; - last_chunk_ = first_chunk_ + ring_state_.size() - 1U; - - if (current_chunk > last_chunk_) { - chunk_notify_.notify_all(); - throw std::runtime_error( - "current chunk must be less than or equal to last chunk"); - } - - current_chunk_ = current_chunk; - ring_state_.set(0U, ring_state_.size(), false); - - chunk_notify_.notify_all(); -} - -void file_manager::ring_buffer_open_file::set_api_path( - const std::string &api_path) { - mutex_lock chunk_lock(chunk_mtx_); - open_file_base::set_api_path(api_path); - chunk_notify_.notify_all(); -} -} // namespace repertory +/* + Copyright <2018-2024> + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ +#include "file_manager/ring_buffer_open_file.hpp" + +#include "app_config.hpp" +#include "file_manager/events.hpp" +#include "file_manager/open_file_base.hpp" +#include "platform/platform.hpp" +#include "providers/i_provider.hpp" +#include "types/repertory.hpp" +#include "utils/common.hpp" +#include "utils/encrypting_reader.hpp" +#include "utils/file_utils.hpp" +#include "utils/path.hpp" +#include "utils/utils.hpp" + +namespace repertory { +ring_buffer_open_file::ring_buffer_open_file(std::string buffer_directory, + std::uint64_t chunk_size, + std::uint8_t chunk_timeout, + filesystem_item fsi, + i_provider &provider) + : ring_buffer_open_file(std::move(buffer_directory), chunk_size, + chunk_timeout, std::move(fsi), provider, + (1024ULL * 1024ULL * 1024ULL) / chunk_size) {} + +ring_buffer_open_file::ring_buffer_open_file(std::string buffer_directory, + std::uint64_t chunk_size, + std::uint8_t chunk_timeout, + filesystem_item fsi, + i_provider &provider, + std::size_t ring_size) + : open_file_base(chunk_size, chunk_timeout, fsi, provider), + ring_state_(ring_size), + total_chunks_(static_cast( + utils::divide_with_ceiling(fsi.size, chunk_size_))) { + if ((ring_size % 2U) != 0U) { + throw std::runtime_error("ring size must be a multiple of 2"); + } + + if (ring_size < 4U) { + throw std::runtime_error("ring size must be greater than or equal to 4"); + } + + if (fsi.size < (ring_state_.size() * chunk_size)) { + throw std::runtime_error("file size is less than ring buffer size"); + } + + last_chunk_ = ring_state_.size() - 1U; + ring_state_.set(0U, ring_state_.size(), true); + + buffer_directory = utils::path::absolute(buffer_directory); + if (not utils::file::directory(buffer_directory).create_directory()) { + throw std::runtime_error("failed to create buffer directory|path|" + + buffer_directory + "|err|" + + std::to_string(utils::get_last_error_code())); + } + + fsi_.source_path = + utils::path::combine(buffer_directory, {utils::create_uuid_string()}); + nf_ = utils::file::file::open_or_create_file(fsi_.source_path); + if (not*nf_) { + throw std::runtime_error("failed to create buffer file|err|" + + std::to_string(utils::get_last_error_code())); + } + + if (not nf_->truncate(ring_state_.size() * chunk_size)) { + nf_->close(); + throw std::runtime_error("failed to resize buffer file|err|" + + std::to_string(utils::get_last_error_code())); + } +} + +ring_buffer_open_file::~ring_buffer_open_file() { + REPERTORY_USES_FUNCTION_NAME(); + + close(); + + nf_->close(); + if (not utils::file::file(fsi_.source_path).remove()) { + utils::error::raise_api_path_error( + function_name, fsi_.api_path, fsi_.source_path, + utils::get_last_error_code(), "failed to delete file"); + } +} + +auto r::ring_buffer_open_file::download_chunk(std::size_t chunk) -> api_error { + unique_mutex_lock chunk_lock(chunk_mtx_); + if (active_downloads_.find(chunk) != active_downloads_.end()) { + auto active_download = active_downloads_.at(chunk); + chunk_notify_.notify_all(); + chunk_lock.unlock(); + + return active_download->wait(); + } + + if (ring_state_[chunk % ring_state_.size()]) { + auto active_download = std::make_shared(); + active_downloads_[chunk] = active_download; + ring_state_[chunk % ring_state_.size()] = false; + chunk_notify_.notify_all(); + chunk_lock.unlock(); + + data_buffer buffer((chunk == (total_chunks_ - 1U)) ? last_chunk_size_ + : chunk_size_); + + stop_type stop_requested = !!ring_state_[chunk % ring_state_.size()]; + auto res = + provider_.read_file_bytes(fsi_.api_path, buffer.size(), + chunk * chunk_size_, buffer, stop_requested); + if (res == api_error::success) { + res = do_io([&]() -> api_error { + std::size_t bytes_written{}; + if (not nf_->write(buffer, (chunk % ring_state_.size()) * chunk_size_, + &bytes_written)) { + return api_error::os_error; + } + + return api_error::success; + }); + } + + active_download->notify(res); + + chunk_lock.lock(); + active_downloads_.erase(chunk); + chunk_notify_.notify_all(); + return res; + } + + chunk_notify_.notify_all(); + chunk_lock.unlock(); + + return api_error::success; +} + +void ring_buffer_open_file::forward(std::size_t count) { + mutex_lock chunk_lock(chunk_mtx_); + if ((current_chunk_ + count) > (total_chunks_ - 1U)) { + count = (total_chunks_ - 1U) - current_chunk_; + } + + if ((current_chunk_ + count) <= last_chunk_) { + current_chunk_ += count; + } else { + const auto added = count - (last_chunk_ - current_chunk_); + if (added >= ring_state_.size()) { + ring_state_.set(0U, ring_state_.size(), true); + current_chunk_ += count; + first_chunk_ += added; + last_chunk_ = + std::min(total_chunks_ - 1U, first_chunk_ + ring_state_.size() - 1U); + } else { + for (std::size_t idx = 0U; idx < added; ++idx) { + ring_state_[(first_chunk_ + idx) % ring_state_.size()] = true; + } + first_chunk_ += added; + current_chunk_ += count; + last_chunk_ = + std::min(total_chunks_ - 1U, first_chunk_ + ring_state_.size() - 1U); + } + } + + chunk_notify_.notify_all(); +} + +auto ring_buffer_open_file::get_read_state() const -> boost::dynamic_bitset<> { + recur_mutex_lock file_lock(file_mtx_); + auto read_state = ring_state_; + return read_state.flip(); +} + +auto ring_buffer_open_file::get_read_state(std::size_t chunk) const -> bool { + recur_mutex_lock file_lock(file_mtx_); + return not ring_state_[chunk % ring_state_.size()]; +} + +auto ring_buffer_open_file::is_download_complete() const -> bool { + return false; +} + +auto ring_buffer_open_file::native_operation( + i_open_file::native_operation_callback callback) -> api_error { + return do_io([&]() -> api_error { return callback(nf_->get_handle()); }); +} + +void ring_buffer_open_file::reverse(std::size_t count) { + mutex_lock chunk_lock(chunk_mtx_); + if (current_chunk_ < count) { + count = current_chunk_; + } + + if ((current_chunk_ - count) >= first_chunk_) { + current_chunk_ -= count; + } else { + const auto removed = count - (current_chunk_ - first_chunk_); + if (removed >= ring_state_.size()) { + ring_state_.set(0U, ring_state_.size(), true); + current_chunk_ -= count; + first_chunk_ = current_chunk_; + last_chunk_ = + std::min(total_chunks_ - 1U, first_chunk_ + ring_state_.size() - 1U); + } else { + for (std::size_t idx = 0U; idx < removed; ++idx) { + ring_state_[(last_chunk_ - idx) % ring_state_.size()] = true; + } + first_chunk_ -= removed; + current_chunk_ -= count; + last_chunk_ = + std::min(total_chunks_ - 1U, first_chunk_ + ring_state_.size() - 1U); + } + } + + chunk_notify_.notify_all(); +} + +auto ring_buffer_open_file::read(std::size_t read_size, + std::uint64_t read_offset, data_buffer &data) + -> api_error { + if (fsi_.directory) { + return api_error::invalid_operation; + } + + reset_timeout(); + + read_size = utils::calculate_read_size(fsi_.size, read_size, read_offset); + if (read_size == 0U) { + return api_error::success; + } + + const auto start_chunk_index = + static_cast(read_offset / chunk_size_); + read_offset = read_offset - (start_chunk_index * chunk_size_); + data_buffer buffer(chunk_size_); + + auto res = api_error::success; + for (std::size_t chunk = start_chunk_index; + (res == api_error::success) && (read_size > 0U); ++chunk) { + if (chunk > current_chunk_) { + forward(chunk - current_chunk_); + } else if (chunk < current_chunk_) { + reverse(current_chunk_ - chunk); + } + + reset_timeout(); + res = download_chunk(chunk); + if (res == api_error::success) { + const auto to_read = std::min( + static_cast(chunk_size_ - read_offset), read_size); + res = do_io([this, &buffer, &chunk, &data, read_offset, + &to_read]() -> api_error { + std::size_t bytes_read{}; + auto ret = + nf_->read(buffer, ((chunk % ring_state_.size()) * chunk_size_), + &bytes_read) + ? api_error::success + : api_error::os_error; + if (ret == api_error::success) { + data.insert(data.end(), + buffer.begin() + static_cast(read_offset), + buffer.begin() + + static_cast(read_offset + to_read)); + reset_timeout(); + } + + return ret; + }); + read_offset = 0U; + read_size -= to_read; + } + } + + return res; +} + +void ring_buffer_open_file::set(std::size_t first_chunk, + std::size_t current_chunk) { + mutex_lock chunk_lock(chunk_mtx_); + if (first_chunk >= total_chunks_) { + chunk_notify_.notify_all(); + throw std::runtime_error("first chunk must be less than total chunks"); + } + + first_chunk_ = first_chunk; + last_chunk_ = first_chunk_ + ring_state_.size() - 1U; + + if (current_chunk > last_chunk_) { + chunk_notify_.notify_all(); + throw std::runtime_error( + "current chunk must be less than or equal to last chunk"); + } + + current_chunk_ = current_chunk; + ring_state_.set(0U, ring_state_.size(), false); + + chunk_notify_.notify_all(); +} + +void ring_buffer_open_file::set_api_path(const std::string &api_path) { + mutex_lock chunk_lock(chunk_mtx_); + open_file_base::set_api_path(api_path); + chunk_notify_.notify_all(); +} +} // namespace repertory diff --git a/repertory/librepertory/src/file_manager/file_manager_upload.cpp b/repertory/librepertory/src/file_manager/upload.cpp similarity index 95% rename from repertory/librepertory/src/file_manager/file_manager_upload.cpp rename to repertory/librepertory/src/file_manager/upload.cpp index ca903242..815f7d23 100644 --- a/repertory/librepertory/src/file_manager/file_manager_upload.cpp +++ b/repertory/librepertory/src/file_manager/upload.cpp @@ -1,65 +1,65 @@ -/* - Copyright <2018-2024> - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. -*/ -#include "file_manager/file_manager.hpp" - -#include "platform/platform.hpp" -#include "providers/i_provider.hpp" -#include "utils/error_utils.hpp" -#include "utils/file_utils.hpp" - -namespace repertory { -using std::bind; - -file_manager::upload::upload(filesystem_item fsi, i_provider &provider) - : fsi_(std::move(fsi)), provider_(provider) { - thread_ = std::make_unique([this] { upload_thread(); }); -} - -file_manager::upload::~upload() { - stop(); - - thread_->join(); - thread_.reset(); -} - -void file_manager::upload::cancel() { - cancelled_ = true; - stop(); -} - -void file_manager::upload::stop() { stop_requested_ = true; } - -void file_manager::upload::upload_thread() { - REPERTORY_USES_FUNCTION_NAME(); - - error_ = - provider_.upload_file(fsi_.api_path, fsi_.source_path, stop_requested_); - if (not utils::file::reset_modified_time(fsi_.source_path)) { - utils::error::raise_api_path_error( - function_name, fsi_.api_path, fsi_.source_path, - utils::get_last_error_code(), "failed to reset modified time"); - } - - event_system::instance().raise( - get_api_path(), get_source_path(), get_api_error(), cancelled_); -} -} // namespace repertory +/* + Copyright <2018-2024> + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ +#include "file_manager/upload.hpp" + +#include "platform/platform.hpp" +#include "providers/i_provider.hpp" +#include "utils/error_utils.hpp" +#include "utils/file_utils.hpp" + +namespace repertory { +using std::bind; + +file_manager::upload::upload(filesystem_item fsi, i_provider &provider) + : fsi_(std::move(fsi)), provider_(provider) { + thread_ = std::make_unique([this] { upload_thread(); }); +} + +file_manager::upload::~upload() { + stop(); + + thread_->join(); + thread_.reset(); +} + +void file_manager::upload::cancel() { + cancelled_ = true; + stop(); +} + +void file_manager::upload::stop() { stop_requested_ = true; } + +void file_manager::upload::upload_thread() { + REPERTORY_USES_FUNCTION_NAME(); + + error_ = + provider_.upload_file(fsi_.api_path, fsi_.source_path, stop_requested_); + if (not utils::file::reset_modified_time(fsi_.source_path)) { + utils::error::raise_api_path_error( + function_name, fsi_.api_path, fsi_.source_path, + utils::get_last_error_code(), "failed to reset modified time"); + } + + event_system::instance().raise( + get_api_path(), get_source_path(), get_api_error(), cancelled_); +} +} // namespace repertory diff --git a/repertory/repertory_test/src/file_manager_open_file_test.cpp b/repertory/repertory_test/src/open_file_test.cpp similarity index 97% rename from repertory/repertory_test/src/file_manager_open_file_test.cpp rename to repertory/repertory_test/src/open_file_test.cpp index d322cc67..05effc1a 100644 --- a/repertory/repertory_test/src/file_manager_open_file_test.cpp +++ b/repertory/repertory_test/src/open_file_test.cpp @@ -1,685 +1,685 @@ -/* - Copyright <2018-2024> - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. -*/ -#include "test_common.hpp" - -#include "file_manager/file_manager.hpp" -#include "mocks/mock_provider.hpp" -#include "mocks/mock_upload_manager.hpp" -#include "types/repertory.hpp" -#include "utils/event_capture.hpp" -#include "utils/path.hpp" - -namespace repertory { -static constexpr const std::size_t test_chunk_size = 1024u; - -static void test_closeable_open_file(const file_manager::open_file &o, - bool directory, const api_error &e, - std::uint64_t size, - const std::string &source_path) { - EXPECT_EQ(directory, o.is_directory()); - EXPECT_EQ(e, o.get_api_error()); - EXPECT_EQ(std::size_t(0u), o.get_open_file_count()); - EXPECT_EQ(std::uint64_t(size), o.get_file_size()); - EXPECT_STREQ(source_path.c_str(), o.get_source_path().c_str()); - EXPECT_TRUE(o.can_close()); -} - -static void validate_write(file_manager::open_file &o, std::size_t offset, - data_buffer data, std::size_t bytes_written) { - EXPECT_EQ(data.size(), bytes_written); - - data_buffer read_data{}; - EXPECT_EQ(api_error::success, o.read(data.size(), offset, read_data)); - - EXPECT_TRUE(std::equal(data.begin(), data.end(), read_data.begin())); -} - -TEST(open_file, properly_initializes_state_for_0_byte_file) { - const auto source_path = - test::generate_test_file_name("file_manager_open_file_test"); - - mock_provider mp; - mock_upload_manager um; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - - filesystem_item fsi; - fsi.directory = false; - fsi.api_path = "/test.txt"; - fsi.size = 0u; - fsi.source_path = source_path; - - file_manager::open_file o(test_chunk_size, 0U, fsi, mp, um); - EXPECT_EQ(std::size_t(0u), o.get_read_state().size()); - EXPECT_FALSE(o.is_modified()); - EXPECT_EQ(test_chunk_size, o.get_chunk_size()); -} - -TEST(open_file, properly_initializes_state_based_on_chunk_size) { - const auto source_path = - test::generate_test_file_name("file_manager_open_file_test"); - - mock_provider mp; - mock_upload_manager um; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - - filesystem_item fsi; - fsi.directory = false; - fsi.api_path = "/test.txt"; - fsi.size = 8u; - fsi.source_path = source_path; - - EXPECT_CALL(um, remove_resume) - .WillOnce( - [&fsi](const std::string &api_path, const std::string &source_path2) { - EXPECT_EQ(fsi.api_path, api_path); - EXPECT_EQ(fsi.source_path, source_path2); - }); - - file_manager::open_file o(1u, 0U, fsi, mp, um); - EXPECT_EQ(std::size_t(8u), o.get_read_state().size()); - EXPECT_TRUE(o.get_read_state().none()); - - EXPECT_FALSE(o.is_modified()); - - EXPECT_CALL(mp, set_item_meta(fsi.api_path, META_SOURCE, _)) - .WillOnce(Return(api_error::success)); - EXPECT_EQ(std::size_t(1u), o.get_chunk_size()); -} - -TEST(open_file, will_not_change_source_path_for_0_byte_file) { - const auto source_path = - test::generate_test_file_name("file_manager_open_file_test"); - - mock_provider mp; - mock_upload_manager um; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - - filesystem_item fsi; - fsi.directory = false; - fsi.api_path = "/test.txt"; - fsi.size = 0u; - fsi.source_path = source_path; - - file_manager::open_file o(0u, 0U, fsi, mp, um); - test_closeable_open_file(o, false, api_error::success, 0u, source_path); - - o.close(); - EXPECT_EQ(api_error::success, o.get_api_error()); - EXPECT_STREQ(source_path.c_str(), o.get_source_path().c_str()); - EXPECT_TRUE(utils::file::file(fsi.source_path).exists()); -} - -TEST(open_file, will_change_source_path_if_file_size_is_greater_than_0) { - const auto source_path = - test::generate_test_file_name("file_manager_open_file_test"); - - mock_provider mp; - mock_upload_manager um; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - - filesystem_item fsi; - fsi.api_path = "/test.txt"; - fsi.size = test_chunk_size; - fsi.source_path = source_path; - - EXPECT_CALL(um, remove_resume) - .WillOnce( - [&fsi](const std::string &api_path, const std::string &source_path2) { - EXPECT_EQ(fsi.api_path, api_path); - EXPECT_EQ(fsi.source_path, source_path2); - }); - - EXPECT_CALL(mp, set_item_meta(fsi.api_path, META_SOURCE, _)) - .WillOnce([&fsi](const std::string &, const std::string &, - const std::string &source_path2) -> api_error { - EXPECT_STRNE(fsi.source_path.c_str(), source_path2.c_str()); - return api_error::success; - }); - - file_manager::open_file o(test_chunk_size, 0U, fsi, mp, um); - test_closeable_open_file(o, false, api_error::success, test_chunk_size, - source_path); - - o.close(); - EXPECT_EQ(api_error::download_stopped, o.get_api_error()); - EXPECT_STRNE(source_path.c_str(), o.get_source_path().c_str()); - EXPECT_FALSE(utils::file::file(source_path).exists()); -} - -TEST(open_file, - will_not_change_source_path_if_file_size_matches_existing_source) { - auto &rf = test::create_random_file(test_chunk_size); - const auto source_path = rf.get_path(); - rf.close(); - - mock_provider mp; - mock_upload_manager um; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - - filesystem_item fsi; - fsi.api_path = "/test.txt"; - fsi.size = test_chunk_size; - fsi.source_path = source_path; - - file_manager::open_file o(test_chunk_size, 0U, fsi, mp, um); - test_closeable_open_file(o, false, api_error::success, test_chunk_size, - source_path); - - o.close(); - EXPECT_EQ(api_error::success, o.get_api_error()); - EXPECT_STREQ(source_path.c_str(), o.get_source_path().c_str()); - EXPECT_TRUE(utils::file::file(source_path).exists()); -} - -TEST(open_file, write_with_incomplete_download) { - const auto source_path = test::generate_test_file_name("test"); - auto &nf = test::create_random_file(test_chunk_size * 2u); - - mock_provider mp; - mock_upload_manager um; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - - filesystem_item fsi; - fsi.api_path = "/test.txt"; - fsi.size = test_chunk_size * 2u; - fsi.source_path = source_path; - - file_manager::open_file o(test_chunk_size, 0U, fsi, mp, um); - test_closeable_open_file(o, false, api_error::success, test_chunk_size * 2u, - source_path); - - EXPECT_CALL(mp, set_item_meta(fsi.api_path, _)) - .WillOnce([](const std::string &, const api_meta_map &meta) -> api_error { - EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_CHANGED).empty())); - EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_MODIFIED).empty())); - EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_WRITTEN).empty())); - return api_error::success; - }); - - EXPECT_CALL(mp, read_file_bytes) - .WillRepeatedly([&nf](const std::string & /* api_path */, - std::size_t size, std::uint64_t offset, - data_buffer &data, - stop_type &stop_requested) -> api_error { - if (stop_requested) { - return api_error::download_stopped; - } - - if (offset == 0u) { - std::size_t bytes_read{}; - data.resize(size); - auto ret = nf.read(data, offset, &bytes_read) ? api_error::success - : api_error::os_error; - EXPECT_EQ(bytes_read, data.size()); - return ret; - } - - while (not stop_requested) { - std::this_thread::sleep_for(100ms); - } - return api_error::download_stopped; - }); - - EXPECT_CALL(um, remove_upload).WillOnce([&fsi](const std::string &api_path) { - EXPECT_EQ(fsi.api_path, api_path); - }); - - EXPECT_CALL(um, store_resume) - .Times(2) - .WillRepeatedly([&fsi](const i_open_file &cur_file) { - EXPECT_EQ(fsi.api_path, cur_file.get_api_path()); - EXPECT_EQ(fsi.source_path, cur_file.get_source_path()); - }); - - data_buffer data = {10, 9, 8}; - std::size_t bytes_written{}; - EXPECT_EQ(api_error::success, o.write(0u, data, bytes_written)); - validate_write(o, 0u, data, bytes_written); - - const auto test_state = [&]() { - EXPECT_STREQ(source_path.c_str(), o.get_source_path().c_str()); - - EXPECT_FALSE(o.can_close()); - - EXPECT_TRUE(o.is_modified()); - - EXPECT_TRUE(o.get_read_state(0u)); - EXPECT_FALSE(o.get_read_state(1u)); - }; - test_state(); - - o.close(); - nf.close(); - - test_state(); - - EXPECT_EQ(api_error::download_incomplete, o.get_api_error()); - - EXPECT_TRUE(utils::file::file(fsi.source_path).exists()); -} - -TEST(open_file, write_new_file) { - const auto source_path = - test::generate_test_file_name("file_manager_open_file_test"); - - mock_provider mp; - mock_upload_manager um; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - - filesystem_item fsi; - fsi.api_path = "/test.txt"; - fsi.size = 0u; - fsi.source_path = source_path; - - EXPECT_CALL(um, store_resume).WillOnce([&fsi](const i_open_file &o) { - EXPECT_EQ(fsi.api_path, o.get_api_path()); - EXPECT_EQ(fsi.source_path, o.get_source_path()); - }); - - file_manager::open_file o(test_chunk_size, 0U, fsi, mp, um); - test_closeable_open_file(o, false, api_error::success, 0u, source_path); - data_buffer data = {10, 9, 8}; - - EXPECT_CALL(mp, set_item_meta(fsi.api_path, _)) - .WillOnce([&data](const std::string &, - const api_meta_map &meta) -> api_error { - EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_CHANGED).empty())); - EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_MODIFIED).empty())); - EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_SIZE).empty())); - EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_WRITTEN).empty())); - EXPECT_EQ(data.size(), utils::string::to_size_t(meta.at(META_SIZE))); - return api_error::success; - }) - .WillOnce([](const std::string &, const api_meta_map &meta) -> api_error { - EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_CHANGED).empty())); - EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_MODIFIED).empty())); - EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_WRITTEN).empty())); - return api_error::success; - }); - - EXPECT_CALL(um, remove_upload).WillOnce([&fsi](const std::string &api_path) { - EXPECT_EQ(fsi.api_path, api_path); - }); - - EXPECT_CALL(um, queue_upload).WillOnce([&fsi](const i_open_file &cur_file) { - EXPECT_EQ(fsi.api_path, cur_file.get_api_path()); - EXPECT_EQ(fsi.source_path, cur_file.get_source_path()); - }); - - std::size_t bytes_written{}; - EXPECT_EQ(api_error::success, o.write(0u, data, bytes_written)); - - const auto test_state = [&]() { - EXPECT_STREQ(source_path.c_str(), o.get_source_path().c_str()); - - EXPECT_FALSE(o.can_close()); - EXPECT_TRUE(o.is_modified()); - - EXPECT_TRUE(o.get_read_state(0u)); - EXPECT_EQ(std::size_t(1u), o.get_read_state().size()); - EXPECT_EQ(data.size(), o.get_file_size()); - }; - test_state(); - - o.close(); - - test_state(); - - EXPECT_EQ(api_error::success, o.get_api_error()); - - EXPECT_TRUE(utils::file::file(fsi.source_path).exists()); -} - -TEST(open_file, write_new_file_multiple_chunks) { - const auto source_path = - test::generate_test_file_name("file_manager_open_file_test"); - - mock_provider mp; - mock_upload_manager um; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - - filesystem_item fsi; - fsi.api_path = "/test.txt"; - fsi.size = 0u; - fsi.source_path = source_path; - - EXPECT_CALL(um, store_resume).WillOnce([&fsi](const i_open_file &o) { - EXPECT_EQ(fsi.api_path, o.get_api_path()); - EXPECT_EQ(fsi.source_path, o.get_source_path()); - }); - file_manager::open_file o(test_chunk_size, 0U, fsi, mp, um); - test_closeable_open_file(o, false, api_error::success, 0u, source_path); - data_buffer data = {10, 9, 8}; - - EXPECT_CALL(mp, set_item_meta(fsi.api_path, _)) - .WillOnce([&data](const std::string &, - const api_meta_map &meta) -> api_error { - EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_CHANGED).empty())); - EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_MODIFIED).empty())); - EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_SIZE).empty())); - EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_WRITTEN).empty())); - EXPECT_EQ(data.size(), utils::string::to_size_t(meta.at(META_SIZE))); - return api_error::success; - }) - .WillOnce([](const std::string &, const api_meta_map &meta) -> api_error { - EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_CHANGED).empty())); - EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_MODIFIED).empty())); - EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_WRITTEN).empty())); - return api_error::success; - }) - .WillOnce( - [&data](const std::string &, const api_meta_map &meta) -> api_error { - EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_CHANGED).empty())); - EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_MODIFIED).empty())); - EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_SIZE).empty())); - EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_WRITTEN).empty())); - EXPECT_EQ(data.size() + test_chunk_size, - utils::string::to_size_t(meta.at(META_SIZE))); - return api_error::success; - }) - .WillOnce([](const std::string &, const api_meta_map &meta) -> api_error { - EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_CHANGED).empty())); - EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_MODIFIED).empty())); - EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_WRITTEN).empty())); - return api_error::success; - }); - - EXPECT_CALL(um, remove_upload).WillOnce([&fsi](const std::string &api_path) { - EXPECT_EQ(fsi.api_path, api_path); - }); - - EXPECT_CALL(um, queue_upload).WillOnce([&fsi](const i_open_file &cur_file) { - EXPECT_EQ(fsi.api_path, cur_file.get_api_path()); - EXPECT_EQ(fsi.source_path, cur_file.get_source_path()); - }); - - std::size_t bytes_written{}; - EXPECT_EQ(api_error::success, o.write(0u, data, bytes_written)); - EXPECT_EQ(api_error::success, o.write(test_chunk_size, data, bytes_written)); - - const auto test_state = [&]() { - EXPECT_STREQ(source_path.c_str(), o.get_source_path().c_str()); - - EXPECT_FALSE(o.can_close()); - EXPECT_TRUE(o.is_modified()); - - EXPECT_EQ(std::size_t(2u), o.get_read_state().size()); - for (std::size_t i = 0u; i < 2u; i++) { - EXPECT_TRUE(o.get_read_state(i)); - } - - EXPECT_EQ(data.size() + test_chunk_size, o.get_file_size()); - }; - test_state(); - - o.close(); - - test_state(); - - EXPECT_EQ(api_error::success, o.get_api_error()); - - EXPECT_TRUE(utils::file::file(fsi.source_path).exists()); -} - -TEST(open_file, resize_file_to_0_bytes) { - auto &rf = test::create_random_file(test_chunk_size * 4u); - const auto source_path = rf.get_path(); - rf.close(); - - mock_provider mp; - mock_upload_manager um; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - - filesystem_item fsi; - fsi.api_path = "/test.txt"; - fsi.size = test_chunk_size * 4u; - fsi.source_path = source_path; - - file_manager::open_file o(test_chunk_size, 0U, fsi, mp, um); - test_closeable_open_file(o, false, api_error::success, fsi.size, source_path); - EXPECT_CALL(mp, set_item_meta(fsi.api_path, _)) - .WillOnce([](const std::string &, const api_meta_map &meta) -> api_error { - EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_CHANGED).empty())); - EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_MODIFIED).empty())); - EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_SIZE).empty())); - EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_WRITTEN).empty())); - EXPECT_EQ(std::size_t(0u), - utils::string::to_size_t(meta.at(META_SIZE))); - return api_error::success; - }); - - EXPECT_CALL(um, remove_upload).WillOnce([&fsi](const std ::string &api_path) { - EXPECT_EQ(fsi.api_path, api_path); - }); - - EXPECT_CALL(um, queue_upload).WillOnce([&fsi](const i_open_file &cur_file) { - EXPECT_EQ(fsi.api_path, cur_file.get_api_path()); - EXPECT_EQ(fsi.source_path, cur_file.get_source_path()); - }); - EXPECT_CALL(um, store_resume).WillOnce([&fsi](const i_open_file &cur_file) { - EXPECT_EQ(fsi.api_path, cur_file.get_api_path()); - EXPECT_EQ(fsi.source_path, cur_file.get_source_path()); - }); - - EXPECT_EQ(api_error::success, o.resize(0u)); - - EXPECT_EQ(std::size_t(0u), o.get_file_size()); - EXPECT_FALSE(o.can_close()); - EXPECT_TRUE(o.is_modified()); - - EXPECT_EQ(std::size_t(0u), o.get_read_state().size()); -} - -TEST(open_file, resize_file_by_full_chunk) { - auto &rf = test::create_random_file(test_chunk_size * 4u); - const auto source_path = rf.get_path(); - rf.close(); - - mock_provider mp; - mock_upload_manager um; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - - filesystem_item fsi; - fsi.api_path = "/test.txt"; - fsi.size = test_chunk_size * 4u; - fsi.source_path = source_path; - - EXPECT_CALL(um, store_resume).WillOnce([&fsi](const i_open_file &o) { - EXPECT_EQ(fsi.api_path, o.get_api_path()); - EXPECT_EQ(fsi.source_path, o.get_source_path()); - }); - - file_manager::open_file o(test_chunk_size, 0U, fsi, mp, um); - test_closeable_open_file(o, false, api_error::success, fsi.size, source_path); - EXPECT_CALL(mp, set_item_meta(fsi.api_path, _)) - .WillOnce([](const std::string &, const api_meta_map &meta) -> api_error { - EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_CHANGED).empty())); - EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_MODIFIED).empty())); - EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_SIZE).empty())); - EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_WRITTEN).empty())); - EXPECT_EQ(std::size_t(test_chunk_size * 3u), - utils::string::to_size_t(meta.at(META_SIZE))); - return api_error::success; - }); - - EXPECT_CALL(um, remove_upload).WillOnce([&fsi](const std::string &api_path) { - EXPECT_EQ(fsi.api_path, api_path); - }); - - EXPECT_CALL(um, queue_upload).WillOnce([&fsi](const i_open_file &cur_file) { - EXPECT_EQ(fsi.api_path, cur_file.get_api_path()); - EXPECT_EQ(fsi.source_path, cur_file.get_source_path()); - }); - - EXPECT_EQ(api_error::success, o.resize(test_chunk_size * 3u)); - - EXPECT_EQ(std::size_t(test_chunk_size * 3u), o.get_file_size()); - EXPECT_FALSE(o.can_close()); - EXPECT_TRUE(o.is_modified()); - EXPECT_EQ(std::size_t(3u), o.get_read_state().size()); -} - -TEST(open_file, can_add_handle) { - event_system::instance().start(); - console_consumer c; - const auto source_path = - test::generate_test_file_name("file_manager_open_file_test"); - - mock_provider mp; - mock_upload_manager um; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - - filesystem_item fsi; - fsi.api_path = "/test.txt"; - fsi.size = test_chunk_size * 4u; - fsi.source_path = source_path; - - event_consumer ec("filesystem_item_opened", [&fsi](const event &e) { - const auto &ee = dynamic_cast(e); - EXPECT_STREQ(fsi.api_path.c_str(), - ee.get_api_path().get().c_str()); - EXPECT_STREQ(fsi.source_path.c_str(), - ee.get_source().get().c_str()); - EXPECT_STREQ("0", ee.get_directory().get().c_str()); - }); - - event_consumer ec2("filesystem_item_handle_opened", [&fsi](const event &e) { - const auto &ee = dynamic_cast(e); - EXPECT_STREQ(fsi.api_path.c_str(), - ee.get_api_path().get().c_str()); - EXPECT_STREQ(fsi.source_path.c_str(), - ee.get_source().get().c_str()); - EXPECT_STREQ("0", ee.get_directory().get().c_str()); - EXPECT_STREQ("1", ee.get_handle().get().c_str()); - }); - - EXPECT_CALL(mp, set_item_meta(fsi.api_path, META_SOURCE, _)) - .WillOnce(Return(api_error::success)); - EXPECT_CALL(um, remove_resume) - .WillOnce( - [&fsi](const std::string &api_path, const std::string &source_path2) { - EXPECT_EQ(fsi.api_path, api_path); - EXPECT_EQ(fsi.source_path, source_path2); - }); - - event_capture capture( - {"filesystem_item_opened", "filesystem_item_handle_opened"}); - - file_manager::open_file o(test_chunk_size, 0U, fsi, mp, um); -#if defined(_WIN32) - o.add(1u, {}); - EXPECT_EQ(nullptr, o.get_open_data(1u).directory_buffer); -#else - o.add(1u, O_RDWR | O_SYNC); - EXPECT_EQ(O_RDWR | O_SYNC, o.get_open_data(1u)); -#endif - - capture.wait_for_empty(); - - event_system::instance().stop(); -} - -TEST(open_file, can_remove_handle) { - event_system::instance().start(); - console_consumer c; - - const auto source_path = - test::generate_test_file_name("file_manager_open_file_test"); - - mock_provider mp; - mock_upload_manager um; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - - filesystem_item fsi; - fsi.api_path = "/test.txt"; - fsi.size = test_chunk_size * 4u; - fsi.source_path = source_path; - - event_consumer ec("filesystem_item_closed", [&fsi](const event &e) { - const auto &ee = dynamic_cast(e); - EXPECT_STREQ(fsi.api_path.c_str(), - ee.get_api_path().get().c_str()); - EXPECT_STREQ(fsi.source_path.c_str(), - ee.get_source().get().c_str()); - EXPECT_STREQ("0", ee.get_directory().get().c_str()); - }); - - event_consumer ec2("filesystem_item_handle_closed", [&fsi](const event &e) { - const auto &ee = dynamic_cast(e); - EXPECT_STREQ(fsi.api_path.c_str(), - ee.get_api_path().get().c_str()); - EXPECT_STREQ(fsi.source_path.c_str(), - ee.get_source().get().c_str()); - EXPECT_STREQ("0", ee.get_directory().get().c_str()); - EXPECT_STREQ("1", ee.get_handle().get().c_str()); - }); - - EXPECT_CALL(um, remove_resume) - .WillOnce( - [&fsi](const std::string &api_path, const std::string &source_path2) { - EXPECT_EQ(fsi.api_path, api_path); - EXPECT_EQ(fsi.source_path, source_path2); - }); - EXPECT_CALL(mp, set_item_meta(fsi.api_path, META_SOURCE, _)) - .WillOnce(Return(api_error::success)); - - event_capture capture({ - "filesystem_item_opened", - "filesystem_item_handle_opened", - "filesystem_item_handle_closed", - "filesystem_item_closed", - }); - - file_manager::open_file o(test_chunk_size, 0U, fsi, mp, um); -#if defined(_WIN32) - o.add(1u, {}); -#else - o.add(1u, O_RDWR | O_SYNC); -#endif - o.remove(1u); - - capture.wait_for_empty(); - - event_system::instance().stop(); -} - -TEST(open_file, - can_read_locally_after_write_with_file_size_greater_than_existing_size) {} - -TEST(open_file, test_valid_download_chunks) {} - -TEST(open_file, test_full_download_with_partial_chunk) {} - -TEST(open_file, source_is_read_after_full_download) {} -} // namespace repertory +/* + Copyright <2018-2024> + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ +#include "test_common.hpp" + +#include "file_manager/file_manager.hpp" +#include "mocks/mock_provider.hpp" +#include "mocks/mock_upload_manager.hpp" +#include "types/repertory.hpp" +#include "utils/event_capture.hpp" +#include "utils/path.hpp" + +namespace repertory { +static constexpr const std::size_t test_chunk_size = 1024u; + +static void test_closeable_open_file(const file_manager::open_file &o, + bool directory, const api_error &e, + std::uint64_t size, + const std::string &source_path) { + EXPECT_EQ(directory, o.is_directory()); + EXPECT_EQ(e, o.get_api_error()); + EXPECT_EQ(std::size_t(0u), o.get_open_file_count()); + EXPECT_EQ(std::uint64_t(size), o.get_file_size()); + EXPECT_STREQ(source_path.c_str(), o.get_source_path().c_str()); + EXPECT_TRUE(o.can_close()); +} + +static void validate_write(file_manager::open_file &o, std::size_t offset, + data_buffer data, std::size_t bytes_written) { + EXPECT_EQ(data.size(), bytes_written); + + data_buffer read_data{}; + EXPECT_EQ(api_error::success, o.read(data.size(), offset, read_data)); + + EXPECT_TRUE(std::equal(data.begin(), data.end(), read_data.begin())); +} + +TEST(open_file, properly_initializes_state_for_0_byte_file) { + const auto source_path = + test::generate_test_file_name("file_manager_open_file_test"); + + mock_provider mp; + mock_upload_manager um; + + EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); + + filesystem_item fsi; + fsi.directory = false; + fsi.api_path = "/test.txt"; + fsi.size = 0u; + fsi.source_path = source_path; + + file_manager::open_file o(test_chunk_size, 0U, fsi, mp, um); + EXPECT_EQ(std::size_t(0u), o.get_read_state().size()); + EXPECT_FALSE(o.is_modified()); + EXPECT_EQ(test_chunk_size, o.get_chunk_size()); +} + +TEST(open_file, properly_initializes_state_based_on_chunk_size) { + const auto source_path = + test::generate_test_file_name("file_manager_open_file_test"); + + mock_provider mp; + mock_upload_manager um; + + EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); + + filesystem_item fsi; + fsi.directory = false; + fsi.api_path = "/test.txt"; + fsi.size = 8u; + fsi.source_path = source_path; + + EXPECT_CALL(um, remove_resume) + .WillOnce( + [&fsi](const std::string &api_path, const std::string &source_path2) { + EXPECT_EQ(fsi.api_path, api_path); + EXPECT_EQ(fsi.source_path, source_path2); + }); + + file_manager::open_file o(1u, 0U, fsi, mp, um); + EXPECT_EQ(std::size_t(8u), o.get_read_state().size()); + EXPECT_TRUE(o.get_read_state().none()); + + EXPECT_FALSE(o.is_modified()); + + EXPECT_CALL(mp, set_item_meta(fsi.api_path, META_SOURCE, _)) + .WillOnce(Return(api_error::success)); + EXPECT_EQ(std::size_t(1u), o.get_chunk_size()); +} + +TEST(open_file, will_not_change_source_path_for_0_byte_file) { + const auto source_path = + test::generate_test_file_name("file_manager_open_file_test"); + + mock_provider mp; + mock_upload_manager um; + + EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); + + filesystem_item fsi; + fsi.directory = false; + fsi.api_path = "/test.txt"; + fsi.size = 0u; + fsi.source_path = source_path; + + file_manager::open_file o(0u, 0U, fsi, mp, um); + test_closeable_open_file(o, false, api_error::success, 0u, source_path); + + o.close(); + EXPECT_EQ(api_error::success, o.get_api_error()); + EXPECT_STREQ(source_path.c_str(), o.get_source_path().c_str()); + EXPECT_TRUE(utils::file::file(fsi.source_path).exists()); +} + +TEST(open_file, will_change_source_path_if_file_size_is_greater_than_0) { + const auto source_path = + test::generate_test_file_name("file_manager_open_file_test"); + + mock_provider mp; + mock_upload_manager um; + + EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); + + filesystem_item fsi; + fsi.api_path = "/test.txt"; + fsi.size = test_chunk_size; + fsi.source_path = source_path; + + EXPECT_CALL(um, remove_resume) + .WillOnce( + [&fsi](const std::string &api_path, const std::string &source_path2) { + EXPECT_EQ(fsi.api_path, api_path); + EXPECT_EQ(fsi.source_path, source_path2); + }); + + EXPECT_CALL(mp, set_item_meta(fsi.api_path, META_SOURCE, _)) + .WillOnce([&fsi](const std::string &, const std::string &, + const std::string &source_path2) -> api_error { + EXPECT_STRNE(fsi.source_path.c_str(), source_path2.c_str()); + return api_error::success; + }); + + file_manager::open_file o(test_chunk_size, 0U, fsi, mp, um); + test_closeable_open_file(o, false, api_error::success, test_chunk_size, + source_path); + + o.close(); + EXPECT_EQ(api_error::download_stopped, o.get_api_error()); + EXPECT_STRNE(source_path.c_str(), o.get_source_path().c_str()); + EXPECT_FALSE(utils::file::file(source_path).exists()); +} + +TEST(open_file, + will_not_change_source_path_if_file_size_matches_existing_source) { + auto &rf = test::create_random_file(test_chunk_size); + const auto source_path = rf.get_path(); + rf.close(); + + mock_provider mp; + mock_upload_manager um; + + EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); + + filesystem_item fsi; + fsi.api_path = "/test.txt"; + fsi.size = test_chunk_size; + fsi.source_path = source_path; + + file_manager::open_file o(test_chunk_size, 0U, fsi, mp, um); + test_closeable_open_file(o, false, api_error::success, test_chunk_size, + source_path); + + o.close(); + EXPECT_EQ(api_error::success, o.get_api_error()); + EXPECT_STREQ(source_path.c_str(), o.get_source_path().c_str()); + EXPECT_TRUE(utils::file::file(source_path).exists()); +} + +TEST(open_file, write_with_incomplete_download) { + const auto source_path = test::generate_test_file_name("test"); + auto &nf = test::create_random_file(test_chunk_size * 2u); + + mock_provider mp; + mock_upload_manager um; + + EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); + + filesystem_item fsi; + fsi.api_path = "/test.txt"; + fsi.size = test_chunk_size * 2u; + fsi.source_path = source_path; + + file_manager::open_file o(test_chunk_size, 0U, fsi, mp, um); + test_closeable_open_file(o, false, api_error::success, test_chunk_size * 2u, + source_path); + + EXPECT_CALL(mp, set_item_meta(fsi.api_path, _)) + .WillOnce([](const std::string &, const api_meta_map &meta) -> api_error { + EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_CHANGED).empty())); + EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_MODIFIED).empty())); + EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_WRITTEN).empty())); + return api_error::success; + }); + + EXPECT_CALL(mp, read_file_bytes) + .WillRepeatedly([&nf](const std::string & /* api_path */, + std::size_t size, std::uint64_t offset, + data_buffer &data, + stop_type &stop_requested) -> api_error { + if (stop_requested) { + return api_error::download_stopped; + } + + if (offset == 0u) { + std::size_t bytes_read{}; + data.resize(size); + auto ret = nf.read(data, offset, &bytes_read) ? api_error::success + : api_error::os_error; + EXPECT_EQ(bytes_read, data.size()); + return ret; + } + + while (not stop_requested) { + std::this_thread::sleep_for(100ms); + } + return api_error::download_stopped; + }); + + EXPECT_CALL(um, remove_upload).WillOnce([&fsi](const std::string &api_path) { + EXPECT_EQ(fsi.api_path, api_path); + }); + + EXPECT_CALL(um, store_resume) + .Times(2) + .WillRepeatedly([&fsi](const i_open_file &cur_file) { + EXPECT_EQ(fsi.api_path, cur_file.get_api_path()); + EXPECT_EQ(fsi.source_path, cur_file.get_source_path()); + }); + + data_buffer data = {10, 9, 8}; + std::size_t bytes_written{}; + EXPECT_EQ(api_error::success, o.write(0u, data, bytes_written)); + validate_write(o, 0u, data, bytes_written); + + const auto test_state = [&]() { + EXPECT_STREQ(source_path.c_str(), o.get_source_path().c_str()); + + EXPECT_FALSE(o.can_close()); + + EXPECT_TRUE(o.is_modified()); + + EXPECT_TRUE(o.get_read_state(0u)); + EXPECT_FALSE(o.get_read_state(1u)); + }; + test_state(); + + o.close(); + nf.close(); + + test_state(); + + EXPECT_EQ(api_error::download_incomplete, o.get_api_error()); + + EXPECT_TRUE(utils::file::file(fsi.source_path).exists()); +} + +TEST(open_file, write_new_file) { + const auto source_path = + test::generate_test_file_name("file_manager_open_file_test"); + + mock_provider mp; + mock_upload_manager um; + + EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); + + filesystem_item fsi; + fsi.api_path = "/test.txt"; + fsi.size = 0u; + fsi.source_path = source_path; + + EXPECT_CALL(um, store_resume).WillOnce([&fsi](const i_open_file &o) { + EXPECT_EQ(fsi.api_path, o.get_api_path()); + EXPECT_EQ(fsi.source_path, o.get_source_path()); + }); + + file_manager::open_file o(test_chunk_size, 0U, fsi, mp, um); + test_closeable_open_file(o, false, api_error::success, 0u, source_path); + data_buffer data = {10, 9, 8}; + + EXPECT_CALL(mp, set_item_meta(fsi.api_path, _)) + .WillOnce([&data](const std::string &, + const api_meta_map &meta) -> api_error { + EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_CHANGED).empty())); + EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_MODIFIED).empty())); + EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_SIZE).empty())); + EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_WRITTEN).empty())); + EXPECT_EQ(data.size(), utils::string::to_size_t(meta.at(META_SIZE))); + return api_error::success; + }) + .WillOnce([](const std::string &, const api_meta_map &meta) -> api_error { + EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_CHANGED).empty())); + EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_MODIFIED).empty())); + EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_WRITTEN).empty())); + return api_error::success; + }); + + EXPECT_CALL(um, remove_upload).WillOnce([&fsi](const std::string &api_path) { + EXPECT_EQ(fsi.api_path, api_path); + }); + + EXPECT_CALL(um, queue_upload).WillOnce([&fsi](const i_open_file &cur_file) { + EXPECT_EQ(fsi.api_path, cur_file.get_api_path()); + EXPECT_EQ(fsi.source_path, cur_file.get_source_path()); + }); + + std::size_t bytes_written{}; + EXPECT_EQ(api_error::success, o.write(0u, data, bytes_written)); + + const auto test_state = [&]() { + EXPECT_STREQ(source_path.c_str(), o.get_source_path().c_str()); + + EXPECT_FALSE(o.can_close()); + EXPECT_TRUE(o.is_modified()); + + EXPECT_TRUE(o.get_read_state(0u)); + EXPECT_EQ(std::size_t(1u), o.get_read_state().size()); + EXPECT_EQ(data.size(), o.get_file_size()); + }; + test_state(); + + o.close(); + + test_state(); + + EXPECT_EQ(api_error::success, o.get_api_error()); + + EXPECT_TRUE(utils::file::file(fsi.source_path).exists()); +} + +TEST(open_file, write_new_file_multiple_chunks) { + const auto source_path = + test::generate_test_file_name("file_manager_open_file_test"); + + mock_provider mp; + mock_upload_manager um; + + EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); + + filesystem_item fsi; + fsi.api_path = "/test.txt"; + fsi.size = 0u; + fsi.source_path = source_path; + + EXPECT_CALL(um, store_resume).WillOnce([&fsi](const i_open_file &o) { + EXPECT_EQ(fsi.api_path, o.get_api_path()); + EXPECT_EQ(fsi.source_path, o.get_source_path()); + }); + file_manager::open_file o(test_chunk_size, 0U, fsi, mp, um); + test_closeable_open_file(o, false, api_error::success, 0u, source_path); + data_buffer data = {10, 9, 8}; + + EXPECT_CALL(mp, set_item_meta(fsi.api_path, _)) + .WillOnce([&data](const std::string &, + const api_meta_map &meta) -> api_error { + EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_CHANGED).empty())); + EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_MODIFIED).empty())); + EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_SIZE).empty())); + EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_WRITTEN).empty())); + EXPECT_EQ(data.size(), utils::string::to_size_t(meta.at(META_SIZE))); + return api_error::success; + }) + .WillOnce([](const std::string &, const api_meta_map &meta) -> api_error { + EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_CHANGED).empty())); + EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_MODIFIED).empty())); + EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_WRITTEN).empty())); + return api_error::success; + }) + .WillOnce( + [&data](const std::string &, const api_meta_map &meta) -> api_error { + EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_CHANGED).empty())); + EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_MODIFIED).empty())); + EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_SIZE).empty())); + EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_WRITTEN).empty())); + EXPECT_EQ(data.size() + test_chunk_size, + utils::string::to_size_t(meta.at(META_SIZE))); + return api_error::success; + }) + .WillOnce([](const std::string &, const api_meta_map &meta) -> api_error { + EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_CHANGED).empty())); + EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_MODIFIED).empty())); + EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_WRITTEN).empty())); + return api_error::success; + }); + + EXPECT_CALL(um, remove_upload).WillOnce([&fsi](const std::string &api_path) { + EXPECT_EQ(fsi.api_path, api_path); + }); + + EXPECT_CALL(um, queue_upload).WillOnce([&fsi](const i_open_file &cur_file) { + EXPECT_EQ(fsi.api_path, cur_file.get_api_path()); + EXPECT_EQ(fsi.source_path, cur_file.get_source_path()); + }); + + std::size_t bytes_written{}; + EXPECT_EQ(api_error::success, o.write(0u, data, bytes_written)); + EXPECT_EQ(api_error::success, o.write(test_chunk_size, data, bytes_written)); + + const auto test_state = [&]() { + EXPECT_STREQ(source_path.c_str(), o.get_source_path().c_str()); + + EXPECT_FALSE(o.can_close()); + EXPECT_TRUE(o.is_modified()); + + EXPECT_EQ(std::size_t(2u), o.get_read_state().size()); + for (std::size_t i = 0u; i < 2u; i++) { + EXPECT_TRUE(o.get_read_state(i)); + } + + EXPECT_EQ(data.size() + test_chunk_size, o.get_file_size()); + }; + test_state(); + + o.close(); + + test_state(); + + EXPECT_EQ(api_error::success, o.get_api_error()); + + EXPECT_TRUE(utils::file::file(fsi.source_path).exists()); +} + +TEST(open_file, resize_file_to_0_bytes) { + auto &rf = test::create_random_file(test_chunk_size * 4u); + const auto source_path = rf.get_path(); + rf.close(); + + mock_provider mp; + mock_upload_manager um; + + EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); + + filesystem_item fsi; + fsi.api_path = "/test.txt"; + fsi.size = test_chunk_size * 4u; + fsi.source_path = source_path; + + file_manager::open_file o(test_chunk_size, 0U, fsi, mp, um); + test_closeable_open_file(o, false, api_error::success, fsi.size, source_path); + EXPECT_CALL(mp, set_item_meta(fsi.api_path, _)) + .WillOnce([](const std::string &, const api_meta_map &meta) -> api_error { + EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_CHANGED).empty())); + EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_MODIFIED).empty())); + EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_SIZE).empty())); + EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_WRITTEN).empty())); + EXPECT_EQ(std::size_t(0u), + utils::string::to_size_t(meta.at(META_SIZE))); + return api_error::success; + }); + + EXPECT_CALL(um, remove_upload).WillOnce([&fsi](const std ::string &api_path) { + EXPECT_EQ(fsi.api_path, api_path); + }); + + EXPECT_CALL(um, queue_upload).WillOnce([&fsi](const i_open_file &cur_file) { + EXPECT_EQ(fsi.api_path, cur_file.get_api_path()); + EXPECT_EQ(fsi.source_path, cur_file.get_source_path()); + }); + EXPECT_CALL(um, store_resume).WillOnce([&fsi](const i_open_file &cur_file) { + EXPECT_EQ(fsi.api_path, cur_file.get_api_path()); + EXPECT_EQ(fsi.source_path, cur_file.get_source_path()); + }); + + EXPECT_EQ(api_error::success, o.resize(0u)); + + EXPECT_EQ(std::size_t(0u), o.get_file_size()); + EXPECT_FALSE(o.can_close()); + EXPECT_TRUE(o.is_modified()); + + EXPECT_EQ(std::size_t(0u), o.get_read_state().size()); +} + +TEST(open_file, resize_file_by_full_chunk) { + auto &rf = test::create_random_file(test_chunk_size * 4u); + const auto source_path = rf.get_path(); + rf.close(); + + mock_provider mp; + mock_upload_manager um; + + EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); + + filesystem_item fsi; + fsi.api_path = "/test.txt"; + fsi.size = test_chunk_size * 4u; + fsi.source_path = source_path; + + EXPECT_CALL(um, store_resume).WillOnce([&fsi](const i_open_file &o) { + EXPECT_EQ(fsi.api_path, o.get_api_path()); + EXPECT_EQ(fsi.source_path, o.get_source_path()); + }); + + file_manager::open_file o(test_chunk_size, 0U, fsi, mp, um); + test_closeable_open_file(o, false, api_error::success, fsi.size, source_path); + EXPECT_CALL(mp, set_item_meta(fsi.api_path, _)) + .WillOnce([](const std::string &, const api_meta_map &meta) -> api_error { + EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_CHANGED).empty())); + EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_MODIFIED).empty())); + EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_SIZE).empty())); + EXPECT_NO_THROW(EXPECT_FALSE(meta.at(META_WRITTEN).empty())); + EXPECT_EQ(std::size_t(test_chunk_size * 3u), + utils::string::to_size_t(meta.at(META_SIZE))); + return api_error::success; + }); + + EXPECT_CALL(um, remove_upload).WillOnce([&fsi](const std::string &api_path) { + EXPECT_EQ(fsi.api_path, api_path); + }); + + EXPECT_CALL(um, queue_upload).WillOnce([&fsi](const i_open_file &cur_file) { + EXPECT_EQ(fsi.api_path, cur_file.get_api_path()); + EXPECT_EQ(fsi.source_path, cur_file.get_source_path()); + }); + + EXPECT_EQ(api_error::success, o.resize(test_chunk_size * 3u)); + + EXPECT_EQ(std::size_t(test_chunk_size * 3u), o.get_file_size()); + EXPECT_FALSE(o.can_close()); + EXPECT_TRUE(o.is_modified()); + EXPECT_EQ(std::size_t(3u), o.get_read_state().size()); +} + +TEST(open_file, can_add_handle) { + event_system::instance().start(); + console_consumer c; + const auto source_path = + test::generate_test_file_name("file_manager_open_file_test"); + + mock_provider mp; + mock_upload_manager um; + + EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); + + filesystem_item fsi; + fsi.api_path = "/test.txt"; + fsi.size = test_chunk_size * 4u; + fsi.source_path = source_path; + + event_consumer ec("filesystem_item_opened", [&fsi](const event &e) { + const auto &ee = dynamic_cast(e); + EXPECT_STREQ(fsi.api_path.c_str(), + ee.get_api_path().get().c_str()); + EXPECT_STREQ(fsi.source_path.c_str(), + ee.get_source().get().c_str()); + EXPECT_STREQ("0", ee.get_directory().get().c_str()); + }); + + event_consumer ec2("filesystem_item_handle_opened", [&fsi](const event &e) { + const auto &ee = dynamic_cast(e); + EXPECT_STREQ(fsi.api_path.c_str(), + ee.get_api_path().get().c_str()); + EXPECT_STREQ(fsi.source_path.c_str(), + ee.get_source().get().c_str()); + EXPECT_STREQ("0", ee.get_directory().get().c_str()); + EXPECT_STREQ("1", ee.get_handle().get().c_str()); + }); + + EXPECT_CALL(mp, set_item_meta(fsi.api_path, META_SOURCE, _)) + .WillOnce(Return(api_error::success)); + EXPECT_CALL(um, remove_resume) + .WillOnce( + [&fsi](const std::string &api_path, const std::string &source_path2) { + EXPECT_EQ(fsi.api_path, api_path); + EXPECT_EQ(fsi.source_path, source_path2); + }); + + event_capture capture( + {"filesystem_item_opened", "filesystem_item_handle_opened"}); + + file_manager::open_file o(test_chunk_size, 0U, fsi, mp, um); +#if defined(_WIN32) + o.add(1u, {}); + EXPECT_EQ(nullptr, o.get_open_data(1u).directory_buffer); +#else + o.add(1u, O_RDWR | O_SYNC); + EXPECT_EQ(O_RDWR | O_SYNC, o.get_open_data(1u)); +#endif + + capture.wait_for_empty(); + + event_system::instance().stop(); +} + +TEST(open_file, can_remove_handle) { + event_system::instance().start(); + console_consumer c; + + const auto source_path = + test::generate_test_file_name("file_manager_open_file_test"); + + mock_provider mp; + mock_upload_manager um; + + EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); + + filesystem_item fsi; + fsi.api_path = "/test.txt"; + fsi.size = test_chunk_size * 4u; + fsi.source_path = source_path; + + event_consumer ec("filesystem_item_closed", [&fsi](const event &e) { + const auto &ee = dynamic_cast(e); + EXPECT_STREQ(fsi.api_path.c_str(), + ee.get_api_path().get().c_str()); + EXPECT_STREQ(fsi.source_path.c_str(), + ee.get_source().get().c_str()); + EXPECT_STREQ("0", ee.get_directory().get().c_str()); + }); + + event_consumer ec2("filesystem_item_handle_closed", [&fsi](const event &e) { + const auto &ee = dynamic_cast(e); + EXPECT_STREQ(fsi.api_path.c_str(), + ee.get_api_path().get().c_str()); + EXPECT_STREQ(fsi.source_path.c_str(), + ee.get_source().get().c_str()); + EXPECT_STREQ("0", ee.get_directory().get().c_str()); + EXPECT_STREQ("1", ee.get_handle().get().c_str()); + }); + + EXPECT_CALL(um, remove_resume) + .WillOnce( + [&fsi](const std::string &api_path, const std::string &source_path2) { + EXPECT_EQ(fsi.api_path, api_path); + EXPECT_EQ(fsi.source_path, source_path2); + }); + EXPECT_CALL(mp, set_item_meta(fsi.api_path, META_SOURCE, _)) + .WillOnce(Return(api_error::success)); + + event_capture capture({ + "filesystem_item_opened", + "filesystem_item_handle_opened", + "filesystem_item_handle_closed", + "filesystem_item_closed", + }); + + file_manager::open_file o(test_chunk_size, 0U, fsi, mp, um); +#if defined(_WIN32) + o.add(1u, {}); +#else + o.add(1u, O_RDWR | O_SYNC); +#endif + o.remove(1u); + + capture.wait_for_empty(); + + event_system::instance().stop(); +} + +TEST(open_file, + can_read_locally_after_write_with_file_size_greater_than_existing_size) {} + +TEST(open_file, test_valid_download_chunks) {} + +TEST(open_file, test_full_download_with_partial_chunk) {} + +TEST(open_file, source_is_read_after_full_download) {} +} // namespace repertory diff --git a/repertory/repertory_test/src/file_manager_ring_buffer_open_file_test.cpp b/repertory/repertory_test/src/ring_buffer_open_file_test.cpp similarity index 97% rename from repertory/repertory_test/src/file_manager_ring_buffer_open_file_test.cpp rename to repertory/repertory_test/src/ring_buffer_open_file_test.cpp index 2ad27551..21a38af1 100644 --- a/repertory/repertory_test/src/file_manager_ring_buffer_open_file_test.cpp +++ b/repertory/repertory_test/src/ring_buffer_open_file_test.cpp @@ -1,588 +1,588 @@ -/* - Copyright <2018-2024> - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. -*/ -#include "test_common.hpp" - -#include "file_manager/file_manager.hpp" -#include "mocks/mock_provider.hpp" -#include "mocks/mock_upload_manager.hpp" -#include "platform/platform.hpp" -#include "utils/file_utils.hpp" -#include "utils/path.hpp" - -namespace repertory { -static constexpr const std::size_t test_chunk_size = 1024u; -static std::string ring_buffer_dir = utils::path::combine( - test::get_test_output_dir(), {"file_manager_ring_buffer_open_file_test"}); - -TEST(ring_buffer_open_file, can_forward_to_last_chunk) { - const auto source_path = - test::generate_test_file_name("ring_buffer_open_file"); - - mock_provider mp; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - - filesystem_item fsi; - fsi.directory = false; - fsi.api_path = "/test.txt"; - fsi.size = test_chunk_size * 16u; - fsi.source_path = source_path; - - { - file_manager::ring_buffer_open_file rb(ring_buffer_dir, test_chunk_size, - 30U, fsi, mp, 8u); - rb.set(0u, 3u); - rb.forward(4u); - - EXPECT_EQ(std::size_t(7u), rb.get_current_chunk()); - EXPECT_EQ(std::size_t(0u), rb.get_first_chunk()); - EXPECT_EQ(std::size_t(7u), rb.get_last_chunk()); - for (std::size_t chunk = 0u; chunk < 8u; chunk++) { - EXPECT_TRUE(rb.get_read_state(chunk)); - } - } - - EXPECT_TRUE(utils::file::directory(ring_buffer_dir).remove_recursively()); -} - -TEST(ring_buffer_open_file, - can_forward_to_last_chunk_if_count_is_greater_than_remaining) { - const auto source_path = - test::generate_test_file_name("ring_buffer_open_file"); - - mock_provider mp; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - - filesystem_item fsi; - fsi.directory = false; - fsi.api_path = "/test.txt"; - fsi.size = test_chunk_size * 16u; - fsi.source_path = source_path; - - { - file_manager::ring_buffer_open_file rb(ring_buffer_dir, test_chunk_size, - 30U, fsi, mp, 8u); - rb.set(0u, 3u); - rb.forward(100u); - - EXPECT_EQ(std::size_t(15u), rb.get_current_chunk()); - EXPECT_EQ(std::size_t(8u), rb.get_first_chunk()); - EXPECT_EQ(std::size_t(15u), rb.get_last_chunk()); - for (std::size_t chunk = 8u; chunk <= 15u; chunk++) { - EXPECT_FALSE(rb.get_read_state(chunk)); - } - } - - EXPECT_TRUE(utils::file::directory(ring_buffer_dir).remove_recursively()); -} - -TEST(ring_buffer_open_file, can_forward_after_last_chunk) { - const auto source_path = - test::generate_test_file_name("ring_buffer_open_file"); - - mock_provider mp; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - - filesystem_item fsi; - fsi.directory = false; - fsi.api_path = "/test.txt"; - fsi.size = test_chunk_size * 16u; - fsi.source_path = source_path; - - { - file_manager::ring_buffer_open_file rb(ring_buffer_dir, test_chunk_size, - 30U, fsi, mp, 8u); - rb.set(0u, 3u); - rb.forward(5u); - - EXPECT_EQ(std::size_t(8u), rb.get_current_chunk()); - EXPECT_EQ(std::size_t(1u), rb.get_first_chunk()); - EXPECT_EQ(std::size_t(8u), rb.get_last_chunk()); - EXPECT_FALSE(rb.get_read_state(8u)); - for (std::size_t chunk = 1u; chunk < 8u; chunk++) { - EXPECT_TRUE(rb.get_read_state(chunk)); - } - } - - EXPECT_TRUE(utils::file::directory(ring_buffer_dir).remove_recursively()); -} - -TEST(ring_buffer_open_file, can_forward_and_rollover_after_last_chunk) { - const auto source_path = - test::generate_test_file_name("ring_buffer_open_file"); - - mock_provider mp; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - - filesystem_item fsi; - fsi.directory = false; - fsi.api_path = "/test.txt"; - fsi.size = test_chunk_size * 32u; - fsi.source_path = source_path; - - { - file_manager::ring_buffer_open_file rb(ring_buffer_dir, test_chunk_size, - 30U, fsi, mp, 8u); - rb.set(16u, 20u); - rb.forward(8u); - - EXPECT_EQ(std::size_t(28u), rb.get_current_chunk()); - EXPECT_EQ(std::size_t(21u), rb.get_first_chunk()); - EXPECT_EQ(std::size_t(28u), rb.get_last_chunk()); - } - - EXPECT_TRUE(utils::file::directory(ring_buffer_dir).remove_recursively()); -} - -TEST(ring_buffer_open_file, can_reverse_to_first_chunk) { - const auto source_path = - test::generate_test_file_name("ring_buffer_open_file"); - - mock_provider mp; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - - filesystem_item fsi; - fsi.directory = false; - fsi.api_path = "/test.txt"; - fsi.size = test_chunk_size * 16u; - fsi.source_path = source_path; - - { - file_manager::ring_buffer_open_file rb(ring_buffer_dir, test_chunk_size, - 30U, fsi, mp, 8u); - rb.set(0u, 3u); - rb.reverse(3u); - - EXPECT_EQ(std::size_t(0u), rb.get_current_chunk()); - EXPECT_EQ(std::size_t(0u), rb.get_first_chunk()); - EXPECT_EQ(std::size_t(7u), rb.get_last_chunk()); - for (std::size_t chunk = 0u; chunk < 8u; chunk++) { - EXPECT_TRUE(rb.get_read_state(chunk)); - } - } - - EXPECT_TRUE(utils::file::directory(ring_buffer_dir).remove_recursively()); -} - -TEST(ring_buffer_open_file, - can_reverse_to_first_chunk_if_count_is_greater_than_remaining) { - const auto source_path = - test::generate_test_file_name("ring_buffer_open_file"); - - mock_provider mp; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - - filesystem_item fsi; - fsi.directory = false; - fsi.api_path = "/test.txt"; - fsi.size = test_chunk_size * 16u; - fsi.source_path = source_path; - - { - file_manager::ring_buffer_open_file rb(ring_buffer_dir, test_chunk_size, - 30U, fsi, mp, 8u); - rb.set(0u, 3u); - rb.reverse(13u); - - EXPECT_EQ(std::size_t(0u), rb.get_current_chunk()); - EXPECT_EQ(std::size_t(0u), rb.get_first_chunk()); - EXPECT_EQ(std::size_t(7u), rb.get_last_chunk()); - for (std::size_t chunk = 0u; chunk < 8u; chunk++) { - EXPECT_TRUE(rb.get_read_state(chunk)); - } - } - - EXPECT_TRUE(utils::file::directory(ring_buffer_dir).remove_recursively()); -} - -TEST(ring_buffer_open_file, can_reverse_before_first_chunk) { - const auto source_path = - test::generate_test_file_name("ring_buffer_open_file"); - - mock_provider mp; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - - filesystem_item fsi; - fsi.directory = false; - fsi.api_path = "/test.txt"; - fsi.size = test_chunk_size * 16u; - fsi.source_path = source_path; - - { - file_manager::ring_buffer_open_file rb(ring_buffer_dir, test_chunk_size, - 30U, fsi, mp, 8u); - rb.set(1u, 3u); - rb.reverse(3u); - - EXPECT_EQ(std::size_t(0u), rb.get_current_chunk()); - EXPECT_EQ(std::size_t(0u), rb.get_first_chunk()); - EXPECT_EQ(std::size_t(7u), rb.get_last_chunk()); - EXPECT_FALSE(rb.get_read_state(0u)); - for (std::size_t chunk = 1u; chunk < 8u; chunk++) { - EXPECT_TRUE(rb.get_read_state(chunk)); - } - } - - EXPECT_TRUE(utils::file::directory(ring_buffer_dir).remove_recursively()); -} - -TEST(ring_buffer_open_file, can_reverse_and_rollover_before_first_chunk) { - const auto source_path = - test::generate_test_file_name("ring_buffer_open_file"); - - mock_provider mp; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - - filesystem_item fsi; - fsi.directory = false; - fsi.api_path = "/test.txt"; - fsi.size = test_chunk_size * 32u; - fsi.source_path = source_path; - - { - file_manager::ring_buffer_open_file rb(ring_buffer_dir, test_chunk_size, - 30U, fsi, mp, 8u); - rb.set(16u, 20u); - rb.reverse(8u); - - EXPECT_EQ(std::size_t(12u), rb.get_current_chunk()); - EXPECT_EQ(std::size_t(12u), rb.get_first_chunk()); - EXPECT_EQ(std::size_t(19u), rb.get_last_chunk()); - - EXPECT_FALSE(rb.get_read_state(12u)); - EXPECT_FALSE(rb.get_read_state(13u)); - EXPECT_FALSE(rb.get_read_state(14u)); - EXPECT_FALSE(rb.get_read_state(15u)); - for (std::size_t chunk = 16u; chunk <= rb.get_last_chunk(); chunk++) { - EXPECT_TRUE(rb.get_read_state(chunk)); - } - } - - EXPECT_TRUE(utils::file::directory(ring_buffer_dir).remove_recursively()); -} - -TEST(ring_buffer_open_file, can_reverse_full_ring) { - const auto source_path = - test::generate_test_file_name("ring_buffer_open_file"); - - mock_provider mp; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - - filesystem_item fsi; - fsi.directory = false; - fsi.api_path = "/test.txt"; - fsi.size = test_chunk_size * 32u; - fsi.source_path = source_path; - - { - file_manager::ring_buffer_open_file rb(ring_buffer_dir, test_chunk_size, - 30U, fsi, mp, 8u); - rb.set(8u, 15u); - rb.reverse(16u); - - EXPECT_EQ(std::size_t(0u), rb.get_current_chunk()); - EXPECT_EQ(std::size_t(0u), rb.get_first_chunk()); - EXPECT_EQ(std::size_t(7u), rb.get_last_chunk()); - - for (std::size_t chunk = 0u; chunk <= rb.get_last_chunk(); chunk++) { - EXPECT_FALSE(rb.get_read_state(chunk)); - } - } - - EXPECT_TRUE(utils::file::directory(ring_buffer_dir).remove_recursively()); -} - -TEST(ring_buffer_open_file, read_full_file) { - auto &nf = test::create_random_file(test_chunk_size * 32u); - const auto download_source_path = nf.get_path(); - - const auto dest_path = test::generate_test_file_name("ring_buffer_open_file"); - - mock_provider mp; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - - filesystem_item fsi; - fsi.directory = false; - fsi.api_path = "/test.txt"; - fsi.size = test_chunk_size * 32u; - fsi.source_path = test::generate_test_file_name("ring_buffer_open_file"); - - EXPECT_CALL(mp, read_file_bytes) - .WillRepeatedly([&nf](const std::string & /* api_path */, - std::size_t size, std::uint64_t offset, - data_buffer &data, - stop_type &stop_requested) -> api_error { - EXPECT_FALSE(stop_requested); - std::size_t bytes_read{}; - data.resize(size); - auto ret = nf.read(data, offset, &bytes_read) ? api_error::success - : api_error::os_error; - EXPECT_EQ(bytes_read, data.size()); - return ret; - }); - { - file_manager::ring_buffer_open_file rb(ring_buffer_dir, test_chunk_size, - 30U, fsi, mp, 8u); - - auto ptr = utils::file::file::open_or_create_file(dest_path); - auto &nf2 = *ptr; - EXPECT_TRUE(nf2); - - auto to_read = fsi.size; - std::size_t chunk = 0u; - while (to_read) { - data_buffer data{}; - EXPECT_EQ(api_error::success, - rb.read(test_chunk_size, chunk * test_chunk_size, data)); - - std::size_t bytes_written{}; - EXPECT_TRUE(nf2.write(data, chunk * test_chunk_size, &bytes_written)); - chunk++; - to_read -= data.size(); - } - nf2.close(); - nf.close(); - - auto hash1 = utils::file::file(download_source_path).sha256(); - auto hash2 = utils::file::file(dest_path).sha256(); - - EXPECT_TRUE(hash1.has_value()); - EXPECT_TRUE(hash2.has_value()); - if (hash1.has_value() && hash2.has_value()) { - EXPECT_STREQ(hash1.value().c_str(), hash2.value().c_str()); - } - } - - EXPECT_TRUE(utils::file::directory(ring_buffer_dir).remove_recursively()); -} - -TEST(ring_buffer_open_file, read_full_file_in_reverse) { - auto &nf = test::create_random_file(test_chunk_size * 32u); - const auto download_source_path = nf.get_path(); - - const auto dest_path = test::generate_test_file_name("ring_buffer_open_file"); - - mock_provider mp; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - - filesystem_item fsi; - fsi.directory = false; - fsi.api_path = "/test.txt"; - fsi.size = test_chunk_size * 32u; - fsi.source_path = test::generate_test_file_name("ring_buffer_open_file"); - - EXPECT_CALL(mp, read_file_bytes) - .WillRepeatedly([&nf](const std::string & /* api_path */, - std::size_t size, std::uint64_t offset, - data_buffer &data, - stop_type &stop_requested) -> api_error { - EXPECT_FALSE(stop_requested); - std::size_t bytes_read{}; - data.resize(size); - auto ret = nf.read(data, offset, &bytes_read) ? api_error::success - : api_error::os_error; - EXPECT_EQ(bytes_read, data.size()); - return ret; - }); - { - file_manager::ring_buffer_open_file rb(ring_buffer_dir, test_chunk_size, - 30U, fsi, mp, 8u); - - auto ptr = utils::file::file::open_or_create_file(dest_path); - auto &nf2 = *ptr; - EXPECT_TRUE(nf2); - - auto to_read = fsi.size; - std::size_t chunk = rb.get_total_chunks() - 1u; - while (to_read) { - data_buffer data{}; - EXPECT_EQ(api_error::success, - rb.read(test_chunk_size, chunk * test_chunk_size, data)); - - std::size_t bytes_written{}; - EXPECT_TRUE(nf2.write(data, chunk * test_chunk_size, &bytes_written)); - chunk--; - to_read -= data.size(); - } - nf2.close(); - nf.close(); - - auto hash1 = utils::file::file(download_source_path).sha256(); - auto hash2 = utils::file::file(dest_path).sha256(); - - EXPECT_TRUE(hash1.has_value()); - EXPECT_TRUE(hash2.has_value()); - if (hash1.has_value() && hash2.has_value()) { - EXPECT_STREQ(hash1.value().c_str(), hash2.value().c_str()); - } - } - - EXPECT_TRUE(utils::file::directory(ring_buffer_dir).remove_recursively()); -} - -TEST(ring_buffer_open_file, read_full_file_in_partial_chunks) { - auto &nf = test::create_random_file(test_chunk_size * 32u); - const auto download_source_path = nf.get_path(); - - const auto dest_path = test::generate_test_file_name("test"); - - mock_provider mp; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - - filesystem_item fsi; - fsi.directory = false; - fsi.api_path = "/test.txt"; - fsi.size = test_chunk_size * 32u; - fsi.source_path = test::generate_test_file_name("test"); - - EXPECT_CALL(mp, read_file_bytes) - .WillRepeatedly([&nf](const std::string & /* api_path */, - std::size_t size, std::uint64_t offset, - data_buffer &data, - stop_type &stop_requested) -> api_error { - EXPECT_FALSE(stop_requested); - std::size_t bytes_read{}; - data.resize(size); - auto ret = nf.read(data, offset, &bytes_read) ? api_error::success - : api_error::os_error; - EXPECT_EQ(bytes_read, data.size()); - return ret; - }); - { - file_manager::ring_buffer_open_file rb(ring_buffer_dir, test_chunk_size, - 30U, fsi, mp, 8u); - - auto ptr = utils::file::file::open_or_create_file(dest_path); - auto &nf2 = *ptr; - EXPECT_TRUE(nf2); - // EXPECT_EQ(api_error::success, native_file::create_or_open(dest_path, - // nf2)); - - auto total_read = std::uint64_t(0u); - - while (total_read < fsi.size) { - data_buffer data{}; - EXPECT_EQ(api_error::success, rb.read(3u, total_read, data)); - - std::size_t bytes_written{}; - EXPECT_TRUE( - nf2.write(data.data(), data.size(), total_read, &bytes_written)); - total_read += data.size(); - } - nf2.close(); - nf.close(); - - auto hash1 = utils::file::file(download_source_path).sha256(); - auto hash2 = utils::file::file(dest_path).sha256(); - - EXPECT_TRUE(hash1.has_value()); - EXPECT_TRUE(hash2.has_value()); - if (hash1.has_value() && hash2.has_value()) { - EXPECT_STREQ(hash1.value().c_str(), hash2.value().c_str()); - } - } - - EXPECT_TRUE(utils::file::directory(ring_buffer_dir).remove_recursively()); -} - -TEST(ring_buffer_open_file, read_full_file_in_partial_chunks_in_reverse) { - auto &nf = test::create_random_file(test_chunk_size * 32u); - const auto download_source_path = nf.get_path(); - - const auto dest_path = test::generate_test_file_name("ring_buffer_open_file"); - - mock_provider mp; - - EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); - - filesystem_item fsi; - fsi.directory = false; - fsi.api_path = "/test.txt"; - fsi.size = test_chunk_size * 32u; - fsi.source_path = test::generate_test_file_name("ring_buffer_open_file"); - - EXPECT_CALL(mp, read_file_bytes) - .WillRepeatedly([&nf](const std::string & /* api_path */, - std::size_t size, std::uint64_t offset, - data_buffer &data, - stop_type &stop_requested) -> api_error { - EXPECT_FALSE(stop_requested); - std::size_t bytes_read{}; - data.resize(size); - auto ret = nf.read(data, offset, &bytes_read) ? api_error::success - : api_error::os_error; - EXPECT_EQ(bytes_read, data.size()); - return ret; - }); - { - file_manager::ring_buffer_open_file rb(ring_buffer_dir, test_chunk_size, - 30U, fsi, mp, 8u); - - auto ptr = utils::file::file::open_or_create_file(dest_path); - auto &nf2 = *ptr; - EXPECT_TRUE(nf2); - - std::uint64_t total_read{0U}; - const auto read_size{3U}; - - while (total_read < fsi.size) { - const auto offset = fsi.size - total_read - read_size; - const auto remain = fsi.size - total_read; - - data_buffer data{}; - EXPECT_EQ(api_error::success, - rb.read(static_cast( - std::min(remain, std::uint64_t(read_size))), - (remain >= read_size) ? offset : 0u, data)); - - std::size_t bytes_written{}; - EXPECT_TRUE( - nf2.write(data, (remain >= read_size) ? offset : 0u, &bytes_written)); - total_read += data.size(); - } - nf2.close(); - nf.close(); - - auto hash1 = utils::file::file(download_source_path).sha256(); - auto hash2 = utils::file::file(dest_path).sha256(); - - EXPECT_TRUE(hash1.has_value()); - EXPECT_TRUE(hash2.has_value()); - if (hash1.has_value() && hash2.has_value()) { - EXPECT_STREQ(hash1.value().c_str(), hash2.value().c_str()); - } - } - - EXPECT_TRUE(utils::file::directory(ring_buffer_dir).remove_recursively()); -} -} // namespace repertory +/* + Copyright <2018-2024> + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ +#include "test_common.hpp" + +#include "file_manager/file_manager.hpp" +#include "mocks/mock_provider.hpp" +#include "mocks/mock_upload_manager.hpp" +#include "platform/platform.hpp" +#include "utils/file_utils.hpp" +#include "utils/path.hpp" + +namespace repertory { +static constexpr const std::size_t test_chunk_size = 1024u; +static std::string ring_buffer_dir = utils::path::combine( + test::get_test_output_dir(), {"file_manager_ring_buffer_open_file_test"}); + +TEST(ring_buffer_open_file, can_forward_to_last_chunk) { + const auto source_path = + test::generate_test_file_name("ring_buffer_open_file"); + + mock_provider mp; + + EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); + + filesystem_item fsi; + fsi.directory = false; + fsi.api_path = "/test.txt"; + fsi.size = test_chunk_size * 16u; + fsi.source_path = source_path; + + { + file_manager::ring_buffer_open_file rb(ring_buffer_dir, test_chunk_size, + 30U, fsi, mp, 8u); + rb.set(0u, 3u); + rb.forward(4u); + + EXPECT_EQ(std::size_t(7u), rb.get_current_chunk()); + EXPECT_EQ(std::size_t(0u), rb.get_first_chunk()); + EXPECT_EQ(std::size_t(7u), rb.get_last_chunk()); + for (std::size_t chunk = 0u; chunk < 8u; chunk++) { + EXPECT_TRUE(rb.get_read_state(chunk)); + } + } + + EXPECT_TRUE(utils::file::directory(ring_buffer_dir).remove_recursively()); +} + +TEST(ring_buffer_open_file, + can_forward_to_last_chunk_if_count_is_greater_than_remaining) { + const auto source_path = + test::generate_test_file_name("ring_buffer_open_file"); + + mock_provider mp; + + EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); + + filesystem_item fsi; + fsi.directory = false; + fsi.api_path = "/test.txt"; + fsi.size = test_chunk_size * 16u; + fsi.source_path = source_path; + + { + file_manager::ring_buffer_open_file rb(ring_buffer_dir, test_chunk_size, + 30U, fsi, mp, 8u); + rb.set(0u, 3u); + rb.forward(100u); + + EXPECT_EQ(std::size_t(15u), rb.get_current_chunk()); + EXPECT_EQ(std::size_t(8u), rb.get_first_chunk()); + EXPECT_EQ(std::size_t(15u), rb.get_last_chunk()); + for (std::size_t chunk = 8u; chunk <= 15u; chunk++) { + EXPECT_FALSE(rb.get_read_state(chunk)); + } + } + + EXPECT_TRUE(utils::file::directory(ring_buffer_dir).remove_recursively()); +} + +TEST(ring_buffer_open_file, can_forward_after_last_chunk) { + const auto source_path = + test::generate_test_file_name("ring_buffer_open_file"); + + mock_provider mp; + + EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); + + filesystem_item fsi; + fsi.directory = false; + fsi.api_path = "/test.txt"; + fsi.size = test_chunk_size * 16u; + fsi.source_path = source_path; + + { + file_manager::ring_buffer_open_file rb(ring_buffer_dir, test_chunk_size, + 30U, fsi, mp, 8u); + rb.set(0u, 3u); + rb.forward(5u); + + EXPECT_EQ(std::size_t(8u), rb.get_current_chunk()); + EXPECT_EQ(std::size_t(1u), rb.get_first_chunk()); + EXPECT_EQ(std::size_t(8u), rb.get_last_chunk()); + EXPECT_FALSE(rb.get_read_state(8u)); + for (std::size_t chunk = 1u; chunk < 8u; chunk++) { + EXPECT_TRUE(rb.get_read_state(chunk)); + } + } + + EXPECT_TRUE(utils::file::directory(ring_buffer_dir).remove_recursively()); +} + +TEST(ring_buffer_open_file, can_forward_and_rollover_after_last_chunk) { + const auto source_path = + test::generate_test_file_name("ring_buffer_open_file"); + + mock_provider mp; + + EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); + + filesystem_item fsi; + fsi.directory = false; + fsi.api_path = "/test.txt"; + fsi.size = test_chunk_size * 32u; + fsi.source_path = source_path; + + { + file_manager::ring_buffer_open_file rb(ring_buffer_dir, test_chunk_size, + 30U, fsi, mp, 8u); + rb.set(16u, 20u); + rb.forward(8u); + + EXPECT_EQ(std::size_t(28u), rb.get_current_chunk()); + EXPECT_EQ(std::size_t(21u), rb.get_first_chunk()); + EXPECT_EQ(std::size_t(28u), rb.get_last_chunk()); + } + + EXPECT_TRUE(utils::file::directory(ring_buffer_dir).remove_recursively()); +} + +TEST(ring_buffer_open_file, can_reverse_to_first_chunk) { + const auto source_path = + test::generate_test_file_name("ring_buffer_open_file"); + + mock_provider mp; + + EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); + + filesystem_item fsi; + fsi.directory = false; + fsi.api_path = "/test.txt"; + fsi.size = test_chunk_size * 16u; + fsi.source_path = source_path; + + { + file_manager::ring_buffer_open_file rb(ring_buffer_dir, test_chunk_size, + 30U, fsi, mp, 8u); + rb.set(0u, 3u); + rb.reverse(3u); + + EXPECT_EQ(std::size_t(0u), rb.get_current_chunk()); + EXPECT_EQ(std::size_t(0u), rb.get_first_chunk()); + EXPECT_EQ(std::size_t(7u), rb.get_last_chunk()); + for (std::size_t chunk = 0u; chunk < 8u; chunk++) { + EXPECT_TRUE(rb.get_read_state(chunk)); + } + } + + EXPECT_TRUE(utils::file::directory(ring_buffer_dir).remove_recursively()); +} + +TEST(ring_buffer_open_file, + can_reverse_to_first_chunk_if_count_is_greater_than_remaining) { + const auto source_path = + test::generate_test_file_name("ring_buffer_open_file"); + + mock_provider mp; + + EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); + + filesystem_item fsi; + fsi.directory = false; + fsi.api_path = "/test.txt"; + fsi.size = test_chunk_size * 16u; + fsi.source_path = source_path; + + { + file_manager::ring_buffer_open_file rb(ring_buffer_dir, test_chunk_size, + 30U, fsi, mp, 8u); + rb.set(0u, 3u); + rb.reverse(13u); + + EXPECT_EQ(std::size_t(0u), rb.get_current_chunk()); + EXPECT_EQ(std::size_t(0u), rb.get_first_chunk()); + EXPECT_EQ(std::size_t(7u), rb.get_last_chunk()); + for (std::size_t chunk = 0u; chunk < 8u; chunk++) { + EXPECT_TRUE(rb.get_read_state(chunk)); + } + } + + EXPECT_TRUE(utils::file::directory(ring_buffer_dir).remove_recursively()); +} + +TEST(ring_buffer_open_file, can_reverse_before_first_chunk) { + const auto source_path = + test::generate_test_file_name("ring_buffer_open_file"); + + mock_provider mp; + + EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); + + filesystem_item fsi; + fsi.directory = false; + fsi.api_path = "/test.txt"; + fsi.size = test_chunk_size * 16u; + fsi.source_path = source_path; + + { + file_manager::ring_buffer_open_file rb(ring_buffer_dir, test_chunk_size, + 30U, fsi, mp, 8u); + rb.set(1u, 3u); + rb.reverse(3u); + + EXPECT_EQ(std::size_t(0u), rb.get_current_chunk()); + EXPECT_EQ(std::size_t(0u), rb.get_first_chunk()); + EXPECT_EQ(std::size_t(7u), rb.get_last_chunk()); + EXPECT_FALSE(rb.get_read_state(0u)); + for (std::size_t chunk = 1u; chunk < 8u; chunk++) { + EXPECT_TRUE(rb.get_read_state(chunk)); + } + } + + EXPECT_TRUE(utils::file::directory(ring_buffer_dir).remove_recursively()); +} + +TEST(ring_buffer_open_file, can_reverse_and_rollover_before_first_chunk) { + const auto source_path = + test::generate_test_file_name("ring_buffer_open_file"); + + mock_provider mp; + + EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); + + filesystem_item fsi; + fsi.directory = false; + fsi.api_path = "/test.txt"; + fsi.size = test_chunk_size * 32u; + fsi.source_path = source_path; + + { + file_manager::ring_buffer_open_file rb(ring_buffer_dir, test_chunk_size, + 30U, fsi, mp, 8u); + rb.set(16u, 20u); + rb.reverse(8u); + + EXPECT_EQ(std::size_t(12u), rb.get_current_chunk()); + EXPECT_EQ(std::size_t(12u), rb.get_first_chunk()); + EXPECT_EQ(std::size_t(19u), rb.get_last_chunk()); + + EXPECT_FALSE(rb.get_read_state(12u)); + EXPECT_FALSE(rb.get_read_state(13u)); + EXPECT_FALSE(rb.get_read_state(14u)); + EXPECT_FALSE(rb.get_read_state(15u)); + for (std::size_t chunk = 16u; chunk <= rb.get_last_chunk(); chunk++) { + EXPECT_TRUE(rb.get_read_state(chunk)); + } + } + + EXPECT_TRUE(utils::file::directory(ring_buffer_dir).remove_recursively()); +} + +TEST(ring_buffer_open_file, can_reverse_full_ring) { + const auto source_path = + test::generate_test_file_name("ring_buffer_open_file"); + + mock_provider mp; + + EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); + + filesystem_item fsi; + fsi.directory = false; + fsi.api_path = "/test.txt"; + fsi.size = test_chunk_size * 32u; + fsi.source_path = source_path; + + { + file_manager::ring_buffer_open_file rb(ring_buffer_dir, test_chunk_size, + 30U, fsi, mp, 8u); + rb.set(8u, 15u); + rb.reverse(16u); + + EXPECT_EQ(std::size_t(0u), rb.get_current_chunk()); + EXPECT_EQ(std::size_t(0u), rb.get_first_chunk()); + EXPECT_EQ(std::size_t(7u), rb.get_last_chunk()); + + for (std::size_t chunk = 0u; chunk <= rb.get_last_chunk(); chunk++) { + EXPECT_FALSE(rb.get_read_state(chunk)); + } + } + + EXPECT_TRUE(utils::file::directory(ring_buffer_dir).remove_recursively()); +} + +TEST(ring_buffer_open_file, read_full_file) { + auto &nf = test::create_random_file(test_chunk_size * 32u); + const auto download_source_path = nf.get_path(); + + const auto dest_path = test::generate_test_file_name("ring_buffer_open_file"); + + mock_provider mp; + + EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); + + filesystem_item fsi; + fsi.directory = false; + fsi.api_path = "/test.txt"; + fsi.size = test_chunk_size * 32u; + fsi.source_path = test::generate_test_file_name("ring_buffer_open_file"); + + EXPECT_CALL(mp, read_file_bytes) + .WillRepeatedly([&nf](const std::string & /* api_path */, + std::size_t size, std::uint64_t offset, + data_buffer &data, + stop_type &stop_requested) -> api_error { + EXPECT_FALSE(stop_requested); + std::size_t bytes_read{}; + data.resize(size); + auto ret = nf.read(data, offset, &bytes_read) ? api_error::success + : api_error::os_error; + EXPECT_EQ(bytes_read, data.size()); + return ret; + }); + { + file_manager::ring_buffer_open_file rb(ring_buffer_dir, test_chunk_size, + 30U, fsi, mp, 8u); + + auto ptr = utils::file::file::open_or_create_file(dest_path); + auto &nf2 = *ptr; + EXPECT_TRUE(nf2); + + auto to_read = fsi.size; + std::size_t chunk = 0u; + while (to_read) { + data_buffer data{}; + EXPECT_EQ(api_error::success, + rb.read(test_chunk_size, chunk * test_chunk_size, data)); + + std::size_t bytes_written{}; + EXPECT_TRUE(nf2.write(data, chunk * test_chunk_size, &bytes_written)); + chunk++; + to_read -= data.size(); + } + nf2.close(); + nf.close(); + + auto hash1 = utils::file::file(download_source_path).sha256(); + auto hash2 = utils::file::file(dest_path).sha256(); + + EXPECT_TRUE(hash1.has_value()); + EXPECT_TRUE(hash2.has_value()); + if (hash1.has_value() && hash2.has_value()) { + EXPECT_STREQ(hash1.value().c_str(), hash2.value().c_str()); + } + } + + EXPECT_TRUE(utils::file::directory(ring_buffer_dir).remove_recursively()); +} + +TEST(ring_buffer_open_file, read_full_file_in_reverse) { + auto &nf = test::create_random_file(test_chunk_size * 32u); + const auto download_source_path = nf.get_path(); + + const auto dest_path = test::generate_test_file_name("ring_buffer_open_file"); + + mock_provider mp; + + EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); + + filesystem_item fsi; + fsi.directory = false; + fsi.api_path = "/test.txt"; + fsi.size = test_chunk_size * 32u; + fsi.source_path = test::generate_test_file_name("ring_buffer_open_file"); + + EXPECT_CALL(mp, read_file_bytes) + .WillRepeatedly([&nf](const std::string & /* api_path */, + std::size_t size, std::uint64_t offset, + data_buffer &data, + stop_type &stop_requested) -> api_error { + EXPECT_FALSE(stop_requested); + std::size_t bytes_read{}; + data.resize(size); + auto ret = nf.read(data, offset, &bytes_read) ? api_error::success + : api_error::os_error; + EXPECT_EQ(bytes_read, data.size()); + return ret; + }); + { + file_manager::ring_buffer_open_file rb(ring_buffer_dir, test_chunk_size, + 30U, fsi, mp, 8u); + + auto ptr = utils::file::file::open_or_create_file(dest_path); + auto &nf2 = *ptr; + EXPECT_TRUE(nf2); + + auto to_read = fsi.size; + std::size_t chunk = rb.get_total_chunks() - 1u; + while (to_read) { + data_buffer data{}; + EXPECT_EQ(api_error::success, + rb.read(test_chunk_size, chunk * test_chunk_size, data)); + + std::size_t bytes_written{}; + EXPECT_TRUE(nf2.write(data, chunk * test_chunk_size, &bytes_written)); + chunk--; + to_read -= data.size(); + } + nf2.close(); + nf.close(); + + auto hash1 = utils::file::file(download_source_path).sha256(); + auto hash2 = utils::file::file(dest_path).sha256(); + + EXPECT_TRUE(hash1.has_value()); + EXPECT_TRUE(hash2.has_value()); + if (hash1.has_value() && hash2.has_value()) { + EXPECT_STREQ(hash1.value().c_str(), hash2.value().c_str()); + } + } + + EXPECT_TRUE(utils::file::directory(ring_buffer_dir).remove_recursively()); +} + +TEST(ring_buffer_open_file, read_full_file_in_partial_chunks) { + auto &nf = test::create_random_file(test_chunk_size * 32u); + const auto download_source_path = nf.get_path(); + + const auto dest_path = test::generate_test_file_name("test"); + + mock_provider mp; + + EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); + + filesystem_item fsi; + fsi.directory = false; + fsi.api_path = "/test.txt"; + fsi.size = test_chunk_size * 32u; + fsi.source_path = test::generate_test_file_name("test"); + + EXPECT_CALL(mp, read_file_bytes) + .WillRepeatedly([&nf](const std::string & /* api_path */, + std::size_t size, std::uint64_t offset, + data_buffer &data, + stop_type &stop_requested) -> api_error { + EXPECT_FALSE(stop_requested); + std::size_t bytes_read{}; + data.resize(size); + auto ret = nf.read(data, offset, &bytes_read) ? api_error::success + : api_error::os_error; + EXPECT_EQ(bytes_read, data.size()); + return ret; + }); + { + file_manager::ring_buffer_open_file rb(ring_buffer_dir, test_chunk_size, + 30U, fsi, mp, 8u); + + auto ptr = utils::file::file::open_or_create_file(dest_path); + auto &nf2 = *ptr; + EXPECT_TRUE(nf2); + // EXPECT_EQ(api_error::success, native_file::create_or_open(dest_path, + // nf2)); + + auto total_read = std::uint64_t(0u); + + while (total_read < fsi.size) { + data_buffer data{}; + EXPECT_EQ(api_error::success, rb.read(3u, total_read, data)); + + std::size_t bytes_written{}; + EXPECT_TRUE( + nf2.write(data.data(), data.size(), total_read, &bytes_written)); + total_read += data.size(); + } + nf2.close(); + nf.close(); + + auto hash1 = utils::file::file(download_source_path).sha256(); + auto hash2 = utils::file::file(dest_path).sha256(); + + EXPECT_TRUE(hash1.has_value()); + EXPECT_TRUE(hash2.has_value()); + if (hash1.has_value() && hash2.has_value()) { + EXPECT_STREQ(hash1.value().c_str(), hash2.value().c_str()); + } + } + + EXPECT_TRUE(utils::file::directory(ring_buffer_dir).remove_recursively()); +} + +TEST(ring_buffer_open_file, read_full_file_in_partial_chunks_in_reverse) { + auto &nf = test::create_random_file(test_chunk_size * 32u); + const auto download_source_path = nf.get_path(); + + const auto dest_path = test::generate_test_file_name("ring_buffer_open_file"); + + mock_provider mp; + + EXPECT_CALL(mp, is_direct_only()).WillRepeatedly(Return(false)); + + filesystem_item fsi; + fsi.directory = false; + fsi.api_path = "/test.txt"; + fsi.size = test_chunk_size * 32u; + fsi.source_path = test::generate_test_file_name("ring_buffer_open_file"); + + EXPECT_CALL(mp, read_file_bytes) + .WillRepeatedly([&nf](const std::string & /* api_path */, + std::size_t size, std::uint64_t offset, + data_buffer &data, + stop_type &stop_requested) -> api_error { + EXPECT_FALSE(stop_requested); + std::size_t bytes_read{}; + data.resize(size); + auto ret = nf.read(data, offset, &bytes_read) ? api_error::success + : api_error::os_error; + EXPECT_EQ(bytes_read, data.size()); + return ret; + }); + { + file_manager::ring_buffer_open_file rb(ring_buffer_dir, test_chunk_size, + 30U, fsi, mp, 8u); + + auto ptr = utils::file::file::open_or_create_file(dest_path); + auto &nf2 = *ptr; + EXPECT_TRUE(nf2); + + std::uint64_t total_read{0U}; + const auto read_size{3U}; + + while (total_read < fsi.size) { + const auto offset = fsi.size - total_read - read_size; + const auto remain = fsi.size - total_read; + + data_buffer data{}; + EXPECT_EQ(api_error::success, + rb.read(static_cast( + std::min(remain, std::uint64_t(read_size))), + (remain >= read_size) ? offset : 0u, data)); + + std::size_t bytes_written{}; + EXPECT_TRUE( + nf2.write(data, (remain >= read_size) ? offset : 0u, &bytes_written)); + total_read += data.size(); + } + nf2.close(); + nf.close(); + + auto hash1 = utils::file::file(download_source_path).sha256(); + auto hash2 = utils::file::file(dest_path).sha256(); + + EXPECT_TRUE(hash1.has_value()); + EXPECT_TRUE(hash2.has_value()); + if (hash1.has_value() && hash2.has_value()) { + EXPECT_STREQ(hash1.value().c_str(), hash2.value().c_str()); + } + } + + EXPECT_TRUE(utils::file::directory(ring_buffer_dir).remove_recursively()); +} +} // namespace repertory diff --git a/repertory/repertory_test/src/file_manager_upload_test.cpp b/repertory/repertory_test/src/upload_test.cpp similarity index 97% rename from repertory/repertory_test/src/file_manager_upload_test.cpp rename to repertory/repertory_test/src/upload_test.cpp index f52d81f6..3f391e81 100644 --- a/repertory/repertory_test/src/file_manager_upload_test.cpp +++ b/repertory/repertory_test/src/upload_test.cpp @@ -1,182 +1,182 @@ -/* - Copyright <2018-2024> - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. -*/ -#include "test_common.hpp" - -#include "file_manager/file_manager.hpp" -#include "mocks/mock_provider.hpp" -#include "utils/event_capture.hpp" - -namespace repertory { -static constexpr const std::size_t test_chunk_size{1024U}; - -TEST(upload, can_upload_a_valid_file) { - console_consumer con; - - event_system::instance().start(); - - const auto source_path = test::generate_test_file_name("upload_test"); - - mock_provider mock_prov; - - EXPECT_CALL(mock_prov, is_direct_only()).WillRepeatedly(Return(false)); - - filesystem_item fsi; - fsi.api_path = "/test.txt"; - fsi.size = test_chunk_size * 4U; - fsi.source_path = source_path; - - event_consumer evt_com("file_upload_completed", [&fsi](const event &evt) { - const auto &comp_evt = dynamic_cast(evt); - EXPECT_STREQ(fsi.api_path.c_str(), - comp_evt.get_api_path().get().c_str()); - EXPECT_STREQ(fsi.source_path.c_str(), - comp_evt.get_source().get().c_str()); - EXPECT_STREQ("success", comp_evt.get_result().get().c_str()); - EXPECT_STREQ("0", comp_evt.get_cancelled().get().c_str()); - }); - - EXPECT_CALL(mock_prov, upload_file(fsi.api_path, fsi.source_path, _)) - .WillOnce([](const std::string &, const std::string &, - stop_type &stop_requested) -> api_error { - EXPECT_FALSE(stop_requested); - return api_error::success; - }); - file_manager::upload upload(fsi, mock_prov); - - event_capture evt_cap({"file_upload_completed"}); - evt_cap.wait_for_empty(); - - EXPECT_EQ(api_error::success, upload.get_api_error()); - EXPECT_FALSE(upload.is_cancelled()); - - event_system::instance().stop(); -} - -TEST(upload, can_cancel_upload) { - console_consumer con; - - event_system::instance().start(); - - const auto source_path = test::generate_test_file_name("upload_test"); - - mock_provider mock_provider; - - EXPECT_CALL(mock_provider, is_direct_only()).WillRepeatedly(Return(false)); - - filesystem_item fsi; - fsi.api_path = "/test.txt"; - fsi.size = test_chunk_size * 4U; - fsi.source_path = source_path; - - event_consumer evt_con("file_upload_completed", [&fsi](const event &evt) { - const auto &comp_evt = dynamic_cast(evt); - EXPECT_STREQ(fsi.api_path.c_str(), - comp_evt.get_api_path().get().c_str()); - EXPECT_STREQ(fsi.source_path.c_str(), - comp_evt.get_source().get().c_str()); - EXPECT_STREQ("comm_error", - comp_evt.get_result().get().c_str()); - EXPECT_STREQ("1", comp_evt.get_cancelled().get().c_str()); - }); - - std::mutex mtx; - std::condition_variable notify; - - EXPECT_CALL(mock_provider, upload_file(fsi.api_path, fsi.source_path, _)) - .WillOnce([¬ify, &mtx](const std::string &, const std::string &, - stop_type &stop_requested) -> api_error { - EXPECT_FALSE(stop_requested); - - unique_mutex_lock lock(mtx); - notify.notify_one(); - lock.unlock(); - - lock.lock(); - notify.wait(lock); - lock.unlock(); - - EXPECT_TRUE(stop_requested); - - return api_error::comm_error; - }); - - unique_mutex_lock lock(mtx); - file_manager::upload upload(fsi, mock_provider); - notify.wait(lock); - - upload.cancel(); - - notify.notify_one(); - lock.unlock(); - - event_capture evt_cap({"file_upload_completed"}); - evt_cap.wait_for_empty(); - - EXPECT_EQ(api_error::comm_error, upload.get_api_error()); - EXPECT_TRUE(upload.is_cancelled()); - - event_system::instance().stop(); -} - -TEST(upload, can_stop_upload) { - console_consumer con; - - event_system::instance().start(); - - const auto source_path = test::generate_test_file_name("upload_test"); - - mock_provider mock_provider; - - EXPECT_CALL(mock_provider, is_direct_only()).WillRepeatedly(Return(false)); - - filesystem_item fsi; - fsi.api_path = "/test.txt"; - fsi.size = test_chunk_size * 4U; - fsi.source_path = source_path; - - event_consumer evt_con("file_upload_completed", [&fsi](const event &evt) { - const auto &evt_com = dynamic_cast(evt); - EXPECT_STREQ(fsi.api_path.c_str(), - evt_com.get_api_path().get().c_str()); - EXPECT_STREQ(fsi.source_path.c_str(), - evt_com.get_source().get().c_str()); - EXPECT_STREQ("comm_error", evt_com.get_result().get().c_str()); - EXPECT_STREQ("0", evt_com.get_cancelled().get().c_str()); - }); - - EXPECT_CALL(mock_provider, upload_file(fsi.api_path, fsi.source_path, _)) - .WillOnce([](const std::string &, const std::string &, - stop_type &stop_requested) -> api_error { - std::this_thread::sleep_for(3s); - EXPECT_TRUE(stop_requested); - return api_error::comm_error; - }); - - event_capture evt_cap({"file_upload_completed"}); - - { file_manager::upload upload(fsi, mock_provider); } - - evt_cap.wait_for_empty(); - - event_system::instance().stop(); -} -} // namespace repertory +/* + Copyright <2018-2024> + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ +#include "test_common.hpp" + +#include "file_manager/file_manager.hpp" +#include "mocks/mock_provider.hpp" +#include "utils/event_capture.hpp" + +namespace repertory { +static constexpr const std::size_t test_chunk_size{1024U}; + +TEST(upload, can_upload_a_valid_file) { + console_consumer con; + + event_system::instance().start(); + + const auto source_path = test::generate_test_file_name("upload_test"); + + mock_provider mock_prov; + + EXPECT_CALL(mock_prov, is_direct_only()).WillRepeatedly(Return(false)); + + filesystem_item fsi; + fsi.api_path = "/test.txt"; + fsi.size = test_chunk_size * 4U; + fsi.source_path = source_path; + + event_consumer evt_com("file_upload_completed", [&fsi](const event &evt) { + const auto &comp_evt = dynamic_cast(evt); + EXPECT_STREQ(fsi.api_path.c_str(), + comp_evt.get_api_path().get().c_str()); + EXPECT_STREQ(fsi.source_path.c_str(), + comp_evt.get_source().get().c_str()); + EXPECT_STREQ("success", comp_evt.get_result().get().c_str()); + EXPECT_STREQ("0", comp_evt.get_cancelled().get().c_str()); + }); + + EXPECT_CALL(mock_prov, upload_file(fsi.api_path, fsi.source_path, _)) + .WillOnce([](const std::string &, const std::string &, + stop_type &stop_requested) -> api_error { + EXPECT_FALSE(stop_requested); + return api_error::success; + }); + file_manager::upload upload(fsi, mock_prov); + + event_capture evt_cap({"file_upload_completed"}); + evt_cap.wait_for_empty(); + + EXPECT_EQ(api_error::success, upload.get_api_error()); + EXPECT_FALSE(upload.is_cancelled()); + + event_system::instance().stop(); +} + +TEST(upload, can_cancel_upload) { + console_consumer con; + + event_system::instance().start(); + + const auto source_path = test::generate_test_file_name("upload_test"); + + mock_provider mock_provider; + + EXPECT_CALL(mock_provider, is_direct_only()).WillRepeatedly(Return(false)); + + filesystem_item fsi; + fsi.api_path = "/test.txt"; + fsi.size = test_chunk_size * 4U; + fsi.source_path = source_path; + + event_consumer evt_con("file_upload_completed", [&fsi](const event &evt) { + const auto &comp_evt = dynamic_cast(evt); + EXPECT_STREQ(fsi.api_path.c_str(), + comp_evt.get_api_path().get().c_str()); + EXPECT_STREQ(fsi.source_path.c_str(), + comp_evt.get_source().get().c_str()); + EXPECT_STREQ("comm_error", + comp_evt.get_result().get().c_str()); + EXPECT_STREQ("1", comp_evt.get_cancelled().get().c_str()); + }); + + std::mutex mtx; + std::condition_variable notify; + + EXPECT_CALL(mock_provider, upload_file(fsi.api_path, fsi.source_path, _)) + .WillOnce([¬ify, &mtx](const std::string &, const std::string &, + stop_type &stop_requested) -> api_error { + EXPECT_FALSE(stop_requested); + + unique_mutex_lock lock(mtx); + notify.notify_one(); + lock.unlock(); + + lock.lock(); + notify.wait(lock); + lock.unlock(); + + EXPECT_TRUE(stop_requested); + + return api_error::comm_error; + }); + + unique_mutex_lock lock(mtx); + file_manager::upload upload(fsi, mock_provider); + notify.wait(lock); + + upload.cancel(); + + notify.notify_one(); + lock.unlock(); + + event_capture evt_cap({"file_upload_completed"}); + evt_cap.wait_for_empty(); + + EXPECT_EQ(api_error::comm_error, upload.get_api_error()); + EXPECT_TRUE(upload.is_cancelled()); + + event_system::instance().stop(); +} + +TEST(upload, can_stop_upload) { + console_consumer con; + + event_system::instance().start(); + + const auto source_path = test::generate_test_file_name("upload_test"); + + mock_provider mock_provider; + + EXPECT_CALL(mock_provider, is_direct_only()).WillRepeatedly(Return(false)); + + filesystem_item fsi; + fsi.api_path = "/test.txt"; + fsi.size = test_chunk_size * 4U; + fsi.source_path = source_path; + + event_consumer evt_con("file_upload_completed", [&fsi](const event &evt) { + const auto &evt_com = dynamic_cast(evt); + EXPECT_STREQ(fsi.api_path.c_str(), + evt_com.get_api_path().get().c_str()); + EXPECT_STREQ(fsi.source_path.c_str(), + evt_com.get_source().get().c_str()); + EXPECT_STREQ("comm_error", evt_com.get_result().get().c_str()); + EXPECT_STREQ("0", evt_com.get_cancelled().get().c_str()); + }); + + EXPECT_CALL(mock_provider, upload_file(fsi.api_path, fsi.source_path, _)) + .WillOnce([](const std::string &, const std::string &, + stop_type &stop_requested) -> api_error { + std::this_thread::sleep_for(3s); + EXPECT_TRUE(stop_requested); + return api_error::comm_error; + }); + + event_capture evt_cap({"file_upload_completed"}); + + { file_manager::upload upload(fsi, mock_provider); } + + evt_cap.wait_for_empty(); + + event_system::instance().stop(); +} +} // namespace repertory diff --git a/support/include/utils/db/sqlite/db_where_t.hpp b/support/include/utils/db/sqlite/db_where_t.hpp index 5178eee2..40ea6119 100644 --- a/support/include/utils/db/sqlite/db_where_t.hpp +++ b/support/include/utils/db/sqlite/db_where_t.hpp @@ -157,8 +157,8 @@ template struct db_where_t final { using action_t = std::variant; - [[nodiscard]] static auto dump(std::int32_t &idx, - auto &&actions) -> std::string { + [[nodiscard]] static auto dump(std::int32_t &idx, auto &&actions) + -> std::string { std::stringstream stream; for (auto &&action : actions) { diff --git a/support/src/utils/error.cpp b/support/src/utils/error.cpp index 04913335..50fe05c7 100644 --- a/support/src/utils/error.cpp +++ b/support/src/utils/error.cpp @@ -27,13 +27,12 @@ std::atomic exception_handler{ auto create_error_message(std::vector items) -> std::string { std::stringstream stream{}; - stream << function_name; for (std::size_t idx = 0U; idx < items.size(); ++idx) { if (idx > 0) { stream << '|'; } - stream << item; + stream << items.at(idx); } return stream.str(); diff --git a/support/src/utils/file_file.cpp b/support/src/utils/file_file.cpp index e7f2d966..10eadd49 100644 --- a/support/src/utils/file_file.cpp +++ b/support/src/utils/file_file.cpp @@ -397,6 +397,8 @@ auto file::sha256() -> std::optional { #endif // defined(PROJECT_ENABLE_LIBSODIUM) auto file::remove() -> bool { + REPERTORY_USES_FUNCTION_NAME(); + close(); return utils::retry_action([this]() -> bool {