Compare commits
1 Commits
924875d0b2
...
v2.0.2-rc
Author | SHA1 | Date | |
---|---|---|---|
8dd46b8ad8 |
@ -21,12 +21,12 @@
|
||||
* \#23 \[bug\] Incorrect file size displayed while upload is pending
|
||||
* \#24 RocksDB implementations should be transactional
|
||||
* \#25 Writes should block when maximum cache size is reached
|
||||
* \#26 Complete ring buffer and direct download support
|
||||
|
||||
### Changes from v2.0.1-rc
|
||||
|
||||
* Ability to choose between RocksDB and SQLite databases
|
||||
* Added direct reads and implemented download fallback
|
||||
* Comprehensive WinFSP and FUSE unit tests, including remote testing
|
||||
* Corrected file times on S3 and Sia providers
|
||||
* Corrected handling of `chown()` and `chmod()`
|
||||
* Fixed erroneous download of chunks after resize
|
||||
|
@ -34,7 +34,7 @@ public:
|
||||
private:
|
||||
struct open_directory final {
|
||||
std::shared_ptr<directory_iterator> iterator;
|
||||
std::vector<std::uint64_t> handles{};
|
||||
std::vector<std::uint64_t> handles;
|
||||
std::chrono::system_clock::time_point last_update{
|
||||
std::chrono::system_clock::now()};
|
||||
};
|
||||
@ -60,8 +60,8 @@ public:
|
||||
void execute_action(const std::string &api_path,
|
||||
const execute_callback &execute);
|
||||
|
||||
[[nodiscard]] auto
|
||||
get_directory(std::uint64_t handle) -> std::shared_ptr<directory_iterator>;
|
||||
[[nodiscard]] auto get_directory(std::uint64_t handle)
|
||||
-> std::shared_ptr<directory_iterator>;
|
||||
|
||||
[[nodiscard]] auto remove_directory(const std::string &api_path)
|
||||
-> std::shared_ptr<directory_iterator>;
|
||||
|
@ -44,6 +44,7 @@ using event_consumer = event_system::event_consumer;
|
||||
#define E_FROM_STRING(t) t
|
||||
#define E_FROM_UINT16(t) std::to_string(t)
|
||||
#define E_FROM_UINT64(t) std::to_string(t)
|
||||
#define E_FROM_DOWNLOAD_TYPE(t) download_type_to_string(t)
|
||||
|
||||
#define E_PROP(type, name, short_name, ts) \
|
||||
private: \
|
||||
|
@ -22,7 +22,7 @@
|
||||
#ifndef REPERTORY_INCLUDE_FILE_MANAGER_DIRECT_OPEN_FILE_HPP_
|
||||
#define REPERTORY_INCLUDE_FILE_MANAGER_DIRECT_OPEN_FILE_HPP_
|
||||
|
||||
#include "file_manager/open_file_base.hpp"
|
||||
#include "file_manager/ring_buffer_base.hpp"
|
||||
|
||||
#include "types/repertory.hpp"
|
||||
|
||||
@ -30,7 +30,7 @@ namespace repertory {
|
||||
class i_provider;
|
||||
class i_upload_manager;
|
||||
|
||||
class direct_open_file final : public open_file_base {
|
||||
class direct_open_file final : public ring_buffer_base {
|
||||
public:
|
||||
direct_open_file(std::uint64_t chunk_size, std::uint8_t chunk_timeout,
|
||||
filesystem_item fsi, i_provider &provider);
|
||||
@ -46,59 +46,37 @@ public:
|
||||
operator=(const direct_open_file &) noexcept -> direct_open_file & = delete;
|
||||
|
||||
private:
|
||||
std::size_t total_chunks_;
|
||||
stop_type stop_requested_{false};
|
||||
std::array<data_buffer, min_ring_size> ring_data_;
|
||||
|
||||
protected:
|
||||
[[nodiscard]] auto is_download_complete() const -> bool override {
|
||||
return false;
|
||||
[[nodiscard]] auto on_check_start() -> bool override;
|
||||
|
||||
[[nodiscard]] auto
|
||||
on_chunk_downloaded(std::size_t /* chunk */,
|
||||
const data_buffer & /* buffer */) -> api_error override {
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
[[nodiscard]] auto
|
||||
on_read_chunk(std::size_t chunk, std::size_t read_size,
|
||||
std::uint64_t read_offset, data_buffer &data,
|
||||
std::size_t &bytes_read) -> api_error override;
|
||||
|
||||
[[nodiscard]] auto use_buffer(std::size_t chunk,
|
||||
std::function<api_error(data_buffer &)> func)
|
||||
-> api_error override;
|
||||
|
||||
public:
|
||||
auto close() -> bool override;
|
||||
|
||||
[[nodiscard]] auto is_complete() const -> bool override { return true; }
|
||||
|
||||
[[nodiscard]] auto is_write_supported() const -> bool override {
|
||||
return false;
|
||||
}
|
||||
|
||||
[[nodiscard]] auto
|
||||
get_read_state() const -> boost::dynamic_bitset<> override {
|
||||
return {};
|
||||
}
|
||||
|
||||
[[nodiscard]] auto
|
||||
get_read_state(std::size_t /* chunk */) const -> bool override {
|
||||
return false;
|
||||
}
|
||||
|
||||
[[nodiscard]] auto get_total_chunks() const -> std::uint64_t {
|
||||
return total_chunks_;
|
||||
}
|
||||
|
||||
[[nodiscard]] auto native_operation(native_operation_callback /* callback */)
|
||||
-> api_error override {
|
||||
return api_error::not_supported;
|
||||
}
|
||||
|
||||
[[nodiscard]] auto native_operation(std::uint64_t /* new_file_size */,
|
||||
native_operation_callback /*callback*/)
|
||||
native_operation_callback /* callback */)
|
||||
-> api_error override {
|
||||
return api_error::not_supported;
|
||||
}
|
||||
|
||||
[[nodiscard]] auto read(std::size_t read_size, std::uint64_t read_offset,
|
||||
data_buffer &data) -> api_error override;
|
||||
|
||||
[[nodiscard]] auto resize(std::uint64_t /*size*/) -> api_error override {
|
||||
return api_error::not_supported;
|
||||
}
|
||||
|
||||
[[nodiscard]] auto write(std::uint64_t, const data_buffer &,
|
||||
std::size_t &) -> api_error override {
|
||||
return api_error::not_supported;
|
||||
}
|
||||
};
|
||||
} // namespace repertory
|
||||
|
||||
|
@ -32,23 +32,6 @@ E_SIMPLE2(download_begin, info, true,
|
||||
std::string, dest_path, dest, E_FROM_STRING
|
||||
);
|
||||
|
||||
E_SIMPLE5(download_chunk_begin, debug, true,
|
||||
std::string, api_path, ap, E_FROM_STRING,
|
||||
std::string, dest_path, dest, E_FROM_STRING,
|
||||
std::size_t, chunk, chunk, E_FROM_SIZE_T,
|
||||
std::size_t, total, total, E_FROM_SIZE_T,
|
||||
std::size_t, complete, complete, E_FROM_SIZE_T
|
||||
);
|
||||
|
||||
E_SIMPLE6(download_chunk_end, debug, true,
|
||||
std::string, api_path, ap, E_FROM_STRING,
|
||||
std::string, dest_path, dest, E_FROM_STRING,
|
||||
std::size_t, chunk, chunk, E_FROM_SIZE_T,
|
||||
std::size_t, total, total, E_FROM_SIZE_T,
|
||||
std::size_t, complete, complete, E_FROM_SIZE_T,
|
||||
api_error, result, result, E_FROM_API_FILE_ERROR
|
||||
);
|
||||
|
||||
E_SIMPLE3(download_end, info, true,
|
||||
std::string, api_path, ap, E_FROM_STRING,
|
||||
std::string, dest_path, dest, E_FROM_STRING,
|
||||
@ -91,6 +74,12 @@ E_SIMPLE2(download_resume_removed, debug, true,
|
||||
E_SIMPLE1(item_timeout, trace, true,
|
||||
std::string, api_path, ap, E_FROM_STRING
|
||||
);
|
||||
|
||||
E_SIMPLE3(download_type_selected, debug, true,
|
||||
std::string, api_path, ap, E_FROM_STRING,
|
||||
std::string, source, src, E_FROM_STRING,
|
||||
download_type, download_type, type, E_FROM_DOWNLOAD_TYPE
|
||||
);
|
||||
// clang-format on
|
||||
} // namespace repertory
|
||||
|
||||
|
@ -68,7 +68,7 @@ private:
|
||||
std::unique_ptr<std::thread> upload_thread_;
|
||||
|
||||
private:
|
||||
void close_all(const std::string &api_path);
|
||||
[[nodiscard]] auto close_all(const std::string &api_path) -> bool;
|
||||
|
||||
void close_timed_out_files();
|
||||
|
||||
@ -108,6 +108,11 @@ public:
|
||||
void remove_resume(const std::string &api_path,
|
||||
const std::string &source_path) override;
|
||||
|
||||
static auto remove_source_and_shrink_cache(const std::string &api_path,
|
||||
const std::string &source_path,
|
||||
std::uint64_t file_size,
|
||||
bool allocated) -> bool;
|
||||
|
||||
void remove_upload(const std::string &api_path) override;
|
||||
|
||||
void store_resume(const i_open_file &file) override;
|
||||
|
@ -40,30 +40,34 @@ public:
|
||||
|
||||
[[nodiscard]] virtual auto get_filesystem_item() const -> filesystem_item = 0;
|
||||
|
||||
[[nodiscard]] virtual auto
|
||||
get_open_data() -> std::map<std::uint64_t, open_file_data> & = 0;
|
||||
[[nodiscard]] virtual auto get_open_data()
|
||||
-> std::map<std::uint64_t, open_file_data> & = 0;
|
||||
|
||||
[[nodiscard]] virtual auto
|
||||
get_open_data() const -> const std::map<std::uint64_t, open_file_data> & = 0;
|
||||
[[nodiscard]] virtual auto get_open_data() const
|
||||
-> const std::map<std::uint64_t, open_file_data> & = 0;
|
||||
|
||||
[[nodiscard]] virtual auto
|
||||
get_open_data(std::uint64_t handle) -> open_file_data & = 0;
|
||||
[[nodiscard]] virtual auto get_open_data(std::uint64_t handle)
|
||||
-> open_file_data & = 0;
|
||||
|
||||
[[nodiscard]] virtual auto
|
||||
get_open_data(std::uint64_t handle) const -> const open_file_data & = 0;
|
||||
[[nodiscard]] virtual auto get_open_data(std::uint64_t handle) const
|
||||
-> const open_file_data & = 0;
|
||||
|
||||
[[nodiscard]] virtual auto get_open_file_count() const -> std::size_t = 0;
|
||||
|
||||
[[nodiscard]] virtual auto
|
||||
get_read_state() const -> boost::dynamic_bitset<> = 0;
|
||||
[[nodiscard]] virtual auto get_read_state() const
|
||||
-> boost::dynamic_bitset<> = 0;
|
||||
|
||||
[[nodiscard]] virtual auto
|
||||
get_read_state(std::size_t chunk) const -> bool = 0;
|
||||
[[nodiscard]] virtual auto get_read_state(std::size_t chunk) const
|
||||
-> bool = 0;
|
||||
|
||||
[[nodiscard]] virtual auto get_source_path() const -> std::string = 0;
|
||||
|
||||
[[nodiscard]] virtual auto is_complete() const -> bool = 0;
|
||||
|
||||
[[nodiscard]] virtual auto is_directory() const -> bool = 0;
|
||||
|
||||
[[nodiscard]] virtual auto is_write_supported() const -> bool = 0;
|
||||
|
||||
[[nodiscard]] virtual auto has_handle(std::uint64_t handle) const -> bool = 0;
|
||||
|
||||
[[nodiscard]] virtual auto
|
||||
@ -74,11 +78,11 @@ public:
|
||||
native_operation_callback callback) -> api_error = 0;
|
||||
|
||||
[[nodiscard]] virtual auto read(std::size_t read_size,
|
||||
std::uint64_t read_offset,
|
||||
data_buffer &data) -> api_error = 0;
|
||||
std::uint64_t read_offset, data_buffer &data)
|
||||
-> api_error = 0;
|
||||
|
||||
[[nodiscard]] virtual auto
|
||||
resize(std::uint64_t new_file_size) -> api_error = 0;
|
||||
[[nodiscard]] virtual auto resize(std::uint64_t new_file_size)
|
||||
-> api_error = 0;
|
||||
|
||||
virtual void set_api_path(const std::string &api_path) = 0;
|
||||
|
||||
@ -93,19 +97,17 @@ class i_closeable_open_file : public i_open_file {
|
||||
public:
|
||||
virtual void add(std::uint64_t handle, open_file_data ofd) = 0;
|
||||
|
||||
[[nodiscard]] virtual auto get_allocated() const -> bool = 0;
|
||||
|
||||
[[nodiscard]] virtual auto can_close() const -> bool = 0;
|
||||
|
||||
virtual auto close() -> bool = 0;
|
||||
|
||||
[[nodiscard]] virtual auto
|
||||
get_handles() const -> std::vector<std::uint64_t> = 0;
|
||||
|
||||
[[nodiscard]] virtual auto is_complete() const -> bool = 0;
|
||||
[[nodiscard]] virtual auto get_handles() const
|
||||
-> std::vector<std::uint64_t> = 0;
|
||||
|
||||
[[nodiscard]] virtual auto is_modified() const -> bool = 0;
|
||||
|
||||
[[nodiscard]] virtual auto is_write_supported() const -> bool = 0;
|
||||
|
||||
virtual void remove(std::uint64_t handle) = 0;
|
||||
|
||||
virtual void remove_all() = 0;
|
||||
|
@ -25,6 +25,7 @@
|
||||
#include "file_manager/open_file_base.hpp"
|
||||
|
||||
#include "types/repertory.hpp"
|
||||
#include "utils/types/file/i_file.hpp"
|
||||
|
||||
namespace repertory {
|
||||
class i_provider;
|
||||
@ -67,31 +68,39 @@ private:
|
||||
i_upload_manager &mgr_;
|
||||
|
||||
private:
|
||||
bool notified_ = false;
|
||||
bool allocated{false};
|
||||
std::unique_ptr<utils::file::i_file> nf_;
|
||||
bool notified_{false};
|
||||
std::size_t read_chunk_{};
|
||||
boost::dynamic_bitset<> read_state_;
|
||||
std::unique_ptr<std::thread> reader_thread_;
|
||||
std::unique_ptr<std::thread> download_thread_;
|
||||
stop_type stop_requested_ = false;
|
||||
mutable std::recursive_mutex rw_mtx_;
|
||||
stop_type stop_requested_{false};
|
||||
|
||||
private:
|
||||
[[nodiscard]] auto adjust_cache_size(std::uint64_t file_size,
|
||||
bool shrink) -> api_error;
|
||||
|
||||
[[nodiscard]] auto check_start() -> api_error;
|
||||
|
||||
void download_chunk(std::size_t chunk, bool skip_active, bool should_reset);
|
||||
|
||||
void download_range(std::size_t start_chunk, std::size_t end_chunk,
|
||||
void download_range(std::size_t begin_chunk, std::size_t end_chunk,
|
||||
bool should_reset);
|
||||
|
||||
void set_modified();
|
||||
|
||||
void update_background_reader(std::size_t read_chunk);
|
||||
void set_read_state(std::size_t chunk);
|
||||
|
||||
protected:
|
||||
auto is_download_complete() const -> bool override {
|
||||
return read_state_.all();
|
||||
}
|
||||
void set_read_state(boost::dynamic_bitset<> read_state);
|
||||
|
||||
void update_reader(std::size_t chunk);
|
||||
|
||||
public:
|
||||
auto close() -> bool override;
|
||||
|
||||
[[nodiscard]] auto get_allocated() const -> bool override;
|
||||
|
||||
[[nodiscard]] auto get_read_state() const -> boost::dynamic_bitset<> override;
|
||||
|
||||
[[nodiscard]] auto get_read_state(std::size_t chunk) const -> bool override;
|
||||
|
@ -24,21 +24,18 @@
|
||||
|
||||
#include "file_manager/i_open_file.hpp"
|
||||
|
||||
#include "utils/types/file/i_file.hpp"
|
||||
|
||||
namespace repertory {
|
||||
class i_provider;
|
||||
|
||||
class open_file_base : public i_closeable_open_file {
|
||||
public:
|
||||
open_file_base(std::uint64_t chunk_size, std::uint8_t chunk_timeout,
|
||||
filesystem_item fsi, i_provider &provider,
|
||||
bool disable_io = false);
|
||||
filesystem_item fsi, i_provider &provider, bool disable_io);
|
||||
|
||||
open_file_base(std::uint64_t chunk_size, std::uint8_t chunk_timeout,
|
||||
filesystem_item fsi,
|
||||
std::map<std::uint64_t, open_file_data> open_data,
|
||||
i_provider &provider, bool disable_io = false);
|
||||
i_provider &provider, bool disable_io);
|
||||
|
||||
~open_file_base() override = default;
|
||||
|
||||
@ -99,7 +96,7 @@ public:
|
||||
[[nodiscard]] auto get_result() -> api_error;
|
||||
};
|
||||
|
||||
protected:
|
||||
private:
|
||||
std::uint64_t chunk_size_;
|
||||
std::uint8_t chunk_timeout_;
|
||||
filesystem_item fsi_;
|
||||
@ -108,21 +105,19 @@ protected:
|
||||
i_provider &provider_;
|
||||
|
||||
private:
|
||||
std::unordered_map<std::size_t, std::shared_ptr<download>> active_downloads_;
|
||||
api_error error_{api_error::success};
|
||||
mutable std::mutex error_mtx_;
|
||||
mutable std::recursive_mutex file_mtx_;
|
||||
stop_type io_stop_requested_{false};
|
||||
std::unique_ptr<std::thread> io_thread_;
|
||||
|
||||
protected:
|
||||
std::unordered_map<std::size_t, std::shared_ptr<download>> active_downloads_;
|
||||
mutable std::recursive_mutex file_mtx_;
|
||||
std::atomic<std::chrono::system_clock::time_point> last_access_{
|
||||
std::chrono::system_clock::now()};
|
||||
bool modified_{false};
|
||||
std::unique_ptr<utils::file::i_file> nf_;
|
||||
mutable std::mutex io_thread_mtx_;
|
||||
std::condition_variable io_thread_notify_;
|
||||
std::deque<std::shared_ptr<io_item>> io_thread_queue_;
|
||||
std::atomic<std::chrono::system_clock::time_point> last_access_{
|
||||
std::chrono::system_clock::now(),
|
||||
};
|
||||
bool modified_{false};
|
||||
bool removed_{false};
|
||||
|
||||
private:
|
||||
@ -131,11 +126,42 @@ private:
|
||||
protected:
|
||||
[[nodiscard]] auto do_io(std::function<api_error()> action) -> api_error;
|
||||
|
||||
[[nodiscard]] virtual auto is_download_complete() const -> bool = 0;
|
||||
[[nodiscard]] auto get_active_downloads()
|
||||
-> std::unordered_map<std::size_t, std::shared_ptr<download>> & {
|
||||
return active_downloads_;
|
||||
}
|
||||
|
||||
[[nodiscard]] auto get_mutex() const -> std::recursive_mutex & {
|
||||
return file_mtx_;
|
||||
}
|
||||
|
||||
[[nodiscard]] auto get_last_chunk_size() const -> std::size_t;
|
||||
|
||||
[[nodiscard]] auto get_provider() -> i_provider & { return provider_; }
|
||||
|
||||
[[nodiscard]] auto get_provider() const -> const i_provider & {
|
||||
return provider_;
|
||||
}
|
||||
|
||||
[[nodiscard]] auto is_removed() const -> bool;
|
||||
|
||||
void notify_io();
|
||||
|
||||
void reset_timeout();
|
||||
|
||||
auto set_api_error(const api_error &e) -> api_error;
|
||||
auto set_api_error(const api_error &err) -> api_error;
|
||||
|
||||
void set_file_size(std::uint64_t size);
|
||||
|
||||
void set_last_chunk_size(std::size_t size);
|
||||
|
||||
void set_modified(bool modified);
|
||||
|
||||
void set_removed(bool removed);
|
||||
|
||||
void set_source_path(std::string source_path);
|
||||
|
||||
void wait_for_io(stop_type &stop_requested);
|
||||
|
||||
public:
|
||||
void add(std::uint64_t handle, open_file_data ofd) override;
|
||||
@ -144,6 +170,8 @@ public:
|
||||
|
||||
auto close() -> bool override;
|
||||
|
||||
[[nodiscard]] auto get_allocated() const -> bool override { return false; }
|
||||
|
||||
[[nodiscard]] auto get_api_error() const -> api_error;
|
||||
|
||||
[[nodiscard]] auto get_api_path() const -> std::string override;
|
||||
@ -172,13 +200,9 @@ public:
|
||||
|
||||
[[nodiscard]] auto get_open_file_count() const -> std::size_t override;
|
||||
|
||||
[[nodiscard]] auto get_source_path() const -> std::string override {
|
||||
return fsi_.source_path;
|
||||
}
|
||||
[[nodiscard]] auto get_source_path() const -> std::string override;
|
||||
|
||||
[[nodiscard]] auto has_handle(std::uint64_t handle) const -> bool override {
|
||||
return open_data_.find(handle) != open_data_.end();
|
||||
}
|
||||
[[nodiscard]] auto has_handle(std::uint64_t handle) const -> bool override;
|
||||
|
||||
[[nodiscard]] auto is_directory() const -> bool override {
|
||||
return fsi_.directory;
|
||||
|
150
repertory/librepertory/include/file_manager/ring_buffer_base.hpp
Normal file
150
repertory/librepertory/include/file_manager/ring_buffer_base.hpp
Normal file
@ -0,0 +1,150 @@
|
||||
/*
|
||||
Copyright <2018-2024> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
*/
|
||||
#ifndef REPERTORY_INCLUDE_FILE_MANAGER_RING_BUFFER_BASE_HPP_
|
||||
#define REPERTORY_INCLUDE_FILE_MANAGER_RING_BUFFER_BASE_HPP_
|
||||
|
||||
#include "file_manager/open_file_base.hpp"
|
||||
|
||||
#include "types/repertory.hpp"
|
||||
#include "utils/file.hpp"
|
||||
|
||||
namespace repertory {
|
||||
class i_provider;
|
||||
class i_upload_manager;
|
||||
|
||||
class ring_buffer_base : public open_file_base {
|
||||
public:
|
||||
ring_buffer_base(std::uint64_t chunk_size, std::uint8_t chunk_timeout,
|
||||
filesystem_item fsi, i_provider &provider,
|
||||
std::size_t ring_size, bool disable_io);
|
||||
|
||||
~ring_buffer_base() override = default;
|
||||
|
||||
public:
|
||||
ring_buffer_base() = delete;
|
||||
ring_buffer_base(const ring_buffer_base &) noexcept = delete;
|
||||
ring_buffer_base(ring_buffer_base &&) noexcept = delete;
|
||||
auto operator=(ring_buffer_base &&) noexcept -> ring_buffer_base & = delete;
|
||||
auto
|
||||
operator=(const ring_buffer_base &) noexcept -> ring_buffer_base & = delete;
|
||||
|
||||
public:
|
||||
static constexpr const auto min_ring_size{5U};
|
||||
|
||||
private:
|
||||
boost::dynamic_bitset<> read_state_;
|
||||
std::size_t total_chunks_;
|
||||
|
||||
private:
|
||||
std::condition_variable chunk_notify_;
|
||||
mutable std::mutex chunk_mtx_;
|
||||
std::mutex read_mtx_;
|
||||
std::unique_ptr<std::thread> reader_thread_;
|
||||
std::size_t ring_begin_{};
|
||||
std::size_t ring_end_{};
|
||||
std::size_t ring_pos_{};
|
||||
stop_type stop_requested_{false};
|
||||
|
||||
private:
|
||||
[[nodiscard]] auto check_start() -> api_error;
|
||||
|
||||
auto download_chunk(std::size_t chunk, bool skip_active) -> api_error;
|
||||
|
||||
void reader_thread();
|
||||
|
||||
void update_position(std::size_t count, bool is_forward);
|
||||
|
||||
protected:
|
||||
[[nodiscard]] auto has_reader_thread() const -> bool {
|
||||
return reader_thread_ != nullptr;
|
||||
}
|
||||
|
||||
[[nodiscard]] auto get_ring_size() const -> std::size_t {
|
||||
return read_state_.size();
|
||||
}
|
||||
|
||||
[[nodiscard]] virtual auto on_check_start() -> bool = 0;
|
||||
|
||||
[[nodiscard]] virtual auto
|
||||
on_chunk_downloaded(std::size_t chunk,
|
||||
const data_buffer &buffer) -> api_error = 0;
|
||||
|
||||
[[nodiscard]] virtual auto
|
||||
on_read_chunk(std::size_t chunk, std::size_t read_size,
|
||||
std::uint64_t read_offset, data_buffer &data,
|
||||
std::size_t &bytes_read) -> api_error = 0;
|
||||
|
||||
[[nodiscard]] virtual auto
|
||||
use_buffer(std::size_t chunk,
|
||||
std::function<api_error(data_buffer &)> func) -> api_error = 0;
|
||||
|
||||
public:
|
||||
auto close() -> bool override;
|
||||
|
||||
void forward(std::size_t count);
|
||||
|
||||
[[nodiscard]] auto get_current_chunk() const -> std::size_t {
|
||||
return ring_pos_;
|
||||
}
|
||||
|
||||
[[nodiscard]] auto get_first_chunk() const -> std::size_t {
|
||||
return ring_begin_;
|
||||
}
|
||||
|
||||
[[nodiscard]] auto get_last_chunk() const -> std::size_t { return ring_end_; }
|
||||
|
||||
[[nodiscard]] auto get_read_state() const -> boost::dynamic_bitset<> override;
|
||||
|
||||
[[nodiscard]] auto get_read_state(std::size_t chunk) const -> bool override;
|
||||
|
||||
[[nodiscard]] auto get_total_chunks() const -> std::size_t {
|
||||
return total_chunks_;
|
||||
}
|
||||
|
||||
[[nodiscard]] auto is_complete() const -> bool override { return false; }
|
||||
|
||||
[[nodiscard]] auto is_write_supported() const -> bool override {
|
||||
return false;
|
||||
}
|
||||
|
||||
[[nodiscard]] auto read(std::size_t read_size, std::uint64_t read_offset,
|
||||
data_buffer &data) -> api_error override;
|
||||
|
||||
[[nodiscard]] auto resize(std::uint64_t /* size */) -> api_error override {
|
||||
return api_error::not_supported;
|
||||
}
|
||||
|
||||
void reverse(std::size_t count);
|
||||
|
||||
void set(std::size_t first_chunk, std::size_t current_chunk);
|
||||
|
||||
void set_api_path(const std::string &api_path) override;
|
||||
|
||||
[[nodiscard]] auto
|
||||
write(std::uint64_t /* write_offset */, const data_buffer & /* data */,
|
||||
std::size_t & /* bytes_written */) -> api_error override {
|
||||
return api_error::not_supported;
|
||||
}
|
||||
};
|
||||
} // namespace repertory
|
||||
|
||||
#endif // REPERTORY_INCLUDE_FILE_MANAGER_RING_BUFFER_BASE_HPP_
|
@ -22,20 +22,17 @@
|
||||
#ifndef REPERTORY_INCLUDE_FILE_MANAGER_RING_BUFFER_OPEN_FILE_HPP_
|
||||
#define REPERTORY_INCLUDE_FILE_MANAGER_RING_BUFFER_OPEN_FILE_HPP_
|
||||
|
||||
#include "file_manager/open_file_base.hpp"
|
||||
#include "file_manager/ring_buffer_base.hpp"
|
||||
|
||||
#include "types/repertory.hpp"
|
||||
#include "utils/file.hpp"
|
||||
|
||||
namespace repertory {
|
||||
class i_provider;
|
||||
class i_upload_manager;
|
||||
|
||||
class ring_buffer_open_file final : public open_file_base {
|
||||
class ring_buffer_open_file final : public ring_buffer_base {
|
||||
public:
|
||||
ring_buffer_open_file(std::string buffer_directory, std::uint64_t chunk_size,
|
||||
std::uint8_t chunk_timeout, filesystem_item fsi,
|
||||
i_provider &provider);
|
||||
|
||||
ring_buffer_open_file(std::string buffer_directory, std::uint64_t chunk_size,
|
||||
std::uint8_t chunk_timeout, filesystem_item fsi,
|
||||
i_provider &provider, std::size_t ring_size);
|
||||
@ -52,90 +49,43 @@ public:
|
||||
-> ring_buffer_open_file & = delete;
|
||||
|
||||
private:
|
||||
boost::dynamic_bitset<> ring_state_;
|
||||
std::size_t total_chunks_;
|
||||
std::string source_path_;
|
||||
|
||||
private:
|
||||
std::unique_ptr<std::thread> chunk_forward_thread_;
|
||||
std::unique_ptr<std::thread> chunk_reverse_thread_;
|
||||
std::condition_variable chunk_notify_;
|
||||
mutable std::mutex chunk_mtx_;
|
||||
std::size_t current_chunk_{};
|
||||
std::size_t first_chunk_{};
|
||||
std::size_t last_chunk_;
|
||||
stop_type stop_requested_{false};
|
||||
|
||||
private:
|
||||
auto download_chunk(std::size_t chunk) -> api_error;
|
||||
|
||||
void forward_reader_thread(std::size_t count);
|
||||
|
||||
void reverse_reader_thread(std::size_t count);
|
||||
std::unique_ptr<utils::file::i_file> nf_;
|
||||
|
||||
protected:
|
||||
[[nodiscard]] auto is_download_complete() const -> bool override {
|
||||
return false;
|
||||
}
|
||||
[[nodiscard]] auto on_check_start() -> bool override;
|
||||
|
||||
[[nodiscard]] auto
|
||||
on_chunk_downloaded(std::size_t chunk,
|
||||
const data_buffer &buffer) -> api_error override;
|
||||
|
||||
[[nodiscard]] auto
|
||||
on_read_chunk(std::size_t chunk, std::size_t read_size,
|
||||
std::uint64_t read_offset, data_buffer &data,
|
||||
std::size_t &bytes_read) -> api_error override;
|
||||
|
||||
[[nodiscard]] auto use_buffer(std::size_t chunk,
|
||||
std::function<api_error(data_buffer &)> func)
|
||||
-> api_error override;
|
||||
|
||||
public:
|
||||
[[nodiscard]] static auto can_handle_file(std::uint64_t file_size,
|
||||
std::size_t chunk_size,
|
||||
std::size_t ring_size) -> bool;
|
||||
|
||||
auto close() -> bool override;
|
||||
|
||||
void forward(std::size_t count);
|
||||
|
||||
[[nodiscard]] auto get_current_chunk() const -> std::size_t {
|
||||
return current_chunk_;
|
||||
}
|
||||
|
||||
[[nodiscard]] auto get_first_chunk() const -> std::size_t {
|
||||
return first_chunk_;
|
||||
}
|
||||
|
||||
[[nodiscard]] auto get_last_chunk() const -> std::size_t {
|
||||
return last_chunk_;
|
||||
}
|
||||
|
||||
[[nodiscard]] auto get_read_state() const -> boost::dynamic_bitset<> override;
|
||||
|
||||
[[nodiscard]] auto get_read_state(std::size_t chunk) const -> bool override;
|
||||
|
||||
[[nodiscard]] auto get_total_chunks() const -> std::size_t {
|
||||
return total_chunks_;
|
||||
}
|
||||
|
||||
[[nodiscard]] auto is_complete() const -> bool override { return true; }
|
||||
|
||||
[[nodiscard]] auto is_write_supported() const -> bool override {
|
||||
return false;
|
||||
}
|
||||
|
||||
[[nodiscard]] auto
|
||||
native_operation(native_operation_callback callback) -> api_error override;
|
||||
|
||||
[[nodiscard]] auto native_operation(std::uint64_t, native_operation_callback)
|
||||
[[nodiscard]] auto native_operation(std::uint64_t /* new_file_size */,
|
||||
native_operation_callback /* callback */)
|
||||
-> api_error override {
|
||||
return api_error::not_supported;
|
||||
}
|
||||
|
||||
[[nodiscard]] auto read(std::size_t read_size, std::uint64_t read_offset,
|
||||
data_buffer &data) -> api_error override;
|
||||
|
||||
[[nodiscard]] auto resize(std::uint64_t) -> api_error override {
|
||||
return api_error::not_supported;
|
||||
}
|
||||
|
||||
void reverse(std::size_t count);
|
||||
|
||||
void set(std::size_t first_chunk, std::size_t current_chunk);
|
||||
|
||||
void set_api_path(const std::string &api_path) override;
|
||||
|
||||
[[nodiscard]] auto write(std::uint64_t, const data_buffer &,
|
||||
std::size_t &) -> api_error override {
|
||||
return api_error::not_supported;
|
||||
[[nodiscard]] auto get_source_path() const -> std::string override {
|
||||
return source_path_;
|
||||
}
|
||||
};
|
||||
} // namespace repertory
|
||||
|
@ -29,7 +29,7 @@ constexpr const auto default_eviction_delay_mins{1U};
|
||||
constexpr const auto default_high_freq_interval_secs{30U};
|
||||
constexpr const auto default_low_freq_interval_secs{0U * 60U};
|
||||
constexpr const auto default_max_cache_size_bytes{
|
||||
20UL * 1024UL * 1024UL * 1024UL,
|
||||
std::uint64_t(20UL * 1024UL * 1024UL * 1024UL),
|
||||
};
|
||||
constexpr const auto default_max_upload_count{5U};
|
||||
constexpr const auto default_med_freq_interval_secs{2U * 60U};
|
||||
@ -194,6 +194,7 @@ enum class api_error {
|
||||
invalid_handle,
|
||||
invalid_operation,
|
||||
invalid_ring_buffer_multiple,
|
||||
invalid_ring_buffer_position,
|
||||
invalid_ring_buffer_size,
|
||||
invalid_version,
|
||||
item_exists,
|
||||
@ -230,13 +231,13 @@ enum class database_type {
|
||||
database_type_to_string(const database_type &type) -> std::string;
|
||||
|
||||
enum class download_type {
|
||||
default_,
|
||||
direct,
|
||||
fallback,
|
||||
ring_buffer,
|
||||
};
|
||||
[[nodiscard]] auto download_type_from_string(
|
||||
std::string type,
|
||||
download_type default_type = download_type::fallback) -> download_type;
|
||||
download_type default_type = download_type::default_) -> download_type;
|
||||
|
||||
[[nodiscard]] auto
|
||||
download_type_to_string(const download_type &type) -> std::string;
|
||||
|
@ -23,6 +23,7 @@
|
||||
|
||||
#include "events/event_system.hpp"
|
||||
#include "events/events.hpp"
|
||||
#include "file_manager/cache_size_mgr.hpp"
|
||||
#include "platform/platform.hpp"
|
||||
#include "types/startup_exception.hpp"
|
||||
#include "utils/common.hpp"
|
||||
@ -79,7 +80,7 @@ app_config::app_config(const provider_type &prov,
|
||||
med_freq_interval_secs_(default_med_freq_interval_secs),
|
||||
online_check_retry_secs_(default_online_check_retry_secs),
|
||||
orphaned_file_retention_days_(default_orphaned_file_retention_days),
|
||||
preferred_download_type_(download_type::fallback),
|
||||
preferred_download_type_(download_type::default_),
|
||||
retry_read_count_(default_retry_read_count),
|
||||
ring_buffer_file_size_(default_ring_buffer_file_size),
|
||||
task_wait_ms_(default_task_wait_ms) {
|
||||
@ -1170,7 +1171,13 @@ void app_config::set_low_frequency_interval_secs(std::uint16_t value) {
|
||||
}
|
||||
|
||||
void app_config::set_max_cache_size_bytes(std::uint64_t value) {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
set_value(max_cache_size_bytes_, value);
|
||||
auto res = cache_size_mgr::instance().shrink(0U);
|
||||
if (res != api_error::success) {
|
||||
utils::error::raise_error(function_name, res, "failed to shrink cache");
|
||||
}
|
||||
}
|
||||
|
||||
void app_config::set_max_upload_count(std::uint8_t value) {
|
||||
|
@ -30,6 +30,7 @@
|
||||
#include "utils/file_utils.hpp"
|
||||
#include "utils/time.hpp"
|
||||
#include "utils/utils.hpp"
|
||||
#include <spdlog/fmt/bundled/base.h>
|
||||
|
||||
namespace repertory {
|
||||
auto eviction::check_minimum_requirements(const std::string &file_path)
|
||||
@ -75,8 +76,8 @@ void eviction::service_function() {
|
||||
|
||||
try {
|
||||
std::string api_path;
|
||||
if (provider_.get_api_path_from_source(file_path, api_path) !=
|
||||
api_error::success) {
|
||||
auto res = provider_.get_api_path_from_source(file_path, api_path);
|
||||
if (res != api_error::success) {
|
||||
continue;
|
||||
}
|
||||
|
||||
|
@ -81,8 +81,8 @@ auto fuse_drive::chown_impl(std::string api_path, uid_t uid, gid_t gid,
|
||||
struct fuse_file_info * /*file_info*/)
|
||||
-> api_error {
|
||||
#else
|
||||
auto fuse_drive::chown_impl(std::string api_path, uid_t uid, gid_t gid)
|
||||
-> api_error {
|
||||
auto fuse_drive::chown_impl(std::string api_path, uid_t uid,
|
||||
gid_t gid) -> api_error {
|
||||
#endif
|
||||
return check_and_perform(
|
||||
api_path, X_OK, [&](api_meta_map &meta) -> api_error {
|
||||
@ -481,8 +481,8 @@ auto fuse_drive::getattr_impl(std::string api_path, struct stat *unix_st,
|
||||
struct fuse_file_info * /*file_info*/)
|
||||
-> api_error {
|
||||
#else
|
||||
auto fuse_drive::getattr_impl(std::string api_path, struct stat *unix_st)
|
||||
-> api_error {
|
||||
auto fuse_drive::getattr_impl(std::string api_path,
|
||||
struct stat *unix_st) -> api_error {
|
||||
#endif
|
||||
auto parent = utils::path::get_parent_api_path(api_path);
|
||||
|
||||
@ -565,8 +565,8 @@ auto fuse_drive::getxtimes_impl(std::string api_path, struct timespec *bkuptime,
|
||||
#endif // __APPLE__
|
||||
|
||||
#if FUSE_USE_VERSION >= 30
|
||||
auto fuse_drive::init_impl(struct fuse_conn_info *conn, struct fuse_config *cfg)
|
||||
-> void * {
|
||||
auto fuse_drive::init_impl(struct fuse_conn_info *conn,
|
||||
struct fuse_config *cfg) -> void * {
|
||||
#else
|
||||
void *fuse_drive::init_impl(struct fuse_conn_info *conn) {
|
||||
#endif
|
||||
@ -800,9 +800,8 @@ auto fuse_drive::release_impl(std::string /*api_path*/,
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
auto fuse_drive::releasedir_impl(std::string /*api_path*/,
|
||||
struct fuse_file_info *file_info)
|
||||
-> api_error {
|
||||
auto fuse_drive::releasedir_impl(
|
||||
std::string /*api_path*/, struct fuse_file_info *file_info) -> api_error {
|
||||
auto iter = directory_cache_->get_directory(file_info->fh);
|
||||
if (iter == nullptr) {
|
||||
return api_error::invalid_handle;
|
||||
@ -820,8 +819,8 @@ auto fuse_drive::rename_directory(const std::string &from_api_path,
|
||||
}
|
||||
|
||||
auto fuse_drive::rename_file(const std::string &from_api_path,
|
||||
const std::string &to_api_path, bool overwrite)
|
||||
-> int {
|
||||
const std::string &to_api_path,
|
||||
bool overwrite) -> int {
|
||||
auto res = fm_->rename_file(from_api_path, to_api_path, overwrite);
|
||||
errno = std::abs(utils::from_api_error(res));
|
||||
return (res == api_error::success) ? 0 : -1;
|
||||
@ -831,8 +830,8 @@ auto fuse_drive::rename_file(const std::string &from_api_path,
|
||||
auto fuse_drive::rename_impl(std::string from_api_path, std::string to_api_path,
|
||||
unsigned int /*flags*/) -> api_error {
|
||||
#else
|
||||
auto fuse_drive::rename_impl(std::string from_api_path, std::string to_api_path)
|
||||
-> api_error {
|
||||
auto fuse_drive::rename_impl(std::string from_api_path,
|
||||
std::string to_api_path) -> api_error {
|
||||
#endif
|
||||
auto res = check_parent_access(to_api_path, W_OK | X_OK);
|
||||
if (res != api_error::success) {
|
||||
@ -946,15 +945,15 @@ auto fuse_drive::getxattr_impl(std::string api_path, const char *name,
|
||||
}
|
||||
#else // __APPLE__
|
||||
auto fuse_drive::getxattr_impl(std::string api_path, const char *name,
|
||||
char *value, size_t size, int &attribute_size)
|
||||
-> api_error {
|
||||
char *value, size_t size,
|
||||
int &attribute_size) -> api_error {
|
||||
return getxattr_common(api_path, name, value, size, attribute_size, nullptr);
|
||||
}
|
||||
#endif // __APPLE__
|
||||
|
||||
auto fuse_drive::listxattr_impl(std::string api_path, char *buffer, size_t size,
|
||||
int &required_size, bool &return_size)
|
||||
-> api_error {
|
||||
int &required_size,
|
||||
bool &return_size) -> api_error {
|
||||
auto check_size = (size == 0);
|
||||
|
||||
auto res = check_parent_access(api_path, X_OK);
|
||||
@ -994,8 +993,8 @@ auto fuse_drive::listxattr_impl(std::string api_path, char *buffer, size_t size,
|
||||
return res;
|
||||
}
|
||||
|
||||
auto fuse_drive::removexattr_impl(std::string api_path, const char *name)
|
||||
-> api_error {
|
||||
auto fuse_drive::removexattr_impl(std::string api_path,
|
||||
const char *name) -> api_error {
|
||||
std::string attribute_name;
|
||||
#if defined(__APPLE__)
|
||||
auto res = parse_xattr_parameters(name, 0, attribute_name, api_path);
|
||||
@ -1023,8 +1022,8 @@ auto fuse_drive::setxattr_impl(std::string api_path, const char *name,
|
||||
uint32_t position) -> api_error {
|
||||
#else // __APPLE__
|
||||
auto fuse_drive::setxattr_impl(std::string api_path, const char *name,
|
||||
const char *value, size_t size, int flags)
|
||||
-> api_error {
|
||||
const char *value, size_t size,
|
||||
int flags) -> api_error {
|
||||
#endif
|
||||
std::string attribute_name;
|
||||
#if defined(__APPLE__)
|
||||
@ -1102,8 +1101,8 @@ void fuse_drive::set_item_meta(const std::string &api_path,
|
||||
}
|
||||
|
||||
#if defined(__APPLE__)
|
||||
auto fuse_drive::setattr_x_impl(std::string api_path, struct setattr_x *attr)
|
||||
-> api_error {
|
||||
auto fuse_drive::setattr_x_impl(std::string api_path,
|
||||
struct setattr_x *attr) -> api_error {
|
||||
bool exists{};
|
||||
auto res = provider_.is_file(api_path, exists);
|
||||
if (res != api_error::success) {
|
||||
@ -1157,7 +1156,7 @@ auto fuse_drive::setattr_x_impl(std::string api_path, struct setattr_x *attr)
|
||||
ts[0].tv_sec = attr->acctime.tv_sec;
|
||||
ts[0].tv_nsec = attr->acctime.tv_nsec;
|
||||
} else {
|
||||
struct timeval tv{};
|
||||
struct timeval tv {};
|
||||
gettimeofday(&tv, NULL);
|
||||
ts[0].tv_sec = tv.tv_sec;
|
||||
ts[0].tv_nsec = tv.tv_usec * 1000;
|
||||
@ -1202,9 +1201,8 @@ auto fuse_drive::setattr_x_impl(std::string api_path, struct setattr_x *attr)
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
auto fuse_drive::setbkuptime_impl(std::string api_path,
|
||||
const struct timespec *bkuptime)
|
||||
-> api_error {
|
||||
auto fuse_drive::setbkuptime_impl(
|
||||
std::string api_path, const struct timespec *bkuptime) -> api_error {
|
||||
return check_and_perform(
|
||||
api_path, X_OK, [&](api_meta_map &meta) -> api_error {
|
||||
auto nanos = bkuptime->tv_nsec +
|
||||
@ -1240,8 +1238,8 @@ auto fuse_drive::setvolname_impl(const char * /*volname*/) -> api_error {
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
auto fuse_drive::statfs_x_impl(std::string /*api_path*/, struct statfs *stbuf)
|
||||
-> api_error {
|
||||
auto fuse_drive::statfs_x_impl(std::string /*api_path*/,
|
||||
struct statfs *stbuf) -> api_error {
|
||||
if (statfs(&config_.get_cache_directory()[0], stbuf) != 0) {
|
||||
return api_error::os_error;
|
||||
}
|
||||
@ -1266,8 +1264,8 @@ auto fuse_drive::statfs_x_impl(std::string /*api_path*/, struct statfs *stbuf)
|
||||
return api_error::success;
|
||||
}
|
||||
#else // __APPLE__
|
||||
auto fuse_drive::statfs_impl(std::string /*api_path*/, struct statvfs *stbuf)
|
||||
-> api_error {
|
||||
auto fuse_drive::statfs_impl(std::string /*api_path*/,
|
||||
struct statvfs *stbuf) -> api_error {
|
||||
if (statvfs(config_.get_cache_directory().data(), stbuf) != 0) {
|
||||
return api_error::os_error;
|
||||
}
|
||||
@ -1317,6 +1315,10 @@ auto fuse_drive::truncate_impl(std::string api_path, off_t size) -> api_error {
|
||||
return res;
|
||||
}
|
||||
|
||||
if (not fm_->get_open_file(handle, true, open_file)) {
|
||||
return api_error::invalid_handle;
|
||||
}
|
||||
|
||||
res = open_file->resize(static_cast<std::uint64_t>(size));
|
||||
}
|
||||
|
||||
@ -1347,8 +1349,8 @@ auto fuse_drive::utimens_impl(std::string api_path, const struct timespec tv[2],
|
||||
struct fuse_file_info * /*file_info*/)
|
||||
-> api_error {
|
||||
#else
|
||||
auto fuse_drive::utimens_impl(std::string api_path, const struct timespec tv[2])
|
||||
-> api_error {
|
||||
auto fuse_drive::utimens_impl(std::string api_path,
|
||||
const struct timespec tv[2]) -> api_error {
|
||||
#endif
|
||||
api_meta_map meta;
|
||||
auto res = provider_.get_item_meta(api_path, meta);
|
||||
|
@ -42,6 +42,7 @@ E_SIMPLE2(max_cache_size_reached, warn, true,
|
||||
|
||||
cache_size_mgr cache_size_mgr::instance_{};
|
||||
|
||||
// TODO add timeout
|
||||
auto cache_size_mgr::expand(std::uint64_t size) -> api_error {
|
||||
if (size == 0U) {
|
||||
return api_error::success;
|
||||
@ -56,16 +57,17 @@ auto cache_size_mgr::expand(std::uint64_t size) -> api_error {
|
||||
|
||||
auto max_cache_size = cfg_->get_max_cache_size_bytes();
|
||||
|
||||
while (not stop_requested_ && cache_size_ > max_cache_size) {
|
||||
auto cache_dir = utils::file::directory{cfg_->get_cache_directory()};
|
||||
while (not stop_requested_ && cache_size_ > max_cache_size &&
|
||||
cache_dir.count() > 1U) {
|
||||
event_system::instance().raise<max_cache_size_reached>(cache_size_,
|
||||
max_cache_size);
|
||||
notify_.notify_all();
|
||||
notify_.wait(lock);
|
||||
}
|
||||
|
||||
notify_.notify_all();
|
||||
|
||||
return stop_requested_ ? api_error::error : api_error::success;
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
void cache_size_mgr::initialize(app_config *cfg) {
|
||||
@ -90,11 +92,12 @@ void cache_size_mgr::initialize(app_config *cfg) {
|
||||
}
|
||||
|
||||
auto cache_size_mgr::shrink(std::uint64_t size) -> api_error {
|
||||
mutex_lock lock(mtx_);
|
||||
if (size == 0U) {
|
||||
notify_.notify_all();
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
mutex_lock lock(mtx_);
|
||||
if (cache_size_ >= size) {
|
||||
cache_size_ -= size;
|
||||
} else {
|
||||
@ -104,7 +107,7 @@ auto cache_size_mgr::shrink(std::uint64_t size) -> api_error {
|
||||
|
||||
notify_.notify_all();
|
||||
|
||||
return stop_requested_ ? api_error::error : api_error::success;
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
auto cache_size_mgr::size() const -> std::uint64_t {
|
||||
|
@ -24,40 +24,40 @@
|
||||
#include "file_manager/open_file_base.hpp"
|
||||
#include "providers/i_provider.hpp"
|
||||
#include "types/repertory.hpp"
|
||||
#include "utils/common.hpp"
|
||||
|
||||
namespace repertory {
|
||||
direct_open_file::direct_open_file(std::uint64_t chunk_size,
|
||||
std::uint8_t chunk_timeout,
|
||||
filesystem_item fsi, i_provider &provider)
|
||||
: open_file_base(chunk_size, chunk_timeout, fsi, provider, true),
|
||||
total_chunks_(static_cast<std::size_t>(
|
||||
utils::divide_with_ceiling(fsi.size, chunk_size))) {}
|
||||
: ring_buffer_base(chunk_size, chunk_timeout, fsi, provider,
|
||||
min_ring_size, true) {}
|
||||
|
||||
direct_open_file::~direct_open_file() { close(); }
|
||||
direct_open_file::~direct_open_file() {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
auto direct_open_file::close() -> bool {
|
||||
stop_requested_ = true;
|
||||
return open_file_base::close();
|
||||
close();
|
||||
}
|
||||
|
||||
auto direct_open_file::read(std::size_t read_size, std::uint64_t read_offset,
|
||||
data_buffer &data) -> api_error {
|
||||
if (fsi_.directory) {
|
||||
return api_error::invalid_operation;
|
||||
}
|
||||
auto direct_open_file::on_check_start() -> bool {
|
||||
return (get_file_size() == 0U || has_reader_thread());
|
||||
}
|
||||
|
||||
reset_timeout();
|
||||
auto direct_open_file::on_read_chunk(std::size_t chunk, std::size_t read_size,
|
||||
std::uint64_t read_offset,
|
||||
data_buffer &data,
|
||||
std::size_t &bytes_read) -> api_error {
|
||||
auto &buffer = ring_data_.at(chunk % get_ring_size());
|
||||
auto begin =
|
||||
std::next(buffer.begin(), static_cast<std::int64_t>(read_offset));
|
||||
auto end = std::next(begin, static_cast<std::int64_t>(read_size));
|
||||
data.insert(data.end(), begin, end);
|
||||
bytes_read = read_size;
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
read_size = utils::calculate_read_size(fsi_.size, read_size, read_offset);
|
||||
if (read_size == 0U) {
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
auto res = provider_.read_file_bytes(fsi_.api_path, read_size, read_offset,
|
||||
data, stop_requested_);
|
||||
reset_timeout();
|
||||
|
||||
return res;
|
||||
auto direct_open_file::use_buffer(std::size_t chunk,
|
||||
std::function<api_error(data_buffer &)> func)
|
||||
-> api_error {
|
||||
return func(ring_data_.at(chunk % get_ring_size()));
|
||||
}
|
||||
} // namespace repertory
|
||||
|
@ -39,8 +39,6 @@
|
||||
#include "utils/file.hpp"
|
||||
#include "utils/path.hpp"
|
||||
#include "utils/polling.hpp"
|
||||
#include "utils/time.hpp"
|
||||
#include <utils/unix.hpp>
|
||||
|
||||
namespace repertory {
|
||||
file_manager::file_manager(app_config &config, i_provider &provider)
|
||||
@ -75,13 +73,13 @@ void file_manager::close(std::uint64_t handle) {
|
||||
closeable_file->remove(handle);
|
||||
}
|
||||
|
||||
void file_manager::close_all(const std::string &api_path) {
|
||||
auto file_manager::close_all(const std::string &api_path) -> bool {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
unique_recur_mutex_lock file_lock(open_file_mtx_);
|
||||
auto file_iter = open_file_lookup_.find(api_path);
|
||||
if (file_iter == open_file_lookup_.end()) {
|
||||
return;
|
||||
return false;
|
||||
}
|
||||
|
||||
auto closeable_file = file_iter->second;
|
||||
@ -90,6 +88,8 @@ void file_manager::close_all(const std::string &api_path) {
|
||||
|
||||
closeable_file->remove_all();
|
||||
closeable_file->close();
|
||||
|
||||
return closeable_file->get_allocated();
|
||||
}
|
||||
|
||||
void file_manager::close_timed_out_files() {
|
||||
@ -104,12 +104,12 @@ void file_manager::close_timed_out_files() {
|
||||
}
|
||||
return items;
|
||||
});
|
||||
for (auto &&closeable_file : closeable_list) {
|
||||
for (const auto &closeable_file : closeable_list) {
|
||||
open_file_lookup_.erase(closeable_file->get_api_path());
|
||||
}
|
||||
file_lock.unlock();
|
||||
|
||||
for (auto &&closeable_file : closeable_list) {
|
||||
for (auto &closeable_file : closeable_list) {
|
||||
closeable_file->close();
|
||||
event_system::instance().raise<item_timeout>(
|
||||
closeable_file->get_api_path());
|
||||
@ -142,7 +142,7 @@ auto file_manager::evict_file(const std::string &api_path) -> bool {
|
||||
return false;
|
||||
}
|
||||
|
||||
recur_mutex_lock open_lock(open_file_mtx_);
|
||||
unique_recur_mutex_lock open_lock(open_file_mtx_);
|
||||
if (is_processing(api_path)) {
|
||||
return false;
|
||||
}
|
||||
@ -151,8 +151,18 @@ auto file_manager::evict_file(const std::string &api_path) -> bool {
|
||||
return false;
|
||||
}
|
||||
|
||||
filesystem_item fsi{};
|
||||
auto res = provider_.get_filesystem_item(api_path, false, fsi);
|
||||
if (res != api_error::success) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (fsi.source_path.empty()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
std::string pinned;
|
||||
auto res = provider_.get_item_meta(api_path, META_PINNED, pinned);
|
||||
res = provider_.get_item_meta(api_path, META_PINNED, pinned);
|
||||
if (res != api_error::success && res != api_error::item_not_found) {
|
||||
utils::error::raise_api_path_error(std::string{function_name}, api_path,
|
||||
res, "failed to get pinned status");
|
||||
@ -163,31 +173,22 @@ auto file_manager::evict_file(const std::string &api_path) -> bool {
|
||||
return false;
|
||||
}
|
||||
|
||||
std::string source_path{};
|
||||
res = provider_.get_item_meta(api_path, META_SOURCE, source_path);
|
||||
if (res != api_error::success) {
|
||||
utils::error::raise_api_path_error(std::string{function_name}, api_path,
|
||||
res, "failed to get source path");
|
||||
return false;
|
||||
}
|
||||
if (source_path.empty()) {
|
||||
return false;
|
||||
std::shared_ptr<i_closeable_open_file> closeable_file;
|
||||
if (open_file_lookup_.contains(api_path)) {
|
||||
closeable_file = open_file_lookup_.at(api_path);
|
||||
}
|
||||
|
||||
open_file_lookup_.erase(api_path);
|
||||
open_lock.unlock();
|
||||
|
||||
auto file = utils::file::file{source_path};
|
||||
auto file_size = file.size().value_or(0U);
|
||||
auto removed = file.remove();
|
||||
auto allocated = closeable_file ? closeable_file->get_allocated() : true;
|
||||
closeable_file.reset();
|
||||
|
||||
auto removed = remove_source_and_shrink_cache(api_path, fsi.source_path,
|
||||
fsi.size, allocated);
|
||||
if (removed) {
|
||||
res = cache_size_mgr::instance().shrink(file_size);
|
||||
if (res != api_error::success) {
|
||||
utils::error::raise_api_path_error(function_name, api_path, res,
|
||||
"failed to shrink cache");
|
||||
}
|
||||
|
||||
event_system::instance().raise<filesystem_item_evicted>(api_path,
|
||||
source_path);
|
||||
fsi.source_path);
|
||||
}
|
||||
|
||||
return removed;
|
||||
@ -218,7 +219,7 @@ auto file_manager::get_open_file_by_handle(std::uint64_t handle) const
|
||||
-> std::shared_ptr<i_closeable_open_file> {
|
||||
auto file_iter =
|
||||
std::find_if(open_file_lookup_.begin(), open_file_lookup_.end(),
|
||||
[&handle](const auto &item) -> bool {
|
||||
[&handle](auto &&item) -> bool {
|
||||
return item.second->has_handle(handle);
|
||||
});
|
||||
return (file_iter == open_file_lookup_.end()) ? nullptr : file_iter->second;
|
||||
@ -234,7 +235,7 @@ auto file_manager::get_open_file_count(const std::string &api_path) const
|
||||
|
||||
auto file_manager::get_open_file(std::uint64_t handle, bool write_supported,
|
||||
std::shared_ptr<i_open_file> &file) -> bool {
|
||||
recur_mutex_lock open_lock(open_file_mtx_);
|
||||
unique_recur_mutex_lock open_lock(open_file_mtx_);
|
||||
auto file_ptr = get_open_file_by_handle(handle);
|
||||
if (not file_ptr) {
|
||||
return false;
|
||||
@ -267,7 +268,7 @@ auto file_manager::get_open_files() const
|
||||
std::unordered_map<std::string, std::size_t> ret;
|
||||
|
||||
recur_mutex_lock open_lock(open_file_mtx_);
|
||||
for (auto &&item : open_file_lookup_) {
|
||||
for (const auto &item : open_file_lookup_) {
|
||||
ret[item.first] = item.second->get_open_file_count();
|
||||
}
|
||||
|
||||
@ -357,12 +358,19 @@ auto file_manager::is_processing(const std::string &api_path) const -> bool {
|
||||
return true;
|
||||
};
|
||||
|
||||
recur_mutex_lock open_lock(open_file_mtx_);
|
||||
unique_recur_mutex_lock open_lock(open_file_mtx_);
|
||||
auto file_iter = open_file_lookup_.find(api_path);
|
||||
return (file_iter == open_file_lookup_.end())
|
||||
? false
|
||||
: file_iter->second->is_modified() ||
|
||||
not file_iter->second->is_complete();
|
||||
if (file_iter == open_file_lookup_.end()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
auto closeable_file = file_iter->second;
|
||||
open_lock.unlock();
|
||||
|
||||
return closeable_file->is_write_supported()
|
||||
? closeable_file->is_modified() ||
|
||||
not closeable_file->is_complete()
|
||||
: false;
|
||||
}
|
||||
|
||||
auto file_manager::open(const std::string &api_path, bool directory,
|
||||
@ -415,6 +423,10 @@ auto file_manager::open(
|
||||
utils::encryption::encrypting_reader::get_data_chunk_size(),
|
||||
};
|
||||
|
||||
auto chunk_timeout = config_.get_enable_download_timeout()
|
||||
? config_.get_download_timeout_secs()
|
||||
: 0U;
|
||||
|
||||
auto ring_buffer_file_size{
|
||||
static_cast<std::uint64_t>(config_.get_ring_buffer_file_size()) *
|
||||
1024UL * 1024UL,
|
||||
@ -422,22 +434,20 @@ auto file_manager::open(
|
||||
|
||||
auto ring_size{ring_buffer_file_size / chunk_size};
|
||||
|
||||
const auto get_download_type =
|
||||
[this, &buffer_directory, &chunk_size, &fsi, &ring_buffer_file_size,
|
||||
&ring_size](download_type type) -> download_type {
|
||||
if (fsi.size == 0U) {
|
||||
return download_type::fallback;
|
||||
const auto get_download_type = [&](download_type type) -> download_type {
|
||||
if (directory || fsi.size == 0U || is_processing(api_path)) {
|
||||
return download_type::default_;
|
||||
}
|
||||
|
||||
if (type == download_type::direct) {
|
||||
return type;
|
||||
}
|
||||
|
||||
if (type == download_type::fallback) {
|
||||
if (type == download_type::default_) {
|
||||
auto free_space =
|
||||
utils::file::get_free_drive_space(config_.get_cache_directory());
|
||||
if (fsi.size < free_space) {
|
||||
return download_type::fallback;
|
||||
return download_type::default_;
|
||||
}
|
||||
}
|
||||
|
||||
@ -462,10 +472,17 @@ auto file_manager::open(
|
||||
return download_type::direct;
|
||||
};
|
||||
|
||||
auto chunk_timeout = config_.get_enable_download_timeout()
|
||||
? config_.get_download_timeout_secs()
|
||||
: 0U;
|
||||
switch (get_download_type(config_.get_preferred_download_type())) {
|
||||
auto preferred_type = config_.get_preferred_download_type();
|
||||
auto type = get_download_type(directory ? download_type::default_
|
||||
: preferred_type == download_type::default_
|
||||
? download_type::ring_buffer
|
||||
: preferred_type);
|
||||
if (not directory) {
|
||||
event_system::instance().raise<download_type_selected>(
|
||||
fsi.api_path, fsi.source_path, type);
|
||||
}
|
||||
|
||||
switch (type) {
|
||||
case repertory::download_type::direct: {
|
||||
closeable_file = std::make_shared<direct_open_file>(
|
||||
chunk_size, chunk_timeout, fsi, provider_);
|
||||
@ -490,7 +507,7 @@ auto file_manager::open(
|
||||
}
|
||||
|
||||
void file_manager::queue_upload(const i_open_file &file) {
|
||||
return queue_upload(file.get_api_path(), file.get_source_path(), false);
|
||||
queue_upload(file.get_api_path(), file.get_source_path(), false);
|
||||
}
|
||||
|
||||
void file_manager::queue_upload(const std::string &api_path,
|
||||
@ -499,9 +516,9 @@ void file_manager::queue_upload(const std::string &api_path,
|
||||
return;
|
||||
}
|
||||
|
||||
std::unique_ptr<mutex_lock> lock;
|
||||
std::unique_ptr<mutex_lock> upload_lock;
|
||||
if (not no_lock) {
|
||||
lock = std::make_unique<mutex_lock>(upload_mtx_);
|
||||
upload_lock = std::make_unique<mutex_lock>(upload_mtx_);
|
||||
}
|
||||
|
||||
remove_upload(api_path, true);
|
||||
@ -525,47 +542,35 @@ void file_manager::queue_upload(const std::string &api_path,
|
||||
auto file_manager::remove_file(const std::string &api_path) -> api_error {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
recur_mutex_lock open_lock(open_file_mtx_);
|
||||
|
||||
filesystem_item fsi{};
|
||||
auto res = provider_.get_filesystem_item(api_path, false, fsi);
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
|
||||
close_all(api_path);
|
||||
auto allocated = close_all(api_path);
|
||||
|
||||
mutex_lock lock(upload_mtx_);
|
||||
unique_mutex_lock upload_lock(upload_mtx_);
|
||||
remove_upload(api_path, true);
|
||||
remove_resume(api_path, fsi.source_path, true);
|
||||
upload_notify_.notify_all();
|
||||
upload_lock.unlock();
|
||||
|
||||
recur_mutex_lock open_lock(open_file_mtx_);
|
||||
|
||||
res = provider_.remove_file(api_path);
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
|
||||
auto file = utils::file::file{fsi.source_path};
|
||||
auto file_size = file.size().value_or(0U);
|
||||
if (file.remove()) {
|
||||
res = cache_size_mgr::instance().shrink(file_size);
|
||||
if (res != api_error::success) {
|
||||
utils::error::raise_api_path_error(function_name, api_path, res,
|
||||
"failed to shrink cache");
|
||||
}
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
utils::error::raise_api_path_error(
|
||||
function_name, fsi.api_path, fsi.source_path,
|
||||
utils::get_last_error_code(), "failed to delete source");
|
||||
|
||||
remove_source_and_shrink_cache(api_path, fsi.source_path, fsi.size,
|
||||
allocated);
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
void file_manager::remove_resume(const std::string &api_path,
|
||||
const std::string &source_path) {
|
||||
return remove_resume(api_path, source_path, false);
|
||||
remove_resume(api_path, source_path, false);
|
||||
}
|
||||
|
||||
void file_manager::remove_resume(const std::string &api_path,
|
||||
@ -574,9 +579,9 @@ void file_manager::remove_resume(const std::string &api_path,
|
||||
return;
|
||||
}
|
||||
|
||||
std::unique_ptr<mutex_lock> lock;
|
||||
std::unique_ptr<mutex_lock> upload_lock;
|
||||
if (not no_lock) {
|
||||
lock = std::make_unique<mutex_lock>(upload_mtx_);
|
||||
upload_lock = std::make_unique<mutex_lock>(upload_mtx_);
|
||||
}
|
||||
|
||||
if (mgr_db_->remove_resume(api_path)) {
|
||||
@ -589,6 +594,40 @@ void file_manager::remove_resume(const std::string &api_path,
|
||||
}
|
||||
}
|
||||
|
||||
auto file_manager::remove_source_and_shrink_cache(
|
||||
const std::string &api_path, const std::string &source_path,
|
||||
std::uint64_t file_size, bool allocated) -> bool {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
auto file = utils::file::file{source_path};
|
||||
auto source_size = file.exists() ? file.size().value_or(0U) : 0U;
|
||||
|
||||
if (not file.remove()) {
|
||||
utils::error::raise_api_path_error(function_name, api_path, source_path,
|
||||
utils::get_last_error_code(),
|
||||
"failed to delete source");
|
||||
return false;
|
||||
}
|
||||
|
||||
if (not allocated || source_size == 0U) {
|
||||
auto res = cache_size_mgr::instance().shrink(0U);
|
||||
if (res != api_error::success) {
|
||||
utils::error::raise_api_path_error(function_name, api_path, source_path,
|
||||
res, "failed to shrink cache");
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
auto res = cache_size_mgr::instance().shrink(file_size);
|
||||
if (res != api_error::success) {
|
||||
utils::error::raise_api_path_error(function_name, api_path, source_path,
|
||||
res, "failed to shrink cache");
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void file_manager::remove_upload(const std::string &api_path) {
|
||||
remove_upload(api_path, false);
|
||||
}
|
||||
@ -600,9 +639,9 @@ void file_manager::remove_upload(const std::string &api_path, bool no_lock) {
|
||||
return;
|
||||
}
|
||||
|
||||
std::unique_ptr<mutex_lock> lock;
|
||||
std::unique_ptr<mutex_lock> upload_lock;
|
||||
if (not no_lock) {
|
||||
lock = std::make_unique<mutex_lock>(upload_mtx_);
|
||||
upload_lock = std::make_unique<mutex_lock>(upload_mtx_);
|
||||
}
|
||||
|
||||
if (not mgr_db_->remove_upload(api_path)) {
|
||||
@ -868,7 +907,6 @@ void file_manager::stop() {
|
||||
|
||||
stop_requested_ = true;
|
||||
|
||||
polling::instance().remove_callback("db_cleanup");
|
||||
polling::instance().remove_callback("timed_out_close");
|
||||
|
||||
unique_mutex_lock upload_lock(upload_mtx_);
|
||||
@ -882,7 +920,7 @@ void file_manager::stop() {
|
||||
open_file_lookup_.clear();
|
||||
|
||||
upload_lock.lock();
|
||||
for (auto &&item : upload_lookup_) {
|
||||
for (auto &item : upload_lookup_) {
|
||||
item.second->stop();
|
||||
}
|
||||
upload_notify_.notify_all();
|
||||
@ -928,10 +966,10 @@ void file_manager::swap_renamed_items(std::string from_api_path,
|
||||
|
||||
auto file_iter = open_file_lookup_.find(from_api_path);
|
||||
if (file_iter != open_file_lookup_.end()) {
|
||||
auto ptr = std::move(open_file_lookup_[from_api_path]);
|
||||
auto closeable_file = std::move(open_file_lookup_[from_api_path]);
|
||||
open_file_lookup_.erase(from_api_path);
|
||||
ptr->set_api_path(to_api_path);
|
||||
open_file_lookup_[to_api_path] = std::move(ptr);
|
||||
closeable_file->set_api_path(to_api_path);
|
||||
open_file_lookup_[to_api_path] = std::move(closeable_file);
|
||||
}
|
||||
|
||||
if (directory) {
|
||||
|
@ -23,17 +23,15 @@
|
||||
|
||||
#include "file_manager/cache_size_mgr.hpp"
|
||||
#include "file_manager/events.hpp"
|
||||
#include "file_manager/file_manager.hpp"
|
||||
#include "file_manager/i_upload_manager.hpp"
|
||||
#include "platform/platform.hpp"
|
||||
#include "providers/i_provider.hpp"
|
||||
#include "types/repertory.hpp"
|
||||
#include "types/startup_exception.hpp"
|
||||
#include "utils/common.hpp"
|
||||
#include "utils/error_utils.hpp"
|
||||
#include "utils/file_utils.hpp"
|
||||
#include "utils/path.hpp"
|
||||
#include "utils/time.hpp"
|
||||
#include "utils/utils.hpp"
|
||||
|
||||
namespace repertory {
|
||||
open_file::open_file(std::uint64_t chunk_size, std::uint8_t chunk_timeout,
|
||||
@ -62,19 +60,23 @@ open_file::open_file(std::uint64_t chunk_size, std::uint8_t chunk_timeout,
|
||||
i_provider &provider,
|
||||
std::optional<boost::dynamic_bitset<>> read_state,
|
||||
i_upload_manager &mgr)
|
||||
: open_file_base(chunk_size, chunk_timeout, fsi, open_data, provider),
|
||||
: open_file_base(chunk_size, chunk_timeout, fsi, open_data, provider,
|
||||
false),
|
||||
mgr_(mgr) {
|
||||
if (fsi_.directory && read_state.has_value()) {
|
||||
throw startup_exception(
|
||||
fmt::format("cannot resume a directory|sp|", fsi.api_path));
|
||||
}
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
if (fsi.directory) {
|
||||
if (read_state.has_value()) {
|
||||
utils::error::raise_api_path_error(
|
||||
function_name, fsi.api_path, fsi.source_path,
|
||||
fmt::format("cannot resume a directory|sp|", fsi.api_path));
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
nf_ = utils::file::file::open_or_create_file(fsi.source_path,
|
||||
provider_.is_read_only());
|
||||
get_provider().is_read_only());
|
||||
set_api_error(*nf_ ? api_error::success : api_error::os_error);
|
||||
if (get_api_error() != api_error::success) {
|
||||
return;
|
||||
@ -83,25 +85,30 @@ open_file::open_file(std::uint64_t chunk_size, std::uint8_t chunk_timeout,
|
||||
if (read_state.has_value()) {
|
||||
read_state_ = read_state.value();
|
||||
set_modified();
|
||||
} else if (fsi_.size > 0U) {
|
||||
read_state_.resize(static_cast<std::size_t>(
|
||||
utils::divide_with_ceiling(fsi_.size, chunk_size)),
|
||||
false);
|
||||
allocated = true;
|
||||
return;
|
||||
}
|
||||
|
||||
auto file_size = nf_->size();
|
||||
if (provider_.is_read_only() || file_size.value() == fsi.size) {
|
||||
read_state_.set(0U, read_state_.size(), true);
|
||||
} else if (nf_->truncate(fsi.size)) {
|
||||
if (file_size.value() > fsi.size) {
|
||||
set_api_error(
|
||||
cache_size_mgr::instance().shrink(file_size.value() - fsi.size));
|
||||
} else {
|
||||
set_api_error(
|
||||
cache_size_mgr::instance().expand(fsi.size - file_size.value()));
|
||||
}
|
||||
} else {
|
||||
set_api_error(api_error::os_error);
|
||||
}
|
||||
if (fsi.size == 0U) {
|
||||
return;
|
||||
}
|
||||
|
||||
read_state_.resize(static_cast<std::size_t>(
|
||||
utils::divide_with_ceiling(fsi.size, chunk_size)),
|
||||
false);
|
||||
|
||||
auto file_size = nf_->size();
|
||||
if (not file_size.has_value()) {
|
||||
utils::error::raise_api_path_error(
|
||||
function_name, fsi.api_path, fsi.source_path,
|
||||
utils::get_last_error_code(), "failed to get file size");
|
||||
set_api_error(api_error::os_error);
|
||||
return;
|
||||
}
|
||||
|
||||
if (get_provider().is_read_only() || file_size.value() == fsi.size) {
|
||||
read_state_.set(0U, read_state_.size(), true);
|
||||
allocated = true;
|
||||
}
|
||||
|
||||
if (get_api_error() != api_error::success && *nf_) {
|
||||
@ -111,18 +118,93 @@ open_file::open_file(std::uint64_t chunk_size, std::uint8_t chunk_timeout,
|
||||
|
||||
open_file::~open_file() { close(); }
|
||||
|
||||
auto open_file::adjust_cache_size(std::uint64_t file_size,
|
||||
bool shrink) -> api_error {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
if (file_size == get_file_size()) {
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
if (file_size > get_file_size()) {
|
||||
auto size = file_size - get_file_size();
|
||||
auto res = shrink ? cache_size_mgr::instance().shrink(size)
|
||||
: cache_size_mgr::instance().expand(size);
|
||||
if (res == api_error::success) {
|
||||
return res;
|
||||
}
|
||||
|
||||
utils::error::raise_api_path_error(
|
||||
function_name, get_api_path(), get_source_path(), res,
|
||||
fmt::format("failed to {} cache|size|{}",
|
||||
(shrink ? "shrink" : "expand"), size));
|
||||
return set_api_error(res);
|
||||
}
|
||||
|
||||
auto size = get_file_size() - file_size;
|
||||
auto res = shrink ? cache_size_mgr::instance().expand(size)
|
||||
: cache_size_mgr::instance().shrink(size);
|
||||
if (res == api_error::success) {
|
||||
return res;
|
||||
}
|
||||
|
||||
utils::error::raise_api_path_error(
|
||||
function_name, get_api_path(), get_source_path(), res,
|
||||
fmt::format("failed to {} cache|size|{}", (shrink ? "expand" : "shrink"),
|
||||
size));
|
||||
return set_api_error(res);
|
||||
}
|
||||
|
||||
auto open_file::check_start() -> api_error {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
unique_recur_mutex_lock file_lock(get_mutex());
|
||||
if (allocated) {
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
auto file_size = nf_->size();
|
||||
if (not file_size.has_value()) {
|
||||
utils::error::raise_api_path_error(
|
||||
function_name, get_api_path(), get_source_path(),
|
||||
utils::get_last_error_code(), "failed to get file size");
|
||||
return set_api_error(api_error::os_error);
|
||||
}
|
||||
|
||||
if (file_size.value() == get_file_size()) {
|
||||
allocated = true;
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
file_lock.unlock();
|
||||
auto res = adjust_cache_size(file_size.value(), true);
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
file_lock.lock();
|
||||
|
||||
if (not nf_->truncate(get_file_size())) {
|
||||
utils::error::raise_api_path_error(
|
||||
function_name, get_api_path(), get_source_path(),
|
||||
utils::get_last_error_code(),
|
||||
fmt::format("failed to truncate file|size|{}", get_file_size()));
|
||||
return set_api_error(res);
|
||||
}
|
||||
|
||||
allocated = true;
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
auto open_file::close() -> bool {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
if (fsi_.directory || stop_requested_) {
|
||||
if (is_directory() || stop_requested_) {
|
||||
return false;
|
||||
}
|
||||
|
||||
stop_requested_ = true;
|
||||
|
||||
unique_mutex_lock reader_lock(io_thread_mtx_);
|
||||
io_thread_notify_.notify_all();
|
||||
reader_lock.unlock();
|
||||
notify_io();
|
||||
|
||||
if (reader_thread_) {
|
||||
reader_thread_->join();
|
||||
@ -133,12 +215,14 @@ auto open_file::close() -> bool {
|
||||
return false;
|
||||
}
|
||||
|
||||
auto read_state = get_read_state();
|
||||
auto err = get_api_error();
|
||||
if (err == api_error::success || err == api_error::download_incomplete ||
|
||||
err == api_error::download_stopped) {
|
||||
if (modified_ && not read_state_.all()) {
|
||||
if (is_modified() && not read_state.all()) {
|
||||
set_api_error(api_error::download_incomplete);
|
||||
} else if (not modified_ && (fsi_.size > 0U) && not read_state_.all()) {
|
||||
} else if (not is_modified() && (get_file_size() > 0U) &&
|
||||
not read_state.all()) {
|
||||
set_api_error(api_error::download_stopped);
|
||||
}
|
||||
|
||||
@ -147,7 +231,7 @@ auto open_file::close() -> bool {
|
||||
|
||||
nf_->close();
|
||||
|
||||
if (modified_) {
|
||||
if (is_modified()) {
|
||||
if (err == api_error::success) {
|
||||
mgr_.queue_upload(*this);
|
||||
return true;
|
||||
@ -159,37 +243,25 @@ auto open_file::close() -> bool {
|
||||
}
|
||||
}
|
||||
|
||||
if (err != api_error::success || read_state_.all()) {
|
||||
mgr_.remove_resume(fsi_.api_path, get_source_path());
|
||||
if (err != api_error::success || read_state.all()) {
|
||||
mgr_.remove_resume(get_api_path(), get_source_path());
|
||||
}
|
||||
|
||||
if (err == api_error::success) {
|
||||
return true;
|
||||
}
|
||||
|
||||
auto file = utils::file::file{fsi_.source_path};
|
||||
auto file_size = file.size().value_or(0U);
|
||||
if (file.remove()) {
|
||||
auto res = cache_size_mgr::instance().shrink(file_size);
|
||||
if (res != api_error::success) {
|
||||
utils::error::raise_api_path_error(function_name, fsi_.api_path,
|
||||
fsi_.source_path, res,
|
||||
"failed to shrink cache");
|
||||
}
|
||||
} else {
|
||||
utils::error::raise_api_path_error(
|
||||
function_name, fsi_.api_path, fsi_.source_path,
|
||||
utils::get_last_error_code(), "failed to delete source file");
|
||||
}
|
||||
file_manager::remove_source_and_shrink_cache(
|
||||
get_api_path(), get_source_path(), get_file_size(), allocated);
|
||||
|
||||
auto parent = utils::path::get_parent_path(fsi_.source_path);
|
||||
fsi_.source_path =
|
||||
utils::path::combine(parent, {utils::create_uuid_string()});
|
||||
auto res =
|
||||
provider_.set_item_meta(fsi_.api_path, META_SOURCE, fsi_.source_path);
|
||||
auto parent = utils::path::get_parent_path(get_source_path());
|
||||
set_source_path(utils::path::combine(parent, {utils::create_uuid_string()}));
|
||||
|
||||
auto res = get_provider().set_item_meta(get_api_path(), META_SOURCE,
|
||||
get_source_path());
|
||||
if (res != api_error::success) {
|
||||
utils::error::raise_api_path_error(function_name, fsi_.api_path,
|
||||
fsi_.source_path, res,
|
||||
utils::error::raise_api_path_error(function_name, get_api_path(),
|
||||
get_source_path(), res,
|
||||
"failed to set new source path");
|
||||
}
|
||||
|
||||
@ -202,34 +274,32 @@ void open_file::download_chunk(std::size_t chunk, bool skip_active,
|
||||
reset_timeout();
|
||||
}
|
||||
|
||||
unique_recur_mutex_lock download_lock(file_mtx_);
|
||||
if ((get_api_error() == api_error::success) && (chunk < read_state_.size()) &&
|
||||
not read_state_[chunk]) {
|
||||
if (active_downloads_.find(chunk) != active_downloads_.end()) {
|
||||
unique_recur_mutex_lock rw_lock(rw_mtx_);
|
||||
auto read_state = get_read_state();
|
||||
if ((get_api_error() == api_error::success) && (chunk < read_state.size()) &&
|
||||
not read_state[chunk]) {
|
||||
if (get_active_downloads().find(chunk) != get_active_downloads().end()) {
|
||||
if (skip_active) {
|
||||
return;
|
||||
}
|
||||
|
||||
auto active_download = active_downloads_.at(chunk);
|
||||
download_lock.unlock();
|
||||
auto active_download = get_active_downloads().at(chunk);
|
||||
rw_lock.unlock();
|
||||
|
||||
active_download->wait();
|
||||
return;
|
||||
}
|
||||
|
||||
auto data_offset = chunk * chunk_size_;
|
||||
auto data_size =
|
||||
(chunk == read_state_.size() - 1U) ? last_chunk_size_ : chunk_size_;
|
||||
if (active_downloads_.empty() && (read_state_.count() == 0U)) {
|
||||
event_system::instance().raise<download_begin>(fsi_.api_path,
|
||||
fsi_.source_path);
|
||||
auto data_offset = chunk * get_chunk_size();
|
||||
auto data_size = (chunk == read_state.size() - 1U) ? get_last_chunk_size()
|
||||
: get_chunk_size();
|
||||
if (get_active_downloads().empty() && (read_state.count() == 0U)) {
|
||||
event_system::instance().raise<download_begin>(get_api_path(),
|
||||
get_source_path());
|
||||
}
|
||||
event_system::instance().raise<download_chunk_begin>(
|
||||
fsi_.api_path, fsi_.source_path, chunk, read_state_.size(),
|
||||
read_state_.count());
|
||||
|
||||
active_downloads_[chunk] = std::make_shared<download>();
|
||||
download_lock.unlock();
|
||||
get_active_downloads()[chunk] = std::make_shared<download>();
|
||||
rw_lock.unlock();
|
||||
|
||||
if (should_reset) {
|
||||
reset_timeout();
|
||||
@ -238,28 +308,28 @@ void open_file::download_chunk(std::size_t chunk, bool skip_active,
|
||||
std::async(std::launch::async, [this, chunk, data_size, data_offset,
|
||||
should_reset]() {
|
||||
const auto notify_complete = [this, chunk, should_reset]() {
|
||||
unique_recur_mutex_lock file_lock(file_mtx_);
|
||||
auto active_download = active_downloads_.at(chunk);
|
||||
active_downloads_.erase(chunk);
|
||||
event_system::instance().raise<download_chunk_end>(
|
||||
fsi_.api_path, fsi_.source_path, chunk, read_state_.size(),
|
||||
read_state_.count(), get_api_error());
|
||||
auto state = get_read_state();
|
||||
|
||||
unique_recur_mutex_lock lock(rw_mtx_);
|
||||
auto active_download = get_active_downloads().at(chunk);
|
||||
get_active_downloads().erase(chunk);
|
||||
if (get_api_error() == api_error::success) {
|
||||
auto progress = (static_cast<double>(read_state_.count()) /
|
||||
static_cast<double>(read_state_.size()) * 100.0);
|
||||
auto progress = (static_cast<double>(state.count()) /
|
||||
static_cast<double>(state.size())) *
|
||||
100.0;
|
||||
event_system::instance().raise<download_progress>(
|
||||
fsi_.api_path, fsi_.source_path, progress);
|
||||
if (read_state_.all() && not notified_) {
|
||||
get_api_path(), get_source_path(), progress);
|
||||
if (state.all() && not notified_) {
|
||||
notified_ = true;
|
||||
event_system::instance().raise<download_end>(
|
||||
fsi_.api_path, fsi_.source_path, get_api_error());
|
||||
get_api_path(), get_source_path(), get_api_error());
|
||||
}
|
||||
} else if (not notified_) {
|
||||
notified_ = true;
|
||||
event_system::instance().raise<download_end>(
|
||||
fsi_.api_path, fsi_.source_path, get_api_error());
|
||||
get_api_path(), get_source_path(), get_api_error());
|
||||
}
|
||||
file_lock.unlock();
|
||||
lock.unlock();
|
||||
|
||||
active_download->notify(get_api_error());
|
||||
|
||||
@ -268,9 +338,9 @@ void open_file::download_chunk(std::size_t chunk, bool skip_active,
|
||||
}
|
||||
};
|
||||
|
||||
data_buffer data;
|
||||
auto res = provider_.read_file_bytes(get_api_path(), data_size,
|
||||
data_offset, data, stop_requested_);
|
||||
data_buffer buffer;
|
||||
auto res = get_provider().read_file_bytes(
|
||||
get_api_path(), data_size, data_offset, buffer, stop_requested_);
|
||||
if (res != api_error::success) {
|
||||
set_api_error(res);
|
||||
notify_complete();
|
||||
@ -283,7 +353,7 @@ void open_file::download_chunk(std::size_t chunk, bool skip_active,
|
||||
|
||||
res = do_io([&]() -> api_error {
|
||||
std::size_t bytes_written{};
|
||||
if (not nf_->write(data, data_offset, &bytes_written)) {
|
||||
if (not nf_->write(buffer, data_offset, &bytes_written)) {
|
||||
return api_error::os_error;
|
||||
}
|
||||
|
||||
@ -298,47 +368,50 @@ void open_file::download_chunk(std::size_t chunk, bool skip_active,
|
||||
return;
|
||||
}
|
||||
|
||||
unique_recur_mutex_lock file_lock(file_mtx_);
|
||||
read_state_.set(chunk);
|
||||
file_lock.unlock();
|
||||
set_read_state(chunk);
|
||||
|
||||
notify_complete();
|
||||
}).wait();
|
||||
}
|
||||
}
|
||||
|
||||
void open_file::download_range(std::size_t start_chunk, std::size_t end_chunk,
|
||||
void open_file::download_range(std::size_t begin_chunk, std::size_t end_chunk,
|
||||
bool should_reset) {
|
||||
for (std::size_t chunk = start_chunk;
|
||||
for (std::size_t chunk = begin_chunk;
|
||||
(get_api_error() == api_error::success) && (chunk <= end_chunk);
|
||||
++chunk) {
|
||||
download_chunk(chunk, false, should_reset);
|
||||
}
|
||||
}
|
||||
|
||||
auto open_file::get_allocated() const -> bool {
|
||||
recur_mutex_lock file_lock(get_mutex());
|
||||
return allocated;
|
||||
}
|
||||
|
||||
auto open_file::get_read_state() const -> boost::dynamic_bitset<> {
|
||||
recur_mutex_lock file_lock(file_mtx_);
|
||||
recur_mutex_lock file_lock(get_mutex());
|
||||
return read_state_;
|
||||
}
|
||||
|
||||
auto open_file::get_read_state(std::size_t chunk) const -> bool {
|
||||
recur_mutex_lock file_lock(file_mtx_);
|
||||
return read_state_[chunk];
|
||||
return get_read_state()[chunk];
|
||||
}
|
||||
|
||||
auto open_file::is_complete() const -> bool {
|
||||
recur_mutex_lock file_lock(file_mtx_);
|
||||
return read_state_.all();
|
||||
}
|
||||
auto open_file::is_complete() const -> bool { return get_read_state().all(); }
|
||||
|
||||
auto open_file::native_operation(
|
||||
i_open_file::native_operation_callback callback) -> api_error {
|
||||
unique_recur_mutex_lock file_lock(file_mtx_);
|
||||
if (stop_requested_) {
|
||||
return api_error::download_stopped;
|
||||
return set_api_error(api_error::download_stopped);
|
||||
}
|
||||
file_lock.unlock();
|
||||
|
||||
auto res = check_start();
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
|
||||
unique_recur_mutex_lock rw_lock(rw_mtx_);
|
||||
return do_io([&]() -> api_error { return callback(nf_->get_handle()); });
|
||||
}
|
||||
|
||||
@ -347,38 +420,48 @@ auto open_file::native_operation(
|
||||
i_open_file::native_operation_callback callback) -> api_error {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
if (fsi_.directory) {
|
||||
return api_error::invalid_operation;
|
||||
if (is_directory()) {
|
||||
return set_api_error(api_error::invalid_operation);
|
||||
}
|
||||
|
||||
unique_recur_mutex_lock file_lock(file_mtx_);
|
||||
if (stop_requested_) {
|
||||
return api_error::download_stopped;
|
||||
return set_api_error(api_error::download_stopped);
|
||||
}
|
||||
|
||||
auto res = check_start();
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
|
||||
res = adjust_cache_size(new_file_size, false);
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
file_lock.unlock();
|
||||
|
||||
auto is_empty_file = new_file_size == 0U;
|
||||
auto last_chunk = is_empty_file
|
||||
? std::size_t(0U)
|
||||
: static_cast<std::size_t>(utils::divide_with_ceiling(
|
||||
new_file_size, chunk_size_)) -
|
||||
new_file_size, get_chunk_size())) -
|
||||
1U;
|
||||
|
||||
file_lock.lock();
|
||||
if (not is_empty_file && (last_chunk < read_state_.size())) {
|
||||
file_lock.unlock();
|
||||
update_background_reader(0U);
|
||||
unique_recur_mutex_lock rw_lock(rw_mtx_);
|
||||
auto read_state = get_read_state();
|
||||
if (not is_empty_file && (last_chunk < read_state.size())) {
|
||||
rw_lock.unlock();
|
||||
update_reader(0U);
|
||||
|
||||
download_chunk(last_chunk, false, true);
|
||||
if (get_api_error() != api_error::success) {
|
||||
return get_api_error();
|
||||
}
|
||||
file_lock.lock();
|
||||
rw_lock.lock();
|
||||
}
|
||||
|
||||
read_state = get_read_state();
|
||||
auto original_file_size = get_file_size();
|
||||
|
||||
auto res = do_io([&]() -> api_error { return callback(nf_->get_handle()); });
|
||||
res = do_io([&]() -> api_error { return callback(nf_->get_handle()); });
|
||||
if (res != api_error::success) {
|
||||
utils::error::raise_api_path_error(function_name, get_api_path(),
|
||||
utils::get_last_error_code(),
|
||||
@ -387,59 +470,73 @@ auto open_file::native_operation(
|
||||
}
|
||||
|
||||
{
|
||||
auto file_size = nf_->size().value_or(0U);
|
||||
if (file_size != new_file_size) {
|
||||
auto file_size = nf_->size();
|
||||
if (not file_size.has_value()) {
|
||||
utils::error::raise_api_path_error(
|
||||
function_name, get_api_path(), api_error::file_size_mismatch,
|
||||
"allocated file size mismatch|expected|" +
|
||||
std::to_string(new_file_size) + "|actual|" +
|
||||
std::to_string(file_size));
|
||||
fmt::format("failed to get file size|error|{}",
|
||||
utils::get_last_error_code()));
|
||||
return set_api_error(api_error::error);
|
||||
}
|
||||
|
||||
if (file_size.value() != new_file_size) {
|
||||
utils::error::raise_api_path_error(
|
||||
function_name, get_api_path(), api_error::file_size_mismatch,
|
||||
fmt::format("file size mismatch|expected|{}|actual|{}", new_file_size,
|
||||
file_size.value()));
|
||||
return set_api_error(api_error::error);
|
||||
}
|
||||
}
|
||||
|
||||
if (is_empty_file || (read_state_.size() != (last_chunk + 1U))) {
|
||||
auto old_size = read_state_.size();
|
||||
read_state_.resize(is_empty_file ? 0U : last_chunk + 1U);
|
||||
if (is_empty_file || (read_state.size() != (last_chunk + 1U))) {
|
||||
auto old_size = read_state.size();
|
||||
read_state.resize(is_empty_file ? 0U : last_chunk + 1U);
|
||||
|
||||
if (not is_empty_file) {
|
||||
for (std::size_t chunk = old_size; chunk <= last_chunk; ++chunk) {
|
||||
read_state_.set(chunk);
|
||||
read_state.set(chunk);
|
||||
}
|
||||
}
|
||||
set_read_state(read_state);
|
||||
|
||||
last_chunk_size_ = static_cast<std::size_t>(
|
||||
new_file_size <= chunk_size_ ? new_file_size
|
||||
: (new_file_size % chunk_size_) == 0U ? chunk_size_
|
||||
: new_file_size % chunk_size_);
|
||||
set_last_chunk_size(static_cast<std::size_t>(
|
||||
new_file_size <= get_chunk_size() ? new_file_size
|
||||
: (new_file_size % get_chunk_size()) == 0U
|
||||
? get_chunk_size()
|
||||
: new_file_size % get_chunk_size()));
|
||||
}
|
||||
|
||||
if (original_file_size != new_file_size) {
|
||||
set_modified();
|
||||
if (original_file_size == new_file_size) {
|
||||
return res;
|
||||
}
|
||||
set_modified();
|
||||
|
||||
fsi_.size = new_file_size;
|
||||
auto now = std::to_string(utils::time::get_time_now());
|
||||
res = provider_.set_item_meta(
|
||||
fsi_.api_path, {
|
||||
{META_CHANGED, now},
|
||||
{META_MODIFIED, now},
|
||||
{META_SIZE, std::to_string(new_file_size)},
|
||||
{META_WRITTEN, now},
|
||||
});
|
||||
if (res != api_error::success) {
|
||||
utils::error::raise_api_path_error(function_name, get_api_path(), res,
|
||||
"failed to set file meta");
|
||||
return set_api_error(res);
|
||||
}
|
||||
set_file_size(new_file_size);
|
||||
auto now = std::to_string(utils::time::get_time_now());
|
||||
res = get_provider().set_item_meta(
|
||||
get_api_path(), {
|
||||
{META_CHANGED, now},
|
||||
{META_MODIFIED, now},
|
||||
{META_SIZE, std::to_string(new_file_size)},
|
||||
{META_WRITTEN, now},
|
||||
});
|
||||
if (res == api_error::success) {
|
||||
return res;
|
||||
}
|
||||
|
||||
return res;
|
||||
utils::error::raise_api_path_error(function_name, get_api_path(), res,
|
||||
"failed to set file meta");
|
||||
return set_api_error(res);
|
||||
}
|
||||
|
||||
auto open_file::read(std::size_t read_size, std::uint64_t read_offset,
|
||||
data_buffer &data) -> api_error {
|
||||
if (fsi_.directory) {
|
||||
return api_error::invalid_operation;
|
||||
if (is_directory()) {
|
||||
return set_api_error(api_error::invalid_operation);
|
||||
}
|
||||
|
||||
if (stop_requested_) {
|
||||
return set_api_error(api_error::download_stopped);
|
||||
}
|
||||
|
||||
read_size =
|
||||
@ -448,12 +545,17 @@ auto open_file::read(std::size_t read_size, std::uint64_t read_offset,
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
auto res = check_start();
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
|
||||
const auto read_from_source = [this, &data, &read_offset,
|
||||
&read_size]() -> api_error {
|
||||
return do_io([this, &data, &read_offset, &read_size]() -> api_error {
|
||||
if (provider_.is_read_only()) {
|
||||
return provider_.read_file_bytes(fsi_.api_path, read_size, read_offset,
|
||||
data, stop_requested_);
|
||||
if (get_provider().is_read_only()) {
|
||||
return get_provider().read_file_bytes(
|
||||
get_api_path(), read_size, read_offset, data, stop_requested_);
|
||||
}
|
||||
|
||||
data.resize(read_size);
|
||||
@ -464,49 +566,48 @@ auto open_file::read(std::size_t read_size, std::uint64_t read_offset,
|
||||
});
|
||||
};
|
||||
|
||||
unique_recur_mutex_lock file_lock(file_mtx_);
|
||||
if (read_state_.all()) {
|
||||
if (get_read_state().all()) {
|
||||
reset_timeout();
|
||||
return read_from_source();
|
||||
}
|
||||
file_lock.unlock();
|
||||
|
||||
auto start_chunk = static_cast<std::size_t>(read_offset / chunk_size_);
|
||||
auto begin_chunk = static_cast<std::size_t>(read_offset / get_chunk_size());
|
||||
auto end_chunk =
|
||||
static_cast<std::size_t>((read_size + read_offset) / chunk_size_);
|
||||
static_cast<std::size_t>((read_size + read_offset) / get_chunk_size());
|
||||
|
||||
update_background_reader(start_chunk);
|
||||
update_reader(begin_chunk);
|
||||
|
||||
download_range(start_chunk, end_chunk, true);
|
||||
download_range(begin_chunk, end_chunk, true);
|
||||
if (get_api_error() != api_error::success) {
|
||||
return get_api_error();
|
||||
}
|
||||
|
||||
file_lock.lock();
|
||||
unique_recur_mutex_lock rw_lock(rw_mtx_);
|
||||
return get_api_error() == api_error::success ? read_from_source()
|
||||
: get_api_error();
|
||||
}
|
||||
|
||||
void open_file::remove(std::uint64_t handle) {
|
||||
recur_mutex_lock file_lock(file_mtx_);
|
||||
open_file_base::remove(handle);
|
||||
if (modified_ && read_state_.all() &&
|
||||
|
||||
recur_mutex_lock rw_lock(rw_mtx_);
|
||||
if (is_modified() && get_read_state().all() &&
|
||||
(get_api_error() == api_error::success)) {
|
||||
mgr_.queue_upload(*this);
|
||||
modified_ = false;
|
||||
open_file_base::set_modified(false);
|
||||
}
|
||||
|
||||
if (removed_ && (get_open_file_count() == 0U)) {
|
||||
removed_ = false;
|
||||
if (is_removed() && (get_open_file_count() == 0U)) {
|
||||
open_file_base::set_removed(false);
|
||||
}
|
||||
}
|
||||
|
||||
void open_file::remove_all() {
|
||||
recur_mutex_lock file_lock(file_mtx_);
|
||||
open_file_base::remove_all();
|
||||
|
||||
modified_ = false;
|
||||
removed_ = true;
|
||||
recur_mutex_lock rw_lock(rw_mtx_);
|
||||
open_file_base::set_modified(false);
|
||||
open_file_base::set_removed(true);
|
||||
|
||||
mgr_.remove_upload(get_api_path());
|
||||
|
||||
@ -514,26 +615,14 @@ void open_file::remove_all() {
|
||||
}
|
||||
|
||||
auto open_file::resize(std::uint64_t new_file_size) -> api_error {
|
||||
if (fsi_.directory) {
|
||||
return api_error::invalid_operation;
|
||||
if (is_directory()) {
|
||||
return set_api_error(api_error::invalid_operation);
|
||||
}
|
||||
|
||||
if (new_file_size == fsi_.size) {
|
||||
if (new_file_size == get_file_size()) {
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
if (new_file_size > fsi_.size) {
|
||||
auto res = cache_size_mgr::instance().expand(new_file_size - fsi_.size);
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
} else {
|
||||
auto res = cache_size_mgr::instance().shrink(fsi_.size - new_file_size);
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
}
|
||||
|
||||
return native_operation(
|
||||
new_file_size, [this, &new_file_size](native_handle) -> api_error {
|
||||
return nf_->truncate(new_file_size) ? api_error::success
|
||||
@ -542,48 +631,58 @@ auto open_file::resize(std::uint64_t new_file_size) -> api_error {
|
||||
}
|
||||
|
||||
void open_file::set_modified() {
|
||||
if (not modified_) {
|
||||
modified_ = true;
|
||||
if (not is_modified()) {
|
||||
open_file_base::set_modified(true);
|
||||
mgr_.store_resume(*this);
|
||||
}
|
||||
|
||||
if (not removed_) {
|
||||
removed_ = true;
|
||||
if (not is_removed()) {
|
||||
open_file_base::set_removed(true);
|
||||
mgr_.remove_upload(get_api_path());
|
||||
}
|
||||
}
|
||||
|
||||
void open_file::update_background_reader(std::size_t read_chunk) {
|
||||
recur_mutex_lock reader_lock(file_mtx_);
|
||||
read_chunk_ = read_chunk;
|
||||
void open_file::set_read_state(std::size_t chunk) {
|
||||
recur_mutex_lock file_lock(get_mutex());
|
||||
read_state_.set(chunk);
|
||||
}
|
||||
|
||||
void open_file::set_read_state(boost::dynamic_bitset<> read_state) {
|
||||
recur_mutex_lock file_lock(get_mutex());
|
||||
read_state_ = std::move(read_state);
|
||||
}
|
||||
|
||||
void open_file::update_reader(std::size_t chunk) {
|
||||
recur_mutex_lock rw_lock(rw_mtx_);
|
||||
read_chunk_ = chunk;
|
||||
|
||||
if (reader_thread_ || stop_requested_) {
|
||||
return;
|
||||
}
|
||||
|
||||
reader_thread_ = std::make_unique<std::thread>([this]() {
|
||||
std::size_t next_chunk{};
|
||||
while (not stop_requested_) {
|
||||
unique_recur_mutex_lock file_lock(file_mtx_);
|
||||
if ((fsi_.size == 0U) || read_state_.all()) {
|
||||
file_lock.unlock();
|
||||
unique_recur_mutex_lock lock(rw_mtx_);
|
||||
auto next_chunk{read_chunk_};
|
||||
auto read_chunk{read_chunk_};
|
||||
lock.unlock();
|
||||
|
||||
unique_mutex_lock io_lock(io_thread_mtx_);
|
||||
if (not stop_requested_ && io_thread_queue_.empty()) {
|
||||
io_thread_notify_.wait(io_lock);
|
||||
}
|
||||
io_thread_notify_.notify_all();
|
||||
io_lock.unlock();
|
||||
while (not stop_requested_) {
|
||||
lock.lock();
|
||||
|
||||
auto read_state = get_read_state();
|
||||
if ((get_file_size() == 0U) || read_state.all()) {
|
||||
lock.unlock();
|
||||
wait_for_io(stop_requested_);
|
||||
continue;
|
||||
}
|
||||
|
||||
do {
|
||||
next_chunk = read_chunk_ =
|
||||
((read_chunk_ + 1U) >= read_state_.size()) ? 0U : read_chunk_ + 1U;
|
||||
} while ((next_chunk != 0U) &&
|
||||
(active_downloads_.find(next_chunk) != active_downloads_.end()));
|
||||
if (read_chunk != read_chunk_) {
|
||||
next_chunk = read_chunk = read_chunk_;
|
||||
}
|
||||
|
||||
next_chunk = next_chunk + 1U >= read_state.size() ? 0U : next_chunk + 1U;
|
||||
lock.unlock();
|
||||
|
||||
file_lock.unlock();
|
||||
download_chunk(next_chunk, true, false);
|
||||
}
|
||||
});
|
||||
@ -595,41 +694,44 @@ auto open_file::write(std::uint64_t write_offset, const data_buffer &data,
|
||||
|
||||
bytes_written = 0U;
|
||||
|
||||
if (fsi_.directory || provider_.is_read_only()) {
|
||||
return api_error::invalid_operation;
|
||||
if (is_directory() || get_provider().is_read_only()) {
|
||||
return set_api_error(api_error::invalid_operation);
|
||||
}
|
||||
|
||||
if (data.empty()) {
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
unique_recur_mutex_lock write_lock(file_mtx_);
|
||||
if (stop_requested_) {
|
||||
return api_error::download_stopped;
|
||||
return set_api_error(api_error::download_stopped);
|
||||
}
|
||||
write_lock.unlock();
|
||||
|
||||
auto start_chunk = static_cast<std::size_t>(write_offset / chunk_size_);
|
||||
auto res = check_start();
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
|
||||
auto begin_chunk = static_cast<std::size_t>(write_offset / get_chunk_size());
|
||||
auto end_chunk =
|
||||
static_cast<std::size_t>((write_offset + data.size()) / chunk_size_);
|
||||
static_cast<std::size_t>((write_offset + data.size()) / get_chunk_size());
|
||||
|
||||
update_background_reader(start_chunk);
|
||||
update_reader(begin_chunk);
|
||||
|
||||
download_range(start_chunk, std::min(read_state_.size() - 1U, end_chunk),
|
||||
download_range(begin_chunk, std::min(get_read_state().size() - 1U, end_chunk),
|
||||
true);
|
||||
if (get_api_error() != api_error::success) {
|
||||
return get_api_error();
|
||||
}
|
||||
|
||||
write_lock.lock();
|
||||
if ((write_offset + data.size()) > fsi_.size) {
|
||||
auto res = resize(write_offset + data.size());
|
||||
unique_recur_mutex_lock rw_lock(rw_mtx_);
|
||||
if ((write_offset + data.size()) > get_file_size()) {
|
||||
res = resize(write_offset + data.size());
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
}
|
||||
|
||||
auto res = do_io([&]() -> api_error {
|
||||
res = do_io([&]() -> api_error {
|
||||
if (not nf_->write(data, write_offset, &bytes_written)) {
|
||||
return api_error::os_error;
|
||||
}
|
||||
@ -642,11 +744,11 @@ auto open_file::write(std::uint64_t write_offset, const data_buffer &data,
|
||||
}
|
||||
|
||||
auto now = std::to_string(utils::time::get_time_now());
|
||||
res = provider_.set_item_meta(fsi_.api_path, {
|
||||
{META_CHANGED, now},
|
||||
{META_MODIFIED, now},
|
||||
{META_WRITTEN, now},
|
||||
});
|
||||
res = get_provider().set_item_meta(get_api_path(), {
|
||||
{META_CHANGED, now},
|
||||
{META_MODIFIED, now},
|
||||
{META_WRITTEN, now},
|
||||
});
|
||||
if (res != api_error::success) {
|
||||
utils::error::raise_api_path_error(function_name, get_api_path(), res,
|
||||
"failed to set file meta");
|
||||
|
@ -119,7 +119,7 @@ auto open_file_base::can_close() const -> bool {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (is_download_complete()) {
|
||||
if (is_complete()) {
|
||||
return true;
|
||||
}
|
||||
|
||||
@ -127,12 +127,30 @@ auto open_file_base::can_close() const -> bool {
|
||||
return true;
|
||||
}
|
||||
|
||||
const std::chrono::system_clock::time_point last_access = last_access_;
|
||||
const auto duration = std::chrono::duration_cast<std::chrono::seconds>(
|
||||
std::chrono::system_clock::time_point last_access{last_access_};
|
||||
auto duration = std::chrono::duration_cast<std::chrono::seconds>(
|
||||
std::chrono::system_clock::now() - last_access);
|
||||
return (duration.count() >= chunk_timeout_);
|
||||
}
|
||||
|
||||
auto open_file_base::close() -> bool {
|
||||
unique_mutex_lock io_lock(io_thread_mtx_);
|
||||
if (io_stop_requested_ || not io_thread_) {
|
||||
io_thread_notify_.notify_all();
|
||||
io_lock.unlock();
|
||||
return false;
|
||||
}
|
||||
|
||||
io_stop_requested_ = true;
|
||||
io_thread_notify_.notify_all();
|
||||
io_lock.unlock();
|
||||
|
||||
io_thread_->join();
|
||||
io_thread_.reset();
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
auto open_file_base::do_io(std::function<api_error()> action) -> api_error {
|
||||
unique_mutex_lock io_lock(io_thread_mtx_);
|
||||
auto item = std::make_shared<io_item>(action);
|
||||
@ -191,6 +209,36 @@ auto open_file_base::get_file_size() const -> std::uint64_t {
|
||||
return fsi_.size;
|
||||
}
|
||||
|
||||
[[nodiscard]] auto open_file_base::get_last_chunk_size() const -> std::size_t {
|
||||
recur_mutex_lock file_lock(file_mtx_);
|
||||
return last_chunk_size_;
|
||||
}
|
||||
|
||||
void open_file_base::set_file_size(std::uint64_t size) {
|
||||
recur_mutex_lock file_lock(file_mtx_);
|
||||
fsi_.size = size;
|
||||
}
|
||||
|
||||
void open_file_base::set_last_chunk_size(std::size_t size) {
|
||||
recur_mutex_lock file_lock(file_mtx_);
|
||||
last_chunk_size_ = size;
|
||||
}
|
||||
|
||||
void open_file_base::set_modified(bool modified) {
|
||||
recur_mutex_lock file_lock(file_mtx_);
|
||||
modified_ = modified;
|
||||
}
|
||||
|
||||
void open_file_base::set_removed(bool removed) {
|
||||
recur_mutex_lock file_lock(file_mtx_);
|
||||
removed_ = removed;
|
||||
}
|
||||
|
||||
void open_file_base::set_source_path(std::string source_path) {
|
||||
recur_mutex_lock file_lock(file_mtx_);
|
||||
fsi_.source_path = std::move(source_path);
|
||||
}
|
||||
|
||||
auto open_file_base::get_filesystem_item() const -> filesystem_item {
|
||||
recur_mutex_lock file_lock(file_mtx_);
|
||||
return fsi_;
|
||||
@ -198,6 +246,7 @@ auto open_file_base::get_filesystem_item() const -> filesystem_item {
|
||||
|
||||
auto open_file_base::get_handles() const -> std::vector<std::uint64_t> {
|
||||
recur_mutex_lock file_lock(file_mtx_);
|
||||
|
||||
std::vector<std::uint64_t> ret;
|
||||
for (const auto &item : open_data_) {
|
||||
ret.emplace_back(item.first);
|
||||
@ -234,11 +283,31 @@ auto open_file_base::get_open_file_count() const -> std::size_t {
|
||||
return open_data_.size();
|
||||
}
|
||||
|
||||
auto open_file_base::get_source_path() const -> std::string {
|
||||
recur_mutex_lock file_lock(file_mtx_);
|
||||
return fsi_.source_path;
|
||||
}
|
||||
|
||||
auto open_file_base::has_handle(std::uint64_t handle) const -> bool {
|
||||
recur_mutex_lock file_lock(file_mtx_);
|
||||
return open_data_.find(handle) != open_data_.end();
|
||||
}
|
||||
|
||||
auto open_file_base::is_modified() const -> bool {
|
||||
recur_mutex_lock file_lock(file_mtx_);
|
||||
return modified_;
|
||||
}
|
||||
|
||||
auto open_file_base::is_removed() const -> bool {
|
||||
recur_mutex_lock file_lock(file_mtx_);
|
||||
return removed_;
|
||||
}
|
||||
|
||||
void open_file_base::notify_io() {
|
||||
mutex_lock io_lock(io_thread_mtx_);
|
||||
io_thread_notify_.notify_all();
|
||||
}
|
||||
|
||||
void open_file_base::remove(std::uint64_t handle) {
|
||||
recur_mutex_lock file_lock(file_mtx_);
|
||||
if (open_data_.find(handle) == open_data_.end()) {
|
||||
@ -297,21 +366,12 @@ void open_file_base::set_api_path(const std::string &api_path) {
|
||||
fsi_.api_parent = utils::path::get_parent_api_path(api_path);
|
||||
}
|
||||
|
||||
auto open_file_base::close() -> bool {
|
||||
void open_file_base::wait_for_io(stop_type &stop_requested) {
|
||||
unique_mutex_lock io_lock(io_thread_mtx_);
|
||||
if (io_stop_requested_ || not io_thread_) {
|
||||
io_thread_notify_.notify_all();
|
||||
io_lock.unlock();
|
||||
return false;
|
||||
if (not stop_requested && io_thread_queue_.empty()) {
|
||||
io_thread_notify_.wait(io_lock);
|
||||
}
|
||||
|
||||
io_stop_requested_ = true;
|
||||
io_thread_notify_.notify_all();
|
||||
io_lock.unlock();
|
||||
|
||||
io_thread_->join();
|
||||
io_thread_.reset();
|
||||
|
||||
return true;
|
||||
}
|
||||
} // namespace repertory
|
||||
|
367
repertory/librepertory/src/file_manager/ring_buffer_base.cpp
Normal file
367
repertory/librepertory/src/file_manager/ring_buffer_base.cpp
Normal file
@ -0,0 +1,367 @@
|
||||
/*
|
||||
Copyright <2018-2024> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
*/
|
||||
#include "file_manager/ring_buffer_base.hpp"
|
||||
|
||||
#include "events/event_system.hpp"
|
||||
#include "file_manager/events.hpp"
|
||||
#include "file_manager/open_file_base.hpp"
|
||||
#include "platform/platform.hpp"
|
||||
#include "providers/i_provider.hpp"
|
||||
#include "types/repertory.hpp"
|
||||
#include "utils/common.hpp"
|
||||
#include "utils/error_utils.hpp"
|
||||
|
||||
namespace repertory {
|
||||
ring_buffer_base::ring_buffer_base(std::uint64_t chunk_size,
|
||||
std::uint8_t chunk_timeout,
|
||||
filesystem_item fsi, i_provider &provider,
|
||||
std::size_t ring_size, bool disable_io)
|
||||
: open_file_base(chunk_size, chunk_timeout, fsi, provider, disable_io),
|
||||
read_state_(ring_size),
|
||||
total_chunks_(static_cast<std::size_t>(
|
||||
utils::divide_with_ceiling(fsi.size, chunk_size))) {
|
||||
if (disable_io) {
|
||||
if (fsi.size > 0U) {
|
||||
read_state_.resize(std::min(total_chunks_, read_state_.size()));
|
||||
|
||||
ring_end_ =
|
||||
std::min(total_chunks_ - 1U, ring_begin_ + read_state_.size() - 1U);
|
||||
read_state_.set(0U, read_state_.size(), false);
|
||||
}
|
||||
} else {
|
||||
if (ring_size < min_ring_size) {
|
||||
throw std::runtime_error("ring size must be greater than or equal to 5");
|
||||
}
|
||||
|
||||
ring_end_ = std::min(total_chunks_ - 1U, ring_begin_ + ring_size - 1U);
|
||||
read_state_.set(0U, ring_size, false);
|
||||
}
|
||||
}
|
||||
|
||||
auto ring_buffer_base::check_start() -> api_error {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
try {
|
||||
if (on_check_start()) {
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
event_system::instance().raise<download_begin>(get_api_path(),
|
||||
get_source_path());
|
||||
reader_thread_ =
|
||||
std::make_unique<std::thread>([this]() { reader_thread(); });
|
||||
return api_error::success;
|
||||
} catch (const std::exception &ex) {
|
||||
utils::error::raise_api_path_error(function_name, get_api_path(),
|
||||
get_source_path(), ex,
|
||||
"failed to start");
|
||||
return api_error::error;
|
||||
}
|
||||
}
|
||||
|
||||
auto ring_buffer_base::close() -> bool {
|
||||
stop_requested_ = true;
|
||||
|
||||
unique_mutex_lock chunk_lock(chunk_mtx_);
|
||||
chunk_notify_.notify_all();
|
||||
chunk_lock.unlock();
|
||||
|
||||
auto res = open_file_base::close();
|
||||
|
||||
if (reader_thread_) {
|
||||
reader_thread_->join();
|
||||
reader_thread_.reset();
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
auto ring_buffer_base::download_chunk(std::size_t chunk,
|
||||
bool skip_active) -> api_error {
|
||||
unique_mutex_lock chunk_lock(chunk_mtx_);
|
||||
const auto unlock_and_notify = [this, &chunk_lock]() {
|
||||
chunk_notify_.notify_all();
|
||||
chunk_lock.unlock();
|
||||
};
|
||||
|
||||
const auto unlock_and_return =
|
||||
[&unlock_and_notify](api_error res) -> api_error {
|
||||
unlock_and_notify();
|
||||
return res;
|
||||
};
|
||||
|
||||
if (chunk < ring_begin_ || chunk > ring_end_) {
|
||||
return unlock_and_return(api_error::invalid_ring_buffer_position);
|
||||
}
|
||||
|
||||
if (get_active_downloads().find(chunk) != get_active_downloads().end()) {
|
||||
if (skip_active) {
|
||||
return unlock_and_return(api_error::success);
|
||||
}
|
||||
|
||||
auto active_download = get_active_downloads().at(chunk);
|
||||
unlock_and_notify();
|
||||
|
||||
return active_download->wait();
|
||||
}
|
||||
|
||||
if (read_state_[chunk % read_state_.size()]) {
|
||||
return unlock_and_return(api_error::success);
|
||||
}
|
||||
|
||||
auto active_download{std::make_shared<download>()};
|
||||
get_active_downloads()[chunk] = active_download;
|
||||
|
||||
return use_buffer(chunk, [&](data_buffer &buffer) -> api_error {
|
||||
auto data_offset{chunk * get_chunk_size()};
|
||||
auto data_size{
|
||||
chunk == (total_chunks_ - 1U) ? get_last_chunk_size()
|
||||
: get_chunk_size(),
|
||||
};
|
||||
unlock_and_notify();
|
||||
|
||||
auto result{
|
||||
get_provider().read_file_bytes(get_api_path(), data_size, data_offset,
|
||||
buffer, stop_requested_),
|
||||
};
|
||||
|
||||
chunk_lock.lock();
|
||||
if (chunk < ring_begin_ || chunk > ring_end_) {
|
||||
result = api_error::invalid_ring_buffer_position;
|
||||
}
|
||||
|
||||
if (result == api_error::success) {
|
||||
result = on_chunk_downloaded(chunk, buffer);
|
||||
if (result == api_error::success) {
|
||||
read_state_[chunk % read_state_.size()] = true;
|
||||
auto progress = (static_cast<double>(chunk + 1U) /
|
||||
static_cast<double>(total_chunks_)) *
|
||||
100.0;
|
||||
event_system::instance().raise<download_progress>(
|
||||
get_api_path(), get_source_path(), progress);
|
||||
}
|
||||
}
|
||||
|
||||
get_active_downloads().erase(chunk);
|
||||
unlock_and_notify();
|
||||
|
||||
active_download->notify(result);
|
||||
return result;
|
||||
});
|
||||
}
|
||||
|
||||
void ring_buffer_base::forward(std::size_t count) {
|
||||
update_position(count, true);
|
||||
}
|
||||
|
||||
auto ring_buffer_base::get_read_state() const -> boost::dynamic_bitset<> {
|
||||
recur_mutex_lock file_lock(get_mutex());
|
||||
return read_state_;
|
||||
}
|
||||
|
||||
auto ring_buffer_base::get_read_state(std::size_t chunk) const -> bool {
|
||||
recur_mutex_lock file_lock(get_mutex());
|
||||
return read_state_[chunk % read_state_.size()];
|
||||
}
|
||||
|
||||
auto ring_buffer_base::read(std::size_t read_size, std::uint64_t read_offset,
|
||||
data_buffer &data) -> api_error {
|
||||
if (is_directory()) {
|
||||
return api_error::invalid_operation;
|
||||
}
|
||||
|
||||
reset_timeout();
|
||||
|
||||
read_size =
|
||||
utils::calculate_read_size(get_file_size(), read_size, read_offset);
|
||||
if (read_size == 0U) {
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
auto begin_chunk{static_cast<std::size_t>(read_offset / get_chunk_size())};
|
||||
read_offset = read_offset - (begin_chunk * get_chunk_size());
|
||||
|
||||
unique_mutex_lock read_lock(read_mtx_);
|
||||
auto res = check_start();
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
|
||||
for (std::size_t chunk = begin_chunk;
|
||||
not stop_requested_ && (res == api_error::success) && (read_size > 0U);
|
||||
++chunk) {
|
||||
reset_timeout();
|
||||
|
||||
if (chunk > ring_pos_) {
|
||||
forward(chunk - ring_pos_);
|
||||
} else if (chunk < ring_pos_) {
|
||||
reverse(ring_pos_ - chunk);
|
||||
}
|
||||
|
||||
res = download_chunk(chunk, false);
|
||||
if (res != api_error::success) {
|
||||
if (res == api_error::invalid_ring_buffer_position) {
|
||||
read_lock.unlock();
|
||||
|
||||
// TODO limit retry
|
||||
return read(read_size, read_offset, data);
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
reset_timeout();
|
||||
|
||||
std::size_t bytes_read{};
|
||||
res = on_read_chunk(
|
||||
chunk,
|
||||
std::min(static_cast<std::size_t>(get_chunk_size() - read_offset),
|
||||
read_size),
|
||||
read_offset, data, bytes_read);
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
|
||||
reset_timeout();
|
||||
|
||||
read_size -= bytes_read;
|
||||
read_offset = 0U;
|
||||
}
|
||||
|
||||
return stop_requested_ ? api_error::download_stopped : res;
|
||||
}
|
||||
|
||||
void ring_buffer_base::reader_thread() {
|
||||
unique_mutex_lock chunk_lock(chunk_mtx_);
|
||||
auto next_chunk{ring_pos_};
|
||||
chunk_notify_.notify_all();
|
||||
chunk_lock.unlock();
|
||||
|
||||
while (not stop_requested_) {
|
||||
chunk_lock.lock();
|
||||
|
||||
next_chunk = next_chunk + 1U > ring_end_ ? ring_begin_ : next_chunk + 1U;
|
||||
const auto check_and_wait = [this, &chunk_lock, &next_chunk]() {
|
||||
if (stop_requested_) {
|
||||
chunk_notify_.notify_all();
|
||||
chunk_lock.unlock();
|
||||
return;
|
||||
}
|
||||
|
||||
if (get_read_state().all()) {
|
||||
chunk_notify_.wait(chunk_lock);
|
||||
next_chunk = ring_pos_;
|
||||
}
|
||||
|
||||
chunk_notify_.notify_all();
|
||||
chunk_lock.unlock();
|
||||
};
|
||||
|
||||
if (read_state_[next_chunk % read_state_.size()]) {
|
||||
check_and_wait();
|
||||
continue;
|
||||
}
|
||||
|
||||
chunk_notify_.notify_all();
|
||||
chunk_lock.unlock();
|
||||
|
||||
download_chunk(next_chunk, true);
|
||||
}
|
||||
|
||||
event_system::instance().raise<download_end>(
|
||||
get_api_path(), get_source_path(), api_error::download_stopped);
|
||||
}
|
||||
|
||||
void ring_buffer_base::reverse(std::size_t count) {
|
||||
update_position(count, false);
|
||||
}
|
||||
|
||||
void ring_buffer_base::set(std::size_t first_chunk, std::size_t current_chunk) {
|
||||
mutex_lock chunk_lock(chunk_mtx_);
|
||||
if (first_chunk >= total_chunks_) {
|
||||
chunk_notify_.notify_all();
|
||||
throw std::runtime_error("first chunk must be less than total chunks");
|
||||
}
|
||||
|
||||
ring_begin_ = first_chunk;
|
||||
ring_end_ =
|
||||
std::min(total_chunks_ - 1U, ring_begin_ + read_state_.size() - 1U);
|
||||
|
||||
if (current_chunk > ring_end_) {
|
||||
chunk_notify_.notify_all();
|
||||
throw std::runtime_error(
|
||||
"current chunk must be less than or equal to last chunk");
|
||||
}
|
||||
|
||||
ring_pos_ = current_chunk;
|
||||
read_state_.set(0U, read_state_.size(), true);
|
||||
|
||||
chunk_notify_.notify_all();
|
||||
}
|
||||
|
||||
void ring_buffer_base::set_api_path(const std::string &api_path) {
|
||||
mutex_lock chunk_lock(chunk_mtx_);
|
||||
open_file_base::set_api_path(api_path);
|
||||
chunk_notify_.notify_all();
|
||||
}
|
||||
|
||||
void ring_buffer_base::update_position(std::size_t count, bool is_forward) {
|
||||
mutex_lock chunk_lock(chunk_mtx_);
|
||||
|
||||
if (is_forward) {
|
||||
if ((ring_pos_ + count) > (total_chunks_ - 1U)) {
|
||||
count = (total_chunks_ - 1U) - ring_pos_;
|
||||
}
|
||||
} else {
|
||||
count = std::min(ring_pos_, count);
|
||||
}
|
||||
|
||||
if (is_forward ? (ring_pos_ + count) <= ring_end_
|
||||
: (ring_pos_ - count) >= ring_begin_) {
|
||||
ring_pos_ += is_forward ? count : -count;
|
||||
} else {
|
||||
auto delta = is_forward ? count - (ring_end_ - ring_pos_)
|
||||
: count - (ring_pos_ - ring_begin_);
|
||||
|
||||
if (delta >= read_state_.size()) {
|
||||
read_state_.set(0U, read_state_.size(), false);
|
||||
ring_pos_ += is_forward ? count : -count;
|
||||
ring_begin_ += is_forward ? delta : -delta;
|
||||
} else {
|
||||
for (std::size_t idx = 0U; idx < delta; ++idx) {
|
||||
if (is_forward) {
|
||||
read_state_[(ring_begin_ + idx) % read_state_.size()] = false;
|
||||
} else {
|
||||
read_state_[(ring_end_ - idx) % read_state_.size()] = false;
|
||||
}
|
||||
}
|
||||
ring_begin_ += is_forward ? delta : -delta;
|
||||
ring_pos_ += is_forward ? count : -count;
|
||||
}
|
||||
|
||||
ring_end_ =
|
||||
std::min(total_chunks_ - 1U, ring_begin_ + read_state_.size() - 1U);
|
||||
}
|
||||
|
||||
chunk_notify_.notify_all();
|
||||
}
|
||||
} // namespace repertory
|
@ -21,73 +21,30 @@
|
||||
*/
|
||||
#include "file_manager/ring_buffer_open_file.hpp"
|
||||
|
||||
#include "app_config.hpp"
|
||||
#include "file_manager/events.hpp"
|
||||
#include "file_manager/open_file_base.hpp"
|
||||
#include "platform/platform.hpp"
|
||||
#include "providers/i_provider.hpp"
|
||||
#include "types/repertory.hpp"
|
||||
#include "utils/common.hpp"
|
||||
#include "utils/error_utils.hpp"
|
||||
#include "utils/file_utils.hpp"
|
||||
#include "utils/path.hpp"
|
||||
#include "utils/utils.hpp"
|
||||
|
||||
namespace repertory {
|
||||
ring_buffer_open_file::ring_buffer_open_file(std::string buffer_directory,
|
||||
std::uint64_t chunk_size,
|
||||
std::uint8_t chunk_timeout,
|
||||
filesystem_item fsi,
|
||||
i_provider &provider)
|
||||
: ring_buffer_open_file(std::move(buffer_directory), chunk_size,
|
||||
chunk_timeout, std::move(fsi), provider,
|
||||
(1024ULL * 1024ULL * 1024ULL) / chunk_size) {}
|
||||
|
||||
ring_buffer_open_file::ring_buffer_open_file(std::string buffer_directory,
|
||||
std::uint64_t chunk_size,
|
||||
std::uint8_t chunk_timeout,
|
||||
filesystem_item fsi,
|
||||
i_provider &provider,
|
||||
std::size_t ring_size)
|
||||
: open_file_base(chunk_size, chunk_timeout, fsi, provider),
|
||||
ring_state_(ring_size),
|
||||
total_chunks_(static_cast<std::size_t>(
|
||||
utils::divide_with_ceiling(fsi.size, chunk_size))) {
|
||||
if ((ring_size % 2U) != 0U) {
|
||||
throw std::runtime_error("ring size must be a multiple of 2");
|
||||
}
|
||||
|
||||
if (ring_size < 4U) {
|
||||
throw std::runtime_error("ring size must be greater than or equal to 4");
|
||||
}
|
||||
|
||||
: ring_buffer_base(chunk_size, chunk_timeout, fsi, provider, ring_size,
|
||||
false),
|
||||
source_path_(utils::path::combine(buffer_directory,
|
||||
{
|
||||
utils::create_uuid_string(),
|
||||
})) {
|
||||
if (not can_handle_file(fsi.size, chunk_size, ring_size)) {
|
||||
throw std::runtime_error("file size is less than ring buffer size");
|
||||
}
|
||||
|
||||
last_chunk_ = ring_state_.size() - 1U;
|
||||
ring_state_.set(0U, ring_state_.size(), true);
|
||||
|
||||
buffer_directory = utils::path::absolute(buffer_directory);
|
||||
if (not utils::file::directory(buffer_directory).create_directory()) {
|
||||
throw std::runtime_error(
|
||||
fmt::format("failed to create buffer directory|path|{}|err|{}",
|
||||
buffer_directory, utils::get_last_error_code()));
|
||||
}
|
||||
|
||||
fsi_.source_path =
|
||||
utils::path::combine(buffer_directory, {utils::create_uuid_string()});
|
||||
nf_ = utils::file::file::open_or_create_file(fsi_.source_path);
|
||||
if (not *nf_) {
|
||||
throw std::runtime_error(fmt::format("failed to create buffer file|err|{}",
|
||||
utils::get_last_error_code()));
|
||||
}
|
||||
|
||||
if (not nf_->truncate(ring_state_.size() * chunk_size)) {
|
||||
nf_->close();
|
||||
throw std::runtime_error(fmt::format("failed to resize buffer file|err|{}",
|
||||
utils::get_last_error_code()));
|
||||
}
|
||||
}
|
||||
|
||||
ring_buffer_open_file::~ring_buffer_open_file() {
|
||||
@ -95,10 +52,16 @@ ring_buffer_open_file::~ring_buffer_open_file() {
|
||||
|
||||
close();
|
||||
|
||||
if (not nf_) {
|
||||
return;
|
||||
}
|
||||
|
||||
nf_->close();
|
||||
if (not utils::file::file(fsi_.source_path).remove()) {
|
||||
nf_.reset();
|
||||
|
||||
if (not utils::file::file(source_path_).remove()) {
|
||||
utils::error::raise_api_path_error(
|
||||
function_name, fsi_.api_path, fsi_.source_path,
|
||||
function_name, get_api_path(), source_path_,
|
||||
utils::get_last_error_code(), "failed to delete file");
|
||||
}
|
||||
}
|
||||
@ -106,102 +69,7 @@ ring_buffer_open_file::~ring_buffer_open_file() {
|
||||
auto ring_buffer_open_file::can_handle_file(std::uint64_t file_size,
|
||||
std::size_t chunk_size,
|
||||
std::size_t ring_size) -> bool {
|
||||
return file_size <= (static_cast<std::uint64_t>(ring_size) * chunk_size);
|
||||
}
|
||||
|
||||
auto ring_buffer_open_file::close() -> bool {
|
||||
stop_requested_ = true;
|
||||
return open_file_base::close();
|
||||
}
|
||||
|
||||
auto ring_buffer_open_file::download_chunk(std::size_t chunk) -> api_error {
|
||||
unique_mutex_lock chunk_lock(chunk_mtx_);
|
||||
if (active_downloads_.find(chunk) != active_downloads_.end()) {
|
||||
auto active_download = active_downloads_.at(chunk);
|
||||
chunk_notify_.notify_all();
|
||||
chunk_lock.unlock();
|
||||
|
||||
return active_download->wait();
|
||||
}
|
||||
|
||||
if (ring_state_[chunk % ring_state_.size()]) {
|
||||
auto active_download = std::make_shared<download>();
|
||||
active_downloads_[chunk] = active_download;
|
||||
ring_state_[chunk % ring_state_.size()] = false;
|
||||
chunk_notify_.notify_all();
|
||||
chunk_lock.unlock();
|
||||
|
||||
data_buffer buffer((chunk == (total_chunks_ - 1U)) ? last_chunk_size_
|
||||
: chunk_size_);
|
||||
|
||||
auto res =
|
||||
provider_.read_file_bytes(fsi_.api_path, buffer.size(),
|
||||
chunk * chunk_size_, buffer, stop_requested_);
|
||||
if (res == api_error::success) {
|
||||
res = do_io([&]() -> api_error {
|
||||
std::size_t bytes_written{};
|
||||
if (nf_->write(buffer, (chunk % ring_state_.size()) * chunk_size_,
|
||||
&bytes_written)) {
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
return api_error::os_error;
|
||||
});
|
||||
}
|
||||
|
||||
active_download->notify(res);
|
||||
|
||||
chunk_lock.lock();
|
||||
active_downloads_.erase(chunk);
|
||||
chunk_notify_.notify_all();
|
||||
return res;
|
||||
}
|
||||
|
||||
chunk_notify_.notify_all();
|
||||
chunk_lock.unlock();
|
||||
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
void ring_buffer_open_file::forward(std::size_t count) {
|
||||
mutex_lock chunk_lock(chunk_mtx_);
|
||||
if ((current_chunk_ + count) > (total_chunks_ - 1U)) {
|
||||
count = (total_chunks_ - 1U) - current_chunk_;
|
||||
}
|
||||
|
||||
if ((current_chunk_ + count) <= last_chunk_) {
|
||||
current_chunk_ += count;
|
||||
} else {
|
||||
const auto added = count - (last_chunk_ - current_chunk_);
|
||||
if (added >= ring_state_.size()) {
|
||||
ring_state_.set(0U, ring_state_.size(), true);
|
||||
current_chunk_ += count;
|
||||
first_chunk_ += added;
|
||||
last_chunk_ =
|
||||
std::min(total_chunks_ - 1U, first_chunk_ + ring_state_.size() - 1U);
|
||||
} else {
|
||||
for (std::size_t idx = 0U; idx < added; ++idx) {
|
||||
ring_state_[(first_chunk_ + idx) % ring_state_.size()] = true;
|
||||
}
|
||||
first_chunk_ += added;
|
||||
current_chunk_ += count;
|
||||
last_chunk_ =
|
||||
std::min(total_chunks_ - 1U, first_chunk_ + ring_state_.size() - 1U);
|
||||
}
|
||||
}
|
||||
|
||||
chunk_notify_.notify_all();
|
||||
}
|
||||
|
||||
auto ring_buffer_open_file::get_read_state() const -> boost::dynamic_bitset<> {
|
||||
recur_mutex_lock file_lock(file_mtx_);
|
||||
auto read_state = ring_state_;
|
||||
return read_state.flip();
|
||||
}
|
||||
|
||||
auto ring_buffer_open_file::get_read_state(std::size_t chunk) const -> bool {
|
||||
recur_mutex_lock file_lock(file_mtx_);
|
||||
return not ring_state_[chunk % ring_state_.size()];
|
||||
return file_size >= (static_cast<std::uint64_t>(ring_size) * chunk_size);
|
||||
}
|
||||
|
||||
auto ring_buffer_open_file::native_operation(
|
||||
@ -209,121 +77,75 @@ auto ring_buffer_open_file::native_operation(
|
||||
return do_io([&]() -> api_error { return callback(nf_->get_handle()); });
|
||||
}
|
||||
|
||||
void ring_buffer_open_file::reverse(std::size_t count) {
|
||||
mutex_lock chunk_lock(chunk_mtx_);
|
||||
if (current_chunk_ < count) {
|
||||
count = current_chunk_;
|
||||
auto ring_buffer_open_file::on_check_start() -> bool {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
if (nf_) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if ((current_chunk_ - count) >= first_chunk_) {
|
||||
current_chunk_ -= count;
|
||||
} else {
|
||||
const auto removed = count - (current_chunk_ - first_chunk_);
|
||||
if (removed >= ring_state_.size()) {
|
||||
ring_state_.set(0U, ring_state_.size(), true);
|
||||
current_chunk_ -= count;
|
||||
first_chunk_ = current_chunk_;
|
||||
last_chunk_ =
|
||||
std::min(total_chunks_ - 1U, first_chunk_ + ring_state_.size() - 1U);
|
||||
} else {
|
||||
for (std::size_t idx = 0U; idx < removed; ++idx) {
|
||||
ring_state_[(last_chunk_ - idx) % ring_state_.size()] = true;
|
||||
}
|
||||
first_chunk_ -= removed;
|
||||
current_chunk_ -= count;
|
||||
last_chunk_ =
|
||||
std::min(total_chunks_ - 1U, first_chunk_ + ring_state_.size() - 1U);
|
||||
}
|
||||
}
|
||||
|
||||
chunk_notify_.notify_all();
|
||||
}
|
||||
|
||||
auto ring_buffer_open_file::read(std::size_t read_size,
|
||||
std::uint64_t read_offset,
|
||||
data_buffer &data) -> api_error {
|
||||
if (fsi_.directory) {
|
||||
return api_error::invalid_operation;
|
||||
}
|
||||
|
||||
reset_timeout();
|
||||
|
||||
read_size = utils::calculate_read_size(fsi_.size, read_size, read_offset);
|
||||
if (read_size == 0U) {
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
const auto start_chunk_index =
|
||||
static_cast<std::size_t>(read_offset / chunk_size_);
|
||||
read_offset = read_offset - (start_chunk_index * chunk_size_);
|
||||
data_buffer buffer(chunk_size_);
|
||||
|
||||
auto res = api_error::success;
|
||||
for (std::size_t chunk = start_chunk_index;
|
||||
(res == api_error::success) && (read_size > 0U); ++chunk) {
|
||||
if (chunk > current_chunk_) {
|
||||
forward(chunk - current_chunk_);
|
||||
} else if (chunk < current_chunk_) {
|
||||
reverse(current_chunk_ - chunk);
|
||||
}
|
||||
|
||||
reset_timeout();
|
||||
res = download_chunk(chunk);
|
||||
if (res != api_error::success) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const auto to_read = std::min(
|
||||
static_cast<std::size_t>(chunk_size_ - read_offset), read_size);
|
||||
res = do_io([this, &buffer, &chunk, &data, read_offset,
|
||||
&to_read]() -> api_error {
|
||||
std::size_t bytes_read{};
|
||||
auto ret = nf_->read(buffer, ((chunk % ring_state_.size()) * chunk_size_),
|
||||
&bytes_read)
|
||||
? api_error::success
|
||||
: api_error::os_error;
|
||||
if (ret == api_error::success) {
|
||||
data.insert(
|
||||
data.end(), buffer.begin() + static_cast<std::int64_t>(read_offset),
|
||||
buffer.begin() + static_cast<std::int64_t>(read_offset + to_read));
|
||||
reset_timeout();
|
||||
}
|
||||
|
||||
return ret;
|
||||
});
|
||||
read_offset = 0U;
|
||||
read_size -= to_read;
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
void ring_buffer_open_file::set(std::size_t first_chunk,
|
||||
std::size_t current_chunk) {
|
||||
mutex_lock chunk_lock(chunk_mtx_);
|
||||
if (first_chunk >= total_chunks_) {
|
||||
chunk_notify_.notify_all();
|
||||
throw std::runtime_error("first chunk must be less than total chunks");
|
||||
}
|
||||
|
||||
first_chunk_ = first_chunk;
|
||||
last_chunk_ = first_chunk_ + ring_state_.size() - 1U;
|
||||
|
||||
if (current_chunk > last_chunk_) {
|
||||
chunk_notify_.notify_all();
|
||||
auto buffer_directory{utils::path::get_parent_path(source_path_)};
|
||||
if (not utils::file::directory(buffer_directory).create_directory()) {
|
||||
throw std::runtime_error(
|
||||
"current chunk must be less than or equal to last chunk");
|
||||
fmt::format("failed to create buffer directory|path|{}|err|{}",
|
||||
buffer_directory, utils::get_last_error_code()));
|
||||
}
|
||||
|
||||
current_chunk_ = current_chunk;
|
||||
ring_state_.set(0U, ring_state_.size(), false);
|
||||
nf_ = utils::file::file::open_or_create_file(source_path_);
|
||||
if (not nf_ || not *nf_) {
|
||||
throw std::runtime_error(fmt::format("failed to create buffer file|err|{}",
|
||||
utils::get_last_error_code()));
|
||||
}
|
||||
|
||||
chunk_notify_.notify_all();
|
||||
if (not nf_->truncate(get_ring_size() * get_chunk_size())) {
|
||||
nf_->close();
|
||||
nf_.reset();
|
||||
|
||||
throw std::runtime_error(fmt::format("failed to resize buffer file|err|{}",
|
||||
utils::get_last_error_code()));
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
void ring_buffer_open_file::set_api_path(const std::string &api_path) {
|
||||
mutex_lock chunk_lock(chunk_mtx_);
|
||||
open_file_base::set_api_path(api_path);
|
||||
chunk_notify_.notify_all();
|
||||
auto ring_buffer_open_file::on_chunk_downloaded(
|
||||
std::size_t chunk, const data_buffer &buffer) -> api_error {
|
||||
return do_io([&]() -> api_error {
|
||||
std::size_t bytes_written{};
|
||||
if (nf_->write(buffer, (chunk % get_ring_size()) * get_chunk_size(),
|
||||
&bytes_written)) {
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
return api_error::os_error;
|
||||
});
|
||||
}
|
||||
|
||||
auto ring_buffer_open_file::on_read_chunk(
|
||||
std::size_t chunk, std::size_t read_size, std::uint64_t read_offset,
|
||||
data_buffer &data, std::size_t &bytes_read) -> api_error {
|
||||
data_buffer buffer(read_size);
|
||||
auto res = do_io([&]() -> api_error {
|
||||
return nf_->read(
|
||||
buffer,
|
||||
(((chunk % get_ring_size()) * get_chunk_size()) + read_offset),
|
||||
&bytes_read)
|
||||
? api_error::success
|
||||
: api_error::os_error;
|
||||
});
|
||||
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
|
||||
data.insert(data.end(), buffer.begin(), buffer.end());
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
auto ring_buffer_open_file::use_buffer(
|
||||
std::size_t /* chunk */,
|
||||
std::function<api_error(data_buffer &)> func) -> api_error {
|
||||
data_buffer buffer;
|
||||
return func(buffer);
|
||||
}
|
||||
} // namespace repertory
|
||||
|
@ -53,7 +53,8 @@ void upload::upload_thread() {
|
||||
|
||||
error_ =
|
||||
provider_.upload_file(fsi_.api_path, fsi_.source_path, stop_requested_);
|
||||
if (not utils::file::reset_modified_time(fsi_.source_path)) {
|
||||
if (error_ == api_error::success &&
|
||||
not utils::file::reset_modified_time(fsi_.source_path)) {
|
||||
utils::error::raise_api_path_error(
|
||||
function_name, fsi_.api_path, fsi_.source_path,
|
||||
utils::get_last_error_code(), "failed to reset modified time");
|
||||
|
@ -51,8 +51,8 @@ void base_provider::add_all_items(const stop_type &stop_requested) {
|
||||
}
|
||||
|
||||
auto base_provider::create_api_file(std::string path, std::string key,
|
||||
std::uint64_t size,
|
||||
std::uint64_t file_time) -> api_file {
|
||||
std::uint64_t size, std::uint64_t file_time)
|
||||
-> api_file {
|
||||
api_file file{};
|
||||
file.api_path = utils::path::create_api_path(path);
|
||||
file.api_parent = utils::path::get_parent_api_path(file.api_path);
|
||||
@ -84,8 +84,8 @@ auto base_provider::create_api_file(std::string path, std::uint64_t size,
|
||||
}
|
||||
|
||||
auto base_provider::create_directory_clone_source_meta(
|
||||
const std::string &source_api_path,
|
||||
const std::string &api_path) -> api_error {
|
||||
const std::string &source_api_path, const std::string &api_path)
|
||||
-> api_error {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
bool exists{};
|
||||
@ -182,8 +182,8 @@ auto base_provider::create_directory(const std::string &api_path,
|
||||
return set_item_meta(api_path, meta);
|
||||
}
|
||||
|
||||
auto base_provider::create_file(const std::string &api_path,
|
||||
api_meta_map &meta) -> api_error {
|
||||
auto base_provider::create_file(const std::string &api_path, api_meta_map &meta)
|
||||
-> api_error {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
bool exists{};
|
||||
@ -240,8 +240,9 @@ auto base_provider::create_file(const std::string &api_path,
|
||||
return api_error::error;
|
||||
}
|
||||
|
||||
auto base_provider::get_api_path_from_source(
|
||||
const std::string &source_path, std::string &api_path) const -> api_error {
|
||||
auto base_provider::get_api_path_from_source(const std::string &source_path,
|
||||
std::string &api_path) const
|
||||
-> api_error {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
if (source_path.empty()) {
|
||||
@ -254,8 +255,9 @@ auto base_provider::get_api_path_from_source(
|
||||
return db3_->get_api_path(source_path, api_path);
|
||||
}
|
||||
|
||||
auto base_provider::get_directory_items(
|
||||
const std::string &api_path, directory_item_list &list) const -> api_error {
|
||||
auto base_provider::get_directory_items(const std::string &api_path,
|
||||
directory_item_list &list) const
|
||||
-> api_error {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
bool exists{};
|
||||
@ -319,9 +321,10 @@ auto base_provider::get_file_size(const std::string &api_path,
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
auto base_provider::get_filesystem_item(
|
||||
const std::string &api_path, bool directory,
|
||||
filesystem_item &fsi) const -> api_error {
|
||||
auto base_provider::get_filesystem_item(const std::string &api_path,
|
||||
bool directory,
|
||||
filesystem_item &fsi) const
|
||||
-> api_error {
|
||||
bool exists{};
|
||||
auto res = is_directory(api_path, exists);
|
||||
if (res != api_error::success) {
|
||||
@ -354,9 +357,10 @@ auto base_provider::get_filesystem_item(
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
auto base_provider::get_filesystem_item_and_file(
|
||||
const std::string &api_path, api_file &file,
|
||||
filesystem_item &fsi) const -> api_error {
|
||||
auto base_provider::get_filesystem_item_and_file(const std::string &api_path,
|
||||
api_file &file,
|
||||
filesystem_item &fsi) const
|
||||
-> api_error {
|
||||
auto res = get_file(api_path, file);
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
|
@ -51,8 +51,6 @@ void full_server::handle_get_directory_items(const httplib::Request &req,
|
||||
|
||||
void full_server::handle_get_drive_information(const httplib::Request & /*req*/,
|
||||
httplib::Response &res) {
|
||||
auto dir_size =
|
||||
utils::file::directory(get_config().get_cache_directory()).size();
|
||||
res.set_content(
|
||||
json({
|
||||
{"cache_space_used", cache_size_mgr::instance().size()},
|
||||
|
@ -53,12 +53,12 @@ auto database_type_to_string(const database_type &type) -> std::string {
|
||||
auto download_type_from_string(std::string type,
|
||||
download_type default_type) -> download_type {
|
||||
type = utils::string::to_lower(utils::string::trim(type));
|
||||
if (type == "direct") {
|
||||
return download_type::direct;
|
||||
if (type == "default") {
|
||||
return download_type::default_;
|
||||
}
|
||||
|
||||
if (type == "fallback") {
|
||||
return download_type::fallback;
|
||||
if (type == "direct") {
|
||||
return download_type::direct;
|
||||
}
|
||||
|
||||
if (type == "ring_buffer") {
|
||||
@ -70,14 +70,14 @@ auto download_type_from_string(std::string type,
|
||||
|
||||
auto download_type_to_string(const download_type &type) -> std::string {
|
||||
switch (type) {
|
||||
case download_type::default_:
|
||||
return "default";
|
||||
case download_type::direct:
|
||||
return "direct";
|
||||
case download_type::fallback:
|
||||
return "fallback";
|
||||
case download_type::ring_buffer:
|
||||
return "ring_buffer";
|
||||
default:
|
||||
return "fallback";
|
||||
return "default";
|
||||
}
|
||||
}
|
||||
|
||||
@ -106,6 +106,7 @@ static const std::unordered_map<api_error, std::string> LOOKUP = {
|
||||
{api_error::invalid_handle, "invalid_handle"},
|
||||
{api_error::invalid_operation, "invalid_operation"},
|
||||
{api_error::invalid_ring_buffer_multiple, "invalid_ring_buffer_multiple"},
|
||||
{api_error::invalid_ring_buffer_position, "invalid_ring_buffer_position"},
|
||||
{api_error::invalid_ring_buffer_size, "invalid_ring_buffer_size"},
|
||||
{api_error::invalid_version, "invalid_version"},
|
||||
{api_error::item_exists, "item_exists"},
|
||||
|
@ -52,6 +52,8 @@ auto from_api_error(const api_error &err) -> int {
|
||||
return -EEXIST;
|
||||
case api_error::file_in_use:
|
||||
return -EBUSY;
|
||||
case api_error::invalid_handle:
|
||||
return -EBADF;
|
||||
case api_error::invalid_operation:
|
||||
return -EINVAL;
|
||||
case api_error::item_not_found:
|
||||
|
@ -29,6 +29,13 @@
|
||||
namespace repertory {
|
||||
class mock_open_file : public virtual i_closeable_open_file {
|
||||
public:
|
||||
MOCK_METHOD(void, add, (std::uint64_t handle, open_file_data ofd),
|
||||
(override));
|
||||
|
||||
MOCK_METHOD(bool, can_close, (), (const, override));
|
||||
|
||||
MOCK_METHOD(bool, close, (), (override));
|
||||
|
||||
MOCK_METHOD(std::string, get_api_path, (), (const, override));
|
||||
|
||||
MOCK_METHOD(std::size_t, get_chunk_size, (), (const, override));
|
||||
@ -47,14 +54,30 @@ public:
|
||||
|
||||
MOCK_METHOD(boost::dynamic_bitset<>, get_read_state, (), (const, override));
|
||||
|
||||
MOCK_METHOD(bool, get_allocated, (), (const, override));
|
||||
|
||||
MOCK_METHOD(std::vector<std::uint64_t>, get_handles, (), (const, override));
|
||||
|
||||
MOCK_METHOD((std::map<std::uint64_t, open_file_data> &), get_open_data, (),
|
||||
(override));
|
||||
|
||||
MOCK_METHOD((const std::map<std::uint64_t, open_file_data> &), get_open_data,
|
||||
(), (const, override));
|
||||
|
||||
MOCK_METHOD(bool, get_read_state, (std::size_t chunk), (const, override));
|
||||
|
||||
MOCK_METHOD(std::string, get_source_path, (), (const, override));
|
||||
|
||||
MOCK_METHOD(bool, has_handle, (std::uint64_t handle), (const, override));
|
||||
|
||||
MOCK_METHOD(bool, is_complete, (), (const, override));
|
||||
|
||||
MOCK_METHOD(bool, is_directory, (), (const, override));
|
||||
|
||||
MOCK_METHOD(bool, is_modified, (), (const, override));
|
||||
|
||||
MOCK_METHOD(bool, is_write_supported, (), (const, override));
|
||||
|
||||
MOCK_METHOD(api_error, native_operation, (native_operation_callback callback),
|
||||
(override));
|
||||
|
||||
@ -67,6 +90,10 @@ public:
|
||||
data_buffer &data),
|
||||
(override));
|
||||
|
||||
MOCK_METHOD(void, remove, (std::uint64_t handle), (override));
|
||||
|
||||
MOCK_METHOD(void, remove_all, (), (override));
|
||||
|
||||
MOCK_METHOD(api_error, resize, (std::uint64_t new_file_size), (override));
|
||||
|
||||
MOCK_METHOD(void, set_api_path, (const std::string &api_path), (override));
|
||||
@ -75,31 +102,6 @@ public:
|
||||
(std::uint64_t write_offset, const data_buffer &data,
|
||||
std::size_t &bytes_written),
|
||||
(override));
|
||||
|
||||
MOCK_METHOD(void, add, (std::uint64_t handle, open_file_data ofd),
|
||||
(override));
|
||||
|
||||
MOCK_METHOD(bool, can_close, (), (const, override));
|
||||
|
||||
MOCK_METHOD(bool, close, (), (override));
|
||||
|
||||
MOCK_METHOD(std::vector<std::uint64_t>, get_handles, (), (const, override));
|
||||
|
||||
MOCK_METHOD((std::map<std::uint64_t, open_file_data> &), get_open_data, (),
|
||||
(override));
|
||||
|
||||
MOCK_METHOD((const std::map<std::uint64_t, open_file_data> &), get_open_data,
|
||||
(), (const, override));
|
||||
|
||||
MOCK_METHOD(bool, is_complete, (), (const, override));
|
||||
|
||||
MOCK_METHOD(bool, is_modified, (), (const, override));
|
||||
|
||||
MOCK_METHOD(bool, is_write_supported, (), (const, override));
|
||||
|
||||
MOCK_METHOD(void, remove, (std::uint64_t handle), (override));
|
||||
|
||||
MOCK_METHOD(void, remove_all, (), (override));
|
||||
};
|
||||
} // namespace repertory
|
||||
|
||||
|
@ -29,12 +29,22 @@ constexpr const std::size_t test_chunk_size{1024U};
|
||||
} // namespace
|
||||
|
||||
namespace repertory {
|
||||
TEST(direct_open_file, read_full_file) {
|
||||
class direct_open_file_test : public ::testing::Test {
|
||||
public:
|
||||
console_consumer con_consumer;
|
||||
mock_provider provider;
|
||||
|
||||
protected:
|
||||
void SetUp() override { event_system::instance().start(); }
|
||||
|
||||
void TearDown() override { event_system::instance().stop(); }
|
||||
};
|
||||
|
||||
TEST_F(direct_open_file_test, read_full_file) {
|
||||
auto &source_file = test::create_random_file(test_chunk_size * 32U);
|
||||
|
||||
auto dest_path = test::generate_test_file_name("direct_open_file");
|
||||
|
||||
mock_provider provider;
|
||||
EXPECT_CALL(provider, is_read_only()).WillRepeatedly(Return(false));
|
||||
|
||||
filesystem_item fsi;
|
||||
@ -42,11 +52,14 @@ TEST(direct_open_file, read_full_file) {
|
||||
fsi.directory = false;
|
||||
fsi.size = test_chunk_size * 32U;
|
||||
|
||||
std::mutex read_mtx;
|
||||
EXPECT_CALL(provider, read_file_bytes)
|
||||
.WillRepeatedly([&source_file](const std::string & /* api_path */,
|
||||
std::size_t size, std::uint64_t offset,
|
||||
data_buffer &data,
|
||||
stop_type &stop_requested) -> api_error {
|
||||
.WillRepeatedly([&read_mtx, &source_file](
|
||||
const std::string & /* api_path */, std::size_t size,
|
||||
std::uint64_t offset, data_buffer &data,
|
||||
stop_type &stop_requested) -> api_error {
|
||||
mutex_lock lock(read_mtx);
|
||||
|
||||
EXPECT_FALSE(stop_requested);
|
||||
std::size_t bytes_read{};
|
||||
data.resize(size);
|
||||
@ -89,12 +102,11 @@ TEST(direct_open_file, read_full_file) {
|
||||
}
|
||||
}
|
||||
|
||||
TEST(direct_open_file, read_full_file_in_reverse) {
|
||||
TEST_F(direct_open_file_test, read_full_file_in_reverse) {
|
||||
auto &source_file = test::create_random_file(test_chunk_size * 32U);
|
||||
|
||||
auto dest_path = test::generate_test_file_name("direct_open_file");
|
||||
|
||||
mock_provider provider;
|
||||
EXPECT_CALL(provider, is_read_only()).WillRepeatedly(Return(false));
|
||||
|
||||
filesystem_item fsi;
|
||||
@ -102,11 +114,14 @@ TEST(direct_open_file, read_full_file_in_reverse) {
|
||||
fsi.directory = false;
|
||||
fsi.size = test_chunk_size * 32U;
|
||||
|
||||
std::mutex read_mtx;
|
||||
EXPECT_CALL(provider, read_file_bytes)
|
||||
.WillRepeatedly([&source_file](const std::string & /* api_path */,
|
||||
std::size_t size, std::uint64_t offset,
|
||||
data_buffer &data,
|
||||
stop_type &stop_requested) -> api_error {
|
||||
.WillRepeatedly([&read_mtx, &source_file](
|
||||
const std::string & /* api_path */, std::size_t size,
|
||||
std::uint64_t offset, data_buffer &data,
|
||||
stop_type &stop_requested) -> api_error {
|
||||
mutex_lock lock(read_mtx);
|
||||
|
||||
EXPECT_FALSE(stop_requested);
|
||||
std::size_t bytes_read{};
|
||||
data.resize(size);
|
||||
@ -149,12 +164,11 @@ TEST(direct_open_file, read_full_file_in_reverse) {
|
||||
}
|
||||
}
|
||||
|
||||
TEST(direct_open_file, read_full_file_in_partial_chunks) {
|
||||
TEST_F(direct_open_file_test, read_full_file_in_partial_chunks) {
|
||||
auto &source_file = test::create_random_file(test_chunk_size * 32U);
|
||||
|
||||
auto dest_path = test::generate_test_file_name("test");
|
||||
|
||||
mock_provider provider;
|
||||
EXPECT_CALL(provider, is_read_only()).WillRepeatedly(Return(false));
|
||||
|
||||
filesystem_item fsi;
|
||||
@ -162,11 +176,14 @@ TEST(direct_open_file, read_full_file_in_partial_chunks) {
|
||||
fsi.api_path = "/test.txt";
|
||||
fsi.size = test_chunk_size * 32U;
|
||||
|
||||
std::mutex read_mtx;
|
||||
EXPECT_CALL(provider, read_file_bytes)
|
||||
.WillRepeatedly([&source_file](const std::string & /* api_path */,
|
||||
std::size_t size, std::uint64_t offset,
|
||||
data_buffer &data,
|
||||
stop_type &stop_requested) -> api_error {
|
||||
.WillRepeatedly([&read_mtx, &source_file](
|
||||
const std::string & /* api_path */, std::size_t size,
|
||||
std::uint64_t offset, data_buffer &data,
|
||||
stop_type &stop_requested) -> api_error {
|
||||
mutex_lock lock(read_mtx);
|
||||
|
||||
EXPECT_FALSE(stop_requested);
|
||||
std::size_t bytes_read{};
|
||||
data.resize(size);
|
||||
@ -206,12 +223,11 @@ TEST(direct_open_file, read_full_file_in_partial_chunks) {
|
||||
}
|
||||
}
|
||||
|
||||
TEST(direct_open_file, read_full_file_in_partial_chunks_in_reverse) {
|
||||
TEST_F(direct_open_file_test, read_full_file_in_partial_chunks_in_reverse) {
|
||||
auto &source_file = test::create_random_file(test_chunk_size * 32U);
|
||||
|
||||
auto dest_path = test::generate_test_file_name("direct_open_file");
|
||||
|
||||
mock_provider provider;
|
||||
EXPECT_CALL(provider, is_read_only()).WillRepeatedly(Return(false));
|
||||
|
||||
filesystem_item fsi;
|
||||
@ -219,11 +235,14 @@ TEST(direct_open_file, read_full_file_in_partial_chunks_in_reverse) {
|
||||
fsi.api_path = "/test.txt";
|
||||
fsi.size = test_chunk_size * 32U;
|
||||
|
||||
std::mutex read_mtx;
|
||||
EXPECT_CALL(provider, read_file_bytes)
|
||||
.WillRepeatedly([&source_file](const std::string & /* api_path */,
|
||||
std::size_t size, std::uint64_t offset,
|
||||
data_buffer &data,
|
||||
stop_type &stop_requested) -> api_error {
|
||||
.WillRepeatedly([&read_mtx, &source_file](
|
||||
const std::string & /* api_path */, std::size_t size,
|
||||
std::uint64_t offset, data_buffer &data,
|
||||
stop_type &stop_requested) -> api_error {
|
||||
mutex_lock lock(read_mtx);
|
||||
|
||||
EXPECT_FALSE(stop_requested);
|
||||
std::size_t bytes_read{};
|
||||
data.resize(size);
|
||||
|
@ -431,16 +431,6 @@ TEST_F(file_manager_test,
|
||||
return api_error::success;
|
||||
});
|
||||
|
||||
std::uint64_t handle{};
|
||||
std::shared_ptr<i_open_file> open_file;
|
||||
#if defined(_WIN32)
|
||||
EXPECT_EQ(api_error::success, mgr.open("/test_write_partial_download.txt",
|
||||
false, {}, handle, open_file));
|
||||
#else
|
||||
EXPECT_EQ(api_error::success, mgr.open("/test_write_partial_download.txt",
|
||||
false, O_RDWR, handle, open_file));
|
||||
#endif
|
||||
|
||||
EXPECT_CALL(mp, read_file_bytes)
|
||||
.WillRepeatedly([&file](const std::string & /* api_path */,
|
||||
std::size_t size, std::uint64_t offset,
|
||||
@ -465,6 +455,17 @@ TEST_F(file_manager_test,
|
||||
|
||||
return api_error::download_stopped;
|
||||
});
|
||||
|
||||
std::uint64_t handle{};
|
||||
std::shared_ptr<i_open_file> open_file;
|
||||
#if defined(_WIN32)
|
||||
EXPECT_EQ(api_error::success, mgr.open("/test_write_partial_download.txt",
|
||||
false, {}, handle, open_file));
|
||||
#else
|
||||
EXPECT_EQ(api_error::success, mgr.open("/test_write_partial_download.txt",
|
||||
false, O_RDWR, handle, open_file));
|
||||
#endif
|
||||
|
||||
EXPECT_CALL(mp, set_item_meta("/test_write_partial_download.txt", _))
|
||||
.WillOnce(
|
||||
[](const std::string &, const api_meta_map &meta2) -> api_error {
|
||||
@ -475,6 +476,10 @@ TEST_F(file_manager_test,
|
||||
});
|
||||
EXPECT_CALL(mp, upload_file).Times(0u);
|
||||
|
||||
if (not open_file->is_write_supported()) {
|
||||
EXPECT_TRUE(mgr.get_open_file(handle, true, open_file));
|
||||
}
|
||||
|
||||
std::size_t bytes_written{};
|
||||
data_buffer data = {0, 1, 2};
|
||||
EXPECT_EQ(api_error::success, open_file->write(0u, data, bytes_written));
|
||||
@ -560,7 +565,6 @@ TEST_F(file_manager_test, upload_occurs_after_write_if_fully_downloaded) {
|
||||
EXPECT_STREQ(source_path.c_str(),
|
||||
evt2.get_source().get<std::string>().c_str());
|
||||
});
|
||||
event_capture capture({"download_end"});
|
||||
|
||||
auto now = utils::time::get_time_now();
|
||||
auto meta = create_meta_attributes(
|
||||
@ -584,16 +588,6 @@ TEST_F(file_manager_test, upload_occurs_after_write_if_fully_downloaded) {
|
||||
return api_error::success;
|
||||
});
|
||||
|
||||
std::uint64_t handle{};
|
||||
std::shared_ptr<i_open_file> open_file;
|
||||
#if defined(_WIN32)
|
||||
EXPECT_EQ(api_error::success, mgr.open("/test_write_full_download.txt", false,
|
||||
{}, handle, open_file));
|
||||
#else
|
||||
EXPECT_EQ(api_error::success, mgr.open("/test_write_full_download.txt", false,
|
||||
O_RDWR, handle, open_file));
|
||||
#endif
|
||||
|
||||
EXPECT_CALL(mp, read_file_bytes)
|
||||
.WillRepeatedly([&file](const std::string & /* api_path */,
|
||||
std::size_t size, std::uint64_t offset,
|
||||
@ -606,6 +600,17 @@ TEST_F(file_manager_test, upload_occurs_after_write_if_fully_downloaded) {
|
||||
EXPECT_EQ(bytes_read, data.size());
|
||||
return ret;
|
||||
});
|
||||
|
||||
std::uint64_t handle{};
|
||||
std::shared_ptr<i_open_file> open_file;
|
||||
#if defined(_WIN32)
|
||||
EXPECT_EQ(api_error::success, mgr.open("/test_write_full_download.txt", false,
|
||||
{}, handle, open_file));
|
||||
#else
|
||||
EXPECT_EQ(api_error::success, mgr.open("/test_write_full_download.txt", false,
|
||||
O_RDWR, handle, open_file));
|
||||
#endif
|
||||
|
||||
EXPECT_CALL(mp, set_item_meta("/test_write_full_download.txt", _))
|
||||
.WillOnce(
|
||||
[](const std::string &, const api_meta_map &meta2) -> api_error {
|
||||
@ -614,25 +619,33 @@ TEST_F(file_manager_test, upload_occurs_after_write_if_fully_downloaded) {
|
||||
EXPECT_NO_THROW(EXPECT_FALSE(meta2.at(META_WRITTEN).empty()));
|
||||
return api_error::success;
|
||||
});
|
||||
std::size_t bytes_written{};
|
||||
data_buffer data = {0, 1, 2};
|
||||
EXPECT_EQ(api_error::success, open_file->write(0u, data, bytes_written));
|
||||
EXPECT_EQ(std::size_t(3u), bytes_written);
|
||||
open_file.reset();
|
||||
|
||||
capture.wait_for_empty();
|
||||
if (not open_file->is_write_supported()) {
|
||||
EXPECT_TRUE(mgr.get_open_file(handle, true, open_file));
|
||||
}
|
||||
|
||||
EXPECT_CALL(mp, upload_file("/test_write_full_download.txt", source_path, _))
|
||||
.WillOnce(Return(api_error::success));
|
||||
|
||||
event_capture ec2({
|
||||
event_capture capture({
|
||||
"item_timeout",
|
||||
"file_upload_queued",
|
||||
"file_upload_completed",
|
||||
});
|
||||
|
||||
EXPECT_CALL(mp, upload_file("/test_write_full_download.txt", source_path, _))
|
||||
.WillOnce(Return(api_error::success));
|
||||
|
||||
std::size_t bytes_written{};
|
||||
data_buffer data = {0, 1, 2};
|
||||
EXPECT_EQ(api_error::success, open_file->write(0u, data, bytes_written));
|
||||
EXPECT_EQ(std::size_t(3u), bytes_written);
|
||||
|
||||
while (not open_file->is_complete()) {
|
||||
std::this_thread::sleep_for(10ms);
|
||||
}
|
||||
open_file.reset();
|
||||
|
||||
mgr.close(handle);
|
||||
|
||||
ec2.wait_for_empty();
|
||||
capture.wait_for_empty();
|
||||
|
||||
EXPECT_EQ(std::size_t(0U), mgr.get_open_file_count());
|
||||
EXPECT_EQ(std::size_t(0U), mgr.get_open_handle_count());
|
||||
@ -697,6 +710,10 @@ TEST_F(file_manager_test, can_evict_file) {
|
||||
.WillRepeatedly(Return(api_error::success));
|
||||
EXPECT_CALL(mp, upload_file(_, _, _)).WillOnce(Return(api_error::success));
|
||||
|
||||
if (not open_file->is_write_supported()) {
|
||||
EXPECT_TRUE(mgr.get_open_file(handle, true, open_file));
|
||||
}
|
||||
|
||||
data_buffer data{{0, 1, 1}};
|
||||
std::size_t bytes_written{};
|
||||
auto res = open_file->write(0U, data, bytes_written);
|
||||
@ -713,15 +730,6 @@ TEST_F(file_manager_test, can_evict_file) {
|
||||
EXPECT_TRUE(utils::retry_action(
|
||||
[&mgr]() -> bool { return not mgr.is_processing("/test_evict.txt"); }));
|
||||
|
||||
EXPECT_CALL(mp, get_item_meta(_, META_SOURCE, _))
|
||||
.WillOnce([&source_path](const std::string &api_path,
|
||||
const std::string &key,
|
||||
std::string &value) -> api_error {
|
||||
EXPECT_STREQ("/test_evict.txt", api_path.c_str());
|
||||
EXPECT_STREQ(META_SOURCE.c_str(), key.c_str());
|
||||
value = source_path;
|
||||
return api_error::success;
|
||||
});
|
||||
EXPECT_CALL(mp, get_item_meta(_, META_PINNED, _))
|
||||
.WillOnce([](const std::string &api_path, const std::string &key,
|
||||
std::string &value) -> api_error {
|
||||
@ -740,6 +748,17 @@ TEST_F(file_manager_test, evict_file_fails_if_file_is_pinned) {
|
||||
EXPECT_CALL(mp, is_read_only()).WillRepeatedly(Return(false));
|
||||
file_manager mgr(*cfg, mp);
|
||||
|
||||
EXPECT_CALL(mp, get_filesystem_item)
|
||||
.WillRepeatedly([](const std::string &api_path, bool directory,
|
||||
filesystem_item &fsi) -> api_error {
|
||||
fsi.api_path = api_path;
|
||||
fsi.api_parent = utils::path::get_parent_api_path(api_path);
|
||||
fsi.directory = directory;
|
||||
fsi.size = 2U;
|
||||
fsi.source_path = "/test/test_open.src";
|
||||
return api_error::success;
|
||||
});
|
||||
|
||||
EXPECT_CALL(mp, get_item_meta(_, META_PINNED, _))
|
||||
.WillOnce([](const std::string &api_path, const std::string &key,
|
||||
std::string &value) -> api_error {
|
||||
@ -798,28 +817,17 @@ TEST_F(file_manager_test, evict_file_fails_if_file_is_open) {
|
||||
mgr.close(handle);
|
||||
}
|
||||
|
||||
TEST_F(file_manager_test,
|
||||
evict_file_fails_if_unable_to_get_source_path_from_item_meta) {
|
||||
TEST_F(file_manager_test, evict_file_fails_if_unable_to_get_filesystem_item) {
|
||||
EXPECT_CALL(mp, is_read_only()).WillRepeatedly(Return(false));
|
||||
file_manager mgr(*cfg, mp);
|
||||
|
||||
EXPECT_CALL(mp, get_item_meta(_, META_SOURCE, _))
|
||||
.WillOnce([](const std::string &api_path, const std::string &key,
|
||||
std::string & /*value*/) -> api_error {
|
||||
EXPECT_STREQ("/test_open.txt", api_path.c_str());
|
||||
EXPECT_STREQ(META_SOURCE.c_str(), key.c_str());
|
||||
EXPECT_CALL(mp, get_filesystem_item)
|
||||
.WillRepeatedly([](const std::string & /* api_path */,
|
||||
bool /* directory */,
|
||||
filesystem_item & /* fsi */) -> api_error {
|
||||
return api_error::error;
|
||||
});
|
||||
|
||||
EXPECT_CALL(mp, get_item_meta(_, META_PINNED, _))
|
||||
.WillOnce([](const std::string &api_path, const std::string &key,
|
||||
std::string &value) -> api_error {
|
||||
EXPECT_STREQ("/test_open.txt", api_path.c_str());
|
||||
EXPECT_STREQ(META_PINNED.c_str(), key.c_str());
|
||||
value = "0";
|
||||
return api_error::success;
|
||||
});
|
||||
|
||||
EXPECT_FALSE(mgr.evict_file("/test_open.txt"));
|
||||
}
|
||||
|
||||
@ -827,20 +835,13 @@ TEST_F(file_manager_test, evict_file_fails_if_source_path_is_empty) {
|
||||
EXPECT_CALL(mp, is_read_only()).WillRepeatedly(Return(false));
|
||||
file_manager mgr(*cfg, mp);
|
||||
|
||||
EXPECT_CALL(mp, get_item_meta(_, META_SOURCE, _))
|
||||
.WillOnce([](const std::string &api_path, const std::string &key,
|
||||
std::string &value) -> api_error {
|
||||
EXPECT_STREQ("/test_open.txt", api_path.c_str());
|
||||
EXPECT_STREQ(META_SOURCE.c_str(), key.c_str());
|
||||
value = "";
|
||||
return api_error::success;
|
||||
});
|
||||
EXPECT_CALL(mp, get_item_meta(_, META_PINNED, _))
|
||||
.WillOnce([](const std::string &api_path, const std::string &key,
|
||||
std::string &value) -> api_error {
|
||||
EXPECT_STREQ("/test_open.txt", api_path.c_str());
|
||||
EXPECT_STREQ(META_PINNED.c_str(), key.c_str());
|
||||
value = "0";
|
||||
EXPECT_CALL(mp, get_filesystem_item)
|
||||
.WillRepeatedly([](const std::string &api_path, bool directory,
|
||||
filesystem_item &fsi) -> api_error {
|
||||
fsi.api_path = api_path;
|
||||
fsi.api_parent = utils::path::get_parent_api_path(api_path);
|
||||
fsi.directory = directory;
|
||||
fsi.size = 20U;
|
||||
return api_error::success;
|
||||
});
|
||||
|
||||
@ -908,6 +909,10 @@ TEST_F(file_manager_test, evict_file_fails_if_file_is_uploading) {
|
||||
return api_error::success;
|
||||
});
|
||||
|
||||
if (not open_file->is_write_supported()) {
|
||||
EXPECT_TRUE(mgr.get_open_file(handle, true, open_file));
|
||||
}
|
||||
|
||||
data_buffer data{{0, 1, 1}};
|
||||
std::size_t bytes_written{};
|
||||
EXPECT_EQ(api_error::success, open_file->write(0U, data, bytes_written));
|
||||
@ -951,6 +956,7 @@ TEST_F(file_manager_test, evict_file_fails_if_file_is_modified) {
|
||||
EXPECT_CALL(mp, is_read_only()).WillRepeatedly(Return(false));
|
||||
|
||||
file_manager mgr(*cfg, mp);
|
||||
|
||||
EXPECT_CALL(mp, get_filesystem_item)
|
||||
.WillOnce([](const std::string &api_path, bool directory,
|
||||
filesystem_item &fsi) -> api_error {
|
||||
@ -965,11 +971,12 @@ TEST_F(file_manager_test, evict_file_fails_if_file_is_modified) {
|
||||
});
|
||||
|
||||
auto file = std::make_shared<mock_open_file>();
|
||||
EXPECT_CALL(*file, is_directory).WillOnce(Return(false));
|
||||
EXPECT_CALL(*file, add).WillOnce(Return());
|
||||
EXPECT_CALL(*file, get_api_path).WillRepeatedly(Return("/test_evict.txt"));
|
||||
EXPECT_CALL(*file, get_source_path).WillRepeatedly(Return("/test_evict.src"));
|
||||
EXPECT_CALL(*file, is_modified).Times(2).WillRepeatedly(Return(true));
|
||||
EXPECT_CALL(*file, is_directory).WillOnce(Return(false));
|
||||
EXPECT_CALL(*file, is_modified).WillRepeatedly(Return(true));
|
||||
EXPECT_CALL(*file, is_write_supported).WillRepeatedly(Return(true));
|
||||
|
||||
std::uint64_t handle{};
|
||||
std::shared_ptr<i_open_file> open_file;
|
||||
@ -992,20 +999,21 @@ TEST_F(file_manager_test, evict_file_fails_if_file_is_not_complete) {
|
||||
filesystem_item &fsi) -> api_error {
|
||||
EXPECT_STREQ("/test_evict.txt", api_path.c_str());
|
||||
EXPECT_FALSE(directory);
|
||||
fsi.api_path = api_path;
|
||||
fsi.api_parent = utils::path::get_parent_api_path(api_path);
|
||||
fsi.api_path = api_path;
|
||||
fsi.directory = directory;
|
||||
fsi.size = 1U;
|
||||
return api_error::success;
|
||||
});
|
||||
|
||||
auto file = std::make_shared<mock_open_file>();
|
||||
EXPECT_CALL(*file, is_directory).WillOnce(Return(false));
|
||||
EXPECT_CALL(*file, add).WillOnce(Return());
|
||||
EXPECT_CALL(*file, get_api_path).WillRepeatedly(Return("/test_evict.txt"));
|
||||
EXPECT_CALL(*file, get_source_path).WillRepeatedly(Return("/test_evict.src"));
|
||||
EXPECT_CALL(*file, is_modified).Times(2).WillRepeatedly(Return(false));
|
||||
EXPECT_CALL(*file, is_complete).Times(2).WillRepeatedly(Return(false));
|
||||
EXPECT_CALL(*file, is_complete).WillRepeatedly(Return(false));
|
||||
EXPECT_CALL(*file, is_directory).WillOnce(Return(false));
|
||||
EXPECT_CALL(*file, is_modified).WillRepeatedly(Return(false));
|
||||
EXPECT_CALL(*file, is_write_supported).WillRepeatedly(Return(true));
|
||||
EXPECT_CALL(mp, set_item_meta("/test_evict.txt", META_SOURCE, _))
|
||||
.WillOnce(Return(api_error::success));
|
||||
|
||||
@ -1461,26 +1469,16 @@ TEST_F(file_manager_test, file_is_closed_after_download_timeout) {
|
||||
|
||||
event_capture capture({"item_timeout"});
|
||||
|
||||
std::uint64_t handle{};
|
||||
std::shared_ptr<i_open_file> open_file;
|
||||
#if defined(_WIN32)
|
||||
EXPECT_EQ(api_error::success, mgr.open("/test_download_timeout.txt", false,
|
||||
{}, handle, open_file));
|
||||
#else
|
||||
EXPECT_EQ(api_error::success, mgr.open("/test_download_timeout.txt", false,
|
||||
O_RDWR, handle, open_file));
|
||||
#endif
|
||||
|
||||
EXPECT_CALL(mp, read_file_bytes)
|
||||
.WillRepeatedly([](const std::string & /* api_path */,
|
||||
std::size_t /*size*/, std::uint64_t offset,
|
||||
data_buffer & /*data*/,
|
||||
.WillRepeatedly([](const std::string & /* api_path */, std::size_t size,
|
||||
std::uint64_t offset, data_buffer &data,
|
||||
stop_type &stop_requested) -> api_error {
|
||||
if (stop_requested) {
|
||||
return api_error::download_stopped;
|
||||
}
|
||||
|
||||
if (offset == 0U) {
|
||||
data.resize(size);
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
@ -1491,13 +1489,25 @@ TEST_F(file_manager_test, file_is_closed_after_download_timeout) {
|
||||
return api_error::download_stopped;
|
||||
});
|
||||
|
||||
std::uint64_t handle{};
|
||||
std::shared_ptr<i_open_file> open_file;
|
||||
#if defined(_WIN32)
|
||||
EXPECT_EQ(api_error::success, mgr.open("/test_download_timeout.txt", false,
|
||||
{}, handle, open_file));
|
||||
#else
|
||||
EXPECT_EQ(api_error::success, mgr.open("/test_download_timeout.txt", false,
|
||||
O_RDWR, handle, open_file));
|
||||
#endif
|
||||
|
||||
data_buffer data{};
|
||||
EXPECT_EQ(api_error::success, open_file->read(1U, 0U, data));
|
||||
|
||||
mgr.close(handle);
|
||||
|
||||
EXPECT_CALL(mp, set_item_meta("/test_download_timeout.txt", META_SOURCE, _))
|
||||
.WillOnce(Return(api_error::success));
|
||||
if (open_file->is_write_supported()) {
|
||||
EXPECT_CALL(mp, set_item_meta("/test_download_timeout.txt", META_SOURCE, _))
|
||||
.WillOnce(Return(api_error::success));
|
||||
}
|
||||
|
||||
EXPECT_EQ(std::size_t(1U), mgr.get_open_file_count());
|
||||
capture.wait_for_empty();
|
||||
|
@ -213,9 +213,9 @@ TEST(json_serialize, can_handle_download_type) {
|
||||
EXPECT_EQ(download_type::direct, data.get<download_type>());
|
||||
EXPECT_STREQ("direct", data.get<std::string>().c_str());
|
||||
|
||||
data = download_type::fallback;
|
||||
EXPECT_EQ(download_type::fallback, data.get<download_type>());
|
||||
EXPECT_STREQ("fallback", data.get<std::string>().c_str());
|
||||
data = download_type::default_;
|
||||
EXPECT_EQ(download_type::default_, data.get<download_type>());
|
||||
EXPECT_STREQ("default", data.get<std::string>().c_str());
|
||||
|
||||
data = download_type::ring_buffer;
|
||||
EXPECT_EQ(download_type::ring_buffer, data.get<download_type>());
|
||||
@ -237,9 +237,9 @@ TEST(json_serialize, can_handle_atomic_download_type) {
|
||||
EXPECT_EQ(download_type::direct, data.get<atomic<download_type>>());
|
||||
EXPECT_STREQ("direct", data.get<std::string>().c_str());
|
||||
|
||||
data = atomic<download_type>{download_type::fallback};
|
||||
EXPECT_EQ(download_type::fallback, data.get<download_type>());
|
||||
EXPECT_STREQ("fallback", data.get<std::string>().c_str());
|
||||
data = atomic<download_type>{download_type::default_};
|
||||
EXPECT_EQ(download_type::default_, data.get<download_type>());
|
||||
EXPECT_STREQ("default", data.get<std::string>().c_str());
|
||||
|
||||
data = atomic<download_type>{download_type::ring_buffer};
|
||||
EXPECT_EQ(download_type::ring_buffer, data.get<atomic<download_type>>());
|
||||
|
@ -30,9 +30,11 @@
|
||||
#include "utils/event_capture.hpp"
|
||||
#include "utils/path.hpp"
|
||||
|
||||
namespace repertory {
|
||||
static constexpr const std::size_t test_chunk_size{1024U};
|
||||
namespace {
|
||||
constexpr const std::size_t test_chunk_size{1024U};
|
||||
}
|
||||
|
||||
namespace repertory {
|
||||
class open_file_test : public ::testing::Test {
|
||||
public:
|
||||
console_consumer con_consumer;
|
||||
|
@ -31,18 +31,29 @@
|
||||
namespace {
|
||||
constexpr const std::size_t test_chunk_size{1024U};
|
||||
|
||||
std::string ring_buffer_dir = repertory::utils::path::combine(
|
||||
repertory::test::get_test_output_dir(),
|
||||
{"file_manager_ring_buffer_open_file_test"});
|
||||
const auto ring_buffer_dir{
|
||||
repertory::utils::path::combine(
|
||||
repertory::test::get_test_output_dir(),
|
||||
{"file_manager_ring_buffer_open_file_test"}),
|
||||
};
|
||||
} // namespace
|
||||
|
||||
namespace repertory {
|
||||
TEST(ring_buffer_open_file, can_forward_to_last_chunk) {
|
||||
class ring_buffer_open_file_test : public ::testing::Test {
|
||||
public:
|
||||
console_consumer con_consumer;
|
||||
mock_provider provider;
|
||||
|
||||
protected:
|
||||
void SetUp() override { event_system::instance().start(); }
|
||||
|
||||
void TearDown() override { event_system::instance().stop(); }
|
||||
};
|
||||
|
||||
TEST_F(ring_buffer_open_file_test, can_forward_to_last_chunk) {
|
||||
auto source_path = test::generate_test_file_name("ring_buffer_open_file");
|
||||
|
||||
mock_provider prov;
|
||||
|
||||
EXPECT_CALL(prov, is_read_only()).WillRepeatedly(Return(false));
|
||||
EXPECT_CALL(provider, is_read_only()).WillRepeatedly(Return(false));
|
||||
|
||||
filesystem_item fsi;
|
||||
fsi.directory = false;
|
||||
@ -51,8 +62,8 @@ TEST(ring_buffer_open_file, can_forward_to_last_chunk) {
|
||||
fsi.source_path = source_path;
|
||||
|
||||
{
|
||||
ring_buffer_open_file file(ring_buffer_dir, test_chunk_size, 30U, fsi, prov,
|
||||
8U);
|
||||
ring_buffer_open_file file(ring_buffer_dir, test_chunk_size, 30U, fsi,
|
||||
provider, 8U);
|
||||
file.set(0U, 3U);
|
||||
file.forward(4U);
|
||||
|
||||
@ -63,17 +74,13 @@ TEST(ring_buffer_open_file, can_forward_to_last_chunk) {
|
||||
EXPECT_TRUE(file.get_read_state(chunk));
|
||||
}
|
||||
}
|
||||
|
||||
EXPECT_TRUE(utils::file::directory(ring_buffer_dir).remove_recursively());
|
||||
}
|
||||
|
||||
TEST(ring_buffer_open_file,
|
||||
can_forward_to_last_chunk_if_count_is_greater_than_remaining) {
|
||||
TEST_F(ring_buffer_open_file_test,
|
||||
can_forward_to_last_chunk_if_count_is_greater_than_remaining) {
|
||||
auto source_path = test::generate_test_file_name("ring_buffer_open_file");
|
||||
|
||||
mock_provider prov;
|
||||
|
||||
EXPECT_CALL(prov, is_read_only()).WillRepeatedly(Return(false));
|
||||
EXPECT_CALL(provider, is_read_only()).WillRepeatedly(Return(false));
|
||||
|
||||
filesystem_item fsi;
|
||||
fsi.directory = false;
|
||||
@ -82,8 +89,8 @@ TEST(ring_buffer_open_file,
|
||||
fsi.source_path = source_path;
|
||||
|
||||
{
|
||||
ring_buffer_open_file file(ring_buffer_dir, test_chunk_size, 30U, fsi, prov,
|
||||
8U);
|
||||
ring_buffer_open_file file(ring_buffer_dir, test_chunk_size, 30U, fsi,
|
||||
provider, 8U);
|
||||
file.set(0U, 3U);
|
||||
file.forward(100U);
|
||||
|
||||
@ -94,16 +101,12 @@ TEST(ring_buffer_open_file,
|
||||
EXPECT_FALSE(file.get_read_state(chunk));
|
||||
}
|
||||
}
|
||||
|
||||
EXPECT_TRUE(utils::file::directory(ring_buffer_dir).remove_recursively());
|
||||
}
|
||||
|
||||
TEST(ring_buffer_open_file, can_forward_after_last_chunk) {
|
||||
TEST_F(ring_buffer_open_file_test, can_forward_after_last_chunk) {
|
||||
auto source_path = test::generate_test_file_name("ring_buffer_open_file");
|
||||
|
||||
mock_provider prov;
|
||||
|
||||
EXPECT_CALL(prov, is_read_only()).WillRepeatedly(Return(false));
|
||||
EXPECT_CALL(provider, is_read_only()).WillRepeatedly(Return(false));
|
||||
|
||||
filesystem_item fsi;
|
||||
fsi.directory = false;
|
||||
@ -112,8 +115,8 @@ TEST(ring_buffer_open_file, can_forward_after_last_chunk) {
|
||||
fsi.source_path = source_path;
|
||||
|
||||
{
|
||||
ring_buffer_open_file file(ring_buffer_dir, test_chunk_size, 30U, fsi, prov,
|
||||
8U);
|
||||
ring_buffer_open_file file(ring_buffer_dir, test_chunk_size, 30U, fsi,
|
||||
provider, 8U);
|
||||
file.set(0U, 3U);
|
||||
file.forward(5U);
|
||||
|
||||
@ -125,16 +128,12 @@ TEST(ring_buffer_open_file, can_forward_after_last_chunk) {
|
||||
EXPECT_TRUE(file.get_read_state(chunk));
|
||||
}
|
||||
}
|
||||
|
||||
EXPECT_TRUE(utils::file::directory(ring_buffer_dir).remove_recursively());
|
||||
}
|
||||
|
||||
TEST(ring_buffer_open_file, can_forward_and_rollover_after_last_chunk) {
|
||||
TEST_F(ring_buffer_open_file_test, can_forward_and_rollover_after_last_chunk) {
|
||||
auto source_path = test::generate_test_file_name("ring_buffer_open_file");
|
||||
|
||||
mock_provider prov;
|
||||
|
||||
EXPECT_CALL(prov, is_read_only()).WillRepeatedly(Return(false));
|
||||
EXPECT_CALL(provider, is_read_only()).WillRepeatedly(Return(false));
|
||||
|
||||
filesystem_item fsi;
|
||||
fsi.directory = false;
|
||||
@ -143,8 +142,8 @@ TEST(ring_buffer_open_file, can_forward_and_rollover_after_last_chunk) {
|
||||
fsi.source_path = source_path;
|
||||
|
||||
{
|
||||
ring_buffer_open_file file(ring_buffer_dir, test_chunk_size, 30U, fsi, prov,
|
||||
8U);
|
||||
ring_buffer_open_file file(ring_buffer_dir, test_chunk_size, 30U, fsi,
|
||||
provider, 8U);
|
||||
file.set(16U, 20U);
|
||||
file.forward(8U);
|
||||
|
||||
@ -152,16 +151,12 @@ TEST(ring_buffer_open_file, can_forward_and_rollover_after_last_chunk) {
|
||||
EXPECT_EQ(std::size_t(21U), file.get_first_chunk());
|
||||
EXPECT_EQ(std::size_t(28U), file.get_last_chunk());
|
||||
}
|
||||
|
||||
EXPECT_TRUE(utils::file::directory(ring_buffer_dir).remove_recursively());
|
||||
}
|
||||
|
||||
TEST(ring_buffer_open_file, can_reverse_to_first_chunk) {
|
||||
TEST_F(ring_buffer_open_file_test, can_reverse_to_first_chunk) {
|
||||
auto source_path = test::generate_test_file_name("ring_buffer_open_file");
|
||||
|
||||
mock_provider prov;
|
||||
|
||||
EXPECT_CALL(prov, is_read_only()).WillRepeatedly(Return(false));
|
||||
EXPECT_CALL(provider, is_read_only()).WillRepeatedly(Return(false));
|
||||
|
||||
filesystem_item fsi;
|
||||
fsi.directory = false;
|
||||
@ -170,8 +165,8 @@ TEST(ring_buffer_open_file, can_reverse_to_first_chunk) {
|
||||
fsi.source_path = source_path;
|
||||
|
||||
{
|
||||
ring_buffer_open_file file(ring_buffer_dir, test_chunk_size, 30U, fsi, prov,
|
||||
8U);
|
||||
ring_buffer_open_file file(ring_buffer_dir, test_chunk_size, 30U, fsi,
|
||||
provider, 8U);
|
||||
file.set(0U, 3U);
|
||||
file.reverse(3U);
|
||||
|
||||
@ -182,17 +177,13 @@ TEST(ring_buffer_open_file, can_reverse_to_first_chunk) {
|
||||
EXPECT_TRUE(file.get_read_state(chunk));
|
||||
}
|
||||
}
|
||||
|
||||
EXPECT_TRUE(utils::file::directory(ring_buffer_dir).remove_recursively());
|
||||
}
|
||||
|
||||
TEST(ring_buffer_open_file,
|
||||
can_reverse_to_first_chunk_if_count_is_greater_than_remaining) {
|
||||
TEST_F(ring_buffer_open_file_test,
|
||||
can_reverse_to_first_chunk_if_count_is_greater_than_remaining) {
|
||||
auto source_path = test::generate_test_file_name("ring_buffer_open_file");
|
||||
|
||||
mock_provider prov;
|
||||
|
||||
EXPECT_CALL(prov, is_read_only()).WillRepeatedly(Return(false));
|
||||
EXPECT_CALL(provider, is_read_only()).WillRepeatedly(Return(false));
|
||||
|
||||
filesystem_item fsi;
|
||||
fsi.directory = false;
|
||||
@ -201,8 +192,8 @@ TEST(ring_buffer_open_file,
|
||||
fsi.source_path = source_path;
|
||||
|
||||
{
|
||||
ring_buffer_open_file file(ring_buffer_dir, test_chunk_size, 30U, fsi, prov,
|
||||
8U);
|
||||
ring_buffer_open_file file(ring_buffer_dir, test_chunk_size, 30U, fsi,
|
||||
provider, 8U);
|
||||
file.set(0U, 3U);
|
||||
file.reverse(13U);
|
||||
|
||||
@ -213,16 +204,12 @@ TEST(ring_buffer_open_file,
|
||||
EXPECT_TRUE(file.get_read_state(chunk));
|
||||
}
|
||||
}
|
||||
|
||||
EXPECT_TRUE(utils::file::directory(ring_buffer_dir).remove_recursively());
|
||||
}
|
||||
|
||||
TEST(ring_buffer_open_file, can_reverse_before_first_chunk) {
|
||||
TEST_F(ring_buffer_open_file_test, can_reverse_before_first_chunk) {
|
||||
auto source_path = test::generate_test_file_name("ring_buffer_open_file");
|
||||
|
||||
mock_provider prov;
|
||||
|
||||
EXPECT_CALL(prov, is_read_only()).WillRepeatedly(Return(false));
|
||||
EXPECT_CALL(provider, is_read_only()).WillRepeatedly(Return(false));
|
||||
|
||||
filesystem_item fsi;
|
||||
fsi.directory = false;
|
||||
@ -231,8 +218,8 @@ TEST(ring_buffer_open_file, can_reverse_before_first_chunk) {
|
||||
fsi.source_path = source_path;
|
||||
|
||||
{
|
||||
ring_buffer_open_file file(ring_buffer_dir, test_chunk_size, 30U, fsi, prov,
|
||||
8U);
|
||||
ring_buffer_open_file file(ring_buffer_dir, test_chunk_size, 30U, fsi,
|
||||
provider, 8U);
|
||||
file.set(1U, 3U);
|
||||
file.reverse(3U);
|
||||
|
||||
@ -244,16 +231,13 @@ TEST(ring_buffer_open_file, can_reverse_before_first_chunk) {
|
||||
EXPECT_TRUE(file.get_read_state(chunk));
|
||||
}
|
||||
}
|
||||
|
||||
EXPECT_TRUE(utils::file::directory(ring_buffer_dir).remove_recursively());
|
||||
}
|
||||
|
||||
TEST(ring_buffer_open_file, can_reverse_and_rollover_before_first_chunk) {
|
||||
TEST_F(ring_buffer_open_file_test,
|
||||
can_reverse_and_rollover_before_first_chunk) {
|
||||
auto source_path = test::generate_test_file_name("ring_buffer_open_file");
|
||||
|
||||
mock_provider prov;
|
||||
|
||||
EXPECT_CALL(prov, is_read_only()).WillRepeatedly(Return(false));
|
||||
EXPECT_CALL(provider, is_read_only()).WillRepeatedly(Return(false));
|
||||
|
||||
filesystem_item fsi;
|
||||
fsi.directory = false;
|
||||
@ -262,8 +246,8 @@ TEST(ring_buffer_open_file, can_reverse_and_rollover_before_first_chunk) {
|
||||
fsi.source_path = source_path;
|
||||
|
||||
{
|
||||
ring_buffer_open_file file(ring_buffer_dir, test_chunk_size, 30U, fsi, prov,
|
||||
8U);
|
||||
ring_buffer_open_file file(ring_buffer_dir, test_chunk_size, 30U, fsi,
|
||||
provider, 8U);
|
||||
file.set(16U, 20U);
|
||||
file.reverse(8U);
|
||||
|
||||
@ -279,16 +263,12 @@ TEST(ring_buffer_open_file, can_reverse_and_rollover_before_first_chunk) {
|
||||
EXPECT_TRUE(file.get_read_state(chunk));
|
||||
}
|
||||
}
|
||||
|
||||
EXPECT_TRUE(utils::file::directory(ring_buffer_dir).remove_recursively());
|
||||
}
|
||||
|
||||
TEST(ring_buffer_open_file, can_reverse_full_ring) {
|
||||
TEST_F(ring_buffer_open_file_test, can_reverse_full_ring) {
|
||||
auto source_path = test::generate_test_file_name("ring_buffer_open_file");
|
||||
|
||||
mock_provider prov;
|
||||
|
||||
EXPECT_CALL(prov, is_read_only()).WillRepeatedly(Return(false));
|
||||
EXPECT_CALL(provider, is_read_only()).WillRepeatedly(Return(false));
|
||||
|
||||
filesystem_item fsi;
|
||||
fsi.directory = false;
|
||||
@ -297,8 +277,8 @@ TEST(ring_buffer_open_file, can_reverse_full_ring) {
|
||||
fsi.source_path = source_path;
|
||||
|
||||
{
|
||||
ring_buffer_open_file file(ring_buffer_dir, test_chunk_size, 30U, fsi, prov,
|
||||
8U);
|
||||
ring_buffer_open_file file(ring_buffer_dir, test_chunk_size, 30U, fsi,
|
||||
provider, 8U);
|
||||
file.set(8U, 15U);
|
||||
file.reverse(16U);
|
||||
|
||||
@ -310,12 +290,10 @@ TEST(ring_buffer_open_file, can_reverse_full_ring) {
|
||||
EXPECT_FALSE(file.get_read_state(chunk));
|
||||
}
|
||||
}
|
||||
|
||||
EXPECT_TRUE(utils::file::directory(ring_buffer_dir).remove_recursively());
|
||||
}
|
||||
|
||||
TEST(ring_buffer_open_file, read_full_file) {
|
||||
auto &nf = test::create_random_file(test_chunk_size * 32u);
|
||||
TEST_F(ring_buffer_open_file_test, read_full_file) {
|
||||
auto &nf = test::create_random_file(test_chunk_size * 33u + 11u);
|
||||
auto download_source_path = nf.get_path();
|
||||
|
||||
auto dest_path = test::generate_test_file_name("ring_buffer_open_file");
|
||||
@ -327,14 +305,17 @@ TEST(ring_buffer_open_file, read_full_file) {
|
||||
filesystem_item fsi;
|
||||
fsi.directory = false;
|
||||
fsi.api_path = "/test.txt";
|
||||
fsi.size = test_chunk_size * 32u;
|
||||
fsi.size = test_chunk_size * 33u + 11u;
|
||||
fsi.source_path = test::generate_test_file_name("ring_buffer_open_file");
|
||||
|
||||
std::mutex read_mtx;
|
||||
EXPECT_CALL(mp, read_file_bytes)
|
||||
.WillRepeatedly([&nf](const std::string & /* api_path */,
|
||||
std::size_t size, std::uint64_t offset,
|
||||
data_buffer &data,
|
||||
stop_type &stop_requested) -> api_error {
|
||||
.WillRepeatedly([&read_mtx, &nf](const std::string & /* api_path */,
|
||||
std::size_t size, std::uint64_t offset,
|
||||
data_buffer &data,
|
||||
stop_type &stop_requested) -> api_error {
|
||||
mutex_lock lock(read_mtx);
|
||||
|
||||
EXPECT_FALSE(stop_requested);
|
||||
std::size_t bytes_read{};
|
||||
data.resize(size);
|
||||
@ -375,11 +356,9 @@ TEST(ring_buffer_open_file, read_full_file) {
|
||||
EXPECT_STREQ(hash1.value().c_str(), hash2.value().c_str());
|
||||
}
|
||||
}
|
||||
|
||||
EXPECT_TRUE(utils::file::directory(ring_buffer_dir).remove_recursively());
|
||||
}
|
||||
|
||||
TEST(ring_buffer_open_file, read_full_file_in_reverse) {
|
||||
TEST_F(ring_buffer_open_file_test, read_full_file_in_reverse) {
|
||||
auto &nf = test::create_random_file(test_chunk_size * 32u);
|
||||
auto download_source_path = nf.get_path();
|
||||
|
||||
@ -395,11 +374,14 @@ TEST(ring_buffer_open_file, read_full_file_in_reverse) {
|
||||
fsi.size = test_chunk_size * 32u;
|
||||
fsi.source_path = test::generate_test_file_name("ring_buffer_open_file");
|
||||
|
||||
std::mutex read_mtx;
|
||||
EXPECT_CALL(mp, read_file_bytes)
|
||||
.WillRepeatedly([&nf](const std::string & /* api_path */,
|
||||
std::size_t size, std::uint64_t offset,
|
||||
data_buffer &data,
|
||||
stop_type &stop_requested) -> api_error {
|
||||
.WillRepeatedly([&read_mtx, &nf](const std::string & /* api_path */,
|
||||
std::size_t size, std::uint64_t offset,
|
||||
data_buffer &data,
|
||||
stop_type &stop_requested) -> api_error {
|
||||
mutex_lock lock(read_mtx);
|
||||
|
||||
EXPECT_FALSE(stop_requested);
|
||||
std::size_t bytes_read{};
|
||||
data.resize(size);
|
||||
@ -417,15 +399,15 @@ TEST(ring_buffer_open_file, read_full_file_in_reverse) {
|
||||
EXPECT_TRUE(nf2);
|
||||
|
||||
auto to_read = fsi.size;
|
||||
std::size_t chunk = rb.get_total_chunks() - 1u;
|
||||
while (to_read) {
|
||||
std::size_t chunk = rb.get_total_chunks() - 1U;
|
||||
while (to_read > 0U) {
|
||||
data_buffer data{};
|
||||
EXPECT_EQ(api_error::success,
|
||||
rb.read(test_chunk_size, chunk * test_chunk_size, data));
|
||||
|
||||
std::size_t bytes_written{};
|
||||
EXPECT_TRUE(nf2.write(data, chunk * test_chunk_size, &bytes_written));
|
||||
chunk--;
|
||||
--chunk;
|
||||
to_read -= data.size();
|
||||
}
|
||||
nf2.close();
|
||||
@ -440,11 +422,9 @@ TEST(ring_buffer_open_file, read_full_file_in_reverse) {
|
||||
EXPECT_STREQ(hash1.value().c_str(), hash2.value().c_str());
|
||||
}
|
||||
}
|
||||
|
||||
EXPECT_TRUE(utils::file::directory(ring_buffer_dir).remove_recursively());
|
||||
}
|
||||
|
||||
TEST(ring_buffer_open_file, read_full_file_in_partial_chunks) {
|
||||
TEST_F(ring_buffer_open_file_test, read_full_file_in_partial_chunks) {
|
||||
auto &nf = test::create_random_file(test_chunk_size * 32u);
|
||||
auto download_source_path = nf.get_path();
|
||||
|
||||
@ -460,11 +440,14 @@ TEST(ring_buffer_open_file, read_full_file_in_partial_chunks) {
|
||||
fsi.size = test_chunk_size * 32u;
|
||||
fsi.source_path = test::generate_test_file_name("test");
|
||||
|
||||
std::mutex read_mtx;
|
||||
EXPECT_CALL(mp, read_file_bytes)
|
||||
.WillRepeatedly([&nf](const std::string & /* api_path */,
|
||||
std::size_t size, std::uint64_t offset,
|
||||
data_buffer &data,
|
||||
stop_type &stop_requested) -> api_error {
|
||||
.WillRepeatedly([&read_mtx, &nf](const std::string & /* api_path */,
|
||||
std::size_t size, std::uint64_t offset,
|
||||
data_buffer &data,
|
||||
stop_type &stop_requested) -> api_error {
|
||||
mutex_lock lock(read_mtx);
|
||||
|
||||
EXPECT_FALSE(stop_requested);
|
||||
std::size_t bytes_read{};
|
||||
data.resize(size);
|
||||
@ -506,11 +489,10 @@ TEST(ring_buffer_open_file, read_full_file_in_partial_chunks) {
|
||||
EXPECT_STREQ(hash1.value().c_str(), hash2.value().c_str());
|
||||
}
|
||||
}
|
||||
|
||||
EXPECT_TRUE(utils::file::directory(ring_buffer_dir).remove_recursively());
|
||||
}
|
||||
|
||||
TEST(ring_buffer_open_file, read_full_file_in_partial_chunks_in_reverse) {
|
||||
TEST_F(ring_buffer_open_file_test,
|
||||
read_full_file_in_partial_chunks_in_reverse) {
|
||||
auto &nf = test::create_random_file(test_chunk_size * 32u);
|
||||
auto download_source_path = nf.get_path();
|
||||
|
||||
@ -526,11 +508,14 @@ TEST(ring_buffer_open_file, read_full_file_in_partial_chunks_in_reverse) {
|
||||
fsi.size = test_chunk_size * 32u;
|
||||
fsi.source_path = test::generate_test_file_name("ring_buffer_open_file");
|
||||
|
||||
std::mutex read_mtx;
|
||||
EXPECT_CALL(mp, read_file_bytes)
|
||||
.WillRepeatedly([&nf](const std::string & /* api_path */,
|
||||
std::size_t size, std::uint64_t offset,
|
||||
data_buffer &data,
|
||||
stop_type &stop_requested) -> api_error {
|
||||
.WillRepeatedly([&read_mtx, &nf](const std::string & /* api_path */,
|
||||
std::size_t size, std::uint64_t offset,
|
||||
data_buffer &data,
|
||||
stop_type &stop_requested) -> api_error {
|
||||
mutex_lock lock(read_mtx);
|
||||
|
||||
EXPECT_FALSE(stop_requested);
|
||||
std::size_t bytes_read{};
|
||||
data.resize(size);
|
||||
@ -577,7 +562,5 @@ TEST(ring_buffer_open_file, read_full_file_in_partial_chunks_in_reverse) {
|
||||
EXPECT_STREQ(hash1.value().c_str(), hash2.value().c_str());
|
||||
}
|
||||
}
|
||||
|
||||
EXPECT_TRUE(utils::file::directory(ring_buffer_dir).remove_recursively());
|
||||
}
|
||||
} // namespace repertory
|
||||
|
Reference in New Issue
Block a user