3 Commits

Author SHA1 Message Date
5df2a5c3c0 Complete ring buffer and direct download support #26
All checks were successful
BlockStorage/repertory/pipeline/head This commit looks good
2024-12-22 13:45:32 -06:00
9bead305cf Complete ring buffer and direct download support #26 2024-12-22 13:44:50 -06:00
49fa2e6637 Complete ring buffer and direct download support #26 2024-12-22 13:44:22 -06:00
5 changed files with 23 additions and 12 deletions

View File

@ -21,6 +21,7 @@
* \#23 \[bug\] Incorrect file size displayed while upload is pending * \#23 \[bug\] Incorrect file size displayed while upload is pending
* \#24 RocksDB implementations should be transactional * \#24 RocksDB implementations should be transactional
* \#25 Writes should block when maximum cache size is reached * \#25 Writes should block when maximum cache size is reached
* \#26 Complete ring buffer and direct download support
### Changes from v2.0.1-rc ### Changes from v2.0.1-rc

View File

@ -44,6 +44,7 @@ using event_consumer = event_system::event_consumer;
#define E_FROM_STRING(t) t #define E_FROM_STRING(t) t
#define E_FROM_UINT16(t) std::to_string(t) #define E_FROM_UINT16(t) std::to_string(t)
#define E_FROM_UINT64(t) std::to_string(t) #define E_FROM_UINT64(t) std::to_string(t)
#define E_FROM_DOWNLOAD_TYPE(t) download_type_to_string(t)
#define E_PROP(type, name, short_name, ts) \ #define E_PROP(type, name, short_name, ts) \
private: \ private: \

View File

@ -74,6 +74,12 @@ E_SIMPLE1(drive_unmounted, info, true,
std::string, location, loc, E_FROM_STRING std::string, location, loc, E_FROM_STRING
); );
E_SIMPLE3(download_type_selected, info, true,
std::string, api_path, ap, E_FROM_STRING,
std::string, source, src, E_FROM_STRING,
download_type, download_type, type, E_FROM_DOWNLOAD_TYPE
);
E_SIMPLE1(event_level_changed, info, true, E_SIMPLE1(event_level_changed, info, true,
std::string, new_event_level, level, E_FROM_STRING std::string, new_event_level, level, E_FROM_STRING
); );

View File

@ -104,12 +104,12 @@ void file_manager::close_timed_out_files() {
} }
return items; return items;
}); });
for (auto &&closeable_file : closeable_list) { for (const auto &closeable_file : closeable_list) {
open_file_lookup_.erase(closeable_file->get_api_path()); open_file_lookup_.erase(closeable_file->get_api_path());
} }
file_lock.unlock(); file_lock.unlock();
for (auto &&closeable_file : closeable_list) { for (auto &closeable_file : closeable_list) {
closeable_file->close(); closeable_file->close();
event_system::instance().raise<item_timeout>( event_system::instance().raise<item_timeout>(
closeable_file->get_api_path()); closeable_file->get_api_path());
@ -267,7 +267,7 @@ auto file_manager::get_open_files() const
std::unordered_map<std::string, std::size_t> ret; std::unordered_map<std::string, std::size_t> ret;
recur_mutex_lock open_lock(open_file_mtx_); recur_mutex_lock open_lock(open_file_mtx_);
for (auto &&item : open_file_lookup_) { for (const auto &item : open_file_lookup_) {
ret[item.first] = item.second->get_open_file_count(); ret[item.first] = item.second->get_open_file_count();
} }
@ -415,6 +415,10 @@ auto file_manager::open(
utils::encryption::encrypting_reader::get_data_chunk_size(), utils::encryption::encrypting_reader::get_data_chunk_size(),
}; };
auto chunk_timeout = config_.get_enable_download_timeout()
? config_.get_download_timeout_secs()
: 0U;
auto ring_buffer_file_size{ auto ring_buffer_file_size{
static_cast<std::uint64_t>(config_.get_ring_buffer_file_size()) * static_cast<std::uint64_t>(config_.get_ring_buffer_file_size()) *
1024UL * 1024UL, 1024UL * 1024UL,
@ -422,9 +426,7 @@ auto file_manager::open(
auto ring_size{ring_buffer_file_size / chunk_size}; auto ring_size{ring_buffer_file_size / chunk_size};
const auto get_download_type = const auto get_download_type = [&](download_type type) -> download_type {
[this, &buffer_directory, &chunk_size, &fsi, &ring_buffer_file_size,
&ring_size](download_type type) -> download_type {
if (fsi.size == 0U) { if (fsi.size == 0U) {
return download_type::fallback; return download_type::fallback;
} }
@ -462,10 +464,11 @@ auto file_manager::open(
return download_type::direct; return download_type::direct;
}; };
auto chunk_timeout = config_.get_enable_download_timeout() auto type = get_download_type(config_.get_preferred_download_type());
? config_.get_download_timeout_secs() event_system::instance().raise<download_type_selected>(
: 0U; fsi.api_path, fsi.source_path, type);
switch (get_download_type(config_.get_preferred_download_type())) {
switch (type) {
case repertory::download_type::direct: { case repertory::download_type::direct: {
closeable_file = std::make_shared<direct_open_file>( closeable_file = std::make_shared<direct_open_file>(
chunk_size, chunk_timeout, fsi, provider_); chunk_size, chunk_timeout, fsi, provider_);
@ -882,7 +885,7 @@ void file_manager::stop() {
open_file_lookup_.clear(); open_file_lookup_.clear();
upload_lock.lock(); upload_lock.lock();
for (auto &&item : upload_lookup_) { for (auto &item : upload_lookup_) {
item.second->stop(); item.second->stop();
} }
upload_notify_.notify_all(); upload_notify_.notify_all();

View File

@ -106,7 +106,7 @@ ring_buffer_open_file::~ring_buffer_open_file() {
auto ring_buffer_open_file::can_handle_file(std::uint64_t file_size, auto ring_buffer_open_file::can_handle_file(std::uint64_t file_size,
std::size_t chunk_size, std::size_t chunk_size,
std::size_t ring_size) -> bool { std::size_t ring_size) -> bool {
return file_size <= (static_cast<std::uint64_t>(ring_size) * chunk_size); return file_size >= (static_cast<std::uint64_t>(ring_size) * chunk_size * 2U);
} }
auto ring_buffer_open_file::close() -> bool { auto ring_buffer_open_file::close() -> bool {