BlockStorage/repertory
1
0
Fork 0
Code Issues 10 Pull Requests 1 Releases 7 Wiki Activity

v2.0.3-rc (#32)
Some checks reported errors
BlockStorage/repertory/pipeline/head Something is wrong with the build of this commit
Details

Browse Source 
# Changelog

## v2.0.3-rc

### Issues

* \#28 \[bug\] Address slow directory responses in S3 mounts for deeply nested directories
* \#29 \[bug\] S3 error responses are not being logged
* \#30 \[bug\] Sia provider error responses are not logged
* \#31 \[bug\] S3 provider should limit max key size to 1024

### Changes from v2.0.2-rc

* Always use direct for read-only providers
* Fixed externally removed files not being processed during cleanup
* Fixed http headers not being added for requests
* Fixed incorrect `stat` values for remote mounts
* Fixed invalid directory nullptr error on remote mounts
* Fixed memory leak in event system
* Refactored application shutdown
* Refactored event system
* Updated build system to Alpine 3.21.0
* Updated build system to MinGW-w64 12.0.0
* Updated copyright to 2018-2025

Reviewed-on: #32
This commit is contained in:
Scott E. Graves
2025-02-11 17:26:24 -06:00
parent 8dd46b8ad8
commit fa439c634f
380 changed files with 9644 additions and 4217 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
repertory/librepertory/include/file_manager/cache_size_mgr.hpp
View File

@@ -1,5 +1,5 @@
/*
Copyright <2018-2024> <scott.e.graves@protonmail.com>
Copyright <2018-2025> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
@@ -28,6 +28,11 @@ namespace repertory {
class app_config;
class cache_size_mgr final {
private:
static constexpr const std::chrono::seconds cache_wait_secs{
5s,
};
public:
cache_size_mgr(const cache_size_mgr &) = delete;
cache_size_mgr(cache_size_mgr &&) = delete;
@@ -49,6 +54,9 @@ private:
std::condition_variable notify_;
stop_type stop_requested_{false};
private:
[[nodiscard]] auto get_stop_requested() const -> bool;
public:
[[nodiscard]] auto expand(std::uint64_t size) -> api_error;

24
repertory/librepertory/include/file_manager/direct_open_file.hpp
View File

@@ -1,5 +1,5 @@
/*
Copyright <2018-2024> <scott.e.graves@protonmail.com>
Copyright <2018-2025> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
@@ -42,8 +42,8 @@ public:
direct_open_file(const direct_open_file &) noexcept = delete;
direct_open_file(direct_open_file &&) noexcept = delete;
auto operator=(direct_open_file &&) noexcept -> direct_open_file & = delete;
auto
operator=(const direct_open_file &) noexcept -> direct_open_file & = delete;
auto operator=(const direct_open_file &) noexcept
-> direct_open_file & = delete;
private:
std::array<data_buffer, min_ring_size> ring_data_;
@@ -51,22 +51,26 @@ private:
protected:
[[nodiscard]] auto on_check_start() -> bool override;
[[nodiscard]] auto
on_chunk_downloaded(std::size_t /* chunk */,
const data_buffer & /* buffer */) -> api_error override {
[[nodiscard]] auto on_chunk_downloaded(std::size_t /* chunk */,
const data_buffer & /* buffer */)
-> api_error override {
return api_error::success;
}
[[nodiscard]] auto
on_read_chunk(std::size_t chunk, std::size_t read_size,
std::uint64_t read_offset, data_buffer &data,
std::size_t &bytes_read) -> api_error override;
[[nodiscard]] auto on_read_chunk(std::size_t chunk, std::size_t read_size,
std::uint64_t read_offset, data_buffer &data,
std::size_t &bytes_read)
-> api_error override;
[[nodiscard]] auto use_buffer(std::size_t chunk,
std::function<api_error(data_buffer &)> func)
-> api_error override;
public:
[[nodiscard]] auto get_source_path() const -> std::string override {
return "direct";
}
[[nodiscard]] auto native_operation(native_operation_callback /* callback */)
-> api_error override {
return api_error::not_supported;

86
repertory/librepertory/include/file_manager/events.hpp
View File

@@ -1,86 +0,0 @@
/*
Copyright <2018-2024> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#ifndef REPERTORY_INCLUDE_FILE_MANAGER_EVENTS_HPP_
#define REPERTORY_INCLUDE_FILE_MANAGER_EVENTS_HPP_
#include "events/events.hpp"
#include "types/repertory.hpp"
namespace repertory {
// clang-format off
E_SIMPLE2(download_begin, info, true,
std::string, api_path, ap, E_FROM_STRING,
std::string, dest_path, dest, E_FROM_STRING
);
E_SIMPLE3(download_end, info, true,
std::string, api_path, ap, E_FROM_STRING,
std::string, dest_path, dest, E_FROM_STRING,
api_error, result, result, E_FROM_API_FILE_ERROR
);
E_SIMPLE3(download_progress, info, true,
std::string, api_path, ap, E_FROM_STRING,
std::string, dest_path, dest, E_FROM_STRING,
double, progress, prog, E_FROM_DOUBLE_PRECISE
);
E_SIMPLE2(download_restored, info, true,
std::string, api_path, ap, E_FROM_STRING,
std::string, dest_path, dest, E_FROM_STRING
);
E_SIMPLE3(download_restore_failed, error, true,
std::string, api_path, ap, E_FROM_STRING,
std::string, dest_path, dest, E_FROM_STRING,
std::string, error, err, E_FROM_STRING
);
E_SIMPLE3(download_resume_add_failed, error, true,
std::string, api_path, ap, E_FROM_STRING,
std::string, dest_path, dest, E_FROM_STRING,
std::string, error, err, E_FROM_STRING
);
E_SIMPLE2(download_resume_added, debug, true,
std::string, api_path, ap, E_FROM_STRING,
std::string, dest_path, dest, E_FROM_STRING
);
E_SIMPLE2(download_resume_removed, debug, true,
std::string, api_path, ap, E_FROM_STRING,
std::string, dest_path, dest, E_FROM_STRING
);
E_SIMPLE1(item_timeout, trace, true,
std::string, api_path, ap, E_FROM_STRING
);
E_SIMPLE3(download_type_selected, debug, true,
std::string, api_path, ap, E_FROM_STRING,
std::string, source, src, E_FROM_STRING,
download_type, download_type, type, E_FROM_DOWNLOAD_TYPE
);
// clang-format on
} // namespace repertory
#endif // REPERTORY_INCLUDE_FILE_MANAGER_EVENTS_HPP_

25
repertory/librepertory/include/file_manager/file_manager.hpp
View File

@@ -1,5 +1,5 @@
/*
Copyright <2018-2024> <scott.e.graves@protonmail.com>
Copyright <2018-2025> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
@@ -24,7 +24,7 @@
#include "db/i_file_mgr_db.hpp"
#include "events/event_system.hpp"
#include "events/events.hpp"
#include "events/types/file_upload_completed.hpp"
#include "file_manager/i_file_manager.hpp"
#include "file_manager/i_open_file.hpp"
#include "file_manager/i_upload_manager.hpp"
@@ -39,6 +39,11 @@ class i_provider;
class file_manager final : public i_file_manager, public i_upload_manager {
E_CONSUMER();
private:
static constexpr const std::chrono::seconds queue_wait_secs{
5s,
};
public:
file_manager(app_config &config, i_provider &provider);
@@ -72,15 +77,19 @@ private:
void close_timed_out_files();
auto get_open_file_by_handle(std::uint64_t handle) const
[[nodiscard]] auto get_open_file_by_handle(std::uint64_t handle) const
-> std::shared_ptr<i_closeable_open_file>;
auto get_open_file_count(const std::string &api_path) const -> std::size_t;
[[nodiscard]] auto get_open_file_count(const std::string &api_path) const
-> std::size_t;
auto open(const std::string &api_path, bool directory,
const open_file_data &ofd, std::uint64_t &handle,
std::shared_ptr<i_open_file> &file,
std::shared_ptr<i_closeable_open_file> closeable_file) -> api_error;
[[nodiscard]] auto get_stop_requested() const -> bool;
[[nodiscard]] auto open(const std::string &api_path, bool directory,
const open_file_data &ofd, std::uint64_t &handle,
std::shared_ptr<i_open_file> &file,
std::shared_ptr<i_closeable_open_file> closeable_file)
-> api_error;
void queue_upload(const std::string &api_path, const std::string &source_path,
bool no_lock);

2
repertory/librepertory/include/file_manager/i_file_manager.hpp
View File

@@ -1,5 +1,5 @@
/*
Copyright <2018-2024> <scott.e.graves@protonmail.com>
Copyright <2018-2025> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal

2
repertory/librepertory/include/file_manager/i_open_file.hpp
View File

@@ -1,5 +1,5 @@
/*
Copyright <2018-2024> <scott.e.graves@protonmail.com>
Copyright <2018-2025> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal

2
repertory/librepertory/include/file_manager/i_upload_manager.hpp
View File

@@ -1,5 +1,5 @@
/*
Copyright <2018-2024> <scott.e.graves@protonmail.com>
Copyright <2018-2025> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal

18
repertory/librepertory/include/file_manager/open_file.hpp
View File

@@ -1,5 +1,5 @@
/*
Copyright <2018-2024> <scott.e.graves@protonmail.com>
Copyright <2018-2025> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
@@ -78,8 +78,8 @@ private:
stop_type stop_requested_{false};
private:
[[nodiscard]] auto adjust_cache_size(std::uint64_t file_size,
bool shrink) -> api_error;
[[nodiscard]] auto adjust_cache_size(std::uint64_t file_size, bool shrink)
-> api_error;
[[nodiscard]] auto check_start() -> api_error;
@@ -88,6 +88,8 @@ private:
void download_range(std::size_t begin_chunk, std::size_t end_chunk,
bool should_reset);
[[nodiscard]] auto get_stop_requested() const -> bool;
void set_modified();
void set_read_state(std::size_t chunk);
@@ -111,12 +113,12 @@ public:
return true;
}
[[nodiscard]] auto
native_operation(native_operation_callback callback) -> api_error override;
[[nodiscard]] auto native_operation(native_operation_callback callback)
-> api_error override;
[[nodiscard]] auto
native_operation(std::uint64_t new_file_size,
native_operation_callback callback) -> api_error override;
[[nodiscard]] auto native_operation(std::uint64_t new_file_size,
native_operation_callback callback)
-> api_error override;
void remove(std::uint64_t handle) override;

16
repertory/librepertory/include/file_manager/open_file_base.hpp
View File

@@ -1,5 +1,5 @@
/*
Copyright <2018-2024> <scott.e.graves@protonmail.com>
Copyright <2018-2025> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
@@ -161,7 +161,7 @@ protected:
void set_source_path(std::string source_path);
void wait_for_io(stop_type &stop_requested);
void wait_for_io(stop_type_callback stop_requested_cb);
public:
void add(std::uint64_t handle, open_file_data ofd) override;
@@ -186,17 +186,17 @@ public:
[[nodiscard]] auto get_handles() const -> std::vector<std::uint64_t> override;
[[nodiscard]] auto
get_open_data() -> std::map<std::uint64_t, open_file_data> & override;
[[nodiscard]] auto get_open_data()
-> std::map<std::uint64_t, open_file_data> & override;
[[nodiscard]] auto get_open_data() const
-> const std::map<std::uint64_t, open_file_data> & override;
[[nodiscard]] auto
get_open_data(std::uint64_t handle) -> open_file_data & override;
[[nodiscard]] auto get_open_data(std::uint64_t handle)
-> open_file_data & override;
[[nodiscard]] auto
get_open_data(std::uint64_t handle) const -> const open_file_data & override;
[[nodiscard]] auto get_open_data(std::uint64_t handle) const
-> const open_file_data & override;
[[nodiscard]] auto get_open_file_count() const -> std::size_t override;

25
repertory/librepertory/include/file_manager/ring_buffer_base.hpp
View File

@@ -1,5 +1,5 @@
/*
Copyright <2018-2024> <scott.e.graves@protonmail.com>
Copyright <2018-2025> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
@@ -44,8 +44,8 @@ public:
ring_buffer_base(const ring_buffer_base &) noexcept = delete;
ring_buffer_base(ring_buffer_base &&) noexcept = delete;
auto operator=(ring_buffer_base &&) noexcept -> ring_buffer_base & = delete;
auto
operator=(const ring_buffer_base &) noexcept -> ring_buffer_base & = delete;
auto operator=(const ring_buffer_base &) noexcept
-> ring_buffer_base & = delete;
public:
static constexpr const auto min_ring_size{5U};
@@ -73,6 +73,8 @@ private:
void update_position(std::size_t count, bool is_forward);
[[nodiscard]] auto get_stop_requested() const -> bool;
protected:
[[nodiscard]] auto has_reader_thread() const -> bool {
return reader_thread_ != nullptr;
@@ -84,9 +86,9 @@ protected:
[[nodiscard]] virtual auto on_check_start() -> bool = 0;
[[nodiscard]] virtual auto
on_chunk_downloaded(std::size_t chunk,
const data_buffer &buffer) -> api_error = 0;
[[nodiscard]] virtual auto on_chunk_downloaded(std::size_t chunk,
const data_buffer &buffer)
-> api_error = 0;
[[nodiscard]] virtual auto
on_read_chunk(std::size_t chunk, std::size_t read_size,
@@ -94,8 +96,8 @@ protected:
std::size_t &bytes_read) -> api_error = 0;
[[nodiscard]] virtual auto
use_buffer(std::size_t chunk,
std::function<api_error(data_buffer &)> func) -> api_error = 0;
use_buffer(std::size_t chunk, std::function<api_error(data_buffer &)> func)
-> api_error = 0;
public:
auto close() -> bool override;
@@ -139,9 +141,10 @@ public:
void set_api_path(const std::string &api_path) override;
[[nodiscard]] auto
write(std::uint64_t /* write_offset */, const data_buffer & /* data */,
std::size_t & /* bytes_written */) -> api_error override {
[[nodiscard]] auto write(std::uint64_t /* write_offset */,
const data_buffer & /* data */,
std::size_t & /* bytes_written */)
-> api_error override {
return api_error::not_supported;
}
};

2
repertory/librepertory/include/file_manager/ring_buffer_open_file.hpp
View File

@@ -1,5 +1,5 @@
/*
Copyright <2018-2024> <scott.e.graves@protonmail.com>
Copyright <2018-2025> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal

4
repertory/librepertory/include/file_manager/upload.hpp
View File

@@ -1,5 +1,5 @@
/*
Copyright <2018-2024> <scott.e.graves@protonmail.com>
Copyright <2018-2025> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
@@ -47,8 +47,8 @@ private:
private:
bool cancelled_{false};
api_error error_{api_error::success};
std::unique_ptr<std::thread> thread_;
stop_type stop_requested_{false};
std::unique_ptr<std::thread> thread_;
private:
void upload_thread();