BlockStorage/repertory
1
0
Fork 0
Code Issues 3 Pull Requests Releases 9 Wiki Activity

updated build system

Browse Source
This commit is contained in:
sgraves76
2025-08-12 18:28:15 -05:00
parent 02d59cb047
commit 3effb4c80a
50 changed files with 1915 additions and 536 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
repertory/librepertory/include/app_config.hpp
View File

@@ -31,24 +31,22 @@ private:
static stop_type stop_requested;
public:
[[nodiscard]] static auto default_agent_name(const provider_type &prov)
[[nodiscard]] static auto default_agent_name(provider_type prov)
-> std::string;
[[nodiscard]] static auto default_api_port(const provider_type &prov)
[[nodiscard]] static auto default_api_port(provider_type prov)
-> std::uint16_t;
[[nodiscard]] static auto default_data_directory(const provider_type &prov)
[[nodiscard]] static auto default_data_directory(provider_type prov)
-> std::string;
[[nodiscard]] static auto default_remote_api_port(const provider_type &prov)
[[nodiscard]] static auto default_remote_api_port(provider_type prov)
-> std::uint16_t;
[[nodiscard]] static auto default_rpc_port() -> std::uint16_t;
[[nodiscard]] static auto get_provider_display_name(const provider_type &prov)
[[nodiscard]] static auto get_provider_display_name(provider_type prov)
-> std::string;
[[nodiscard]] static auto get_provider_name(const provider_type &prov)
[[nodiscard]] static auto get_provider_name(provider_type prov)
-> std::string;
[[nodiscard]] static auto get_root_data_directory() -> std::string;
@@ -59,7 +57,7 @@ public:
static void set_stop_requested();
public:
app_config(const provider_type &prov, std::string_view data_directory = "");
app_config(provider_type prov, std::string_view data_directory = "");
app_config() = delete;
app_config(app_config &&) = delete;
@@ -72,9 +70,9 @@ public:
private:
provider_type prov_;
atomic<std::string> api_password_;
utils::atomic<std::string> api_password_;
std::atomic<std::uint16_t> api_port_;
atomic<std::string> api_user_;
utils::atomic<std::string> api_user_;
std::atomic<bool> config_changed_;
std::atomic<database_type> db_type_{database_type::rocksdb};
std::atomic<std::uint8_t> download_timeout_secs_;
@@ -100,14 +98,14 @@ private:
private:
std::string cache_directory_;
std::string data_directory_;
atomic<encrypt_config> encrypt_config_;
atomic<host_config> host_config_;
utils::atomic<encrypt_config> encrypt_config_;
utils::atomic<host_config> host_config_;
std::string log_directory_;
mutable std::recursive_mutex read_write_mutex_;
atomic<remote::remote_config> remote_config_;
atomic<remote::remote_mount> remote_mount_;
atomic<s3_config> s3_config_;
atomic<sia_config> sia_config_;
utils::atomic<remote::remote_config> remote_config_;
utils::atomic<remote::remote_mount> remote_mount_;
utils::atomic<s3_config> s3_config_;
utils::atomic<sia_config> sia_config_;
std::unordered_map<std::string, std::function<std::string()>>
value_get_lookup_;
std::unordered_map<std::string,

7
repertory/librepertory/include/comm/curl/curl_comm.hpp
View File

@@ -43,7 +43,7 @@ private:
using write_callback = size_t (*)(char *, size_t, size_t, void *);
struct read_write_info final {
data_buffer data{};
data_buffer data;
stop_type_callback stop_requested_cb;
};
@@ -93,9 +93,8 @@ public:
}
data_buffer data{};
const auto key =
utils::encryption::generate_key<utils::encryption::hash_256_t>(
request.decryption_token.value());
const auto key = utils::encryption::generate_key<utils::hash::hash_256_t>(
request.decryption_token.value());
if (not utils::encryption::read_encrypted_range(
request.range.value(), key,
[&](data_buffer &ct, std::uint64_t start_offset,

5
repertory/librepertory/include/providers/provider.hpp
View File

@@ -28,9 +28,8 @@ namespace repertory {
class app_config;
class i_provider;
[[nodiscard]] auto
create_provider(const provider_type &prov,
app_config &config) -> std::unique_ptr<i_provider>;
[[nodiscard]] auto create_provider(provider_type prov, app_config &config)
-> std::unique_ptr<i_provider>;
} // namespace repertory
#endif // REPERTORY_INCLUDE_PROVIDERS_PROVIDER_HPP_

101
repertory/librepertory/include/types/repertory.hpp
View File

@@ -22,6 +22,8 @@
#ifndef REPERTORY_INCLUDE_TYPES_REPERTORY_HPP_
#define REPERTORY_INCLUDE_TYPES_REPERTORY_HPP_
#include "utils/atomic.hpp"
namespace repertory {
inline constexpr auto default_api_password_size{48U};
inline constexpr auto default_download_timeout_secs{30U};
@@ -40,6 +42,7 @@ inline constexpr auto default_med_freq_interval_secs{
inline constexpr auto default_online_check_retry_secs{60U};
inline constexpr auto default_retry_read_count{6U};
inline constexpr auto default_ring_buffer_file_size{512U};
inline constexpr auto default_rpc_port{std::uint16_t(10000U)};
inline constexpr auto default_task_wait_ms{100U};
inline constexpr auto default_timeout_ms{60000U};
inline constexpr auto default_ui_mgmt_port{std::uint16_t{30000U}};
@@ -54,95 +57,6 @@ inline constexpr auto min_retry_read_count{std::uint16_t(2U)};
inline constexpr auto min_ring_buffer_file_size{std::uint16_t(64U)};
inline constexpr auto min_task_wait_ms{std::uint16_t(50U)};
template <typename data_t> class atomic final {
public:
atomic() : mtx_(std::make_shared<std::mutex>()) {}
atomic(const atomic &at_data)
: data_(at_data.load()), mtx_(std::make_shared<std::mutex>()) {}
atomic(data_t data)
: data_(std::move(data)), mtx_(std::make_shared<std::mutex>()) {}
atomic(atomic &&) = default;
~atomic() = default;
private:
data_t data_;
std::shared_ptr<std::mutex> mtx_;
public:
[[nodiscard]] auto load() const -> data_t {
mutex_lock lock(*mtx_);
return data_;
}
auto store(data_t data) -> data_t {
mutex_lock lock(*mtx_);
data_ = std::move(data);
return data_;
}
auto operator=(const atomic &at_data) -> atomic & {
if (&at_data == this) {
return *this;
}
store(at_data.load());
return *this;
}
auto operator=(atomic &&) -> atomic & = default;
auto operator=(data_t data) -> atomic & {
if (&data == &data_) {
return *this;
}
store(std::move(data));
return *this;
}
[[nodiscard]] auto operator==(const atomic &at_data) const -> bool {
if (&at_data == this) {
return true;
}
mutex_lock lock(*mtx_);
return at_data.load() == data_;
}
[[nodiscard]] auto operator==(const data_t &data) const -> bool {
if (&data == &data_) {
return true;
}
mutex_lock lock(*mtx_);
return data == data_;
}
[[nodiscard]] auto operator!=(const atomic &at_data) const -> bool {
if (&at_data == this) {
return false;
}
mutex_lock lock(*mtx_);
return at_data.load() != data_;
}
[[nodiscard]] auto operator!=(const data_t &data) const -> bool {
if (&data == &data_) {
return false;
}
mutex_lock lock(*mtx_);
return data != data_;
}
[[nodiscard]] operator data_t() const { return load(); }
};
inline constexpr auto max_time{
std::numeric_limits<std::uint64_t>::max(),
};
@@ -617,12 +531,15 @@ template <> struct adl_serializer<repertory::sia_config> {
}
};
template <typename data_t> struct adl_serializer<repertory::atomic<data_t>> {
static void to_json(json &data, const repertory::atomic<data_t> &value) {
template <typename data_t>
struct adl_serializer<repertory::utils::atomic<data_t>> {
static void to_json(json &data,
const repertory::utils::atomic<data_t> &value) {
data = value.load();
}
static void from_json(const json &data, repertory::atomic<data_t> &value) {
static void from_json(const json &data,
repertory::utils::atomic<data_t> &value) {
value.store(data.get<data_t>());
}
};

2
repertory/librepertory/include/utils/cli_utils.hpp
View File

@@ -92,7 +92,7 @@ inline const std::vector<option> option_list = {
// Prototypes
void get_api_authentication_data(std::string &user, std::string &password,
std::uint16_t &port, const provider_type &prov,
std::uint16_t &port, provider_type prov,
const std::string &data_directory);
[[nodiscard]] auto get_provider_type_from_args(std::vector<const char *> args)

3
repertory/librepertory/include/utils/utils.hpp
View File

@@ -38,8 +38,7 @@ create_rocksdb(const app_config &cfg, const std::string &name,
std::vector<rocksdb::ColumnFamilyHandle *> &handles, bool clear)
-> std::unique_ptr<rocksdb::TransactionDB>;
[[nodiscard]] auto create_volume_label(const provider_type &prov)
-> std::string;
[[nodiscard]] auto create_volume_label(provider_type prov) -> std::string;
[[nodiscard]] auto get_attributes_from_meta(const api_meta_map &meta) -> DWORD;
} // namespace utils

21
repertory/librepertory/src/app_config.cpp
View File

@@ -25,6 +25,7 @@
#include "events/types/event_level_changed.hpp"
#include "file_manager/cache_size_mgr.hpp"
#include "platform/platform.hpp"
#include "types/repertory.hpp"
#include "types/startup_exception.hpp"
#include "utils/common.hpp"
#include "utils/error_utils.hpp"
@@ -39,7 +40,7 @@ auto get_value(const json &data, const std::string &name, dest &dst,
REPERTORY_USES_FUNCTION_NAME();
try {
if (data.find(name) == data.end()) {
if (not data.contains(name)) {
found = false;
return false;
}
@@ -63,11 +64,11 @@ auto app_config::get_stop_requested() -> bool { return stop_requested.load(); }
void app_config::set_stop_requested() { stop_requested.store(true); }
app_config::app_config(const provider_type &prov,
app_config::app_config(provider_type prov,
std::string_view data_directory)
: prov_(prov),
api_password_(utils::generate_random_string(default_api_password_size)),
api_port_(default_rpc_port()),
api_port_(default_rpc_port),
api_user_(std::string{REPERTORY}),
config_changed_(false),
download_timeout_secs_(default_download_timeout_secs),
@@ -674,7 +675,7 @@ app_config::app_config(const provider_type &prov,
};
}
auto app_config::default_agent_name(const provider_type &prov) -> std::string {
auto app_config::default_agent_name(provider_type prov) -> std::string {
static const std::array<std::string,
static_cast<std::size_t>(provider_type::unknown)>
PROVIDER_AGENT_NAMES = {
@@ -687,7 +688,7 @@ auto app_config::default_agent_name(const provider_type &prov) -> std::string {
return PROVIDER_AGENT_NAMES.at(static_cast<std::size_t>(prov));
}
auto app_config::default_api_port(const provider_type &prov) -> std::uint16_t {
auto app_config::default_api_port(provider_type prov) -> std::uint16_t {
static const std::array<std::uint16_t,
static_cast<std::size_t>(provider_type::unknown)>
PROVIDER_API_PORTS = {
@@ -722,7 +723,7 @@ auto app_config::get_root_data_directory() -> std::string {
return data_directory;
}
auto app_config::default_data_directory(const provider_type &prov)
auto app_config::default_data_directory(provider_type prov)
-> std::string {
return utils::path::combine(app_config::get_root_data_directory(),
{
@@ -730,7 +731,7 @@ auto app_config::default_data_directory(const provider_type &prov)
});
}
auto app_config::default_remote_api_port(const provider_type &prov)
auto app_config::default_remote_api_port(provider_type prov)
-> std::uint16_t {
static const std::array<std::uint16_t,
static_cast<std::size_t>(provider_type::unknown)>
@@ -743,8 +744,6 @@ auto app_config::default_remote_api_port(const provider_type &prov)
return PROVIDER_REMOTE_PORTS.at(static_cast<std::size_t>(prov));
}
auto app_config::default_rpc_port() -> std::uint16_t { return 10000U; }
auto app_config::get_api_password() const -> std::string {
return api_password_;
}
@@ -930,7 +929,7 @@ auto app_config::get_preferred_download_type() const -> download_type {
return preferred_download_type_;
}
auto app_config::get_provider_display_name(const provider_type &prov)
auto app_config::get_provider_display_name(provider_type prov)
-> std::string {
static const std::array<std::string,
static_cast<std::size_t>(provider_type::unknown) + 1U>
@@ -940,7 +939,7 @@ auto app_config::get_provider_display_name(const provider_type &prov)
return PROVIDER_DISPLAY_NAMES.at(static_cast<std::size_t>(prov));
}
auto app_config::get_provider_name(const provider_type &prov) -> std::string {
auto app_config::get_provider_name(provider_type prov) -> std::string {
static const std::array<std::string,
static_cast<std::size_t>(provider_type::unknown) + 1U>
PROVIDER_NAMES = {

1
repertory/librepertory/src/drives/fuse/fuse_base.cpp
View File

@@ -19,7 +19,6 @@
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include <unistd.h>
#if !defined(_WIN32)
#include "drives/fuse/fuse_base.hpp"

85
repertory/librepertory/src/providers/encrypt/encrypt_provider.cpp
View File

@@ -40,7 +40,6 @@
#include "utils/file_utils.hpp"
#include "utils/path.hpp"
#include "utils/polling.hpp"
#include <chrono>
namespace repertory {
encrypt_provider::encrypt_provider(app_config &config)
@@ -65,7 +64,7 @@ auto encrypt_provider::create_api_file(const std::string &api_path,
directory
? 0U
: utils::encryption::encrypting_reader::calculate_encrypted_size(
source_path);
source_path, false);
file.modified_date = times->get(utils::file::time_type::written);
file.source_path = source_path;
@@ -285,26 +284,28 @@ auto encrypt_provider::get_directory_items(const std::string &api_path,
(item1.api_path.compare(item2.api_path) < 0));
});
list.insert(list.begin(),
directory_item{
"..",
"",
true,
0U,
{
{META_DIRECTORY, utils::string::from_bool(true)},
},
});
list.insert(list.begin(),
directory_item{
".",
"",
true,
0U,
{
{META_DIRECTORY, utils::string::from_bool(true)},
},
});
list.insert(list.begin(), directory_item{
.api_path = "..",
.api_parent = "",
.directory = true,
.size = 0U,
.meta =
{
{META_DIRECTORY,
utils::string::from_bool(true)},
},
});
list.insert(list.begin(), directory_item{
.api_path = ".",
.api_parent = "",
.directory = true,
.size = 0U,
.meta =
{
{META_DIRECTORY,
utils::string::from_bool(true)},
},
});
return api_error::success;
});
}
@@ -387,7 +388,7 @@ auto encrypt_provider::get_file_size(const std::string &api_path,
}
file_size = utils::encryption::encrypting_reader::calculate_encrypted_size(
source_path);
source_path, false);
return api_error::success;
} catch (const std::exception &ex) {
utils::error::raise_error(function_name, ex, api_path,
@@ -425,7 +426,7 @@ auto encrypt_provider::get_filesystem_item(const std::string &api_path,
fsi.api_parent = utils::path::get_parent_api_path(fsi.api_path);
fsi.directory = false;
fsi.size = utils::encryption::encrypting_reader::calculate_encrypted_size(
source_path);
source_path, false);
fsi.source_path = source_path;
return api_error::success;
@@ -729,12 +730,12 @@ auto encrypt_provider::process_directory_entry(
api_parent + "/" + reader.get_encrypted_file_name());
file_res = db_->add_or_update_file(i_file_db::file_data{
api_path,
dynamic_cast<const utils::file::i_file *>(&dir_entry)
->size()
.value_or(0U),
reader.get_iv_list(),
dir_entry.get_path(),
.api_path = api_path,
.file_size = dynamic_cast<const utils::file::i_file *>(&dir_entry)
->size()
.value_or(0U),
.iv_list = reader.get_iv_list(),
.source_path = dir_entry.get_path(),
});
if (file_res != api_error::success) {
// TODO raise error
@@ -802,8 +803,7 @@ auto encrypt_provider::read_file_bytes(const std::string &api_path,
"failed to update file");
return result;
}
} else if (reader_lookup_.find(file_data.source_path) ==
reader_lookup_.end()) {
} else if (not reader_lookup_.contains(file_data.source_path)) {
auto info{std::make_shared<reader_info>()};
info->reader = std::make_unique<utils::encryption::encrypting_reader>(
api_path, file_data.source_path,
@@ -827,8 +827,9 @@ auto encrypt_provider::read_file_bytes(const std::string &api_path,
data.resize(size);
auto res{
info->reader->reader_function(reinterpret_cast<char *>(data.data()), 1U,
data.size(), info->reader.get()),
utils::encryption::encrypting_reader::reader_function(
reinterpret_cast<char *>(data.data()), 1U, data.size(),
info->reader.get()),
};
return res == 0 ? api_error::os_error : api_error::success;
}
@@ -941,15 +942,19 @@ auto encrypt_provider::start(api_item_added_callback /*api_item_added*/,
}
polling::instance().set_callback({
"check_deleted",
polling::frequency::low,
[this](auto &&stop_requested) { remove_deleted_files(stop_requested); },
.name = "check_deleted",
.freq = polling::frequency::low,
.action =
[this](auto &&stop_requested) {
remove_deleted_files(stop_requested);
},
});
polling::instance().set_callback({
"remove_expired",
polling::frequency::high,
[this](auto && /* stop_requested */) { remove_expired_files(); },
.name = "remove_expired",
.freq = polling::frequency::high,
.action =
[this](auto && /* stop_requested */) { remove_expired_files(); },
});
event_system::instance().raise<service_start_end>(function_name,

2
repertory/librepertory/src/providers/provider.cpp
View File

@@ -42,7 +42,7 @@ inline void create_comm(std::unique_ptr<intf_t> &comm, const config_t &config) {
} // namespace
namespace repertory {
auto create_provider(const provider_type &prov,
auto create_provider(provider_type prov,
app_config &config) -> std::unique_ptr<i_provider> {
static std::mutex mutex;
mutex_lock lock(mutex);

8
repertory/librepertory/src/providers/s3/s3_provider.cpp
View File

@@ -367,7 +367,7 @@ auto s3_provider::get_directory_items_impl(const std::string &api_path,
auto size{node.select_node("Size").node().text().as_ullong()};
dir_item.size = is_encrypted ? utils::encryption::encrypting_reader::
calculate_decrypted_size(size)
calculate_decrypted_size(size, false)
: size;
}
@@ -529,7 +529,7 @@ auto s3_provider::get_file(const std::string &api_path, api_file &file) const
file.file_size =
is_encrypted
? utils::encryption::encrypting_reader::calculate_decrypted_size(
result.content_length)
result.content_length, false)
: result.content_length;
}
@@ -613,7 +613,7 @@ auto s3_provider::get_file_list(api_file_list &list, std::string &marker) const
file.file_size =
is_encrypted
? utils::encryption::encrypting_reader::calculate_decrypted_size(
size)
size, false)
: size;
file.key = is_encrypted ? utils::path::create_api_path(object_name) : "";
auto res{add_if_not_found(file, file.key)};
@@ -914,7 +914,7 @@ auto s3_provider::read_file_bytes(const std::string &api_path, std::size_t size,
auto total_size{utils::string::to_uint64(temp)};
return utils::encryption::read_encrypted_range(
{offset, offset + size - 1U},
utils::encryption::generate_key<utils::encryption::hash_256_t>(
utils::encryption::generate_key<utils::hash::hash_256_t>(
cfg.encryption_token),
[&](data_buffer &ct_buffer, std::uint64_t start_offset,
std::uint64_t end_offset) -> bool {

2
repertory/librepertory/src/rpc/common.cpp
View File

@@ -6,6 +6,6 @@
namespace repertory::rpc {
auto create_password_hash(std::string_view password) -> std::string {
return utils::collection::to_hex_string(
utils::encryption::create_hash_blake2b_384(password));
utils::hash::create_hash_blake2b_384(password));
}
} // namespace repertory::rpc

2
repertory/librepertory/src/utils/cli_utils.cpp
View File

@@ -31,7 +31,7 @@
namespace repertory::utils::cli {
void get_api_authentication_data(std::string &user, std::string &password,
std::uint16_t &port, const provider_type &prov,
std::uint16_t &port, provider_type prov,
const std::string &data_directory) {
const auto cfg_file_path = utils::path::combine(
data_directory.empty() ? app_config::default_data_directory(prov)

2
repertory/librepertory/src/utils/unix/unix_utils.cpp
View File

@@ -19,11 +19,11 @@
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include <utils/common.hpp>
#if !defined(_WIN32)
#include "utils/unix/unix_utils.hpp"
#include "utils/common.hpp"
#include "utils/path.hpp"
namespace repertory::utils {

2
repertory/librepertory/src/utils/utils.cpp
View File

@@ -83,7 +83,7 @@ auto create_rocksdb(
return std::unique_ptr<rocksdb::TransactionDB>(ptr);
}
auto create_volume_label(const provider_type &prov) -> std::string {
auto create_volume_label(provider_type prov) -> std::string {
return "repertory_" + app_config::get_provider_name(prov);
}