new_build_system (#18)
All checks were successful
BlockStorage/repertory/pipeline/head This commit looks good

Reviewed-on: #18
This commit is contained in:
2024-09-06 15:05:48 +00:00
parent 9d3e4b8767
commit a7239558bd
191 changed files with 10683 additions and 10598 deletions

View File

@ -45,9 +45,6 @@ public:
[[nodiscard]] static auto
default_rpc_port(const provider_type &prov) -> std::uint16_t;
[[nodiscard]] static auto
get_provider_api_password(const provider_type &prov) -> std::string;
[[nodiscard]] static auto
get_provider_display_name(const provider_type &prov) -> std::string;

View File

@ -26,8 +26,7 @@
#include "comm/i_http_comm.hpp"
#include "events/event_system.hpp"
#include "events/events.hpp"
#include "utils/encrypt.hpp"
#include "utils/utils.hpp"
#include "utils/encryption.hpp"
namespace repertory {
class curl_comm final : public i_http_comm {
@ -42,8 +41,8 @@ private:
using write_callback = size_t (*)(char *, size_t, size_t, void *);
struct read_write_info final {
repertory::data_buffer data{};
repertory::stop_type &stop_requested;
data_buffer data{};
stop_type &stop_requested;
};
static const write_callback write_data;
@ -56,6 +55,11 @@ private:
private:
bool use_s3_path_style_{false};
public:
[[nodiscard]] static auto create_curl() -> CURL *;
[[nodiscard]] static auto reset_curl(CURL *curl_handle) -> CURL *;
public:
[[nodiscard]] static auto
construct_url(CURL *curl, const std::string &relative_path,
@ -92,32 +96,31 @@ public:
const auto key =
utils::encryption::generate_key<utils::encryption::hash_256_t>(
request.decryption_token.value());
const auto result = utils::encryption::read_encrypted_range(
request.range.value(), key,
[&](data_buffer &ct, std::uint64_t start_offset,
std::uint64_t end_offset) -> api_error {
auto encrypted_request = request;
encrypted_request.decryption_token = std::nullopt;
encrypted_request.range = {{start_offset, end_offset}};
encrypted_request.response_handler = [&ct](const auto &encrypted_data,
long /*response_code*/) {
ct = encrypted_data;
};
encrypted_request.total_size = std::nullopt;
if (not utils::encryption::read_encrypted_range(
request.range.value(), key,
[&](data_buffer &ct, std::uint64_t start_offset,
std::uint64_t end_offset) -> bool {
auto encrypted_request = request;
encrypted_request.decryption_token = std::nullopt;
encrypted_request.range = {{start_offset, end_offset}};
encrypted_request.response_handler =
[&ct](const auto &encrypted_data, long /*response_code*/) {
ct = encrypted_data;
};
encrypted_request.total_size = std::nullopt;
if (not make_request(cfg, encrypted_request, response_code,
stop_requested)) {
return api_error::comm_error;
}
if (not make_request(cfg, encrypted_request, response_code,
stop_requested)) {
return false;
}
if (response_code != 200) {
return api_error::comm_error;
}
if (response_code != 200) {
return false;
}
return api_error::success;
},
request.total_size.value(), data);
if (result != api_error::success) {
return true;
},
request.total_size.value(), data)) {
return false;
}
@ -140,7 +143,7 @@ public:
response_code = 0;
auto *curl = utils::create_curl();
auto *curl = create_curl();
if (not request.set_method(curl, stop_requested)) {
return false;
}

View File

@ -23,7 +23,7 @@
#define INCLUDE_COMM_CURL_CURL_REQUESTS_HTTP_REQUEST_BASE_HPP_
#include "types/repertory.hpp"
#include "utils/native_file.hpp"
#include "utils/file.hpp"
namespace repertory::curl::requests {
using read_callback = size_t (*)(char *, size_t, size_t, void *);
@ -33,7 +33,7 @@ using response_callback =
struct read_file_info final {
stop_type &stop_requested;
native_file::native_file_ptr nf{};
std::unique_ptr<utils::file::i_file> file{};
std::uint64_t offset{};
};
@ -41,9 +41,9 @@ inline const auto read_file_data = static_cast<read_callback>(
[](char *buffer, size_t size, size_t nitems, void *instream) -> size_t {
auto *read_info = reinterpret_cast<read_file_info *>(instream);
std::size_t bytes_read{};
auto ret = read_info->nf->read_bytes(
reinterpret_cast<unsigned char *>(buffer), size * nitems,
read_info->offset, bytes_read);
auto ret =
read_info->file->read(reinterpret_cast<unsigned char *>(buffer),
size * nitems, read_info->offset, &bytes_read);
if (ret) {
read_info->offset += bytes_read;
}
@ -66,7 +66,7 @@ struct http_request_base {
std::optional<std::string> decryption_token{};
http_headers headers{};
std::string path{};
query_parameters query{};
http_query_parameters query{};
std::optional<http_range> range{};
std::optional<response_callback> response_handler;
std::optional<http_headers> response_headers;

View File

@ -61,17 +61,7 @@ inline constexpr const std::uint64_t REPERTORY_CONFIG_VERSION = 0ULL;
inline constexpr const std::string_view REPERTORY_DATA_NAME = "repertory2";
inline constexpr const std::string_view REPERTORY_MIN_REMOTE_VERSION = "2.0.0";
#if defined(_WIN32)
#define REPERTORY_INVALID_HANDLE INVALID_HANDLE_VALUE
#define REPERTORY_API_INVALID_HANDLE static_cast<std::uint64_t>(-1)
using native_handle = HANDLE;
#else
#define REPERTORY_INVALID_HANDLE (-1)
#define REPERTORY_API_INVALID_HANDLE REPERTORY_INVALID_HANDLE
using native_handle = int;
#endif
inline constexpr const auto NANOS_PER_SECOND = 1000000000L;
#define WINFSP_ALLOCATION_UNIT UINT64(4096U)

View File

@ -64,15 +64,15 @@ private:
struct fuse_operations fuse_ops_ {};
private:
[[nodiscard]] auto execute_callback(
std::string_view function_name, const char *from, const char *to,
const std::function<api_error(const std::string &, const std::string &)>
&cb,
bool disable_logging = false) -> int;
[[nodiscard]] auto
execute_callback(std::string_view function_name, const char *from,
const char *to,
const std::function<api_error(std::string, std::string)> &cb,
bool disable_logging = false) -> int;
[[nodiscard]] auto
execute_callback(std::string_view function_name, const char *path,
const std::function<api_error(const std::string &)> &cb,
const std::function<api_error(std::string)> &cb,
bool disable_logging = false) -> int;
static void execute_void_callback(std::string_view function_name,
@ -83,7 +83,7 @@ private:
const std::function<void *()> &cb) -> void *;
void raise_fuse_event(std::string_view function_name,
const std::string &api_path, int ret,
std::string_view api_path, int ret,
bool disable_logging);
private:

View File

@ -1269,7 +1269,7 @@ public:
data_buffer buffer(write_size);
if ((ret = request->decode(buffer.data(), buffer.size())) == 0) {
buffer = macaron::Base64::Decode(
std::string{buffer.begin(), buffer.end()});
std::string(buffer.begin(), buffer.end()));
write_size = buffer.size();
remote::file_offset write_offset{};

View File

@ -42,9 +42,9 @@ private:
private:
#if defined(_WIN32)
#define to_handle(x) (x)
#else
#else // !defined(_WIN32)
static auto to_handle(PVOID file_desc) -> native_handle;
#endif
#endif // defined(_WIN32)
public:
auto json_create_directory_snapshot(const std::string &path, json &json_data)

View File

@ -30,7 +30,7 @@
#include "file_manager/i_upload_manager.hpp"
#include "platform/platform.hpp"
#include "types/repertory.hpp"
#include "utils/native_file.hpp"
#include "utils/file.hpp"
namespace repertory {
class app_config;
@ -131,7 +131,7 @@ public:
std::atomic<std::chrono::system_clock::time_point> last_access_{
std::chrono::system_clock::now()};
bool modified_{false};
native_file_ptr nf_;
std::unique_ptr<utils::file::i_file> nf_;
mutable std::mutex io_thread_mtx_;
std::condition_variable io_thread_notify_;
std::deque<std::shared_ptr<io_item>> io_thread_queue_;

View File

@ -29,6 +29,9 @@
namespace repertory {
class encrypt_provider final : public i_provider {
public:
static const constexpr auto type{provider_type::encrypt};
public:
explicit encrypt_provider(app_config &config);
@ -70,10 +73,9 @@ private:
const std::string &source_path)>
callback) const -> api_error;
auto
process_directory_entry(const std::filesystem::directory_entry &dir_entry,
const encrypt_config &cfg,
std::string &api_path) const -> bool;
auto process_directory_entry(const utils::file::i_fs_item &dir_entry,
const encrypt_config &cfg,
std::string &api_path) const -> bool;
void remove_deleted_files();
@ -81,74 +83,68 @@ public:
[[nodiscard]] auto create_directory(const std::string &api_path,
api_meta_map &meta) -> api_error override;
[[nodiscard]] auto
create_directory_clone_source_meta(const std::string & /*source_api_path*/,
const std::string & /*api_path*/)
-> api_error override {
return api_error::not_implemented;
}
[[nodiscard]] auto create_file(const std::string & /*api_path*/,
api_meta_map & /*meta*/)
-> api_error override {
[[nodiscard]] auto create_directory_clone_source_meta(
const std::string & /*source_api_path*/,
const std::string & /*api_path*/) -> api_error override {
return api_error::not_implemented;
}
[[nodiscard]] auto
get_api_path_from_source(const std::string & /*source_path*/,
std::string & /*api_path*/) const
-> api_error override;
create_file(const std::string & /*api_path*/,
api_meta_map & /*meta*/) -> api_error override {
return api_error::not_implemented;
}
[[nodiscard]] auto get_api_path_from_source(
const std::string & /*source_path*/,
std::string & /*api_path*/) const -> api_error override;
[[nodiscard]] auto get_directory_item_count(const std::string &api_path) const
-> std::uint64_t override;
[[nodiscard]] auto get_directory_items(const std::string &api_path,
directory_item_list &list) const
-> api_error override;
[[nodiscard]] auto
get_directory_items(const std::string &api_path,
directory_item_list &list) const -> api_error override;
[[nodiscard]] auto get_file(const std::string &api_path, api_file &file) const
-> api_error override;
[[nodiscard]] auto get_file_list(api_file_list &list) const
-> api_error override;
[[nodiscard]] auto get_file_size(const std::string &api_path,
std::uint64_t &file_size) const
-> api_error override;
[[nodiscard]] auto get_filesystem_item(const std::string &api_path,
bool directory,
filesystem_item &fsi) const
-> api_error override;
[[nodiscard]] auto get_filesystem_item_and_file(const std::string &api_path,
api_file &file,
filesystem_item &fsi) const
-> api_error override;
[[nodiscard]] auto get_file(const std::string &api_path,
api_file &file) const -> api_error override;
[[nodiscard]] auto
get_filesystem_item_from_source_path(const std::string &source_path,
filesystem_item &fsi) const
-> api_error override;
get_file_list(api_file_list &list) const -> api_error override;
[[nodiscard]] auto get_pinned_files() const
-> std::vector<std::string> override;
[[nodiscard]] auto
get_file_size(const std::string &api_path,
std::uint64_t &file_size) const -> api_error override;
[[nodiscard]] auto get_item_meta(const std::string &api_path,
api_meta_map &meta) const
-> api_error override;
[[nodiscard]] auto
get_filesystem_item(const std::string &api_path, bool directory,
filesystem_item &fsi) const -> api_error override;
[[nodiscard]] auto get_item_meta(const std::string &api_path,
const std::string &key,
std::string &value) const
-> api_error override;
[[nodiscard]] auto get_filesystem_item_and_file(
const std::string &api_path, api_file &file,
filesystem_item &fsi) const -> api_error override;
[[nodiscard]] auto get_filesystem_item_from_source_path(
const std::string &source_path,
filesystem_item &fsi) const -> api_error override;
[[nodiscard]] auto
get_pinned_files() const -> std::vector<std::string> override;
[[nodiscard]] auto
get_item_meta(const std::string &api_path,
api_meta_map &meta) const -> api_error override;
[[nodiscard]] auto
get_item_meta(const std::string &api_path, const std::string &key,
std::string &value) const -> api_error override;
[[nodiscard]] auto get_total_drive_space() const -> std::uint64_t override;
[[nodiscard]] auto get_total_item_count() const -> std::uint64_t override;
[[nodiscard]] auto get_provider_type() const -> provider_type override {
return provider_type::encrypt;
return type;
}
[[nodiscard]] auto get_used_drive_space() const -> std::uint64_t override;
@ -158,54 +154,52 @@ public:
[[nodiscard]] auto is_directory(const std::string &api_path,
bool &exists) const -> api_error override;
[[nodiscard]] auto is_file(const std::string &api_path, bool &exists) const
-> api_error override;
[[nodiscard]] auto is_file(const std::string &api_path,
bool &exists) const -> api_error override;
[[nodiscard]] auto is_file_writeable(const std::string &api_path) const
-> bool override;
[[nodiscard]] auto
is_file_writeable(const std::string &api_path) const -> bool override;
[[nodiscard]] auto is_online() const -> bool override;
[[nodiscard]] auto is_rename_supported() const -> bool override;
[[nodiscard]] auto read_file_bytes(const std::string &api_path,
std::size_t size, std::uint64_t offset,
data_buffer &data,
stop_type &stop_requested)
-> api_error override;
[[nodiscard]] auto
read_file_bytes(const std::string &api_path, std::size_t size,
std::uint64_t offset, data_buffer &data,
stop_type &stop_requested) -> api_error override;
[[nodiscard]] auto remove_directory(const std::string & /*api_path*/)
-> api_error override {
[[nodiscard]] auto
remove_directory(const std::string & /*api_path*/) -> api_error override {
return api_error::not_implemented;
}
[[nodiscard]] auto remove_file(const std::string & /*api_path*/)
-> api_error override {
[[nodiscard]] auto
remove_file(const std::string & /*api_path*/) -> api_error override {
return api_error::not_implemented;
}
[[nodiscard]] auto remove_item_meta(const std::string & /*api_path*/,
const std::string & /*key*/)
-> api_error override {
[[nodiscard]] auto
remove_item_meta(const std::string & /*api_path*/,
const std::string & /*key*/) -> api_error override {
return api_error::success;
}
[[nodiscard]] auto rename_file(const std::string & /*from_api_path*/,
const std::string & /*to_api_path*/)
-> api_error override {
[[nodiscard]] auto
rename_file(const std::string & /*from_api_path*/,
const std::string & /*to_api_path*/) -> api_error override {
return api_error::not_implemented;
}
[[nodiscard]] auto set_item_meta(const std::string & /*api_path*/,
const std::string & /*key*/,
const std::string & /*value*/)
-> api_error override {
[[nodiscard]] auto
set_item_meta(const std::string & /*api_path*/, const std::string & /*key*/,
const std::string & /*value*/) -> api_error override {
return api_error::success;
}
[[nodiscard]] auto set_item_meta(const std::string & /*api_path*/,
const api_meta_map & /*meta*/)
-> api_error override {
[[nodiscard]] auto
set_item_meta(const std::string & /*api_path*/,
const api_meta_map & /*meta*/) -> api_error override {
return api_error::success;
}
@ -214,10 +208,10 @@ public:
void stop() override;
[[nodiscard]] auto upload_file(const std::string & /*api_path*/,
const std::string & /*source_path*/,
stop_type & /*stop_requested*/)
-> api_error override {
[[nodiscard]] auto
upload_file(const std::string & /*api_path*/,
const std::string & /*source_path*/,
stop_type & /*stop_requested*/) -> api_error override {
return api_error::not_implemented;
}
};

View File

@ -32,6 +32,9 @@ class i_http_comm;
struct head_object_result;
class s3_provider final : public base_provider {
public:
static const constexpr auto type{provider_type::s3};
public:
s3_provider(app_config &config, i_http_comm &comm);
@ -92,6 +95,9 @@ protected:
stop_type &stop_requested) -> api_error override;
public:
[[nodiscard]] static auto
convert_api_date(std::string_view date) -> std::uint64_t;
[[nodiscard]] auto get_directory_item_count(const std::string &api_path) const
-> std::uint64_t override;
@ -104,7 +110,7 @@ public:
[[nodiscard]] auto get_total_drive_space() const -> std::uint64_t override;
[[nodiscard]] auto get_provider_type() const -> provider_type override {
return provider_type::s3;
return type;
}
[[nodiscard]] auto is_direct_only() const -> bool override { return false; }

View File

@ -31,6 +31,9 @@ class i_file_manager;
class i_http_comm;
class sia_provider : public base_provider {
public:
static const constexpr auto type{provider_type::sia};
public:
sia_provider(app_config &config, i_http_comm &comm);
@ -50,40 +53,39 @@ private:
nlohmann::json &object_list) const -> bool;
protected:
[[nodiscard]] auto create_directory_impl(const std::string &api_path,
api_meta_map &meta)
-> api_error override;
[[nodiscard]] auto
create_directory_impl(const std::string &api_path,
api_meta_map &meta) -> api_error override;
[[nodiscard]] auto get_directory_items_impl(const std::string &api_path,
directory_item_list &list) const
-> api_error override;
[[nodiscard]] auto get_used_drive_space_impl() const
-> std::uint64_t override;
[[nodiscard]] auto
get_used_drive_space_impl() const -> std::uint64_t override;
[[nodiscard]] auto remove_directory_impl(const std::string &api_path)
-> api_error override;
[[nodiscard]] auto
remove_directory_impl(const std::string &api_path) -> api_error override;
[[nodiscard]] auto remove_file_impl(const std::string &api_path)
-> api_error override;
[[nodiscard]] auto
remove_file_impl(const std::string &api_path) -> api_error override;
[[nodiscard]] auto upload_file_impl(const std::string &api_path,
const std::string &source_path,
stop_type &stop_requested)
-> api_error override;
[[nodiscard]] auto
upload_file_impl(const std::string &api_path, const std::string &source_path,
stop_type &stop_requested) -> api_error override;
public:
[[nodiscard]] auto get_directory_item_count(const std::string &api_path) const
-> std::uint64_t override;
[[nodiscard]] auto get_file(const std::string &api_path, api_file &file) const
-> api_error override;
[[nodiscard]] auto get_file(const std::string &api_path,
api_file &file) const -> api_error override;
[[nodiscard]] auto get_file_list(api_file_list &list) const
-> api_error override;
[[nodiscard]] auto
get_file_list(api_file_list &list) const -> api_error override;
[[nodiscard]] auto get_provider_type() const -> provider_type override {
return provider_type::sia;
return type;
}
[[nodiscard]] auto get_total_drive_space() const -> std::uint64_t override;
@ -93,8 +95,8 @@ public:
[[nodiscard]] auto is_directory(const std::string &api_path,
bool &exists) const -> api_error override;
[[nodiscard]] auto is_file(const std::string &api_path, bool &exists) const
-> api_error override;
[[nodiscard]] auto is_file(const std::string &api_path,
bool &exists) const -> api_error override;
[[nodiscard]] auto is_online() const -> bool override;
@ -102,15 +104,14 @@ public:
return true;
}
[[nodiscard]] auto read_file_bytes(const std::string &api_path,
std::size_t size, std::uint64_t offset,
data_buffer &buffer,
stop_type &stop_requested)
-> api_error override;
[[nodiscard]] auto
read_file_bytes(const std::string &api_path, std::size_t size,
std::uint64_t offset, data_buffer &buffer,
stop_type &stop_requested) -> api_error override;
[[nodiscard]] auto rename_file(const std::string &from_api_path,
const std::string &to_api_path)
-> api_error override;
[[nodiscard]] auto
rename_file(const std::string &from_api_path,
const std::string &to_api_path) -> api_error override;
[[nodiscard]] auto start(api_item_added_callback api_item_added,
i_file_manager *mgr) -> bool override;

View File

@ -22,14 +22,14 @@
#ifndef INCLUDE_TYPES_REMOTE_HPP_
#define INCLUDE_TYPES_REMOTE_HPP_
#define PACKET_SERVICE_FUSE 1U
#define PACKET_SERVICE_WINFSP 2U
inline constexpr const auto PACKET_SERVICE_FUSE{1U};
inline constexpr const auto PACKET_SERVICE_WINFSP{2U};
#if defined(_WIN32)
#define PACKET_SERVICE_FLAGS PACKET_SERVICE_WINFSP
#else
#define PACKET_SERVICE_FLAGS PACKET_SERVICE_FUSE
#endif
inline constexpr const auto PACKET_SERVICE_FLAGS{PACKET_SERVICE_WINFSP};
#else // !defined(_WIN32)
inline constexpr const auto PACKET_SERVICE_FLAGS{PACKET_SERVICE_FUSE};
#endif // defined(_WIN32)
namespace repertory::remote {
using block_count = std::uint64_t;
@ -65,16 +65,19 @@ enum class open_flags : std::uint32_t {
dsync = 131072U,
};
inline auto operator|(const open_flags &flag_1,
const open_flags &flag_2) -> open_flags {
using t = std::underlying_type_t<open_flags>;
return static_cast<open_flags>(static_cast<t>(flag_1) |
static_cast<t>(flag_2));
#if defined(__GNUG__)
__attribute__((unused))
#endif // defined(__GNUG__)
inline auto
operator|(const open_flags &flag_1, const open_flags &flag_2) -> open_flags {
using flag_t = std::underlying_type_t<open_flags>;
return static_cast<open_flags>(static_cast<flag_t>(flag_1) |
static_cast<flag_t>(flag_2));
}
#if defined(__GNUG__)
__attribute__((unused))
#endif
#endif // defined(__GNUG__)
inline auto
operator|=(open_flags &flag_1, const open_flags &flag_2) -> open_flags & {
flag_1 = flag_1 | flag_2;
@ -83,69 +86,69 @@ operator|=(open_flags &flag_1, const open_flags &flag_2) -> open_flags & {
#if defined(__GNUG__)
__attribute__((unused))
#endif
#endif // defined(__GNUG__)
inline auto
operator&(const open_flags &flag_1, const open_flags &flag_2) -> open_flags {
using t = std::underlying_type_t<open_flags>;
return static_cast<open_flags>(static_cast<t>(flag_1) &
static_cast<t>(flag_2));
using flag_t = std::underlying_type_t<open_flags>;
return static_cast<open_flags>(static_cast<flag_t>(flag_1) &
static_cast<flag_t>(flag_2));
}
#pragma pack(1)
struct file_info {
UINT32 FileAttributes;
UINT32 ReparseTag;
UINT64 AllocationSize;
UINT64 FileSize;
UINT64 CreationTime;
UINT64 LastAccessTime;
UINT64 LastWriteTime;
UINT64 ChangeTime;
UINT64 IndexNumber;
UINT32 HardLinks;
UINT32 EaSize;
struct file_info final {
UINT32 FileAttributes{};
UINT32 ReparseTag{};
UINT64 AllocationSize{};
UINT64 FileSize{};
UINT64 CreationTime{};
UINT64 LastAccessTime{};
UINT64 LastWriteTime{};
UINT64 ChangeTime{};
UINT64 IndexNumber{};
UINT32 HardLinks{};
UINT32 EaSize{};
};
struct setattr_x {
std::int32_t valid;
file_mode mode;
user_id uid;
group_id gid;
file_size size;
file_time acctime;
file_time modtime;
file_time crtime;
file_time chgtime;
file_time bkuptime;
std::uint32_t flags;
struct setattr_x final {
std::int32_t valid{};
file_mode mode{};
user_id uid{};
group_id gid{};
file_size size{};
file_time acctime{};
file_time modtime{};
file_time crtime{};
file_time chgtime{};
file_time bkuptime{};
std::uint32_t flags{};
};
struct stat {
file_mode st_mode;
file_nlink st_nlink;
user_id st_uid;
group_id st_gid;
file_time st_atimespec;
file_time st_mtimespec;
file_time st_ctimespec;
file_time st_birthtimespec;
file_size st_size;
block_count st_blocks;
block_size st_blksize;
std::uint32_t st_flags;
struct stat final {
file_mode st_mode{};
file_nlink st_nlink{};
user_id st_uid{};
group_id st_gid{};
file_time st_atimespec{};
file_time st_mtimespec{};
file_time st_ctimespec{};
file_time st_birthtimespec{};
file_size st_size{};
block_count st_blocks{};
block_size st_blksize{};
std::uint32_t st_flags{};
};
struct statfs {
std::uint64_t f_bavail;
std::uint64_t f_bfree;
std::uint64_t f_blocks;
std::uint64_t f_favail;
std::uint64_t f_ffree;
std::uint64_t f_files;
std::uint64_t f_bavail{};
std::uint64_t f_bfree{};
std::uint64_t f_blocks{};
std::uint64_t f_favail{};
std::uint64_t f_ffree{};
std::uint64_t f_files{};
};
struct statfs_x : public statfs {
char f_mntfromname[1024];
struct statfs_x final : public statfs {
std::array<char, 1024U> f_mntfromname{};
};
#pragma pack()
@ -154,7 +157,7 @@ struct statfs_x : public statfs {
[[nodiscard]] auto
create_os_open_flags(const open_flags &flags) -> std::uint32_t;
#endif
#endif // !defined(_WIN32)
} // namespace repertory::remote
#endif // INCLUDE_TYPES_REMOTE_HPP_

View File

@ -23,26 +23,26 @@
#define INCLUDE_TYPES_REPERTORY_HPP_
namespace repertory {
constexpr const auto max_time = 0xFFFFFFFFFFFFFFFFULL;
inline constexpr const auto max_time{std::numeric_limits<std::uint64_t>::max()};
const std::string META_ACCESSED = "accessed";
const std::string META_ATTRIBUTES = "attributes";
const std::string META_BACKUP = "backup";
const std::string META_CHANGED = "changed";
const std::string META_CREATION = "creation";
const std::string META_DIRECTORY = "directory";
const std::string META_GID = "gid";
const std::string META_KEY = "key";
const std::string META_MODE = "mode";
const std::string META_MODIFIED = "modified";
const std::string META_OSXFLAGS = "flags";
const std::string META_PINNED = "pinned";
const std::string META_SIZE = "size";
const std::string META_SOURCE = "source";
const std::string META_UID = "uid";
const std::string META_WRITTEN = "written";
inline constexpr const std::string META_ACCESSED{"accessed"};
inline constexpr const std::string META_ATTRIBUTES{"attributes"};
inline constexpr const std::string META_BACKUP{"backup"};
inline constexpr const std::string META_CHANGED{"changed"};
inline constexpr const std::string META_CREATION{"creation"};
inline constexpr const std::string META_DIRECTORY{"directory"};
inline constexpr const std::string META_GID{"gid"};
inline constexpr const std::string META_KEY{"key"};
inline constexpr const std::string META_MODE{"mode"};
inline constexpr const std::string META_MODIFIED{"modified"};
inline constexpr const std::string META_OSXFLAGS{"flags"};
inline constexpr const std::string META_PINNED{"pinned"};
inline constexpr const std::string META_SIZE{"size"};
inline constexpr const std::string META_SOURCE{"source"};
inline constexpr const std::string META_UID{"uid"};
inline constexpr const std::string META_WRITTEN{"written"};
const std::vector<std::string> META_USED_NAMES = {
inline constexpr const std::array<std::string, 16U> META_USED_NAMES = {
META_ACCESSED, META_ATTRIBUTES, META_BACKUP, META_CHANGED,
META_CREATION, META_DIRECTORY, META_GID, META_KEY,
META_MODE, META_MODIFIED, META_OSXFLAGS, META_PINNED,
@ -51,8 +51,6 @@ const std::vector<std::string> META_USED_NAMES = {
using api_meta_map = std::map<std::string, std::string>;
using stop_type = std::atomic<bool>;
enum class api_error {
success = 0,
access_denied,
@ -152,28 +150,28 @@ enum class provider_type : std::size_t {
};
#if defined(_WIN32)
struct open_file_data {
struct open_file_data final {
void *directory_buffer{};
};
#else
using open_file_data = int;
#endif
struct api_file {
std::string api_path;
std::string api_parent;
struct api_file final {
std::string api_path{};
std::string api_parent{};
std::uint64_t accessed_date{};
std::uint64_t changed_date{};
std::uint64_t creation_date{};
std::uint64_t file_size{};
std::string key;
std::string key{};
std::uint64_t modified_date{};
std::string source_path;
};
struct directory_item {
std::string api_path;
std::string api_parent;
struct directory_item final {
std::string api_path{};
std::string api_parent{};
bool directory{false};
std::uint64_t size{};
api_meta_map meta{};
@ -190,26 +188,30 @@ struct directory_item {
}
[[nodiscard]] auto to_json() const -> json {
return {{"path", api_path},
{"parent", api_parent},
{"size", size},
{"directory", directory},
{"meta", meta}};
return {
{"path", api_path}, {"parent", api_parent}, {"size", size},
{"directory", directory}, {"meta", meta},
};
}
};
struct filesystem_item {
std::string api_path;
std::string api_parent;
bool directory{false};
std::uint64_t size{};
std::string source_path;
struct encrypt_config final {
std::string encryption_token{};
std::string path{};
};
struct host_config {
std::string agent_string;
std::string api_password;
std::string api_user;
struct filesystem_item final {
std::string api_path{};
std::string api_parent{};
bool directory{false};
std::uint64_t size{};
std::string source_path{};
};
struct host_config final {
std::string agent_string{};
std::string api_password{};
std::string api_user{};
std::uint16_t api_port{};
std::string host_name_or_ip{"localhost"};
std::string path{};
@ -265,25 +267,15 @@ from_json(const json &j, host_config &hc) {
j.at("TimeoutMs").get_to(hc.timeout_ms);
}
struct http_range {
std::uint64_t begin;
std::uint64_t end;
};
struct encrypt_config {
std::string encryption_token;
std::string path;
};
struct s3_config {
std::string access_key;
std::string bucket;
struct s3_config final {
std::string access_key{};
std::string bucket{};
std::uint16_t cache_timeout_secs{60U};
std::string encryption_token;
std::string encryption_token{};
std::string region{"any"};
std::string secret_key;
std::string secret_key{};
std::uint32_t timeout_ms{60000U};
std::string url;
std::string url{};
bool use_path_style{false};
bool use_region_in_url{false};
};
@ -292,11 +284,7 @@ using api_file_list = std::vector<api_file>;
using api_file_provider_callback = std::function<void(api_file &)>;
using api_item_added_callback = std::function<api_error(bool, api_file &)>;
using directory_item_list = std::vector<directory_item>;
using http_headers = std::unordered_map<std::string, std::string>;
using http_parameters = std::unordered_map<std::string, std::string>;
using http_ranges = std::vector<http_range>;
using meta_provider_callback = std::function<void(directory_item &)>;
using query_parameters = std::map<std::string, std::string>;
} // namespace repertory
#endif // INCLUDE_TYPES_REPERTORY_HPP_

View File

@ -79,8 +79,8 @@ struct head_object_result {
#else
strptime(date.c_str(), "%a, %d %b %Y %H:%M:%S %Z", &tm1);
#endif
last_modified =
static_cast<std::uint64_t>(mktime(&tm1)) * NANOS_PER_SECOND;
last_modified = static_cast<std::uint64_t>(mktime(&tm1)) *
utils::time::NANOS_PER_SECOND;
}
return *this;
}

View File

@ -1,49 +0,0 @@
/*
Copyright <2018-2024> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#ifndef INCLUDE_UTILS_ACTION_QUEUE_HPP_
#define INCLUDE_UTILS_ACTION_QUEUE_HPP_
#include "utils/single_thread_service_base.hpp"
namespace repertory::utils::action_queue {
class action_queue final : single_thread_service_base {
explicit action_queue(const std::string &id,
std::uint8_t max_concurrent_actions = 5u);
private:
std::string id_;
std::uint8_t max_concurrent_actions_;
private:
std::deque<std::function<void()>> queue_;
mutable std::mutex queue_mtx_;
std::condition_variable queue_notify_;
protected:
void service_function() override;
public:
void push(std::function<void()> action);
};
} // namespace repertory::utils::action_queue
#endif // INCLUDE_UTILS_ACTION_QUEUE_HPP_

View File

@ -1,46 +0,0 @@
/*
Copyright <2018-2024> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#ifndef INCLUDE_UTILS_ENCRYPT_HPP_
#define INCLUDE_UTILS_ENCRYPT_HPP_
#include "types/repertory.hpp"
#include "utils/encryption.hpp"
namespace repertory::utils::encryption {
using reader_func = std::function<api_error(data_buffer &cypher_text,
std::uint64_t start_offset,
std::uint64_t end_offset)>;
// Prototypes
[[nodiscard]] auto decrypt_file_path(std::string_view encryption_token,
std::string &file_path) -> api_error;
[[nodiscard]] auto decrypt_file_name(std::string_view encryption_token,
std::string &file_name) -> api_error;
[[nodiscard]] auto
read_encrypted_range(const http_range &range,
const utils::encryption::hash_256_t &key,
reader_func reader, std::uint64_t total_size,
data_buffer &data) -> api_error;
} // namespace repertory::utils::encryption
#endif // INCLUDE_UTILS_ENCRYPT_HPP_

View File

@ -1,140 +0,0 @@
/*
Copyright <2018-2024> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#ifndef INCLUDE_UTILS_ENCRYPTING_READER_HPP_
#define INCLUDE_UTILS_ENCRYPTING_READER_HPP_
#include "types/repertory.hpp"
#include "utils/encryption.hpp"
#include "utils/native_file.hpp"
namespace repertory::utils::encryption {
class encrypting_reader final {
public:
encrypting_reader(std::string_view file_name, std::string_view source_path,
stop_type &stop_requested, std::string_view token,
std::optional<std::string_view> relative_parent_path,
std::size_t error_return = 0U);
encrypting_reader(std::string_view encrypted_file_path,
std::string_view source_path, stop_type &stop_requested,
std::string_view token, std::size_t error_return = 0U);
encrypting_reader(
std::string_view encrypted_file_path, std::string_view source_path,
stop_type &stop_requested, std::string_view token,
std::vector<std::array<unsigned char,
crypto_aead_xchacha20poly1305_IETF_NPUBBYTES>>
iv_list,
std::size_t error_return = 0U);
encrypting_reader(const encrypting_reader &reader);
encrypting_reader(encrypting_reader &&) = delete;
auto operator=(const encrypting_reader &) -> encrypting_reader & = delete;
auto operator=(encrypting_reader &&) -> encrypting_reader & = delete;
~encrypting_reader();
public:
using iostream = std::basic_iostream<char, std::char_traits<char>>;
using streambuf = std::basic_streambuf<char, std::char_traits<char>>;
private:
utils::encryption::hash_256_t key_;
stop_type &stop_requested_;
size_t error_return_;
std::unordered_map<std::size_t, data_buffer> chunk_buffers_;
std::string encrypted_file_name_;
std::string encrypted_file_path_;
std::vector<
std::array<unsigned char, crypto_aead_xchacha20poly1305_IETF_NPUBBYTES>>
iv_list_;
std::size_t last_data_chunk_{};
std::size_t last_data_chunk_size_{};
std::uint64_t read_offset_{};
native_file_ptr source_file_;
std::uint64_t total_size_{};
private:
static const std::size_t header_size_;
static const std::size_t data_chunk_size_;
static const std::size_t encrypted_chunk_size_;
private:
auto reader_function(char *buffer, size_t size, size_t nitems) -> size_t;
public:
[[nodiscard]] static auto
calculate_decrypted_size(std::uint64_t total_size) -> std::uint64_t;
[[nodiscard]] static auto
calculate_encrypted_size(std::string_view source_path) -> std::uint64_t;
[[nodiscard]] auto create_iostream() const -> std::shared_ptr<iostream>;
[[nodiscard]] static constexpr auto
get_encrypted_chunk_size() -> std::size_t {
return encrypted_chunk_size_;
}
[[nodiscard]] static constexpr auto get_data_chunk_size() -> std::size_t {
return data_chunk_size_;
}
[[nodiscard]] auto get_encrypted_file_name() const -> std::string {
return encrypted_file_name_;
}
[[nodiscard]] auto get_encrypted_file_path() const -> std::string {
return encrypted_file_path_;
}
[[nodiscard]] auto get_error_return() const -> std::size_t {
return error_return_;
}
[[nodiscard]] auto get_iv_list()
-> std::vector<std::array<unsigned char,
crypto_aead_xchacha20poly1305_IETF_NPUBBYTES>> {
return iv_list_;
}
[[nodiscard]] auto get_stop_requested() const -> bool {
return stop_requested_;
}
[[nodiscard]] auto get_total_size() const -> std::uint64_t {
return total_size_;
}
[[nodiscard]] static auto reader_function(char *buffer, size_t size,
size_t nitems,
void *instream) -> size_t {
return reinterpret_cast<encrypting_reader *>(instream)->reader_function(
buffer, size, nitems);
}
void set_read_position(std::uint64_t position) { read_offset_ = position; }
};
} // namespace repertory::utils::encryption
#endif // INCLUDE_UTILS_ENCRYPTING_READER_HPP_

View File

@ -22,64 +22,17 @@
#ifndef INCLUDE_UTILS_FILE_UTILS_HPP_
#define INCLUDE_UTILS_FILE_UTILS_HPP_
#include "types/repertory.hpp"
#include "utils/file.hpp"
#include "utils/native_file.hpp"
namespace repertory::utils::file {
// Prototypes
[[nodiscard]] auto calculate_used_space(std::string path,
bool recursive) -> std::uint64_t;
void change_to_process_directory();
[[nodiscard]] auto copy_directory_recursively(std::string from_path,
std::string to_path) -> bool;
[[nodiscard]] auto copy_file(std::string from_path,
std::string to_path) -> bool;
[[nodiscard]] auto create_full_directory_path(std::string path) -> bool;
[[nodiscard]] auto delete_directory(std::string path,
bool recursive = false) -> bool;
[[nodiscard]] auto delete_directory_recursively(std::string path) -> bool;
[[nodiscard]] auto delete_file(std::string path) -> bool;
[[nodiscard]] auto generate_sha256(const std::string &file_path) -> std::string;
[[nodiscard]] auto get_accessed_time(const std::string &path,
std::uint64_t &accessed) -> bool;
[[nodiscard]] auto
get_directory_files(std::string path, bool oldest_first,
get_directory_files(std::string_view path, bool oldest_first,
bool recursive = false) -> std::deque<std::string>;
[[nodiscard]] auto
get_free_drive_space(const std::string &path) -> std::uint64_t;
[[nodiscard]] auto
get_total_drive_space(const std::string &path) -> std::uint64_t;
[[nodiscard]] auto get_modified_time(const std::string &path,
std::uint64_t &modified) -> bool;
[[nodiscard]] auto
is_modified_date_older_than(const std::string &path,
const std::chrono::hours &hours) -> bool;
[[nodiscard]] auto move_file(std::string from, std::string to) -> bool;
[[nodiscard]] auto
read_file_lines(const std::string &path) -> std::vector<std::string>;
[[nodiscard]] auto reset_modified_time(const std::string &path) -> bool;
[[nodiscard]] auto retry_delete_directory(const std::string &dir) -> bool;
[[nodiscard]] auto retry_delete_file(const std::string &file) -> bool;
} // namespace repertory::utils::file
#endif // INCLUDE_UTILS_FILE_UTILS_HPP_

View File

@ -1,117 +0,0 @@
/*
Copyright <2018-2024> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#ifndef INCLUDE_UTILS_NATIVEFILE_HPP_
#define INCLUDE_UTILS_NATIVEFILE_HPP_
#include "types/repertory.hpp"
namespace repertory {
class native_file final {
public:
native_file(const native_file &) = delete;
native_file(native_file &&) = delete;
auto operator=(const native_file &) -> native_file & = delete;
auto operator=(native_file &&) -> native_file & = delete;
using native_file_ptr = std::shared_ptr<native_file>;
public:
[[nodiscard]] static auto attach(native_handle handle) -> native_file_ptr {
return std::shared_ptr<native_file>(new native_file(handle));
}
[[nodiscard]] static auto
clone(const native_file_ptr &ptr) -> native_file_ptr;
[[nodiscard]] static auto create_or_open(std::string_view source_path,
bool read_only,
native_file_ptr &ptr) -> api_error;
[[nodiscard]] static auto create_or_open(std::string_view source_path,
native_file_ptr &ptr) -> api_error;
[[nodiscard]] static auto open(std::string_view source_path,
native_file_ptr &ptr) -> api_error;
[[nodiscard]] static auto open(std::string_view source_path, bool read_only,
native_file_ptr &ptr) -> api_error;
private:
explicit native_file(const native_handle &handle) : handle_(handle) {}
public:
~native_file();
private:
native_handle handle_;
private:
bool auto_close{false};
#if defined(_WIN32)
std::recursive_mutex read_write_mutex_;
#endif
public:
[[nodiscard]] auto allocate(std::uint64_t file_size) -> bool;
void close();
[[nodiscard]] auto copy_from(const native_file_ptr &ptr) -> bool;
[[nodiscard]] auto copy_from(const std::string &path) -> bool;
void flush();
[[nodiscard]] auto get_file_size(std::uint64_t &file_size) -> bool;
[[nodiscard]] auto get_handle() -> native_handle;
#if defined(_WIN32)
[[nodiscard]] auto read_bytes(unsigned char *buffer, std::size_t read_size,
std::uint64_t read_offset,
std::size_t &bytes_read) -> bool;
#else
[[nodiscard]] auto read_bytes(unsigned char *buffer, std::size_t read_size,
std::uint64_t read_offset,
std::size_t &bytes_read) -> bool;
#endif
void set_auto_close(bool b) { auto_close = b; }
[[nodiscard]] auto truncate(std::uint64_t file_size) -> bool;
#if defined(_WIN32)
[[nodiscard]] auto write_bytes(const unsigned char *buffer,
std::size_t write_size,
std::uint64_t write_offset,
std::size_t &bytes_written) -> bool;
#else
[[nodiscard]] auto write_bytes(const unsigned char *buffer,
std::size_t write_size,
std::uint64_t write_offset,
std::size_t &bytes_written) -> bool;
#endif
};
using native_file_ptr = native_file::native_file_ptr;
} // namespace repertory
#endif // INCLUDE_UTILS_NATIVEFILE_HPP_

View File

@ -43,15 +43,9 @@ inline const std::array<std::string, 4U> attribute_namespaces = {
[[nodiscard]] auto unix_error_to_windows(int err) -> std::uint32_t;
[[nodiscard]] auto
unix_time_to_windows_time(const remote::file_time &file_time) -> UINT64;
void windows_create_to_unix(const UINT32 &create_options,
const UINT32 &granted_access, std::uint32_t &flags,
remote::file_mode &mode);
[[nodiscard]] auto
windows_time_to_unix_time(std::uint64_t win_time) -> remote::file_time;
} // namespace repertory::utils
#endif // !_WIN32

View File

@ -29,27 +29,10 @@ void calculate_allocation_size(bool directory, std::uint64_t file_size,
UINT64 allocation_size,
std::string &allocation_meta_size);
[[nodiscard]] auto convert_api_date(const std::string &date) -> std::uint64_t;
[[nodiscard]] auto create_curl() -> CURL *;
[[nodiscard]] auto
create_volume_label(const provider_type &prov) -> std::string;
[[nodiscard]] auto get_attributes_from_meta(const api_meta_map &meta) -> DWORD;
[[nodiscard]] auto reset_curl(CURL *curl_handle) -> CURL *;
[[nodiscard]] auto
retryable_action(const std::function<bool()> &action) -> bool;
void spin_wait_for_mutex(std::function<bool()> complete,
std::condition_variable &cond, std::mutex &mtx,
const std::string &text = "");
void spin_wait_for_mutex(bool &complete, std::condition_variable &cond,
std::mutex &mtx, const std::string &text = "");
} // namespace repertory::utils
#endif // INCLUDE_UTILS_UTILS_HPP_