BlockStorage/repertory
1
0
Fork 0
Code Issues 10 Pull Requests Releases 7 Wiki Activity

refactor s3 provider

Browse Source
This commit is contained in:
Scott E. Graves
2023-11-11 11:32:14 -06:00
parent f2c1f64f02
commit 68476cbc00
55 changed files with 1504 additions and 5050 deletions
Download Patch File Download Diff File Expand all files Collapse all files

192
src/app_config.cpp
View File

@@ -29,19 +29,19 @@
#include "utils/utils.hpp"
namespace repertory {
app_config::app_config(const provider_type &pt,
app_config::app_config(const provider_type &prov,
const std::string &data_directory)
: pt_(pt),
api_auth_(utils::generate_random_string(48u)),
api_port_(default_rpc_port(pt)),
: prov_(prov),
api_auth_(utils::generate_random_string(48U)),
api_port_(default_rpc_port(prov)),
api_user_("repertory"),
config_changed_(false),
data_directory_(
data_directory.empty() ? default_data_directory(pt)
: ((pt == provider_type::remote) || (pt == provider_type::s3))
data_directory.empty() ? default_data_directory(prov)
: ((prov == provider_type::remote) || (prov == provider_type::s3))
? utils::path::absolute(data_directory)
: utils::path::absolute(utils::path::combine(
data_directory, {get_provider_name(pt)}))),
data_directory, {get_provider_name(prov)}))),
download_timeout_secs_(30),
enable_chunk_downloader_timeout_(true),
enable_comm_duration_events_(false),
@@ -58,7 +58,7 @@ app_config::app_config(const provider_type &pt,
is_remote_mount_(false),
low_freq_interval_secs_(60 * 60),
max_cache_size_bytes_(20 * 1024 * 1024 * 1024ULL),
max_upload_count_(5u),
max_upload_count_(5U),
min_download_timeout_secs_(5),
online_check_retry_secs_(60),
orphaned_file_retention_days_(15),
@@ -68,10 +68,10 @@ app_config::app_config(const provider_type &pt,
remote_client_pool_size_(10),
remote_host_name_or_ip_(""),
remote_max_connections_(20),
remote_port_((pt == provider_type::sia) ? 20000
: (pt == provider_type::s3) ? 20001
: (pt == provider_type::encrypt) ? 20002
: 20003),
remote_port_((prov == provider_type::sia) ? 20000
: (prov == provider_type::s3) ? 20001
: (prov == provider_type::encrypt) ? 20002
: 20003),
remote_receive_timeout_secs_(120),
remote_send_timeout_secs_(30),
remote_token_(""),
@@ -80,18 +80,21 @@ app_config::app_config(const provider_type &pt,
cache_directory_ = utils::path::combine(data_directory_, {"cache"});
log_directory_ = utils::path::combine(data_directory_, {"logs"});
hc_.agent_string = default_agent_name(pt_);
hc_.api_password = get_provider_api_password(pt_);
hc_.api_port = default_api_port(pt_);
hc_.agent_string = default_agent_name(prov_);
hc_.api_password = get_provider_api_password(prov_);
hc_.api_port = default_api_port(prov_);
if (not utils::file::create_full_directory_path(data_directory_))
if (not utils::file::create_full_directory_path(data_directory_)) {
throw startup_exception("unable to create: " + data_directory_);
}
if (not utils::file::create_full_directory_path(cache_directory_))
if (not utils::file::create_full_directory_path(cache_directory_)) {
throw startup_exception("unable to create: " + cache_directory_);
}
if (not utils::file::create_full_directory_path(log_directory_))
if (not utils::file::create_full_directory_path(log_directory_)) {
throw startup_exception("unable to create: " + log_directory_);
}
if (not load()) {
save();
@@ -102,48 +105,51 @@ auto app_config::get_config_file_path() const -> std::string {
return utils::path::combine(data_directory_, {"config.json"});
}
auto app_config::default_agent_name(const provider_type &pt) -> std::string {
auto app_config::default_agent_name(const provider_type &prov) -> std::string {
static const std::array<std::string,
static_cast<std::size_t>(provider_type::unknown)>
PROVIDER_AGENT_NAMES = {"Sia-Agent", "", "", "", ""};
PROVIDER_AGENT_NAMES = {"Sia-Agent", "", "", ""};
return PROVIDER_AGENT_NAMES[static_cast<std::size_t>(pt)];
return PROVIDER_AGENT_NAMES[static_cast<std::size_t>(prov)];
}
auto app_config::default_api_port(const provider_type &pt) -> std::uint16_t {
auto app_config::default_api_port(const provider_type &prov) -> std::uint16_t {
static const std::array<std::uint16_t,
static_cast<std::size_t>(provider_type::unknown)>
PROVIDER_API_PORTS = {9980u, 0u, 0u, 0u, 0u};
return PROVIDER_API_PORTS[static_cast<std::size_t>(pt)];
PROVIDER_API_PORTS = {9980U, 0U, 0U, 0U};
return PROVIDER_API_PORTS[static_cast<std::size_t>(prov)];
}
auto app_config::default_data_directory(const provider_type &pt)
auto app_config::default_data_directory(const provider_type &prov)
-> std::string {
#ifdef _WIN32
auto data_directory = utils::path::combine(
utils::get_local_app_data_directory(),
{REPERTORY_DATA_NAME, app_config::get_provider_name(pt)});
{REPERTORY_DATA_NAME, app_config::get_provider_name(prov)});
#else
#ifdef __APPLE__
auto data_directory = utils::path::resolve(
std::string("~/Library/Application Support/") + REPERTORY_DATA_NAME +
'/' + app_config::get_provider_name(pt));
'/' + app_config::get_provider_name(prov));
#else
auto data_directory =
utils::path::resolve(std::string("~/.local/") + REPERTORY_DATA_NAME +
'/' + app_config::get_provider_name(pt));
'/' + app_config::get_provider_name(prov));
#endif
#endif
return data_directory;
}
auto app_config::default_rpc_port(const provider_type &pt) -> std::uint16_t {
auto app_config::default_rpc_port(const provider_type &prov) -> std::uint16_t {
static const std::array<std::uint16_t,
static_cast<std::size_t>(provider_type::unknown)>
PROVIDER_RPC_PORTS = {
11101u, 11102u, 11103u, 11104u, 11105u,
11101U,
11102U,
11103U,
11104U,
};
return PROVIDER_RPC_PORTS[static_cast<std::size_t>(pt)];
return PROVIDER_RPC_PORTS[static_cast<std::size_t>(prov)];
}
auto app_config::get_json() const -> json {
@@ -209,11 +215,12 @@ auto app_config::get_json() const -> json {
{"SecretKey", s3_config_.secret_key},
{"TimeoutMs", s3_config_.timeout_ms},
{"URL", s3_config_.url},
{"UsePathStyle", s3_config_.use_path_style},
{"UseRegionInURL", s3_config_.use_region_in_url},
}},
{"Version", version_}};
if (pt_ == provider_type::encrypt) {
if (prov_ == provider_type::encrypt) {
ret.erase("ChunkDownloaderTimeoutSeconds");
ret.erase("EnableChunkDownloaderTimeout");
ret.erase("EnableMaxCacheSize");
@@ -229,13 +236,13 @@ auto app_config::get_json() const -> json {
ret.erase("RetryReadCount");
ret.erase("RingBufferFileSize");
ret.erase("S3Config");
} else if (pt_ == provider_type::s3) {
} else if (prov_ == provider_type::s3) {
ret.erase("EncryptConfig");
ret.erase("HostConfig");
} else if (pt_ == provider_type::sia) {
} else if (prov_ == provider_type::sia) {
ret.erase("EncryptConfig");
ret.erase("S3Config");
} else if (pt_ == provider_type::remote) {
} else if (prov_ == provider_type::remote) {
ret.erase("ChunkDownloaderTimeoutSeconds");
ret.erase("EnableChunkDownloaderTimeout");
ret.erase("EnableChunkDownloaderTimeout");
@@ -261,50 +268,56 @@ auto app_config::get_json() const -> json {
auto app_config::get_max_cache_size_bytes() const -> std::uint64_t {
const auto max_space =
std::max(static_cast<std::uint64_t>(100ull * 1024ull * 1024ull),
std::max(static_cast<std::uint64_t>(100ULL * 1024ULL * 1024ULL),
max_cache_size_bytes_);
return std::min(utils::file::get_free_drive_space(get_cache_directory()),
max_space);
}
auto app_config::get_provider_api_password(const provider_type &pt)
auto app_config::get_provider_api_password(const provider_type &prov)
-> std::string {
#ifdef _WIN32
auto api_file =
utils::path::combine(utils::get_local_app_data_directory(),
{get_provider_display_name(pt), "apipassword"});
{get_provider_display_name(prov), "apipassword"});
#else
#ifdef __APPLE__
auto api_file =
utils::path::combine(utils::path::resolve("~"),
{"/Library/Application Support",
get_provider_display_name(pt), "apipassword"});
get_provider_display_name(prov), "apipassword"});
#else
auto api_file = utils::path::combine(utils::path::resolve("~/."),
{get_provider_name(pt), "apipassword"});
auto api_file = utils::path::combine(
utils::path::resolve("~/."), {get_provider_name(prov), "apipassword"});
#endif
#endif
auto lines = utils::file::read_file_lines(api_file);
return lines.empty() ? "" : utils::string::trim(lines[0]);
}
auto app_config::get_provider_display_name(const provider_type &pt)
auto app_config::get_provider_display_name(const provider_type &prov)
-> std::string {
static const std::array<std::string,
static_cast<std::size_t>(provider_type::unknown)>
PROVIDER_DISPLAY_NAMES = {
"Sia", "Remote", "S3", "Passthrough", "Encrypt",
"Sia",
"Remote",
"S3",
"Encrypt",
};
return PROVIDER_DISPLAY_NAMES[static_cast<std::size_t>(pt)];
return PROVIDER_DISPLAY_NAMES[static_cast<std::size_t>(prov)];
}
auto app_config::get_provider_name(const provider_type &pt) -> std::string {
auto app_config::get_provider_name(const provider_type &prov) -> std::string {
static const std::array<std::string,
static_cast<std::size_t>(provider_type::unknown)>
PROVIDER_NAMES = {
"sia", "remote", "s3", "passthrough", "encrypt",
"sia",
"remote",
"s3",
"encrypt",
};
return PROVIDER_NAMES[static_cast<std::size_t>(pt)];
return PROVIDER_NAMES[static_cast<std::size_t>(prov)];
}
auto app_config::get_value_by_name(const std::string &name) -> std::string {
@@ -447,8 +460,11 @@ auto app_config::get_value_by_name(const std::string &name) -> std::string {
if (name == "S3Config.URL") {
return s3_config_.url;
}
if (name == "S3Config.UsePathStyle") {
return utils::string::from_bool(s3_config_.use_path_style);
}
if (name == "S3Config.UseRegionInURL") {
return std::to_string(s3_config_.use_region_in_url);
return utils::string::from_bool(s3_config_.use_region_in_url);
}
if (name == "S3Config.TimeoutMs") {
return std::to_string(s3_config_.timeout_ms);
@@ -463,14 +479,14 @@ auto app_config::load() -> bool {
auto ret = false;
const auto config_file_path = get_config_file_path();
recur_mutex_lock l(read_write_mutex_);
recur_mutex_lock lock(read_write_mutex_);
if (utils::file::is_file(config_file_path)) {
try {
std::ifstream config_file(&config_file_path[0]);
std::ifstream config_file(config_file_path.data());
if (config_file.is_open()) {
std::stringstream ss;
ss << config_file.rdbuf();
const auto json_text = ss.str();
std::stringstream stream;
stream << config_file.rdbuf();
const auto json_text = stream.str();
config_file.close();
if ((ret = not json_text.empty())) {
const auto json_document = json::parse(json_text);
@@ -510,20 +526,20 @@ auto app_config::load() -> bool {
if (json_document.find("HostConfig") != json_document.end()) {
auto host_config_json = json_document["HostConfig"];
auto hc = hc_;
get_value(host_config_json, "AgentString", hc.agent_string, ret);
get_value(host_config_json, "ApiPassword", hc.api_password, ret);
get_value(host_config_json, "ApiPort", hc.api_port, ret);
get_value(host_config_json, "HostNameOrIp", hc.host_name_or_ip,
auto cfg = hc_;
get_value(host_config_json, "AgentString", cfg.agent_string, ret);
get_value(host_config_json, "ApiPassword", cfg.api_password, ret);
get_value(host_config_json, "ApiPort", cfg.api_port, ret);
get_value(host_config_json, "HostNameOrIp", cfg.host_name_or_ip,
ret);
get_value(host_config_json, "TimeoutMs", hc.timeout_ms, ret);
hc_ = hc;
get_value(host_config_json, "TimeoutMs", cfg.timeout_ms, ret);
hc_ = cfg;
} else {
ret = false;
}
if (hc_.api_password.empty()) {
hc_.api_password = get_provider_api_password(pt_);
hc_.api_password = get_provider_api_password(prov_);
if (hc_.api_password.empty()) {
ret = false;
}
@@ -531,20 +547,22 @@ auto app_config::load() -> bool {
if (json_document.find("S3Config") != json_document.end()) {
auto s3_config_json = json_document["S3Config"];
auto s3 = s3_config_;
get_value(s3_config_json, "AccessKey", s3.access_key, ret);
get_value(s3_config_json, "Bucket", s3.bucket, ret);
auto s3_cfg = s3_config_;
get_value(s3_config_json, "AccessKey", s3_cfg.access_key, ret);
get_value(s3_config_json, "Bucket", s3_cfg.bucket, ret);
get_value(s3_config_json, "CacheTimeoutSeconds",
s3.cache_timeout_secs, ret);
get_value(s3_config_json, "EncryptionToken", s3.encryption_token,
s3_cfg.cache_timeout_secs, ret);
get_value(s3_config_json, "EncryptionToken",
s3_cfg.encryption_token, ret);
get_value(s3_config_json, "Region", s3_cfg.region, ret);
get_value(s3_config_json, "SecretKey", s3_cfg.secret_key, ret);
get_value(s3_config_json, "TimeoutMs", s3_cfg.timeout_ms, ret);
get_value(s3_config_json, "URL", s3_cfg.url, ret);
get_value(s3_config_json, "UsePathStyle", s3_cfg.use_path_style,
ret);
get_value(s3_config_json, "Region", s3.region, ret);
get_value(s3_config_json, "SecretKey", s3.secret_key, ret);
get_value(s3_config_json, "TimeoutMs", s3.timeout_ms, ret);
get_value(s3_config_json, "URL", s3.url, ret);
get_value(s3_config_json, "UseRegionInURL", s3.use_region_in_url,
ret);
s3_config_ = s3;
get_value(s3_config_json, "UseRegionInURL",
s3_cfg.use_region_in_url, ret);
s3_config_ = s3_cfg;
} else {
ret = false;
}
@@ -593,13 +611,13 @@ auto app_config::load() -> bool {
ret = false;
}
std::uint64_t version = 0u;
std::uint64_t version{};
get_value(json_document, "Version", version, ret);
// Handle configuration defaults for new config versions
if (version != REPERTORY_CONFIG_VERSION) {
if (version > REPERTORY_CONFIG_VERSION) {
version = 0u;
version = 0U;
}
version_ = version;
@@ -622,7 +640,7 @@ auto app_config::load() -> bool {
void app_config::save() {
const auto configFilePath = get_config_file_path();
recur_mutex_lock l(read_write_mutex_);
recur_mutex_lock lock(read_write_mutex_);
if (config_changed_ || not utils::file::is_file(configFilePath)) {
if (not utils::file::is_directory(data_directory_)) {
if (not utils::file::create_full_directory_path(data_directory_)) {
@@ -684,19 +702,19 @@ auto app_config::set_value_by_name(const std::string &name,
}
if (name == "EnableChunkDownloaderTimeout") {
set_enable_chunk_downloader_timeout(utils::string::to_bool(value));
return std::to_string(get_enable_chunk_download_timeout());
return utils::string::from_bool(get_enable_chunk_download_timeout());
}
if (name == "EnableCommDurationEvents") {
set_enable_comm_duration_events(utils::string::to_bool(value));
return std::to_string(get_enable_comm_duration_events());
return utils::string::from_bool(get_enable_comm_duration_events());
}
if (name == "EnableDriveEvents") {
set_enable_drive_events(utils::string::to_bool(value));
return std::to_string(get_enable_drive_events());
return utils::string::from_bool(get_enable_drive_events());
}
if (name == "EnableMaxCacheSize") {
set_enable_max_cache_size(utils::string::to_bool(value));
return std::to_string(get_enable_max_cache_size());
return utils::string::from_bool(get_enable_max_cache_size());
#ifdef _WIN32
}
if (name == "EnableMountManager") {
@@ -722,7 +740,7 @@ auto app_config::set_value_by_name(const std::string &name,
}
if (name == "EvictionUsesAccessedTime") {
set_eviction_uses_accessed_time(utils::string::to_bool(value));
return std::to_string(get_eviction_uses_accessed_time());
return utils::string::from_bool(get_eviction_uses_accessed_time());
}
if (name == "HighFreqIntervalSeconds") {
set_high_frequency_interval_secs(utils::string::to_uint8(value));
@@ -779,11 +797,11 @@ auto app_config::set_value_by_name(const std::string &name,
}
if (name == "RemoteMount.EnableRemoteMount") {
set_enable_remote_mount(utils::string::to_bool(value));
return std::to_string(get_enable_remote_mount());
return utils::string::from_bool(get_enable_remote_mount());
}
if (name == "RemoteMount.IsRemoteMount") {
set_is_remote_mount(utils::string::to_bool(value));
return std::to_string(get_is_remote_mount());
return utils::string::from_bool(get_is_remote_mount());
}
if (name == "RemoteMount.RemoteClientPoolSize") {
set_remote_client_pool_size(utils::string::to_uint8(value));
@@ -831,7 +849,7 @@ auto app_config::set_value_by_name(const std::string &name,
}
if (name == "S3Config.CacheTimeoutSeconds") {
const auto timeout =
std::max(std::uint16_t(5u), utils::string::to_uint16(value));
std::max(std::uint16_t(5U), utils::string::to_uint16(value));
set_value(s3_config_.cache_timeout_secs, timeout);
return std::to_string(s3_config_.cache_timeout_secs);
}
@@ -847,9 +865,13 @@ auto app_config::set_value_by_name(const std::string &name,
set_value(s3_config_.url, value);
return s3_config_.url;
}
if (name == "S3Config.UsePathStyle") {
set_value(s3_config_.use_path_style, utils::string::to_bool(value));
return utils::string::from_bool(s3_config_.use_path_style);
}
if (name == "S3Config.UseRegionInURL") {
set_value(s3_config_.use_region_in_url, utils::string::to_bool(value));
return std::to_string(s3_config_.use_region_in_url);
return utils::string::from_bool(s3_config_.use_region_in_url);
}
if (name == "S3Config.TimeoutMs") {
set_value(s3_config_.timeout_ms, utils::string::to_uint32(value));

16
src/comm/curl/requests/http_put_file.cpp
View File

@@ -24,27 +24,13 @@
#include "utils/string_utils.hpp"
namespace repertory::curl::requests {
auto http_put_file::get_path() const -> std::string {
if (reader) {
auto updated_path = path;
return utils::string::replace(updated_path, file_name,
reader->get_encrypted_file_name());
}
return http_request_base::get_path();
}
auto http_put_file::set_method(CURL *curl, stop_type &stop_requested) const
-> bool {
curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "PUT");
if (not source_path.empty()) {
curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L);
if (not encryption_token.value_or("").empty()) {
if (not reader) {
reader = std::make_shared<utils::encryption::encrypting_reader>(
file_name, source_path, stop_requested, encryption_token.value());
}
if (reader) {
curl_easy_setopt(curl, CURLOPT_READDATA, reader.get());
curl_easy_setopt(
curl, CURLOPT_READFUNCTION,

4
src/comm/packet/client_pool.cpp
View File

@@ -148,8 +148,8 @@ void client_pool::shutdown() {
unique_mutex_lock pool_lock(pool_mutex_);
if (not shutdown_) {
shutdown_ = true;
for (auto &pool : pool_lookup_) {
pool.second->shutdown();
for (auto &pool_entry : pool_lookup_) {
pool_entry.second->shutdown();
}
pool_lookup_.clear();
}

577
src/comm/s3/s3_comm.cpp
View File

@@ -1,577 +0,0 @@
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#if defined(REPERTORY_ENABLE_S3)
#include "comm/s3/s3_comm.hpp"
#include "app_config.hpp"
#include "comm/curl/curl_comm.hpp"
#include "comm/s3/s3_requests.hpp"
#include "events/event_system.hpp"
#include "events/events.hpp"
#include "providers/i_provider.hpp"
#include "types/repertory.hpp"
#include "types/s3.hpp"
#include "utils/encryption.hpp"
#include "utils/error_utils.hpp"
#include "utils/path_utils.hpp"
#include "utils/polling.hpp"
#include "utils/string_utils.hpp"
namespace repertory {
static const get_key_callback empty_key = []() { return ""; };
s3_comm::s3_comm(const app_config &config)
: config_(config), s3_config_(config.get_s3_config()) {
s3_config_.bucket = utils::string::trim(s3_config_.bucket);
// TODO make configurable
const auto enable_path_style =
utils::string::begins_with(s3_config_.url,
"https://gateway.storjshare.io") ||
utils::string::begins_with(s3_config_.url, "http://localhost") ||
utils::string::begins_with(s3_config_.url, "https://localhost") ||
utils::string::begins_with(s3_config_.url, "http://127.0.0.1") ||
utils::string::begins_with(s3_config_.url, "https://127.0.0.1");
s3_client_ = std::make_unique<curl_comm>(s3_config_);
s3_client_->enable_s3_path_style(enable_path_style);
polling::instance().set_callback(
{"s3_directory_cache", polling::frequency::high,
[this]() { this->clear_expired_directories(); }});
}
s3_comm::s3_comm(s3_comm &&comm)
: config_(std::move(comm.config_)),
s3_config_(std::move(comm.s3_config_)),
s3_client_(std::move(comm.s3_client_)) {
comm.active_ = false;
polling::instance().set_callback(
{"s3_directory_cache", polling::frequency::high,
[this]() { this->clear_expired_directories(); }});
}
s3_comm::~s3_comm() {
if (active_) {
polling::instance().remove_callback("s3_directory_cache");
}
}
void s3_comm::clear_expired_directories() {
recur_mutex_lock l(cached_directories_mutex_);
std::vector<std::string> expired_list;
for (const auto &kv : cached_directories_) {
if (kv.second.expiration <= std::chrono::system_clock::now()) {
expired_list.emplace_back(kv.first);
}
}
for (const auto &expired : expired_list) {
event_system::instance().raise<debug_log>(__FUNCTION__, expired, "expired");
cached_directories_.erase(expired);
}
}
auto s3_comm::create_directory(const std::string &api_path) -> api_error {
raise_begin(__FUNCTION__, api_path);
long response_code{};
auto object_name = get_object_name(api_path, empty_key) + '/';
if (not create_directory_object_request(*s3_client_, s3_config_, object_name,
response_code)) {
return raise_end(__FUNCTION__, api_path, api_error::comm_error,
response_code);
}
return raise_end(__FUNCTION__, api_path,
response_code == 200 ? api_error::success
: api_error::comm_error,
response_code);
}
auto s3_comm::directory_exists(const std::string &api_path) const -> api_error {
raise_begin(__FUNCTION__, api_path);
auto object_name = get_object_name(api_path, empty_key) + "/";
head_object_result result{};
long response_code{};
if (head_object_request(*s3_client_, s3_config_, object_name, result,
response_code)) {
if (response_code == 404) {
return raise_end(__FUNCTION__, api_path, api_error::directory_not_found,
response_code);
}
return raise_end(__FUNCTION__, api_path, api_error::directory_exists,
response_code);
}
return raise_end(__FUNCTION__, api_path, api_error::comm_error,
response_code);
}
auto s3_comm::file_exists(const std::string &api_path,
const get_key_callback &get_key) const -> api_error {
raise_begin(__FUNCTION__, api_path);
if (get_cached_file_exists(api_path)) {
return raise_end(__FUNCTION__, api_path, api_error::item_exists, 200);
}
auto object_name = get_object_name(api_path, get_key);
head_object_result result{};
long response_code{};
if (head_object_request(*s3_client_, s3_config_, object_name, result,
response_code)) {
if (response_code == 404) {
return raise_end(__FUNCTION__, api_path, api_error::item_not_found,
response_code);
}
return raise_end(__FUNCTION__, api_path, api_error::item_exists,
response_code);
}
return raise_end(__FUNCTION__, api_path, api_error::directory_exists,
response_code);
}
auto s3_comm::get_object_list(std::vector<directory_item> &list) const
-> api_error {
raise_begin(__FUNCTION__, "/");
long response_code{};
auto success =
list_objects_request(*s3_client_, s3_config_, list, response_code);
return raise_end(__FUNCTION__, "/",
success ? api_error::success : api_error::comm_error,
response_code);
}
auto s3_comm::get_object_name(const std::string &api_path,
const get_key_callback &get_key) const
-> std::string {
auto object_name = utils::path::create_api_path(api_path).substr(1);
const auto key = get_key();
if (not key.empty()) {
auto parts = utils::string::split(object_name, '/', false);
parts[parts.size() - 1u] = key;
object_name = utils::string::join(parts, '/');
}
return object_name;
}
auto s3_comm::get_cached_directory_item_count(const std::string &api_path,
std::size_t &count) const
-> bool {
recur_mutex_lock l(cached_directories_mutex_);
if (cached_directories_.find(api_path) != cached_directories_.end()) {
count = cached_directories_.at(api_path).items.size();
return true;
}
return false;
}
auto s3_comm::get_cached_directory_items(const std::string &api_path,
meta_provider_callback meta_provider,
directory_item_list &list) const
-> bool {
unique_recur_mutex_lock l(cached_directories_mutex_);
if (cached_directories_.find(api_path) != cached_directories_.end()) {
auto &cachedEntry = cached_directories_.at(api_path);
list = cachedEntry.items;
cached_directories_[api_path].reset_timeout(
std::chrono::seconds(config_.get_s3_config().cache_timeout_secs));
l.unlock();
for (auto &item : list) {
meta_provider(item);
}
return true;
}
return false;
}
auto s3_comm::get_cached_file_exists(const std::string &api_path) const
-> bool {
unique_recur_mutex_lock l(cached_directories_mutex_);
const auto parent_api_path = utils::path::get_parent_api_path(api_path);
if (cached_directories_.find(parent_api_path) != cached_directories_.end()) {
auto &entry = cached_directories_.at(parent_api_path);
if (std::find_if(entry.items.begin(), entry.items.end(),
[&api_path](const auto &item) -> bool {
return not item.directory && (api_path == item.api_path);
}) != entry.items.end()) {
cached_directories_[api_path].reset_timeout(
std::chrono::seconds(config_.get_s3_config().cache_timeout_secs));
return true;
}
}
return false;
}
auto s3_comm::get_directory_item_count(
const std::string &api_path, meta_provider_callback meta_provider) const
-> std::size_t {
raise_begin(__FUNCTION__, api_path);
std::size_t ret = 0u;
if (not get_cached_directory_item_count(api_path, ret)) {
directory_item_list list;
const auto res = grab_directory_items(api_path, meta_provider, list);
if (res != api_error::success) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, res,
"failed to grab directory items");
}
return list.size();
}
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(__FUNCTION__, api_path,
"end|" + std::to_string(ret));
}
return ret;
}
auto s3_comm::get_directory_items(const std::string &api_path,
meta_provider_callback meta_provider,
directory_item_list &list) const
-> api_error {
raise_begin(__FUNCTION__, api_path);
auto ret = api_error::success;
if (not get_cached_directory_items(api_path, meta_provider, list)) {
ret = grab_directory_items(api_path, meta_provider, list);
}
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(
__FUNCTION__, api_path, "end|" + api_error_to_string(ret));
}
return ret;
}
auto s3_comm::get_directory_list(api_file_list &list) const -> api_error {
raise_begin(__FUNCTION__, "/");
long response_code{};
auto success =
list_directories_request(*s3_client_, s3_config_, list, response_code);
return raise_end(__FUNCTION__, "/",
success ? api_error::success : api_error::comm_error,
response_code);
}
auto s3_comm::get_file(const std::string &api_path,
const get_key_callback &get_key,
const get_name_callback &get_name,
const get_token_callback &get_token,
api_file &file) const -> api_error {
raise_begin(__FUNCTION__, api_path);
auto ret = api_error::success;
auto object_name = get_object_name(api_path, get_key);
head_object_result result{};
long response_code{};
if (head_object_request(*s3_client_, s3_config_, object_name, result,
response_code)) {
const auto key = get_key();
object_name = get_name(key, object_name);
file.accessed_date = utils::get_file_time_now();
file.api_path = utils::path::create_api_path(object_name);
file.api_parent = utils::path::get_parent_api_path(file.api_path);
file.changed_date = utils::aws::format_time(result.last_modified);
file.creation_date = utils::aws::format_time(result.last_modified);
file.encryption_token = get_token();
file.file_size =
file.encryption_token.empty()
? result.content_length
: utils::encryption::encrypting_reader::calculate_decrypted_size(
result.content_length);
file.modified_date = utils::aws::format_time(result.last_modified);
} else {
utils::error::raise_api_path_error(__FUNCTION__, api_path, response_code,
"head object request failed");
ret = api_error::comm_error;
}
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(
__FUNCTION__, api_path, "end|" + api_error_to_string(ret));
}
return ret;
}
auto s3_comm::get_file_list(
const get_api_file_token_callback &get_api_file_token,
const get_name_callback &get_name, api_file_list &list) const -> api_error {
raise_begin(__FUNCTION__, "/");
long response_code{};
auto success = list_files_request(*s3_client_, s3_config_, get_api_file_token,
get_name, list, response_code);
return raise_end(__FUNCTION__, "/",
success ? api_error::success : api_error::comm_error,
response_code);
}
auto s3_comm::grab_directory_items(const std::string &api_path,
meta_provider_callback meta_provider,
directory_item_list &list) const
-> api_error {
auto object_name = get_object_name(api_path, empty_key);
long response_code{};
if (list_objects_in_directory_request(*s3_client_, s3_config_, object_name,
meta_provider, list, response_code)) {
if (response_code == 404) {
return api_error::directory_not_found;
}
if (response_code != 200) {
return api_error::comm_error;
}
set_cached_directory_items(api_path, list);
return api_error::success;
}
return api_error::comm_error;
}
void s3_comm::raise_begin(const std::string &function_name,
const std::string &api_path) const {
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(function_name, api_path,
"begin|");
}
}
auto s3_comm::raise_end(const std::string &function_name,
const std::string &api_path, const api_error &error,
long code) const -> api_error {
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(
function_name, api_path,
"end|" + api_error_to_string(error) + '|' + std::to_string(code));
}
return error;
}
auto s3_comm::read_file_bytes(const std::string &api_path, std::size_t size,
std::uint64_t offset, data_buffer &data,
const get_key_callback &get_key,
const get_size_callback &get_size,
const get_token_callback &get_token,
stop_type &stop_requested) const -> api_error {
data.clear();
auto object_name = get_object_name(api_path, get_key);
const auto encryption_token = get_token();
const auto data_size = get_size();
if (encryption_token.empty()) {
long response_code{};
if (not read_object_request(*s3_client_, s3_config_, object_name, size,
offset, data, response_code, stop_requested)) {
auto res =
stop_requested ? api_error::download_stopped : api_error::comm_error;
utils::error::raise_api_path_error(__FUNCTION__, api_path, res,
"failed to read file bytes");
return res;
}
if (response_code < 200 || response_code >= 300) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, response_code,
"failed to read file bytes");
return api_error::comm_error;
}
return api_error::success;
}
const auto key = utils::encryption::generate_key(encryption_token);
return utils::encryption::read_encrypted_range(
{offset, offset + size - 1}, key,
[&](data_buffer &ct, std::uint64_t start_offset,
std::uint64_t end_offset) -> api_error {
return read_file_bytes(
api_path, (end_offset - start_offset + 1u), start_offset, ct,
get_key, get_size, []() -> std::string { return ""; },
stop_requested);
},
data_size, data);
}
void s3_comm::remove_cached_directory(const std::string &api_path) {
recur_mutex_lock l(cached_directories_mutex_);
cached_directories_.erase(api_path);
}
auto s3_comm::remove_directory(const std::string &api_path) -> api_error {
raise_begin(__FUNCTION__, api_path);
auto object_name = get_object_name(api_path, empty_key) + "/";
long response_code{};
if (delete_object_request(*s3_client_, s3_config_, object_name,
response_code)) {
if (response_code == 404) {
return raise_end(__FUNCTION__, api_path, api_error::directory_not_found,
response_code);
}
if (response_code != 204) {
return raise_end(__FUNCTION__, api_path, api_error::comm_error,
response_code);
}
remove_cached_directory(utils::path::get_parent_api_path(api_path));
remove_cached_directory(api_path);
return raise_end(__FUNCTION__, api_path, api_error::success, response_code);
}
return raise_end(__FUNCTION__, api_path, api_error::comm_error,
response_code);
}
auto s3_comm::remove_file(const std::string &api_path,
const get_key_callback &get_key) -> api_error {
raise_begin(__FUNCTION__, api_path);
auto object_name = get_object_name(api_path, get_key);
long response_code{};
if (delete_object_request(*s3_client_, s3_config_, object_name,
response_code)) {
if (response_code == 404) {
return raise_end(__FUNCTION__, api_path, api_error::item_not_found,
response_code);
}
if (response_code != 204) {
return raise_end(__FUNCTION__, api_path, api_error::comm_error,
response_code);
}
remove_cached_directory(utils::path::get_parent_api_path(api_path));
return raise_end(__FUNCTION__, api_path, api_error::success, response_code);
}
return raise_end(__FUNCTION__, api_path, api_error::comm_error,
response_code);
}
auto s3_comm::rename_file(const std::string & /*api_path*/,
const std::string & /*new_api_path*/) -> api_error {
return api_error::not_implemented;
/* if (config_.get_event_level() >= event_level::debug) { */
/* event_system::instance().raise<debug_log>(__FUNCTION__, api_path,
* "begin"); */
/* } */
/* auto ret = api_error::success; */
/* */
/* std::string bucket_name, object_name; */
/* get_object_name(api_path, bucket_name, object_name); */
/* */
/* std::string new_object_name; */
/* get_object_name(new_api_path, bucket_name, new_object_name); */
/* */
/* Aws::S3::Model::CopyObjectRequest request{}; */
/* request.SetBucket(bucket_name); */
/* request.SetCopySource(bucket_name + '/' + object_name); */
/* request.SetKey(new_object_name); */
/* */
/* const auto outcome = s3_client_->CopyObject(request); */
/* if (outcome.IsSuccess()) { */
/* ret = remove_file(api_path); */
/* } else { */
/* const auto &error = outcome.GetError(); */
/* event_system::instance().raise<repertory_exception>(__FUNCTION__,
* error.GetExceptionName()
* +
* "|" + */
/* error.GetMessage());
*/
/* ret = api_error::comm_error; */
/* } */
/* */
/* if (config_.get_event_level() >= event_level::debug) { */
/* event_system::instance().raise<debug_log>(__FUNCTION__, api_path, */
/* "end|" +
* std::to_string(std::uint8_t(ret))); */
/* } */
/* return ret; */
}
void s3_comm::set_cached_directory_items(const std::string &api_path,
directory_item_list list) const {
recur_mutex_lock l(cached_directories_mutex_);
cached_directories_[api_path].items = std::move(list);
cached_directories_[api_path].reset_timeout(
std::chrono::seconds(config_.get_s3_config().cache_timeout_secs));
}
auto s3_comm::upload_file(const std::string &api_path,
const std::string &source_path,
const std::string &encryption_token,
const get_key_callback &get_key,
const set_key_callback &set_key,
stop_type &stop_requested) -> api_error {
raise_begin(__FUNCTION__, api_path);
auto object_name = get_object_name(api_path, get_key);
long response_code{};
if (not put_object_request(*s3_client_, s3_config_, object_name, source_path,
encryption_token, get_key, set_key, response_code,
stop_requested)) {
return raise_end(__FUNCTION__, api_path,
stop_requested ? api_error::upload_stopped
: api_error::upload_failed,
response_code);
}
if (response_code != 200) {
return raise_end(__FUNCTION__, api_path, api_error::comm_error,
response_code);
}
remove_cached_directory(utils::path::get_parent_api_path(api_path));
return raise_end(__FUNCTION__, api_path, api_error::success, response_code);
}
} // namespace repertory
#endif // REPERTORY_ENABLE_S3

19
src/comm/s3/s3_requests_curl.cpp
View File

@@ -65,8 +65,6 @@ auto create_directory_object_request_impl(i_http_comm &client,
curl::requests::http_put_file put_file{};
put_file.allow_timeout = true;
put_file.aws_service = "aws:amz:" + config.region + ":s3";
put_file.file_name =
*(utils::string::split(object_name, '/', false).end() - 1U);
put_file.path = '/' + object_name;
stop_type stop_requested{false};
@@ -212,12 +210,12 @@ auto list_files_request_impl(
file.api_path = utils::path::create_api_path(object_name);
file.api_parent = utils::path::get_parent_api_path(file.api_path);
file.accessed_date = utils::get_file_time_now();
file.encryption_token = get_api_file_token(file.api_path);
// file.encryption_token = get_api_file_token(file.api_path);
auto size = node.node().select_node("Size").node().text().as_ullong();
file.file_size = file.encryption_token.empty()
? size
: utils::encryption::encrypting_reader::
calculate_decrypted_size(size);
// file.file_size = file.encryption_token.empty()
// ? size
// : utils::encryption::encrypting_reader::
// calculate_decrypted_size(size);
file.changed_date = utils::convert_api_date(
node.node().select_node("LastModified").node().text().as_string());
file.creation_date = file.changed_date;
@@ -349,9 +347,6 @@ auto put_object_request_impl(i_http_comm &client, const s3_config &config,
try {
curl::requests::http_put_file put_file{};
put_file.aws_service = "aws:amz:" + config.region + ":s3";
put_file.encryption_token = encryption_token;
put_file.file_name =
*(utils::string::split(object_name, '/', false).end() - 1U);
put_file.path = '/' + object_name;
put_file.source_path = source_path;
@@ -359,8 +354,8 @@ auto put_object_request_impl(i_http_comm &client, const s3_config &config,
static stop_type no_stop{false};
put_file.reader = std::make_shared<utils::encryption::encrypting_reader>(
put_file.file_name, source_path, no_stop, encryption_token,
std::nullopt, -1);
*(utils::string::split(object_name, '/', false).end() - 1U),
source_path, no_stop, encryption_token, std::nullopt, -1);
auto key = get_key();
if (key.empty()) {
key = put_file.reader->get_encrypted_file_name();

370
src/db/directory_db.cpp
View File

@@ -1,370 +0,0 @@
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "db/directory_db.hpp"
#include "utils/path_utils.hpp"
namespace repertory {
void directory_db::directory_tree::add_path(
const std::string &api_path, const std::vector<std::string> &files,
rocksdb::DB &db) {
const auto create_not_found = [&](const auto &create_path) {
std::string value;
if (not db.Get(rocksdb::ReadOptions(), create_path, &value).ok()) {
json directoryData = {
{"path", api_path},
{"files", files},
};
db.Put(rocksdb::WriteOptions(), create_path, directoryData.dump());
}
};
const auto parts = utils::string::split(api_path, '/', false);
std::string previous_directory;
for (const auto &directory_part : parts) {
if (directory_part.empty()) {
sub_directory_lookup_["/"];
previous_directory = "/";
create_not_found("/");
} else {
auto &sub_directories = sub_directory_lookup_[previous_directory];
if (std::find(sub_directories.begin(), sub_directories.end(),
directory_part) == sub_directories.end()) {
sub_directories.emplace_back(directory_part);
}
previous_directory = utils::path::create_api_path(
utils::path::combine(previous_directory, {directory_part}));
sub_directory_lookup_[previous_directory];
create_not_found(previous_directory);
}
}
}
auto directory_db::directory_tree::get_count(const std::string &api_path) const
-> std::size_t {
return (sub_directory_lookup_.find(api_path) == sub_directory_lookup_.end())
? 0
: sub_directory_lookup_.at(api_path).size();
}
auto directory_db::directory_tree::get_directories() const
-> std::vector<std::string> {
std::vector<std::string> ret;
std::transform(sub_directory_lookup_.begin(), sub_directory_lookup_.end(),
std::back_inserter(ret),
[](const auto &kv) { return kv.first; });
return ret;
}
auto directory_db::directory_tree::get_sub_directories(
const std::string &api_path) const -> std::vector<std::string> {
std::vector<std::string> ret;
if (sub_directory_lookup_.find(api_path) != sub_directory_lookup_.end()) {
const auto &lookup = sub_directory_lookup_.at(api_path);
std::transform(lookup.begin(), lookup.end(), std::back_inserter(ret),
[&api_path](const auto &directory) {
return utils::path::create_api_path(
utils::path::combine(api_path, {directory}));
});
}
return ret;
}
auto directory_db::directory_tree::is_directory(
const std::string &api_path) const -> bool {
return sub_directory_lookup_.find(api_path) != sub_directory_lookup_.end();
}
void directory_db::directory_tree::remove_directory(const std::string &api_path,
rocksdb::DB &db,
bool allow_remove_root) {
if ((allow_remove_root || (api_path != "/")) && is_directory(api_path)) {
sub_directory_lookup_.erase(api_path);
db.Delete(rocksdb::WriteOptions(), api_path);
const auto parent_api_path = utils::path::get_parent_api_path(api_path);
const auto parts = utils::string::split(api_path, '/', false);
utils::remove_element_from(sub_directory_lookup_[parent_api_path],
parts[parts.size() - 1]);
}
}
directory_db::directory_db(const app_config &config) {
utils::db::create_rocksdb(config, DIRDB_NAME, db_);
auto iterator = std::unique_ptr<rocksdb::Iterator>(
db_->NewIterator(rocksdb::ReadOptions()));
for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
auto directory_data = json::parse(iterator->value().ToString());
tree_.add_path(directory_data["path"].get<std::string>(),
directory_data["files"].get<std::vector<std::string>>(),
*db_);
}
}
directory_db::~directory_db() { db_.reset(); }
auto directory_db::create_directory(const std::string &api_path,
bool create_always) -> api_error {
recur_mutex_lock directory_lock(directory_mutex_);
auto ret = api_error::directory_exists;
if (not is_directory(api_path)) {
ret = api_error::directory_not_found;
if (create_always || (api_path == "/") ||
is_directory(utils::path::get_parent_api_path(api_path))) {
ret = api_error::item_exists;
if (not is_file(api_path)) {
tree_.add_path(api_path, {}, *db_);
ret = api_error::success;
}
}
}
return ret;
}
auto directory_db::create_file(const std::string &api_path) -> api_error {
recur_mutex_lock directory_lock(directory_mutex_);
if (is_directory(api_path)) {
return api_error::directory_exists;
}
const auto parent_api_path = utils::path::get_parent_api_path(api_path);
auto directory_data = get_directory_data(parent_api_path);
if (directory_data.empty()) {
return api_error::directory_not_found;
}
const auto file_name = utils::path::strip_to_file_name(api_path);
if (utils::collection_includes(directory_data["files"], file_name)) {
return api_error::item_exists;
}
directory_data["files"].emplace_back(file_name);
db_->Put(rocksdb::WriteOptions(), parent_api_path, directory_data.dump());
return api_error::success;
}
auto directory_db::get_directory_data(const std::string &api_path) const
-> json {
std::string data;
db_->Get(rocksdb::ReadOptions(), api_path, &data);
if (data.empty()) {
return {};
}
return json::parse(data);
}
auto directory_db::get_directory_item_count(const std::string &api_path) const
-> std::uint64_t {
auto directory_data = get_directory_data(api_path);
const auto sub_directory_count = get_sub_directory_count(api_path);
const auto file_count =
(directory_data.empty() ? 0 : directory_data["files"].size());
return sub_directory_count + file_count;
}
auto directory_db::get_file(const std::string &api_path, api_file &file,
api_file_provider_callback api_file_provider) const
-> api_error {
const auto parent_api_path = utils::path::get_parent_api_path(api_path);
auto directory_data = get_directory_data(parent_api_path);
if (not directory_data.empty()) {
const auto file_name = utils::path::strip_to_file_name(api_path);
if (utils::collection_includes(directory_data["files"], file_name)) {
file.api_path = utils::path::create_api_path(
utils::path::combine(parent_api_path, {file_name})),
api_file_provider(file);
return api_error::success;
}
}
return api_error::item_not_found;
}
auto directory_db::get_file_list(
api_file_list &list, api_file_provider_callback api_file_provider) const
-> api_error {
auto iterator = std::unique_ptr<rocksdb::Iterator>(
db_->NewIterator(rocksdb::ReadOptions()));
for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
auto directory_data = json::parse(iterator->value().ToString());
for (const auto &directory_file : directory_data["files"]) {
api_file file{
utils::path::create_api_path(utils::path::combine(
iterator->key().ToString(), {directory_file.get<std::string>()})),
};
api_file_provider(file);
list.emplace_back(file);
}
}
return api_error::success;
}
auto directory_db::get_sub_directory_count(const std::string &api_path) const
-> std::size_t {
recur_mutex_lock directoryLock(directory_mutex_);
return tree_.get_count(api_path);
}
auto directory_db::get_total_item_count() const -> std::uint64_t {
unique_recur_mutex_lock directory_lock(directory_mutex_);
const auto directories = tree_.get_directories();
directory_lock.unlock();
return std::accumulate(
directories.begin(), directories.end(), std::uint64_t(directories.size()),
[this](std::uint64_t c, const std::string &directory) {
const auto dirData = this->get_directory_data(directory);
return c + (dirData.empty() ? 0 : dirData["files"].size());
});
}
auto directory_db::is_directory(const std::string &api_path) const -> bool {
recur_mutex_lock directory_lock(directory_mutex_);
return tree_.is_directory(api_path);
}
auto directory_db::is_file(const std::string &api_path) const -> bool {
auto directory_data =
get_directory_data(utils::path::get_parent_api_path(api_path));
if (directory_data.empty()) {
return false;
}
const auto file_name = utils::path::strip_to_file_name(api_path);
return utils::collection_includes(directory_data["files"], file_name);
}
void directory_db::populate_directory_files(
const std::string &api_path, meta_provider_callback meta_provider,
directory_item_list &list) const {
auto directory_data = get_directory_data(api_path);
if (not directory_data.empty()) {
for (const auto &directory_file : directory_data["files"]) {
directory_item di{};
di.api_path = utils::path::create_api_path(
utils::path::combine(api_path, {directory_file.get<std::string>()}));
di.directory = false;
meta_provider(di);
di.size = utils::string::to_uint64(di.meta[META_SIZE]);
list.emplace_back(std::move(di));
}
}
}
void directory_db::populate_sub_directories(
const std::string &api_path, meta_provider_callback meta_provider,
directory_item_list &list) const {
unique_recur_mutex_lock directory_lock(directory_mutex_);
const auto directories = tree_.get_sub_directories(api_path);
directory_lock.unlock();
std::size_t offset{};
for (const auto &directory : directories) {
if (std::find_if(list.begin(), list.end(),
[&directory](const auto &di) -> bool {
return directory == di.api_path;
}) == list.end()) {
directory_item di{};
di.api_path = directory;
di.api_parent = utils::path::get_parent_api_path(directory);
di.directory = true;
di.size = get_sub_directory_count(directory);
meta_provider(di);
list.insert(list.begin() + static_cast<std::int64_t>(offset++),
std::move(di));
}
}
}
auto directory_db::remove_directory(const std::string &api_path,
bool allow_remove_root) -> api_error {
recur_mutex_lock directory_lock(directory_mutex_);
if ((api_path == "/") && not allow_remove_root) {
return api_error::access_denied;
}
if (is_file(api_path) || not is_directory(api_path)) {
return api_error::directory_not_found;
}
if (tree_.get_count(api_path) == 0) {
auto directory_data = get_directory_data(api_path);
if (directory_data.empty() || directory_data["files"].empty()) {
tree_.remove_directory(api_path, *db_, allow_remove_root);
return api_error::success;
}
}
return api_error::directory_not_empty;
}
auto directory_db::remove_file(const std::string &api_path) -> bool {
recur_mutex_lock directory_lock(directory_mutex_);
if (is_directory(api_path)) {
return false;
}
const auto parent_api_path = utils::path::get_parent_api_path(api_path);
auto directory_data = get_directory_data(parent_api_path);
if (directory_data.empty()) {
return false;
}
const auto file_name = utils::path::strip_to_file_name(api_path);
if (utils::collection_excludes(directory_data["files"], file_name)) {
return false;
}
utils::remove_element_from(directory_data["files"], file_name);
db_->Put(rocksdb::WriteOptions(), parent_api_path, directory_data.dump());
return true;
}
auto directory_db::rename_file(const std::string &from_api_path,
const std::string &to_api_path) -> api_error {
recur_mutex_lock directory_lock(directory_mutex_);
if (is_directory(from_api_path) || is_directory(to_api_path)) {
return api_error::directory_exists;
}
if (not is_directory(utils::path::get_parent_api_path(to_api_path))) {
return api_error::directory_not_found;
}
if (is_file(to_api_path)) {
return api_error::item_exists;
}
if (not remove_file(from_api_path)) {
return api_error::item_not_found;
}
return create_file(to_api_path);
}
} // namespace repertory

347
src/db/meta_db.cpp
View File

@@ -1,347 +0,0 @@
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "db/meta_db.hpp"
#include "types/repertory.hpp"
#include "types/startup_exception.hpp"
#include "utils/error_utils.hpp"
#include "utils/file_utils.hpp"
#include "utils/path_utils.hpp"
#include "utils/utils.hpp"
namespace repertory {
meta_db::meta_db(const app_config &config) {
const auto create_resources = [this, &config](const std::string &name) {
auto families = std::vector<rocksdb::ColumnFamilyDescriptor>();
families.emplace_back(rocksdb::kDefaultColumnFamilyName,
rocksdb::ColumnFamilyOptions());
families.emplace_back("keys", rocksdb::ColumnFamilyOptions());
families.emplace_back("source", rocksdb::ColumnFamilyOptions());
auto handles = std::vector<rocksdb::ColumnFamilyHandle *>();
utils::db::create_rocksdb(config, name, families, handles, db_);
std::size_t idx{};
default_family_ = handles[idx++];
keys_family_ = handles[idx++];
source_family_ = handles[idx++];
};
create_resources(METADB_NAME);
}
meta_db::~meta_db() { db_.reset(); }
auto meta_db::create_iterator(bool source_family) const
-> std::shared_ptr<rocksdb::Iterator> {
return std::shared_ptr<rocksdb::Iterator>(
db_->NewIterator(rocksdb::ReadOptions(),
source_family ? source_family_ : default_family_));
}
auto meta_db::get_api_path_from_key(const std::string &key,
std::string &api_path) const -> api_error {
if (key.empty()) {
return api_error::item_not_found;
}
return perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
return db_->Get(rocksdb::ReadOptions(), keys_family_, key, &api_path);
});
}
auto meta_db::get_api_path_from_source(const std::string &source_path,
std::string &api_path) const
-> api_error {
if (source_path.empty()) {
return api_error::item_not_found;
}
return perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
return db_->Get(rocksdb::ReadOptions(), source_family_, source_path,
&api_path);
});
}
auto meta_db::get_item_meta_json(const std::string &api_path,
json &json_data) const -> api_error {
std::string value;
const auto res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
return db_->Get(rocksdb::ReadOptions(), default_family_, api_path, &value);
});
if (res != api_error::success) {
return res;
}
json_data = json::parse(value);
return api_error::success;
}
auto meta_db::get_item_meta(const std::string &api_path,
api_meta_map &meta) const -> api_error {
json json_data;
const auto ret = get_item_meta_json(api_path, json_data);
if (ret == api_error::success) {
for (auto it = json_data.begin(); it != json_data.end(); it++) {
meta[it.key()] = it.value().get<std::string>();
}
}
return ret;
}
auto meta_db::get_item_meta(const std::string &api_path, const std::string &key,
std::string &value) const -> api_error {
json json_data;
const auto ret = get_item_meta_json(api_path, json_data);
if (ret == api_error::success) {
if (json_data.find(key) != json_data.end()) {
value = json_data[key].get<std::string>();
}
}
return ret;
}
auto meta_db::get_item_meta_exists(const std::string &api_path) const -> bool {
std::string value;
return db_->Get(rocksdb::ReadOptions(), api_path, &value).ok();
}
auto meta_db::get_pinned_files() const -> std::vector<std::string> {
std::vector<std::string> ret;
auto iterator = const_cast<meta_db *>(this)->create_iterator(false);
for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
auto api_path = iterator->key().ToString();
std::string pinned;
const auto res = get_item_meta(api_path, META_PINNED, pinned);
if ((res == api_error::success) && not pinned.empty() &&
utils::string::to_bool(pinned)) {
ret.emplace_back(api_path);
}
}
return ret;
}
auto meta_db::get_source_path_exists(const std::string &source_path) const
-> bool {
std::string value;
return db_->Get(rocksdb::ReadOptions(), source_family_, source_path, &value)
.ok();
}
auto meta_db::perform_action(
const std::string &function_name,
const std::function<rocksdb::Status()> &action) const -> api_error {
const auto res = action();
if (res.ok()) {
return api_error::success;
}
if (not res.IsNotFound()) {
utils::error::raise_error(function_name, res.ToString());
}
return res.IsNotFound() ? api_error::item_not_found : api_error::error;
}
auto meta_db::get_total_item_count() const -> std::uint64_t {
std::uint64_t ret = 0u;
auto iter = create_iterator(false);
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
ret++;
}
return ret;
}
auto meta_db::remove_item_meta(const std::string &api_path) -> api_error {
json json_data;
auto res = get_item_meta_json(api_path, json_data);
if (res != api_error::success) {
return res == api_error::item_not_found ? api_error::success : res;
}
if (not json_data[META_KEY].empty()) {
if ((res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
return db_->Delete(rocksdb::WriteOptions(), keys_family_,
json_data[META_KEY].get<std::string>());
})) != api_error::success) {
return res;
}
}
if (not json_data[META_SOURCE].empty()) {
if ((res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
return db_->Delete(rocksdb::WriteOptions(), source_family_,
json_data[META_SOURCE].get<std::string>());
})) != api_error::success) {
return res;
}
}
return perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
return db_->Delete(rocksdb::WriteOptions(), default_family_, api_path);
});
}
auto meta_db::remove_item_meta(const std::string &api_path,
const std::string &key) -> api_error {
json json_data;
auto res = get_item_meta_json(api_path, json_data);
if (res != api_error::success) {
return res == api_error::item_not_found ? api_error::success : res;
}
if ((key == META_KEY) && not json_data[META_KEY].empty()) {
if ((res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
return db_->Delete(rocksdb::WriteOptions(), keys_family_,
json_data[META_KEY].get<std::string>());
})) != api_error::success) {
return res;
}
}
if ((key == META_SOURCE) && not json_data[META_SOURCE].empty()) {
if ((res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
return db_->Delete(rocksdb::WriteOptions(), source_family_,
json_data[META_SOURCE].get<std::string>());
})) != api_error::success) {
return res;
}
}
json_data.erase(key);
return perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
return db_->Put(rocksdb::WriteOptions(), default_family_, api_path,
json_data.dump());
});
}
auto meta_db::rename_item_meta(const std::string &source_path,
const std::string &from_api_path,
const std::string &to_api_path) -> api_error {
api_meta_map meta{};
auto res = get_item_meta(from_api_path, meta);
if (res != api_error::success) {
return res;
}
if ((res = remove_item_meta(from_api_path)) != api_error::success) {
return res;
}
if (not source_path.empty()) {
meta[META_SOURCE] = source_path;
}
return set_item_meta(to_api_path, meta);
}
auto meta_db::set_item_meta(const std::string &api_path, const std::string &key,
const std::string &value) -> api_error {
if (key == META_SOURCE) {
return set_source_path(api_path, value);
}
if (key == META_KEY) {
const auto res = remove_item_meta(api_path, META_KEY);
if ((res != api_error::success) && (res != api_error::item_not_found)) {
return res;
}
}
auto res = store_item_meta(api_path, key, value);
if (res != api_error::success) {
return res;
}
if (key == META_KEY) {
return perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
return db_->Put(rocksdb::WriteOptions(), keys_family_, value, api_path);
});
}
return api_error::success;
}
auto meta_db::set_item_meta(const std::string &api_path,
const api_meta_map &meta) -> api_error {
auto ret = api_error::success;
auto it = meta.begin();
for (std::size_t i = 0u; (ret == api_error::success) && (i < meta.size());
i++) {
ret = set_item_meta(api_path, it->first, it->second);
it++;
}
return ret;
}
auto meta_db::set_source_path(const std::string &api_path,
const std::string &source_path) -> api_error {
std::string current_source_path;
auto res = get_item_meta(api_path, META_SOURCE, current_source_path);
if ((res != api_error::success) && (res != api_error::item_not_found)) {
return res;
}
// TODO multiple db ops should be in transaction
if (not current_source_path.empty()) {
if ((res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
return db_->Delete(rocksdb::WriteOptions(), source_family_,
current_source_path);
})) != api_error::success) {
return res;
}
}
if (not source_path.empty()) {
if ((res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
return db_->Put(rocksdb::WriteOptions(), source_family_, source_path,
api_path);
})) != api_error::success) {
return res;
}
}
return store_item_meta(api_path, META_SOURCE, source_path);
}
auto meta_db::store_item_meta(const std::string &api_path,
const std::string &key, const std::string &value)
-> api_error {
json json_data;
auto res = get_item_meta_json(api_path, json_data);
if ((res != api_error::success) && (res != api_error::item_not_found)) {
return res;
}
json_data[key] = value;
return perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
return db_->Put(rocksdb::WriteOptions(), default_family_, api_path,
json_data.dump());
});
}
} // namespace repertory

9
src/drives/fuse/fuse_drive.cpp
View File

@@ -170,8 +170,7 @@ auto fuse_drive::create_impl(std::string api_path, mode_t mode,
auto meta = create_meta_attributes(
now, FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_ARCHIVE, now, now,
is_directory_op, "", get_effective_gid(), "", mode, now, 0U, osx_flags,
0U,
is_directory_op, get_effective_gid(), "", mode, now, 0U, osx_flags, 0U,
utils::path::combine(config_.get_cache_directory(),
{utils::create_uuid_string()}),
get_effective_uid(), now);
@@ -589,9 +588,9 @@ auto fuse_drive::mkdir_impl(std::string api_path, mode_t mode) -> api_error {
}
const auto now = utils::get_file_time_now();
auto meta = create_meta_attributes(
now, FILE_ATTRIBUTE_DIRECTORY, now, now, true, "", get_effective_gid(),
"", mode, now, 0U, 0U, 0U, "", get_effective_uid(), now);
auto meta = create_meta_attributes(now, FILE_ATTRIBUTE_DIRECTORY, now, now,
true, get_effective_gid(), "", mode, now,
0U, 0U, 0U, "", get_effective_uid(), now);
if ((res = provider_.create_directory(api_path, meta)) !=
api_error::success) {
return res;

4
src/drives/winfsp/winfsp_drive.cpp
View File

@@ -281,8 +281,8 @@ auto winfsp_drive::Create(PWSTR file_name, UINT32 create_options,
const auto now = utils::get_file_time_now();
auto meta = create_meta_attributes(
now, attributes, now, now, attributes & FILE_ATTRIBUTE_DIRECTORY, "", 0U,
"", 0U, now, 0U, 0U, 0U,
now, attributes, now, now, attributes & FILE_ATTRIBUTE_DIRECTORY, 0U, "",
0U, now, 0U, 0U, 0U,
(attributes & FILE_ATTRIBUTE_DIRECTORY)
? ""
: utils::path::combine(config_.get_cache_directory(),

4
src/file_manager/file_manager_upload.cpp
View File

@@ -48,8 +48,8 @@ void file_manager::upload::cancel() {
void file_manager::upload::stop() { stop_requested_ = true; }
void file_manager::upload::upload_thread() {
error_ = provider_.upload_file(fsi_.api_path, fsi_.source_path,
fsi_.encryption_token, stop_requested_);
error_ =
provider_.upload_file(fsi_.api_path, fsi_.source_path, stop_requested_);
if (not utils::file::reset_modified_time(fsi_.source_path)) {
utils::error::raise_api_path_error(
__FUNCTION__, fsi_.api_path, fsi_.source_path,

13
src/platform/unix_platform.cpp
View File

@@ -185,11 +185,10 @@ auto lock_data::wait_for_lock(int fd, std::uint8_t retry_count) -> int {
auto create_meta_attributes(
std::uint64_t accessed_date, std::uint32_t attributes,
std::uint64_t changed_date, std::uint64_t creation_date, bool directory,
const std::string &encryption_token, std::uint32_t gid,
const std::string &key, std::uint32_t mode, std::uint64_t modified_date,
std::uint32_t osx_backup, std::uint32_t osx_flags, std::uint64_t size,
const std::string &source_path, std::uint32_t uid,
std::uint64_t written_date) -> api_meta_map {
std::uint32_t gid, const std::string &key, std::uint32_t mode,
std::uint64_t modified_date, std::uint32_t osx_backup,
std::uint32_t osx_flags, std::uint64_t size, const std::string &source_path,
std::uint32_t uid, std::uint64_t written_date) -> api_meta_map {
return {
{META_ACCESSED, std::to_string(accessed_date)},
{META_ATTRIBUTES, std::to_string(attributes)},
@@ -197,7 +196,6 @@ auto create_meta_attributes(
{META_CHANGED, std::to_string(changed_date)},
{META_CREATION, std::to_string(creation_date)},
{META_DIRECTORY, utils::string::from_bool(directory)},
{META_ENCRYPTION_TOKEN, encryption_token},
{META_GID, std::to_string(gid)},
{META_KEY, key},
{META_MODE, std::to_string(mode)},
@@ -217,8 +215,7 @@ auto provider_meta_handler(i_provider &provider, bool directory,
file.accessed_date,
directory ? FILE_ATTRIBUTE_DIRECTORY
: FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_ARCHIVE,
file.changed_date, file.creation_date, directory, file.encryption_token,
getgid(), file.key,
file.changed_date, file.creation_date, directory, getgid(), file.key,
directory ? S_IFDIR | S_IRUSR | S_IWUSR | S_IXUSR
: S_IFREG | S_IRUSR | S_IWUSR,
file.modified_date, 0u, 0u, file.file_size, file.source_path, getuid(),

16
src/platform/win32_platform.cpp
View File

@@ -166,11 +166,10 @@ auto lock_data::set_mount_state(bool active, const std::string &mount_location,
auto create_meta_attributes(
std::uint64_t accessed_date, std::uint32_t attributes,
std::uint64_t changed_date, std::uint64_t creation_date, bool directory,
const std::string &encryption_token, std::uint32_t gid,
const std::string &key, std::uint32_t mode, std::uint64_t modified_date,
std::uint32_t osx_backup, std::uint32_t osx_flags, std::uint64_t size,
const std::string &source_path, std::uint32_t uid,
std::uint64_t written_date) -> api_meta_map {
std::uint32_t gid, const std::string &key, std::uint32_t mode,
std::uint64_t modified_date, std::uint32_t osx_backup,
std::uint32_t osx_flags, std::uint64_t size, const std::string &source_path,
std::uint32_t uid, std::uint64_t written_date) -> api_meta_map {
return {
{META_ACCESSED, std::to_string(accessed_date)},
{META_ATTRIBUTES, std::to_string(attributes)},
@@ -178,7 +177,6 @@ auto create_meta_attributes(
{META_CHANGED, std::to_string(changed_date)},
{META_CREATION, std::to_string(creation_date)},
{META_DIRECTORY, utils::string::from_bool(directory)},
{META_ENCRYPTION_TOKEN, encryption_token},
{META_GID, std::to_string(gid)},
{META_KEY, key},
{META_MODE, std::to_string(mode)},
@@ -198,9 +196,9 @@ auto provider_meta_handler(i_provider &provider, bool directory,
file.accessed_date,
directory ? FILE_ATTRIBUTE_DIRECTORY
: FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_ARCHIVE,
file.changed_date, file.creation_date, directory, file.encryption_token,
0u, file.key, directory ? S_IFDIR : S_IFREG, file.modified_date, 0u, 0u,
file.file_size, file.source_path, 0u, file.modified_date);
file.changed_date, file.creation_date, directory, 0u, file.key,
directory ? S_IFDIR : S_IFREG, file.modified_date, 0u, 0u, file.file_size,
file.source_path, 0u, file.modified_date);
auto res = provider.set_item_meta(file.api_path, meta);
if (res == api_error::success) {
event_system::instance().raise<filesystem_item_added>(

546
src/providers/base_provider.cpp
View File

@@ -1,546 +0,0 @@
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "providers/base_provider.hpp"
#include "app_config.hpp"
#include "file_manager/i_file_manager.hpp"
#include "types/repertory.hpp"
#include "types/startup_exception.hpp"
#include "utils/error_utils.hpp"
#include "utils/file_utils.hpp"
#include "utils/native_file.hpp"
#include "utils/path_utils.hpp"
namespace repertory {
base_provider::base_provider(app_config &config) : config_(config) {}
void base_provider::calculate_used_drive_space(bool add_missing) {
api_file_list list{};
if (get_file_list(list) != api_error::success) {
return;
}
used_space_ = std::accumulate(
list.begin(), list.end(), std::uint64_t(0U),
[this, &add_missing](std::uint64_t total_size, const auto &file) {
if (add_missing && not meta_db_->get_item_meta_exists(file.api_path)) {
[[maybe_unused]] auto res = this->notify_file_added(
file.api_path, utils::path::get_parent_api_path(file.api_path),
0);
}
return total_size + file.file_size;
});
}
void base_provider::cleanup() {
remove_deleted_files();
remove_unknown_source_files();
remove_expired_orphaned_files();
}
auto base_provider::create_directory_clone_source_meta(
const std::string &source_api_path, const std::string &api_path)
-> api_error {
api_meta_map meta{};
auto ret = get_item_meta(source_api_path, meta);
if (ret == api_error::success) {
ret = create_directory(api_path, meta);
}
return ret == api_error::item_not_found ? api_error::directory_not_found
: ret;
}
auto base_provider::create_file(const std::string &api_path, api_meta_map &meta)
-> api_error {
bool exists{};
auto res = is_directory(api_path, exists);
if (res != api_error::success) {
return res;
}
if (exists) {
return api_error::directory_exists;
}
res = is_file(api_path, exists);
if (res != api_error::success) {
return res;
}
if (exists) {
return api_error::item_exists;
}
if ((res = meta_db_->set_item_meta(api_path, meta)) != api_error::success) {
return res;
}
{
native_file_ptr nf;
res = native_file::create_or_open(meta[META_SOURCE], nf);
if (res != api_error::success) {
return res;
}
nf->close();
}
stop_type stop_requested = false;
return upload_file(api_path, meta[META_SOURCE], meta[META_ENCRYPTION_TOKEN],
stop_requested);
}
auto base_provider::get_api_path_from_source(const std::string &source_path,
std::string &api_path) const
-> api_error {
return meta_db_->get_api_path_from_source(source_path, api_path);
}
auto base_provider::get_directory_items(const std::string &api_path,
directory_item_list &list) const
-> api_error {
auto res = populate_directory_items(api_path, list);
if (res != api_error::success) {
return res;
}
std::sort(list.begin(), list.end(), [](const auto &a, const auto &b) -> bool {
return (a.directory && not b.directory) ||
(not(b.directory && not a.directory) &&
(a.api_path.compare(b.api_path) < 0));
});
list.insert(list.begin(), directory_item{
"..",
"",
true,
});
list.insert(list.begin(), directory_item{
".",
"",
true,
});
return api_error::success;
}
auto base_provider::get_file(const std::string &api_path, api_file &file) const
-> api_error {
auto ret = api_error::success;
try {
if ((ret = populate_file(api_path, file)) != api_error::success) {
event_system::instance().raise<file_get_failed>(api_path,
api_error_to_string(ret));
}
std::string sz;
if ((ret = get_item_meta(api_path, META_SIZE, sz)) != api_error::success) {
return ret;
}
file.file_size = utils::string::to_uint64(sz);
return ret;
} catch (const std::exception &e) {
event_system::instance().raise<file_get_failed>(
api_path, e.what() ? e.what() : "failed to get file");
}
return api_error::error;
}
auto base_provider::get_file_size(const std::string &api_path,
std::uint64_t &file_size) const -> api_error {
api_file file{};
const auto ret = get_file(api_path, file);
if (ret == api_error::success) {
file_size = file.file_size;
} else {
event_system::instance().raise<file_get_size_failed>(
api_path, api_error_to_string(ret));
}
return ret;
}
auto base_provider::get_filesystem_item(const std::string &api_path,
bool directory,
filesystem_item &fsi) const
-> api_error {
auto ret = api_error::error;
if (directory) {
bool exists{};
ret = is_directory(api_path, exists);
if (ret != api_error::success) {
return ret;
}
ret = exists ? api_error::success : api_error::item_not_found;
update_filesystem_item(true, ret, api_path, fsi);
} else {
api_file file{};
ret = get_filesystem_item_and_file(api_path, file, fsi);
}
return ret;
}
auto base_provider::get_filesystem_item_and_file(const std::string &api_path,
api_file &file,
filesystem_item &fsi) const
-> api_error {
auto ret = get_item_meta(api_path, META_SOURCE, fsi.source_path);
if (ret == api_error::success) {
ret = get_file(api_path, file);
if (ret == api_error::success) {
fsi.encryption_token = file.encryption_token;
fsi.size = file.file_size;
} else {
bool exists{};
ret = is_file(api_path, exists);
if (ret != api_error::success) {
return ret;
}
if (not exists) {
ret = api_error::item_not_found;
}
}
}
update_filesystem_item(false, ret, api_path, fsi);
return ret;
}
auto base_provider::get_filesystem_item_from_source_path(
const std::string &source_path, filesystem_item &fsi) const -> api_error {
auto ret = api_error::item_not_found;
if (not source_path.empty()) {
std::string api_path;
if ((ret = get_api_path_from_source(source_path, api_path)) ==
api_error::success) {
ret = get_filesystem_item(api_path, false, fsi);
}
}
return ret;
}
auto base_provider::get_item_meta(const std::string &api_path,
api_meta_map &meta) const -> api_error {
auto ret = meta_db_->get_item_meta(api_path, meta);
if (ret == api_error::item_not_found) {
auto get_meta = false;
bool exists{};
ret = is_directory(api_path, exists);
if (ret != api_error::success) {
return ret;
}
if (exists) {
ret = notify_directory_added(api_path,
utils::path::get_parent_api_path(api_path));
if (ret == api_error::success) {
get_meta = true;
}
} else {
ret = is_file(api_path, exists);
if (ret != api_error::success) {
return ret;
}
if (exists) {
std::uint64_t file_size{};
if ((ret = get_file_size(api_path, file_size)) == api_error::success) {
get_meta = ((ret = notify_file_added(
api_path, utils::path::get_parent_api_path(api_path),
file_size)) == api_error::success);
}
}
}
ret = get_meta ? meta_db_->get_item_meta(api_path, meta)
: api_error::item_not_found;
}
return ret;
}
auto base_provider::get_item_meta(const std::string &api_path,
const std::string &key,
std::string &value) const -> api_error {
auto ret = meta_db_->get_item_meta(api_path, key, value);
if (ret == api_error::item_not_found) {
auto get_meta = false;
bool exists{};
ret = is_directory(api_path, exists);
if (ret != api_error::success) {
return ret;
}
if (exists) {
ret = notify_directory_added(api_path,
utils::path::get_parent_api_path(api_path));
if (ret == api_error::success) {
get_meta = true;
}
} else {
ret = is_file(api_path, exists);
if (ret != api_error::success) {
return ret;
}
if (exists) {
std::uint64_t file_size{};
if ((ret = get_file_size(api_path, file_size)) == api_error::success) {
get_meta = ((ret = notify_file_added(
api_path, utils::path::get_parent_api_path(api_path),
file_size)) == api_error::success);
}
}
}
ret = get_meta ? meta_db_->get_item_meta(api_path, key, value)
: api_error::item_not_found;
}
return ret;
}
auto base_provider::get_used_drive_space() const -> std::uint64_t {
std::uint64_t used_space = used_space_;
fm_->update_used_space(used_space);
return used_space;
}
auto base_provider::notify_directory_added(const std::string &api_path,
const std::string &api_parent)
-> api_error {
recur_mutex_lock l(notify_added_mutex_);
const auto now = utils::get_file_time_now();
api_file file{};
file.api_path = api_path;
file.api_parent = api_parent;
file.accessed_date = now;
file.changed_date = now;
file.creation_date = now;
file.file_size = 0U;
file.modified_date = now;
return api_item_added_(true, file);
}
auto base_provider::processed_orphaned_file(const std::string &source_path,
const std::string &api_path) const
-> bool {
const auto orphaned_directory =
utils::path::combine(get_config().get_data_directory(), {"orphaned"});
if (utils::file::create_full_directory_path(orphaned_directory)) {
event_system::instance().raise<orphaned_file_detected>(source_path);
const auto parts = utils::string::split(api_path, '/', false);
const auto orphaned_file = utils::path::combine(
orphaned_directory, {utils::path::strip_to_file_name(source_path) +
'_' + parts[parts.size() - 1U]});
if (utils::file::reset_modified_time(source_path) &&
utils::file::move_file(source_path, orphaned_file)) {
event_system::instance().raise<orphaned_file_processed>(source_path,
orphaned_file);
return true;
}
event_system::instance().raise<orphaned_file_processing_failed>(
source_path, orphaned_file,
std::to_string(utils::get_last_error_code()));
return false;
}
utils::error::raise_error(
__FUNCTION__, std::to_string(utils::get_last_error_code()),
"failed to create orphaned director|sp|" + orphaned_directory);
return false;
}
void base_provider::remove_deleted_files() {
std::vector<std::string> removed_files{};
api_file_list list{};
if (get_file_list(list) == api_error::success) {
if (not list.empty()) {
auto iterator = meta_db_->create_iterator(false);
for (iterator->SeekToFirst(); not stop_requested_ && iterator->Valid();
iterator->Next()) {
const auto meta_api_path = iterator->key().ToString();
if (meta_api_path.empty()) {
const auto res = meta_db_->remove_item_meta(meta_api_path);
if (res != api_error::success) {
utils::error::raise_api_path_error(__FUNCTION__, meta_api_path, res,
"failed to remove item meta");
}
} else {
auto api_path = meta_api_path;
const auto it = std::find_if(list.begin(), list.end(),
[&api_path](const auto &file) -> bool {
return file.api_path == api_path;
});
if (it == list.end()) {
removed_files.emplace_back(api_path);
}
}
}
}
}
while (not stop_requested_ && not removed_files.empty()) {
const auto api_path = removed_files.back();
removed_files.pop_back();
bool exists{};
if (is_directory(api_path, exists) != api_error::success) {
continue;
}
std::string source_path;
if (not exists &&
(check_file_exists(api_path) == api_error::item_not_found) &&
(meta_db_->get_item_meta(api_path, META_SOURCE, source_path) ==
api_error::success)) {
if (not source_path.empty()) {
fm_->perform_locked_operation(
[this, &api_path, &source_path](i_provider &) -> bool {
if (fm_->has_no_open_file_handles()) {
auto res = meta_db_->remove_item_meta(api_path);
if (res == api_error::success) {
event_system::instance().raise<file_removed_externally>(
api_path, source_path);
processed_orphaned_file(source_path, api_path);
} else {
utils::error::raise_api_path_error(
__FUNCTION__, api_path, source_path, res,
"failed to remove item meta for externally removed file");
}
}
return true;
});
}
}
}
}
void base_provider::remove_expired_orphaned_files() {
const auto orphaned_directory =
utils::path::combine(get_config().get_data_directory(), {"orphaned"});
const auto files = utils::file::get_directory_files(orphaned_directory, true);
for (const auto &file : files) {
if (utils::file::is_modified_date_older_than(
file, std::chrono::hours(
get_config().get_orphaned_file_retention_days() * 24))) {
if (utils::file::retry_delete_file(file)) {
event_system::instance().raise<orphaned_file_deleted>(file);
}
}
if (stop_requested_) {
break;
}
}
}
void base_provider::remove_unknown_source_files() {
auto files = utils::file::get_directory_files(
get_config().get_cache_directory(), true);
while (not stop_requested_ && not files.empty()) {
const auto file = files.front();
files.pop_front();
std::string api_path;
if (not meta_db_->get_source_path_exists(file)) {
processed_orphaned_file(file);
}
}
}
auto base_provider::rename_file(const std::string &from_api_path,
const std::string &to_api_path) -> api_error {
std::string source_path;
auto ret = get_item_meta(from_api_path, META_SOURCE, source_path);
if (ret != api_error::success) {
return ret;
}
std::string encryption_token;
ret = get_item_meta(from_api_path, META_ENCRYPTION_TOKEN, encryption_token);
if (ret != api_error::success) {
return ret;
}
ret = handle_rename_file(from_api_path, to_api_path, source_path);
return ret;
}
auto base_provider::start(api_item_added_callback api_item_added,
i_file_manager *fm) -> bool {
meta_db_ = std::make_unique<meta_db>(config_);
api_item_added_ = api_item_added;
fm_ = fm;
auto unmount_requested = false;
{
repertory::event_consumer ec(
"unmount_requested",
[&unmount_requested](const event &) { unmount_requested = true; });
for (std::uint16_t i = 0U; not unmount_requested && not is_online() &&
(i < get_config().get_online_check_retry_secs());
i++) {
event_system::instance().raise<provider_offline>(
get_config().get_host_config().host_name_or_ip,
get_config().get_host_config().api_port);
std::this_thread::sleep_for(1s);
}
}
auto ret = not unmount_requested && is_online();
if (ret) {
// Force root creation
api_meta_map meta{};
auto res = get_item_meta("/", meta);
if (res != api_error::success) {
throw startup_exception("failed to create root|err|" +
api_error_to_string(res));
}
calculate_used_drive_space(false);
}
return ret;
}
void base_provider::stop() { meta_db_.reset(); }
void base_provider::update_filesystem_item(bool directory,
const api_error &error,
const std::string &api_path,
filesystem_item &fsi) const {
if (error == api_error::success) {
fsi.directory = directory;
fsi.api_path = api_path;
fsi.api_parent = utils::path::get_parent_api_path(api_path);
} else {
event_system::instance().raise<filesystem_item_get_failed>(
api_path, std::to_string(static_cast<int>(error)));
}
}
} // namespace repertory

2
src/providers/encrypt/encrypt_provider.cpp
View File

@@ -33,7 +33,7 @@
namespace repertory {
encrypt_provider::encrypt_provider(app_config &config) : config_(config) {}
auto encrypt_provider::create_api_file(const std::string api_path,
auto encrypt_provider::create_api_file(const std::string &api_path,
bool directory,
const std::string &source_path)
-> api_file {

10
src/providers/provider.cpp
View File

@@ -24,8 +24,6 @@
#include "app_config.hpp"
#include "comm/curl/curl_comm.hpp"
#include "comm/i_http_comm.hpp"
#include "comm/i_s3_comm.hpp"
#include "comm/s3/s3_comm.hpp"
#include "events/events.hpp"
#include "providers/encrypt/encrypt_provider.hpp"
#include "providers/s3/s3_provider.hpp"
@@ -51,9 +49,6 @@ auto create_provider(const provider_type &pt, app_config &config)
mutex_lock lock(mutex);
static std::unique_ptr<i_http_comm> comm;
#if defined(REPERTORY_ENABLE_S3)
static std::unique_ptr<i_s3_comm> s3_comm_;
#endif // defined(REPERTORY_ENABLE_S3)
switch (pt) {
case provider_type::sia: {
@@ -64,9 +59,10 @@ auto create_provider(const provider_type &pt, app_config &config)
}
#if defined(REPERTORY_ENABLE_S3)
case provider_type::s3: {
create_comm<i_s3_comm, s3_comm, app_config>(s3_comm_, config);
create_comm<i_http_comm, curl_comm, s3_config>(comm,
config.get_s3_config());
return std::unique_ptr<i_provider>(
dynamic_cast<i_provider *>(new s3_provider(config, *s3_comm_)));
dynamic_cast<i_provider *>(new s3_provider(config, *comm)));
}
#endif // defined(REPERTORY_ENABLE_S3)
case provider_type::encrypt: {

1480
src/providers/s3/s3_provider.cpp
View File

File diff suppressed because it is too large Load Diff

83
src/providers/sia/sia_provider.cpp
View File

@@ -23,7 +23,6 @@
#include "app_config.hpp"
#include "comm/i_http_comm.hpp"
#include "db/meta_db.hpp"
#include "events/events.hpp"
#include "file_manager/i_file_manager.hpp"
#include "types/repertory.hpp"
@@ -31,6 +30,7 @@
#include "utils/file_utils.hpp"
#include "utils/path_utils.hpp"
#include "utils/polling.hpp"
#include "utils/rocksdb_utils.hpp"
#include "utils/string_utils.hpp"
#include "utils/utils.hpp"
@@ -47,7 +47,7 @@ auto sia_provider::get_object_info(const std::string &api_path,
get.response_handler = [&object_info](const data_buffer &data,
long response_code) {
if (response_code == 200) {
if (response_code == http_error_codes::ok) {
object_info = nlohmann::json::parse(data.begin(), data.end());
}
};
@@ -58,11 +58,11 @@ auto sia_provider::get_object_info(const std::string &api_path,
return api_error::comm_error;
}
if (response_code == 404) {
if (response_code == http_error_codes::not_found) {
return api_error::item_not_found;
}
if (response_code != 200) {
if (response_code != http_error_codes::ok) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, response_code,
"failed to get object info");
return api_error::comm_error;
@@ -77,7 +77,7 @@ auto sia_provider::get_object_info(const std::string &api_path,
return api_error::error;
}
auto sia_provider::get_object_list(const std::string api_path,
auto sia_provider::get_object_list(const std::string &api_path,
nlohmann::json &object_list) const -> bool {
curl::requests::http_get get{};
get.allow_timeout = true;
@@ -85,7 +85,7 @@ auto sia_provider::get_object_list(const std::string api_path,
get.response_handler = [&object_list](const data_buffer &data,
long response_code) {
if (response_code == 200) {
if (response_code == http_error_codes::ok) {
object_list = nlohmann::json::parse(data.begin(), data.end());
}
};
@@ -99,7 +99,7 @@ auto sia_provider::get_object_list(const std::string api_path,
return false;
}
if (response_code != 200) {
if (response_code != http_error_codes::ok) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, response_code,
"failed to get object list");
return false;
@@ -166,8 +166,6 @@ auto sia_provider::create_directory(const std::string &api_path,
try {
curl::requests::http_put_file put_file{};
put_file.file_name =
*(utils::string::split(api_path, '/', false).end() - 1u);
put_file.path = "/api/worker/objects" + api_path + "/";
long response_code{};
@@ -179,7 +177,7 @@ auto sia_provider::create_directory(const std::string &api_path,
return api_error::comm_error;
}
if (response_code != 200) {
if (response_code != http_error_codes::ok) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, response_code,
"failed to create directory");
return api_error::comm_error;
@@ -272,8 +270,6 @@ auto sia_provider::create_file(const std::string &api_path, api_meta_map &meta)
try {
curl::requests::http_put_file put_file{};
put_file.file_name =
*(utils::string::split(api_path, '/', false).end() - 1u);
put_file.path = "/api/worker/objects" + api_path;
long response_code{};
@@ -285,7 +281,7 @@ auto sia_provider::create_file(const std::string &api_path, api_meta_map &meta)
return api_error::comm_error;
}
if (response_code != 200) {
if (response_code != http_error_codes::ok) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, response_code,
"failed to create file");
return api_error::comm_error;
@@ -716,7 +712,7 @@ auto sia_provider::get_total_drive_space() const -> std::uint64_t {
json config_data{};
get.response_handler = [&config_data](const data_buffer &data,
long response_code) {
if (response_code == 200) {
if (response_code == http_error_codes::ok) {
config_data = nlohmann::json::parse(data.begin(), data.end());
}
};
@@ -727,7 +723,7 @@ auto sia_provider::get_total_drive_space() const -> std::uint64_t {
return 0U;
}
if (response_code != 200) {
if (response_code != http_error_codes::ok) {
utils::error::raise_error(__FUNCTION__, response_code,
"failed to get total drive space");
return 0U;
@@ -762,7 +758,7 @@ auto sia_provider::get_used_drive_space() const -> std::uint64_t {
json object_data{};
get.response_handler = [&object_data](const data_buffer &data,
long response_code) {
if (response_code == 200) {
if (response_code == http_error_codes::ok) {
object_data = nlohmann::json::parse(data.begin(), data.end());
}
};
@@ -773,7 +769,7 @@ auto sia_provider::get_used_drive_space() const -> std::uint64_t {
return 0U;
}
if (response_code != 200) {
if (response_code != http_error_codes::ok) {
utils::error::raise_error(__FUNCTION__, response_code,
"failed to get used drive space");
return 0U;
@@ -797,6 +793,8 @@ auto sia_provider::is_directory(const std::string &api_path, bool &exists) const
return api_error::success;
}
exists = false;
try {
json object_list{};
if (not get_object_list(utils::path::get_parent_api_path(api_path),
@@ -821,24 +819,31 @@ auto sia_provider::is_directory(const std::string &api_path, bool &exists) const
auto sia_provider::is_file(const std::string &api_path, bool &exists) const
-> api_error {
exists = false;
if (api_path == "/") {
return api_error::success;
}
json file_data{};
auto res = get_object_info(api_path, file_data);
if (res == api_error::item_not_found) {
exists = false;
try {
json file_data{};
auto res = get_object_info(api_path, file_data);
if (res == api_error::item_not_found) {
return api_error::success;
}
if (res != api_error::success) {
return res;
}
exists = not file_data.contains("entries");
return api_error::success;
} catch (const std::exception &e) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, e,
"failed to determine path is directory");
}
if (res != api_error::success) {
return res;
}
exists = not file_data.contains("entries");
return api_error::success;
return api_error::error;
}
auto sia_provider::is_file_writeable(const std::string &api_path) const
@@ -861,7 +866,7 @@ auto sia_provider::is_online() const -> bool {
json state_data{};
get.response_handler = [&state_data](const data_buffer &data,
long response_code) {
if (response_code == 200) {
if (response_code == http_error_codes::ok) {
state_data = nlohmann::json::parse(data.begin(), data.end());
}
};
@@ -874,7 +879,7 @@ auto sia_provider::is_online() const -> bool {
return false;
}
if (response_code != 200) {
if (response_code != http_error_codes::ok) {
utils::error::raise_error(__FUNCTION__, response_code,
"failed to determine if provider is online");
return false;
@@ -897,7 +902,10 @@ auto sia_provider::read_file_bytes(const std::string &api_path,
stop_type &stop_requested) -> api_error {
curl::requests::http_get get{};
get.path = "/api/worker/objects" + api_path;
get.range = {{offset, offset + size - 1U}};
get.range = {{
offset,
offset + size - 1U,
}};
get.response_handler = [&buffer](const data_buffer &data,
long /*response_code*/) { buffer = data; };
@@ -928,7 +936,8 @@ auto sia_provider::read_file_bytes(const std::string &api_path,
continue;
}
if (response_code < 200 || response_code >= 300) {
if (response_code < http_error_codes::ok ||
response_code >= http_error_codes::multiple_choices) {
notify_retry();
continue;
}
@@ -1064,7 +1073,7 @@ auto sia_provider::remove_directory(const std::string &api_path) -> api_error {
return notify_end(api_error::comm_error);
}
if (response_code != 200) {
if (response_code != http_error_codes::ok) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, response_code,
"failed to remove directory");
return notify_end(api_error::comm_error);
@@ -1138,7 +1147,8 @@ auto sia_provider::remove_file(const std::string &api_path) -> api_error {
return notify_end(api_error::comm_error);
}
if (response_code != 200 && response_code != 404) {
if (response_code != http_error_codes::ok &&
response_code != http_error_codes::not_found) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, response_code,
"failed to remove file");
return notify_end(api_error::comm_error);
@@ -1283,7 +1293,6 @@ void sia_provider::stop() {
auto sia_provider::upload_file(const std::string &api_path,
const std::string &source_path,
const std::string & /* encryption_token */,
stop_type &stop_requested) -> api_error {
event_system::instance().raise<provider_upload_begin>(api_path, source_path);
@@ -1296,8 +1305,6 @@ auto sia_provider::upload_file(const std::string &api_path,
try {
curl::requests::http_put_file put_file{};
put_file.file_name =
*(utils::string::split(api_path, '/', false).end() - 1u);
put_file.path = "/api/worker/objects" + api_path;
put_file.source_path = source_path;
@@ -1309,7 +1316,7 @@ auto sia_provider::upload_file(const std::string &api_path,
return notify_end(api_error::comm_error);
}
if (response_code != 200) {
if (response_code != http_error_codes::ok) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, source_path,
response_code,
"failed to upload file");

120
src/utils/encrypting_reader.cpp
View File

@@ -31,6 +31,12 @@
namespace repertory::utils::encryption {
class encrypting_streambuf final : public encrypting_reader::streambuf {
public:
encrypting_streambuf(const encrypting_streambuf &) = default;
encrypting_streambuf(encrypting_streambuf &&) = delete;
auto operator=(const encrypting_streambuf &)
-> encrypting_streambuf & = delete;
auto operator=(encrypting_streambuf &&) -> encrypting_streambuf & = delete;
explicit encrypting_streambuf(const encrypting_reader &reader)
: reader_(reader) {
setg(reinterpret_cast<char *>(0), reinterpret_cast<char *>(0),
@@ -102,7 +108,7 @@ protected:
static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(gptr())));
char c{};
const auto res = reader_.reader_function(&c, 1u, 1u, &reader_);
const auto res = encrypting_reader::reader_function(&c, 1U, 1U, &reader_);
if (res != 1) {
return traits_type::eof();
}
@@ -118,7 +124,8 @@ protected:
reader_.set_read_position(
static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(gptr())));
const auto res = reader_.reader_function(ptr, 1u, count, &reader_);
const auto res = encrypting_reader::reader_function(
ptr, 1U, static_cast<std::size_t>(count), &reader_);
if ((res == reader_.get_error_return()) ||
(reader_.get_stop_requested() && (res == CURL_READFUNC_ABORT))) {
return traits_type::eof();
@@ -126,12 +133,20 @@ protected:
setg(eback(), gptr() + res,
reinterpret_cast<char *>(reader_.get_total_size()));
return res;
return static_cast<std::streamsize>(res);
}
};
class encrypting_reader_iostream final : public encrypting_reader::iostream {
public:
encrypting_reader_iostream(const encrypting_reader_iostream &) = delete;
encrypting_reader_iostream(encrypting_reader_iostream &&) = delete;
auto operator=(const encrypting_reader_iostream &)
-> encrypting_reader_iostream & = delete;
auto operator=(encrypting_reader_iostream &&)
-> encrypting_reader_iostream & = delete;
explicit encrypting_reader_iostream(
std::unique_ptr<encrypting_streambuf> buffer)
: encrypting_reader::iostream(buffer.get()), buffer_(std::move(buffer)) {}
@@ -146,14 +161,14 @@ const std::size_t encrypting_reader::header_size_ = ([]() {
return crypto_aead_xchacha20poly1305_IETF_NPUBBYTES +
crypto_aead_xchacha20poly1305_IETF_ABYTES;
})();
const std::size_t encrypting_reader::data_chunk_size_ = (8u * 1024u * 1024u);
const std::size_t encrypting_reader::data_chunk_size_ = (8UL * 1024UL * 1024UL);
const std::size_t encrypting_reader::encrypted_chunk_size_ =
data_chunk_size_ + header_size_;
encrypting_reader::encrypting_reader(
const std::string &file_name, const std::string &source_path,
stop_type &stop_requested, const std::string &token,
std::optional<std::string> relative_parent_path, const size_t error_return)
std::optional<std::string> relative_parent_path, std::size_t error_return)
: key_(utils::encryption::generate_key(token)),
stop_requested_(stop_requested),
error_return_(error_return) {
@@ -191,24 +206,22 @@ encrypting_reader::encrypting_reader(
file_size, static_cast<std::uint64_t>(data_chunk_size_)));
total_size_ =
file_size + (total_chunks * encrypting_reader::get_header_size());
last_data_chunk_ = total_chunks - 1u;
last_data_chunk_ = total_chunks - 1U;
last_data_chunk_size_ = static_cast<std::size_t>(
(file_size <= data_chunk_size_) ? file_size
: (file_size % data_chunk_size_) ? file_size % data_chunk_size_
: data_chunk_size_);
(file_size <= data_chunk_size_) ? file_size
: (file_size % data_chunk_size_) == 0U ? data_chunk_size_
: file_size % data_chunk_size_);
iv_list_.resize(total_chunks);
for (auto &iv : iv_list_) {
randombytes_buf(iv.data(), iv.size());
}
}
encrypting_reader::encrypting_reader(
const std::string &encrypted_file_path, const std::string &source_path,
stop_type &stop_requested, const std::string &token,
std::vector<
std::array<unsigned char, crypto_aead_xchacha20poly1305_IETF_NPUBBYTES>>
iv_list,
const size_t error_return)
encrypting_reader::encrypting_reader(const std::string &encrypted_file_path,
const std::string &source_path,
stop_type &stop_requested,
const std::string &token,
std::size_t error_return)
: key_(utils::encryption::generate_key(token)),
stop_requested_(stop_requested),
error_return_(error_return) {
@@ -232,27 +245,68 @@ encrypting_reader::encrypting_reader(
file_size, static_cast<std::uint64_t>(data_chunk_size_)));
total_size_ =
file_size + (total_chunks * encrypting_reader::get_header_size());
last_data_chunk_ = total_chunks - 1u;
last_data_chunk_ = total_chunks - 1U;
last_data_chunk_size_ = static_cast<std::size_t>(
(file_size <= data_chunk_size_) ? file_size
: (file_size % data_chunk_size_) ? file_size % data_chunk_size_
: data_chunk_size_);
(file_size <= data_chunk_size_) ? file_size
: (file_size % data_chunk_size_) == 0U ? data_chunk_size_
: file_size % data_chunk_size_);
iv_list_.resize(total_chunks);
for (auto &iv : iv_list_) {
randombytes_buf(iv.data(), iv.size());
}
}
encrypting_reader::encrypting_reader(
const std::string &encrypted_file_path, const std::string &source_path,
stop_type &stop_requested, const std::string &token,
std::vector<
std::array<unsigned char, crypto_aead_xchacha20poly1305_IETF_NPUBBYTES>>
iv_list,
std::size_t error_return)
: key_(utils::encryption::generate_key(token)),
stop_requested_(stop_requested),
error_return_(error_return) {
const auto res = native_file::open(source_path, false, source_file_);
if (res != api_error::success) {
throw std::runtime_error("file open failed|src|" + source_path + '|' +
api_error_to_string(res));
}
encrypted_file_path_ = encrypted_file_path;
encrypted_file_name_ =
std::filesystem::path(encrypted_file_path_).filename().string();
std::uint64_t file_size{};
if (not utils::file::get_file_size(source_path, file_size)) {
throw std::runtime_error("get file size failed|src|" + source_path + '|' +
std::to_string(utils::get_last_error_code()));
}
const auto total_chunks = static_cast<std::size_t>(utils::divide_with_ceiling(
file_size, static_cast<std::uint64_t>(data_chunk_size_)));
total_size_ =
file_size + (total_chunks * encrypting_reader::get_header_size());
last_data_chunk_ = total_chunks - 1U;
last_data_chunk_size_ = static_cast<std::size_t>(
(file_size <= data_chunk_size_) ? file_size
: (file_size % data_chunk_size_) == 0U ? data_chunk_size_
: file_size % data_chunk_size_);
iv_list_ = std::move(iv_list);
}
encrypting_reader::encrypting_reader(const encrypting_reader &r)
: key_(r.key_),
stop_requested_(r.stop_requested_),
error_return_(r.error_return_),
chunk_buffers_(r.chunk_buffers_),
encrypted_file_name_(r.encrypted_file_name_),
encrypted_file_path_(r.encrypted_file_path_),
iv_list_(r.iv_list_),
last_data_chunk_(r.last_data_chunk_),
last_data_chunk_size_(r.last_data_chunk_size_),
read_offset_(r.read_offset_),
source_file_(native_file::clone(r.source_file_)),
total_size_(r.total_size_) {}
encrypting_reader::encrypting_reader(const encrypting_reader &reader)
: key_(reader.key_),
stop_requested_(reader.stop_requested_),
error_return_(reader.error_return_),
chunk_buffers_(reader.chunk_buffers_),
encrypted_file_name_(reader.encrypted_file_name_),
encrypted_file_path_(reader.encrypted_file_path_),
iv_list_(reader.iv_list_),
last_data_chunk_(reader.last_data_chunk_),
last_data_chunk_size_(reader.last_data_chunk_size_),
read_offset_(reader.read_offset_),
source_file_(native_file::clone(reader.source_file_)),
total_size_(reader.total_size_) {}
encrypting_reader::~encrypting_reader() {
if (source_file_) {

6
src/utils/encryption.cpp
View File

@@ -104,7 +104,7 @@ auto read_encrypted_range(
const auto start_chunk =
static_cast<std::size_t>(range.begin / data_chunk_size);
const auto end_chunk = static_cast<std::size_t>(range.end / data_chunk_size);
auto remain = range.end - range.begin + 1u;
auto remain = range.end - range.begin + 1U;
auto source_offset = static_cast<std::size_t>(range.begin % data_chunk_size);
for (std::size_t chunk = start_chunk; chunk <= end_chunk; chunk++) {
@@ -112,8 +112,8 @@ auto read_encrypted_range(
const auto start_offset = chunk * encrypted_chunk_size;
const auto end_offset = std::min(
start_offset + (total_size - (chunk * data_chunk_size)) + header_size -
1u,
static_cast<std::uint64_t>(start_offset + encrypted_chunk_size - 1u));
1U,
static_cast<std::uint64_t>(start_offset + encrypted_chunk_size - 1U));
const auto result = reader(ct, start_offset, end_offset);
if (result != api_error::success) {