BlockStorage/repertory
1
0
Fork 0
Code Issues 10 Pull Requests Releases 7 Wiki Activity

2.0.0-rc (#9)
Some checks failed
BlockStorage/repertory_osx/pipeline/head This commit looks good
Details
BlockStorage/repertory_windows/pipeline/head This commit looks good
Details
BlockStorage/repertory/pipeline/head There was a failure building this commit
Details
BlockStorage/repertory_linux_builds/pipeline/head This commit looks good
Details
BlockStorage/repertory_osx_builds/pipeline/head There was a failure building this commit
Details

Browse Source 
### Issues

* \#1 \[bug\] Unable to mount S3 due to 'item_not_found' exception
* \#2 Require bucket name for S3 mounts
* \#3 \[bug\] File size is not being updated in S3 mount
* \#4 Upgrade to libfuse-3.x.x
* \#5 Switch to renterd for Sia support
* \#6 Switch to cpp-httplib to further reduce dependencies
* \#7 Remove global_data and calculate used disk space per provider
* \#8 Switch to libcurl for S3 mount support

### Changes from v1.x.x

* Added read-only encrypt provider
  * Pass-through mount point that transparently encrypts source data using `XChaCha20-Poly1305`
* Added S3 encryption support via `XChaCha20-Poly1305`
* Added replay protection to remote mounts
* Added support base64 writes in remote FUSE
* Created static linked Linux binaries for `amd64` and `aarch64` using `musl-libc`
* Removed legacy Sia renter support
* Removed Skynet support
* Fixed multiple remote mount WinFSP API issues on \*NIX servers
* Implemented chunked read and write
  * Writes for non-cached files are performed in chunks of 8Mib
* Removed `repertory-ui` support
* Removed `FreeBSD` support
* Switched to `libsodium` over `CryptoPP`
* Switched to `XChaCha20-Poly1305` for remote mounts
* Updated `GoogleTest` to v1.14.0
* Updated `JSON for Modern C++` to v3.11.2
* Updated `OpenSSL` to v1.1.1w
* Updated `RocksDB` to v8.5.3
* Updated `WinFSP` to 2023
* Updated `boost` to v1.78.0
* Updated `cURL` to v8.3.0
* Updated `zlib` to v1.3
* Use `upload_manager` for all providers
  * Adds a delay to uploads to prevent excessive API calls
  * Supports re-upload after mount restart for incomplete uploads
  * NOTE: Uploads for all providers are full file (no resume support)
    * Multipart upload support is planned for S3

Reviewed-on: #9
This commit is contained in:
Scott E. Graves
2023-10-29 06:55:59 +00:00
parent 3ff46723b8
commit f43c41f88a
839 changed files with 98214 additions and 92959 deletions
Download Patch File Download Diff File Expand all files Collapse all files

796
src/app_config.cpp
View File

File diff suppressed because it is too large Load Diff

677
src/comm/aws_s3/aws_s3_comm.cpp
View File

@@ -1,677 +0,0 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#if defined(REPERTORY_ENABLE_S3)
#include "comm/aws_s3/aws_s3_comm.hpp"
#include "app_config.hpp"
#include "events/events.hpp"
#include "events/event_system.hpp"
#include "providers/i_provider.hpp"
#include "types/repertory.hpp"
#include "utils/encryption.hpp"
#include "utils/path_utils.hpp"
#include "utils/polling.hpp"
#include "utils/string_utils.hpp"
namespace repertory {
static const i_s3_comm::get_key_callback empty_key = []() { return ""; };
aws_s3_comm::aws_s3_comm(const app_config &config)
: config_(config), s3_config_(config.get_s3_config()) {
s3_config_.bucket = utils::string::trim(s3_config_.bucket);
Aws::InitAPI(sdk_options_);
if (config_.get_event_level() >= event_level::debug) {
Aws::Utils::Logging::InitializeAWSLogging(
Aws::MakeShared<Aws::Utils::Logging::DefaultLogSystem>(
"Repertory", Aws::Utils::Logging::LogLevel::Trace,
utils::path::combine(config_.get_log_directory(), {"aws_sdk_"})));
}
Aws::Auth::AWSCredentials credentials;
credentials.SetAWSAccessKeyId(s3_config_.access_key);
credentials.SetAWSSecretKey(s3_config_.secret_key);
Aws::Client::ClientConfiguration configuration;
configuration.endpointOverride = s3_config_.url;
configuration.httpRequestTimeoutMs = s3_config_.timeout_ms;
configuration.region = s3_config_.region;
configuration.requestTimeoutMs = s3_config_.timeout_ms;
configuration.connectTimeoutMs = s3_config_.timeout_ms;
// TODO make configurable
const auto enable_path_style = utils::string::begins_with(s3_config_.url, "http://localhost") ||
utils::string::begins_with(s3_config_.url, "https://localhost") ||
utils::string::begins_with(s3_config_.url, "http://127.0.0.1") ||
utils::string::begins_with(s3_config_.url, "https://127.0.0.1");
s3_client_ = std::make_unique<Aws::S3::S3Client>(
credentials, configuration, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
not enable_path_style);
polling::instance().set_callback(
{"s3_directory_cache", false, [this]() { this->clear_expired_directories(); }});
}
aws_s3_comm::~aws_s3_comm() {
polling::instance().remove_callback("s3_directory_cache");
Aws::ShutdownAPI(sdk_options_);
Aws::Utils::Logging::ShutdownAWSLogging();
s3_client_.reset();
}
void aws_s3_comm::clear_expired_directories() {
recur_mutex_lock l(cached_directories_mutex_);
std::vector<std::string> expired_list;
for (const auto &kv : cached_directories_) {
if (kv.second.expiration <= std::chrono::system_clock::now()) {
expired_list.emplace_back(kv.first);
}
}
for (const auto &expired : expired_list) {
event_system::instance().raise<debug_log>(__FUNCTION__, expired, "expired");
cached_directories_.erase(expired);
}
}
api_error aws_s3_comm::create_bucket(const std::string &api_path) {
std::string bucket_name, object_name;
get_bucket_name_and_object_name(api_path, empty_key, bucket_name, object_name);
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(__FUNCTION__, bucket_name, "begin");
}
auto ret = api_error::access_denied;
if (s3_config_.bucket.empty()) {
Aws::S3::Model::CreateBucketRequest request{};
request.SetBucket(bucket_name);
const auto outcome = s3_client_->CreateBucket(request);
if (outcome.IsSuccess()) {
remove_cached_directory(utils::path::get_parent_api_path(api_path));
ret = api_error::success;
} else {
const auto &error = outcome.GetError();
event_system::instance().raise<repertory_exception>(
__FUNCTION__, error.GetExceptionName() + "|" + error.GetMessage());
ret = api_error::comm_error;
}
}
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(__FUNCTION__, bucket_name,
"end|" + std::to_string(static_cast<int>(ret)));
}
return ret;
}
bool aws_s3_comm::exists(const std::string &api_path, const get_key_callback &get_key) const {
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(__FUNCTION__, api_path, "begin");
}
if (api_path == "/") {
return false;
}
auto ret = false;
if (not get_cached_file_exists(api_path, ret)) {
std::string bucket_name, object_name;
get_bucket_name_and_object_name(api_path, get_key, bucket_name, object_name);
Aws::S3::Model::HeadObjectRequest request{};
request.SetBucket(bucket_name);
request.SetKey(object_name);
ret = s3_client_->HeadObject(request).IsSuccess();
}
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(__FUNCTION__, api_path, "end|" + std::to_string(ret));
}
return ret;
}
void aws_s3_comm::get_bucket_name_and_object_name(const std::string &api_path,
const get_key_callback &get_key,
std::string &bucket_name,
std::string &object_name) const {
bucket_name = s3_config_.bucket;
object_name = api_path.substr(1);
if (bucket_name.empty()) {
bucket_name = utils::string::split(api_path, '/')[1];
object_name = object_name.substr(bucket_name.size());
if (not object_name.empty() && (object_name[0] == '/')) {
object_name = object_name.substr(1);
}
}
const auto key = get_key();
if (not key.empty()) {
auto parts = utils::string::split(object_name, '/', false);
parts[parts.size() - 1u] = key;
object_name = utils::string::join(parts, '/');
}
}
bool aws_s3_comm::get_cached_directory_item_count(const std::string &api_path,
std::size_t &count) const {
recur_mutex_lock l(cached_directories_mutex_);
if (cached_directories_.find(api_path) != cached_directories_.end()) {
count = cached_directories_.at(api_path).items.size();
return true;
}
return false;
}
bool aws_s3_comm::get_cached_directory_items(const std::string &api_path,
const meta_provider_callback &meta_provider,
directory_item_list &list) const {
unique_recur_mutex_lock l(cached_directories_mutex_);
if (cached_directories_.find(api_path) != cached_directories_.end()) {
auto &cachedEntry = cached_directories_.at(api_path);
list = cachedEntry.items;
cached_directories_[api_path].reset_timeout(
std::chrono::seconds(config_.get_s3_config().cache_timeout_secs));
l.unlock();
for (auto &item : list) {
meta_provider(item, false);
}
return true;
}
return false;
}
bool aws_s3_comm::get_cached_file_exists(const std::string &api_path, bool &exists) const {
exists = false;
unique_recur_mutex_lock l(cached_directories_mutex_);
const auto parent_api_path = utils::path::get_parent_api_path(api_path);
if (cached_directories_.find(parent_api_path) != cached_directories_.end()) {
auto &entry = cached_directories_.at(parent_api_path);
exists =
std::find_if(entry.items.begin(), entry.items.end(), [&api_path](const auto &item) -> bool {
return not item.directory && (api_path == item.api_path);
}) != entry.items.end();
if (exists) {
cached_directories_[api_path].reset_timeout(
std::chrono::seconds(config_.get_s3_config().cache_timeout_secs));
return true;
}
}
return false;
}
std::size_t
aws_s3_comm::get_directory_item_count(const std::string &api_path,
const meta_provider_callback &meta_provider) const {
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(__FUNCTION__, api_path, "begin");
}
std::size_t ret = 0u;
if (not(s3_config_.bucket.empty() && (api_path == "/"))) {
if (not get_cached_directory_item_count(api_path, ret)) {
directory_item_list list;
grab_directory_items(api_path, meta_provider, list);
return list.size();
}
}
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(__FUNCTION__, api_path, "end|" + std::to_string(ret));
}
return ret;
}
api_error aws_s3_comm::get_directory_items(const std::string &api_path,
const meta_provider_callback &meta_provider,
directory_item_list &list) const {
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(__FUNCTION__, api_path, "begin");
}
auto ret = api_error::success;
if (not get_cached_directory_items(api_path, meta_provider, list)) {
ret = grab_directory_items(api_path, meta_provider, list);
}
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(__FUNCTION__, api_path,
"end|" + std::to_string(std::uint8_t(ret)));
}
return ret;
}
api_error aws_s3_comm::get_file(const std::string &api_path, const get_key_callback &get_key,
const get_name_callback &get_name,
const get_token_callback &get_token, api_file &file) const {
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(__FUNCTION__, api_path, "begin");
}
auto ret = api_error::success;
std::string bucket_name, object_name;
get_bucket_name_and_object_name(api_path, get_key, bucket_name, object_name);
Aws::S3::Model::HeadObjectRequest request{};
request.SetBucket(bucket_name);
request.SetKey(object_name);
const auto outcome = s3_client_->HeadObject(request);
if (outcome.IsSuccess()) {
const auto key = get_key();
auto object = outcome.GetResult();
object_name = get_name(key, object_name);
file.accessed_date = utils::get_file_time_now();
file.api_path = utils::path::create_api_path(utils::path::combine(bucket_name, {object_name}));
file.api_parent = utils::path::get_parent_api_path(file.api_path);
file.changed_date = object.GetLastModified().Millis() * 1000u * 1000u;
file.created_date = object.GetLastModified().Millis() * 1000u * 1000u;
file.encryption_token = get_token();
if (file.encryption_token.empty()) {
file.file_size = object.GetContentLength();
} else {
file.file_size =
utils::encryption::encrypting_reader::calculate_decrypted_size(object.GetContentLength());
}
file.modified_date = object.GetLastModified().Millis() * 1000u * 1000u;
file.recoverable = true;
file.redundancy = 3.0;
} else {
const auto &error = outcome.GetError();
event_system::instance().raise<repertory_exception>(__FUNCTION__, error.GetExceptionName() +
"|" + error.GetMessage());
ret = api_error::comm_error;
}
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(__FUNCTION__, api_path,
"end|" + std::to_string(std::uint8_t(ret)));
}
return ret;
}
api_error aws_s3_comm::get_file_list(const get_api_file_token_callback &get_api_file_token,
const get_name_callback &get_name, api_file_list &list) const {
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(__FUNCTION__, "/", "begin");
}
list.clear();
auto ret = api_error::success;
const auto bucket_name = s3_config_.bucket;
if (bucket_name.empty()) {
const auto outcome = s3_client_->ListBuckets();
if (outcome.IsSuccess()) {
const auto &bucket_list = outcome.GetResult().GetBuckets();
for (const auto &bucket : bucket_list) {
get_file_list(bucket.GetName(), get_api_file_token, get_name, list);
}
} else {
const auto &error = outcome.GetError();
event_system::instance().raise<repertory_exception>(
__FUNCTION__, error.GetExceptionName() + "|" + error.GetMessage());
ret = api_error::comm_error;
}
} else {
ret = get_file_list("", get_api_file_token, get_name, list);
}
return ret;
}
api_error aws_s3_comm::get_file_list(const std::string &bucket_name,
const get_api_file_token_callback &get_api_file_token,
const get_name_callback &get_name, api_file_list &list) const {
auto ret = api_error::success;
Aws::S3::Model::ListObjectsRequest request{};
request.SetBucket(bucket_name.empty() ? s3_config_.bucket : bucket_name);
const auto outcome = s3_client_->ListObjects(request);
if (outcome.IsSuccess()) {
const auto &object_list = outcome.GetResult().GetContents();
for (auto const &object : object_list) {
api_file file{};
file.accessed_date = utils::get_file_time_now();
std::string object_name = object.GetKey();
object_name =
get_name(*(utils::string::split(object_name, '/', false).end() - 1u), object_name);
file.api_path = utils::path::create_api_path(utils::path::combine(
bucket_name.empty() ? s3_config_.bucket : bucket_name, {object_name}));
file.api_parent = utils::path::get_parent_api_path(file.api_path);
file.changed_date = object.GetLastModified().Millis() * 1000u * 1000u;
file.created_date = object.GetLastModified().Millis() * 1000u * 1000u;
file.encryption_token = get_api_file_token(file.api_path);
if (file.encryption_token.empty()) {
file.file_size = object.GetSize();
} else {
file.file_size =
object.GetSize() -
(utils::divide_with_ceiling(
static_cast<std::uint64_t>(object.GetSize()),
static_cast<std::uint64_t>(
utils::encryption::encrypting_reader::get_encrypted_chunk_size())) *
utils::encryption::encrypting_reader::get_header_size());
}
file.modified_date = object.GetLastModified().Millis() * 1000u * 1000u;
file.recoverable = true;
file.redundancy = 3.0;
list.emplace_back(std::move(file));
}
} else {
const auto &error = outcome.GetError();
event_system::instance().raise<repertory_exception>(__FUNCTION__, error.GetExceptionName() +
"|" + error.GetMessage());
ret = api_error::comm_error;
}
return ret;
}
api_error aws_s3_comm::grab_directory_items(const std::string &api_path,
const meta_provider_callback &meta_provider,
directory_item_list &list) const {
auto ret = api_error::success;
if (s3_config_.bucket.empty() && (api_path == "/")) {
const auto outcome = s3_client_->ListBuckets();
if (outcome.IsSuccess()) {
const auto &bucket_list = outcome.GetResult().GetBuckets();
for (const auto &bucket : bucket_list) {
directory_item di{};
di.api_path = utils::path::create_api_path(utils::path::combine("/", {bucket.GetName()}));
di.api_parent = utils::path::get_parent_api_path(di.api_path);
di.directory = true;
di.size = get_directory_item_count(di.api_path, meta_provider);
meta_provider(di, true);
list.emplace_back(std::move(di));
}
set_cached_directory_items(api_path, list);
} else {
const auto &error = outcome.GetError();
event_system::instance().raise<repertory_exception>(
__FUNCTION__, error.GetExceptionName() + "|" + error.GetMessage());
ret = api_error::comm_error;
}
} else {
std::string bucket_name, object_name;
get_bucket_name_and_object_name(api_path, empty_key, bucket_name, object_name);
Aws::S3::Model::ListObjectsRequest request{};
request.SetBucket(bucket_name);
request.SetDelimiter("/");
request.SetPrefix(object_name.empty() ? object_name : object_name + "/");
const auto outcome = s3_client_->ListObjects(request);
if (outcome.IsSuccess()) {
const auto &object_list = outcome.GetResult().GetContents();
for (auto const &object : object_list) {
directory_item item{};
item.api_path =
utils::path::create_api_path(utils::path::combine(bucket_name, {object.GetKey()}));
item.api_parent = utils::path::get_parent_api_path(item.api_path);
item.directory = false;
item.size = object.GetSize();
meta_provider(item, true);
list.emplace_back(std::move(item));
}
set_cached_directory_items(api_path, list);
} else {
const auto &error = outcome.GetError();
event_system::instance().raise<repertory_exception>(
__FUNCTION__, error.GetExceptionName() + "|" + error.GetMessage());
ret = api_error::comm_error;
}
}
return ret;
}
api_error aws_s3_comm::read_file_bytes(const std::string &api_path, const std::size_t &size,
const std::uint64_t &offset, std::vector<char> &data,
const get_key_callback &get_key,
const get_size_callback &get_size,
const get_token_callback &get_token,
const bool &stop_requested) const {
auto ret = api_error::success;
data.clear();
std::string bucket_name, object_name;
get_bucket_name_and_object_name(api_path, get_key, bucket_name, object_name);
const auto encryption_token = get_token();
const auto data_size = get_size();
if (encryption_token.empty()) {
Aws::S3::Model::GetObjectRequest request{};
request.SetBucket(bucket_name);
request.SetKey(object_name);
request.SetResponseContentType("application/octet-stream");
request.SetRange("bytes=" + utils::string::from_uint64(offset) + "-" +
utils::string::from_uint64(offset + size - 1u));
request.SetContinueRequestHandler(
[&stop_requested](const Aws::Http::HttpRequest *) { return not stop_requested; });
auto outcome = s3_client_->GetObject(request);
if (outcome.IsSuccess()) {
auto result = outcome.GetResultWithOwnership();
const auto len = result.GetContentLength();
data.resize(len);
result.GetBody().read(&data[0], len);
} else {
const auto &error = outcome.GetError();
event_system::instance().raise<repertory_exception>(
__FUNCTION__, error.GetExceptionName() + "|" + error.GetMessage());
ret = api_error::comm_error;
}
} else {
const auto key = utils::encryption::generate_key(encryption_token);
ret = utils::encryption::read_encrypted_range(
{offset, offset + size - 1}, key,
[&](std::vector<char> &ct, const std::uint64_t &start_offset,
const std::uint64_t &end_offset) -> api_error {
return read_file_bytes(
api_path, (end_offset - start_offset + 1u), start_offset, ct, get_key, get_size,
[]() -> std::string { return ""; }, stop_requested);
},
data_size, data);
}
return ret;
}
api_error aws_s3_comm::remove_bucket(const std::string &api_path) {
std::string bucket_name, object_name;
get_bucket_name_and_object_name(api_path, empty_key, bucket_name, object_name);
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(__FUNCTION__, bucket_name, "begin");
}
auto ret = api_error::access_denied;
if (s3_config_.bucket.empty()) {
Aws::S3::Model::DeleteBucketRequest request{};
request.SetBucket(bucket_name);
const auto outcome = s3_client_->DeleteBucket(request);
if (outcome.IsSuccess()) {
remove_cached_directory(api_path);
ret = api_error::success;
} else {
const auto &error = outcome.GetError();
event_system::instance().raise<repertory_exception>(
__FUNCTION__, error.GetExceptionName() + "|" + error.GetMessage());
ret = api_error::comm_error;
}
}
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(__FUNCTION__, bucket_name,
"end|" + std::to_string(static_cast<int>(ret)));
}
return ret;
}
void aws_s3_comm::remove_cached_directory(const std::string &api_path) {
recur_mutex_lock l(cached_directories_mutex_);
cached_directories_.erase(api_path);
}
api_error aws_s3_comm::remove_file(const std::string &api_path, const get_key_callback &get_key) {
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(__FUNCTION__, api_path, "begin");
}
auto ret = api_error::success;
std::string bucket_name, object_name;
get_bucket_name_and_object_name(api_path, get_key, bucket_name, object_name);
Aws::S3::Model::DeleteObjectRequest request{};
request.SetBucket(bucket_name);
request.SetKey(object_name);
const auto outcome = s3_client_->DeleteObject(request);
if (outcome.IsSuccess()) {
remove_cached_directory(utils::path::get_parent_api_path(api_path));
} else {
const auto &error = outcome.GetError();
event_system::instance().raise<repertory_exception>(__FUNCTION__, error.GetExceptionName() +
"|" + error.GetMessage());
ret = api_error::comm_error;
}
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(__FUNCTION__, api_path,
"end|" + std::to_string(std::uint8_t(ret)));
}
return ret;
}
api_error aws_s3_comm::rename_file(const std::string & /*api_path*/,
const std::string & /*new_api_path*/) {
return api_error::not_implemented;
/* if (config_.get_event_level() >= event_level::debug) { */
/* event_system::instance().raise<debug_log>(__FUNCTION__, api_path, "begin"); */
/* } */
/* auto ret = api_error::success; */
/* */
/* std::string bucket_name, object_name; */
/* get_bucket_name_and_object_name(api_path, bucket_name, object_name); */
/* */
/* std::string new_object_name; */
/* get_bucket_name_and_object_name(new_api_path, bucket_name, new_object_name); */
/* */
/* Aws::S3::Model::CopyObjectRequest request{}; */
/* request.SetBucket(bucket_name); */
/* request.SetCopySource(bucket_name + '/' + object_name); */
/* request.SetKey(new_object_name); */
/* */
/* const auto outcome = s3_client_->CopyObject(request); */
/* if (outcome.IsSuccess()) { */
/* ret = remove_file(api_path); */
/* } else { */
/* const auto &error = outcome.GetError(); */
/* event_system::instance().raise<repertory_exception>(__FUNCTION__, error.GetExceptionName()
* +
* "|" + */
/* error.GetMessage()); */
/* ret = api_error::comm_error; */
/* } */
/* */
/* if (config_.get_event_level() >= event_level::debug) { */
/* event_system::instance().raise<debug_log>(__FUNCTION__, api_path, */
/* "end|" + std::to_string(std::uint8_t(ret))); */
/* } */
/* return ret; */
}
void aws_s3_comm::set_cached_directory_items(const std::string &api_path,
directory_item_list list) const {
recur_mutex_lock l(cached_directories_mutex_);
cached_directories_[api_path].items = std::move(list);
cached_directories_[api_path].reset_timeout(
std::chrono::seconds(config_.get_s3_config().cache_timeout_secs));
}
api_error aws_s3_comm::upload_file(const std::string &api_path, const std::string &source_path,
const std::string &encryption_token,
const get_key_callback &get_key, const set_key_callback &set_key,
const bool &stop_requested) {
static const auto no_stop = false;
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(__FUNCTION__, api_path, "begin");
}
auto ret = api_error::success;
std::string bucket_name, object_name;
get_bucket_name_and_object_name(api_path, get_key, bucket_name, object_name);
std::shared_ptr<Aws::IOStream> file_stream;
if (encryption_token.empty()) {
file_stream = Aws::MakeShared<Aws::FStream>(&source_path[0], &source_path[0],
std::ios_base::in | std::ios_base::binary);
} else {
const auto file_name = ([&api_path]() -> std::string {
return *(utils::string::split(api_path, '/', false).end() - 1u);
})();
const auto reader =
utils::encryption::encrypting_reader(file_name, source_path, no_stop, encryption_token, -1);
auto key = get_key();
if (key.empty()) {
key = reader.get_encrypted_file_name();
set_key(key);
auto parts = utils::string::split(object_name, '/', false);
parts[parts.size() - 1u] = key;
object_name = utils::string::join(parts, '/');
}
file_stream = reader.create_iostream();
}
file_stream->seekg(0);
Aws::S3::Model::PutObjectRequest request{};
request.SetBucket(bucket_name);
request.SetKey(object_name);
request.SetBody(file_stream);
request.SetContinueRequestHandler(
[&stop_requested](const Aws::Http::HttpRequest *) { return not stop_requested; });
const auto outcome = s3_client_->PutObject(request);
if (outcome.IsSuccess()) {
remove_cached_directory(utils::path::get_parent_api_path(api_path));
} else {
const auto &error = outcome.GetError();
event_system::instance().raise<repertory_exception>(__FUNCTION__, error.GetExceptionName() +
"|" + error.GetMessage());
ret = api_error::upload_failed;
}
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(__FUNCTION__, api_path,
"end|" + std::to_string(std::uint8_t(ret)));
}
return ret;
}
} // namespace repertory
#endif // REPERTORY_ENABLE_S3

949
src/comm/curl/curl_comm.cpp
View File

@@ -1,849 +1,176 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "comm/curl/curl_comm.hpp"
#include "comm/curl/curl_resolver.hpp"
#include "curl/curl.h"
#include "types/repertory.hpp"
#include "utils/Base64.hpp"
#include "utils/encryption.hpp"
#include "utils/encrypting_reader.hpp"
#include "utils/file_utils.hpp"
#include "utils/path_utils.hpp"
#include "utils/string_utils.hpp"
namespace repertory {
struct curl_setup {
bool allow_timeout = false;
const host_config &hc;
const http_parameters *parameters = nullptr;
http_headers *headers = nullptr;
bool post = false;
raw_write_data *write_data = nullptr;
std::string *write_string = nullptr;
std::unique_ptr<curl_resolver> resolver;
};
const curl_comm::write_callback curl_comm::write_data =
static_cast<curl_comm::write_callback>([](char *buffer, size_t size,
size_t nitems,
void *outstream) -> size_t {
auto &info = *reinterpret_cast<read_write_info *>(outstream);
std::copy(buffer, buffer + (size * nitems),
std::back_inserter(info.data));
return info.stop_requested ? 0 : size * nitems;
});
struct raw_write_data {
std::vector<char> *buffer;
const bool &stop_requested;
};
struct read_data {
const bool *stop_requested = nullptr;
native_file *nf = nullptr;
std::uint64_t offset = 0u;
};
CURL *curl_comm::common_curl_setup(const std::string &path, curl_setup &setup, std::string &url,
std::string &fields) {
return common_curl_setup(utils::create_curl(), path, setup, url, fields);
}
CURL *curl_comm::common_curl_setup(CURL *curl_handle, const std::string &path, curl_setup &setup,
std::string &url, std::string &fields) {
url = construct_url(curl_handle, path, setup.hc);
if (setup.post) {
curl_easy_setopt(curl_handle, CURLOPT_HTTPGET, 0);
curl_easy_setopt(curl_handle, CURLOPT_POST, 1L);
if (setup.parameters && not setup.parameters->empty()) {
for (const auto &param : *setup.parameters) {
if (not fields.empty()) {
fields += "&";
}
if (param.first ==
"new" + app_config::get_provider_path_name(config_.get_provider_type())) {
fields += (param.first + "=" + url_encode(curl_handle, param.second));
} else {
fields += (param.first + "=" + param.second);
}
const curl_comm::write_callback curl_comm::write_headers =
static_cast<curl_comm::write_callback>([](char *buffer, size_t size,
size_t nitems,
void *outstream) -> size_t {
auto &headers = *reinterpret_cast<http_headers *>(outstream);
const auto header = std::string(buffer, size * nitems);
const auto parts = utils::string::split(header, ':');
if (parts.size() > 1u) {
auto data = header.substr(parts[0u].size() + 1u);
utils::string::left_trim(data);
utils::string::right_trim(data, '\r');
utils::string::right_trim(data, '\n');
utils::string::right_trim(data, '\r');
headers[utils::string::to_lower(parts[0u])] = data;
}
curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, &fields[0]);
} else {
curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, "");
return size * nitems;
});
curl_comm::curl_comm(host_config hc)
: host_config_(std::move(hc)), s3_config_(std::nullopt) {}
curl_comm::curl_comm(s3_config s3)
: host_config_(std::nullopt), s3_config_(std::move(s3)) {}
auto curl_comm::construct_url(CURL *curl, const std::string &relative_path,
const host_config &hc) -> std::string {
auto custom_port =
(((hc.protocol == "http") && (hc.api_port == 80U || hc.api_port == 0U)) ||
((hc.protocol == "https") && (hc.api_port == 443U || hc.api_port == 0U)))
? ""
: ":" + std::to_string(hc.api_port);
auto url = hc.protocol + "://" +
utils::string::trim_copy(hc.host_name_or_ip) + custom_port;
static const auto complete_url = [](const std::string &current_path,
const std::string &parent_path,
std::string &final_url) -> std::string & {
final_url += utils::path::create_api_path(current_path);
if (utils::string::ends_with(parent_path, "/")) {
final_url += '/';
}
} else {
curl_easy_setopt(curl_handle, CURLOPT_POST, 0);
curl_easy_setopt(curl_handle, CURLOPT_HTTPGET, 1L);
if (setup.parameters && not setup.parameters->empty()) {
url += "?";
for (const auto &param : *setup.parameters) {
if (url[url.size() - 1] != '?') {
url += "&";
}
url += (param.first + "=" + url_encode(curl_handle, param.second));
}
}
}
if (not setup.hc.agent_string.empty()) {
curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, &setup.hc.agent_string[0]);
}
if (setup.write_data) {
setup.write_data->buffer->clear();
curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, setup.write_data);
curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, write_data_callback_);
} else if (setup.write_string) {
setup.write_string->clear();
curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, setup.write_string);
curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, write_string_callback_);
}
curl_easy_setopt(curl_handle, CURLOPT_URL, url.c_str());
if (setup.allow_timeout) {
curl_easy_setopt(curl_handle, CURLOPT_TIMEOUT_MS, setup.hc.timeout_ms);
}
if (not setup.hc.api_password.empty()) {
curl_easy_setopt(curl_handle, CURLOPT_USERNAME, "");
curl_easy_setopt(curl_handle, CURLOPT_PASSWORD, &setup.hc.api_password[0]);
}
if (setup.headers) {
curl_easy_setopt(curl_handle, CURLOPT_HEADERDATA, setup.headers);
curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, write_header_callback_);
}
std::vector<std::string> items = {"localhost:" + std::to_string(setup.hc.api_port) +
":127.0.0.1"};
setup.resolver = std::make_unique<curl_resolver>(curl_handle, items);
return curl_handle;
}
std::string curl_comm::construct_url(CURL *curl_handle, const std::string &relative_path,
const host_config &hc) {
auto custom_port = (((hc.protocol == "http") && (hc.api_port == 80u)) ||
((hc.protocol == "https") && (hc.api_port == 443u)))
? ""
: ":" + std::to_string(hc.api_port);
auto ret = hc.protocol + "://" + utils::string::trim_copy(hc.host_name_or_ip) + custom_port;
auto path = utils::path::combine("/", {hc.path});
if (relative_path.empty()) {
ret += utils::path::create_api_path(path);
if (utils::string::ends_with(hc.path, "/")) {
ret += '/';
}
} else {
path = utils::path::combine(path, {url_encode(curl_handle, relative_path, true)});
ret += utils::path::create_api_path(path);
if (utils::string::ends_with(relative_path, "/")) {
ret += '/';
}
}
return ret;
}
bool curl_comm::create_auth_session(CURL *&curl_handle, const app_config &config, host_config hc,
std::string &session) {
auto ret = true;
if (not curl_handle) {
curl_handle = utils::create_curl();
}
if (not hc.auth_url.empty() && not hc.auth_user.empty()) {
event_system::instance().raise<comm_auth_begin>(hc.auth_url, hc.auth_user);
ret = false;
session = utils::create_uuid_string();
const auto cookie_path = utils::path::combine(config.get_data_directory(), {session + ".txt"});
http_headers headers{};
auto url = utils::string::right_trim(utils::string::trim(hc.auth_url), '/') + "/api/login";
if (not hc.agent_string.empty()) {
curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, &hc.agent_string[0]);
}
if (not hc.api_password.empty()) {
curl_easy_setopt(curl_handle, CURLOPT_USERNAME, "");
curl_easy_setopt(curl_handle, CURLOPT_PASSWORD, &hc.api_password[0]);
}
struct curl_slist *hs = nullptr;
hs = curl_slist_append(hs, "Content-Type: application/json");
curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, hs);
const auto payload = json({
{"email", hc.auth_user},
{"password", hc.auth_password},
})
.dump(2);
curl_easy_setopt(curl_handle, CURLOPT_COOKIEFILE, cookie_path.c_str());
curl_easy_setopt(curl_handle, CURLOPT_COOKIEJAR, cookie_path.c_str());
curl_easy_setopt(curl_handle, CURLOPT_HEADERDATA, &headers);
curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, write_header_callback_);
curl_easy_setopt(curl_handle, CURLOPT_HTTPGET, 0L);
curl_easy_setopt(curl_handle, CURLOPT_POST, 1L);
curl_easy_setopt(curl_handle, CURLOPT_TIMEOUT_MS, hc.timeout_ms);
curl_easy_setopt(curl_handle, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, payload.c_str());
long code{};
auto res = curl_easy_perform(curl_handle);
curl_easy_getinfo(curl_handle, CURLINFO_HTTP_CODE, &code);
if ((res == CURLE_OK) && ((code >= 200) && (code < 300))) {
auto cookie = headers["set-cookie"];
if (cookie.empty()) {
code = 400;
} else {
cookie = headers["set-cookie"];
if (cookie.empty() || not utils::string::contains(cookie, "skynet-jwt=")) {
code = 401;
} else {
ret = true;
}
}
}
if (ret) {
curl_easy_setopt(curl_handle, CURLOPT_COOKIELIST, "FLUSH");
update_auth_session(utils::reset_curl(curl_handle), config, session);
} else {
curl_easy_cleanup(curl_handle);
release_auth_session(config, hc, session);
}
curl_slist_free_all(hs);
event_system::instance().raise<comm_auth_end>(hc.auth_url, hc.auth_user, res, code);
}
return ret;
}
api_error curl_comm::get_or_post(const host_config &hc, const bool &post, const std::string &path,
const http_parameters &parameters, json &data, json &error,
http_headers *headers, std::function<void(CURL *curl_handle)> cb) {
std::string result;
auto setup = curl_setup{not post, hc, &parameters, headers, post, nullptr, &result, nullptr};
std::string fields;
std::string url;
auto *curl_handle = common_curl_setup(path, setup, url, fields);
if (cb) {
cb(curl_handle);
}
if (config_.get_event_level() >= event_level::verbose) {
if (post) {
event_system::instance().raise<comm_post_begin>(url, fields);
} else {
event_system::instance().raise<comm_get_begin>(url);
}
}
std::unique_ptr<std::chrono::system_clock::time_point> tp(nullptr);
if (config_.get_enable_comm_duration_events()) {
tp = std::make_unique<std::chrono::system_clock::time_point>(std::chrono::system_clock::now());
}
const auto curl_code = curl_easy_perform(curl_handle);
if (curl_code == CURLE_OPERATION_TIMEDOUT) {
event_system::instance().raise<repertory_exception>(__FUNCTION__, "CURL timeout: " + path);
}
long http_code = -1;
curl_easy_getinfo(curl_handle, CURLINFO_RESPONSE_CODE, &http_code);
if (tp != nullptr) {
const auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::system_clock::now() - *tp);
event_system::instance().raise<comm_duration>(url, std::to_string(duration.count()));
}
const auto ret = process_json_response(url, curl_code, http_code, result, data, error);
if (config_.get_event_level() >= event_level::verbose) {
if (post) {
event_system::instance().raise<comm_post_end>(
url, curl_code, http_code, ((ret == api_error::success) ? data.dump(2) : error.dump(2)));
} else {
event_system::instance().raise<comm_get_end>(
url, curl_code, http_code, ((ret == api_error::success) ? data.dump(2) : error.dump(2)));
}
}
curl_easy_cleanup(curl_handle);
return ret;
}
api_error curl_comm::get_range(const host_config &hc, const std::string &path,
const std::uint64_t &data_size, const http_parameters &parameters,
const std::string &encryption_token, std::vector<char> &data,
const http_ranges &ranges, json &error, http_headers *headers,
const bool &stop_requested) {
if (encryption_token.empty()) {
return get_range_unencrypted(hc, path, parameters, data, ranges, error, headers,
stop_requested);
}
if (ranges.empty()) {
return api_error::error;
}
const auto key = utils::encryption::generate_key(encryption_token);
for (const auto &range : ranges) {
const auto result = utils::encryption::read_encrypted_range(
range, key,
[&](std::vector<char> &ct, const std::uint64_t &start_offset,
const std::uint64_t &end_offset) -> api_error {
const auto ret =
get_range_unencrypted(hc, path, parameters, ct, {{start_offset, end_offset}}, error,
headers, stop_requested);
headers = nullptr;
return ret;
},
data_size, data);
if (result != api_error::success) {
return result;
}
}
return api_error::success;
}
api_error curl_comm::get_range_unencrypted(const host_config &hc, const std::string &path,
const http_parameters &parameters,
std::vector<char> &data, const http_ranges &ranges,
json &error, http_headers *headers,
const bool &stop_requested) {
raw_write_data wd = {&data, stop_requested};
auto setup = curl_setup{false, hc, &parameters, headers, false, &wd, nullptr, nullptr};
std::string fields;
std::string url;
auto *curl_handle = common_curl_setup(path, setup, url, fields);
std::string range_list;
if (not ranges.empty()) {
range_list = std::accumulate(
std::next(ranges.begin()), ranges.end(), http_range_to_string(ranges[0]),
[](const auto &l, const auto &r) { return l + ',' + http_range_to_string(r); });
curl_easy_setopt(curl_handle, CURLOPT_RANGE, &range_list[0]);
}
return execute_binary_operation<comm_get_range_begin, comm_get_range_end>(curl_handle, url, data,
error, stop_requested);
}
api_error curl_comm::get_raw(const host_config &hc, const std::string &path,
const http_parameters &parameters, std::vector<char> &data,
json &error, const bool &stop_requested) {
raw_write_data wd = {&data, stop_requested};
auto setup = curl_setup{false, hc, &parameters, nullptr, false, &wd, nullptr, nullptr};
std::string fields;
std::string url;
return execute_binary_operation<comm_get_range_begin, comm_get_range_end>(
common_curl_setup(path, setup, url, fields), url, data, error, stop_requested);
}
std::string curl_comm::http_range_to_string(const http_range &range) {
return std::to_string(range.begin) + '-' + std::to_string(range.end);
}
api_error curl_comm::post_file(const host_config &hc, const std::string &path,
const std::string &source_path, const http_parameters &parameters,
json &data, json &error, const bool &stop_requested) {
auto ret = api_error::os_error;
std::uint64_t file_size{};
if (utils::file::get_file_size(source_path, file_size)) {
std::string result;
auto setup = curl_setup{false, hc, &parameters, nullptr, true, nullptr, &result, nullptr};
std::string fields;
std::string url;
auto *curl_handle = common_curl_setup(path, setup, url, fields);
curl_easy_setopt(curl_handle, CURLOPT_UPLOAD, 1L);
curl_easy_setopt(curl_handle, CURLOPT_INFILESIZE_LARGE, file_size);
native_file::native_file_ptr nf;
native_file::create_or_open(source_path, nf);
if (nf) {
read_data rd = {&stop_requested};
rd.nf = nf.get();
curl_easy_setopt(curl_handle, CURLOPT_READDATA, &rd);
curl_easy_setopt(curl_handle, CURLOPT_READFUNCTION, read_data_callback_);
ret = execute_json_operation<comm_post_file_begin, comm_post_file_end>(
curl_handle, url, result, data, error, stop_requested);
nf->close();
}
}
return ret;
}
api_error curl_comm::post_multipart_file(const host_config &hc, const std::string &path,
const std::string &file_name,
const std::string &source_path,
const std::string &encryption_token, json &data,
json &error, const bool &stop_requested) {
std::string result;
std::uint64_t file_size{};
std::unique_ptr<utils::encryption::encrypting_reader> reader;
if (encryption_token.empty()) {
if (not utils::file::get_file_size(source_path, file_size)) {
return api_error::os_error;
}
} else {
try {
reader = std::make_unique<utils::encryption::encrypting_reader>(
file_name, source_path, stop_requested, encryption_token);
file_size = reader->get_total_size();
} catch (const std::exception &e) {
event_system::instance().raise<repertory_exception>(__FUNCTION__, e.what());
return api_error::error;
}
}
std::string session;
CURL *curl_handle = nullptr;
if (not session_manager_.create_auth_session(curl_handle, config_, hc, session)) {
return api_error::access_denied;
}
if (config_.get_provider_type() == provider_type::skynet) {
curl_easy_cleanup(curl_handle);
const auto fn = reader ? reader->get_encrypted_file_name() : file_name;
const auto fs = reader ? reader->get_total_size() : file_size;
std::string location;
if (tus_upload_create(hc, fn, fs, location)) {
std::string skylink;
if (tus_upload(hc, source_path, fn, fs, location, skylink, stop_requested, reader.get())) {
data["skylink"] = skylink;
return api_error::success;
}
}
return api_error::comm_error;
}
auto setup = curl_setup{false, hc, nullptr, nullptr, true, nullptr, &result, nullptr};
std::string fields;
std::string url;
common_curl_setup(curl_handle, path, setup, url, fields);
auto *curl_mime = curl_mime_init(curl_handle);
auto *mime_part = curl_mime_addpart(curl_mime);
curl_mime_name(mime_part, "file");
auto ret = api_error::success;
if (encryption_token.empty()) {
curl_mime_filename(mime_part, &file_name[0]);
curl_mime_filedata(mime_part, &source_path[0]);
} else {
try {
curl_mime_filename(mime_part, reader->get_encrypted_file_name().c_str());
curl_mime_data_cb(
mime_part, reader->get_total_size(),
static_cast<curl_read_callback>(utils::encryption::encrypting_reader::reader_function),
nullptr, nullptr, reader.get());
} catch (const std::exception &e) {
event_system::instance().raise<repertory_exception>(__FUNCTION__, e.what());
ret = api_error::error;
}
}
if (ret == api_error::success) {
curl_easy_setopt(curl_handle, CURLOPT_MIMEPOST, curl_mime);
ret = execute_json_operation<comm_post_multi_part_file_begin, comm_post_multi_part_file_end>(
curl_handle, url, result, data, error, stop_requested,
(ret == api_error::success) ? CURLE_OK : CURLE_SEND_ERROR);
}
curl_mime_free(curl_mime);
curl_easy_cleanup(curl_handle);
/* if (not session.empty() && (ret == api_error::success)) { */
/* auto pin_hc = hc; */
/* */
/* const auto skylink = data["skylink"].get<std::string>(); */
/* utils::string::replace(pin_hc.path, "/skyfile", "/pin/" + skylink); */
/* */
/* ret = api_error::comm_error; */
/* for (std::uint8_t i = 0u; not stop_requested && (i < 30u) && (ret != api_error::success);
* i++) { */
/* if (i) { */
/* event_system::instance().raise<repertory_exception>( */
/* __FUNCTION__, "RETRY [" + std::to_string(i) + "] Pin failed for file: " +
* file_name); */
/* std::this_thread::sleep_for(1s); */
/* } */
/* */
/* json response; */
/* http_headers headers{}; */
/* ret = get_or_post(pin_hc, true, "", {}, response, error, &headers, [&](CURL *curl_handle)
* { */
/* session_manager_.update_auth_session(curl_handle, config_, hc); */
/* }); */
/* } */
/* } */
session_manager_.release_auth_session(config_, hc, session);
return ret;
}
api_error curl_comm::process_binary_response(const std::string &url, const CURLcode &res,
const long &http_code, std::vector<char> data,
json &error) {
const auto ret = process_response(
url, res, http_code, data.size(),
[&]() -> std::string { return (data.empty() ? "" : std::string(&data[0], data.size())); },
nullptr, error);
if (ret != api_error::success) {
data.clear();
}
return ret;
}
api_error curl_comm::process_json_response(const std::string &url, const CURLcode &res,
const long &http_code, const std::string &result,
json &data, json &error) {
const auto ret = process_response(
url, res, http_code, result.size(), [&]() -> std::string { return result; },
[&]() {
if (result.length()) {
data = json::parse(result.c_str());
}
},
error);
if (ret != api_error::success) {
data.clear();
}
return ret;
}
api_error curl_comm::process_response(const std::string &url, const CURLcode &res,
const long &http_code, const std::size_t &data_size,
const std::function<std::string()> &to_string_convertor,
const std::function<void()> &success_handler,
json &error) const {
auto ret = api_error::success;
auto construct_error = [&]() {
ret = api_error::comm_error;
const auto *curl_string = curl_easy_strerror(res);
std::string error_string(curl_string ? curl_string : "");
error["message"] = error_string + ":" + std::to_string(http_code);
error["url"] = url;
return final_url;
};
if ((res == CURLE_OK) && ((http_code >= 200) && (http_code < 300))) {
if (success_handler) {
success_handler();
}
} else if (data_size == 0u) {
construct_error();
} else {
try {
const auto tmp = json::parse(to_string_convertor().c_str());
if (tmp.find("message") != tmp.end()) {
ret = api_error::comm_error;
error = tmp;
error["url"] = url;
} else {
construct_error();
}
} catch (...) {
construct_error();
}
}
return ret;
auto path = utils::path::combine("/", {hc.path});
return relative_path.empty()
? complete_url(path, hc.path, url)
: complete_url(utils::path::combine(
path, {url_encode(curl, relative_path, true)}),
relative_path, url);
}
void curl_comm::release_auth_session(const app_config &config, host_config hc,
const std::string &session) {
if (not hc.auth_url.empty() && not hc.auth_user.empty()) {
event_system::instance().raise<comm_auth_logout_begin>(hc.auth_url, hc.auth_user);
const auto cookie_path = utils::path::combine(config.get_data_directory(), {session + ".txt"});
const auto url =
utils::string::right_trim(utils::string::trim(hc.auth_url), '/') + "/api/logout";
auto curl_comm::create_host_config(const s3_config &config,
bool use_s3_path_style) -> host_config {
host_config hc{};
hc.api_password = config.secret_key;
hc.api_user = config.access_key;
auto *curl_handle = utils::create_curl();
if (not hc.agent_string.empty()) {
curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, &hc.agent_string[0]);
auto pos = config.url.find(':');
hc.host_name_or_ip = config.url.substr(pos + 3U);
if (config.use_region_in_url && not config.region.empty()) {
auto parts = utils::string::split(hc.host_name_or_ip, '.', false);
if (parts.size() > 1U) {
parts.insert(parts.begin() + 1U, config.region);
hc.host_name_or_ip = utils::string::join(parts, '.');
}
if (not hc.api_password.empty()) {
curl_easy_setopt(curl_handle, CURLOPT_USERNAME, "");
curl_easy_setopt(curl_handle, CURLOPT_PASSWORD, &hc.api_password[0]);
}
curl_easy_setopt(curl_handle, CURLOPT_COOKIEFILE, cookie_path.c_str());
curl_easy_setopt(curl_handle, CURLOPT_COOKIEJAR, cookie_path.c_str());
curl_easy_setopt(curl_handle, CURLOPT_HTTPGET, 0L);
curl_easy_setopt(curl_handle, CURLOPT_POST, 1L);
curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, "");
curl_easy_setopt(curl_handle, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl_handle, CURLOPT_TIMEOUT_MS, hc.timeout_ms);
curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, write_null_callback_);
const auto curl_code = curl_easy_perform(curl_handle);
long http_code = -1;
curl_easy_getinfo(curl_handle, CURLINFO_HTTP_CODE, &http_code);
curl_easy_cleanup(curl_handle);
event_system::instance().raise<comm_auth_logout_end>(hc.auth_url, hc.auth_user, curl_code,
http_code);
utils::file::delete_file(cookie_path);
}
if (not use_s3_path_style) {
hc.host_name_or_ip = config.bucket + '.' + hc.host_name_or_ip;
}
hc.protocol = config.url.substr(0U, pos);
if (use_s3_path_style) {
hc.path = '/' + config.bucket;
}
return hc;
}
bool curl_comm::tus_upload(host_config hc, const std::string &source_path,
const std::string &file_name, std::uint64_t file_size,
const std::string &location, std::string &skylink,
const bool &stop_requested,
utils::encryption::encrypting_reader *reader) {
static const constexpr std::uint64_t max_skynet_file_size = (1ull << 22ull) * 10ull;
auto tus_hc = hc;
utils::string::replace(tus_hc.path, "/skyfile", "/tus");
auto ret = true;
std::uint64_t offset = 0u;
native_file::native_file_ptr nf;
while (ret && file_size) {
const auto chunk_size = std::min(max_skynet_file_size, file_size);
auto *curl_handle = utils::create_curl();
curl_easy_setopt(curl_handle, CURLOPT_CUSTOMREQUEST, "PATCH");
curl_easy_setopt(curl_handle, CURLOPT_HTTPGET, 0);
curl_easy_setopt(curl_handle, CURLOPT_NOBODY, 0);
const auto upload_offset = "Upload-Offset: " + std::to_string(offset);
const auto content_length = "Content-Length: " + std::to_string(chunk_size);
struct curl_slist *hs = nullptr;
hs = curl_slist_append(hs, "tus-resumable: 1.0.0");
hs = curl_slist_append(hs, "Content-Type: application/offset+octet-stream");
hs = curl_slist_append(hs, content_length.c_str());
hs = curl_slist_append(hs, upload_offset.c_str());
curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, hs);
curl_easy_setopt(curl_handle, CURLOPT_INFILESIZE_LARGE, chunk_size);
curl_easy_setopt(curl_handle, CURLOPT_UPLOAD, 1L);
session_manager_.update_auth_session(curl_handle, config_, hc);
read_data rd = {&stop_requested};
if (reader) {
curl_easy_setopt(curl_handle, CURLOPT_READDATA, reader);
curl_easy_setopt(
curl_handle, CURLOPT_READFUNCTION,
static_cast<curl_read_callback>(utils::encryption::encrypting_reader::reader_function));
} else {
if (not nf) {
native_file::create_or_open(source_path, nf);
}
if (nf) {
rd.nf = nf.get();
rd.offset = offset;
curl_easy_setopt(curl_handle, CURLOPT_READDATA, &rd);
curl_easy_setopt(curl_handle, CURLOPT_READFUNCTION, read_data_callback_);
} else {
ret = false;
}
}
if (ret) {
event_system::instance().raise<comm_tus_upload_begin>(file_name, location, file_size, offset);
curl_easy_setopt(curl_handle, CURLOPT_URL, location.c_str());
const auto curl_code = curl_easy_perform(curl_handle);
long http_code = -1;
curl_easy_getinfo(curl_handle, CURLINFO_HTTP_CODE, &http_code);
event_system::instance().raise<comm_tus_upload_end>(file_name, location, file_size, offset,
curl_code, http_code);
if ((ret = ((curl_code == CURLE_OK) && (http_code >= 200) && (http_code < 300)))) {
file_size -= chunk_size;
offset += chunk_size;
}
}
curl_easy_cleanup(curl_handle);
curl_slist_free_all(hs);
}
if (nf) {
nf->close();
}
if (ret) {
auto *curl_handle = utils::create_curl();
http_headers headers{};
curl_easy_setopt(curl_handle, CURLOPT_CUSTOMREQUEST, "HEAD");
curl_easy_setopt(curl_handle, CURLOPT_NOBODY, 1L);
curl_easy_setopt(curl_handle, CURLOPT_HEADERDATA, &headers);
curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, write_header_callback_);
session_manager_.update_auth_session(curl_handle, config_, hc);
struct curl_slist *hs = nullptr;
hs = curl_slist_append(hs, "tus-resumable: 1.0.0");
curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, hs);
curl_easy_setopt(curl_handle, CURLOPT_URL, location.c_str());
const auto res = curl_easy_perform(curl_handle);
long http_code = -1;
curl_easy_getinfo(curl_handle, CURLINFO_HTTP_CODE, &http_code);
ret = ((res == CURLE_OK) && (http_code >= 200) && (http_code < 300));
skylink = headers["skynet-skylink"];
curl_easy_cleanup(curl_handle);
curl_slist_free_all(hs);
}
return ret;
void curl_comm::enable_s3_path_style(bool enable) {
use_s3_path_style_ = enable;
}
bool curl_comm::tus_upload_create(host_config hc, const std::string &fileName,
const std::uint64_t &fileSize, std::string &location) {
auto tus_hc = hc;
utils::string::replace(tus_hc.path, "/skyfile", "/tus");
auto *curl_handle = utils::create_curl();
const auto url = construct_url(curl_handle, "", tus_hc);
event_system::instance().raise<comm_tus_upload_create_begin>(fileName, url);
http_headers headers{};
curl_easy_setopt(curl_handle, CURLOPT_HTTPGET, 0);
curl_easy_setopt(curl_handle, CURLOPT_POST, 1L);
curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, "");
curl_easy_setopt(curl_handle, CURLOPT_HEADERDATA, &headers);
curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, write_header_callback_);
const auto upload_length = "upload-length: " + utils::string::from_uint64(fileSize);
const auto upload_metadata = "upload-metadata: filename " + macaron::Base64::Encode(fileName) +
",filetype " + macaron::Base64::Encode("application/octet-stream");
struct curl_slist *hs = nullptr;
hs = curl_slist_append(hs, "tus-resumable: 1.0.0");
hs = curl_slist_append(hs, "Content-Length: 0");
hs = curl_slist_append(hs, upload_length.c_str());
hs = curl_slist_append(hs, upload_metadata.c_str());
curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, hs);
session_manager_.update_auth_session(curl_handle, config_, hc);
curl_easy_setopt(curl_handle, CURLOPT_URL, url.c_str());
const auto res = curl_easy_perform(curl_handle);
long http_code = -1;
curl_easy_getinfo(curl_handle, CURLINFO_HTTP_CODE, &http_code);
const auto ret = (res == CURLE_OK && http_code == 201);
if (ret) {
location = headers["location"];
}
curl_easy_cleanup(curl_handle);
curl_slist_free_all(hs);
event_system::instance().raise<comm_tus_upload_create_end>(fileName, url, res, http_code);
return ret;
auto curl_comm::make_request(const curl::requests::http_delete &del,
long &response_code,
stop_type &stop_requested) const -> bool {
return make_request(
s3_config_.has_value()
? create_host_config(s3_config_.value(), use_s3_path_style_)
: host_config_.value(),
del, response_code, stop_requested);
}
void curl_comm::update_auth_session(CURL *curl_handle, const app_config &config,
const std::string &session) {
const auto cookie_path = utils::path::combine(config.get_data_directory(), {session + ".txt"});
curl_easy_setopt(curl_handle, CURLOPT_COOKIEFILE, cookie_path.c_str());
curl_easy_setopt(curl_handle, CURLOPT_COOKIEJAR, cookie_path.c_str());
auto curl_comm::make_request(const curl::requests::http_get &get,
long &response_code,
stop_type &stop_requested) const -> bool {
return make_request(
s3_config_.has_value()
? create_host_config(s3_config_.value(), use_s3_path_style_)
: host_config_.value(),
get, response_code, stop_requested);
}
std::string curl_comm::url_encode(CURL *curl_handle, const std::string &data,
const bool &allow_slash) {
auto *value = curl_easy_escape(curl_handle, data.c_str(), 0);
auto curl_comm::make_request(const curl::requests::http_head &head,
long &response_code,
stop_type &stop_requested) const -> bool {
return make_request(
s3_config_.has_value()
? create_host_config(s3_config_.value(), use_s3_path_style_)
: host_config_.value(),
head, response_code, stop_requested);
}
auto curl_comm::make_request(const curl::requests::http_put_file &put_file,
long &response_code,
stop_type &stop_requested) const -> bool {
return make_request(
s3_config_.has_value()
? create_host_config(s3_config_.value(), use_s3_path_style_)
: host_config_.value(),
put_file, response_code, stop_requested);
}
auto curl_comm::url_encode(CURL *curl, const std::string &data,
bool allow_slash) -> std::string {
auto *value =
curl_easy_escape(curl, data.c_str(), static_cast<int>(data.size()));
std::string ret = value;
curl_free(value);
if (allow_slash) {
utils::string::replace(ret, "%2F", "/");
}
return ret;
}
curl_comm::curl_read_callback curl_comm::read_data_callback_ =
static_cast<curl_comm::curl_read_callback>(
[](char *buffer, size_t size, size_t nitems, void *instream) -> size_t {
auto *rd = reinterpret_cast<read_data *>(instream);
std::size_t bytes_read{};
const auto ret = rd->nf->read_bytes(buffer, size * nitems, rd->offset, bytes_read);
if (ret) {
rd->offset += bytes_read;
}
return ret && not *rd->stop_requested ? bytes_read : CURL_READFUNC_ABORT;
});
curl_comm::curl_write_callback curl_comm::write_data_callback_ =
static_cast<curl_comm::curl_write_callback>(
[](char *buffer, size_t size, size_t nitems, void *outstream) -> size_t {
auto *wd = reinterpret_cast<raw_write_data *>(outstream);
std::copy(buffer, buffer + (size * nitems), std::back_inserter(*wd->buffer));
return wd->stop_requested ? 0 : size * nitems;
});
curl_comm::curl_write_callback curl_comm::write_header_callback_ =
static_cast<curl_comm::curl_write_callback>(
[](char *buffer, size_t size, size_t nitems, void *outstream) -> size_t {
auto &headers = *reinterpret_cast<http_headers *>(outstream);
const auto header = std::string(buffer, size * nitems);
const auto parts = utils::string::split(header, ':');
if (parts.size() > 1u) {
auto data = header.substr(parts[0u].size() + 1u);
utils::string::left_trim(data);
utils::string::right_trim(data, '\r');
utils::string::right_trim(data, '\n');
utils::string::right_trim(data, '\r');
headers[utils::string::to_lower(parts[0u])] = data;
}
return size * nitems;
});
curl_comm::curl_write_callback curl_comm::write_null_callback_ =
static_cast<curl_comm::curl_write_callback>(
[](char *, size_t size, size_t nitems, void *) -> size_t { return size * nitems; });
curl_comm::curl_write_callback curl_comm::write_string_callback_ =
static_cast<curl_comm::curl_write_callback>(
[](char *buffer, size_t size, size_t nitems, void *outstream) -> size_t {
(*reinterpret_cast<std::string *>(outstream)) += std::string(buffer, size * nitems);
return size * nitems;
});
} // namespace repertory

43
src/comm/curl/curl_resolver.cpp
View File

@@ -1,43 +0,0 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#include "comm/curl/curl_resolver.hpp"
namespace repertory {
curl_resolver::curl_resolver(CURL *handle, std::vector<std::string> items, const bool &ignore_root)
: items_(std::move(items)) {
#ifndef _WIN32
if (ignore_root && (getuid() == 0)) {
items_.clear();
}
#endif
for (const auto &item : items_) {
host_list_ = curl_slist_append(host_list_, &item[0u]);
}
if (host_list_) {
curl_easy_setopt(handle, CURLOPT_RESOLVE, host_list_);
}
}
curl_resolver::~curl_resolver() {
if (host_list_) {
curl_slist_free_all(host_list_);
}
}
} // namespace repertory

40
src/comm/curl/multi_request.cpp
View File

@@ -1,27 +1,33 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "comm/curl/multi_request.hpp"
#include "utils/utils.hpp"
namespace repertory {
multi_request::multi_request(CURL *curl_handle, const bool &stop_requested)
: curl_handle_(curl_handle), stop_requested_(stop_requested), multi_handle_(curl_multi_init()) {
multi_request::multi_request(CURL *curl_handle, stop_type &stop_requested)
: curl_handle_(curl_handle),
stop_requested_(stop_requested),
multi_handle_(curl_multi_init()) {
curl_multi_add_handle(multi_handle_, curl_handle);
}
@@ -48,9 +54,11 @@ void multi_request::get_result(CURLcode &curl_code, long &http_code) {
if (not stop_requested_) {
int remaining_messages = 0;
auto *multi_result = curl_multi_info_read(multi_handle_, &remaining_messages);
auto *multi_result =
curl_multi_info_read(multi_handle_, &remaining_messages);
if (multi_result && (multi_result->msg == CURLMSG_DONE)) {
curl_easy_getinfo(multi_result->easy_handle, CURLINFO_RESPONSE_CODE, &http_code);
curl_easy_getinfo(multi_result->easy_handle, CURLINFO_RESPONSE_CODE,
&http_code);
curl_code = multi_result->data.result;
}
}

80
src/comm/curl/requests/http_put_file.cpp Normal file
View File

@@ -0,0 +1,80 @@
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "comm/curl/requests/http_put_file.hpp"
#include "utils/string_utils.hpp"
namespace repertory::curl::requests {
auto http_put_file::get_path() const -> std::string {
if (reader) {
auto updated_path = path;
return utils::string::replace(updated_path, file_name,
reader->get_encrypted_file_name());
}
return http_request_base::get_path();
}
auto http_put_file::set_method(CURL *curl, stop_type &stop_requested) const
-> bool {
curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "PUT");
if (not source_path.empty()) {
curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L);
if (not encryption_token.value_or("").empty()) {
if (not reader) {
reader = std::make_shared<utils::encryption::encrypting_reader>(
file_name, source_path, stop_requested, encryption_token.value());
}
curl_easy_setopt(curl, CURLOPT_READDATA, reader.get());
curl_easy_setopt(
curl, CURLOPT_READFUNCTION,
static_cast<curl_read_callback>(
utils::encryption::encrypting_reader::reader_function));
curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE,
reader->get_total_size());
return true;
}
read_info = std::make_shared<read_file_info>(read_file_info{
stop_requested,
});
if (native_file::open(source_path, read_info->nf) != api_error::success) {
return false;
}
read_info->nf->set_auto_close(true);
std::uint64_t file_size{};
if (not read_info->nf->get_file_size(file_size)) {
return false;
}
curl_easy_setopt(curl, CURLOPT_READDATA, read_info.get());
curl_easy_setopt(curl, CURLOPT_READFUNCTION, read_file_data);
curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, file_size);
}
return true;
}
} // namespace repertory::curl::requests

73
src/comm/curl/session_manager.cpp
View File

@@ -1,73 +0,0 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#include "comm/curl/session_manager.hpp"
#include "app_config.hpp"
#include "comm/curl/curl_comm.hpp"
#include "utils/utils.hpp"
namespace repertory {
bool session_manager::create_auth_session(CURL *&curl_handle, const app_config &config,
host_config hc, std::string &session) {
auto ret = true;
if (not curl_handle) {
curl_handle = utils::create_curl();
}
if (not hc.auth_url.empty() && not hc.auth_user.empty()) {
mutex_lock l(session_mutex_);
if (session_.empty()) {
if ((ret = curl_comm::create_auth_session(curl_handle, config, hc, session))) {
session_ = session;
}
} else {
session = session_;
curl_comm::update_auth_session(curl_handle, config, session_);
}
if (ret) {
session_count_++;
}
}
return ret;
}
void session_manager::release_auth_session(const app_config &config, host_config hc,
const std::string &session) {
if (not hc.auth_url.empty() && not hc.auth_user.empty()) {
mutex_lock l(session_mutex_);
if (not session_.empty() && (session == session_) && session_count_) {
if (not --session_count_) {
curl_comm::release_auth_session(config, hc, session_);
session_.clear();
}
}
}
}
void session_manager::update_auth_session(CURL *curl_handle, const app_config &config,
const host_config &hc) {
mutex_lock l(session_mutex_);
if (hc.auth_url.empty() || hc.auth_user.empty() || session_.empty()) {
return;
}
curl_comm::update_auth_session(curl_handle, config, session_);
}
} // namespace repertory

78
src/comm/packet/client_pool.cpp
View File

@@ -1,29 +1,35 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "comm/packet/client_pool.hpp"
#include "events/events.hpp"
#include "events/event_system.hpp"
#include "events/events.hpp"
#include "utils/error_utils.hpp"
namespace repertory {
void client_pool::pool::execute(const std::uint64_t &thread_id, const worker_callback &worker,
const worker_complete_callback &worker_complete) {
const auto index = static_cast<std::size_t>(thread_id % pool_queues_.size());
void client_pool::pool::execute(
std::uint64_t thread_id, const worker_callback &worker,
const worker_complete_callback &worker_complete) {
const auto index = thread_id % pool_queues_.size();
auto wi = std::make_shared<work_item>(worker, worker_complete);
auto &pool_queue = pool_queues_[index];
@@ -33,14 +39,15 @@ void client_pool::pool::execute(const std::uint64_t &thread_id, const worker_cal
queue_lock.unlock();
}
client_pool::pool::pool(const std::uint8_t &pool_size) {
client_pool::pool::pool(std::uint8_t pool_size) {
event_system::instance().raise<service_started>("client_pool");
thread_index_ = 0u;
for (std::uint8_t i = 0u; i < pool_size; i++) {
pool_queues_.emplace_back(std::make_unique<work_queue>());
}
for (std::size_t i = 0u; i < pool_queues_.size(); i++) {
pool_threads_.emplace_back(std::thread([this]() {
pool_threads_.emplace_back([this]() {
const auto thread_index = thread_index_++;
auto &pool_queue = pool_queues_[thread_index];
@@ -58,18 +65,18 @@ client_pool::pool::pool(const std::uint8_t &pool_size) {
}
while (not queue.empty()) {
auto workItem = queue.front();
auto item = queue.front();
queue.pop_front();
queue_notify.notify_all();
queue_lock.unlock();
try {
const auto result = workItem->work();
workItem->work_complete(result);
const auto result = item->work();
item->work_complete(result);
} catch (const std::exception &e) {
workItem->work_complete(utils::translate_api_error(api_error::error));
event_system::instance().raise<repertory_exception>(__FUNCTION__,
e.what() ? e.what() : "unknown");
item->work_complete(utils::from_api_error(api_error::error));
utils::error::raise_error(__FUNCTION__, e,
"exception occurred in work item");
}
queue_lock.lock();
@@ -86,14 +93,14 @@ client_pool::pool::pool(const std::uint8_t &pool_size) {
queue_notify.notify_all();
queue_lock.unlock();
wi->work_complete(utils::translate_api_error(api_error::download_stopped));
wi->work_complete(utils::from_api_error(api_error::download_stopped));
queue_lock.lock();
}
queue_notify.notify_all();
queue_lock.unlock();
}));
});
}
}
@@ -113,7 +120,7 @@ void client_pool::pool::shutdown() {
pool_threads_.clear();
}
void client_pool::execute(const std::string &client_id, const std::uint64_t &thread_id,
void client_pool::execute(const std::string &client_id, std::uint64_t thread_id,
const worker_callback &worker,
const worker_complete_callback &worker_complete) {
unique_mutex_lock pool_lock(pool_mutex_);
@@ -136,15 +143,18 @@ void client_pool::remove_client(const std::string &client_id) {
}
void client_pool::shutdown() {
unique_mutex_lock pool_lock(pool_mutex_);
if (not shutdown_) {
shutdown_ = true;
event_system::instance().raise<service_shutdown>("client_pool");
for (auto &kv : pool_lookup_) {
kv.second->shutdown();
event_system::instance().raise<service_shutdown_begin>("client_pool");
unique_mutex_lock pool_lock(pool_mutex_);
if (not shutdown_) {
shutdown_ = true;
for (auto &kv : pool_lookup_) {
kv.second->shutdown();
}
pool_lookup_.clear();
}
pool_lookup_.clear();
pool_lock.unlock();
event_system::instance().raise<service_shutdown_end>("client_pool");
}
pool_lock.unlock();
}
} // namespace repertory

143
src/comm/packet/packet.cpp
View File

@@ -1,27 +1,32 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "comm/packet/packet.hpp"
#include "events/events.hpp"
#include "events/event_system.hpp"
#include "events/events.hpp"
#include "types/remote.hpp"
#include "types/repertory.hpp"
#include "utils/encryption.hpp"
#include "utils/error_utils.hpp"
#include "utils/utils.hpp"
namespace repertory {
@@ -30,46 +35,47 @@ void packet::clear() {
decode_offset_ = 0u;
}
packet::error_type packet::decode(std::string &data) {
auto packet::decode(std::string &data) -> packet::error_type {
const auto *str = &buffer_[decode_offset_];
const auto length = strnlen(str, buffer_.size() - decode_offset_);
data = std::string(str, length);
decode_offset_ += (length + 1);
return utils::translate_api_error(api_error::success);
return utils::from_api_error(api_error::success);
}
packet::error_type packet::decode(std::wstring &data) {
auto packet::decode(std::wstring &data) -> packet::error_type {
std::string utf8_string;
const auto ret = decode(utf8_string);
if (ret == 0) {
data = utils::string::from_utf8(utf8_string);
}
return utils::translate_api_error(api_error::success);
return utils::from_api_error(api_error::success);
}
packet::error_type packet::decode(void *&ptr) {
auto packet::decode(void *&ptr) -> packet::error_type {
return decode(reinterpret_cast<std::uint64_t &>(ptr));
}
packet::error_type packet::decode(void *buffer, const size_t &size) {
auto packet::decode(void *buffer, std::size_t size) -> packet::error_type {
if (size) {
const auto read_size = utils::calculate_read_size(buffer_.size(), size, decode_offset_);
const auto read_size =
utils::calculate_read_size(buffer_.size(), size, decode_offset_);
if (read_size == size) {
memcpy(buffer, &buffer_[decode_offset_], size);
decode_offset_ += size;
return utils::translate_api_error(api_error::success);
return utils::from_api_error(api_error::success);
}
return ((decode_offset_ + size) > buffer_.size())
? utils::translate_api_error(api_error::buffer_overflow)
: utils::translate_api_error(api_error::buffer_too_small);
? utils::from_api_error(api_error::buffer_overflow)
: utils::from_api_error(api_error::buffer_too_small);
}
return utils::translate_api_error(api_error::success);
return utils::from_api_error(api_error::success);
}
packet::error_type packet::decode(std::int8_t &i) {
auto packet::decode(std::int8_t &i) -> packet::error_type {
const auto ret = decode(&i, sizeof(i));
if (ret == 0) {
boost::endian::big_to_native_inplace(i);
@@ -77,7 +83,7 @@ packet::error_type packet::decode(std::int8_t &i) {
return ret;
}
packet::error_type packet::decode(std::uint8_t &i) {
auto packet::decode(std::uint8_t &i) -> packet::error_type {
const auto ret = decode(&i, sizeof(i));
if (ret == 0) {
boost::endian::big_to_native_inplace(i);
@@ -85,7 +91,7 @@ packet::error_type packet::decode(std::uint8_t &i) {
return ret;
}
packet::error_type packet::decode(std::int16_t &i) {
auto packet::decode(std::int16_t &i) -> packet::error_type {
const auto ret = decode(&i, sizeof(i));
if (ret == 0) {
boost::endian::big_to_native_inplace(i);
@@ -93,7 +99,7 @@ packet::error_type packet::decode(std::int16_t &i) {
return ret;
}
packet::error_type packet::decode(std::uint16_t &i) {
auto packet::decode(std::uint16_t &i) -> packet::error_type {
const auto ret = decode(&i, sizeof(i));
if (ret == 0) {
boost::endian::big_to_native_inplace(i);
@@ -101,7 +107,7 @@ packet::error_type packet::decode(std::uint16_t &i) {
return ret;
}
packet::error_type packet::decode(std::int32_t &i) {
auto packet::decode(std::int32_t &i) -> packet::error_type {
const auto ret = decode(&i, sizeof(i));
if (ret == 0) {
boost::endian::big_to_native_inplace(i);
@@ -109,7 +115,7 @@ packet::error_type packet::decode(std::int32_t &i) {
return ret;
}
packet::error_type packet::decode(std::uint32_t &i) {
auto packet::decode(std::uint32_t &i) -> packet::error_type {
const auto ret = decode(&i, sizeof(i));
if (ret == 0) {
boost::endian::big_to_native_inplace(i);
@@ -117,7 +123,7 @@ packet::error_type packet::decode(std::uint32_t &i) {
return ret;
}
packet::error_type packet::decode(std::int64_t &i) {
auto packet::decode(std::int64_t &i) -> packet::error_type {
const auto ret = decode(&i, sizeof(i));
if (ret == 0) {
boost::endian::big_to_native_inplace(i);
@@ -125,7 +131,7 @@ packet::error_type packet::decode(std::int64_t &i) {
return ret;
}
packet::error_type packet::decode(std::uint64_t &i) {
auto packet::decode(std::uint64_t &i) -> packet::error_type {
const auto ret = decode(&i, sizeof(i));
if (ret == 0) {
boost::endian::big_to_native_inplace(i);
@@ -133,7 +139,7 @@ packet::error_type packet::decode(std::uint64_t &i) {
return ret;
}
packet::error_type packet::decode(remote::setattr_x &i) {
auto packet::decode(remote::setattr_x &i) -> packet::error_type {
const auto ret = decode(&i, sizeof(i));
if (ret == 0) {
boost::endian::big_to_native_inplace(i.acctime);
@@ -152,7 +158,7 @@ packet::error_type packet::decode(remote::setattr_x &i) {
return ret;
}
packet::error_type packet::decode(remote::stat &i) {
auto packet::decode(remote::stat &i) -> packet::error_type {
const auto ret = decode(&i, sizeof(i));
if (ret == 0) {
boost::endian::big_to_native_inplace(i.st_mode);
@@ -172,7 +178,7 @@ packet::error_type packet::decode(remote::stat &i) {
return ret;
}
packet::error_type packet::decode(remote::statfs &i) {
auto packet::decode(remote::statfs &i) -> packet::error_type {
const auto ret = decode(&i, sizeof(i));
if (ret == 0) {
boost::endian::big_to_native_inplace(i.f_bavail);
@@ -185,7 +191,7 @@ packet::error_type packet::decode(remote::statfs &i) {
return ret;
}
packet::error_type packet::decode(remote::statfs_x &i) {
auto packet::decode(remote::statfs_x &i) -> packet::error_type {
auto ret = decode(*dynamic_cast<remote::statfs *>(&i));
if (ret == 0) {
ret = decode(&i.f_mntfromname[0], 1024);
@@ -193,7 +199,7 @@ packet::error_type packet::decode(remote::statfs_x &i) {
return ret;
}
packet::error_type packet::decode(remote::file_info &i) {
auto packet::decode(remote::file_info &i) -> packet::error_type {
const auto ret = decode(&i, sizeof(i));
if (ret == 0) {
boost::endian::big_to_native_inplace(i.AllocationSize);
@@ -212,15 +218,14 @@ packet::error_type packet::decode(remote::file_info &i) {
return ret;
}
int packet::decode_json(packet &response, json &json_data) {
auto packet::decode_json(packet &response, json &json_data) -> int {
int ret = 0;
std::string data;
if ((ret = response.decode(data)) == 0) {
try {
json_data = json::parse(data);
} catch (const std::exception &e) {
event_system::instance().raise<repertory_exception>(
__FUNCTION__, e.what() ? e.what() : "Failed to parse JSON string");
utils::error::raise_error(__FUNCTION__, e, "failed to parse json string");
ret = -EIO;
}
}
@@ -228,25 +233,26 @@ int packet::decode_json(packet &response, json &json_data) {
return ret;
}
packet::error_type packet::decrypt(const std::string &token) {
auto ret = utils::translate_api_error(api_error::success);
auto packet::decrypt(const std::string &token) -> packet::error_type {
auto ret = utils::from_api_error(api_error::success);
try {
std::vector<char> result;
data_buffer result;
if (not utils::encryption::decrypt_data(token, &buffer_[decode_offset_],
buffer_.size() - decode_offset_, result)) {
throw std::runtime_error("Decryption failed");
buffer_.size() - decode_offset_,
result)) {
throw std::runtime_error("decryption failed");
}
buffer_ = std::move(result);
decode_offset_ = 0;
} catch (const std::exception &e) {
event_system::instance().raise<repertory_exception>(__FUNCTION__, e.what());
ret = utils::translate_api_error(api_error::error);
utils::error::raise_error(__FUNCTION__, e, "exception occurred");
ret = utils::from_api_error(api_error::error);
}
return ret;
}
void packet::encode(const void *buffer, const std::size_t &size, bool should_reserve) {
void packet::encode(const void *buffer, std::size_t size, bool should_reserve) {
if (size) {
if (should_reserve) {
buffer_.reserve(buffer_.size() + size);
@@ -263,11 +269,17 @@ void packet::encode(const std::string &str) {
buffer_.emplace_back(0);
}
void packet::encode(wchar_t *str) { encode(utils::string::to_utf8(str ? str : L"")); }
void packet::encode(wchar_t *str) {
encode(utils::string::to_utf8(str ? str : L""));
}
void packet::encode(const wchar_t *str) { encode(utils::string::to_utf8(str ? str : L"")); }
void packet::encode(const wchar_t *str) {
encode(utils::string::to_utf8(str ? str : L""));
}
void packet::encode(const std::wstring &str) { encode(utils::string::to_utf8(str)); }
void packet::encode(const std::wstring &str) {
encode(utils::string::to_utf8(str));
}
void packet::encode(std::int8_t i) {
boost::endian::native_to_big_inplace(i);
@@ -375,7 +387,8 @@ void packet::encode(remote::file_info i) {
encode(&i, sizeof(i), true);
}
void packet::encode_top(const void *buffer, const std::size_t &size, bool should_reserve) {
void packet::encode_top(const void *buffer, std::size_t size,
bool should_reserve) {
if (size) {
if (should_reserve) {
buffer_.reserve(buffer_.size() + size);
@@ -386,13 +399,15 @@ void packet::encode_top(const void *buffer, const std::size_t &size, bool should
}
void packet::encode_top(const std::string &str) {
const auto len = strnlen(&str[0], str.size());
buffer_.reserve(len + 1 + buffer_.size());
const auto len = strnlen(str.c_str(), str.size());
buffer_.reserve(len + 1U + buffer_.size());
encode_top(&str[0], len, false);
buffer_.insert(buffer_.begin() + len, 0);
buffer_.insert(buffer_.begin() + static_cast<std::int32_t>(len), 0);
}
void packet::encode_top(const std::wstring &str) { encode_top(utils::string::to_utf8(str)); }
void packet::encode_top(const std::wstring &str) {
encode_top(utils::string::to_utf8(str));
}
void packet::encode_top(std::int8_t i) {
boost::endian::native_to_big_inplace(i);
@@ -502,22 +517,22 @@ void packet::encode_top(remote::file_info i) {
void packet::encrypt(const std::string &token) {
try {
std::vector<char> result;
data_buffer result;
utils::encryption::encrypt_data(token, buffer_, result);
buffer_ = std::move(result);
encode_top(static_cast<std::uint32_t>(buffer_.size()));
} catch (const std::exception &e) {
event_system::instance().raise<repertory_exception>(__FUNCTION__, e.what());
utils::error::raise_error(__FUNCTION__, e, "exception occurred");
}
}
void packet::transfer_into(std::vector<char> &buffer) {
void packet::transfer_into(data_buffer &buffer) {
buffer = std::move(buffer_);
buffer_ = std::vector<char>();
buffer_ = data_buffer();
decode_offset_ = 0;
}
packet &packet::operator=(const std::vector<char> &buffer) noexcept {
auto packet::operator=(const data_buffer &buffer) noexcept -> packet & {
if (&buffer_ != &buffer) {
buffer_ = buffer;
decode_offset_ = 0;
@@ -526,7 +541,7 @@ packet &packet::operator=(const std::vector<char> &buffer) noexcept {
return *this;
}
packet &packet::operator=(std::vector<char> &&buffer) noexcept {
auto packet::operator=(data_buffer &&buffer) noexcept -> packet & {
if (&buffer_ != &buffer) {
buffer_ = std::move(buffer);
decode_offset_ = 0;
@@ -535,7 +550,7 @@ packet &packet::operator=(std::vector<char> &&buffer) noexcept {
return *this;
}
packet &packet::operator=(const packet &p) noexcept {
auto packet::operator=(const packet &p) noexcept -> packet & {
if (this != &p) {
buffer_ = p.buffer_;
decode_offset_ = p.decode_offset_;
@@ -544,7 +559,7 @@ packet &packet::operator=(const packet &p) noexcept {
return *this;
}
packet &packet::operator=(packet &&p) noexcept {
auto packet::operator=(packet &&p) noexcept -> packet & {
if (this != &p) {
buffer_ = std::move(p.buffer_);
decode_offset_ = p.decode_offset_;

126
src/comm/packet/packet_client.cpp
View File

@@ -1,37 +1,44 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "comm/packet/packet_client.hpp"
#include "events/events.hpp"
#include "types/repertory.hpp"
#include "utils/error_utils.hpp"
#include "utils/timeout.hpp"
namespace repertory {
// clang-format off
E_SIMPLE2(packet_client_timeout, error, true,
std::string, event_name, en, E_STRING,
E_SIMPLE2(packet_client_timeout, error, true,
std::string, event_name, en, E_STRING,
std::string, message, msg, E_STRING
);
// clang-format on
packet_client::packet_client(std::string host_name_or_ip, const std::uint8_t &max_connections,
const std::uint16_t &port, const std::uint16_t &receive_timeout,
const std::uint16_t &send_timeout, std::string encryption_token)
packet_client::packet_client(std::string host_name_or_ip,
std::uint8_t max_connections, std::uint16_t port,
std::uint16_t receive_timeout,
std::uint16_t send_timeout,
std::string encryption_token)
: io_context_(),
host_name_or_ip_(std::move(host_name_or_ip)),
max_connections_(max_connections ? max_connections : 20u),
@@ -47,10 +54,10 @@ packet_client::~packet_client() {
io_context_.stop();
}
void packet_client::close(client &client) const {
void packet_client::close(client &cli) const {
try {
boost::system::error_code ec;
client.socket.close(ec);
cli.socket.close(ec);
} catch (...) {
}
}
@@ -65,8 +72,7 @@ void packet_client::close_all() {
unique_id_ = utils::create_uuid_string();
}
bool packet_client::connect(client &c) {
auto ret = false;
void packet_client::connect(client &c) {
try {
resolve();
boost::asio::connect(c.socket, resolve_results_);
@@ -74,16 +80,16 @@ bool packet_client::connect(client &c) {
c.socket.set_option(boost::asio::socket_base::linger(false, 0));
packet response;
read_packet(c, response);
ret = true;
const auto res = read_packet(c, response);
if (res != 0) {
throw std::runtime_error(std::to_string(res));
}
} catch (const std::exception &e) {
event_system::instance().raise<repertory_exception>(__FUNCTION__, e.what());
utils::error::raise_error(__FUNCTION__, e, "connection handshake failed");
}
return ret;
}
std::shared_ptr<packet_client::client> packet_client::get_client() {
auto packet_client::get_client() -> std::shared_ptr<packet_client::client> {
std::shared_ptr<client> ret;
unique_mutex_lock clients_lock(clients_mutex_);
@@ -109,22 +115,25 @@ void packet_client::put_client(std::shared_ptr<client> &c) {
}
}
packet::error_type packet_client::read_packet(client &c, packet &response) {
std::vector<char> buffer(sizeof(std::uint32_t));
auto packet_client::read_packet(client &c, packet &response)
-> packet::error_type {
data_buffer buffer(sizeof(std::uint32_t));
const auto read_buffer = [&]() {
std::uint32_t offset = 0u;
while (offset < buffer.size()) {
const auto bytes_read =
boost::asio::read(c.socket, boost::asio::buffer(&buffer[offset], buffer.size() - offset));
const auto bytes_read = boost::asio::read(
c.socket,
boost::asio::buffer(&buffer[offset], buffer.size() - offset));
if (bytes_read <= 0) {
throw std::runtime_error("Read failed: " + std::to_string(bytes_read));
throw std::runtime_error("read failed|" + std::to_string(bytes_read));
}
offset += static_cast<std::uint32_t>(bytes_read);
}
};
read_buffer();
const auto size = boost::endian::big_to_native(*reinterpret_cast<std::uint32_t *>(&buffer[0u]));
const auto size = boost::endian::big_to_native(
*reinterpret_cast<std::uint32_t *>(&buffer[0u]));
buffer.resize(size);
read_buffer();
@@ -140,34 +149,37 @@ packet::error_type packet_client::read_packet(client &c, packet &response) {
void packet_client::resolve() {
if (resolve_results_.empty()) {
resolve_results_ =
tcp::resolver(io_context_).resolve({host_name_or_ip_, std::to_string(port_)});
resolve_results_ = tcp::resolver(io_context_)
.resolve({host_name_or_ip_, std::to_string(port_)});
}
}
packet::error_type packet_client::send(const std::string &method, std::uint32_t &service_flags) {
auto packet_client::send(const std::string &method,
std::uint32_t &service_flags) -> packet::error_type {
packet request;
return send(method, request, service_flags);
}
packet::error_type packet_client::send(const std::string &method, packet &request,
std::uint32_t &service_flags) {
auto packet_client::send(const std::string &method, packet &request,
std::uint32_t &service_flags) -> packet::error_type {
packet response;
return send(method, request, response, service_flags);
}
packet::error_type packet_client::send(const std::string &method, packet &request, packet &response,
std::uint32_t &service_flags) {
auto packet_client::send(const std::string &method, packet &request,
packet &response, std::uint32_t &service_flags)
-> packet::error_type {
auto success = false;
packet::error_type ret = utils::translate_api_error(api_error::error);
packet::error_type ret = utils::from_api_error(api_error::error);
request.encode_top(method);
request.encode_top(utils::get_thread_id());
request.encode_top(unique_id_);
request.encode_top(PACKET_SERVICE_FLAGS);
request.encode_top(get_repertory_version());
static const auto max_attempts = 2;
for (auto i = 1; allow_connections_ && not success && (i <= max_attempts); i++) {
static const std::uint8_t max_attempts = 5u;
for (std::uint8_t i = 1u;
allow_connections_ && not success && (i <= max_attempts); i++) {
auto c = get_client();
if (c) {
try {
@@ -176,7 +188,8 @@ packet::error_type packet_client::send(const std::string &method, packet &reques
timeout request_timeout(
[this, method, c]() {
event_system::instance().raise<packet_client_timeout>("Request", method);
event_system::instance().raise<packet_client_timeout>("request",
method);
close(*c.get());
},
std::chrono::seconds(send_timeout_));
@@ -184,9 +197,11 @@ packet::error_type packet_client::send(const std::string &method, packet &reques
std::uint32_t offset = 0u;
while (offset < request.get_size()) {
const auto bytes_written = boost::asio::write(
c->socket, boost::asio::buffer(&request[offset], request.get_size() - offset));
c->socket, boost::asio::buffer(&request[offset],
request.get_size() - offset));
if (bytes_written <= 0) {
throw std::runtime_error("Write failed: " + std::to_string(bytes_written));
throw std::runtime_error("write failed|" +
std::to_string(bytes_written));
}
offset += static_cast<std::uint32_t>(bytes_written);
}
@@ -194,7 +209,8 @@ packet::error_type packet_client::send(const std::string &method, packet &reques
timeout response_timeout(
[this, method, c]() {
event_system::instance().raise<packet_client_timeout>("Response", method);
event_system::instance().raise<packet_client_timeout>("response",
method);
close(*c.get());
},
std::chrono::seconds(receive_timeout_));
@@ -202,13 +218,17 @@ packet::error_type packet_client::send(const std::string &method, packet &reques
ret = read_packet(*c, response);
response_timeout.disable();
if (ret == 0) {
response.decode(service_flags);
response.decode(ret);
success = true;
put_client(c);
if ((ret = response.decode(service_flags)) == 0) {
packet::error_type res{};
if ((ret = response.decode(res)) == 0) {
ret = res;
success = true;
put_client(c);
}
}
}
} catch (const std::exception &e) {
event_system::instance().raise<repertory_exception>(__FUNCTION__, e.what());
utils::error::raise_error(__FUNCTION__, e, "send failed");
close_all();
if (allow_connections_ && (i < max_attempts)) {
std::this_thread::sleep_for(1s);
@@ -217,7 +237,7 @@ packet::error_type packet_client::send(const std::string &method, packet &reques
}
if (not allow_connections_) {
ret = utils::translate_api_error(api_error::error);
ret = utils::from_api_error(api_error::error);
success = true;
}
}

139
src/comm/packet/packet_server.cpp
View File

@@ -1,37 +1,49 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "comm/packet/packet_server.hpp"
#include "events/events.hpp"
#include "events/event_system.hpp"
#include "comm/packet/packet.hpp"
#include "events/event_system.hpp"
#include "events/events.hpp"
#include "types/repertory.hpp"
#include "utils/error_utils.hpp"
#include "utils/string_utils.hpp"
#include "utils/utils.hpp"
namespace repertory {
packet_server::packet_server(const std::uint16_t &port, std::string token, std::uint8_t pool_size,
closed_callback closed, message_handler_callback message_handler)
: encryption_token_(std::move(token)), closed_(closed), message_handler_(message_handler) {
using std::thread;
packet_server::packet_server(std::uint16_t port, std::string token,
std::uint8_t pool_size, closed_callback closed,
message_handler_callback message_handler)
: encryption_token_(std::move(token)),
closed_(std::move(closed)),
message_handler_(std::move(message_handler)) {
initialize(port, pool_size);
event_system::instance().raise<service_started>("packet_server");
}
packet_server::~packet_server() {
event_system::instance().raise<service_shutdown>("packet_server");
event_system::instance().raise<service_shutdown_begin>("packet_server");
std::thread([this]() {
for (std::size_t i = 0u; i < service_threads_.size(); i++) {
io_context_.stop();
@@ -40,6 +52,7 @@ packet_server::~packet_server() {
server_thread_->join();
server_thread_.reset();
event_system::instance().raise<service_shutdown_end>("packet_server");
}
void packet_server::add_client(connection &c, const std::string &client_id) {
@@ -64,12 +77,13 @@ void packet_server::initialize(const uint16_t &port, uint8_t pool_size) {
acceptor.bind(endpoint);
acceptor.listen();
} catch (const std::exception &e) {
event_system::instance().raise<repertory_exception>(__FUNCTION__, e.what());
repertory::utils::error::raise_error(__FUNCTION__, e,
"exception occurred");
}
listen_for_connection(acceptor);
for (std::uint8_t i = 0u; i < pool_size; i++) {
service_threads_.emplace_back(std::thread([this]() { io_context_.run(); }));
service_threads_.emplace_back([this]() { io_context_.run(); });
}
for (auto &th : service_threads_) {
@@ -80,14 +94,16 @@ void packet_server::initialize(const uint16_t &port, uint8_t pool_size) {
void packet_server::listen_for_connection(tcp::acceptor &acceptor) {
auto c = std::make_shared<packet_server::connection>(io_context_, acceptor);
acceptor.async_accept(
c->socket, boost::bind(&packet_server::on_accept, this, c, boost::asio::placeholders::error));
acceptor.async_accept(c->socket,
boost::bind(&packet_server::on_accept, this, c,
boost::asio::placeholders::error));
}
void packet_server::on_accept(std::shared_ptr<connection> c, boost::system::error_code ec) {
void packet_server::on_accept(std::shared_ptr<connection> c,
boost::system::error_code ec) {
listen_for_connection(c->acceptor);
if (ec) {
event_system::instance().raise<repertory_exception>(__FUNCTION__, ec.message());
utils::error::raise_error(__FUNCTION__, ec.message());
std::this_thread::sleep_for(1s);
} else {
c->socket.set_option(boost::asio::ip::tcp::no_delay(true));
@@ -101,30 +117,34 @@ void packet_server::on_accept(std::shared_ptr<connection> c, boost::system::erro
}
void packet_server::read_header(std::shared_ptr<connection> c) {
static const std::string function_name = __FUNCTION__;
c->buffer.resize(sizeof(std::uint32_t));
boost::asio::async_read(c->socket, boost::asio::buffer(&c->buffer[0u], c->buffer.size()),
[this, c](boost::system::error_code ec, std::size_t) {
if (ec) {
remove_client(*c);
event_system::instance().raise<repertory_exception>(__FUNCTION__,
ec.message());
} else {
auto to_read = *reinterpret_cast<std::uint32_t *>(&c->buffer[0u]);
boost::endian::big_to_native_inplace(to_read);
read_packet(c, to_read);
}
});
boost::asio::async_read(
c->socket, boost::asio::buffer(&c->buffer[0u], c->buffer.size()),
[this, c](boost::system::error_code ec, std::size_t) {
if (ec) {
remove_client(*c);
repertory::utils::error::raise_error(function_name, ec.message());
} else {
auto to_read = *reinterpret_cast<std::uint32_t *>(&c->buffer[0u]);
boost::endian::big_to_native_inplace(to_read);
read_packet(c, to_read);
}
});
}
void packet_server::read_packet(std::shared_ptr<connection> c, const std::uint32_t &data_size) {
void packet_server::read_packet(std::shared_ptr<connection> c,
std::uint32_t data_size) {
try {
const auto read_buffer = [&]() {
std::uint32_t offset = 0u;
while (offset < c->buffer.size()) {
const auto bytes_read = boost::asio::read(
c->socket, boost::asio::buffer(&c->buffer[offset], c->buffer.size() - offset));
c->socket,
boost::asio::buffer(&c->buffer[offset], c->buffer.size() - offset));
if (bytes_read <= 0) {
throw std::runtime_error("read failed: " + std::to_string(bytes_read));
throw std::runtime_error("read failed|" + std::to_string(bytes_read));
}
offset += static_cast<std::uint32_t>(bytes_read);
}
@@ -147,7 +167,8 @@ void packet_server::read_packet(std::shared_ptr<connection> c, const std::uint32
std::string version;
if ((ret = request->decode(version)) == 0) {
if (utils::compare_version_strings(version, MIN_REMOTE_VERSION) >= 0) {
if (utils::compare_version_strings(
version, REPERTORY_MIN_REMOTE_VERSION) >= 0) {
std::uint32_t service_flags = 0u;
DECODE_OR_IGNORE(request, service_flags);
@@ -166,17 +187,18 @@ void packet_server::read_packet(std::shared_ptr<connection> c, const std::uint32
}
should_send_response = false;
message_handler_(service_flags, client_id, thread_id, method, request.get(),
*response,
[this, c, request, response](const packet::error_type &result) {
message_handler_(service_flags, client_id, thread_id, method,
request.get(), *response,
[this, c, request,
response](const packet::error_type &result) {
this->send_response(c, result, *response);
});
}
} else {
ret = utils::translate_api_error(api_error::incompatible_version);
ret = utils::from_api_error(api_error::incompatible_version);
}
} else {
ret = utils::translate_api_error(api_error::invalid_version);
ret = utils::from_api_error(api_error::invalid_version);
}
} else {
throw std::runtime_error("invalid nonce");
@@ -189,7 +211,7 @@ void packet_server::read_packet(std::shared_ptr<connection> c, const std::uint32
}
} catch (const std::exception &e) {
remove_client(*c);
event_system::instance().raise<repertory_exception>(__FUNCTION__, e.what());
utils::error::raise_error(__FUNCTION__, e, "exception occurred");
}
}
@@ -203,23 +225,26 @@ void packet_server::remove_client(connection &c) {
}
}
void packet_server::send_response(std::shared_ptr<connection> c, const packet::error_type &result,
void packet_server::send_response(std::shared_ptr<connection> c,
const packet::error_type &result,
packet &response) {
static const std::string function_name = __FUNCTION__;
response.encode_top(result);
response.encode_top(PACKET_SERVICE_FLAGS);
response.encode_top(c->nonce);
response.encrypt(encryption_token_);
response.transfer_into(c->buffer);
boost::asio::async_write(c->socket, boost::asio::buffer(c->buffer),
[this, c](boost::system::error_code ec, std::size_t /*length*/) {
if (ec) {
remove_client(*c);
event_system::instance().raise<repertory_exception>(__FUNCTION__,
ec.message());
} else {
read_header(c);
}
});
boost::asio::async_write(
c->socket, boost::asio::buffer(c->buffer),
[this, c](boost::system::error_code ec, std::size_t /*length*/) {
if (ec) {
remove_client(*c);
utils::error::raise_error(function_name, ec.message());
} else {
read_header(c);
}
});
}
} // namespace repertory

575
src/comm/s3/s3_comm.cpp Normal file
View File

@@ -0,0 +1,575 @@
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#if defined(REPERTORY_ENABLE_S3)
#include "comm/s3/s3_comm.hpp"
#include "app_config.hpp"
#include "comm/curl/curl_comm.hpp"
#include "comm/s3/s3_requests.hpp"
#include "events/event_system.hpp"
#include "events/events.hpp"
#include "providers/i_provider.hpp"
#include "types/repertory.hpp"
#include "types/s3.hpp"
#include "utils/encryption.hpp"
#include "utils/error_utils.hpp"
#include "utils/path_utils.hpp"
#include "utils/polling.hpp"
#include "utils/string_utils.hpp"
namespace repertory {
static const get_key_callback empty_key = []() { return ""; };
s3_comm::s3_comm(const app_config &config)
: config_(config), s3_config_(config.get_s3_config()) {
s3_config_.bucket = utils::string::trim(s3_config_.bucket);
// TODO make configurable
const auto enable_path_style =
utils::string::begins_with(s3_config_.url, "http://localhost") ||
utils::string::begins_with(s3_config_.url, "https://localhost") ||
utils::string::begins_with(s3_config_.url, "http://127.0.0.1") ||
utils::string::begins_with(s3_config_.url, "https://127.0.0.1");
s3_client_ = std::make_unique<curl_comm>(s3_config_);
s3_client_->enable_s3_path_style(enable_path_style);
polling::instance().set_callback(
{"s3_directory_cache", polling::frequency::high,
[this]() { this->clear_expired_directories(); }});
}
s3_comm::s3_comm(s3_comm &&comm)
: config_(std::move(comm.config_)),
s3_config_(std::move(comm.s3_config_)),
s3_client_(std::move(comm.s3_client_)) {
comm.active_ = false;
polling::instance().set_callback(
{"s3_directory_cache", polling::frequency::high,
[this]() { this->clear_expired_directories(); }});
}
s3_comm::~s3_comm() {
if (active_) {
polling::instance().remove_callback("s3_directory_cache");
}
}
void s3_comm::clear_expired_directories() {
recur_mutex_lock l(cached_directories_mutex_);
std::vector<std::string> expired_list;
for (const auto &kv : cached_directories_) {
if (kv.second.expiration <= std::chrono::system_clock::now()) {
expired_list.emplace_back(kv.first);
}
}
for (const auto &expired : expired_list) {
event_system::instance().raise<debug_log>(__FUNCTION__, expired, "expired");
cached_directories_.erase(expired);
}
}
auto s3_comm::create_directory(const std::string &api_path) -> api_error {
raise_begin(__FUNCTION__, api_path);
long response_code{};
auto object_name = get_object_name(api_path, empty_key) + '/';
if (not create_directory_object_request(*s3_client_, s3_config_, object_name,
response_code)) {
return raise_end(__FUNCTION__, api_path, api_error::comm_error,
response_code);
}
return raise_end(__FUNCTION__, api_path,
response_code == 200 ? api_error::success
: api_error::comm_error,
response_code);
}
auto s3_comm::directory_exists(const std::string &api_path) const -> api_error {
raise_begin(__FUNCTION__, api_path);
auto object_name = get_object_name(api_path, empty_key) + "/";
head_object_result result{};
long response_code{};
if (head_object_request(*s3_client_, s3_config_, object_name, result,
response_code)) {
if (response_code == 404) {
return raise_end(__FUNCTION__, api_path, api_error::directory_not_found,
response_code);
}
return raise_end(__FUNCTION__, api_path, api_error::directory_exists,
response_code);
}
return raise_end(__FUNCTION__, api_path, api_error::comm_error,
response_code);
}
auto s3_comm::file_exists(const std::string &api_path,
const get_key_callback &get_key) const -> api_error {
raise_begin(__FUNCTION__, api_path);
if (get_cached_file_exists(api_path)) {
return raise_end(__FUNCTION__, api_path, api_error::item_exists, 200);
}
auto object_name = get_object_name(api_path, get_key);
head_object_result result{};
long response_code{};
if (head_object_request(*s3_client_, s3_config_, object_name, result,
response_code)) {
if (response_code == 404) {
return raise_end(__FUNCTION__, api_path, api_error::item_not_found,
response_code);
}
return raise_end(__FUNCTION__, api_path, api_error::item_exists,
response_code);
}
return raise_end(__FUNCTION__, api_path, api_error::directory_exists,
response_code);
}
auto s3_comm::get_object_list(std::vector<directory_item> &list) const
-> api_error {
raise_begin(__FUNCTION__, "/");
long response_code{};
auto success =
list_objects_request(*s3_client_, s3_config_, list, response_code);
return raise_end(__FUNCTION__, "/",
success ? api_error::success : api_error::comm_error,
response_code);
}
auto s3_comm::get_object_name(const std::string &api_path,
const get_key_callback &get_key) const
-> std::string {
auto object_name = utils::path::create_api_path(api_path).substr(1);
const auto key = get_key();
if (not key.empty()) {
auto parts = utils::string::split(object_name, '/', false);
parts[parts.size() - 1u] = key;
object_name = utils::string::join(parts, '/');
}
return object_name;
}
auto s3_comm::get_cached_directory_item_count(const std::string &api_path,
std::size_t &count) const
-> bool {
recur_mutex_lock l(cached_directories_mutex_);
if (cached_directories_.find(api_path) != cached_directories_.end()) {
count = cached_directories_.at(api_path).items.size();
return true;
}
return false;
}
auto s3_comm::get_cached_directory_items(const std::string &api_path,
meta_provider_callback meta_provider,
directory_item_list &list) const
-> bool {
unique_recur_mutex_lock l(cached_directories_mutex_);
if (cached_directories_.find(api_path) != cached_directories_.end()) {
auto &cachedEntry = cached_directories_.at(api_path);
list = cachedEntry.items;
cached_directories_[api_path].reset_timeout(
std::chrono::seconds(config_.get_s3_config().cache_timeout_secs));
l.unlock();
for (auto &item : list) {
meta_provider(item);
}
return true;
}
return false;
}
auto s3_comm::get_cached_file_exists(const std::string &api_path) const
-> bool {
unique_recur_mutex_lock l(cached_directories_mutex_);
const auto parent_api_path = utils::path::get_parent_api_path(api_path);
if (cached_directories_.find(parent_api_path) != cached_directories_.end()) {
auto &entry = cached_directories_.at(parent_api_path);
if (std::find_if(entry.items.begin(), entry.items.end(),
[&api_path](const auto &item) -> bool {
return not item.directory && (api_path == item.api_path);
}) != entry.items.end()) {
cached_directories_[api_path].reset_timeout(
std::chrono::seconds(config_.get_s3_config().cache_timeout_secs));
return true;
}
}
return false;
}
auto s3_comm::get_directory_item_count(
const std::string &api_path, meta_provider_callback meta_provider) const
-> std::size_t {
raise_begin(__FUNCTION__, api_path);
std::size_t ret = 0u;
if (not get_cached_directory_item_count(api_path, ret)) {
directory_item_list list;
const auto res = grab_directory_items(api_path, meta_provider, list);
if (res != api_error::success) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, res,
"failed to grab directory items");
}
return list.size();
}
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(__FUNCTION__, api_path,
"end|" + std::to_string(ret));
}
return ret;
}
auto s3_comm::get_directory_items(const std::string &api_path,
meta_provider_callback meta_provider,
directory_item_list &list) const
-> api_error {
raise_begin(__FUNCTION__, api_path);
auto ret = api_error::success;
if (not get_cached_directory_items(api_path, meta_provider, list)) {
ret = grab_directory_items(api_path, meta_provider, list);
}
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(
__FUNCTION__, api_path, "end|" + api_error_to_string(ret));
}
return ret;
}
auto s3_comm::get_directory_list(api_file_list &list) const -> api_error {
raise_begin(__FUNCTION__, "/");
long response_code{};
auto success =
list_directories_request(*s3_client_, s3_config_, list, response_code);
return raise_end(__FUNCTION__, "/",
success ? api_error::success : api_error::comm_error,
response_code);
}
auto s3_comm::get_file(const std::string &api_path,
const get_key_callback &get_key,
const get_name_callback &get_name,
const get_token_callback &get_token,
api_file &file) const -> api_error {
raise_begin(__FUNCTION__, api_path);
auto ret = api_error::success;
auto object_name = get_object_name(api_path, get_key);
head_object_result result{};
long response_code{};
if (head_object_request(*s3_client_, s3_config_, object_name, result,
response_code)) {
const auto key = get_key();
object_name = get_name(key, object_name);
file.accessed_date = utils::get_file_time_now();
file.api_path = utils::path::create_api_path(object_name);
file.api_parent = utils::path::get_parent_api_path(file.api_path);
file.changed_date = utils::aws::format_time(result.last_modified);
file.creation_date = utils::aws::format_time(result.last_modified);
file.encryption_token = get_token();
file.file_size =
file.encryption_token.empty()
? result.content_length
: utils::encryption::encrypting_reader::calculate_decrypted_size(
result.content_length);
file.modified_date = utils::aws::format_time(result.last_modified);
} else {
utils::error::raise_api_path_error(__FUNCTION__, api_path, response_code,
"head object request failed");
ret = api_error::comm_error;
}
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(
__FUNCTION__, api_path, "end|" + api_error_to_string(ret));
}
return ret;
}
auto s3_comm::get_file_list(
const get_api_file_token_callback &get_api_file_token,
const get_name_callback &get_name, api_file_list &list) const -> api_error {
raise_begin(__FUNCTION__, "/");
long response_code{};
auto success = list_files_request(*s3_client_, s3_config_, get_api_file_token,
get_name, list, response_code);
return raise_end(__FUNCTION__, "/",
success ? api_error::success : api_error::comm_error,
response_code);
}
auto s3_comm::grab_directory_items(const std::string &api_path,
meta_provider_callback meta_provider,
directory_item_list &list) const
-> api_error {
auto object_name = get_object_name(api_path, empty_key);
long response_code{};
if (list_objects_in_directory_request(*s3_client_, s3_config_, object_name,
meta_provider, list, response_code)) {
if (response_code == 404) {
return api_error::directory_not_found;
}
if (response_code != 200) {
return api_error::comm_error;
}
set_cached_directory_items(api_path, list);
return api_error::success;
}
return api_error::comm_error;
}
void s3_comm::raise_begin(const std::string &function_name,
const std::string &api_path) const {
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(function_name, api_path,
"begin|");
}
}
auto s3_comm::raise_end(const std::string &function_name,
const std::string &api_path, const api_error &error,
long code) const -> api_error {
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(
function_name, api_path,
"end|" + api_error_to_string(error) + '|' + std::to_string(code));
}
return error;
}
auto s3_comm::read_file_bytes(const std::string &api_path, std::size_t size,
std::uint64_t offset, data_buffer &data,
const get_key_callback &get_key,
const get_size_callback &get_size,
const get_token_callback &get_token,
stop_type &stop_requested) const -> api_error {
data.clear();
auto object_name = get_object_name(api_path, get_key);
const auto encryption_token = get_token();
const auto data_size = get_size();
if (encryption_token.empty()) {
long response_code{};
if (not read_object_request(*s3_client_, s3_config_, object_name, size,
offset, data, response_code, stop_requested)) {
auto res =
stop_requested ? api_error::download_stopped : api_error::comm_error;
utils::error::raise_api_path_error(__FUNCTION__, api_path, res,
"failed to read file bytes");
return res;
}
if (response_code < 200 || response_code >= 300) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, response_code,
"failed to read file bytes");
return api_error::comm_error;
}
return api_error::success;
}
const auto key = utils::encryption::generate_key(encryption_token);
return utils::encryption::read_encrypted_range(
{offset, offset + size - 1}, key,
[&](data_buffer &ct, std::uint64_t start_offset,
std::uint64_t end_offset) -> api_error {
return read_file_bytes(
api_path, (end_offset - start_offset + 1u), start_offset, ct,
get_key, get_size, []() -> std::string { return ""; },
stop_requested);
},
data_size, data);
}
void s3_comm::remove_cached_directory(const std::string &api_path) {
recur_mutex_lock l(cached_directories_mutex_);
cached_directories_.erase(api_path);
}
auto s3_comm::remove_directory(const std::string &api_path) -> api_error {
raise_begin(__FUNCTION__, api_path);
auto object_name = get_object_name(api_path, empty_key) + "/";
long response_code{};
if (delete_object_request(*s3_client_, s3_config_, object_name,
response_code)) {
if (response_code == 404) {
return raise_end(__FUNCTION__, api_path, api_error::directory_not_found,
response_code);
}
if (response_code != 204) {
return raise_end(__FUNCTION__, api_path, api_error::comm_error,
response_code);
}
remove_cached_directory(utils::path::get_parent_api_path(api_path));
remove_cached_directory(api_path);
return raise_end(__FUNCTION__, api_path, api_error::success, response_code);
}
return raise_end(__FUNCTION__, api_path, api_error::comm_error,
response_code);
}
auto s3_comm::remove_file(const std::string &api_path,
const get_key_callback &get_key) -> api_error {
raise_begin(__FUNCTION__, api_path);
auto object_name = get_object_name(api_path, get_key);
long response_code{};
if (delete_object_request(*s3_client_, s3_config_, object_name,
response_code)) {
if (response_code == 404) {
return raise_end(__FUNCTION__, api_path, api_error::item_not_found,
response_code);
}
if (response_code != 204) {
return raise_end(__FUNCTION__, api_path, api_error::comm_error,
response_code);
}
remove_cached_directory(utils::path::get_parent_api_path(api_path));
return raise_end(__FUNCTION__, api_path, api_error::success, response_code);
}
return raise_end(__FUNCTION__, api_path, api_error::comm_error,
response_code);
}
auto s3_comm::rename_file(const std::string & /*api_path*/,
const std::string & /*new_api_path*/) -> api_error {
return api_error::not_implemented;
/* if (config_.get_event_level() >= event_level::debug) { */
/* event_system::instance().raise<debug_log>(__FUNCTION__, api_path,
* "begin"); */
/* } */
/* auto ret = api_error::success; */
/* */
/* std::string bucket_name, object_name; */
/* get_object_name(api_path, bucket_name, object_name); */
/* */
/* std::string new_object_name; */
/* get_object_name(new_api_path, bucket_name, new_object_name); */
/* */
/* Aws::S3::Model::CopyObjectRequest request{}; */
/* request.SetBucket(bucket_name); */
/* request.SetCopySource(bucket_name + '/' + object_name); */
/* request.SetKey(new_object_name); */
/* */
/* const auto outcome = s3_client_->CopyObject(request); */
/* if (outcome.IsSuccess()) { */
/* ret = remove_file(api_path); */
/* } else { */
/* const auto &error = outcome.GetError(); */
/* event_system::instance().raise<repertory_exception>(__FUNCTION__,
* error.GetExceptionName()
* +
* "|" + */
/* error.GetMessage());
*/
/* ret = api_error::comm_error; */
/* } */
/* */
/* if (config_.get_event_level() >= event_level::debug) { */
/* event_system::instance().raise<debug_log>(__FUNCTION__, api_path, */
/* "end|" +
* std::to_string(std::uint8_t(ret))); */
/* } */
/* return ret; */
}
void s3_comm::set_cached_directory_items(const std::string &api_path,
directory_item_list list) const {
recur_mutex_lock l(cached_directories_mutex_);
cached_directories_[api_path].items = std::move(list);
cached_directories_[api_path].reset_timeout(
std::chrono::seconds(config_.get_s3_config().cache_timeout_secs));
}
auto s3_comm::upload_file(const std::string &api_path,
const std::string &source_path,
const std::string &encryption_token,
const get_key_callback &get_key,
const set_key_callback &set_key,
stop_type &stop_requested) -> api_error {
raise_begin(__FUNCTION__, api_path);
auto object_name = get_object_name(api_path, get_key);
long response_code{};
if (not put_object_request(*s3_client_, s3_config_, object_name, source_path,
encryption_token, get_key, set_key, response_code,
stop_requested)) {
return raise_end(__FUNCTION__, api_path,
stop_requested ? api_error::upload_stopped
: api_error::upload_failed,
response_code);
}
if (response_code != 200) {
return raise_end(__FUNCTION__, api_path, api_error::comm_error,
response_code);
}
remove_cached_directory(utils::path::get_parent_api_path(api_path));
return raise_end(__FUNCTION__, api_path, api_error::success, response_code);
}
} // namespace repertory
#endif // REPERTORY_ENABLE_S3

399
src/comm/s3/s3_requests_curl.cpp Normal file
View File

@@ -0,0 +1,399 @@
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#if defined(REPERTORY_ENABLE_S3)
#include "comm/s3/s3_requests.hpp"
#include "comm/curl/curl_comm.hpp"
#include "comm/curl/requests/http_get.hpp"
#include "utils/encryption.hpp"
#include "utils/error_utils.hpp"
#include "utils/path_utils.hpp"
namespace repertory {
namespace {
[[nodiscard]] auto
get_object_list(i_http_comm &client, const s3_config &config,
std::string &response_data, long &response_code,
std::optional<std::string> delimiter = std::nullopt,
std::optional<std::string> prefix = std::nullopt) -> bool {
curl::requests::http_get get{};
get.aws_service = "aws:amz:" + config.region + ":s3";
get.path = '/';
get.query["list-type"] = "2";
if (delimiter.has_value() && not delimiter.value().empty()) {
get.query["delimiter"] = delimiter.value();
}
if (prefix.has_value() && not prefix.value().empty()) {
get.query["prefix"] = prefix.value();
}
get.response_handler = [&response_data](const data_buffer &data,
long /*response_code*/) {
response_data = std::string(data.begin(), data.end());
};
stop_type stop_requested{};
return client.make_request(get, response_code, stop_requested);
}
} // namespace
auto create_directory_object_request_impl(i_http_comm &client,
const s3_config &config,
const std::string &object_name,
long &response_code) -> bool {
try {
curl::requests::http_put_file put_file{};
put_file.aws_service = "aws:amz:" + config.region + ":s3";
put_file.file_name =
*(utils::string::split(object_name, '/', false).end() - 1U);
put_file.path = '/' + object_name;
stop_type stop_requested{false};
return client.make_request(put_file, response_code, stop_requested);
} catch (const std::exception &e) {
utils::error::raise_error(__FUNCTION__, e, "exception occurred");
}
return false;
}
auto delete_object_request_impl(i_http_comm &client, const s3_config &config,
const std::string &object_name,
long &response_code) -> bool {
try {
head_object_result result{};
if (not head_object_request_impl(client, config, object_name, result,
response_code)) {
return false;
}
if (response_code == 404) {
return true;
}
curl::requests::http_delete del{};
del.aws_service = "aws:amz:" + config.region + ":s3";
del.path = '/' + object_name;
stop_type stop_requested{false};
return client.make_request(del, response_code, stop_requested);
} catch (const std::exception &e) {
utils::error::raise_error(__FUNCTION__, e, "exception occurred");
}
return false;
}
auto head_object_request_impl(i_http_comm &client, const s3_config &config,
const std::string &object_name,
head_object_result &result, long &response_code)
-> bool {
try {
curl::requests::http_head head{};
head.aws_service = "aws:amz:" + config.region + ":s3";
head.path = '/' + object_name;
head.response_headers = http_headers{};
stop_type stop_requested{false};
if (not client.make_request(head, response_code, stop_requested)) {
return false;
}
if (response_code == 200) {
result.from_headers(head.response_headers.value());
}
return true;
} catch (const std::exception &e) {
utils::error::raise_error(__FUNCTION__, e, "exception occurred");
}
return false;
}
auto list_directories_request_impl(i_http_comm &client, const s3_config &config,
list_directories_result &result,
long &response_code) -> bool {
try {
std::string response_data{};
if (not get_object_list(client, config, response_data, response_code)) {
return false;
}
if (response_code != 200) {
return false;
}
pugi::xml_document doc;
auto res = doc.load_string(response_data.c_str());
if (res.status != pugi::xml_parse_status::status_ok) {
return false;
}
auto node_list = doc.select_nodes("/ListBucketResult/Contents");
for (const auto &node : node_list) {
auto object_name =
node.node().select_node("Key").node().text().as_string();
if (utils::string::ends_with(object_name, "/")) {
api_file directory{};
directory.api_path = utils::path::create_api_path(object_name);
directory.api_parent =
utils::path::get_parent_api_path(directory.api_path);
directory.accessed_date = utils::get_file_time_now();
directory.changed_date = utils::convert_api_date(
node.node().select_node("LastModified").node().text().as_string());
directory.creation_date = directory.changed_date;
directory.modified_date = directory.changed_date;
result.emplace_back(std::move(directory));
}
}
return true;
} catch (const std::exception &e) {
utils::error::raise_error(__FUNCTION__, e, "exception occurred");
}
return false;
}
auto list_files_request_impl(
i_http_comm &client, const s3_config &config,
const get_api_file_token_callback &get_api_file_token,
const get_name_callback &get_name, list_files_result &result,
long &response_code) -> bool {
try {
std::string response_data{};
if (not get_object_list(client, config, response_data, response_code)) {
return false;
}
if (response_code != 200) {
return false;
}
pugi::xml_document doc;
auto res = doc.load_string(response_data.c_str());
if (res.status != pugi::xml_parse_status::status_ok) {
return false;
}
auto node_list = doc.select_nodes("/ListBucketResult/Contents");
for (const auto &node : node_list) {
std::string object_name =
node.node().select_node("Key").node().text().as_string();
if (not utils::string::ends_with(object_name, "/")) {
api_file file{};
object_name = get_name(
*(utils::string::split(object_name, '/', false).end() - 1u),
object_name);
file.api_path = utils::path::create_api_path(object_name);
file.api_parent = utils::path::get_parent_api_path(file.api_path);
file.accessed_date = utils::get_file_time_now();
file.encryption_token = get_api_file_token(file.api_path);
auto size = node.node().select_node("Size").node().text().as_ullong();
file.file_size = file.encryption_token.empty()
? size
: utils::encryption::encrypting_reader::
calculate_decrypted_size(size);
file.changed_date = utils::convert_api_date(
node.node().select_node("LastModified").node().text().as_string());
file.creation_date = file.changed_date;
file.modified_date = file.changed_date;
result.emplace_back(std::move(file));
}
}
return true;
} catch (const std::exception &e) {
utils::error::raise_error(__FUNCTION__, e, "exception occurred");
}
return false;
}
auto list_objects_in_directory_request_impl(
i_http_comm &client, const s3_config &config,
const std::string &object_name, meta_provider_callback meta_provider,
list_objects_result &result, long &response_code) -> bool {
try {
std::string response_data{};
auto prefix = object_name.empty() ? object_name : object_name + "/";
if (not get_object_list(client, config, response_data, response_code, "/",
prefix)) {
return false;
}
if (response_code != 200) {
return false;
}
pugi::xml_document doc;
auto res = doc.load_string(response_data.c_str());
if (res.status != pugi::xml_parse_status::status_ok) {
return false;
}
const auto add_directory_item =
[&](bool directory, const std::string &name,
std::function<std::uint64_t(const directory_item &)> get_size) {
directory_item di{};
di.api_path =
utils::path::create_api_path(utils::path::combine("/", {name}));
di.api_parent = utils::path::get_parent_api_path(di.api_path);
di.directory = directory;
di.size = get_size(di);
meta_provider(di);
result.emplace_back(std::move(di));
};
auto node_list =
doc.select_nodes("/ListBucketResult/CommonPrefixes/Prefix");
for (const auto &node : node_list) {
add_directory_item(
true, node.node().text().as_string(),
[](const directory_item &) -> std::uint64_t { return 0U; });
}
node_list = doc.select_nodes("/ListBucketResult/Contents");
for (const auto &node : node_list) {
auto child_object_name =
node.node().select_node("Key").node().text().as_string();
if (child_object_name != prefix) {
auto size = node.node().select_node("Size").node().text().as_ullong();
add_directory_item(
false, child_object_name,
[&size](const directory_item &) -> std::uint64_t { return size; });
}
}
return true;
} catch (const std::exception &e) {
utils::error::raise_error(__FUNCTION__, e, "exception occurred");
}
return false;
}
auto list_objects_request_impl(i_http_comm &client, const s3_config &config,
list_objects_result &result, long &response_code)
-> bool {
try {
std::string response_data{};
if (not get_object_list(client, config, response_data, response_code)) {
return false;
}
if (response_code != 200) {
return false;
}
pugi::xml_document doc;
auto res = doc.load_string(response_data.c_str());
if (res.status != pugi::xml_parse_status::status_ok) {
return false;
}
auto node_list = doc.select_nodes("/ListBucketResult/Contents");
for (const auto &node : node_list) {
auto object_name =
node.node().select_node("Key").node().text().as_string();
auto size = node.node().select_node("Size").node().text().as_ullong();
directory_item di{};
di.api_path = utils::path::create_api_path(object_name);
di.api_parent = utils::path::get_parent_api_path(di.api_path);
di.directory = utils::string::ends_with(object_name, "/");
di.size = di.directory ? 0U : size;
di.resolved = false;
result.emplace_back(std::move(di));
}
return true;
} catch (const std::exception &e) {
utils::error::raise_error(__FUNCTION__, e, "exception occurred");
}
return false;
}
auto put_object_request_impl(i_http_comm &client, const s3_config &config,
std::string object_name,
const std::string &source_path,
const std::string &encryption_token,
get_key_callback get_key, set_key_callback set_key,
long &response_code, stop_type &stop_requested)
-> bool {
try {
curl::requests::http_put_file put_file{};
put_file.aws_service = "aws:amz:" + config.region + ":s3";
put_file.encryption_token = encryption_token;
put_file.file_name =
*(utils::string::split(object_name, '/', false).end() - 1U);
put_file.path = '/' + object_name;
put_file.source_path = source_path;
if (not encryption_token.empty()) {
static stop_type no_stop{false};
put_file.reader = std::make_shared<utils::encryption::encrypting_reader>(
put_file.file_name, source_path, no_stop, encryption_token,
std::nullopt, -1);
auto key = get_key();
if (key.empty()) {
key = put_file.reader->get_encrypted_file_name();
set_key(key);
}
}
return client.make_request(put_file, response_code, stop_requested);
} catch (const std::exception &e) {
utils::error::raise_error(__FUNCTION__, e, "exception occurred");
}
return false;
}
auto read_object_request_impl(i_http_comm &client, const s3_config &config,
const std::string &object_name, std::size_t size,
std::uint64_t offset, data_buffer &data,
long &response_code, stop_type &stop_requested)
-> bool {
try {
curl::requests::http_get get{};
get.aws_service = "aws:amz:" + config.region + ":s3";
get.headers["response-content-type"] = "binary/octet-stream";
get.path = '/' + object_name;
get.range = {{offset, offset + size - 1}};
get.response_handler = [&data](const data_buffer &response_data,
long /*response_code*/) {
data = response_data;
};
return client.make_request(get, response_code, stop_requested);
} catch (const std::exception &e) {
utils::error::raise_error(__FUNCTION__, e, "exception occurred");
}
return false;
}
} // namespace repertory
#endif

59
src/common.cpp.in
View File

@@ -1,29 +1,54 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "common.hpp"
#include "types/repertory.hpp"
const std::string &get_repertory_version() {
namespace repertory {
auto get_repertory_git_revision() -> const std::string & {
static const std::string git_revision = "@REPERTORY_GIT_REV@";
return git_revision;
}
auto get_repertory_version() -> const std::string & {
static const std::string version = "@REPERTORY_VERSION@";
return version;
}
const std::string &get_repertory_git_revision() {
static const std::string git_revision = "@REPERTORY_GIT_REV@";
return git_revision;
void repertory_init() {
#ifdef REPERTORY_MUSL
pthread_attr_t a;
memset(&a, 0, sizeof(pthread_attr_t));
pthread_attr_setstacksize(&a, 8U * 1024U * 1024U);
pthread_attr_setguardsize(&a, 4096U);
pthread_setattr_default_np(&a);
#endif
const auto err = sodium_init();
if (err < 0) {
std::cerr << "failed to initialize libsodium|err|" << err << std::endl;
exit(static_cast<std::int32_t>(exit_code::init_failed));
}
curl_global_init(CURL_GLOBAL_DEFAULT);
}
void repertory_shutdown() { curl_global_cleanup(); }
} // namespace repertory

194
src/db/directory_db.cpp
View File

@@ -1,28 +1,32 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "db/directory_db.hpp"
#include "utils/path_utils.hpp"
namespace repertory {
void directory_db::directory_tree::add_path(const std::string &api_path,
const std::vector<std::string> &files,
rocksdb::DB &db) {
void directory_db::directory_tree::add_path(
const std::string &api_path, const std::vector<std::string> &files,
rocksdb::DB &db) {
const auto create_not_found = [&](const auto &create_path) {
std::string value;
if (not db.Get(rocksdb::ReadOptions(), create_path, &value).ok()) {
@@ -43,83 +47,91 @@ void directory_db::directory_tree::add_path(const std::string &api_path,
create_not_found("/");
} else {
auto &sub_directories = sub_directory_lookup_[previous_directory];
if (std::find(sub_directories.begin(), sub_directories.end(), directory_part) ==
sub_directories.end()) {
if (std::find(sub_directories.begin(), sub_directories.end(),
directory_part) == sub_directories.end()) {
sub_directories.emplace_back(directory_part);
}
previous_directory =
utils::path::create_api_path(utils::path::combine(previous_directory, {directory_part}));
previous_directory = utils::path::create_api_path(
utils::path::combine(previous_directory, {directory_part}));
sub_directory_lookup_[previous_directory];
create_not_found(previous_directory);
}
}
}
std::size_t directory_db::directory_tree::get_count(const std::string &api_path) const {
auto directory_db::directory_tree::get_count(const std::string &api_path) const
-> std::size_t {
return (sub_directory_lookup_.find(api_path) == sub_directory_lookup_.end())
? 0
: sub_directory_lookup_.at(api_path).size();
}
std::vector<std::string> directory_db::directory_tree::get_directories() const {
auto directory_db::directory_tree::get_directories() const
-> std::vector<std::string> {
std::vector<std::string> ret;
std::transform(sub_directory_lookup_.begin(), sub_directory_lookup_.end(),
std::back_inserter(ret), [](const auto &kv) { return kv.first; });
std::back_inserter(ret),
[](const auto &kv) { return kv.first; });
return ret;
}
std::vector<std::string>
directory_db::directory_tree::get_sub_directories(const std::string &api_path) const {
auto directory_db::directory_tree::get_sub_directories(
const std::string &api_path) const -> std::vector<std::string> {
std::vector<std::string> ret;
if (sub_directory_lookup_.find(api_path) != sub_directory_lookup_.end()) {
const auto &lookup = sub_directory_lookup_.at(api_path);
std::transform(
lookup.begin(), lookup.end(), std::back_inserter(ret), [&api_path](const auto &directory) {
return utils::path::create_api_path(utils::path::combine(api_path, {directory}));
});
std::transform(lookup.begin(), lookup.end(), std::back_inserter(ret),
[&api_path](const auto &directory) {
return utils::path::create_api_path(
utils::path::combine(api_path, {directory}));
});
}
return ret;
}
bool directory_db::directory_tree::is_directory(const std::string &api_path) const {
auto directory_db::directory_tree::is_directory(
const std::string &api_path) const -> bool {
return sub_directory_lookup_.find(api_path) != sub_directory_lookup_.end();
}
bool directory_db::directory_tree::remove_directory(const std::string &api_path, rocksdb::DB &db,
const bool &allow_remove_root) {
void directory_db::directory_tree::remove_directory(const std::string &api_path,
rocksdb::DB &db,
bool allow_remove_root) {
if ((allow_remove_root || (api_path != "/")) && is_directory(api_path)) {
sub_directory_lookup_.erase(api_path);
db.Delete(rocksdb::WriteOptions(), api_path);
const auto parent_api_path = utils::path::get_parent_api_path(api_path);
const auto parts = utils::string::split(api_path, '/', false);
utils::remove_element_from(sub_directory_lookup_[parent_api_path], parts[parts.size() - 1]);
return true;
utils::remove_element_from(sub_directory_lookup_[parent_api_path],
parts[parts.size() - 1]);
}
return false;
}
directory_db::directory_db(const app_config &config) {
utils::db::create_rocksdb(config, DIRDB_NAME, db_);
auto iterator = std::unique_ptr<rocksdb::Iterator>(db_->NewIterator(rocksdb::ReadOptions()));
auto iterator = std::unique_ptr<rocksdb::Iterator>(
db_->NewIterator(rocksdb::ReadOptions()));
for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
auto directory_data = json::parse(iterator->value().ToString());
tree_.add_path(directory_data["path"].get<std::string>(),
directory_data["files"].get<std::vector<std::string>>(), *db_);
directory_data["files"].get<std::vector<std::string>>(),
*db_);
}
}
directory_db::~directory_db() { db_.reset(); }
api_error directory_db::create_directory(const std::string &api_path, const bool &create_always) {
auto directory_db::create_directory(const std::string &api_path,
bool create_always) -> api_error {
recur_mutex_lock directory_lock(directory_mutex_);
auto ret = api_error::directory_exists;
if (not is_directory(api_path)) {
ret = api_error::directory_not_found;
if (create_always || (api_path == "/") ||
is_directory(utils::path::get_parent_api_path(api_path))) {
ret = api_error::file_exists;
ret = api_error::item_exists;
if (not is_file(api_path)) {
tree_.add_path(api_path, {}, *db_);
ret = api_error::success;
@@ -130,7 +142,7 @@ api_error directory_db::create_directory(const std::string &api_path, const bool
return ret;
}
api_error directory_db::create_file(const std::string &api_path) {
auto directory_db::create_file(const std::string &api_path) -> api_error {
recur_mutex_lock directory_lock(directory_mutex_);
if (is_directory(api_path)) {
return api_error::directory_exists;
@@ -144,7 +156,7 @@ api_error directory_db::create_file(const std::string &api_path) {
const auto file_name = utils::path::strip_to_file_name(api_path);
if (utils::collection_includes(directory_data["files"], file_name)) {
return api_error::file_exists;
return api_error::item_exists;
}
directory_data["files"].emplace_back(file_name);
@@ -152,32 +164,36 @@ api_error directory_db::create_file(const std::string &api_path) {
return api_error::success;
}
json directory_db::get_directory_data(const std::string &api_path) const {
auto directory_db::get_directory_data(const std::string &api_path) const
-> json {
std::string data;
db_->Get(rocksdb::ReadOptions(), api_path, &data);
if (data.empty()) {
return json();
return {};
}
return json::parse(data);
}
std::uint64_t directory_db::get_directory_item_count(const std::string &api_path) const {
auto directory_db::get_directory_item_count(const std::string &api_path) const
-> std::uint64_t {
auto directory_data = get_directory_data(api_path);
const auto sub_directory_count = get_sub_directory_count(api_path);
const auto file_count = (directory_data.empty() ? 0 : directory_data["files"].size());
const auto file_count =
(directory_data.empty() ? 0 : directory_data["files"].size());
return sub_directory_count + file_count;
}
api_error directory_db::get_file(const std::string &api_path, api_file &file,
api_file_provider_callback api_file_provider) const {
auto directory_db::get_file(const std::string &api_path, api_file &file,
api_file_provider_callback api_file_provider) const
-> api_error {
const auto parent_api_path = utils::path::get_parent_api_path(api_path);
auto directory_data = get_directory_data(parent_api_path);
if (not directory_data.empty()) {
const auto file_name = utils::path::strip_to_file_name(api_path);
if (utils::collection_includes(directory_data["files"], file_name)) {
file.api_path =
utils::path::create_api_path(utils::path::combine(parent_api_path, {file_name})),
file.api_path = utils::path::create_api_path(
utils::path::combine(parent_api_path, {file_name})),
api_file_provider(file);
return api_error::success;
}
@@ -186,15 +202,17 @@ api_error directory_db::get_file(const std::string &api_path, api_file &file,
return api_error::item_not_found;
}
api_error directory_db::get_file_list(api_file_list &list,
api_file_provider_callback api_file_provider) const {
auto iterator = std::unique_ptr<rocksdb::Iterator>(db_->NewIterator(rocksdb::ReadOptions()));
auto directory_db::get_file_list(
api_file_list &list, api_file_provider_callback api_file_provider) const
-> api_error {
auto iterator = std::unique_ptr<rocksdb::Iterator>(
db_->NewIterator(rocksdb::ReadOptions()));
for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
auto directory_data = json::parse(iterator->value().ToString());
for (const auto &directory_file : directory_data["files"]) {
api_file file{
utils::path::create_api_path(utils::path::combine(iterator->key().ToString(),
{directory_file.get<std::string>()})),
utils::path::create_api_path(utils::path::combine(
iterator->key().ToString(), {directory_file.get<std::string>()})),
};
api_file_provider(file);
list.emplace_back(file);
@@ -204,30 +222,33 @@ api_error directory_db::get_file_list(api_file_list &list,
return api_error::success;
}
std::size_t directory_db::get_sub_directory_count(const std::string &api_path) const {
auto directory_db::get_sub_directory_count(const std::string &api_path) const
-> std::size_t {
recur_mutex_lock directoryLock(directory_mutex_);
return tree_.get_count(api_path);
}
std::uint64_t directory_db::get_total_item_count() const {
auto directory_db::get_total_item_count() const -> std::uint64_t {
unique_recur_mutex_lock directory_lock(directory_mutex_);
const auto directories = tree_.get_directories();
directory_lock.unlock();
return std::accumulate(directories.begin(), directories.end(), std::uint64_t(directories.size()),
[this](std::uint64_t c, const std::string &directory) {
const auto dirData = this->get_directory_data(directory);
return c + (dirData.empty() ? 0 : dirData["files"].size());
});
return std::accumulate(
directories.begin(), directories.end(), std::uint64_t(directories.size()),
[this](std::uint64_t c, const std::string &directory) {
const auto dirData = this->get_directory_data(directory);
return c + (dirData.empty() ? 0 : dirData["files"].size());
});
}
bool directory_db::is_directory(const std::string &api_path) const {
auto directory_db::is_directory(const std::string &api_path) const -> bool {
recur_mutex_lock directory_lock(directory_mutex_);
return tree_.is_directory(api_path);
}
bool directory_db::is_file(const std::string &api_path) const {
auto directory_data = get_directory_data(utils::path::get_parent_api_path(api_path));
auto directory_db::is_file(const std::string &api_path) const -> bool {
auto directory_data =
get_directory_data(utils::path::get_parent_api_path(api_path));
if (directory_data.empty()) {
return false;
}
@@ -236,9 +257,9 @@ bool directory_db::is_file(const std::string &api_path) const {
return utils::collection_includes(directory_data["files"], file_name);
}
void directory_db::populate_directory_files(const std::string &api_path,
const meta_provider_callback &meta_provider,
directory_item_list &list) const {
void directory_db::populate_directory_files(
const std::string &api_path, meta_provider_callback meta_provider,
directory_item_list &list) const {
auto directory_data = get_directory_data(api_path);
if (not directory_data.empty()) {
for (const auto &directory_file : directory_data["files"]) {
@@ -246,48 +267,47 @@ void directory_db::populate_directory_files(const std::string &api_path,
di.api_path = utils::path::create_api_path(
utils::path::combine(api_path, {directory_file.get<std::string>()}));
di.directory = false;
meta_provider(di, true);
meta_provider(di);
di.size = utils::string::to_uint64(di.meta[META_SIZE]);
list.emplace_back(std::move(di));
}
}
}
void directory_db::populate_sub_directories(const std::string &api_path,
const meta_provider_callback &meta_provider,
directory_item_list &list) const {
void directory_db::populate_sub_directories(
const std::string &api_path, meta_provider_callback meta_provider,
directory_item_list &list) const {
unique_recur_mutex_lock directory_lock(directory_mutex_);
const auto directories = tree_.get_sub_directories(api_path);
directory_lock.unlock();
std::size_t offset = 0u;
std::size_t offset{};
for (const auto &directory : directories) {
if (std::find_if(list.begin(), list.end(), [&directory](const auto &di) -> bool {
return directory == di.api_path;
}) == list.end()) {
if (std::find_if(list.begin(), list.end(),
[&directory](const auto &di) -> bool {
return directory == di.api_path;
}) == list.end()) {
directory_item di{};
di.api_path = directory;
di.api_parent = utils::path::get_parent_api_path(directory);
di.directory = true;
di.size = get_sub_directory_count(directory);
meta_provider(di, true);
list.insert(list.begin() + offset++, std::move(di));
meta_provider(di);
list.insert(list.begin() + static_cast<std::int64_t>(offset++),
std::move(di));
}
}
}
api_error directory_db::remove_directory(const std::string &api_path,
const bool &allow_remove_root) {
auto directory_db::remove_directory(const std::string &api_path,
bool allow_remove_root) -> api_error {
recur_mutex_lock directory_lock(directory_mutex_);
if ((api_path == "/") && not allow_remove_root) {
return api_error::access_denied;
}
if (is_file(api_path)) {
return api_error::item_is_file;
}
if (not is_directory(api_path)) {
if (is_file(api_path) || not is_directory(api_path)) {
return api_error::directory_not_found;
}
@@ -302,7 +322,7 @@ api_error directory_db::remove_directory(const std::string &api_path,
return api_error::directory_not_empty;
}
bool directory_db::remove_file(const std::string &api_path) {
auto directory_db::remove_file(const std::string &api_path) -> bool {
recur_mutex_lock directory_lock(directory_mutex_);
if (is_directory(api_path)) {
@@ -325,8 +345,8 @@ bool directory_db::remove_file(const std::string &api_path) {
return true;
}
api_error directory_db::rename_file(const std::string &from_api_path,
const std::string &to_api_path) {
auto directory_db::rename_file(const std::string &from_api_path,
const std::string &to_api_path) -> api_error {
recur_mutex_lock directory_lock(directory_mutex_);
if (is_directory(from_api_path) || is_directory(to_api_path)) {
@@ -338,7 +358,7 @@ api_error directory_db::rename_file(const std::string &from_api_path,
}
if (is_file(to_api_path)) {
return api_error::file_exists;
return api_error::item_exists;
}
if (not remove_file(from_api_path)) {

370
src/db/meta_db.cpp
View File

@@ -1,24 +1,29 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "db/meta_db.hpp"
#include "types/repertory.hpp"
#include "types/startup_exception.hpp"
#include "utils/error_utils.hpp"
#include "utils/file_utils.hpp"
#include "utils/path_utils.hpp"
#include "utils/utils.hpp"
@@ -27,84 +32,85 @@ namespace repertory {
meta_db::meta_db(const app_config &config) {
const auto create_resources = [this, &config](const std::string &name) {
auto families = std::vector<rocksdb::ColumnFamilyDescriptor>();
families.emplace_back(rocksdb::ColumnFamilyDescriptor{rocksdb::kDefaultColumnFamilyName,
rocksdb::ColumnFamilyOptions()});
families.emplace_back(
rocksdb::ColumnFamilyDescriptor{"source", rocksdb::ColumnFamilyOptions()});
families.emplace_back(rocksdb::ColumnFamilyDescriptor{"keys", rocksdb::ColumnFamilyOptions()});
families.emplace_back(rocksdb::kDefaultColumnFamilyName,
rocksdb::ColumnFamilyOptions());
families.emplace_back("keys", rocksdb::ColumnFamilyOptions());
families.emplace_back("source", rocksdb::ColumnFamilyOptions());
auto handles = std::vector<rocksdb::ColumnFamilyHandle *>();
utils::db::create_rocksdb(config, name, families, handles, db_);
default_family_.reset(handles[0u]);
source_family_.reset(handles[1u]);
keys_family_.reset(handles[2u]);
std::size_t idx{};
default_family_ = handles[idx++];
keys_family_ = handles[idx++];
source_family_ = handles[idx++];
};
create_resources(METADB_NAME);
}
meta_db::~meta_db() { release_resources(); }
meta_db::~meta_db() { db_.reset(); }
void meta_db::release_resources() {
default_family_.reset();
source_family_.reset();
keys_family_.reset();
db_.reset();
auto meta_db::create_iterator(bool source_family) const
-> std::shared_ptr<rocksdb::Iterator> {
return std::shared_ptr<rocksdb::Iterator>(
db_->NewIterator(rocksdb::ReadOptions(),
source_family ? source_family_ : default_family_));
}
std::shared_ptr<rocksdb::Iterator> meta_db::create_iterator(const bool &source_family) {
return std::shared_ptr<rocksdb::Iterator>(db_->NewIterator(
rocksdb::ReadOptions(), source_family ? source_family_.get() : default_family_.get()));
}
api_error meta_db::get_api_path_from_key(const std::string &key, std::string &api_path) const {
auto ret = api_error::item_not_found;
if (not key.empty()) {
if (db_->Get(rocksdb::ReadOptions(), keys_family_.get(), key, &api_path).ok()) {
ret = api_error::success;
}
auto meta_db::get_api_path_from_key(const std::string &key,
std::string &api_path) const -> api_error {
if (key.empty()) {
return api_error::item_not_found;
}
return ret;
return perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
return db_->Get(rocksdb::ReadOptions(), keys_family_, key, &api_path);
});
}
api_error meta_db::get_api_path_from_source(const std::string &source_path,
std::string &api_path) const {
auto ret = api_error::item_not_found;
if (not source_path.empty()) {
if (db_->Get(rocksdb::ReadOptions(), source_family_.get(), source_path, &api_path).ok()) {
ret = api_error::success;
}
auto meta_db::get_api_path_from_source(const std::string &source_path,
std::string &api_path) const
-> api_error {
if (source_path.empty()) {
return api_error::item_not_found;
}
return ret;
return perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
return db_->Get(rocksdb::ReadOptions(), source_family_, source_path,
&api_path);
});
}
api_error meta_db::get_item_meta_json(const std::string &api_path, json &json_data) const {
auto ret = api_error::item_not_found;
auto meta_db::get_item_meta_json(const std::string &api_path,
json &json_data) const -> api_error {
std::string value;
if (db_->Get(rocksdb::ReadOptions(), default_family_.get(), api_path, &value).ok()) {
json_data = json::parse(value);
ret = api_error::success;
const auto res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
return db_->Get(rocksdb::ReadOptions(), default_family_, api_path, &value);
});
if (res != api_error::success) {
return res;
}
return ret;
json_data = json::parse(value);
return api_error::success;
}
api_error meta_db::get_item_meta(const std::string &api_path, api_meta_map &meta) const {
auto meta_db::get_item_meta(const std::string &api_path,
api_meta_map &meta) const -> api_error {
json json_data;
const auto ret = get_item_meta_json(api_path, json_data);
if (ret == api_error::success) {
for (auto it = json_data.begin(); it != json_data.end(); it++) {
meta.insert({it.key(), it.value().get<std::string>()});
meta[it.key()] = it.value().get<std::string>();
}
}
return ret;
}
api_error meta_db::get_item_meta(const std::string &api_path, const std::string &key,
std::string &value) const {
auto meta_db::get_item_meta(const std::string &api_path, const std::string &key,
std::string &value) const -> api_error {
json json_data;
const auto ret = get_item_meta_json(api_path, json_data);
if (ret == api_error::success) {
@@ -116,20 +122,21 @@ api_error meta_db::get_item_meta(const std::string &api_path, const std::string
return ret;
}
bool meta_db::get_item_meta_exists(const std::string &api_path) const {
auto meta_db::get_item_meta_exists(const std::string &api_path) const -> bool {
std::string value;
return db_->Get(rocksdb::ReadOptions(), api_path, &value).ok();
}
std::vector<std::string> meta_db::get_pinned_files() const {
auto meta_db::get_pinned_files() const -> std::vector<std::string> {
std::vector<std::string> ret;
auto iterator = const_cast<meta_db *>(this)->create_iterator(false);
for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
auto api_path = iterator->key().ToString();
std::string pinned;
get_item_meta(api_path, META_PINNED, pinned);
if (not pinned.empty() && utils::string::to_bool(pinned)) {
const auto res = get_item_meta(api_path, META_PINNED, pinned);
if ((res == api_error::success) && not pinned.empty() &&
utils::string::to_bool(pinned)) {
ret.emplace_back(api_path);
}
}
@@ -137,96 +144,154 @@ std::vector<std::string> meta_db::get_pinned_files() const {
return ret;
}
bool meta_db::get_source_path_exists(const std::string &sourceFilePath) const {
auto meta_db::get_source_path_exists(const std::string &source_path) const
-> bool {
std::string value;
return db_->Get(rocksdb::ReadOptions(), source_family_.get(), sourceFilePath, &value).ok();
return db_->Get(rocksdb::ReadOptions(), source_family_, source_path, &value)
.ok();
}
void meta_db::remove_item_meta(const std::string &api_path) {
std::string source;
get_item_meta(api_path, META_SOURCE, source);
auto meta_db::perform_action(
const std::string &function_name,
const std::function<rocksdb::Status()> &action) const -> api_error {
const auto res = action();
if (res.ok()) {
return api_error::success;
}
std::string key;
get_item_meta(api_path, META_SOURCE, key);
if (not res.IsNotFound()) {
utils::error::raise_error(function_name, res.ToString());
}
db_->Delete(rocksdb::WriteOptions(), source_family_.get(), source);
db_->Delete(rocksdb::WriteOptions(), default_family_.get(), api_path);
db_->Delete(rocksdb::WriteOptions(), keys_family_.get(), key);
return res.IsNotFound() ? api_error::item_not_found : api_error::error;
}
api_error meta_db::remove_item_meta(const std::string &api_path, const std::string &key) {
auto meta_db::get_total_item_count() const -> std::uint64_t {
std::uint64_t ret = 0u;
auto iter = create_iterator(false);
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
ret++;
}
return ret;
}
auto meta_db::remove_item_meta(const std::string &api_path) -> api_error {
json json_data;
auto ret = get_item_meta_json(api_path, json_data);
auto res = get_item_meta_json(api_path, json_data);
if (res != api_error::success) {
return res == api_error::item_not_found ? api_error::success : res;
}
if (ret == api_error::success) {
if ((key == META_KEY) && not json_data[META_KEY].empty()) {
db_->Delete(rocksdb::WriteOptions(), keys_family_.get(),
json_data[META_KEY].get<std::string>());
if (not json_data[META_KEY].empty()) {
if ((res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
return db_->Delete(rocksdb::WriteOptions(), keys_family_,
json_data[META_KEY].get<std::string>());
})) != api_error::success) {
return res;
}
json_data.erase(key);
ret = db_->Put(rocksdb::WriteOptions(), default_family_.get(), api_path, json_data.dump()).ok()
? api_error::success
: api_error::error;
}
return ret;
if (not json_data[META_SOURCE].empty()) {
if ((res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
return db_->Delete(rocksdb::WriteOptions(), source_family_,
json_data[META_SOURCE].get<std::string>());
})) != api_error::success) {
return res;
}
}
return perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
return db_->Delete(rocksdb::WriteOptions(), default_family_, api_path);
});
}
api_error meta_db::rename_item_meta(const std::string &source_path,
const std::string &from_api_path,
const std::string &to_api_path) {
auto ret = api_error::success;
std::string key;
get_item_meta(from_api_path, META_KEY, key);
if (not key.empty()) {
remove_item_meta(from_api_path, META_KEY);
}
std::string value;
if (db_->Get(rocksdb::ReadOptions(), default_family_.get(), from_api_path, &value).ok()) {
db_->Delete(rocksdb::WriteOptions(), default_family_.get(), from_api_path);
db_->Put(rocksdb::WriteOptions(), default_family_.get(), to_api_path, value);
}
if (not key.empty()) {
set_item_meta(to_api_path, META_KEY, key);
}
db_->Put(rocksdb::WriteOptions(), source_family_.get(), source_path, to_api_path);
return ret;
}
api_error meta_db::set_item_meta(const std::string &api_path, const std::string &key,
const std::string &value) {
if (key == META_KEY) {
remove_item_meta(api_path, META_KEY);
}
auto meta_db::remove_item_meta(const std::string &api_path,
const std::string &key) -> api_error {
json json_data;
get_item_meta_json(api_path, json_data);
json_data[key] = value;
auto ret =
db_->Put(rocksdb::WriteOptions(), default_family_.get(), api_path, json_data.dump()).ok()
? api_error::success
: api_error::error;
if ((key == META_KEY) && (ret == api_error::success)) {
ret = db_->Put(rocksdb::WriteOptions(), keys_family_.get(), value, api_path).ok()
? api_error::success
: api_error::error;
auto res = get_item_meta_json(api_path, json_data);
if (res != api_error::success) {
return res == api_error::item_not_found ? api_error::success : res;
}
return ret;
if ((key == META_KEY) && not json_data[META_KEY].empty()) {
if ((res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
return db_->Delete(rocksdb::WriteOptions(), keys_family_,
json_data[META_KEY].get<std::string>());
})) != api_error::success) {
return res;
}
}
if ((key == META_SOURCE) && not json_data[META_SOURCE].empty()) {
if ((res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
return db_->Delete(rocksdb::WriteOptions(), source_family_,
json_data[META_SOURCE].get<std::string>());
})) != api_error::success) {
return res;
}
}
json_data.erase(key);
return perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
return db_->Put(rocksdb::WriteOptions(), default_family_, api_path,
json_data.dump());
});
}
api_error meta_db::set_item_meta(const std::string &api_path, const api_meta_map &meta) {
auto meta_db::rename_item_meta(const std::string &source_path,
const std::string &from_api_path,
const std::string &to_api_path) -> api_error {
api_meta_map meta{};
auto res = get_item_meta(from_api_path, meta);
if (res != api_error::success) {
return res;
}
if ((res = remove_item_meta(from_api_path)) != api_error::success) {
return res;
}
if (not source_path.empty()) {
meta[META_SOURCE] = source_path;
}
return set_item_meta(to_api_path, meta);
}
auto meta_db::set_item_meta(const std::string &api_path, const std::string &key,
const std::string &value) -> api_error {
if (key == META_SOURCE) {
return set_source_path(api_path, value);
}
if (key == META_KEY) {
const auto res = remove_item_meta(api_path, META_KEY);
if ((res != api_error::success) && (res != api_error::item_not_found)) {
return res;
}
}
auto res = store_item_meta(api_path, key, value);
if (res != api_error::success) {
return res;
}
if (key == META_KEY) {
return perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
return db_->Put(rocksdb::WriteOptions(), keys_family_, value, api_path);
});
}
return api_error::success;
}
auto meta_db::set_item_meta(const std::string &api_path,
const api_meta_map &meta) -> api_error {
auto ret = api_error::success;
auto it = meta.begin();
for (std::size_t i = 0u; (ret == api_error::success) && (i < meta.size()); i++) {
for (std::size_t i = 0u; (ret == api_error::success) && (i < meta.size());
i++) {
ret = set_item_meta(api_path, it->first, it->second);
it++;
}
@@ -234,18 +299,49 @@ api_error meta_db::set_item_meta(const std::string &api_path, const api_meta_map
return ret;
}
api_error meta_db::set_source_path(const std::string &api_path, const std::string &source_path) {
auto meta_db::set_source_path(const std::string &api_path,
const std::string &source_path) -> api_error {
std::string current_source_path;
get_item_meta(api_path, META_SOURCE, current_source_path);
auto res = get_item_meta(api_path, META_SOURCE, current_source_path);
if ((res != api_error::success) && (res != api_error::item_not_found)) {
return res;
}
// TODO multiple db ops should be in transaction
if (not current_source_path.empty()) {
db_->Delete(rocksdb::WriteOptions(), source_family_.get(), current_source_path);
if ((res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
return db_->Delete(rocksdb::WriteOptions(), source_family_,
current_source_path);
})) != api_error::success) {
return res;
}
}
auto ret = set_item_meta(api_path, META_SOURCE, source_path);
if (ret == api_error::success) {
db_->Put(rocksdb::WriteOptions(), source_family_.get(), source_path, api_path);
if (not source_path.empty()) {
if ((res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
return db_->Put(rocksdb::WriteOptions(), source_family_, source_path,
api_path);
})) != api_error::success) {
return res;
}
}
return ret;
return store_item_meta(api_path, META_SOURCE, source_path);
}
auto meta_db::store_item_meta(const std::string &api_path,
const std::string &key, const std::string &value)
-> api_error {
json json_data;
auto res = get_item_meta_json(api_path, json_data);
if ((res != api_error::success) && (res != api_error::item_not_found)) {
return res;
}
json_data[key] = value;
return perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
return db_->Put(rocksdb::WriteOptions(), default_family_, api_path,
json_data.dump());
});
}
} // namespace repertory

81
src/db/retry_db.cpp
View File

@@ -1,81 +0,0 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#include "db/retry_db.hpp"
namespace repertory {
retry_db::retry_db(const app_config &config) {
utils::db::create_rocksdb(config, ROCKS_DB_NAME, db_);
}
retry_db::~retry_db() { db_.reset(); }
bool retry_db::exists(const std::string &api_path) const {
std::string value;
return db_->Get(rocksdb::ReadOptions(), api_path, &value).ok();
}
void retry_db::pause() {
mutex_lock processing_lock(processing_mutex_);
paused_ = true;
}
bool retry_db::process_all(const process_callback &process) {
auto processed = false;
if (not paused_) {
unique_mutex_lock processing_lock(processing_mutex_);
if (not paused_) {
auto iterator = std::unique_ptr<rocksdb::Iterator>(db_->NewIterator(rocksdb::ReadOptions()));
for (iterator->SeekToFirst(); not paused_ && iterator->Valid(); iterator->Next()) {
const auto api_path = iterator->value().ToString();
if (process(api_path)) {
remove(api_path);
processed = true;
}
processing_lock.unlock();
std::this_thread::sleep_for(1ms);
processing_lock.lock();
}
}
processing_lock.unlock();
}
return processed;
}
void retry_db::remove(const std::string &api_path) {
db_->Delete(rocksdb::WriteOptions(), api_path);
}
void retry_db::rename(const std::string &from_api_path, const std::string &to_api_path) {
if (exists(from_api_path)) {
remove(from_api_path);
set(to_api_path);
}
}
void retry_db::resume() {
mutex_lock processing_lock(processing_mutex_);
paused_ = false;
}
void retry_db::set(const std::string &api_path) {
db_->Put(rocksdb::WriteOptions(), api_path, api_path);
}
} // namespace repertory

204
src/download/buffered_reader.cpp
View File

@@ -1,204 +0,0 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#include "download/buffered_reader.hpp"
#include "app_config.hpp"
#include "download/reader_pool.hpp"
namespace repertory {
buffered_reader::buffered_reader(const app_config &config, const filesystem_item &fsi,
const api_reader_callback &api_reader,
const std::size_t &chunk_size, const std::size_t &total_chunks,
const std::size_t &start_chunk)
: fsi_(fsi),
api_reader_(api_reader),
chunk_size_(chunk_size),
total_chunks_(total_chunks),
ring_state_(config.get_read_ahead_count()) {
ring_data_.resize(ring_state_.size());
const auto read_chunk = [this](const std::size_t &read_offset,
std::unique_ptr<std::vector<char>> &chunk_ptr) -> api_error {
auto ret = api_error::success;
const auto read_size = utils::calculate_read_size(fsi_.size, chunk_size_, read_offset);
if (read_size) {
std::vector<char> data;
if ((ret = api_reader_(fsi_.api_path, read_size, read_offset + read_offset_, data,
stop_requested_)) == api_error::success) {
chunk_ptr = std::make_unique<std::vector<char>>(std::move(data));
}
}
return ret;
};
auto first = std::async(std::launch::async, [this, &read_chunk]() -> api_error {
return read_chunk(0u, first_chunk_data_);
});
if (total_chunks_ > 1u) {
auto last = std::async(std::launch::async, [this, &read_chunk]() -> api_error {
return read_chunk(((total_chunks_ - 1u) * chunk_size_), last_chunk_data_);
});
error_ = last.get();
}
if (error_ == api_error::success) {
if ((error_ = first.get()) == api_error::success) {
read_chunk_index_ = write_chunk_index_ =
(first_chunk_data_ ? ((start_chunk == 0u) ? 1u : start_chunk) : start_chunk);
if (ring_data_.size() > 1u) {
reader_thread_ = std::make_unique<std::thread>([this]() { this->reader_thread(); });
}
}
}
}
buffered_reader::~buffered_reader() {
notify_stop_requested();
unique_mutex_lock write_lock(write_mutex_);
read_notify_.notify_all();
write_lock.unlock();
if (reader_thread_) {
reader_thread_->join();
reader_thread_.reset();
}
}
void buffered_reader::notify_stop_requested() {
error_ = api_error::download_stopped;
stop_requested_ = true;
}
api_error buffered_reader::read_chunk(const std::size_t &chunk_index, std::vector<char> &data) {
if (error_ == api_error::success) {
data.clear();
if (last_chunk_data_ && (chunk_index == (total_chunks_ - 1u))) {
data = *last_chunk_data_;
} else if (first_chunk_data_ && (chunk_index == 0u)) {
data = *first_chunk_data_;
} else if (ring_state_.size() == 1u) {
const auto read_offset = chunk_index * chunk_size_;
std::size_t read_size = 0u;
if ((read_size = utils::calculate_read_size(fsi_.size, chunk_size_, read_offset)) > 0u) {
error_ = api_reader_(fsi_.api_path, read_size, read_offset + read_offset_, data,
stop_requested_);
}
} else {
mutex_lock read_lock(read_mutex_);
const auto read_position = chunk_index % ring_state_.size();
unique_mutex_lock write_lock(write_mutex_);
while ((reset_reader_ ||
((chunk_index < read_chunk_index_) || (chunk_index > write_chunk_index_))) &&
is_active()) {
reset_reader_ = true;
read_chunk_index_ = write_chunk_index_ = chunk_index;
read_notify_.notify_all();
read_notify_.wait(write_lock);
}
if (is_active()) {
read_chunk_index_ = chunk_index;
read_notify_.notify_all();
while (not ring_state_[read_position] && is_active()) {
read_notify_.wait(write_lock);
}
data = ring_data_[read_position];
}
read_notify_.notify_all();
write_lock.unlock();
}
}
return error_;
}
void buffered_reader::reader_thread() {
reader_pool pool(ring_state_.size(), api_reader_);
unique_mutex_lock write_lock(write_mutex_);
read_notify_.notify_all();
write_lock.unlock();
while (is_active()) {
write_lock.lock();
while ((write_chunk_index_ > read_chunk_index_) &&
(write_chunk_index_ > (read_chunk_index_ + ring_state_.size() - 1u)) &&
not reset_reader_ && is_active()) {
read_notify_.wait(write_lock);
}
if (is_active()) {
if (reset_reader_) {
read_notify_.notify_all();
write_lock.unlock();
pool.restart();
write_lock.lock();
write_chunk_index_ = read_chunk_index_;
ring_state_.reset();
reset_reader_ = false;
}
const auto file_offset = write_chunk_index_ * chunk_size_;
const auto write_chunk_index = write_chunk_index_;
const auto write_position = write_chunk_index_ % ring_state_.size();
ring_state_[write_position] = false;
read_notify_.notify_all();
write_lock.unlock();
const auto read_size = utils::calculate_read_size(fsi_.size, chunk_size_, file_offset);
if (read_size > 0u) {
if ((first_chunk_data_ && (write_chunk_index == 0u)) ||
(last_chunk_data_ && (write_chunk_index == (total_chunks_ - 1u)))) {
write_lock.lock();
write_chunk_index_++;
} else {
const auto completed = [this, write_position](api_error error) {
mutex_lock write_lock(write_mutex_);
if (error == api_error::success) {
ring_state_[write_position] = true;
} else if (not reset_reader_) {
error_ = error;
}
read_notify_.notify_all();
};
pool.queue_read_bytes(fsi_.api_path, read_size,
((write_chunk_index * chunk_size_) + read_offset_),
ring_data_[write_position], completed);
write_lock.lock();
write_chunk_index_++;
}
} else {
write_lock.lock();
while (not reset_reader_ && ((write_chunk_index_ * chunk_size_) >= fsi_.size) &&
is_active()) {
read_notify_.wait(write_lock);
}
}
}
read_notify_.notify_all();
write_lock.unlock();
}
}
} // namespace repertory

116
src/download/direct_download.cpp
View File

@@ -1,116 +0,0 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#include "download/direct_download.hpp"
#include "app_config.hpp"
#include "download/buffered_reader.hpp"
#include "download/events.hpp"
#include "download/utils.hpp"
#include "events/event_system.hpp"
#include "utils/encrypting_reader.hpp"
namespace repertory {
direct_download::direct_download(const app_config &config, filesystem_item fsi,
const api_reader_callback &api_reader, const std::uint64_t &handle)
: config_(config), fsi_(std::move(fsi)), api_reader_(api_reader), handle_(handle) {}
direct_download::~direct_download() {
stop_requested_ = true;
unique_mutex_lock read_lock(read_mutex_);
if (buffered_reader_) {
buffered_reader_->notify_stop_requested();
}
read_lock.unlock();
if (buffered_reader_) {
buffered_reader_.reset();
}
}
void direct_download::notify_download_end() {
unique_mutex_lock read_lock(read_mutex_);
if (not disable_download_end_ && not download_end_notified_) {
download_end_notified_ = true;
read_lock.unlock();
event_system::instance().raise<download_end>(fsi_.api_path, "direct", handle_, error_);
}
}
void direct_download::notify_stop_requested() {
set_api_error(api_error::download_stopped);
stop_requested_ = true;
notify_download_end();
}
api_error direct_download::read_bytes(const std::uint64_t &, std::size_t read_size,
const std::uint64_t &read_offset, std::vector<char> &data) {
data.clear();
if ((read_size = utils::calculate_read_size(fsi_.size, read_size, read_offset)) > 0u) {
unique_mutex_lock read_lock(read_mutex_);
if (is_active()) {
if (not buffered_reader_) {
const auto chunk_size = utils::encryption::encrypting_reader::get_data_chunk_size();
const auto read_chunk = static_cast<std::size_t>(read_offset / chunk_size);
event_system::instance().raise<download_begin>(fsi_.api_path, "direct");
buffered_reader_ = std::make_unique<buffered_reader>(
config_, fsi_, api_reader_, chunk_size,
static_cast<std::size_t>(
utils::divide_with_ceiling(fsi_.size, static_cast<std::uint64_t>(chunk_size))),
read_chunk);
}
auto read_chunk = static_cast<std::size_t>(read_offset / buffered_reader_->get_chunk_size());
auto offset = static_cast<std::size_t>(read_offset % buffered_reader_->get_chunk_size());
while (is_active() && (read_size > 0u)) {
std::vector<char> buffer;
auto ret = api_error::success;
if ((ret = buffered_reader_->read_chunk(read_chunk, buffer)) == api_error::success) {
const auto total_read = std::min(buffered_reader_->get_chunk_size() - offset, read_size);
data.insert(data.end(), buffer.begin() + offset, buffer.begin() + offset + total_read);
buffer.clear();
read_chunk++;
offset = 0u;
read_size -= total_read;
} else {
set_api_error(ret);
read_lock.unlock();
notify_download_end();
read_lock.lock();
}
}
if (is_active()) {
utils::download::notify_progress<download_progress>(
config_, fsi_.api_path, "direct", static_cast<double>(data.size() + read_offset),
static_cast<double>(fsi_.size), progress_);
}
}
}
return error_;
}
void direct_download::set_api_error(const api_error &error) {
if ((error_ == api_error::success) && (error_ != error)) {
error_ = error;
}
}
} // namespace repertory

855
src/download/download.cpp
View File

@@ -1,855 +0,0 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#include "download/download.hpp"
#include "download/events.hpp"
#include "download/utils.hpp"
#include "drives/i_open_file_table.hpp"
#include "events/event_system.hpp"
#include "providers/i_provider.hpp"
#include "utils/file_utils.hpp"
#include "utils/global_data.hpp"
#include "utils/path_utils.hpp"
#include "utils/utils.hpp"
namespace repertory {
download::download(const app_config &config, filesystem_item &fsi,
const api_reader_callback &api_reader, const std::size_t &chunk_size,
i_open_file_table &oft)
: config_(config),
fsi_(fsi),
api_reader_(api_reader),
oft_(oft),
chunk_size_(chunk_size),
read_chunk_state_(static_cast<std::size_t>(
utils::divide_with_ceiling(fsi.size, static_cast<std::uint64_t>(chunk_size)))),
last_chunk_size_(static_cast<std::size_t>(fsi.size <= chunk_size ? fsi.size
: fsi.size % chunk_size ? fsi.size % chunk_size
: chunk_size)),
write_chunk_state_(static_cast<std::size_t>(
utils::divide_with_ceiling(fsi.size, static_cast<std::uint64_t>(chunk_size)))) {
/* std::cout << "size:" << fsi.size << ":last:" << last_chunk_size_ << std::endl; */
initialize_download(fsi, true);
}
download::download(const app_config &config, filesystem_item &fsi,
const api_reader_callback &api_reader, const std::size_t &chunk_size,
std::size_t &last_chunk_size, boost::dynamic_bitset<> &read_state,
boost::dynamic_bitset<> &write_state, i_open_file_table &oft)
: config_(config),
fsi_(fsi),
api_reader_(api_reader),
oft_(oft),
chunk_size_(chunk_size),
read_chunk_state_(read_state),
last_chunk_size_(last_chunk_size),
write_chunk_state_(write_state) {
initialize_download(fsi, false);
}
download::~download() {
if (not get_complete()) {
notify_stop_requested();
}
if (io_thread_) {
io_thread_->join();
}
if (auto_close_) {
native_file::attach(fsi_.handle)->close();
fsi_.handle = REPERTORY_INVALID_HANDLE;
}
}
api_error download::allocate(const std::uint64_t &, const std::uint64_t &size,
const allocator_callback &allocator,
const completer_callback &completer) {
reset_timeout(false);
/* std::cout << "allocate called" << std::endl; */
if (size == fsi_.size) {
return api_error::success;
}
if (size > fsi_.size) {
download_chunk(read_chunk_state_.size() - 1u, false);
} else {
download_chunk(static_cast<std::size_t>(
utils::divide_with_ceiling(size, static_cast<std::uint64_t>(chunk_size_))),
false);
}
if (error_ == api_error::success) {
const auto original_size = fsi_.size;
unique_mutex_lock read_write_lock(read_write_mutex_);
const auto allocate_local_file = [&]() -> api_error {
if ((error_ = allocator()) == api_error::success) {
utils::file::get_file_size(fsi_.source_path, fsi_.size);
completer(original_size, fsi_.size, true);
oft_.force_schedule_upload(fsi_);
}
read_write_notify_.notify_all();
read_write_lock.unlock();
return error_;
};
if (processed_) {
return allocate_local_file();
}
read_write_notify_.notify_all();
read_write_lock.unlock();
pause();
read_write_lock.lock();
if (processed_) {
/* std::cout << "recursive allocate" << std::endl; */
return allocate_local_file();
}
if ((error_ = allocator()) == api_error::success) {
utils::file::get_file_size(fsi_.source_path, fsi_.size);
const auto end_chunk_index = static_cast<std::size_t>(
utils::divide_with_ceiling(fsi_.size, static_cast<std::uint64_t>(chunk_size_)));
if (end_chunk_index >= read_chunk_state_.size()) {
write_chunk_state_.resize(end_chunk_index + 1u);
read_chunk_state_.resize(end_chunk_index + 1u);
}
last_chunk_size_ =
static_cast<std::size_t>(fsi_.size <= chunk_size_ ? fsi_.size
: fsi_.size % chunk_size_ ? fsi_.size % chunk_size_
: chunk_size_);
read_chunk_state_[end_chunk_index] = write_chunk_state_[end_chunk_index] = true;
completer(original_size, fsi_.size, true);
}
read_write_notify_.notify_all();
read_write_lock.unlock();
resume();
}
return error_;
}
void download::create_active_chunk(std::size_t chunk_index) {
auto reader = [this, chunk_index]() {
const auto chunk_read_size =
(chunk_index == (read_chunk_state_.size() - 1u)) ? last_chunk_size_ : chunk_size_;
const auto chunk_read_offset = chunk_index * chunk_size_;
api_error ret;
if (not get_complete()) {
std::vector<char> buffer;
if ((ret = api_reader_(fsi_.api_path, chunk_read_size, chunk_read_offset + read_offset_,
buffer, stop_requested_)) == api_error::success) {
unique_mutex_lock read_write_lock(read_write_mutex_);
if (not get_complete()) {
write_queue_.emplace_back(
std::make_shared<write_data>(chunk_index, std::move(buffer), chunk_read_offset));
}
read_write_notify_.notify_all();
read_write_lock.unlock();
}
if (ret != api_error::success) {
error_ = ((error_ == api_error::success) ? ret : error_);
mutex_lock read_write_lock(read_write_mutex_);
read_write_notify_.notify_all();
}
}
};
active_chunks_.insert({chunk_index, std::make_shared<active_chunk>(std::thread(reader))});
}
void download::download_chunk(std::size_t chunk_index, bool inactive_only) {
unique_mutex_lock read_write_lock(read_write_mutex_);
const auto should_download = [&]() -> bool {
return (chunk_index < read_chunk_state_.size()) && not read_chunk_state_[chunk_index] &&
not get_complete();
};
if (should_download()) {
while (paused_ && not get_complete()) {
read_write_notify_.notify_all();
read_write_notify_.wait(read_write_lock);
}
if (should_download()) {
auto created = false;
if (active_chunks_.find(chunk_index) == active_chunks_.end()) {
create_active_chunk(chunk_index);
created = true;
}
if (not inactive_only || created) {
auto chunk = active_chunks_[chunk_index];
read_write_notify_.notify_all();
read_write_lock.unlock();
if (not inactive_only) {
reset_timeout(false);
}
utils::spin_wait_for_mutex(
[this, chunk_index]() -> bool {
return read_chunk_state_[chunk_index] || get_complete();
},
chunk->notify, chunk->mutex, "download_chunk");
}
}
}
}
api_error download::download_all() {
const auto total_chunks = read_chunk_state_.size();
for (std::size_t chunk = 0u; not get_complete() && (chunk < total_chunks); chunk++) {
download_chunk(chunk, false);
}
utils::spin_wait_for_mutex(processed_, processed_notify_, read_write_mutex_, "download_all");
return error_;
}
bool download::get_complete() const {
return read_chunk_state_.all() || (error_ != api_error::success) || stop_requested_ || processed_;
}
bool download::get_timeout_enabled() const {
return not paused_ && not write_chunk_state_.any() &&
config_.get_enable_chunk_download_timeout() && (oft_.get_open_count(fsi_.api_path) == 0u);
}
void download::get_state_information(filesystem_item &fsi, std::size_t &chunk_size,
std::size_t &last_chunk_size,
boost::dynamic_bitset<> &read_state,
boost::dynamic_bitset<> &write_state) {
fsi = fsi_;
chunk_size = chunk_size_;
last_chunk_size = last_chunk_size_;
read_state = read_chunk_state_;
write_state = write_chunk_state_;
}
void download::handle_active_chunk_complete(std::size_t chunk_index, unique_mutex_lock &lock) {
lock.lock();
auto activeChunk = active_chunks_[chunk_index];
active_chunks_.erase(chunk_index);
lock.unlock();
unique_mutex_lock completed_lock(activeChunk->mutex);
activeChunk->notify.notify_all();
completed_lock.unlock();
activeChunk->worker.join();
}
void download::initialize_download(filesystem_item &fsi, const bool &delete_existing) {
const auto current_source_path = fsi.source_path;
if (delete_existing) {
// Handle should always be invalid (just in case it's not - no leaking).
unique_recur_mutex_lock item_lock(*fsi.lock);
if (fsi.handle != REPERTORY_INVALID_HANDLE) {
native_file::attach(fsi.handle)->close();
fsi.handle = REPERTORY_INVALID_HANDLE;
}
item_lock.unlock();
// Update current open file information to non-existing source file and free open handle.
fsi.source_path =
utils::path::combine(config_.get_cache_directory(), {utils::create_uuid_string()});
fsi.source_path_changed = true;
// Update cache space if existing file size is >0 and delete
std::uint64_t file_size = 0u;
utils::file::get_file_size(current_source_path, file_size);
if (file_size) {
global_data::instance().update_used_space(file_size, 0u, true);
}
utils::file::delete_file(current_source_path);
// Update completed file information and create new lock mutex
fsi_.source_path =
utils::path::combine(config_.get_cache_directory(), {utils::create_uuid_string()});
fsi_.lock = std::make_shared<std::recursive_mutex>();
}
// For resume-only, open file to report correct file size in fuse/winfsp operations
else if ((error_ = oft_.open(fsi_, open_file_handle_)) != api_error::success) {
event_system::instance().raise<download_restore_failed>(
fsi_.api_path, fsi_.source_path, "open failed: " + api_error_to_string(error_));
}
if (error_ == api_error::success) {
event_system::instance().raise<download_begin>(fsi_.api_path, fsi_.source_path);
// Create new file for reading and writing
if ((error_ = native_file::create_or_open(fsi_.source_path, read_write_file_)) ==
api_error::success) {
// Pre-allocate full file and update used cache space
if (read_write_file_->allocate(fsi_.size)) {
if (not delete_existing) {
// Update used cache space
global_data::instance().update_used_space(0u, fsi_.size, true);
}
// Launch read-ahead workers
for (std::uint8_t i = 0u; i < config_.get_read_ahead_count(); i++) {
background_workers_.emplace_back(std::thread([this]() { read_ahead_worker(); }));
}
// Launch read-behind worker
background_workers_.emplace_back(std::thread([this]() { read_behind_worker(); }));
// Launch read-end worker
if (fsi.size > (chunk_size_ * config_.get_read_ahead_count())) {
background_workers_.emplace_back(std::thread([this]() { read_end_worker(); }));
}
// Launch read/write IO worker
io_thread_ = std::make_unique<std::thread>([this]() { io_data_worker(); });
} else if (delete_existing) {
const auto temp_error_code = utils::get_last_error_code();
utils::file::delete_file(fsi_.source_path);
utils::set_last_error_code(temp_error_code);
error_ = api_error::os_error;
}
}
if (error_ != api_error::success) {
event_system::instance().raise<download_end>(fsi_.api_path, fsi_.source_path, 0u, error_);
}
}
}
void download::io_data_worker() {
unique_mutex_lock read_write_lock(read_write_mutex_);
read_write_lock.unlock();
do {
process_timeout(read_write_lock);
process_read_queue(read_write_lock);
process_write_queue(read_write_lock);
notify_progress();
process_download_complete(read_write_lock);
wait_for_io(read_write_lock);
} while (not processed_);
shutdown(read_write_lock);
}
void download::notify_progress() {
if (read_chunk_state_.any()) {
utils::download::notify_progress<download_progress>(
config_, fsi_.api_path, fsi_.source_path, static_cast<double>(read_chunk_state_.count()),
static_cast<double>(read_chunk_state_.size()), progress_);
}
}
void download::notify_stop_requested() {
if (not stop_requested_) {
unique_mutex_lock read_write_lock(read_write_mutex_);
error_ =
write_chunk_state_.any() ? api_error::download_incomplete : api_error::download_stopped;
stop_requested_ = true;
paused_ = false;
read_write_notify_.notify_all();
read_write_lock.unlock();
}
}
bool download::pause() {
unique_mutex_lock read_write_lock(read_write_mutex_);
if (not paused_) {
paused_ = true;
while (not active_chunks_.empty() && not stop_requested_) {
read_write_notify_.notify_all();
read_write_notify_.wait_for(read_write_lock, 2s);
}
read_write_notify_.notify_all();
read_write_lock.unlock();
if (stop_requested_) {
paused_ = false;
} else {
event_system::instance().raise<download_paused>(fsi_.api_path, fsi_.source_path);
}
return paused_;
}
return true;
}
void download::process_download_complete(unique_mutex_lock &lock) {
if (get_complete() && not processed_) {
lock.lock();
if (not processed_) {
if ((error_ == api_error::success) && read_chunk_state_.all()) {
const auto assign_handle_and_auto_close = [this]() {
if (fsi_.handle != REPERTORY_INVALID_HANDLE) {
native_file::attach(fsi_.handle)->close();
fsi_.handle = REPERTORY_INVALID_HANDLE;
}
fsi_.handle = read_write_file_->get_handle();
auto_close_ = true;
};
if (not oft_.perform_locked_operation(
[this, &assign_handle_and_auto_close](i_open_file_table &oft,
i_provider &provider) -> bool {
filesystem_item *fsi = nullptr;
if (oft.get_open_file(fsi_.api_path, fsi)) {
unique_recur_mutex_lock item_lock(*fsi->lock);
// Handle should always be invalid (just in case it's not - no leaking).
if (fsi->handle != REPERTORY_INVALID_HANDLE) {
native_file::attach(fsi->handle)->close();
fsi->handle = REPERTORY_INVALID_HANDLE;
}
fsi->handle = read_write_file_->get_handle();
item_lock.unlock();
fsi->source_path = fsi_.source_path;
fsi->source_path_changed = true;
fsi->changed = false;
fsi_ = *fsi;
} else {
provider.set_source_path(fsi_.api_path, fsi_.source_path);
assign_handle_and_auto_close();
}
return true;
})) {
assign_handle_and_auto_close();
}
} else if (error_ == api_error::success) {
error_ = api_error::download_failed;
}
stop_requested_ = processed_ = true;
read_write_notify_.notify_all();
lock.unlock();
process_write_queue(lock);
process_read_queue(lock);
lock.lock();
if (error_ != api_error::success) {
read_write_file_->close();
if (error_ != api_error::download_incomplete) {
if (utils::file::delete_file(fsi_.source_path)) {
global_data::instance().update_used_space(fsi_.size, 0u, true);
}
}
}
processed_notify_.notify_all();
read_write_notify_.notify_all();
lock.unlock();
if (write_chunk_state_.any() && (error_ == api_error::success)) {
oft_.force_schedule_upload(fsi_);
}
if (open_file_handle_ != static_cast<std::uint64_t>(REPERTORY_API_INVALID_HANDLE)) {
oft_.close(open_file_handle_);
open_file_handle_ = REPERTORY_API_INVALID_HANDLE;
}
if (not disable_download_end_) {
event_system::instance().raise<download_end>(fsi_.api_path, fsi_.source_path, 0u, error_);
}
} else {
lock.unlock();
}
}
}
void download::process_read_queue(unique_mutex_lock &lock) {
lock.lock();
while (not read_queue_.empty()) {
auto rd = read_queue_.front();
read_queue_.pop_front();
lock.unlock();
if (error_ == api_error::success) {
std::size_t bytes_read = 0u;
if (not read_write_file_->read_bytes(&rd->data[0u], rd->data.size(), rd->offset,
bytes_read)) {
error_ = ((error_ == api_error::success) ? api_error::os_error : error_);
}
}
rd->complete = true;
unique_mutex_lock read_lock(rd->mutex);
rd->notify.notify_all();
read_lock.unlock();
lock.lock();
}
lock.unlock();
}
void download::process_timeout(unique_mutex_lock &lock) {
if ((std::chrono::system_clock::now() >= timeout_) && not get_complete() &&
get_timeout_enabled()) {
error_ = api_error::download_timeout;
stop_requested_ = true;
lock.lock();
read_write_notify_.notify_all();
lock.unlock();
event_system::instance().raise<download_timeout>(fsi_.api_path, fsi_.source_path);
}
}
void download::process_write_queue(unique_mutex_lock &lock) {
lock.lock();
do {
if (not write_queue_.empty()) {
auto wd = write_queue_.front();
write_queue_.pop_front();
lock.unlock();
if (not get_complete() || (error_ == api_error::download_incomplete)) {
auto &bytes_written = wd->written;
if (read_write_file_->write_bytes(&wd->data[0u], wd->data.size(), wd->offset,
bytes_written)) {
read_write_file_->flush();
if (wd->from_read) {
lock.lock();
read_chunk_state_[wd->chunk_index] = true;
lock.unlock();
} else {
const auto start_chunk_index = static_cast<std::size_t>(wd->offset / chunk_size_);
const auto end_chunk_index =
static_cast<std::size_t>((wd->data.size() + wd->offset) / chunk_size_);
lock.lock();
for (std::size_t chunk_index = start_chunk_index; (chunk_index <= end_chunk_index);
chunk_index++) {
write_chunk_state_[chunk_index] = true;
}
lock.unlock();
}
} else {
error_ = ((error_ == api_error::success) ? api_error::os_error : error_);
}
}
if (wd->from_read) {
handle_active_chunk_complete(wd->chunk_index, lock);
} else {
wd->complete = true;
unique_mutex_lock write_lock(wd->mutex);
wd->notify.notify_all();
write_lock.unlock();
}
lock.lock();
}
} while (not write_queue_.empty() && read_queue_.empty());
lock.unlock();
}
void download::read_ahead_worker() {
const auto total_chunks = read_chunk_state_.size();
auto first_chunk_index = current_chunk_index_;
const auto get_next_chunk =
[&first_chunk_index](const std::size_t &current_chunk_index,
const std::size_t &last_chunk_index) -> std::size_t {
if (current_chunk_index == first_chunk_index) {
return last_chunk_index + 1ul;
}
first_chunk_index = current_chunk_index;
return current_chunk_index + 1ul;
};
while (not get_complete()) {
for (auto chunk_index = get_next_chunk(current_chunk_index_, 0u);
(chunk_index < total_chunks) && not get_complete();) {
download_chunk(chunk_index, true);
chunk_index = get_next_chunk(current_chunk_index_, chunk_index);
}
utils::spin_wait_for_mutex(
[this, &first_chunk_index]() -> bool {
return get_complete() || (first_chunk_index != current_chunk_index_);
},
read_write_notify_, read_write_mutex_, "read_ahead_worker");
}
}
void download::read_behind_worker() {
auto first_chunk_index = current_chunk_index_;
const auto get_next_chunk =
[&first_chunk_index](const std::size_t &current_chunk_index,
const std::size_t &last_chunk_index) -> std::size_t {
if (current_chunk_index == first_chunk_index) {
return last_chunk_index ? last_chunk_index - 1ul : 0u;
}
first_chunk_index = current_chunk_index;
return current_chunk_index ? current_chunk_index - 1ul : 0u;
};
while (not get_complete()) {
for (auto chunk_index = get_next_chunk(current_chunk_index_, current_chunk_index_);
not get_complete();) {
download_chunk(chunk_index, true);
if (not chunk_index) {
break;
}
chunk_index = get_next_chunk(current_chunk_index_, chunk_index);
}
utils::spin_wait_for_mutex(
[this, &first_chunk_index]() -> bool {
return get_complete() || (first_chunk_index != current_chunk_index_);
},
read_write_notify_, read_write_mutex_, "read_behind_worker");
}
}
api_error download::read_bytes(const std::uint64_t &, std::size_t read_size,
const std::uint64_t &read_offset, std::vector<char> &data) {
/* std::cout << "read called:" << read_size << ':' << read_offset << std::endl; */
data.clear();
if ((read_size > 0u) && (error_ == api_error::success)) {
unique_mutex_lock read_write_lock(read_write_mutex_);
const auto read_local_file = [&]() -> api_error {
read_write_notify_.notify_all();
read_write_lock.unlock();
error_ = (error_ == api_error::success)
? utils::file::read_from_source(fsi_, read_size, read_offset, data)
: error_;
return error_;
};
if (processed_) {
return read_local_file();
}
read_write_notify_.notify_all();
read_write_lock.unlock();
const auto start_chunk_index = current_chunk_index_ =
static_cast<std::size_t>(read_offset / chunk_size_);
const auto end_chunk_index = static_cast<std::size_t>((read_size + read_offset) / chunk_size_);
for (std::size_t chunk = start_chunk_index; not get_complete() && (chunk <= end_chunk_index);
chunk++) {
download_chunk(chunk, false);
}
if (error_ == api_error::success) {
data.resize(read_size);
auto rd = std::make_shared<read_data>(data, read_offset);
read_write_lock.lock();
if (processed_) {
/* std::cout << "recursive read:" << read_size << ':' << read_offset << std::endl; */
return read_local_file();
}
read_queue_.emplace_front(rd);
read_write_notify_.notify_all();
read_write_lock.unlock();
reset_timeout(false);
utils::spin_wait_for_mutex(rd->complete, rd->notify, rd->mutex, "read_bytes");
reset_timeout(false);
}
}
return error_;
}
void download::read_end_worker() {
const auto total_chunks = read_chunk_state_.size();
auto to_read = utils::divide_with_ceiling(std::size_t(262144u), chunk_size_);
for (auto chunk_index = total_chunks ? total_chunks - 1ul : 0u;
chunk_index && to_read && not get_complete(); chunk_index--, to_read--) {
download_chunk(chunk_index, true);
}
}
void download::reset_timeout(const bool &) {
mutex_lock read_write_lock(read_write_mutex_);
timeout_ = std::chrono::system_clock::now() +
std::chrono::seconds(config_.get_chunk_downloader_timeout_secs());
read_write_notify_.notify_all();
}
void download::resume() {
reset_timeout(false);
mutex_lock read_write_lock(read_write_mutex_);
if (paused_) {
paused_ = false;
event_system::instance().raise<download_resumed>(fsi_.api_path, fsi_.source_path);
}
read_write_notify_.notify_all();
}
void download::set_api_path(const std::string &api_path) {
mutex_lock read_write_lock(read_write_mutex_);
fsi_.api_path = api_path;
fsi_.api_parent = utils::path::get_parent_api_path(api_path);
read_write_notify_.notify_all();
}
void download::shutdown(unique_mutex_lock &lock) {
lock.lock();
while (not active_chunks_.empty()) {
const auto chunk_index = active_chunks_.begin()->first;
lock.unlock();
handle_active_chunk_complete(chunk_index, lock);
lock.lock();
}
lock.unlock();
for (auto &worker : background_workers_) {
worker.join();
}
background_workers_.clear();
}
void download::wait_for_io(unique_mutex_lock &lock) {
lock.lock();
if (not get_complete() && read_queue_.empty() && write_queue_.empty()) {
if (get_timeout_enabled()) {
read_write_notify_.wait_until(lock, timeout_);
} else {
read_write_notify_.wait(lock);
}
}
lock.unlock();
}
api_error download::write_bytes(const std::uint64_t &, const std::uint64_t &write_offset,
std::vector<char> data, std::size_t &bytes_written,
const completer_callback &completer) {
/* std::cout << "write called:" << write_offset << std::endl; */
bytes_written = 0u;
if (not data.empty() && (error_ == api_error::success)) {
const auto start_chunk_index = current_chunk_index_ =
static_cast<std::size_t>(write_offset / chunk_size_);
const auto end_chunk_index =
static_cast<std::size_t>((data.size() + write_offset) / chunk_size_);
for (std::size_t chunk_index = start_chunk_index;
not get_complete() &&
(chunk_index <= std::min(read_chunk_state_.size() - 1u, end_chunk_index));
chunk_index++) {
download_chunk(chunk_index, false);
}
if (error_ == api_error::success) {
const auto original_size = fsi_.size;
unique_mutex_lock read_write_lock(read_write_mutex_);
const auto write_local_file = [&]() -> api_error {
error_ = (error_ == api_error::success)
? utils::file::write_to_source(fsi_, write_offset, data, bytes_written)
: error_;
if (error_ == api_error::success) {
utils::file::get_file_size(fsi_.source_path, fsi_.size);
completer(original_size, fsi_.size, true);
oft_.force_schedule_upload(fsi_);
}
read_write_notify_.notify_all();
read_write_lock.unlock();
return error_;
};
if (processed_) {
return write_local_file();
}
read_write_notify_.notify_all();
read_write_lock.unlock();
const auto size = (write_offset + data.size());
if (size > fsi_.size) {
pause();
read_write_lock.lock();
if (end_chunk_index >= write_chunk_state_.size()) {
const auto first_new_chunk_index = write_chunk_state_.size();
write_chunk_state_.resize(end_chunk_index + 1u);
read_chunk_state_.resize(end_chunk_index + 1u);
for (std::size_t chunk_index = first_new_chunk_index; chunk_index <= end_chunk_index;
chunk_index++) {
read_chunk_state_[chunk_index] = true;
}
}
fsi_.size = size;
last_chunk_size_ =
static_cast<std::size_t>(fsi_.size <= chunk_size_ ? fsi_.size
: fsi_.size % chunk_size_ ? fsi_.size % chunk_size_
: chunk_size_);
read_write_lock.unlock();
resume();
}
auto wd = std::make_shared<write_data>(write_offset, std::move(data));
read_write_lock.lock();
if (processed_) {
/* std::cout << "recursive write:" << write_offset << std::endl; */
return write_local_file();
}
write_queue_.emplace_back(wd);
read_write_notify_.notify_all();
read_write_lock.unlock();
reset_timeout(false);
utils::spin_wait_for_mutex(wd->complete, wd->notify, wd->mutex, "write_bytes");
reset_timeout(false);
bytes_written = wd->written;
if (bytes_written && (size > fsi_.size)) {
fsi_.size = size;
completer(original_size, fsi_.size, false);
}
}
}
return error_;
}
} // namespace repertory

502
src/download/download_manager.cpp
View File

@@ -1,502 +0,0 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#include "download/download_manager.hpp"
#include "download/direct_download.hpp"
#include "download/download.hpp"
#include "download/events.hpp"
#include "download/ring_download.hpp"
#include "drives/i_open_file_table.hpp"
#include "events/events.hpp"
#include "types/repertory.hpp"
#include "utils/encrypting_reader.hpp"
#include "utils/file_utils.hpp"
#include "utils/global_data.hpp"
#include "utils/path_utils.hpp"
#include "utils/rocksdb_utils.hpp"
#include "utils/utils.hpp"
namespace repertory {
static json create_resume_entry(i_download &d) {
filesystem_item fsi;
std::size_t chunk_size;
std::size_t last_chunk_size;
boost::dynamic_bitset<> read_state, write_state;
d.get_state_information(fsi, chunk_size, last_chunk_size, read_state, write_state);
return {
{"chunk_size", chunk_size},
{"encryption_token", fsi.encryption_token},
{"last_chunk_size", last_chunk_size},
{"path", fsi.api_path},
{"read_state", utils::string::from_dynamic_bitset(read_state)},
{"source", fsi.source_path},
{"write_state", utils::string::from_dynamic_bitset(write_state)},
};
}
static void restore_resume_entry(const json &resume_entry, filesystem_item &fsi,
std::size_t &chunk_size, std::size_t &last_chunk_size,
boost::dynamic_bitset<> &read_state,
boost::dynamic_bitset<> &write_state) {
chunk_size = resume_entry["chunk_size"].get<std::size_t>();
fsi.encryption_token = resume_entry["encryption_token"].get<std::string>();
last_chunk_size = resume_entry["last_chunk_size"].get<std::size_t>();
fsi.api_path = resume_entry["path"].get<std::string>();
read_state = utils::string::to_dynamic_bitset(resume_entry["read_state"].get<std::string>());
fsi.source_path = resume_entry["source"].get<std::string>();
write_state = utils::string::to_dynamic_bitset(resume_entry["write_state"].get<std::string>());
fsi.api_path = utils::path::get_parent_api_path(fsi.api_path);
}
static void create_source_file_if_empty(filesystem_item &fsi, const app_config &config) {
unique_recur_mutex_lock item_lock(*fsi.lock);
if (fsi.source_path.empty()) {
fsi.source_path =
utils::path::combine(config.get_cache_directory(), {utils::create_uuid_string()});
fsi.source_path_changed = true;
}
item_lock.unlock();
}
download_manager::download_manager(const app_config &config, api_reader_callback api_reader,
const bool &force_download)
: config_(config), api_reader_(api_reader), force_download_(force_download) {
utils::db::create_rocksdb(config, "resume_db", restore_db_);
E_SUBSCRIBE_EXACT(download_end, [this](const download_end &downloadEnd) {
this->handle_download_end(downloadEnd);
});
E_SUBSCRIBE_EXACT(filesystem_item_handle_closed,
[this](const filesystem_item_handle_closed &fileHandleClosed) {
this->on_handle_closed(fileHandleClosed);
});
E_SUBSCRIBE_EXACT(filesystem_item_closed, [this](const filesystem_item_closed &fileClosed) {
reset_timeout(fileClosed.get_api_path(), true);
});
E_SUBSCRIBE_EXACT(filesystem_item_opened, [this](const filesystem_item_opened &fileOpened) {
reset_timeout(fileOpened.get_api_path(), false);
});
}
download_manager::~download_manager() {
E_CONSUMER_RELEASE();
stop();
}
api_error download_manager::allocate(const std::uint64_t &handle, filesystem_item &fsi,
const std::uint64_t &size,
const i_download::allocator_callback &allocator) {
create_source_file_if_empty(fsi, config_);
if (stop_requested_) {
return api_error::download_stopped;
}
const auto completer = [&](std::uint64_t old_size, std::uint64_t new_size, bool changed) {
fsi.changed = changed;
if (old_size != new_size) {
fsi.size = new_size;
global_data::instance().update_used_space(old_size, new_size, false);
}
const auto modified = std::to_string(utils::get_file_time_now());
oft_->set_item_meta(fsi.api_path, META_MODIFIED, modified);
oft_->set_item_meta(fsi.api_path, META_WRITTEN, modified);
};
if (utils::file::is_file(fsi.source_path)) {
std::uint64_t file_size = 0u;
if (utils::file::get_file_size(fsi.source_path, file_size)) {
if (fsi.size == file_size) {
native_file_ptr nf;
auto ret = utils::file::assign_and_get_native_file(fsi, nf);
if (ret != api_error::success) {
return ret;
}
if ((ret = allocator()) == api_error::success) {
std::uint64_t new_file_size{};
utils::file::get_file_size(fsi.source_path, new_file_size);
completer(fsi.size, new_file_size, true);
}
return ret;
}
} else {
return api_error::os_error;
}
}
auto d = get_download(handle, fsi, true);
const auto result = d->get_result();
return ((result == api_error::success) ? d->allocate(handle, size, allocator, completer)
: result);
}
bool download_manager::contains_handle(const std::string &api_path,
const std::uint64_t &handle) const {
return ((download_lookup_.find(api_path) != download_lookup_.end()) &&
(download_lookup_.at(api_path).find(handle) != download_lookup_.at(api_path).end()));
}
bool download_manager::contains_restore(const std::string &api_path) const {
std::string value;
restore_db_->Get(rocksdb::ReadOptions(), api_path, &value);
return not value.empty();
}
api_error download_manager::download_file(const std::uint64_t &handle, filesystem_item &fsi) {
create_source_file_if_empty(fsi, config_);
if (stop_requested_) {
return api_error::download_stopped;
} else if (fsi.size > 0u) {
std::uint64_t file_size = 0u;
if (utils::file::get_file_size(fsi.source_path, file_size)) {
if (file_size == fsi.size) {
native_file_ptr nf;
return utils::file::assign_and_get_native_file(fsi, nf);
}
}
auto d = get_download(handle, fsi, true);
return ((d->get_result() == api_error::success) ? d->download_all() : d->get_result());
} else if (fsi.handle == REPERTORY_INVALID_HANDLE) {
native_file_ptr nf;
return utils::file::assign_and_get_native_file(fsi, nf);
} else {
return api_error::success;
}
}
download_ptr download_manager::get_download(std::uint64_t handle, filesystem_item &fsi,
const bool &write_supported) {
const auto create_download = [this, &handle, &fsi, &write_supported]() -> download_ptr {
const auto chunk_size = utils::encryption::encrypting_reader::get_data_chunk_size();
if (write_supported || force_download_) {
handle = 0u;
return std::dynamic_pointer_cast<i_download>(
std::make_shared<download>(config_, fsi, api_reader_, chunk_size, *oft_));
} else {
const auto dt = config_.get_preferred_download_type();
const std::size_t ring_buffer_size = config_.get_ring_buffer_file_size() * 1024ull * 1024ull;
const auto ring_allowed = (fsi.size > ring_buffer_size);
const auto no_space_remaining =
(utils::file::get_available_drive_space(config_.get_cache_directory()) < fsi.size);
if (ring_allowed && (dt != download_type::direct) &&
(no_space_remaining || (dt == download_type::ring_buffer))) {
return std::dynamic_pointer_cast<i_download>(std::make_shared<ring_download>(
config_, fsi, api_reader_, handle, chunk_size, ring_buffer_size));
}
if (no_space_remaining || (dt == download_type::direct)) {
return std::dynamic_pointer_cast<i_download>(
std::make_shared<direct_download>(config_, fsi, api_reader_, handle));
} else {
handle = 0u;
return std::dynamic_pointer_cast<i_download>(
std::make_shared<download>(config_, fsi, api_reader_, chunk_size, *oft_));
}
}
};
download_ptr d;
{
recur_mutex_lock download_lock(download_mutex_);
if (contains_handle(fsi.api_path, 0u)) {
d = download_lookup_[fsi.api_path][0u];
} else if (contains_handle(fsi.api_path, handle)) {
d = download_lookup_[fsi.api_path][handle];
} else {
d = create_download();
download_lookup_[fsi.api_path][handle] = d;
}
if (write_supported && not d->get_write_supported()) {
for (auto &kv : download_lookup_[fsi.api_path]) {
kv.second->set_disable_download_end(true);
}
d = create_download();
download_lookup_[fsi.api_path] = {{0u, d}};
}
}
return d;
}
std::string download_manager::get_source_path(const std::string &api_path) const {
recur_mutex_lock download_lock(download_mutex_);
if (contains_handle(api_path, 0u)) {
return download_lookup_.at(api_path).at(0u)->get_source_path();
}
return "";
}
void download_manager::handle_download_end(const download_end &de) {
download_ptr d;
const auto api_path = de.get_api_path();
const auto handle = utils::string::to_uint64(de.get_handle());
unique_recur_mutex_lock download_lock(download_mutex_);
if (contains_handle(api_path, handle)) {
d = download_lookup_[api_path][handle];
}
download_lock.unlock();
if (d) {
download_lock.lock();
download_lookup_[api_path].erase(handle);
if (download_lookup_[api_path].empty()) {
download_lookup_.erase(api_path);
}
if (d->get_result() == api_error::download_incomplete) {
const auto res = restore_db_->Put(rocksdb::WriteOptions(), api_path.get<std::string>(),
create_resume_entry(*d).dump());
if (res.ok()) {
event_system::instance().raise<download_stored>(api_path, d->get_source_path());
} else {
event_system::instance().raise<download_store_failed>(api_path.get<std::string>(),
d->get_source_path(), res.ToString());
}
}
download_lock.unlock();
}
}
void download_manager::on_handle_closed(const filesystem_item_handle_closed &handle_closed) {
download_ptr d;
const auto api_path = handle_closed.get_api_path();
const auto handle = utils::string::to_uint64(handle_closed.get_handle());
unique_recur_mutex_lock download_lock(download_mutex_);
if (contains_handle(api_path, handle)) {
d = download_lookup_[api_path][handle];
}
download_lock.unlock();
if (d) {
d->notify_stop_requested();
}
}
bool download_manager::is_processing(const std::string &api_path) const {
recur_mutex_lock download_lock(download_mutex_);
return download_lookup_.find(api_path) != download_lookup_.end();
}
bool download_manager::pause_download(const std::string &api_path) {
auto ret = true;
recur_mutex_lock download_lock(download_mutex_);
if (download_lookup_.find(api_path) != download_lookup_.end()) {
ret = false;
if (download_lookup_[api_path].find(0u) != download_lookup_[api_path].end()) {
if ((ret = (download_lookup_[api_path].size() == 1))) {
ret = download_lookup_[api_path][0u]->pause();
}
}
}
return ret;
}
api_error download_manager::read_bytes(const std::uint64_t &handle, filesystem_item &fsi,
std::size_t read_size, const std::uint64_t &read_offset,
std::vector<char> &data) {
create_source_file_if_empty(fsi, config_);
if (stop_requested_) {
return api_error::download_stopped;
}
read_size = utils::calculate_read_size(fsi.size, read_size, read_offset);
if (read_size == 0u) {
return api_error::success;
}
if (utils::file::is_file(fsi.source_path)) {
std::uint64_t file_size = 0u;
if (utils::file::get_file_size(fsi.source_path, file_size)) {
if ((read_offset + read_size) <= file_size) {
return utils::file::read_from_source(fsi, read_size, read_offset, data);
}
} else {
return api_error::os_error;
}
}
auto d = get_download(handle, fsi, false);
const auto result = d->get_result();
return ((result == api_error::success) ? d->read_bytes(handle, read_size, read_offset, data)
: result);
}
void download_manager::rename_download(const std::string &from_api_path,
const std::string &to_api_path) {
recur_mutex_lock download_lock(download_mutex_);
if (download_lookup_.find(from_api_path) != download_lookup_.end()) {
if (download_lookup_[from_api_path].find(0u) != download_lookup_[from_api_path].end()) {
download_lookup_[from_api_path][0u]->set_api_path(to_api_path);
download_lookup_[to_api_path][0u] = download_lookup_[from_api_path][0u];
download_lookup_.erase(from_api_path);
}
}
}
void download_manager::reset_timeout(const std::string &api_path, const bool &file_closed) {
download_ptr d;
unique_recur_mutex_lock download_lock(download_mutex_);
if (contains_handle(api_path, 0u)) {
d = download_lookup_[api_path][0u];
}
download_lock.unlock();
if (d) {
d->reset_timeout(file_closed);
}
}
api_error download_manager::resize(const std::uint64_t &handle, filesystem_item &fsi,
const std::uint64_t &size) {
return allocate(handle, fsi, size,
[&]() -> api_error { return utils::file::truncate_source(fsi, size); });
}
void download_manager::resume_download(const std::string &api_path) {
recur_mutex_lock download_lock(download_mutex_);
if (download_lookup_.find(api_path) != download_lookup_.end()) {
if (download_lookup_[api_path].find(0u) != download_lookup_[api_path].end()) {
download_lookup_[api_path][0u]->resume();
}
}
}
void download_manager::start(i_open_file_table *oft) {
recur_mutex_lock start_stop_lock(start_stop_mutex_);
if (stop_requested_) {
stop_requested_ = false;
oft_ = oft;
start_incomplete();
}
}
void download_manager::start_incomplete() {
auto iterator =
std::shared_ptr<rocksdb::Iterator>(restore_db_->NewIterator(rocksdb::ReadOptions()));
for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
filesystem_item fsi{};
fsi.lock = std::make_shared<std::recursive_mutex>();
std::size_t chunk_size, last_chunk_size;
boost::dynamic_bitset<> readState, writeState;
restore_resume_entry(json::parse(iterator->value().ToString()), fsi, chunk_size,
last_chunk_size, readState, writeState);
if (utils::file::get_file_size(fsi.source_path, fsi.size)) {
download_lookup_[fsi.api_path][0u] = std::make_shared<download>(
config_, fsi, api_reader_, chunk_size, last_chunk_size, readState, writeState, *oft_);
event_system::instance().raise<download_restored>(fsi.api_path, fsi.source_path);
} else {
event_system::instance().raise<download_restore_failed>(
fsi.api_path, fsi.source_path,
"failed to get file size: " + utils::string::from_uint64(utils::get_last_error_code()));
}
restore_db_->Delete(rocksdb::WriteOptions(), iterator->key());
}
}
void download_manager::stop() {
unique_recur_mutex_lock start_stop_lock(start_stop_mutex_);
if (not stop_requested_) {
event_system::instance().raise<service_shutdown>("download_manager");
stop_requested_ = true;
start_stop_lock.unlock();
std::vector<download_ptr> downloads;
unique_recur_mutex_lock download_lock(download_mutex_);
for (auto &d : download_lookup_) {
for (auto &kv : d.second) {
auto download = kv.second;
if (download) {
downloads.emplace_back(download);
}
}
}
download_lock.unlock();
for (auto &d : downloads) {
d->notify_stop_requested();
}
while (not download_lookup_.empty()) {
std::this_thread::sleep_for(1ms);
}
}
}
api_error download_manager::write_bytes(const std::uint64_t &handle, filesystem_item &fsi,
const std::uint64_t &write_offset, std::vector<char> data,
std::size_t &bytes_written) {
bytes_written = 0u;
create_source_file_if_empty(fsi, config_);
if (stop_requested_) {
return api_error::download_stopped;
}
if (data.empty()) {
return api_error::success;
}
const auto completer = [&](std::uint64_t old_size, std::uint64_t new_size, bool changed) {
fsi.changed = changed;
if (old_size != new_size) {
fsi.size = new_size;
global_data::instance().update_used_space(old_size, new_size, false);
}
const auto modified = std::to_string(utils::get_file_time_now());
oft_->set_item_meta(fsi.api_path, META_MODIFIED, modified);
oft_->set_item_meta(fsi.api_path, META_WRITTEN, modified);
};
if (utils::file::is_file(fsi.source_path)) {
std::uint64_t file_size = 0u;
if (utils::file::get_file_size(fsi.source_path, file_size)) {
if (fsi.size == file_size) {
auto ret = utils::file::write_to_source(fsi, write_offset, data, bytes_written);
if (ret == api_error::success) {
std::uint64_t new_size{};
utils::file::get_file_size(fsi.source_path, new_size);
completer(fsi.size, new_size, true);
}
return ret;
}
} else {
return api_error::os_error;
}
}
auto d = get_download(handle, fsi, true);
const auto result = d->get_result();
return ((result == api_error::success)
? d->write_bytes(handle, write_offset, std::move(data), bytes_written, completer)
: result);
}
} // namespace repertory

166
src/download/reader_pool.cpp
View File

@@ -1,166 +0,0 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#include "download/reader_pool.hpp"
namespace repertory {
void reader_pool::pause() {
unique_mutex_lock work_lock(work_mutex_);
if (not paused_) {
paused_ = true;
while (not stop_requested_ && not work_queue_.empty()) {
work_notify_.notify_all();
work_notify_.wait_for(work_lock, 2s);
}
}
work_notify_.notify_all();
}
void reader_pool::process_work_item(pool_work_item &work) {
const auto result =
api_reader_(work.api_path, work.read_size, work.read_offset, work.data, restart_active_);
work.completed(result);
}
void reader_pool::queue_read_bytes(const std::string &api_path, const std::size_t &read_size,
const std::uint64_t &read_offset, std::vector<char> &data,
completed_callback completed) {
data.clear();
unique_mutex_lock work_lock(work_mutex_);
wait_for_resume(work_lock);
if (not stop_requested_) {
if (pool_size_ == 1ull) {
work_notify_.notify_all();
work_lock.unlock();
completed(api_reader_(api_path, read_size, read_offset, data, restart_active_));
} else {
auto work =
std::make_shared<pool_work_item>(api_path, read_size, read_offset, data, completed);
if (not restart_active_) {
while ((work_queue_.size() >= pool_size_) && not restart_active_) {
work_notify_.wait(work_lock);
}
if (not restart_active_) {
work_queue_.emplace_back(work);
work_notify_.notify_all();
}
work_lock.unlock();
}
}
}
}
void reader_pool::restart() {
restart_active_ = true;
unique_mutex_lock work_lock(work_mutex_);
wait_for_resume(work_lock);
work_notify_.notify_all();
work_lock.unlock();
while (not work_queue_.empty() || active_count_) {
work_lock.lock();
if (not work_queue_.empty() || active_count_) {
work_notify_.wait(work_lock);
}
work_lock.unlock();
}
work_lock.lock();
restart_active_ = false;
work_notify_.notify_all();
work_lock.unlock();
}
void reader_pool::resume() {
unique_mutex_lock work_lock(work_mutex_);
if (paused_) {
paused_ = false;
}
work_notify_.notify_all();
}
void reader_pool::start() {
restart_active_ = stop_requested_ = false;
active_count_ = 0u;
if (pool_size_ > 1u) {
for (std::size_t i = 0u; i < pool_size_; i++) {
work_threads_.emplace_back(std::thread([this]() {
unique_mutex_lock work_lock(work_mutex_);
work_notify_.notify_all();
work_lock.unlock();
while (not stop_requested_) {
work_lock.lock();
if (not stop_requested_) {
if (work_queue_.empty()) {
work_notify_.wait(work_lock);
}
if (not work_queue_.empty()) {
active_count_++;
auto work = work_queue_.front();
work_queue_.pop_front();
work_notify_.notify_all();
work_lock.unlock();
process_work_item(*work);
work_lock.lock();
active_count_--;
}
}
work_notify_.notify_all();
work_lock.unlock();
}
work_lock.lock();
work_notify_.notify_all();
work_lock.unlock();
}));
}
}
}
void reader_pool::stop() {
stop_requested_ = true;
paused_ = false;
restart();
restart_active_ = true;
unique_mutex_lock work_lock(work_mutex_);
work_notify_.notify_all();
work_lock.unlock();
for (auto &work_thread : work_threads_) {
work_thread.join();
}
work_threads_.clear();
}
void reader_pool::wait_for_resume(unique_mutex_lock &lock) {
while (paused_ && not stop_requested_) {
work_notify_.notify_all();
work_notify_.wait_for(lock, 2s);
}
}
} // namespace repertory

450
src/download/ring_download.cpp
View File

@@ -1,450 +0,0 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#include "download/ring_download.hpp"
#include "app_config.hpp"
#include "download/buffered_reader.hpp"
#include "download/events.hpp"
#include "download/utils.hpp"
#include "events/event_system.hpp"
#include "utils/file_utils.hpp"
#include "utils/path_utils.hpp"
#include "utils/utils.hpp"
namespace repertory {
ring_download::ring_download(const app_config &config, filesystem_item fsi,
const api_reader_callback &api_reader, const std::uint64_t &handle,
const std::size_t &chunk_size, const std::size_t &ring_buffer_size)
: config_(config),
fsi_(std::move(fsi)),
api_reader_(api_reader),
handle_(handle),
chunk_size_(chunk_size),
ring_state_(ring_buffer_size / chunk_size) {
if (not chunk_size) {
error_ = api_error::empty_ring_buffer_chunk_size;
} else if (not ring_buffer_size) {
error_ = api_error::empty_ring_buffer_size;
} else if (ring_buffer_size % chunk_size) {
error_ = api_error::invalid_ring_buffer_multiple;
} else if (ring_buffer_size < chunk_size) {
error_ = api_error::invalid_ring_buffer_size;
} else {
*const_cast<std::size_t *>(&total_chunks_) =
utils::divide_with_ceiling(static_cast<std::size_t>(fsi.size), chunk_size_);
const auto buffer_directory = utils::path::combine(config_.get_data_directory(), {"buffer"});
if (utils::file::create_full_directory_path(buffer_directory)) {
buffer_file_path_ = utils::path::combine(buffer_directory, {utils::create_uuid_string()});
if ((error_ = native_file::create_or_open(buffer_file_path_, buffer_file_)) ==
api_error::success) {
if (not buffer_file_->allocate(ring_buffer_size)) {
error_ = api_error::os_error;
}
}
} else {
error_ = api_error::os_error;
}
}
}
ring_download::~ring_download() {
stop();
if (buffer_file_) {
buffer_file_->close();
utils::file::delete_file(buffer_file_path_);
}
}
void ring_download::buffer_thread(std::size_t start_chunk_index) {
buffered_reader reader(config_, fsi_, api_reader_, chunk_size_, total_chunks_, start_chunk_index);
unique_mutex_lock write_lock(write_mutex_);
buffered_reader_ = &reader;
read_notify_.notify_all();
write_lock.unlock();
const auto ring_size = ring_state_.size();
const auto half_ring_size = ring_size / 2;
double progress = 0.0;
const auto reset_on_overflow = [&](const std::size_t &amt) -> bool {
if (amt >= ring_size) {
ring_state_.reset();
head_chunk_index_ = read_chunk_;
return true;
}
return false;
};
const auto decrement_head_chunk = [&](std::size_t amt) {
if (not reset_on_overflow(amt)) {
while (amt-- && head_chunk_index_) {
head_chunk_index_--;
ring_state_[head_chunk_index_ % ring_size] = false;
}
}
write_chunk_ = read_chunk_;
};
const auto increment_head_chunk = [&](std::size_t amt) {
if (not reset_on_overflow(amt)) {
while (amt-- && (head_chunk_index_ < (total_chunks_ - 1u))) {
ring_state_[head_chunk_index_ % ring_size] = false;
head_chunk_index_++;
}
}
write_chunk_ = read_chunk_;
};
while (is_active()) {
write_lock.lock();
const auto get_overflow_chunk = [&]() { return head_chunk_index_ + ring_size; };
while ((write_chunk_ > read_chunk_) && (write_chunk_ >= get_overflow_chunk()) && is_active()) {
// Buffer 50% for read-ahead/read-behind
auto buffered = false;
while ((write_chunk_ > read_chunk_) && ((write_chunk_ - read_chunk_) > half_ring_size) &&
is_active()) {
buffered = true;
read_notify_.wait(write_lock);
}
if (not buffered && is_active()) {
read_notify_.wait(write_lock);
}
if ((write_chunk_ > read_chunk_) && ((write_chunk_ - read_chunk_) <= half_ring_size)) {
increment_head_chunk(1u);
}
}
if (is_active()) {
if (read_chunk_ >= get_overflow_chunk()) {
increment_head_chunk(read_chunk_ - get_overflow_chunk() + 1u);
} else if (read_chunk_ < head_chunk_index_) {
decrement_head_chunk(head_chunk_index_ - read_chunk_);
} else if ((write_chunk_ < head_chunk_index_) || (write_chunk_ >= get_overflow_chunk())) {
write_chunk_ = read_chunk_;
}
const auto write_position = write_chunk_ % ring_size;
if (ring_state_[write_position]) {
write_chunk_++;
} else {
const auto file_offset = write_chunk_ * chunk_size_;
const auto write_chunk_index = write_chunk_;
read_notify_.notify_all();
write_lock.unlock();
const auto read_size = utils::calculate_read_size(fsi_.size, chunk_size_, file_offset);
if (read_size > 0u) {
if (((write_chunk_index == 0u) && reader.has_first_chunk()) ||
((write_chunk_index == (total_chunks_ - 1ull)) && reader.has_last_chunk())) {
write_lock.lock();
write_chunk_++;
} else {
std::vector<char> data;
auto error = reader.read_chunk(write_chunk_index, data);
if (error == api_error::success) {
std::mutex job_mutex;
std::condition_variable job_notify;
unique_mutex_lock job_lock(job_mutex);
if (queue_io_item(job_mutex, job_notify, false, [&]() {
#ifdef _DEBUG
write_lock.lock();
if (write_chunk_index >= (head_chunk_index_ + ring_size)) {
throw std::runtime_error("Invalid write: write chunk >= head");
}
if (write_chunk_index < head_chunk_index_) {
throw std::runtime_error("Invalid write: write chunk < head");
}
#endif // _DEBUG
std::size_t bytes_written = 0u;
if (buffer_file_->write_bytes(&data[0u], data.size(),
write_position * chunk_size_, bytes_written)) {
buffer_file_->flush();
utils::download::notify_progress<download_progress>(
config_, fsi_.api_path, buffer_file_path_,
static_cast<double>(write_chunk_index + 1ull),
static_cast<double>(total_chunks_), progress);
} else {
error = api_error::os_error;
}
#ifdef _DEBUG
write_lock.unlock();
#endif // _DEBUG
})) {
job_notify.wait(job_lock);
}
job_notify.notify_all();
job_lock.unlock();
}
set_api_error(error);
write_lock.lock();
if (error == api_error::success) {
ring_state_[write_position] = true;
write_chunk_++;
}
}
} else {
write_lock.lock();
while ((read_chunk_ <= write_chunk_) && (read_chunk_ >= head_chunk_index_) &&
(((write_chunk_ * chunk_size_) >= fsi_.size)) && is_active()) {
read_notify_.wait(write_lock);
}
}
}
}
read_notify_.notify_all();
write_lock.unlock();
}
write_lock.lock();
buffered_reader_ = nullptr;
read_notify_.notify_all();
write_lock.unlock();
if (not disable_download_end_) {
event_system::instance().raise<download_end>(fsi_.api_path, buffer_file_path_, handle_, error_);
}
}
void ring_download::io_thread() {
const auto process_all = [&]() {
unique_mutex_lock io_lock(io_mutex_);
while (not io_queue_.empty()) {
auto &action = io_queue_.front();
io_lock.unlock();
unique_mutex_lock job_lock(action->mutex);
action->action();
action->notify.notify_all();
job_lock.unlock();
io_lock.lock();
utils::remove_element_from(io_queue_, action);
}
io_lock.unlock();
};
while (not stop_requested_) {
if (io_queue_.empty()) {
unique_mutex_lock io_lock(io_mutex_);
if (not stop_requested_ && io_queue_.empty()) {
io_notify_.wait(io_lock);
}
} else {
process_all();
}
}
process_all();
}
void ring_download::notify_stop_requested() {
set_api_error(api_error::download_stopped);
stop_requested_ = true;
unique_mutex_lock write_lock(write_mutex_);
read_notify_.notify_all();
write_lock.unlock();
unique_mutex_lock io_lock(io_mutex_);
io_notify_.notify_all();
io_lock.unlock();
}
bool ring_download::queue_io_item(std::mutex &m, std::condition_variable &cv, const bool &is_read,
std::function<void()> action) {
auto ret = false;
mutex_lock io_lock(io_mutex_);
if (not stop_requested_) {
ret = true;
if (is_read) {
io_queue_.insert(io_queue_.begin(), std::make_unique<io_action>(m, cv, action));
} else {
io_queue_.emplace_back(std::make_unique<io_action>(m, cv, action));
}
}
io_notify_.notify_all();
return ret;
}
void ring_download::read(std::size_t read_chunk_index, std::size_t read_size,
std::size_t read_offset, std::vector<char> &data) {
while (is_active() && (read_size > 0u)) {
unique_mutex_lock write_lock(write_mutex_);
if ((read_chunk_index == (total_chunks_ - 1ull)) && buffered_reader_ &&
buffered_reader_->has_last_chunk()) {
std::vector<char> *buffer = nullptr;
buffered_reader_->get_last_chunk(buffer);
if (buffer) {
data.insert(data.end(), buffer->begin() + read_offset,
buffer->begin() + read_offset + read_size);
}
read_notify_.notify_all();
write_lock.unlock();
read_size = 0u;
} else {
const auto notify_read_location = [&]() {
if (read_chunk_index != read_chunk_) {
write_lock.lock();
read_chunk_ = read_chunk_index;
read_notify_.notify_all();
write_lock.unlock();
}
};
if ((read_chunk_index == 0u) && buffered_reader_ && buffered_reader_->has_first_chunk()) {
const auto to_read = std::min(chunk_size_ - read_offset, read_size);
std::vector<char> *buffer = nullptr;
buffered_reader_->get_first_chunk(buffer);
if (buffer) {
data.insert(data.end(), buffer->begin() + read_offset,
buffer->begin() + read_offset + to_read);
}
read_notify_.notify_all();
write_lock.unlock();
read_size -= to_read;
read_offset = 0u;
read_chunk_index++;
notify_read_location();
} else {
const auto ring_size = ring_state_.size();
while (((read_chunk_index > write_chunk_) || (read_chunk_index < head_chunk_index_) ||
(read_chunk_index >= (head_chunk_index_ + ring_size))) &&
is_active()) {
read_chunk_ = read_chunk_index;
read_notify_.notify_all();
read_notify_.wait(write_lock);
}
read_notify_.notify_all();
write_lock.unlock();
if (is_active()) {
const auto ringPos = read_chunk_index % ring_size;
notify_read_location();
if (ring_state_[ringPos]) {
const auto to_read = std::min(chunk_size_ - read_offset, read_size);
std::mutex job_mutex;
std::condition_variable job_notify;
unique_mutex_lock job_lock(job_mutex);
if (queue_io_item(job_mutex, job_notify, true, [&]() {
std::vector<char> buffer(to_read);
std::size_t bytes_read = 0u;
if (buffer_file_->read_bytes(&buffer[0u], buffer.size(),
(ringPos * chunk_size_) + read_offset, bytes_read)) {
data.insert(data.end(), buffer.begin(), buffer.end());
buffer.clear();
read_size -= to_read;
read_offset = 0u;
} else {
set_api_error(api_error::os_error);
}
})) {
job_notify.wait(job_lock);
read_chunk_index++;
}
job_notify.notify_all();
job_lock.unlock();
} else {
write_lock.lock();
while (not ring_state_[ringPos] && is_active()) {
read_notify_.wait(write_lock);
}
read_notify_.notify_all();
write_lock.unlock();
}
}
}
}
}
}
api_error ring_download::read_bytes(const std::uint64_t &, std::size_t read_size,
const std::uint64_t &read_offset, std::vector<char> &data) {
data.clear();
mutex_lock read_lock(read_mutex_);
if (is_active()) {
if (((read_size = utils::calculate_read_size(fsi_.size, read_size, read_offset)) > 0u)) {
const auto read_chunk_index = static_cast<std::size_t>(read_offset / chunk_size_);
if (not buffer_thread_) {
start(read_chunk_index);
}
read(read_chunk_index, read_size, read_offset % chunk_size_, data);
set_api_error(((error_ == api_error::success) && stop_requested_)
? api_error::download_stopped
: error_);
}
}
return error_;
}
void ring_download::set_api_error(const api_error &error) {
if ((error_ == api_error::success) && (error_ != error)) {
error_ = error;
}
}
void ring_download::start(const std::size_t &startChunk) {
event_system::instance().raise<download_begin>(fsi_.api_path, buffer_file_path_);
error_ = api_error::success;
stop_requested_ = false;
head_chunk_index_ = read_chunk_ = write_chunk_ = startChunk;
ring_state_.reset();
io_thread_ = std::make_unique<std::thread>([this] { this->io_thread(); });
unique_mutex_lock write_lock(write_mutex_);
buffer_thread_ =
std::make_unique<std::thread>([this, startChunk]() { this->buffer_thread(startChunk); });
read_notify_.wait(write_lock);
read_notify_.notify_all();
write_lock.unlock();
}
void ring_download::stop() {
notify_stop_requested();
if (buffer_thread_) {
buffer_thread_->join();
buffer_thread_.reset();
}
if (io_thread_) {
io_thread_->join();
io_thread_.reset();
io_queue_.clear();
}
}
} // namespace repertory

120
src/drives/directory_cache.cpp
View File

@@ -1,63 +1,42 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "drives/directory_cache.hpp"
#include "drives/directory_iterator.hpp"
#include "events/events.hpp"
#include "events/event_system.hpp"
#include "events/events.hpp"
#include "types/repertory.hpp"
namespace repertory {
bool directory_cache::execute_action(const std::string &api_path, const execute_callback &execute) {
auto found = false;
void directory_cache::execute_action(const std::string &api_path,
const execute_callback &execute) {
recur_mutex_lock directory_lock(directory_mutex_);
if ((found = (directory_lookup_.find(api_path) != directory_lookup_.end()))) {
if ((directory_lookup_.find(api_path) != directory_lookup_.end())) {
execute(*directory_lookup_[api_path].iterator);
}
return found;
}
void directory_cache::refresh_thread() {
while (not is_shutdown_) {
unique_recur_mutex_lock directory_lock(directory_mutex_);
auto lookup = directory_lookup_;
directory_lock.unlock();
for (const auto &kv : lookup) {
if (std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now() -
kv.second.last_update) >= 120s) {
directory_lock.lock();
directory_lookup_.erase(kv.first);
directory_lock.unlock();
}
}
if (not is_shutdown_) {
unique_mutex_lock shutdown_lock(shutdown_mutex_);
if (not is_shutdown_) {
shutdown_notify_.wait_for(shutdown_lock, 15s);
}
}
}
}
directory_iterator *directory_cache::remove_directory(const std::string &api_path) {
auto directory_cache::remove_directory(const std::string &api_path)
-> directory_iterator * {
directory_iterator *ret = nullptr;
recur_mutex_lock directory_lock(directory_mutex_);
@@ -72,39 +51,42 @@ directory_iterator *directory_cache::remove_directory(const std::string &api_pat
void directory_cache::remove_directory(directory_iterator *iterator) {
if (iterator) {
recur_mutex_lock directory_lock(directory_mutex_);
const auto it = std::find_if(
directory_lookup_.begin(), directory_lookup_.end(),
[&iterator](const auto &kv) -> bool { return kv.second.iterator == iterator; });
const auto it =
std::find_if(directory_lookup_.begin(), directory_lookup_.end(),
[&iterator](const auto &kv) -> bool {
return kv.second.iterator == iterator;
});
if (it != directory_lookup_.end()) {
directory_lookup_.erase(it->first);
}
}
}
void directory_cache::set_directory(const std::string &api_path, directory_iterator *iterator) {
void directory_cache::service_function() {
unique_recur_mutex_lock directory_lock(directory_mutex_);
auto lookup = directory_lookup_;
directory_lock.unlock();
for (const auto &kv : lookup) {
if (std::chrono::duration_cast<std::chrono::seconds>(
std::chrono::system_clock::now() - kv.second.last_update) >= 120s) {
directory_lock.lock();
directory_lookup_.erase(kv.first);
directory_lock.unlock();
}
}
if (not get_stop_requested()) {
unique_mutex_lock shutdown_lock(get_mutex());
if (not get_stop_requested()) {
get_notify().wait_for(shutdown_lock, 15s);
}
}
}
void directory_cache::set_directory(const std::string &api_path,
directory_iterator *iterator) {
recur_mutex_lock directory_lock(directory_mutex_);
directory_lookup_[api_path] = {iterator};
}
void directory_cache::start() {
unique_mutex_lock shutdown_lock(shutdown_mutex_);
if (is_shutdown_) {
is_shutdown_ = false;
refresh_thread_ = std::make_unique<std::thread>([this]() { this->refresh_thread(); });
}
}
void directory_cache::stop() {
unique_mutex_lock shutdown_lock(shutdown_mutex_);
if (not is_shutdown_) {
event_system::instance().raise<service_shutdown>("directory_cache");
is_shutdown_ = true;
shutdown_notify_.notify_all();
shutdown_lock.unlock();
refresh_thread_->join();
refresh_thread_.reset();
}
}
} // namespace repertory

125
src/drives/directory_iterator.cpp
View File

@@ -1,53 +1,70 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "drives/directory_iterator.hpp"
#include "utils/error_utils.hpp"
#include "utils/path_utils.hpp"
namespace repertory {
#ifndef _WIN32
int directory_iterator::fill_buffer(const remote::file_offset &offset,
fuse_fill_dir_t filler_function, void *buffer,
populate_stat_callback populate_stat) {
auto directory_iterator::fill_buffer(const remote::file_offset &offset,
fuse_fill_dir_t filler_function,
void *buffer,
populate_stat_callback populate_stat)
-> int {
if (offset < items_.size()) {
std::string item_name;
struct stat st {};
struct stat *pst = nullptr;
switch (offset) {
case 0: {
item_name = ".";
} break;
try {
std::string item_name;
struct stat st {};
struct stat *pst = nullptr;
switch (offset) {
case 0: {
item_name = ".";
} break;
case 1: {
item_name = "..";
} break;
case 1: {
item_name = "..";
} break;
default: {
const auto &item = items_[offset];
item_name = utils::path::strip_to_file_name(item.api_path);
populate_stat(item.api_path, item.size, item.meta, item.directory, &st);
pst = &st;
} break;
}
default: {
const auto &item = items_[offset];
item_name = utils::path::strip_to_file_name(item.api_path);
populate_stat(item.api_path, item.size, item.meta, item.directory, &st);
pst = &st;
} break;
}
if (filler_function(buffer, &item_name[0], pst, offset + 1) != 0) {
errno = ENOMEM;
return -1;
#if FUSE_USE_VERSION >= 30
if (filler_function(buffer, &item_name[0], pst, offset + 1,
FUSE_FILL_DIR_PLUS) != 0) {
#else
if (filler_function(buffer, &item_name[0], pst, offset + 1) != 0) {
#endif
errno = ENOMEM;
return -1;
}
} catch (const std::exception &e) {
utils::error::raise_error(__FUNCTION__, e,
"failed to fill fuse directory buffer");
}
return 0;
@@ -58,7 +75,7 @@ int directory_iterator::fill_buffer(const remote::file_offset &offset,
}
#endif // !_WIN32
int directory_iterator::get(const std::size_t &offset, std::string &item) {
auto directory_iterator::get(std::size_t offset, std::string &item) -> int {
if (offset < items_.size()) {
item = items_[offset].api_path;
return 0;
@@ -68,7 +85,8 @@ int directory_iterator::get(const std::size_t &offset, std::string &item) {
return -1;
}
api_error directory_iterator::get_directory_item(const std::size_t &offset, directory_item &di) {
auto directory_iterator::get_directory_item(std::size_t offset,
directory_item &di) -> api_error {
if (offset < items_.size()) {
di = items_[offset];
return api_error::success;
@@ -77,9 +95,12 @@ api_error directory_iterator::get_directory_item(const std::size_t &offset, dire
return api_error::directory_end_of_files;
}
api_error directory_iterator::get_directory_item(const std::string &api_path, directory_item &di) {
auto iter = std::find_if(items_.begin(), items_.end(),
[&](const auto &item) -> bool { return api_path == item.api_path; });
auto directory_iterator::get_directory_item(const std::string &api_path,
directory_item &di) -> api_error {
auto iter =
std::find_if(items_.begin(), items_.end(), [&](const auto &item) -> bool {
return api_path == item.api_path;
});
if (iter != items_.end()) {
di = *iter;
return api_error::success;
@@ -88,7 +109,7 @@ api_error directory_iterator::get_directory_item(const std::string &api_path, di
return api_error::item_not_found;
}
int directory_iterator::get_json(const std::size_t &offset, json &item) {
auto directory_iterator::get_json(std::size_t offset, json &item) -> int {
if (offset < items_.size()) {
item = items_[offset].to_json();
return 0;
@@ -98,29 +119,35 @@ int directory_iterator::get_json(const std::size_t &offset, json &item) {
return -1;
}
std::size_t directory_iterator::get_next_directory_offset(const std::string &api_path) const {
const auto it = std::find_if(items_.begin(), items_.end(), [&api_path](const auto &di) -> bool {
return api_path == di.api_path;
});
auto directory_iterator::get_next_directory_offset(
const std::string &api_path) const -> std::size_t {
const auto it = std::find_if(
items_.begin(), items_.end(),
[&api_path](const auto &di) -> bool { return api_path == di.api_path; });
return (it == items_.end()) ? 0 : std::distance(items_.begin(), it) + std::size_t(1u);
return (it == items_.end())
? 0
: std::distance(items_.begin(), it) + std::size_t(1u);
}
directory_iterator &directory_iterator::operator=(const directory_iterator &iterator) noexcept {
auto directory_iterator::operator=(const directory_iterator &iterator) noexcept
-> directory_iterator & {
if (this != &iterator) {
items_ = iterator.items_;
}
return *this;
}
directory_iterator &directory_iterator::operator=(directory_iterator &&iterator) noexcept {
auto directory_iterator::operator=(directory_iterator &&iterator) noexcept
-> directory_iterator & {
if (this != &iterator) {
items_ = std::move(iterator.items_);
}
return *this;
}
directory_iterator &directory_iterator::operator=(directory_item_list list) noexcept {
auto directory_iterator::operator=(directory_item_list list) noexcept
-> directory_iterator & {
if (&items_ != &list) {
items_ = std::move(list);
}

202
src/drives/eviction.cpp
View File

@@ -1,108 +1,62 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "drives/eviction.hpp"
#include "app_config.hpp"
#include "drives/i_open_file_table.hpp"
#include "file_manager/i_file_manager.hpp"
#include "providers/i_provider.hpp"
#include "types/repertory.hpp"
#include "utils/error_utils.hpp"
#include "utils/file_utils.hpp"
#include "utils/global_data.hpp"
#include "utils/unix/unix_utils.hpp"
#include "utils/utils.hpp"
namespace repertory {
void eviction::check_items_thread() {
while (not stop_requested_) {
auto should_evict = true;
// Handle maximum cache size eviction
auto used_bytes = global_data::instance().get_used_cache_space();
if (config_.get_enable_max_cache_size()) {
should_evict = (used_bytes > config_.get_max_cache_size_bytes());
}
// Evict all items if minimum redundancy eviction is enabled; otherwise, evict
// until required space is reclaimed.
if (should_evict) {
// Remove cached source files that don't meet minimum requirements
auto cached_files_list = get_filtered_cached_files();
if (not cached_files_list.empty()) {
while (not stop_requested_ && should_evict && not cached_files_list.empty()) {
std::string api_path;
if (provider_.get_api_path_from_source(cached_files_list.front(), api_path) ==
api_error::success) {
std::string pinned;
provider_.get_item_meta(api_path, META_PINNED, pinned);
api_file file{};
filesystem_item fsi{};
if ((pinned.empty() || not utils::string::to_bool(pinned)) &&
provider_.get_filesystem_item_and_file(api_path, file, fsi) == api_error::success) {
// Only evict files that match expected size
std::uint64_t file_size = 0u;
utils::file::get_file_size(cached_files_list.front(), file_size);
if (file_size == fsi.size) {
// Ensure minimum file redundancy has been met or source path is not being
// used for local recovery
const auto different_source = file.source_path != fsi.source_path;
if (file.recoverable &&
((file.redundancy >= config_.get_minimum_redundancy()) || different_source)) {
// Try to evict file
if (oft_.evict_file(fsi.api_path) && config_.get_enable_max_cache_size()) {
// Restrict number of items evicted if maximum cache size is enabled
used_bytes -= file_size;
should_evict = (used_bytes > config_.get_max_cache_size_bytes());
}
}
}
}
}
cached_files_list.pop_front();
}
}
}
if (not stop_requested_) {
unique_mutex_lock l(eviction_mutex_);
if (not stop_requested_) {
stop_notify_.wait_for(l, 30s);
}
}
auto eviction::check_minimum_requirements(const std::string &file_path)
-> bool {
std::uint64_t file_size{};
if (not utils::file::get_file_size(file_path, file_size)) {
utils::error::raise_error(__FUNCTION__, utils::get_last_error_code(),
file_path, "failed to get file size");
return false;
}
}
bool eviction::check_minimum_requirements(const std::string &file_path) {
auto ret = false;
// Only evict cachedFileList that are > 0
std::uint64_t file_size = 0u;
utils::file::get_file_size(file_path, file_size);
if (file_size) {
// Check modified/accessed date/time
std::uint64_t reference_time = 0u;
if ((ret = config_.get_eviction_uses_accessed_time()
? utils::file::get_accessed_time(file_path, reference_time)
: utils::file::get_modified_time(file_path, reference_time))) {
std::uint64_t reference_time{};
if ((ret =
config_.get_eviction_uses_accessed_time()
? utils::file::get_accessed_time(file_path, reference_time)
: utils::file::get_modified_time(file_path, reference_time))) {
#ifdef _WIN32
const auto now = std::chrono::system_clock::now();
const auto delay = std::chrono::minutes(config_.get_eviction_delay_mins());
ret = ((std::chrono::system_clock::from_time_t(reference_time) + delay) <= now);
const auto delay =
std::chrono::minutes(config_.get_eviction_delay_mins());
ret = ((std::chrono::system_clock::from_time_t(reference_time) + delay) <=
now);
#else
const auto now = utils::get_time_now();
const auto delay = (config_.get_eviction_delay_mins() * 60L) * NANOS_PER_SECOND;
const auto delay =
(config_.get_eviction_delay_mins() * 60L) * NANOS_PER_SECOND;
ret = ((reference_time + delay) <= now);
#endif
}
@@ -111,8 +65,9 @@ bool eviction::check_minimum_requirements(const std::string &file_path) {
return ret;
}
std::deque<std::string> eviction::get_filtered_cached_files() {
auto list = utils::file::get_directory_files(config_.get_cache_directory(), true);
auto eviction::get_filtered_cached_files() -> std::deque<std::string> {
auto list =
utils::file::get_directory_files(config_.get_cache_directory(), true);
list.erase(std::remove_if(list.begin(), list.end(),
[this](const std::string &path) -> bool {
return not this->check_minimum_requirements(path);
@@ -121,25 +76,66 @@ std::deque<std::string> eviction::get_filtered_cached_files() {
return list;
}
void eviction::start() {
mutex_lock l(start_stop_mutex_);
if (not eviction_thread_) {
stop_requested_ = false;
eviction_thread_ = std::make_unique<std::thread>([this] { this->check_items_thread(); });
}
}
void eviction::service_function() {
auto should_evict = true;
void eviction::stop() {
mutex_lock l(start_stop_mutex_);
if (eviction_thread_) {
event_system::instance().raise<service_shutdown>("eviction");
stop_requested_ = true;
{
mutex_lock l2(eviction_mutex_);
stop_notify_.notify_all();
// Handle maximum cache size eviction
auto used_bytes =
utils::file::calculate_used_space(config_.get_cache_directory(), false);
if (config_.get_enable_max_cache_size()) {
should_evict = (used_bytes > config_.get_max_cache_size_bytes());
}
if (should_evict) {
// Remove cached source files that don't meet minimum requirements
auto cached_files_list = get_filtered_cached_files();
while (not get_stop_requested() && should_evict &&
not cached_files_list.empty()) {
try {
std::string api_path;
if (provider_.get_api_path_from_source(
cached_files_list.front(), api_path) == api_error::success) {
api_file file{};
filesystem_item fsi{};
if (provider_.get_filesystem_item_and_file(api_path, file, fsi) ==
api_error::success) {
// Only evict files that match expected size
std::uint64_t file_size{};
if (utils::file::get_file_size(cached_files_list.front(),
file_size)) {
if (file_size == fsi.size) {
// Try to evict file
if (fm_.evict_file(fsi.api_path) &&
config_.get_enable_max_cache_size()) {
// Restrict number of items evicted if maximum cache size is
// enabled
used_bytes -= file_size;
should_evict =
(used_bytes > config_.get_max_cache_size_bytes());
}
}
} else {
utils::error::raise_api_path_error(
__FUNCTION__, file.api_path, file.source_path,
utils::get_last_error_code(), "failed to get file size");
}
}
}
} catch (const std::exception &ex) {
utils::error::raise_error(__FUNCTION__, ex,
"failed to process cached file|sp|" +
cached_files_list.front());
}
cached_files_list.pop_front();
}
}
if (not get_stop_requested()) {
unique_mutex_lock lock(get_mutex());
if (not get_stop_requested()) {
get_notify().wait_for(lock, 30s);
}
eviction_thread_->join();
eviction_thread_.reset();
}
}
} // namespace repertory

911
src/drives/fuse/fuse_base.cpp
View File

File diff suppressed because it is too large Load Diff

975
src/drives/fuse/fuse_drive.cpp
View File

File diff suppressed because it is too large Load Diff

353
src/drives/fuse/fuse_drive_base.cpp Normal file
View File

@@ -0,0 +1,353 @@
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#ifndef _WIN32
#include "drives/fuse/fuse_drive_base.hpp"
#include "providers/i_provider.hpp"
namespace repertory {
auto fuse_drive_base::access_impl(std::string api_path, int mask) -> api_error {
return check_access(api_path, mask);
}
auto fuse_drive_base::check_access(const std::string &api_path, int mask) const
-> api_error {
api_meta_map meta;
const auto res = get_item_meta(api_path, meta);
if (res != api_error::success) {
return res;
}
// Always allow root
auto current_uid = get_current_uid();
if (current_uid != 0) {
// Always allow forced user
if (not forced_uid_.has_value() || (current_uid != get_effective_uid())) {
// Always allow if checking file exists
if (F_OK != mask) {
const auto effective_uid =
(forced_uid_.has_value() ? forced_uid_.value()
: get_uid_from_meta(meta));
const auto effective_gid =
(forced_gid_.has_value() ? forced_gid_.value()
: get_gid_from_meta(meta));
// Create file mode
mode_t effective_mode =
forced_umask_.has_value()
? ((S_IRWXU | S_IRWXG | S_IRWXO) & (~forced_umask_.value()))
: get_mode_from_meta(meta);
// Create access mask
mode_t active_mask = S_IRWXO;
if (current_uid == effective_uid) {
active_mask |= S_IRWXU;
}
if (get_current_gid() == effective_gid) {
active_mask |= S_IRWXG;
}
if (utils::is_uid_member_of_group(current_uid, effective_gid)) {
active_mask |= S_IRWXG;
}
// Calculate effective file mode
effective_mode &= active_mask;
// Check allow execute
if ((mask & X_OK) == X_OK) {
if ((effective_mode & (S_IXUSR | S_IXGRP | S_IXOTH)) == 0) {
return api_error::permission_denied;
}
}
// Check allow write
if ((mask & W_OK) == W_OK) {
if ((effective_mode & (S_IWUSR | S_IWGRP | S_IWOTH)) == 0) {
return api_error::access_denied;
}
}
// Check allow read
if ((mask & R_OK) == R_OK) {
if ((effective_mode & (S_IRUSR | S_IRGRP | S_IROTH)) == 0) {
return api_error::access_denied;
}
}
if (effective_mode == 0) {
// Deny access if effective mode is 0
return api_error::access_denied;
}
}
}
}
return api_error::success;
}
auto fuse_drive_base::check_and_perform(
const std::string &api_path, int parent_mask,
const std::function<api_error(api_meta_map &meta)> &action) -> api_error {
api_meta_map meta;
auto ret = get_item_meta(api_path, meta);
if (ret != api_error::success) {
return ret;
}
if ((ret = check_parent_access(api_path, parent_mask)) !=
api_error::success) {
return ret;
}
if ((ret = check_owner(meta)) != api_error::success) {
return ret;
}
return action(meta);
}
auto fuse_drive_base::check_open_flags(int flags, int mask,
const api_error &fail_error)
-> api_error {
return ((flags & mask) ? fail_error : api_error::success);
}
auto fuse_drive_base::check_owner(const std::string &api_path) const
-> api_error {
api_meta_map meta{};
auto ret = get_item_meta(api_path, meta);
if (ret == api_error::success) {
ret = check_owner(meta);
}
return ret;
}
auto fuse_drive_base::check_owner(const api_meta_map &meta) const -> api_error {
// Always allow root
auto current_uid = get_current_uid();
if ((current_uid != 0) &&
// Always allow forced UID
(not forced_uid_.has_value() || (current_uid != get_effective_uid())) &&
// Check if current uid matches file uid
(get_uid_from_meta(meta) != get_effective_uid())) {
return api_error::permission_denied;
}
return api_error::success;
}
auto fuse_drive_base::check_parent_access(const std::string &api_path,
int mask) const -> api_error {
auto ret = api_error::success;
// Ignore root
if (api_path != "/") {
if ((mask & X_OK) == X_OK) {
for (auto parent = utils::path::get_parent_directory(api_path);
(ret == api_error::success) && not parent.empty();
parent = utils::path::get_parent_directory(parent)) {
if (((ret = check_access(parent, X_OK)) == api_error::success) &&
(parent == "/")) {
break;
}
}
}
if (ret == api_error::success) {
mask &= (~X_OK);
if (mask != 0) {
ret = check_access(utils::path::get_parent_directory(api_path), mask);
}
}
}
return ret;
}
auto fuse_drive_base::check_readable(int flags, const api_error &fail_error)
-> api_error {
const auto mode = (flags & O_ACCMODE);
return ((mode == O_WRONLY) ? fail_error : api_error::success);
}
auto fuse_drive_base::check_writeable(int flags, const api_error &fail_error)
-> api_error {
return ((flags & O_ACCMODE) ? api_error::success : fail_error);
}
auto fuse_drive_base::get_current_gid() const -> gid_t {
auto *ctx = fuse_get_context();
return ctx ? ctx->gid : getgid();
}
auto fuse_drive_base::get_current_uid() const -> uid_t {
auto *ctx = fuse_get_context();
return ctx ? ctx->uid : getuid();
}
auto fuse_drive_base::get_effective_gid() const -> gid_t {
return forced_gid_.has_value() ? forced_gid_.value() : get_current_gid();
}
auto fuse_drive_base::get_effective_uid() const -> uid_t {
return forced_uid_.has_value() ? forced_uid_.value() : get_current_uid();
}
#ifdef __APPLE__
auto fuse_drive_base::get_flags_from_meta(const api_meta_map &meta)
-> __uint32_t {
return utils::string::to_uint32(meta.at(META_OSXFLAGS));
}
#endif // __APPLE__
auto fuse_drive_base::get_gid_from_meta(const api_meta_map &meta) -> gid_t {
return static_cast<gid_t>(utils::string::to_uint32(meta.at(META_GID)));
}
auto fuse_drive_base::get_mode_from_meta(const api_meta_map &meta) -> mode_t {
return static_cast<mode_t>(utils::string::to_uint32(meta.at(META_MODE)));
}
void fuse_drive_base::get_timespec_from_meta(const api_meta_map &meta,
const std::string &name,
struct timespec &ts) {
const auto t = utils::string::to_uint64(meta.at(name));
ts.tv_nsec = t % NANOS_PER_SECOND;
ts.tv_sec = t / NANOS_PER_SECOND;
}
auto fuse_drive_base::get_uid_from_meta(const api_meta_map &meta) -> uid_t {
return static_cast<uid_t>(utils::string::to_uint32(meta.at(META_UID)));
}
#ifdef __APPLE__
auto fuse_drive_base::parse_xattr_parameters(const char *name,
const uint32_t &position,
std::string &attribute_name,
const std::string &api_path)
-> api_error {
#else
auto fuse_drive_base::parse_xattr_parameters(const char *name,
std::string &attribute_name,
const std::string &api_path)
-> api_error {
#endif
auto res = api_path.empty() ? api_error::bad_address : api_error::success;
if (res != api_error::success) {
return res;
}
if (not name) {
return api_error::bad_address;
}
attribute_name = std::string(name);
#ifdef __APPLE__
if (attribute_name == A_KAUTH_FILESEC_XATTR) {
char new_name[MAXPATHLEN] = {0};
memcpy(new_name, A_KAUTH_FILESEC_XATTR, sizeof(A_KAUTH_FILESEC_XATTR));
memcpy(new_name, G_PREFIX, sizeof(G_PREFIX) - 1);
attribute_name = new_name;
} else if (attribute_name.empty() ||
((attribute_name != XATTR_RESOURCEFORK_NAME) && (position != 0))) {
return api_error::invalid_operation;
}
#endif
return api_error::success;
}
#ifdef __APPLE__
auto fuse_drive_base::parse_xattr_parameters(
const char *name, const char *value, size_t size, const uint32_t &position,
std::string &attribute_name, const std::string &api_path) -> api_error {
auto res = parse_xattr_parameters(name, position, attribute_name, api_path);
#else
auto fuse_drive_base::parse_xattr_parameters(const char *name,
const char *value, size_t size,
std::string &attribute_name,
const std::string &api_path)
-> api_error {
auto res = parse_xattr_parameters(name, attribute_name, api_path);
#endif
if (res != api_error::success) {
return res;
}
return (value ? api_error::success
: (size ? api_error::bad_address : api_error::success));
}
void fuse_drive_base::populate_stat(const std::string &api_path,
std::uint64_t size_or_count,
const api_meta_map &meta, bool directory,
i_provider &provider, struct stat *st) {
memset(st, 0, sizeof(struct stat));
st->st_nlink =
(directory
? 2 + (size_or_count ? size_or_count
: provider.get_directory_item_count(api_path))
: 1);
if (directory) {
st->st_blocks = 0;
} else {
st->st_size = size_or_count;
static const auto block_size_stat = static_cast<std::uint64_t>(512u);
static const auto block_size = static_cast<std::uint64_t>(4096u);
const auto size = utils::divide_with_ceiling(
static_cast<std::uint64_t>(st->st_size), block_size) *
block_size;
st->st_blocks = std::max(block_size / block_size_stat,
utils::divide_with_ceiling(size, block_size_stat));
}
st->st_gid = get_gid_from_meta(meta);
st->st_mode = (directory ? S_IFDIR : S_IFREG) | get_mode_from_meta(meta);
st->st_uid = get_uid_from_meta(meta);
#ifdef __APPLE__
st->st_blksize = 0;
st->st_flags = get_flags_from_meta(meta);
set_timespec_from_meta(meta, META_MODIFIED, st->st_mtimespec);
set_timespec_from_meta(meta, META_CREATION, st->st_birthtimespec);
set_timespec_from_meta(meta, META_CHANGED, st->st_ctimespec);
set_timespec_from_meta(meta, META_ACCESSED, st->st_atimespec);
#else // __APPLE__
st->st_blksize = 4096;
set_timespec_from_meta(meta, META_MODIFIED, st->st_mtim);
set_timespec_from_meta(meta, META_CREATION, st->st_ctim);
set_timespec_from_meta(meta, META_ACCESSED, st->st_atim);
#endif // __APPLE__
}
void fuse_drive_base::set_timespec_from_meta(const api_meta_map &meta,
const std::string &name,
struct timespec &ts) {
const auto t = utils::string::to_uint64(meta.at(name));
ts.tv_nsec = t % NANOS_PER_SECOND;
ts.tv_sec = t / NANOS_PER_SECOND;
}
} // namespace repertory
#endif // _WIN32

652
src/drives/fuse/remotefuse/remote_client.cpp
View File

File diff suppressed because it is too large Load Diff

878
src/drives/fuse/remotefuse/remote_fuse_drive.cpp
View File

@@ -1,52 +1,258 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#ifndef _WIN32
#include "drives/fuse/remotefuse/remote_fuse_drive.hpp"
#include "app_config.hpp"
#include "drives/fuse/events.hpp"
#include "events/consumers/console_consumer.hpp"
#include "events/consumers/logging_consumer.hpp"
#include "events/events.hpp"
#include "events/event_system.hpp"
#include "events/events.hpp"
#include "platform/platform.hpp"
#include "rpc/server/server.hpp"
#include "types/remote.hpp"
#include "utils/error_utils.hpp"
#include "utils/file_utils.hpp"
#include "utils/path_utils.hpp"
#include "utils/utils.hpp"
namespace repertory::remote_fuse {
app_config *remote_fuse_drive::remote_fuse_impl::config_ = nullptr;
lock_data *remote_fuse_drive::remote_fuse_impl::lock_ = nullptr;
std::string *remote_fuse_drive::remote_fuse_impl::mount_location_ = nullptr;
bool remote_fuse_drive::remote_fuse_impl::console_enabled_ = false;
bool remote_fuse_drive::remote_fuse_impl::was_mounted_ = false;
std::optional<gid_t> remote_fuse_drive::remote_fuse_impl::forced_gid_;
std::optional<uid_t> remote_fuse_drive::remote_fuse_impl::forced_uid_;
std::optional<mode_t> remote_fuse_drive::remote_fuse_impl::forced_umask_;
remote_instance_factory *remote_fuse_drive::remote_fuse_impl::factory_ = nullptr;
std::unique_ptr<logging_consumer> remote_fuse_drive::remote_fuse_impl::logging_consumer_;
std::unique_ptr<console_consumer> remote_fuse_drive::remote_fuse_impl::console_consumer_;
std::unique_ptr<i_remote_instance> remote_fuse_drive::remote_fuse_impl::remote_instance_;
std::unique_ptr<server> remote_fuse_drive::remote_fuse_impl::server_;
auto remote_fuse_drive::access_impl(std::string api_path, int mask)
-> api_error {
return utils::to_api_error(
remote_instance_->fuse_access(api_path.c_str(), mask));
}
void remote_fuse_drive::remote_fuse_impl::tear_down(const int &ret) {
#ifdef __APPLE__
api_error remote_fuse_drive::chflags_impl(std::string api_path,
uint32_t flags) {
return utils::to_api_error(
remote_instance_->fuse_chflags(api_path.c_str(), flags));
}
#endif // __APPLE__
#if FUSE_USE_VERSION >= 30
auto remote_fuse_drive::chmod_impl(std::string api_path, mode_t mode,
struct fuse_file_info * /*fi*/)
-> api_error {
#else
auto remote_fuse_drive::chmod_impl(std::string api_path, mode_t mode)
-> api_error {
#endif
return utils::to_api_error(
remote_instance_->fuse_chmod(api_path.c_str(), mode));
}
#if FUSE_USE_VERSION >= 30
auto remote_fuse_drive::chown_impl(std::string api_path, uid_t uid, gid_t gid,
struct fuse_file_info * /*fi*/)
-> api_error {
#else
auto remote_fuse_drive::chown_impl(std::string api_path, uid_t uid, gid_t gid)
-> api_error {
#endif
return utils::to_api_error(
remote_instance_->fuse_chown(api_path.c_str(), uid, gid));
}
auto remote_fuse_drive::create_impl(std::string api_path, mode_t mode,
struct fuse_file_info *fi) -> api_error {
return utils::to_api_error(remote_instance_->fuse_create(
api_path.c_str(), mode, remote::create_open_flags(fi->flags), fi->fh));
}
void remote_fuse_drive::destroy_impl(void * /*ptr*/) {
event_system::instance().raise<drive_unmount_pending>(get_mount_location());
if (server_) {
server_->stop();
server_.reset();
}
if (remote_instance_) {
const auto res = remote_instance_->fuse_destroy();
if (res != 0) {
utils::error::raise_error(__FUNCTION__,
"remote fuse_destroy() failed|err|" +
std::to_string(res));
}
remote_instance_.reset();
}
if (not lock_data_.set_mount_state(false, "", -1)) {
utils::error::raise_error(__FUNCTION__, "failed to set mount state");
}
event_system::instance().raise<drive_unmounted>(get_mount_location());
}
auto remote_fuse_drive::fgetattr_impl(std::string api_path, struct stat *st,
struct fuse_file_info *fi) -> api_error {
remote::stat r{};
auto directory = false;
const auto res =
remote_instance_->fuse_fgetattr(api_path.c_str(), r, directory, fi->fh);
if (res == 0) {
populate_stat(r, directory, *st);
}
return utils::to_api_error(res);
}
#ifdef __APPLE__
api_error remote_fuse_drive::fsetattr_x_impl(std::string api_path,
struct setattr_x *attr,
struct fuse_file_info *fi) {
remote::setattr_x attributes{};
attributes.valid = attr->valid;
attributes.mode = attr->mode;
attributes.uid = attr->uid;
attributes.gid = attr->gid;
attributes.size = attr->size;
attributes.acctime =
((attr->acctime.tv_sec * NANOS_PER_SECOND) + attr->acctime.tv_nsec);
attributes.modtime =
((attr->modtime.tv_sec * NANOS_PER_SECOND) + attr->modtime.tv_nsec);
attributes.crtime =
((attr->crtime.tv_sec * NANOS_PER_SECOND) + attr->crtime.tv_nsec);
attributes.chgtime =
((attr->chgtime.tv_sec * NANOS_PER_SECOND) + attr->chgtime.tv_nsec);
attributes.bkuptime =
((attr->bkuptime.tv_sec * NANOS_PER_SECOND) + attr->bkuptime.tv_nsec);
attributes.flags = attr->flags;
return utils::to_api_error(
remote_instance_->fuse_fsetattr_x(api_path.c_str(), attributes, fi->fh));
}
#endif
auto remote_fuse_drive::fsync_impl(std::string api_path, int datasync,
struct fuse_file_info *fi) -> api_error {
return utils::to_api_error(
remote_instance_->fuse_fsync(api_path.c_str(), datasync, fi->fh));
}
#if FUSE_USE_VERSION < 30
auto remote_fuse_drive::ftruncate_impl(std::string api_path, off_t size,
struct fuse_file_info *fi) -> api_error {
return utils::to_api_error(
remote_instance_->fuse_ftruncate(api_path.c_str(), size, fi->fh));
}
#endif
#if FUSE_USE_VERSION >= 30
auto remote_fuse_drive::getattr_impl(std::string api_path, struct stat *st,
struct fuse_file_info * /*fi*/)
-> api_error {
#else
auto remote_fuse_drive::getattr_impl(std::string api_path, struct stat *st)
-> api_error {
#endif
bool directory = false;
remote::stat r{};
const auto res =
remote_instance_->fuse_getattr(api_path.c_str(), r, directory);
if (res == 0) {
populate_stat(r, directory, *st);
}
return utils::to_api_error(res);
}
#ifdef __APPLE__
api_error remote_fuse_drive::getxtimes_impl(std::string api_path,
struct timespec *bkuptime,
struct timespec *crtime) {
if (not(bkuptime && crtime)) {
return utils::to_api_error(-EFAULT);
}
remote::file_time repertory_bkuptime = 0u;
remote::file_time repertory_crtime = 0u;
const auto res = remote_instance_->fuse_getxtimes(
api_path.c_str(), repertory_bkuptime, repertory_crtime);
if (res == 0) {
bkuptime->tv_nsec =
static_cast<long>(repertory_bkuptime % NANOS_PER_SECOND);
bkuptime->tv_sec = repertory_bkuptime / NANOS_PER_SECOND;
crtime->tv_nsec = static_cast<long>(repertory_crtime % NANOS_PER_SECOND);
crtime->tv_sec = repertory_crtime / NANOS_PER_SECOND;
}
return utils::to_api_error(res);
}
#endif // __APPLE__
#if FUSE_USE_VERSION >= 30
auto remote_fuse_drive::init_impl(struct fuse_conn_info *conn,
struct fuse_config *cfg) -> void * {
#else
auto remote_fuse_drive::init_impl(struct fuse_conn_info *conn) -> void * {
#endif
utils::file::change_to_process_directory();
was_mounted_ = true;
if (console_enabled_) {
console_consumer_ = std::make_shared<console_consumer>();
}
logging_consumer_ = std::make_shared<logging_consumer>(
config_.get_log_directory(), config_.get_event_level());
event_system::instance().start();
if (not lock_data_.set_mount_state(true, get_mount_location(), getpid())) {
utils::error::raise_error(__FUNCTION__, "failed to set mount state");
}
remote_instance_ = factory_();
remote_instance_->set_fuse_uid_gid(getuid(), getgid());
if (remote_instance_->fuse_init() != 0) {
utils::error::raise_error(__FUNCTION__,
"failed to connect to remote server");
event_system::instance().raise<unmount_requested>();
} else {
server_ = std::make_shared<server>(config_);
server_->start();
event_system::instance().raise<drive_mounted>(get_mount_location());
}
#if FUSE_USE_VERSION >= 30
return fuse_base::init_impl(conn, cfg);
#else
return fuse_base::init_impl(conn);
#endif
}
auto remote_fuse_drive::mkdir_impl(std::string api_path, mode_t mode)
-> api_error {
return utils::to_api_error(
remote_instance_->fuse_mkdir(api_path.c_str(), mode));
}
void remote_fuse_drive::notify_fuse_main_exit(int &ret) {
if (was_mounted_) {
event_system::instance().raise<drive_mount_result>(std::to_string(ret));
event_system::instance().stop();
@@ -55,8 +261,21 @@ void remote_fuse_drive::remote_fuse_impl::tear_down(const int &ret) {
}
}
void remote_fuse_drive::remote_fuse_impl::populate_stat(const remote::stat &r,
const bool &directory, struct stat &st) {
auto remote_fuse_drive::open_impl(std::string api_path,
struct fuse_file_info *fi) -> api_error {
return utils::to_api_error(remote_instance_->fuse_open(
api_path.c_str(), remote::create_open_flags(fi->flags), fi->fh));
}
auto remote_fuse_drive::opendir_impl(std::string api_path,
struct fuse_file_info *fi) -> api_error {
return utils::to_api_error(
remote_instance_->fuse_opendir(api_path.c_str(), fi->fh));
}
void remote_fuse_drive::populate_stat(const remote::stat &r, bool directory,
struct stat &st) {
memset(&st, 0, sizeof(struct stat));
#ifdef __APPLE__
@@ -88,12 +307,13 @@ void remote_fuse_drive::remote_fuse_impl::populate_stat(const remote::stat &r,
st.st_mtim.tv_sec = r.st_mtimespec / NANOS_PER_SECOND;
#endif
if (not directory) {
const auto blockSizeStat = static_cast<std::uint64_t>(512);
const auto blockSize = static_cast<std::uint64_t>(4096);
const auto size =
utils::divide_with_ceiling(static_cast<std::uint64_t>(st.st_size), blockSize) * blockSize;
st.st_blocks =
std::max(blockSize / blockSizeStat, utils::divide_with_ceiling(size, blockSizeStat));
const auto block_size_stat = static_cast<std::uint64_t>(512u);
const auto block_size = static_cast<std::uint64_t>(4096u);
const auto size = utils::divide_with_ceiling(
static_cast<std::uint64_t>(st.st_size), block_size) *
block_size;
st.st_blocks = std::max(block_size / block_size_stat,
utils::divide_with_ceiling(size, block_size_stat));
}
st.st_gid = r.st_gid;
@@ -104,316 +324,168 @@ void remote_fuse_drive::remote_fuse_impl::populate_stat(const remote::stat &r,
st.st_uid = r.st_uid;
}
int remote_fuse_drive::remote_fuse_impl::repertory_access(const char *path, int mask) {
return remote_instance_->fuse_access(path, mask);
auto remote_fuse_drive::read_impl(std::string api_path, char *buffer,
size_t read_size, off_t read_offset,
struct fuse_file_info *fi,
std::size_t &bytes_read) -> api_error {
auto res = remote_instance_->fuse_read(api_path.c_str(), buffer, read_size,
read_offset, fi->fh);
if (res >= 0) {
bytes_read = res;
return api_error::success;
}
return utils::to_api_error(res);
}
#ifdef __APPLE__
int remote_fuse_drive::remote_fuse_impl::repertory_chflags(const char *path, uint32_t flags) {
return remote_instance_->fuse_chflags(path, flags);
}
#if FUSE_USE_VERSION >= 30
auto remote_fuse_drive::readdir_impl(std::string api_path, void *buf,
fuse_fill_dir_t fuse_fill_dir,
off_t offset, struct fuse_file_info *fi,
fuse_readdir_flags /*flags*/)
-> api_error {
#else
auto remote_fuse_drive::readdir_impl(std::string api_path, void *buf,
fuse_fill_dir_t fuse_fill_dir,
off_t offset, struct fuse_file_info *fi)
-> api_error {
#endif
int remote_fuse_drive::remote_fuse_impl::repertory_chmod(const char *path, mode_t mode) {
return remote_instance_->fuse_chmod(path, mode);
}
int remote_fuse_drive::remote_fuse_impl::repertory_chown(const char *path, uid_t uid, gid_t gid) {
return remote_instance_->fuse_chown(path, uid, gid);
}
int remote_fuse_drive::remote_fuse_impl::repertory_create(const char *path, mode_t mode,
struct fuse_file_info *fi) {
return remote_instance_->fuse_create(path, mode, remote::open_flags(fi->flags), fi->fh);
}
void remote_fuse_drive::remote_fuse_impl::repertory_destroy(void * /*ptr*/) {
event_system::instance().raise<drive_unmount_pending>(*mount_location_);
if (server_) {
server_->stop();
server_.reset();
}
remote_instance_->fuse_destroy();
remote_instance_.reset();
lock_->set_mount_state(false, "", -1);
event_system::instance().raise<drive_mounted>(*mount_location_);
}
/*int remote_fuse_drive::remote_fuse_impl::repertory_fallocate(const char *path, int mode, off_t
offset, off_t length, struct fuse_file_info *fi) { return remote_instance_->fuse_fallocate(path,
mode, offset, length, fi->fh);
}*/
int remote_fuse_drive::remote_fuse_impl::repertory_fgetattr(const char *path, struct stat *st,
struct fuse_file_info *fi) {
remote::stat r{};
bool directory = false;
auto ret = remote_instance_->fuse_fgetattr(path, r, directory, fi->fh);
if (ret == 0) {
populate_stat(r, directory, *st);
}
return ret;
}
#ifdef __APPLE__
int remote_fuse_drive::remote_fuse_impl::repertory_fsetattr_x(const char *path,
struct setattr_x *attr,
struct fuse_file_info *fi) {
remote::setattr_x attrRepertory{};
attrRepertory.valid = attr->valid;
attrRepertory.mode = attr->mode;
attrRepertory.uid = attr->uid;
attrRepertory.gid = attr->gid;
attrRepertory.size = attr->size;
attrRepertory.acctime = ((attr->acctime.tv_sec * NANOS_PER_SECOND) + attr->acctime.tv_nsec);
attrRepertory.modtime = ((attr->modtime.tv_sec * NANOS_PER_SECOND) + attr->modtime.tv_nsec);
attrRepertory.crtime = ((attr->crtime.tv_sec * NANOS_PER_SECOND) + attr->crtime.tv_nsec);
attrRepertory.chgtime = ((attr->chgtime.tv_sec * NANOS_PER_SECOND) + attr->chgtime.tv_nsec);
attrRepertory.bkuptime = ((attr->bkuptime.tv_sec * NANOS_PER_SECOND) + attr->bkuptime.tv_nsec);
attrRepertory.flags = attr->flags;
return remote_instance_->fuse_fsetattr_x(path, attrRepertory, fi->fh);
}
#endif
int remote_fuse_drive::remote_fuse_impl::repertory_fsync(const char *path, int datasync,
struct fuse_file_info *fi) {
return remote_instance_->fuse_fsync(path, datasync, fi->fh);
}
int remote_fuse_drive::remote_fuse_impl::repertory_ftruncate(const char *path, off_t size,
struct fuse_file_info *fi) {
return remote_instance_->fuse_ftruncate(path, size, fi->fh);
}
int remote_fuse_drive::remote_fuse_impl::repertory_getattr(const char *path, struct stat *st) {
bool directory = false;
remote::stat r{};
const auto ret = remote_instance_->fuse_getattr(path, r, directory);
if (ret == 0) {
populate_stat(r, directory, *st);
}
return ret;
}
#ifdef __APPLE__
int remote_fuse_drive::remote_fuse_impl::repertory_getxtimes(const char *path,
struct timespec *bkuptime,
struct timespec *crtime) {
if (not(bkuptime && crtime)) {
return -EFAULT;
} else {
remote::file_time repertory_bkuptime = 0;
remote::file_time repertory_crtime = 0;
const auto ret = remote_instance_->fuse_getxtimes(path, repertory_bkuptime, repertory_crtime);
if (ret == 0) {
bkuptime->tv_nsec = static_cast<long>(repertory_bkuptime % NANOS_PER_SECOND);
bkuptime->tv_sec = repertory_bkuptime / NANOS_PER_SECOND;
crtime->tv_nsec = static_cast<long>(repertory_crtime % NANOS_PER_SECOND);
crtime->tv_sec = repertory_crtime / NANOS_PER_SECOND;
}
return ret;
}
}
#endif
void *remote_fuse_drive::remote_fuse_impl::repertory_init(struct fuse_conn_info *conn) {
utils::file::change_to_process_directory();
#ifdef __APPLE__
conn->want |= FUSE_CAP_VOL_RENAME;
conn->want |= FUSE_CAP_XTIMES;
#endif
conn->want |= FUSE_CAP_BIG_WRITES;
if (console_enabled_) {
console_consumer_ = std::make_unique<console_consumer>();
}
logging_consumer_ =
std::make_unique<logging_consumer>(config_->get_log_directory(), config_->get_event_level());
event_system::instance().start();
was_mounted_ = true;
lock_->set_mount_state(true, *mount_location_, getpid());
remote_instance_ = (*factory_)();
remote_instance_->set_fuse_uid_gid(getuid(), getgid());
if (remote_instance_->fuse_init() != 0) {
event_system::instance().raise<repertory_exception>(__FUNCTION__,
"Failed to connect to remote server");
event_system::instance().raise<unmount_requested>();
} else {
server_ = std::make_unique<server>(*config_);
server_->start();
event_system::instance().raise<drive_mounted>(*mount_location_);
}
return nullptr;
}
int remote_fuse_drive::remote_fuse_impl::repertory_mkdir(const char *path, mode_t mode) {
return remote_instance_->fuse_mkdir(path, mode);
}
int remote_fuse_drive::remote_fuse_impl::repertory_open(const char *path,
struct fuse_file_info *fi) {
return remote_instance_->fuse_open(path, remote::open_flags(fi->flags), fi->fh);
}
int remote_fuse_drive::remote_fuse_impl::repertory_opendir(const char *path,
struct fuse_file_info *fi) {
return remote_instance_->fuse_opendir(path, fi->fh);
}
int remote_fuse_drive::remote_fuse_impl::repertory_read(const char *path, char *buffer,
size_t readSize, off_t readOffset,
struct fuse_file_info *fi) {
return remote_instance_->fuse_read(path, buffer, readSize, readOffset, fi->fh);
}
int remote_fuse_drive::remote_fuse_impl::repertory_readdir(const char *path, void *buf,
fuse_fill_dir_t fuseFillDir,
off_t offset,
struct fuse_file_info *fi) {
std::string item_path;
int ret = 0;
while ((ret = remote_instance_->fuse_readdir(path, offset, fi->fh, item_path)) == 0) {
int res = 0;
while ((res = remote_instance_->fuse_readdir(api_path.c_str(), offset, fi->fh,
item_path)) == 0) {
if ((item_path != ".") && (item_path != "..")) {
item_path = utils::path::strip_to_file_name(item_path);
}
if (fuseFillDir(buf, &item_path[0], nullptr, ++offset) != 0) {
#if FUSE_USE_VERSION >= 30
if (fuse_fill_dir(buf, item_path.c_str(), nullptr, ++offset,
static_cast<fuse_fill_dir_flags>(0)) != 0) {
#else
if (fuse_fill_dir(buf, item_path.c_str(), nullptr, ++offset) != 0) {
#endif
break;
}
}
if (ret == -120) {
ret = 0;
if (res == -120) {
res = 0;
}
return ret;
return utils::to_api_error(res);
}
int remote_fuse_drive::remote_fuse_impl::repertory_release(const char *path,
struct fuse_file_info *fi) {
return remote_instance_->fuse_release(path, fi->fh);
auto remote_fuse_drive::release_impl(std::string api_path,
struct fuse_file_info *fi) -> api_error {
return utils::to_api_error(
remote_instance_->fuse_release(api_path.c_str(), fi->fh));
}
int remote_fuse_drive::remote_fuse_impl::repertory_releasedir(const char *path,
struct fuse_file_info *fi) {
return remote_instance_->fuse_releasedir(path, fi->fh);
auto remote_fuse_drive::releasedir_impl(std::string api_path,
struct fuse_file_info *fi)
-> api_error {
return utils::to_api_error(
remote_instance_->fuse_releasedir(api_path.c_str(), fi->fh));
}
int remote_fuse_drive::remote_fuse_impl::repertory_rename(const char *from, const char *to) {
return remote_instance_->fuse_rename(from, to);
}
int remote_fuse_drive::remote_fuse_impl::repertory_rmdir(const char *path) {
return remote_instance_->fuse_rmdir(path);
}
/*
#ifdef HAS_SETXATTR
#ifdef __APPLE__
int remote_fuse_drive::remote_fuse_impl::repertory_getxattr(const char *path, const char *name,
char *value, size_t size, uint32_t position) { return remote_instance_->fuse_getxattr_osx(path,
name, value, size, position);
}
#if FUSE_USE_VERSION >= 30
auto remote_fuse_drive::rename_impl(std::string from_api_path,
std::string to_api_path,
unsigned int /*flags*/) -> api_error {
#else
int remote_fuse_drive::remote_fuse_impl::repertory_getxattr(const char *path, const char *name,
char *value, size_t size) { return remote_instance_->fuse_getxattr(path, name, value, size);
}
auto remote_fuse_drive::rename_impl(std::string from_api_path,
std::string to_api_path) -> api_error {
#endif
return utils::to_api_error(remote_instance_->fuse_rename(
from_api_path.c_str(), to_api_path.c_str()));
}
#endif
int remote_fuse_drive::remote_fuse_impl::repertory_listxattr(const char *path, char *buffer,
size_t size) { return remote_instance_->fuse_listxattr(path, buffer, size);
}
auto remote_fuse_drive::rmdir_impl(std::string api_path) -> api_error {
return utils::to_api_error(remote_instance_->fuse_rmdir(api_path.c_str()));
}
remote_fuse_drive::remote_fuse_impl::int repertory_removexattr(const char *path, const char
*name) { return remote_instance_->fuse_removexattr(path, name);
}
#ifdef __APPLE__
int remote_fuse_drive::remote_fuse_impl::repertory_setxattr(const char *path, const char *name,
const char *value, size_t size, int flags, uint32_t position) { return
remote_instance_->fuse_setxattr_osx(path, name, value, size, flags, position);
}
#else
int remote_fuse_drive::remote_fuse_impl::repertory_setxattr(const char *path, const char *name,
const char *value, size_t size, int flags) { return remote_instance_->fuse_setxattr(path, name,
value, size, flags);
}
#endif
#endif
*/
#ifdef __APPLE__
int remote_fuse_drive::remote_fuse_impl::repertory_setattr_x(const char *path,
struct setattr_x *attr) {
api_error remote_fuse_drive::setattr_x_impl(std::string api_path,
struct setattr_x *attr) {
remote::setattr_x attributes{};
attributes.valid = attr->valid;
attributes.mode = attr->mode;
attributes.uid = attr->uid;
attributes.gid = attr->gid;
attributes.size = attr->size;
attributes.acctime = ((attr->acctime.tv_sec * NANOS_PER_SECOND) + attr->acctime.tv_nsec);
attributes.modtime = ((attr->modtime.tv_sec * NANOS_PER_SECOND) + attr->modtime.tv_nsec);
attributes.crtime = ((attr->crtime.tv_sec * NANOS_PER_SECOND) + attr->crtime.tv_nsec);
attributes.chgtime = ((attr->chgtime.tv_sec * NANOS_PER_SECOND) + attr->chgtime.tv_nsec);
attributes.bkuptime = ((attr->bkuptime.tv_sec * NANOS_PER_SECOND) + attr->bkuptime.tv_nsec);
attributes.acctime =
((attr->acctime.tv_sec * NANOS_PER_SECOND) + attr->acctime.tv_nsec);
attributes.modtime =
((attr->modtime.tv_sec * NANOS_PER_SECOND) + attr->modtime.tv_nsec);
attributes.crtime =
((attr->crtime.tv_sec * NANOS_PER_SECOND) + attr->crtime.tv_nsec);
attributes.chgtime =
((attr->chgtime.tv_sec * NANOS_PER_SECOND) + attr->chgtime.tv_nsec);
attributes.bkuptime =
((attr->bkuptime.tv_sec * NANOS_PER_SECOND) + attr->bkuptime.tv_nsec);
attributes.flags = attr->flags;
return remote_instance_->fuse_setattr_x(path, attributes);
return utils::to_api_error(
remote_instance_->fuse_setattr_x(api_path.c_str(), attributes));
}
int remote_fuse_drive::remote_fuse_impl::repertory_setbkuptime(const char *path,
const struct timespec *bkuptime) {
api_error remote_fuse_drive::setbkuptime_impl(std::string api_path,
const struct timespec *bkuptime) {
remote::file_time repertory_bkuptime =
((bkuptime->tv_sec * NANOS_PER_SECOND) + bkuptime->tv_nsec);
return remote_instance_->fuse_setbkuptime(path, repertory_bkuptime);
return utils::to_api_error(
remote_instance_->fuse_setbkuptime(api_path.c_str(), repertory_bkuptime));
}
int remote_fuse_drive::remote_fuse_impl::repertory_setchgtime(const char *path,
const struct timespec *chgtime) {
remote::file_time repertory_chgtime = ((chgtime->tv_sec * NANOS_PER_SECOND) + chgtime->tv_nsec);
return remote_instance_->fuse_setchgtime(path, repertory_chgtime);
api_error remote_fuse_drive::setchgtime_impl(std::string api_path,
const struct timespec *chgtime) {
remote::file_time repertory_chgtime =
((chgtime->tv_sec * NANOS_PER_SECOND) + chgtime->tv_nsec);
return utils::to_api_error(
remote_instance_->fuse_setchgtime(api_path.c_str(), repertory_chgtime));
}
int remote_fuse_drive::remote_fuse_impl::repertory_setcrtime(const char *path,
const struct timespec *crtime) {
remote::file_time repertory_crtime = ((crtime->tv_sec * NANOS_PER_SECOND) + crtime->tv_nsec);
return remote_instance_->fuse_setcrtime(path, repertory_crtime);
api_error remote_fuse_drive::setcrtime_impl(std::string api_path,
const struct timespec *crtime) {
remote::file_time repertory_crtime =
((crtime->tv_sec * NANOS_PER_SECOND) + crtime->tv_nsec);
return utils::to_api_error(
remote_instance_->fuse_setcrtime(api_path.c_str(), repertory_crtime));
}
int remote_fuse_drive::remote_fuse_impl::repertory_setvolname(const char *volname) {
return remote_instance_->fuse_setvolname(volname);
api_error remote_fuse_drive::setvolname_impl(const char *volname) {
return utils::to_api_error(remote_instance_->fuse_setvolname(volname));
}
int remote_fuse_drive::remote_fuse_impl::repertory_statfs_x(const char *path,
struct statfs *stbuf) {
auto ret = statfs(config_->get_data_directory().c_str(), stbuf);
if (ret == 0) {
api_error remote_fuse_drive::statfs_x_impl(std::string api_path,
struct statfs *stbuf) {
auto res = statfs(config_.get_data_directory().c_str(), stbuf);
if (res == 0) {
remote::statfs_x r{};
if ((ret = remote_instance_->fuse_statfs_x(path, stbuf->f_bsize, r)) == 0) {
if ((res = remote_instance_->fuse_statfs_x(api_path.c_str(), stbuf->f_bsize,
r)) == 0) {
stbuf->f_blocks = r.f_blocks;
stbuf->f_bavail = r.f_bavail;
stbuf->f_bfree = r.f_bfree;
stbuf->f_ffree = r.f_ffree;
stbuf->f_files = r.f_files;
stbuf->f_owner = getuid();
strncpy(&stbuf->f_mntonname[0], mount_location_->c_str(), MNAMELEN);
strncpy(&stbuf->f_mntfromname[0], &r.f_mntfromname[0], MNAMELEN);
strncpy(&stbuf->f_mntonname[0u], get_mount_location().c_str(), MNAMELEN);
strncpy(&stbuf->f_mntfromname[0u], &r.f_mntfromname[0], MNAMELEN);
}
} else {
ret = -errno;
res = -errno;
}
return ret;
}
#else
int remote_fuse_drive::remote_fuse_impl::repertory_statfs(const char *path, struct statvfs *stbuf) {
auto ret = statvfs(&config_->get_data_directory()[0], stbuf);
if (ret == 0) {
return utils::to_api_error(res);
}
#else // __APPLE__
auto remote_fuse_drive::statfs_impl(std::string api_path, struct statvfs *stbuf)
-> api_error {
auto res = statvfs(config_.get_data_directory().c_str(), stbuf);
if (res == 0) {
remote::statfs r{};
if ((ret = remote_instance_->fuse_statfs(path, stbuf->f_frsize, r)) == 0) {
if ((res = remote_instance_->fuse_statfs(api_path.c_str(), stbuf->f_frsize,
r)) == 0) {
stbuf->f_blocks = r.f_blocks;
stbuf->f_bavail = r.f_bavail;
stbuf->f_bfree = r.f_bfree;
@@ -422,195 +494,61 @@ int remote_fuse_drive::remote_fuse_impl::repertory_statfs(const char *path, stru
stbuf->f_files = r.f_files;
}
} else {
ret = -errno;
}
return ret;
}
#endif
int remote_fuse_drive::remote_fuse_impl::repertory_truncate(const char *path, off_t size) {
return remote_instance_->fuse_truncate(path, size);
}
int remote_fuse_drive::remote_fuse_impl::repertory_unlink(const char *path) {
return remote_instance_->fuse_unlink(path);
}
int remote_fuse_drive::remote_fuse_impl::repertory_utimens(const char *path,
const struct timespec tv[2]) {
remote::file_time rtv[2] = {0};
if (tv) {
rtv[0] = tv[0].tv_nsec + (tv[0].tv_sec * NANOS_PER_SECOND);
rtv[1] = tv[1].tv_nsec + (tv[1].tv_sec * NANOS_PER_SECOND);
}
return remote_instance_->fuse_utimens(path, rtv, tv ? tv[0].tv_nsec : 0, tv ? tv[1].tv_nsec : 0);
}
int remote_fuse_drive::remote_fuse_impl::repertory_write(const char *path, const char *buffer,
size_t writeSize, off_t writeOffset,
struct fuse_file_info *fi) {
return remote_instance_->fuse_write(path, buffer, writeSize, writeOffset, fi->fh);
}
remote_fuse_drive::remote_fuse_drive(app_config &config, lock_data &lock,
remote_instance_factory factory)
: config_(config), lock_(lock), factory_(std::move(factory)) {
E_SUBSCRIBE_EXACT(unmount_requested, [this](const unmount_requested &) {
std::thread([this]() { remote_fuse_drive::shutdown(mount_location_); }).detach();
});
}
int remote_fuse_drive::mount(std::vector<std::string> drive_args) {
remote_fuse_impl::lock_ = &lock_;
remote_fuse_impl::config_ = &config_;
remote_fuse_impl::mount_location_ = &mount_location_;
remote_fuse_impl::factory_ = &factory_;
#ifdef __APPLE__
fuse_ops_.chflags = remote_fuse_impl::repertory_chflags;
fuse_ops_.fsetattr_x = remote_fuse_impl::repertory_fsetattr_x;
fuse_ops_.getxtimes = remote_fuse_impl::repertory_getxtimes;
fuse_ops_.setattr_x = remote_fuse_impl::repertory_setattr_x;
fuse_ops_.setbkuptime = remote_fuse_impl::repertory_setbkuptime;
fuse_ops_.setchgtime = remote_fuse_impl::repertory_setchgtime;
fuse_ops_.setcrtime = remote_fuse_impl::repertory_setcrtime;
fuse_ops_.setvolname = remote_fuse_impl::repertory_setvolname;
fuse_ops_.statfs_x = remote_fuse_impl::repertory_statfs_x;
#endif
auto force_no_console = false;
for (std::size_t i = 1u; !force_no_console && (i < drive_args.size()); i++) {
if (drive_args[i] == "-nc") {
force_no_console = true;
}
}
utils::remove_element_from(drive_args, "-nc");
for (std::size_t i = 1u; i < drive_args.size(); i++) {
if (drive_args[i] == "-f") {
remote_fuse_impl::console_enabled_ = not force_no_console;
} else if (drive_args[i].find("-o") == 0) {
std::string options;
if (drive_args[i].size() == 2u) {
if ((i + 1) < drive_args.size()) {
options = drive_args[++i];
}
} else {
options = drive_args[i].substr(2);
}
const auto option_list = utils::string::split(options, ',');
for (const auto &option : option_list) {
if (option.find("gid") == 0) {
const auto parts = utils::string::split(option, '=');
if (parts.size() == 2u) {
auto gid = getgrnam(parts[1u].c_str());
if (not gid) {
gid = getgrgid(utils::string::to_uint32(parts[1]));
}
if ((getgid() != 0) && (gid->gr_gid == 0)) {
std::cerr << "'gid=0' requires running as root" << std::endl;
return -1;
} else {
remote_fuse_impl::forced_gid_ = gid->gr_gid;
}
}
} else if (option.find("uid") == 0) {
const auto parts = utils::string::split(option, '=');
if (parts.size() == 2u) {
auto uid = getpwnam(parts[1u].c_str());
if (not uid) {
uid = getpwuid(utils::string::to_uint32(parts[1]));
}
if ((getuid() != 0) && (uid->pw_uid == 0)) {
std::cerr << "'uid=0' requires running as root" << std::endl;
return -1;
} else {
remote_fuse_impl::forced_uid_ = uid->pw_uid;
}
}
} else if (option.find("umask") == 0) {
const auto parts = utils::string::split(option, '=');
if (parts.size() == 2u) {
static const auto numeric_regex = std::regex("[0-9]+");
try {
if (not std::regex_match(parts[1u], numeric_regex)) {
throw std::runtime_error("invalid syntax");
} else {
remote_fuse_impl::forced_umask_ = utils::string::to_uint32(parts[1]);
}
} catch (...) {
std::cerr << ("'" + option + "' invalid syntax") << std::endl;
return -1;
}
}
}
}
}
res = -errno;
}
std::vector<const char *> fuse_argv(drive_args.size());
for (std::size_t i = 0u; i < drive_args.size(); i++) {
fuse_argv[i] = drive_args[i].c_str();
}
struct fuse_args args =
FUSE_ARGS_INIT(static_cast<int>(fuse_argv.size()), (char **)&fuse_argv[0]);
char *mount_location = nullptr;
fuse_parse_cmdline(&args, &mount_location, nullptr, nullptr);
if (mount_location) {
mount_location_ = mount_location;
free(mount_location);
}
std::string args_string;
for (const auto *arg : fuse_argv) {
if (args_string.empty()) {
args_string += arg;
} else {
args_string += (" " + std::string(arg));
}
}
event_system::instance().raise<fuse_args_parsed>(args_string);
umask(0);
const auto ret = fuse_main(static_cast<int>(fuse_argv.size()), (char **)&fuse_argv[0], &fuse_ops_,
&mount_location);
remote_fuse_impl::tear_down(ret);
return ret;
return utils::to_api_error(res);
}
#endif // __APPLE__
void remote_fuse_drive::display_options(int argc, char *argv[]) {
struct fuse_operations fuse_ops {};
fuse_main(argc, argv, &fuse_ops, nullptr);
std::cout << std::endl;
}
void remote_fuse_drive::display_version_information(int argc, char *argv[]) {
struct fuse_operations fuse_ops {};
fuse_main(argc, argv, &fuse_ops, nullptr);
}
void remote_fuse_drive::shutdown(std::string mount_location) {
#if __APPLE__
const auto unmount = "umount \"" + mount_location + "\" >/dev/null 2>&1";
#if FUSE_USE_VERSION >= 30
auto remote_fuse_drive::truncate_impl(std::string api_path, off_t size,
struct fuse_file_info * /*fi*/)
-> api_error {
#else
const auto unmount = "fusermount -u \"" + mount_location + "\" >/dev/null 2>&1";
auto remote_fuse_drive::truncate_impl(std::string api_path, off_t size)
-> api_error {
#endif
int ret;
for (std::uint8_t i = 0u; ((ret = system(unmount.c_str())) != 0) && (i < 10u); i++) {
event_system::instance().raise<unmount_result>(mount_location, std::to_string(ret));
if (i != 9u) {
std::this_thread::sleep_for(1s);
}
return utils::to_api_error(
remote_instance_->fuse_truncate(api_path.c_str(), size));
}
auto remote_fuse_drive::unlink_impl(std::string api_path) -> api_error {
return utils::to_api_error(remote_instance_->fuse_unlink(api_path.c_str()));
}
#if FUSE_USE_VERSION >= 30
auto remote_fuse_drive::utimens_impl(std::string api_path,
const struct timespec tv[2],
struct fuse_file_info * /*fi*/)
-> api_error {
#else
auto remote_fuse_drive::utimens_impl(std::string api_path,
const struct timespec tv[2]) -> api_error {
#endif
remote::file_time rtv[2u] = {0};
if (tv) {
rtv[0u] = tv[0u].tv_nsec + (tv[0u].tv_sec * NANOS_PER_SECOND);
rtv[1u] = tv[1u].tv_nsec + (tv[1u].tv_sec * NANOS_PER_SECOND);
}
if (ret != 0) {
ret = kill(getpid(), SIGINT);
return utils::to_api_error(remote_instance_->fuse_utimens(
api_path.c_str(), rtv, tv ? tv[0u].tv_nsec : 0u,
tv ? tv[1u].tv_nsec : 0u));
}
auto remote_fuse_drive::write_impl(std::string api_path, const char *buffer,
size_t write_size, off_t write_offset,
struct fuse_file_info *fi,
std::size_t &bytes_written) -> api_error {
const auto res = remote_instance_->fuse_write(
api_path.c_str(), buffer, write_size, write_offset, fi->fh);
if (res >= 0) {
bytes_written = res;
return api_error::success;
}
event_system::instance().raise<unmount_result>(mount_location, std::to_string(ret));
return utils::to_api_error(res);
}
} // namespace repertory::remote_fuse

51
src/drives/fuse/remotefuse/remote_fuse_drive2.cpp
View File

@@ -1,51 +0,0 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#ifndef _WIN32
#if 0
#include "drives/fuse/remotefuse/remote_fuse_drive2.hpp"
#include "app_config.hpp"
#include "events/consumers/console_consumer.hpp"
#include "events/consumers/logging_consumer.hpp"
#include "events/events.hpp"
#include "events/event_system.hpp"
#include "platform/platform.hpp"
#include "rpc/server/server.hpp"
#include "types/remote.hpp"
#include "utils/file_utils.hpp"
#include "utils/path_utils.hpp"
namespace repertory::remote_fuse {
api_error remote_fuse_drive2::access_impl(std::string api_path, int mask) {
return utils::to_api_error(remote_instance_->fuse_access(api_path.c_str(), mask));
}
#ifdef __APPLE__
api_error remote_fuse_drive2::chflags_impl(std::string api_path, uint32_t flags) {
return utils::to_api_error(remote_instance_->fuse_chflags(path, flags));
}
#endif // __APPLE__
api_error remote_fuse_drive2::chmod_impl(std::string api_path, mode_t mode) {
return utils::to_api_error(remote_instance_->fuse_chmod(path, mode));
}
} // namespace repertory::remote_fuse
#endif // 0
#endif // _WIN32

1133
src/drives/fuse/remotefuse/remote_server.cpp
View File

File diff suppressed because it is too large Load Diff

176
src/drives/remote/remote_open_file_table.cpp
View File

@@ -1,27 +1,34 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "drives/remote/remote_open_file_table.hpp"
#include "events/event_system.hpp"
#include "events/events.hpp"
#include "utils/utils.hpp"
namespace repertory {
void remote_open_file_table::add_directory(const std::string &client_id, void *dir) {
unique_mutex_lock directory_lock(directory_mutex_);
void remote_open_file_table::add_directory(const std::string &client_id,
void *dir) {
recur_mutex_lock directory_lock(directory_mutex_);
auto &list = directory_lookup_[client_id];
if (utils::collection_excludes(list, dir)) {
directory_lookup_[client_id].emplace_back(dir);
@@ -29,16 +36,16 @@ void remote_open_file_table::add_directory(const std::string &client_id, void *d
}
void remote_open_file_table::close_all(const std::string &client_id) {
std::vector<remote::file_handle> compatHandles;
unique_mutex_lock compat_lock(compat_mutex_);
std::vector<remote::file_handle> compat_handles;
unique_recur_mutex_lock compat_lock(compat_mutex_);
for (auto &kv : compat_lookup_) {
if (kv.second.client_id == client_id) {
compatHandles.emplace_back(kv.first);
compat_handles.emplace_back(kv.first);
}
}
compat_lock.unlock();
for (auto &handle : compatHandles) {
for (auto &handle : compat_handles) {
#ifdef _WIN32
_close(static_cast<int>(handle));
#else
@@ -47,8 +54,8 @@ void remote_open_file_table::close_all(const std::string &client_id) {
remove_compat_open_info(handle);
}
std::vector<OSHandle> handles;
unique_mutex_lock file_lock(file_mutex_);
std::vector<native_handle> handles;
unique_recur_mutex_lock file_lock(file_mutex_);
for (auto &kv : file_lookup_) {
if (kv.second.client_id == client_id) {
handles.emplace_back(kv.first);
@@ -66,7 +73,7 @@ void remote_open_file_table::close_all(const std::string &client_id) {
}
std::vector<void *> dirs;
unique_mutex_lock directory_lock(directory_mutex_);
unique_recur_mutex_lock directory_lock(directory_mutex_);
for (auto &kv : directory_lookup_) {
if (kv.first == client_id) {
dirs.insert(dirs.end(), kv.second.begin(), kv.second.end());
@@ -80,8 +87,9 @@ void remote_open_file_table::close_all(const std::string &client_id) {
}
#ifdef _WIN32
bool remote_open_file_table::get_directory_buffer(const OSHandle &handle, PVOID *&buffer) {
mutex_lock file_lock(file_mutex_);
auto remote_open_file_table::get_directory_buffer(const native_handle &handle,
PVOID *&buffer) -> bool {
recur_mutex_lock file_lock(file_mutex_);
if (file_lookup_.find(handle) != file_lookup_.end()) {
buffer = &file_lookup_[handle].directory_buffer;
return true;
@@ -90,8 +98,31 @@ bool remote_open_file_table::get_directory_buffer(const OSHandle &handle, PVOID
}
#endif
bool remote_open_file_table::get_open_info(const OSHandle &handle, open_info &oi) {
mutex_lock file_lock(file_mutex_);
auto remote_open_file_table::get_open_file_count(
const std::string &file_path) const -> std::size_t {
unique_recur_mutex_lock file_lock(file_mutex_);
const auto count = std::accumulate(
file_lookup_.cbegin(), file_lookup_.cend(), std::size_t(0U),
[&file_path](std::size_t total, const auto &kv) -> std::size_t {
if (kv.second.path == file_path) {
return ++total;
}
return total;
});
return std::accumulate(
compat_lookup_.cbegin(), compat_lookup_.cend(), count,
[&file_path](std::size_t total, const auto &kv) -> std::size_t {
if (kv.second.path == file_path) {
return ++total;
}
return total;
});
}
auto remote_open_file_table::get_open_info(const native_handle &handle,
open_info &oi) -> bool {
recur_mutex_lock file_lock(file_mutex_);
if (file_lookup_.find(handle) != file_lookup_.end()) {
oi = file_lookup_[handle];
return true;
@@ -99,8 +130,9 @@ bool remote_open_file_table::get_open_info(const OSHandle &handle, open_info &oi
return false;
}
std::string remote_open_file_table::get_open_file_path(const OSHandle &handle) {
mutex_lock file_lock(file_mutex_);
auto remote_open_file_table::get_open_file_path(const native_handle &handle)
-> std::string {
recur_mutex_lock file_lock(file_mutex_);
if (file_lookup_.find(handle) != file_lookup_.end()) {
return file_lookup_[handle].path;
}
@@ -108,34 +140,72 @@ std::string remote_open_file_table::get_open_file_path(const OSHandle &handle) {
return "";
}
bool remote_open_file_table::has_open_directory(const std::string &client_id, void *dir) {
unique_mutex_lock directory_lock(directory_mutex_);
auto remote_open_file_table::has_open_directory(const std::string &client_id,
void *dir) -> bool {
recur_mutex_lock directory_lock(directory_mutex_);
auto &list = directory_lookup_[client_id];
return (utils::collection_includes(list, dir));
}
int remote_open_file_table::has_compat_open_info(const remote::file_handle &handle,
const int &errorReturn) {
mutex_lock compat_lock(compat_mutex_);
const auto res = ((compat_lookup_.find(handle) == compat_lookup_.end()) ? -1 : 0);
auto remote_open_file_table::has_compat_open_info(
const remote::file_handle &handle, int error_return) -> int {
recur_mutex_lock compat_lock(compat_mutex_);
const auto res =
((compat_lookup_.find(handle) == compat_lookup_.end()) ? -1 : 0);
if (res == -1) {
errno = errorReturn;
errno = error_return;
}
return res;
}
void remote_open_file_table::remove_compat_open_info(const remote::file_handle &handle) {
mutex_lock compat_lock(compat_mutex_);
void remote_open_file_table::remove_all(const std::string &file_path) {
unique_recur_mutex_lock file_lock(file_mutex_);
const auto open_list = std::accumulate(
file_lookup_.begin(), file_lookup_.end(), std::vector<native_handle>(),
[&file_path](std::vector<native_handle> v,
const auto &kv) -> std::vector<native_handle> {
if (kv.second.path == file_path) {
v.emplace_back(kv.first);
}
return v;
});
const auto compat_open_list = std::accumulate(
compat_lookup_.begin(), compat_lookup_.end(),
std::vector<remote::file_handle>(),
[&file_path](std::vector<remote::file_handle> v,
const auto &kv) -> std::vector<remote::file_handle> {
if (kv.second.path == file_path) {
v.emplace_back(kv.first);
}
return v;
});
file_lock.unlock();
for (auto &handle : open_list) {
remove_open_info(handle);
}
for (auto &handle : compat_open_list) {
remove_compat_open_info(handle);
}
}
void remote_open_file_table::remove_compat_open_info(
const remote::file_handle &handle) {
recur_mutex_lock compat_lock(compat_mutex_);
if (compat_lookup_[handle].count > 0) {
compat_lookup_[handle].count--;
}
if (not compat_lookup_[handle].count) {
compat_lookup_.erase(handle);
}
}
bool remote_open_file_table::remove_directory(const std::string &client_id, void *dir) {
unique_mutex_lock directory_lock(directory_mutex_);
auto remote_open_file_table::remove_directory(const std::string &client_id,
void *dir) -> bool {
recur_mutex_lock directory_lock(directory_mutex_);
auto &list = directory_lookup_[client_id];
if (utils::collection_includes(list, dir)) {
utils::remove_element_from(list, dir);
@@ -148,42 +218,46 @@ bool remote_open_file_table::remove_directory(const std::string &client_id, void
return false;
}
void remote_open_file_table::remove_open_info(const OSHandle &handle) {
mutex_lock file_lock(file_mutex_);
void remote_open_file_table::remove_open_info(const native_handle &handle) {
recur_mutex_lock file_lock(file_mutex_);
if (file_lookup_[handle].count > 0) {
file_lookup_[handle].count--;
}
if (not file_lookup_[handle].count) {
#ifdef _WIN32
if (file_lookup_[handle].directory_buffer) {
FspFileSystemDeleteDirectoryBuffer(&file_lookup_[handle].directory_buffer);
FspFileSystemDeleteDirectoryBuffer(
&file_lookup_[handle].directory_buffer);
}
#endif
file_lookup_.erase(handle);
}
}
void remote_open_file_table::set_compat_client_id(const remote::file_handle &handle,
const std::string &client_id) {
mutex_lock compat_lock(compat_mutex_);
void remote_open_file_table::set_compat_client_id(
const remote::file_handle &handle, const std::string &client_id) {
recur_mutex_lock compat_lock(compat_mutex_);
compat_lookup_[handle].client_id = client_id;
}
void remote_open_file_table::set_client_id(const OSHandle &handle, const std::string &client_id) {
mutex_lock file_lock(file_mutex_);
void remote_open_file_table::set_client_id(const native_handle &handle,
const std::string &client_id) {
recur_mutex_lock file_lock(file_mutex_);
file_lookup_[handle].client_id = client_id;
}
void remote_open_file_table::set_compat_open_info(const remote::file_handle &handle) {
mutex_lock compat_lock(compat_mutex_);
void remote_open_file_table::set_compat_open_info(
const remote::file_handle &handle, const std::string &file_path) {
recur_mutex_lock compat_lock(compat_mutex_);
if (compat_lookup_.find(handle) == compat_lookup_.end()) {
compat_lookup_[handle] = {0, ""};
compat_lookup_[handle] = {0, "", file_path};
}
compat_lookup_[handle].count++;
}
void remote_open_file_table::set_open_info(const OSHandle &handle, open_info oi) {
mutex_lock file_lock(file_mutex_);
void remote_open_file_table::set_open_info(const native_handle &handle,
open_info oi) {
recur_mutex_lock file_lock(file_mutex_);
if (file_lookup_.find(handle) == file_lookup_.end()) {
file_lookup_[handle] = std::move(oi);
}

323
src/drives/winfsp/remotewinfsp/remote_client.cpp
View File

@@ -1,52 +1,59 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "drives/winfsp/remotewinfsp/remote_client.hpp"
#include "app_config.hpp"
#include "drives/winfsp/remotewinfsp/i_remote_instance.hpp"
#include "events/events.hpp"
#include "events/event_system.hpp"
#include "events/events.hpp"
#include "types/repertory.hpp"
#include "utils/path_utils.hpp"
namespace repertory::remote_winfsp {
#define RAISE_REMOTE_WINFSP_CLIENT_EVENT(func, file, ret) \
if (config_.get_enable_drive_events() && \
(((config_.get_event_level() >= remote_winfsp_client_event::level) && \
(ret != STATUS_SUCCESS)) || \
(config_.get_event_level() >= event_level::verbose))) \
#define RAISE_REMOTE_WINFSP_CLIENT_EVENT(func, file, ret) \
if (config_.get_enable_drive_events() && \
(((config_.get_event_level() >= remote_winfsp_client_event::level) && \
(ret != STATUS_SUCCESS)) || \
(config_.get_event_level() >= event_level::verbose))) \
event_system::instance().raise<remote_winfsp_client_event>(func, file, ret)
// clang-format off
E_SIMPLE3(remote_winfsp_client_event, debug, true,
std::string, function, func, E_STRING,
std::string, api_path, ap, E_STRING,
E_SIMPLE3(remote_winfsp_client_event, debug, true,
std::string, function, func, E_STRING,
std::string, api_path, ap, E_STRING,
packet::error_type, result, res, E_FROM_INT32
);
// clang-format on
remote_client::remote_client(const app_config &config)
: config_(config),
packet_client_(config.get_remote_host_name_or_ip(), config.get_remote_max_connections(),
config.get_remote_port(), config.get_remote_receive_timeout_secs(),
config.get_remote_send_timeout_secs(), config.get_remote_token()) {}
packet_client_(
config.get_remote_host_name_or_ip(),
config.get_remote_max_connections(), config.get_remote_port(),
config.get_remote_receive_timeout_secs(),
config.get_remote_send_timeout_secs(), config.get_remote_token()) {}
packet::error_type remote_client::winfsp_can_delete(PVOID file_desc, PWSTR file_name) {
auto remote_client::winfsp_can_delete(PVOID file_desc, PWSTR file_name)
-> packet::error_type {
packet request;
request.encode(file_desc);
request.encode(file_name);
@@ -54,18 +61,21 @@ packet::error_type remote_client::winfsp_can_delete(PVOID file_desc, PWSTR file_
std::uint32_t service_flags = 0u;
const auto ret = packet_client_.send(__FUNCTION__, request, service_flags);
RAISE_REMOTE_WINFSP_CLIENT_EVENT(
__FUNCTION__, utils::path::create_api_path(utils::string::to_utf8(file_name)), ret);
__FUNCTION__,
utils::path::create_api_path(utils::string::to_utf8(file_name)), ret);
return ret;
}
packet::error_type remote_client::json_create_directory_snapshot(const std::string &path,
json &json_data) {
auto remote_client::json_create_directory_snapshot(const std::string &path,
json &json_data)
-> packet::error_type {
packet request;
request.encode(path);
packet response;
std::uint32_t service_flags = 0u;
auto ret = packet_client_.send(__FUNCTION__, request, response, service_flags);
auto ret =
packet_client_.send(__FUNCTION__, request, response, service_flags);
if (ret == 0) {
ret = packet::decode_json(response, json_data);
}
@@ -74,10 +84,9 @@ packet::error_type remote_client::json_create_directory_snapshot(const std::stri
return ret;
}
packet::error_type remote_client::json_read_directory_snapshot(const std::string &path,
const remote::file_handle &handle,
const std::uint32_t &page,
json &json_data) {
auto remote_client::json_read_directory_snapshot(
const std::string &path, const remote::file_handle &handle,
std::uint32_t page, json &json_data) -> packet::error_type {
packet request;
request.encode(path);
request.encode(handle);
@@ -85,7 +94,8 @@ packet::error_type remote_client::json_read_directory_snapshot(const std::string
packet response;
std::uint32_t service_flags = 0u;
auto ret = packet_client_.send(__FUNCTION__, request, response, service_flags);
auto ret =
packet_client_.send(__FUNCTION__, request, response, service_flags);
if (ret == 0) {
ret = packet::decode_json(response, json_data);
}
@@ -94,9 +104,9 @@ packet::error_type remote_client::json_read_directory_snapshot(const std::string
return ret;
}
packet::error_type
remote_client::json_release_directory_snapshot(const std::string &path,
const remote::file_handle &handle) {
auto remote_client::json_release_directory_snapshot(
const std::string &path, const remote::file_handle &handle)
-> packet::error_type {
packet request;
request.encode(path);
request.encode(handle);
@@ -108,9 +118,11 @@ remote_client::json_release_directory_snapshot(const std::string &path,
return ret;
}
packet::error_type remote_client::winfsp_cleanup(PVOID file_desc, PWSTR file_name, UINT32 flags,
BOOLEAN &was_closed) {
const auto file_path = get_open_file_path(to_handle(file_desc));
auto remote_client::winfsp_cleanup(PVOID file_desc, PWSTR file_name,
UINT32 flags, BOOLEAN &was_closed)
-> packet::error_type {
auto handle = to_handle(file_desc);
const auto file_path = get_open_file_path(handle);
was_closed = 0;
packet request;
@@ -120,35 +132,41 @@ packet::error_type remote_client::winfsp_cleanup(PVOID file_desc, PWSTR file_nam
packet response;
std::uint32_t service_flags = 0u;
auto ret = packet_client_.send(__FUNCTION__, request, response, service_flags);
auto ret =
packet_client_.send(__FUNCTION__, request, response, service_flags);
DECODE_OR_IGNORE(&response, was_closed);
if (was_closed) {
remove_open_info(to_handle(file_desc));
remove_all(file_path);
}
RAISE_REMOTE_WINFSP_CLIENT_EVENT(__FUNCTION__, file_path, ret);
return ret;
}
packet::error_type remote_client::winfsp_close(PVOID file_desc) {
const auto file_path = get_open_file_path(to_handle(file_desc));
auto remote_client::winfsp_close(PVOID file_desc) -> packet::error_type {
auto handle = to_handle(file_desc);
if (has_open_info(handle, STATUS_INVALID_HANDLE) == STATUS_SUCCESS) {
const auto file_path = get_open_file_path(handle);
packet request;
request.encode(file_desc);
packet request;
request.encode(file_desc);
std::uint32_t service_flags = 0u;
const auto ret = packet_client_.send(__FUNCTION__, request, service_flags);
if (ret == STATUS_SUCCESS) {
remove_open_info(to_handle(file_desc));
std::uint32_t service_flags = 0u;
const auto ret = packet_client_.send(__FUNCTION__, request, service_flags);
if ((ret == STATUS_SUCCESS) || (ret == STATUS_INVALID_HANDLE)) {
remove_open_info(handle);
RAISE_REMOTE_WINFSP_CLIENT_EVENT(__FUNCTION__, file_path, ret);
}
}
RAISE_REMOTE_WINFSP_CLIENT_EVENT(__FUNCTION__, file_path, ret);
return ret;
return 0;
}
packet::error_type remote_client::winfsp_create(PWSTR file_name, UINT32 create_options,
UINT32 granted_access, UINT32 attributes,
UINT64 allocation_size, PVOID *file_desc,
remote::file_info *file_info,
std::string &normalized_name, BOOLEAN &exists) {
auto remote_client::winfsp_create(PWSTR file_name, UINT32 create_options,
UINT32 granted_access, UINT32 attributes,
UINT64 allocation_size, PVOID *file_desc,
remote::file_info *file_info,
std::string &normalized_name, BOOLEAN &exists)
-> packet::error_type {
packet request;
request.encode(file_name);
request.encode(create_options);
@@ -158,7 +176,8 @@ packet::error_type remote_client::winfsp_create(PWSTR file_name, UINT32 create_o
packet response;
std::uint32_t service_flags = 0u;
auto ret = packet_client_.send(__FUNCTION__, request, response, service_flags);
auto ret =
packet_client_.send(__FUNCTION__, request, response, service_flags);
if (ret == STATUS_SUCCESS) {
HANDLE handle;
DECODE_OR_IGNORE(&response, handle);
@@ -167,8 +186,9 @@ packet::error_type remote_client::winfsp_create(PWSTR file_name, UINT32 create_o
DECODE_OR_IGNORE(&response, exists);
if (ret == STATUS_SUCCESS) {
*file_desc = reinterpret_cast<PVOID>(handle);
set_open_info(to_handle(*file_desc),
open_info{0, "", nullptr, utils::string::to_utf8(file_name)});
set_open_info(
to_handle(*file_desc),
open_info{0, "", nullptr, utils::string::to_utf8(file_name)});
#ifdef _WIN32
if (exists) {
::SetLastError(ERROR_ALREADY_EXISTS);
@@ -177,57 +197,70 @@ packet::error_type remote_client::winfsp_create(PWSTR file_name, UINT32 create_o
}
}
RAISE_REMOTE_WINFSP_CLIENT_EVENT(__FUNCTION__, get_open_file_path(to_handle(*file_desc)), ret);
RAISE_REMOTE_WINFSP_CLIENT_EVENT(
__FUNCTION__, get_open_file_path(to_handle(*file_desc)), ret);
return ret;
}
packet::error_type remote_client::winfsp_flush(PVOID file_desc, remote::file_info *file_info) {
auto remote_client::winfsp_flush(PVOID file_desc, remote::file_info *file_info)
-> packet::error_type {
packet request;
request.encode(file_desc);
packet response;
std::uint32_t service_flags = 0u;
auto ret = packet_client_.send(__FUNCTION__, request, response, service_flags);
auto ret =
packet_client_.send(__FUNCTION__, request, response, service_flags);
DECODE_OR_IGNORE(&response, *file_info);
RAISE_REMOTE_WINFSP_CLIENT_EVENT(__FUNCTION__, get_open_file_path(to_handle(file_desc)), ret);
RAISE_REMOTE_WINFSP_CLIENT_EVENT(
__FUNCTION__, get_open_file_path(to_handle(file_desc)), ret);
return ret;
}
packet::error_type remote_client::winfsp_get_dir_buffer(PVOID file_desc, PVOID *&ptr) {
auto remote_client::winfsp_get_dir_buffer([[maybe_unused]] PVOID file_desc,
[[maybe_unused]] PVOID *&ptr)
-> packet::error_type {
#ifdef _WIN32
if (get_directory_buffer(reinterpret_cast<OSHandle>(file_desc), ptr)) {
if (get_directory_buffer(reinterpret_cast<native_handle>(file_desc), ptr)) {
return STATUS_SUCCESS;
}
#endif
return STATUS_INVALID_HANDLE;
}
packet::error_type remote_client::winfsp_get_file_info(PVOID file_desc,
remote::file_info *file_info) {
auto remote_client::winfsp_get_file_info(PVOID file_desc,
remote::file_info *file_info)
-> packet::error_type {
packet request;
request.encode(file_desc);
packet response;
std::uint32_t service_flags = 0u;
auto ret = packet_client_.send(__FUNCTION__, request, response, service_flags);
auto ret =
packet_client_.send(__FUNCTION__, request, response, service_flags);
DECODE_OR_IGNORE(&response, *file_info);
RAISE_REMOTE_WINFSP_CLIENT_EVENT(__FUNCTION__, get_open_file_path(to_handle(file_desc)), ret);
RAISE_REMOTE_WINFSP_CLIENT_EVENT(
__FUNCTION__, get_open_file_path(to_handle(file_desc)), ret);
return ret;
}
packet::error_type remote_client::winfsp_get_security_by_name(PWSTR file_name, PUINT32 attributes,
std::uint64_t *descriptor_size,
std::wstring &string_descriptor) {
auto remote_client::winfsp_get_security_by_name(PWSTR file_name,
PUINT32 attributes,
std::uint64_t *descriptor_size,
std::wstring &string_descriptor)
-> packet::error_type {
packet request;
request.encode(file_name);
request.encode(static_cast<std::uint64_t>(descriptor_size ? *descriptor_size : 0));
request.encode(
static_cast<std::uint64_t>(descriptor_size ? *descriptor_size : 0));
request.encode(static_cast<std::uint8_t>(attributes != nullptr));
packet response;
std::uint32_t service_flags = 0u;
auto ret = packet_client_.send(__FUNCTION__, request, response, service_flags);
auto ret =
packet_client_.send(__FUNCTION__, request, response, service_flags);
string_descriptor.clear();
DECODE_OR_IGNORE(&response, string_descriptor);
@@ -239,16 +272,20 @@ packet::error_type remote_client::winfsp_get_security_by_name(PWSTR file_name, P
DECODE_OR_IGNORE(&response, *attributes);
}
RAISE_REMOTE_WINFSP_CLIENT_EVENT(__FUNCTION__, utils::string::to_utf8(file_name), ret);
RAISE_REMOTE_WINFSP_CLIENT_EVENT(__FUNCTION__,
utils::string::to_utf8(file_name), ret);
return ret;
}
packet::error_type remote_client::winfsp_get_volume_info(UINT64 &total_size, UINT64 &free_size,
std::string &volume_label) {
auto remote_client::winfsp_get_volume_info(UINT64 &total_size,
UINT64 &free_size,
std::string &volume_label)
-> packet::error_type {
packet request;
packet response;
std::uint32_t service_flags = 0u;
auto ret = packet_client_.send(__FUNCTION__, request, response, service_flags);
auto ret =
packet_client_.send(__FUNCTION__, request, response, service_flags);
DECODE_OR_IGNORE(&response, total_size);
DECODE_OR_IGNORE(&response, free_size);
DECODE_OR_IGNORE(&response, volume_label);
@@ -256,24 +293,26 @@ packet::error_type remote_client::winfsp_get_volume_info(UINT64 &total_size, UIN
return ret;
}
packet::error_type remote_client::winfsp_mounted(const std::wstring &location) {
auto remote_client::winfsp_mounted(const std::wstring &location)
-> packet::error_type {
packet request;
request.encode(get_repertory_version());
request.encode(location);
std::uint32_t service_flags = 0u;
const auto ret = packet_client_.send(__FUNCTION__, request, service_flags);
const auto mountLocation = utils::string::to_utf8(location);
event_system::instance().raise<drive_mounted>(mountLocation);
const auto mount_location = utils::string::to_utf8(location);
event_system::instance().raise<drive_mounted>(mount_location);
RAISE_REMOTE_WINFSP_CLIENT_EVENT(__FUNCTION__, mountLocation, ret);
RAISE_REMOTE_WINFSP_CLIENT_EVENT(__FUNCTION__, mount_location, ret);
return ret;
}
packet::error_type remote_client::winfsp_open(PWSTR file_name, UINT32 create_options,
UINT32 granted_access, PVOID *file_desc,
remote::file_info *file_info,
std::string &normalized_name) {
auto remote_client::winfsp_open(PWSTR file_name, UINT32 create_options,
UINT32 granted_access, PVOID *file_desc,
remote::file_info *file_info,
std::string &normalized_name)
-> packet::error_type {
packet request;
request.encode(file_name);
request.encode(create_options);
@@ -281,7 +320,8 @@ packet::error_type remote_client::winfsp_open(PWSTR file_name, UINT32 create_opt
packet response;
std::uint32_t service_flags = 0u;
auto ret = packet_client_.send(__FUNCTION__, request, response, service_flags);
auto ret =
packet_client_.send(__FUNCTION__, request, response, service_flags);
if (ret == STATUS_SUCCESS) {
HANDLE handle;
DECODE_OR_IGNORE(&response, handle);
@@ -290,19 +330,22 @@ packet::error_type remote_client::winfsp_open(PWSTR file_name, UINT32 create_opt
if (ret == STATUS_SUCCESS) {
*file_desc = reinterpret_cast<PVOID>(handle);
set_open_info(to_handle(*file_desc),
open_info{0, "", nullptr, utils::string::to_utf8(file_name)});
set_open_info(
to_handle(*file_desc),
open_info{0, "", nullptr, utils::string::to_utf8(file_name)});
}
}
RAISE_REMOTE_WINFSP_CLIENT_EVENT(__FUNCTION__, get_open_file_path(to_handle(*file_desc)), ret);
RAISE_REMOTE_WINFSP_CLIENT_EVENT(
__FUNCTION__, get_open_file_path(to_handle(*file_desc)), ret);
return ret;
}
packet::error_type remote_client::winfsp_overwrite(PVOID file_desc, UINT32 attributes,
BOOLEAN replace_attributes,
UINT64 allocation_size,
remote::file_info *file_info) {
auto remote_client::winfsp_overwrite(PVOID file_desc, UINT32 attributes,
BOOLEAN replace_attributes,
UINT64 allocation_size,
remote::file_info *file_info)
-> packet::error_type {
packet request;
request.encode(file_desc);
request.encode(attributes);
@@ -311,15 +354,18 @@ packet::error_type remote_client::winfsp_overwrite(PVOID file_desc, UINT32 attri
packet response;
std::uint32_t service_flags = 0u;
auto ret = packet_client_.send(__FUNCTION__, request, response, service_flags);
auto ret =
packet_client_.send(__FUNCTION__, request, response, service_flags);
DECODE_OR_IGNORE(&response, *file_info);
RAISE_REMOTE_WINFSP_CLIENT_EVENT(__FUNCTION__, get_open_file_path(to_handle(file_desc)), ret);
RAISE_REMOTE_WINFSP_CLIENT_EVENT(
__FUNCTION__, get_open_file_path(to_handle(file_desc)), ret);
return ret;
}
packet::error_type remote_client::winfsp_read(PVOID file_desc, PVOID buffer, UINT64 offset,
UINT32 length, PUINT32 bytes_transferred) {
auto remote_client::winfsp_read(PVOID file_desc, PVOID buffer, UINT64 offset,
UINT32 length, PUINT32 bytes_transferred)
-> packet::error_type {
packet request;
request.encode(file_desc);
request.encode(offset);
@@ -327,26 +373,30 @@ packet::error_type remote_client::winfsp_read(PVOID file_desc, PVOID buffer, UIN
packet response;
std::uint32_t service_flags = 0u;
auto ret = packet_client_.send(__FUNCTION__, request, response, service_flags);
auto ret =
packet_client_.send(__FUNCTION__, request, response, service_flags);
DECODE_OR_IGNORE(&response, *bytes_transferred);
if (ret == STATUS_SUCCESS) {
ret = response.decode(buffer, *bytes_transferred);
#ifdef _WIN32
if ((ret == STATUS_SUCCESS) && (not *bytes_transferred || (*bytes_transferred != length))) {
if ((ret == STATUS_SUCCESS) &&
(not *bytes_transferred || (*bytes_transferred != length))) {
::SetLastError(ERROR_HANDLE_EOF);
}
#endif
}
if (ret != STATUS_SUCCESS) {
RAISE_REMOTE_WINFSP_CLIENT_EVENT(__FUNCTION__, get_open_file_path(to_handle(file_desc)), ret);
RAISE_REMOTE_WINFSP_CLIENT_EVENT(
__FUNCTION__, get_open_file_path(to_handle(file_desc)), ret);
}
return ret;
}
packet::error_type remote_client::winfsp_read_directory(PVOID file_desc, PWSTR pattern,
PWSTR marker, json &item_list) {
auto remote_client::winfsp_read_directory(PVOID file_desc, PWSTR pattern,
PWSTR marker, json &item_list)
-> packet::error_type {
packet request;
request.encode(file_desc);
request.encode(pattern);
@@ -354,17 +404,21 @@ packet::error_type remote_client::winfsp_read_directory(PVOID file_desc, PWSTR p
packet response;
std::uint32_t service_flags = 0u;
auto ret = packet_client_.send(__FUNCTION__, request, response, service_flags);
auto ret =
packet_client_.send(__FUNCTION__, request, response, service_flags);
if (ret == STATUS_SUCCESS) {
ret = packet::decode_json(response, item_list);
}
RAISE_REMOTE_WINFSP_CLIENT_EVENT(__FUNCTION__, get_open_file_path(to_handle(file_desc)), ret);
RAISE_REMOTE_WINFSP_CLIENT_EVENT(
__FUNCTION__, get_open_file_path(to_handle(file_desc)), ret);
return ret;
}
packet::error_type remote_client::winfsp_rename(PVOID file_desc, PWSTR file_name,
PWSTR new_file_name, BOOLEAN replace_if_exists) {
auto remote_client::winfsp_rename(PVOID file_desc, PWSTR file_name,
PWSTR new_file_name,
BOOLEAN replace_if_exists)
-> packet::error_type {
packet request;
request.encode(file_desc);
request.encode(file_name);
@@ -381,11 +435,10 @@ packet::error_type remote_client::winfsp_rename(PVOID file_desc, PWSTR file_name
return ret;
}
packet::error_type remote_client::winfsp_set_basic_info(PVOID file_desc, UINT32 attributes,
UINT64 creation_time,
UINT64 last_access_time,
UINT64 last_write_time, UINT64 change_time,
remote::file_info *file_info) {
auto remote_client::winfsp_set_basic_info(
PVOID file_desc, UINT32 attributes, UINT64 creation_time,
UINT64 last_access_time, UINT64 last_write_time, UINT64 change_time,
remote::file_info *file_info) -> packet::error_type {
packet request;
request.encode(file_desc);
request.encode(attributes);
@@ -396,16 +449,19 @@ packet::error_type remote_client::winfsp_set_basic_info(PVOID file_desc, UINT32
packet response;
std::uint32_t service_flags = 0u;
auto ret = packet_client_.send(__FUNCTION__, request, response, service_flags);
auto ret =
packet_client_.send(__FUNCTION__, request, response, service_flags);
DECODE_OR_IGNORE(&response, *file_info);
RAISE_REMOTE_WINFSP_CLIENT_EVENT(__FUNCTION__, get_open_file_path(to_handle(file_desc)), ret);
RAISE_REMOTE_WINFSP_CLIENT_EVENT(
__FUNCTION__, get_open_file_path(to_handle(file_desc)), ret);
return ret;
}
packet::error_type remote_client::winfsp_set_file_size(PVOID file_desc, UINT64 new_size,
BOOLEAN set_allocation_size,
remote::file_info *file_info) {
auto remote_client::winfsp_set_file_size(PVOID file_desc, UINT64 new_size,
BOOLEAN set_allocation_size,
remote::file_info *file_info)
-> packet::error_type {
packet request;
request.encode(file_desc);
request.encode(new_size);
@@ -413,14 +469,17 @@ packet::error_type remote_client::winfsp_set_file_size(PVOID file_desc, UINT64 n
packet response;
std::uint32_t service_flags = 0u;
auto ret = packet_client_.send(__FUNCTION__, request, response, service_flags);
auto ret =
packet_client_.send(__FUNCTION__, request, response, service_flags);
DECODE_OR_IGNORE(&response, *file_info);
RAISE_REMOTE_WINFSP_CLIENT_EVENT(__FUNCTION__, get_open_file_path(to_handle(file_desc)), ret);
RAISE_REMOTE_WINFSP_CLIENT_EVENT(
__FUNCTION__, get_open_file_path(to_handle(file_desc)), ret);
return ret;
}
packet::error_type remote_client::winfsp_unmounted(const std::wstring &location) {
auto remote_client::winfsp_unmounted(const std::wstring &location)
-> packet::error_type {
const auto mount_location = utils::string::to_utf8(location);
event_system::instance().raise<drive_unmount_pending>(mount_location);
packet request;
@@ -434,10 +493,12 @@ packet::error_type remote_client::winfsp_unmounted(const std::wstring &location)
return ret;
}
packet::error_type remote_client::winfsp_write(PVOID file_desc, PVOID buffer, UINT64 offset,
UINT32 length, BOOLEAN write_to_end,
BOOLEAN constrained_io, PUINT32 bytes_transferred,
remote::file_info *file_info) {
auto remote_client::winfsp_write(PVOID file_desc, PVOID buffer, UINT64 offset,
UINT32 length, BOOLEAN write_to_end,
BOOLEAN constrained_io,
PUINT32 bytes_transferred,
remote::file_info *file_info)
-> packet::error_type {
packet request;
request.encode(file_desc);
request.encode(length);
@@ -450,13 +511,21 @@ packet::error_type remote_client::winfsp_write(PVOID file_desc, PVOID buffer, UI
packet response;
std::uint32_t service_flags = 0u;
auto ret = packet_client_.send(__FUNCTION__, request, response, service_flags);
auto ret =
packet_client_.send(__FUNCTION__, request, response, service_flags);
DECODE_OR_IGNORE(&response, *bytes_transferred);
DECODE_OR_IGNORE(&response, *file_info);
if (ret != STATUS_SUCCESS) {
RAISE_REMOTE_WINFSP_CLIENT_EVENT(__FUNCTION__, get_open_file_path(to_handle(file_desc)), ret);
RAISE_REMOTE_WINFSP_CLIENT_EVENT(
__FUNCTION__, get_open_file_path(to_handle(file_desc)), ret);
}
return ret;
}
#ifndef _WIN32
native_handle remote_client::to_handle(PVOID file_desc) {
return static_cast<native_handle>(reinterpret_cast<std::uint64_t>(file_desc));
}
#endif
} // namespace repertory::remote_winfsp

775
src/drives/winfsp/remotewinfsp/remote_server.cpp
View File

File diff suppressed because it is too large Load Diff

333
src/drives/winfsp/remotewinfsp/remote_winfsp_drive.cpp
View File

@@ -1,144 +1,177 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#ifdef _WIN32
#include <utility>
#include "drives/winfsp/remotewinfsp/remote_winfsp_drive.hpp"
#include "app_config.hpp"
#include "events/consumers/console_consumer.hpp"
#include "events/consumers/logging_consumer.hpp"
#include "events/events.hpp"
#include "platform/platform.hpp"
#include "rpc/server/server.hpp"
#include "utils/error_utils.hpp"
#include "utils/file_utils.hpp"
#include "utils/path_utils.hpp"
#include "utils/utils.hpp"
namespace repertory::remote_winfsp {
remote_winfsp_drive::winfsp_service::winfsp_service(lock_data &lock, remote_winfsp_drive &drive,
std::vector<std::string> drive_args,
app_config &config)
: Service(&(L"RepertoryRemote_" + utils::string::from_utf8(lock.get_unique_id()))[0u]),
remote_winfsp_drive::winfsp_service::winfsp_service(
lock_data &lock, remote_winfsp_drive &drive,
std::vector<std::string> drive_args, app_config &config)
: Service(&(L"RepertoryRemote_" +
utils::string::from_utf8(lock.get_unique_id()))[0u]),
config_(config),
lock_(lock),
drive_(drive),
drive_args_(std::move(drive_args)),
host_(drive) {}
NTSTATUS remote_winfsp_drive::winfsp_service::OnStart(ULONG, PWSTR *) {
const auto mount_location =
utils::path::absolute((drive_args_.size() > 1u) ? drive_args_[1u] : "");
const auto drive_letter = ((mount_location.size() == 2u) ||
((mount_location.size() == 3u) && (mount_location[2u] == '\\'))) &&
(mount_location[1u] == ':');
auto remote_winfsp_drive::winfsp_service::OnStart(ULONG, PWSTR *) -> NTSTATUS {
const auto mount_location = utils::string::to_lower(
utils::path::absolute((drive_args_.size() > 1u) ? drive_args_[1u] : ""));
const auto drive_letter =
((mount_location.size() == 2u) ||
((mount_location.size() == 3u) && (mount_location[2u] == '\\'))) &&
(mount_location[1u] == ':');
auto ret = drive_letter ? STATUS_DEVICE_BUSY : STATUS_NOT_SUPPORTED;
if ((drive_letter && not utils::file::is_directory(mount_location))) {
auto unicode_mount_location = utils::string::from_utf8(mount_location);
host_.SetFileSystemName(&unicode_mount_location[0u]);
if (config_.get_enable_mount_manager()) {
unicode_mount_location =
std::wstring(L"\\\\.\\") + unicode_mount_location[0u] + L":";
}
ret = host_.Mount(&unicode_mount_location[0u]);
} else {
std::cerr << (drive_letter ? "Mount location in use: " : "Mount location not supported: ")
std::cerr << (drive_letter ? "Mount location in use: "
: "Mount location not supported: ")
<< mount_location << std::endl;
}
if (ret != STATUS_SUCCESS) {
event_system::instance().raise<drive_mount_failed>(mount_location, std::to_string(ret));
lock_.set_mount_state(false, "", -1);
event_system::instance().raise<drive_mount_failed>(mount_location,
std::to_string(ret));
if (not lock_.set_mount_state(false, "", -1)) {
utils::error::raise_error(__FUNCTION__, "failed to set mount state");
}
}
return ret;
}
NTSTATUS remote_winfsp_drive::winfsp_service::OnStop() {
auto remote_winfsp_drive::winfsp_service::OnStop() -> NTSTATUS {
host_.Unmount();
lock_.set_mount_state(false, "", -1);
if (not lock_.set_mount_state(false, "", -1)) {
utils::error::raise_error(__FUNCTION__, "failed to set mount state");
}
return STATUS_SUCCESS;
}
remote_winfsp_drive::remote_winfsp_drive(app_config &config, lock_data &lockData,
remote_instance_factory factory)
: FileSystemBase(), config_(config), lock_(lockData), factory_(factory) {
E_SUBSCRIBE_EXACT(unmount_requested, [this](const unmount_requested &e) {
std::thread([this]() { this->shutdown(); }).detach();
});
remote_winfsp_drive::remote_winfsp_drive(app_config &config,
remote_instance_factory factory,
lock_data &lock)
: FileSystemBase(),
config_(config),
lock_(lock),
factory_(std::move(factory)) {
E_SUBSCRIBE_EXACT(unmount_requested,
[this](const unmount_requested & /* e */) {
std::thread([this]() { this->shutdown(); }).detach();
});
}
NTSTATUS remote_winfsp_drive::CanDelete(PVOID file_node, PVOID file_desc, PWSTR file_name) {
auto remote_winfsp_drive::CanDelete(PVOID /*file_node*/, PVOID file_desc,
PWSTR file_name) -> NTSTATUS {
return remote_instance_->winfsp_can_delete(file_desc, file_name);
}
VOID remote_winfsp_drive::Cleanup(PVOID file_node, PVOID file_desc, PWSTR file_name, ULONG flags) {
VOID remote_winfsp_drive::Cleanup(PVOID /*file_node*/, PVOID file_desc,
PWSTR file_name, ULONG flags) {
BOOLEAN was_closed;
remote_instance_->winfsp_cleanup(file_desc, file_name, flags, was_closed);
}
VOID remote_winfsp_drive::Close(PVOID file_node, PVOID file_desc) {
VOID remote_winfsp_drive::Close(PVOID /*file_node*/, PVOID file_desc) {
remote_instance_->winfsp_close(file_desc);
}
NTSTATUS remote_winfsp_drive::Create(PWSTR file_name, UINT32 create_options, UINT32 granted_access,
UINT32 attributes, PSECURITY_DESCRIPTOR descriptor,
UINT64 allocation_size, PVOID *file_node, PVOID *file_desc,
OpenFileInfo *ofi) {
auto remote_winfsp_drive::Create(PWSTR file_name, UINT32 create_options,
UINT32 granted_access, UINT32 attributes,
PSECURITY_DESCRIPTOR /*descriptor*/,
UINT64 allocation_size, PVOID * /*file_node*/,
PVOID *file_desc, OpenFileInfo *ofi)
-> NTSTATUS {
remote::file_info fi{};
std::string normalized_name;
BOOLEAN exists = 0;
const auto ret =
remote_instance_->winfsp_create(file_name, create_options, granted_access, attributes,
allocation_size, file_desc, &fi, normalized_name, exists);
const auto ret = remote_instance_->winfsp_create(
file_name, create_options, granted_access, attributes, allocation_size,
file_desc, &fi, normalized_name, exists);
if (ret == STATUS_SUCCESS) {
SetFileInfo(ofi->FileInfo, fi);
const auto filePath = utils::string::from_utf8(normalized_name);
wcsncpy(ofi->NormalizedName, &filePath[0], wcslen(&filePath[0]));
ofi->NormalizedNameSize = (UINT16)(wcslen(&filePath[0]) * sizeof(WCHAR));
set_file_info(ofi->FileInfo, fi);
const auto file_path = utils::string::from_utf8(normalized_name);
wcsncpy(ofi->NormalizedName, &file_path[0], wcslen(&file_path[0]));
ofi->NormalizedNameSize = (UINT16)(wcslen(&file_path[0]) * sizeof(WCHAR));
}
return ret;
}
NTSTATUS remote_winfsp_drive::Flush(PVOID file_node, PVOID file_desc, FileInfo *file_info) {
auto remote_winfsp_drive::Flush(PVOID /*file_node*/, PVOID file_desc,
FileInfo *file_info) -> NTSTATUS {
remote::file_info fi{};
const auto ret = remote_instance_->winfsp_flush(file_desc, &fi);
SetFileInfo(*file_info, fi);
set_file_info(*file_info, fi);
return ret;
}
NTSTATUS remote_winfsp_drive::GetFileInfo(PVOID file_node, PVOID file_desc, FileInfo *file_info) {
auto remote_winfsp_drive::GetFileInfo(PVOID /*file_node*/, PVOID file_desc,
FileInfo *file_info) -> NTSTATUS {
remote::file_info fi{};
const auto ret = remote_instance_->winfsp_get_file_info(file_desc, &fi);
SetFileInfo(*file_info, fi);
set_file_info(*file_info, fi);
return ret;
}
NTSTATUS remote_winfsp_drive::GetSecurityByName(PWSTR file_name, PUINT32 attributes,
PSECURITY_DESCRIPTOR descriptor,
SIZE_T *descriptor_size) {
auto remote_winfsp_drive::GetSecurityByName(PWSTR file_name, PUINT32 attributes,
PSECURITY_DESCRIPTOR descriptor,
SIZE_T *descriptor_size)
-> NTSTATUS {
std::wstring string_descriptor;
std::uint64_t sds = descriptor_size ? *descriptor_size : 0;
auto ret = remote_instance_->winfsp_get_security_by_name(
file_name, attributes, descriptor_size ? &sds : nullptr, string_descriptor);
file_name, attributes, descriptor_size ? &sds : nullptr,
string_descriptor);
*descriptor_size = static_cast<SIZE_T>(sds);
if ((ret == STATUS_SUCCESS) && *descriptor_size) {
PSECURITY_DESCRIPTOR sd = nullptr;
ULONG sz2 = 0u;
if (::ConvertStringSecurityDescriptorToSecurityDescriptorW(&string_descriptor[0u],
SDDL_REVISION_1, &sd, &sz2)) {
if (::ConvertStringSecurityDescriptorToSecurityDescriptorW(
&string_descriptor[0u], SDDL_REVISION_1, &sd, &sz2)) {
if (sz2 > *descriptor_size) {
ret = STATUS_BUFFER_TOO_SMALL;
} else {
@@ -153,22 +186,28 @@ NTSTATUS remote_winfsp_drive::GetSecurityByName(PWSTR file_name, PUINT32 attribu
return ret;
}
NTSTATUS remote_winfsp_drive::GetVolumeInfo(VolumeInfo *volume_info) {
auto remote_winfsp_drive::GetVolumeInfo(VolumeInfo *volume_info) -> NTSTATUS {
std::string volume_label;
const auto ret = remote_instance_->winfsp_get_volume_info(volume_info->TotalSize,
volume_info->FreeSize, volume_label);
const auto ret = remote_instance_->winfsp_get_volume_info(
volume_info->TotalSize, volume_info->FreeSize, volume_label);
if (ret == STATUS_SUCCESS) {
const auto byte_size = static_cast<UINT16>(volume_label.size() * sizeof(WCHAR));
wcscpy_s(&volume_info->VolumeLabel[0u], 32, &utils::string::from_utf8(volume_label)[0u]);
volume_info->VolumeLabelLength = std::min(static_cast<UINT16>(64u), byte_size);
const auto byte_size =
static_cast<UINT16>(volume_label.size() * sizeof(WCHAR));
wcscpy_s(&volume_info->VolumeLabel[0u], 32,
&utils::string::from_utf8(volume_label)[0u]);
volume_info->VolumeLabelLength =
std::min(static_cast<UINT16>(64u), byte_size);
}
return ret;
}
NTSTATUS remote_winfsp_drive::Init(PVOID host) {
auto remote_winfsp_drive::Init(PVOID host) -> NTSTATUS {
auto *file_system_host = reinterpret_cast<FileSystemHost *>(host);
file_system_host->SetPrefix(
&(L"\\repertory\\" + std::wstring(file_system_host->FileSystemName()).substr(0, 1))[0]);
if (not config_.get_enable_mount_manager()) {
file_system_host->SetPrefix(
&(L"\\repertory\\" +
std::wstring(file_system_host->FileSystemName()).substr(0, 1))[0u]);
}
file_system_host->SetFileSystemName(REPERTORY_W);
file_system_host->SetFlushAndPurgeOnCleanup(TRUE);
file_system_host->SetReparsePoints(FALSE);
@@ -188,7 +227,8 @@ NTSTATUS remote_winfsp_drive::Init(PVOID host) {
return STATUS_SUCCESS;
}
int remote_winfsp_drive::mount(const std::vector<std::string> &drive_args) {
auto remote_winfsp_drive::mount(const std::vector<std::string> &drive_args)
-> int {
std::vector<std::string> parsed_drive_args;
const auto force_no_console = utils::collection_includes(drive_args, "-nc");
@@ -210,50 +250,61 @@ int remote_winfsp_drive::mount(const std::vector<std::string> &drive_args) {
c = std::make_unique<console_consumer>();
}
event_system::instance().start();
const auto ret = winfsp_service(lock_, *this, parsed_drive_args, config_).Run();
const auto ret =
winfsp_service(lock_, *this, parsed_drive_args, config_).Run();
event_system::instance().raise<drive_mount_result>(std::to_string(ret));
event_system::instance().stop();
c.reset();
return static_cast<int>(ret);
}
NTSTATUS remote_winfsp_drive::Mounted(PVOID host) {
auto remote_winfsp_drive::Mounted(PVOID host) -> NTSTATUS {
auto *file_system_host = reinterpret_cast<FileSystemHost *>(host);
remote_instance_ = factory_();
server_ = std::make_unique<server>(config_);
server_->start();
mount_location_ = utils::string::to_utf8(file_system_host->MountPoint());
lock_.set_mount_state(true, mount_location_, ::GetCurrentProcessId());
if (not lock_.set_mount_state(true, mount_location_,
::GetCurrentProcessId())) {
utils::error::raise_error(__FUNCTION__, "failed to set mount state");
}
return remote_instance_->winfsp_mounted(file_system_host->MountPoint());
}
NTSTATUS remote_winfsp_drive::Open(PWSTR file_name, UINT32 create_options, UINT32 granted_access,
PVOID *file_node, PVOID *file_desc, OpenFileInfo *ofi) {
auto remote_winfsp_drive::Open(PWSTR file_name, UINT32 create_options,
UINT32 granted_access, PVOID * /*file_node*/,
PVOID *file_desc, OpenFileInfo *ofi)
-> NTSTATUS {
remote::file_info fi{};
std::string normalize_name;
const auto ret = remote_instance_->winfsp_open(file_name, create_options, granted_access,
file_desc, &fi, normalize_name);
const auto ret =
remote_instance_->winfsp_open(file_name, create_options, granted_access,
file_desc, &fi, normalize_name);
if (ret == STATUS_SUCCESS) {
SetFileInfo(ofi->FileInfo, fi);
const auto filePath = utils::string::from_utf8(normalize_name);
wcsncpy(ofi->NormalizedName, &filePath[0], wcslen(&filePath[0]));
ofi->NormalizedNameSize = (UINT16)(wcslen(&filePath[0]) * sizeof(WCHAR));
set_file_info(ofi->FileInfo, fi);
const auto file_path = utils::string::from_utf8(normalize_name);
wcsncpy(ofi->NormalizedName, &file_path[0], wcslen(&file_path[0]));
ofi->NormalizedNameSize = (UINT16)(wcslen(&file_path[0]) * sizeof(WCHAR));
}
return ret;
}
NTSTATUS remote_winfsp_drive::Overwrite(PVOID file_node, PVOID file_desc, UINT32 attributes,
BOOLEAN replace_attributes, UINT64 allocation_size,
FileInfo *file_info) {
auto remote_winfsp_drive::Overwrite(PVOID /*file_node*/, PVOID file_desc,
UINT32 attributes,
BOOLEAN replace_attributes,
UINT64 allocation_size, FileInfo *file_info)
-> NTSTATUS {
remote::file_info fi{};
const auto ret = remote_instance_->winfsp_overwrite(file_desc, attributes, replace_attributes,
allocation_size, &fi);
SetFileInfo(*file_info, fi);
const auto ret = remote_instance_->winfsp_overwrite(
file_desc, attributes, replace_attributes, allocation_size, &fi);
set_file_info(*file_info, fi);
return ret;
}
void remote_winfsp_drive::PopulateFileInfo(const json &item, FSP_FSCTL_FILE_INFO &file_info) {
void remote_winfsp_drive::populate_file_info(const json &item,
FSP_FSCTL_FILE_INFO &file_info) {
const auto di = directory_item::from_json(item);
file_info.FileSize = di.directory ? 0 : di.size;
file_info.AllocationSize =
@@ -270,28 +321,33 @@ void remote_winfsp_drive::PopulateFileInfo(const json &item, FSP_FSCTL_FILE_INFO
file_info.EaSize = 0;
}
NTSTATUS remote_winfsp_drive::Read(PVOID file_node, PVOID file_desc, PVOID buffer, UINT64 offset,
ULONG length, PULONG bytes_transferred) {
return remote_instance_->winfsp_read(file_desc, buffer, offset, length,
reinterpret_cast<PUINT32>(bytes_transferred));
auto remote_winfsp_drive::Read(PVOID /*file_node*/, PVOID file_desc,
PVOID buffer, UINT64 offset, ULONG length,
PULONG bytes_transferred) -> NTSTATUS {
return remote_instance_->winfsp_read(
file_desc, buffer, offset, length,
reinterpret_cast<PUINT32>(bytes_transferred));
}
NTSTATUS remote_winfsp_drive::ReadDirectory(PVOID file_node, PVOID file_desc, PWSTR pattern,
PWSTR marker, PVOID buffer, ULONG buffer_length,
PULONG bytes_transferred) {
json itemList;
NTSTATUS ret = remote_instance_->winfsp_read_directory(file_desc, pattern, marker, itemList);
auto remote_winfsp_drive::ReadDirectory(PVOID /*file_node*/, PVOID file_desc,
PWSTR pattern, PWSTR marker,
PVOID buffer, ULONG buffer_length,
PULONG bytes_transferred) -> NTSTATUS {
json item_list;
NTSTATUS ret = remote_instance_->winfsp_read_directory(file_desc, pattern,
marker, item_list);
if (ret == STATUS_SUCCESS) {
PVOID *directory_buffer = nullptr;
if ((ret = remote_instance_->winfsp_get_dir_buffer(file_desc, directory_buffer)) ==
STATUS_SUCCESS) {
if (FspFileSystemAcquireDirectoryBuffer(directory_buffer,
static_cast<BOOLEAN>(nullptr == marker), &ret)) {
if ((ret = remote_instance_->winfsp_get_dir_buffer(
file_desc, directory_buffer)) == STATUS_SUCCESS) {
if (FspFileSystemAcquireDirectoryBuffer(
directory_buffer, static_cast<BOOLEAN>(nullptr == marker),
&ret)) {
auto item_found = false;
for (const auto &item : itemList) {
for (const auto &item : item_list) {
const auto item_path = item["path"].get<std::string>();
const auto display_name =
utils::string::from_utf8(utils::path::strip_to_file_name(item_path));
const auto display_name = utils::string::from_utf8(
utils::path::strip_to_file_name(item_path));
if (not marker || (marker && item_found)) {
if (not utils::path::is_ads_file_path(item_path)) {
union {
@@ -304,15 +360,19 @@ NTSTATUS remote_winfsp_drive::ReadDirectory(PVOID file_node, PVOID file_desc, PW
::ZeroMemory(directory_info, sizeof(*directory_info));
directory_info->Size = static_cast<UINT16>(
FIELD_OFFSET(FSP_FSCTL_DIR_INFO, FileNameBuf) +
(std::min((size_t)MAX_PATH, display_name.size()) * sizeof(WCHAR)));
(std::min((size_t)MAX_PATH, display_name.size()) *
sizeof(WCHAR)));
if (not item["meta"].empty() || ((item_path != ".") && (item_path != ".."))) {
PopulateFileInfo(item, directory_info->FileInfo);
if (not item["meta"].empty() ||
((item_path != ".") && (item_path != ".."))) {
populate_file_info(item, directory_info->FileInfo);
}
if (ret == STATUS_SUCCESS) {
::wcscpy_s(&directory_info->FileNameBuf[0], MAX_PATH, &display_name[0]);
::wcscpy_s(&directory_info->FileNameBuf[0], MAX_PATH,
&display_name[0]);
FspFileSystemFillDirectoryBuffer(directory_buffer, directory_info, &ret);
FspFileSystemFillDirectoryBuffer(directory_buffer,
directory_info, &ret);
if (ret != STATUS_SUCCESS) {
break;
}
@@ -327,8 +387,9 @@ NTSTATUS remote_winfsp_drive::ReadDirectory(PVOID file_node, PVOID file_desc, PW
}
if ((ret == STATUS_SUCCESS) || (ret == STATUS_NO_MORE_FILES)) {
FspFileSystemReadDirectoryBuffer(directory_buffer, marker, buffer, buffer_length,
reinterpret_cast<PULONG>(bytes_transferred));
FspFileSystemReadDirectoryBuffer(
directory_buffer, marker, buffer, buffer_length,
reinterpret_cast<PULONG>(bytes_transferred));
ret = STATUS_SUCCESS;
}
}
@@ -337,23 +398,29 @@ NTSTATUS remote_winfsp_drive::ReadDirectory(PVOID file_node, PVOID file_desc, PW
return ret;
}
NTSTATUS remote_winfsp_drive::Rename(PVOID file_node, PVOID file_desc, PWSTR file_name,
PWSTR new_file_name, BOOLEAN replace_if_exists) {
return remote_instance_->winfsp_rename(file_desc, file_name, new_file_name, replace_if_exists);
auto remote_winfsp_drive::Rename(PVOID /*file_node*/, PVOID file_desc,
PWSTR file_name, PWSTR new_file_name,
BOOLEAN replace_if_exists) -> NTSTATUS {
return remote_instance_->winfsp_rename(file_desc, file_name, new_file_name,
replace_if_exists);
}
NTSTATUS remote_winfsp_drive::SetBasicInfo(PVOID file_node, PVOID file_desc, UINT32 attributes,
UINT64 creation_time, UINT64 last_access_time,
UINT64 last_write_time, UINT64 change_time,
FileInfo *file_info) {
auto remote_winfsp_drive::SetBasicInfo(PVOID /*file_node*/, PVOID file_desc,
UINT32 attributes, UINT64 creation_time,
UINT64 last_access_time,
UINT64 last_write_time,
UINT64 change_time, FileInfo *file_info)
-> NTSTATUS {
remote::file_info fi{};
const auto ret = remote_instance_->winfsp_set_basic_info(
file_desc, attributes, creation_time, last_access_time, last_write_time, change_time, &fi);
SetFileInfo(*file_info, fi);
file_desc, attributes, creation_time, last_access_time, last_write_time,
change_time, &fi);
set_file_info(*file_info, fi);
return ret;
}
void remote_winfsp_drive::SetFileInfo(FileInfo &dest, const remote::file_info &src) {
void remote_winfsp_drive::set_file_info(FileInfo &dest,
const remote::file_info &src) {
dest.FileAttributes = src.FileAttributes;
dest.ReparseTag = src.ReparseTag;
dest.AllocationSize = src.AllocationSize;
@@ -367,33 +434,39 @@ void remote_winfsp_drive::SetFileInfo(FileInfo &dest, const remote::file_info &s
dest.EaSize = src.EaSize;
}
NTSTATUS remote_winfsp_drive::SetFileSize(PVOID file_node, PVOID file_desc, UINT64 new_size,
BOOLEAN set_allocation_size, FileInfo *file_info) {
auto remote_winfsp_drive::SetFileSize(PVOID /*file_node*/, PVOID file_desc,
UINT64 new_size,
BOOLEAN set_allocation_size,
FileInfo *file_info) -> NTSTATUS {
remote::file_info fi{};
const auto ret =
remote_instance_->winfsp_set_file_size(file_desc, new_size, set_allocation_size, &fi);
SetFileInfo(*file_info, fi);
const auto ret = remote_instance_->winfsp_set_file_size(
file_desc, new_size, set_allocation_size, &fi);
set_file_info(*file_info, fi);
return ret;
}
VOID remote_winfsp_drive::Unmounted(PVOID host) {
server_->stop();
server_.reset();
auto *fileSystemHost = reinterpret_cast<FileSystemHost *>(host);
lock_.set_mount_state(false, "", -1);
remote_instance_->winfsp_unmounted(fileSystemHost->MountPoint());
auto *file_system_host = reinterpret_cast<FileSystemHost *>(host);
if (not lock_.set_mount_state(false, "", -1)) {
utils::error::raise_error(__FUNCTION__, "failed to set mount state");
}
remote_instance_->winfsp_unmounted(file_system_host->MountPoint());
remote_instance_.reset();
mount_location_ = "";
}
NTSTATUS remote_winfsp_drive::Write(PVOID file_node, PVOID file_desc, PVOID buffer, UINT64 offset,
ULONG length, BOOLEAN write_to_end, BOOLEAN constrained_io,
PULONG bytes_transferred, FileInfo *file_info) {
auto remote_winfsp_drive::Write(PVOID /*file_node*/, PVOID file_desc,
PVOID buffer, UINT64 offset, ULONG length,
BOOLEAN write_to_end, BOOLEAN constrained_io,
PULONG bytes_transferred, FileInfo *file_info)
-> NTSTATUS {
remote::file_info fi{};
const auto ret = remote_instance_->winfsp_write(
file_desc, buffer, offset, length, write_to_end, constrained_io,
reinterpret_cast<PUINT32>(bytes_transferred), &fi);
SetFileInfo(*file_info, fi);
set_file_info(*file_info, fi);
return ret;
}
} // namespace repertory::remote_winfsp

1071
src/drives/winfsp/winfsp_drive.cpp
View File

File diff suppressed because it is too large Load Diff

95
src/events/consumers/logging_consumer.cpp
View File

@@ -1,74 +1,106 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "events/consumers/logging_consumer.hpp"
#include "events/events.hpp"
#include "types/startup_exception.hpp"
#include "utils/error_utils.hpp"
#include "utils/file_utils.hpp"
#include "utils/path_utils.hpp"
#include "utils/unix/unix_utils.hpp"
#include "utils/utils.hpp"
namespace repertory {
logging_consumer::logging_consumer(const std::string &log_directory, const event_level &level)
logging_consumer::logging_consumer(const std::string &log_directory,
const event_level &level)
: event_level_(level),
log_directory_(utils::path::finalize(log_directory)),
log_path_(utils::path::combine(log_directory, {"repertory.log"})) {
utils::file::create_full_directory_path(log_directory_);
if (not utils::file::create_full_directory_path(log_directory_)) {
throw startup_exception("failed to create log directory|sp|" +
log_directory_ + "|err|" +
std::to_string(utils::get_last_error_code()));
}
check_log_roll(0);
reopen_log_file();
E_SUBSCRIBE_ALL(process_event);
E_SUBSCRIBE_EXACT(event_level_changed, [this](const event_level_changed &e) {
event_level_ = event_level_from_string(e.get_new_event_level().get<std::string>());
});
logging_thread_ = std::make_unique<std::thread>([this]() { this->logging_thread(); });
E_SUBSCRIBE_EXACT(event_level_changed,
[this](const event_level_changed &evt) {
event_level_ = event_level_from_string(
evt.get_new_event_level().get<std::string>());
});
logging_thread_ =
std::make_unique<std::thread>([this] { logging_thread(false); });
}
logging_consumer::~logging_consumer() {
E_CONSUMER_RELEASE();
{
mutex_lock l(log_mutex_);
logging_active_ = false;
log_notify_.notify_all();
}
unique_mutex_lock l(log_mutex_);
logging_active_ = false;
log_notify_.notify_all();
l.unlock();
logging_thread_->join();
logging_thread_.reset();
logging_thread(true);
close_log_file();
}
void logging_consumer::check_log_roll(const size_t &count) {
void logging_consumer::check_log_roll(std::size_t count) {
std::uint64_t file_size{};
const auto success = utils::file::get_file_size(log_path_, file_size);
if (success && (file_size + count) >= MAX_LOG_FILE_SIZE) {
close_log_file();
for (std::uint8_t i = MAX_LOG_FILES; i > 0u; i--) {
const auto temp_log_path = utils::path::combine(
log_directory_, {"repertory." + utils::string::from_uint8(i) + ".log"});
log_directory_, {"repertory." + std::to_string(i) + ".log"});
if (utils::file::is_file(temp_log_path)) {
if (i == MAX_LOG_FILES) {
utils::file::delete_file(temp_log_path);
if (not utils::file::retry_delete_file(temp_log_path)) {
}
} else {
const auto next_file_path = utils::path::combine(
log_directory_,
{"repertory." + utils::string::from_uint8(i + std::uint8_t(1)) + ".log"});
utils::file::move_file(temp_log_path, next_file_path);
{"repertory." + std::to_string(i + std::uint8_t(1)) + ".log"});
if (not utils::file::move_file(temp_log_path, next_file_path)) {
utils::error::raise_error(__FUNCTION__,
utils::get_last_error_code(),
temp_log_path + "|dest|" + next_file_path,
"failed to move file");
}
}
}
}
utils::file::move_file(log_path_, utils::path::combine(log_directory_, {"repertory.1.log"}));
auto backup_log_path =
utils::path::combine(log_directory_, {"repertory.1.log"});
if (not utils::file::move_file(log_path_, backup_log_path)) {
utils::error::raise_error(__FUNCTION__, utils::get_last_error_code(),
log_path_ + "|dest|" + backup_log_path,
"failed to move file");
}
reopen_log_file();
}
}
@@ -80,7 +112,7 @@ void logging_consumer::close_log_file() {
}
}
void logging_consumer::logging_thread(const bool &drain) {
void logging_consumer::logging_thread(bool drain) {
do {
std::deque<std::shared_ptr<event>> events;
{
@@ -112,7 +144,8 @@ void logging_consumer::logging_thread(const bool &drain) {
check_log_roll(msg.length());
auto retry = true;
for (int i = 0; retry && (i < 2); i++) {
retry = (not log_file_ || (fwrite(&msg[0], 1, msg.length(), log_file_) != msg.length()));
retry = (not log_file_ || (fwrite(&msg[0], 1, msg.length(),
log_file_) != msg.length()));
if (retry) {
reopen_log_file();
}

32
src/events/event.cpp
View File

@@ -1,26 +1,30 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "events/event.hpp"
#include "utils/string_utils.hpp"
namespace repertory {
event_level event_level_from_string(std::string level) {
auto event_level_from_string(std::string level) -> event_level {
level = utils::string::to_lower(level);
if (level == "debug" || level == "event_level::debug") {
return event_level::debug;
@@ -36,7 +40,7 @@ event_level event_level_from_string(std::string level) {
return event_level::normal;
}
std::string event_level_to_string(const event_level &level) {
auto event_level_to_string(const event_level &level) -> std::string {
switch (level) {
case event_level::debug:
return "debug";

35
src/events/t_event_system.cpp
View File

@@ -1,29 +1,36 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "events/t_event_system.hpp"
#include "events/event.hpp"
namespace repertory {
template <typename event_type> t_event_system<event_type> &t_event_system<event_type>::instance() {
template <typename event_type>
auto t_event_system<event_type>::instance() -> t_event_system<event_type> & {
return event_system_;
}
template <typename event_type> t_event_system<event_type> t_event_system<event_type>::event_system_;
template <typename event_type>
t_event_system<event_type> t_event_system<event_type>::event_system_;
template class t_event_system<event>;
} // namespace repertory

979
src/file_manager/file_manager.cpp Normal file
View File

@@ -0,0 +1,979 @@
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "file_manager/file_manager.hpp"
#include "app_config.hpp"
#include "file_manager/events.hpp"
#include "providers/i_provider.hpp"
#include "types/repertory.hpp"
#include "utils/encrypting_reader.hpp"
#include "utils/error_utils.hpp"
#include "utils/file_utils.hpp"
#include "utils/path_utils.hpp"
#include "utils/polling.hpp"
#include "utils/rocksdb_utils.hpp"
#include "utils/unix/unix_utils.hpp"
namespace repertory {
static auto create_resume_entry(const i_open_file &o) -> json {
return {
{"chunk_size", o.get_chunk_size()},
{"path", o.get_api_path()},
{"read_state", utils::string::from_dynamic_bitset(o.get_read_state())},
{"source", o.get_source_path()},
};
}
static void restore_resume_entry(const json &resume_entry,
std::string &api_path, std::size_t &chunk_size,
boost::dynamic_bitset<> &read_state,
std::string &source_path) {
api_path = resume_entry["path"].get<std::string>();
chunk_size = resume_entry["chunk_size"].get<std::size_t>();
read_state = utils::string::to_dynamic_bitset(
resume_entry["read_state"].get<std::string>());
source_path = resume_entry["source"].get<std::string>();
}
file_manager::file_manager(app_config &config, i_provider &provider)
: config_(config), provider_(provider) {
if (not provider_.is_direct_only()) {
auto families = std::vector<rocksdb::ColumnFamilyDescriptor>();
families.emplace_back(rocksdb::kDefaultColumnFamilyName,
rocksdb::ColumnFamilyOptions());
families.emplace_back("upload", rocksdb::ColumnFamilyOptions());
families.emplace_back("upload_active", rocksdb::ColumnFamilyOptions());
auto handles = std::vector<rocksdb::ColumnFamilyHandle *>();
utils::db::create_rocksdb(config, "upload_db", families, handles, db_);
std::size_t idx{};
default_family_ = handles[idx++];
upload_family_ = handles[idx++];
upload_active_family_ = handles[idx++];
E_SUBSCRIBE_EXACT(file_upload_completed,
[this](const file_upload_completed &evt) {
this->upload_completed(evt);
});
}
}
file_manager::~file_manager() {
stop();
E_CONSUMER_RELEASE();
db_.reset();
}
void file_manager::close(std::uint64_t handle) {
unique_recur_mutex_lock file_lock(open_file_mtx_);
auto it = open_handle_lookup_.find(handle);
if (it != open_handle_lookup_.end()) {
auto *cur_file = it->second;
open_handle_lookup_.erase(handle);
cur_file->remove(handle);
if (cur_file->can_close()) {
const auto api_path = cur_file->get_api_path();
cur_file = nullptr;
auto closeable_file = open_file_lookup_.at(api_path);
open_file_lookup_.erase(api_path);
file_lock.unlock();
closeable_file->close();
}
}
}
void file_manager::close_all(const std::string &api_path) {
recur_mutex_lock file_lock(open_file_mtx_);
std::vector<std::uint64_t> handles;
auto it = open_file_lookup_.find(api_path);
if (it != open_file_lookup_.end()) {
handles = it->second->get_handles();
}
for (auto &handle : handles) {
open_file_lookup_[api_path]->remove(handle);
open_handle_lookup_.erase(handle);
}
open_file_lookup_.erase(api_path);
}
void file_manager::close_timed_out_files() {
unique_recur_mutex_lock file_lock(open_file_mtx_);
auto closeable_list = std::accumulate(
open_file_lookup_.begin(), open_file_lookup_.end(),
std::vector<std::string>{}, [](auto items, const auto &kv) -> auto {
if (kv.second->get_open_file_count() == 0U && kv.second->can_close()) {
items.emplace_back(kv.first);
}
return items;
});
std::vector<std::shared_ptr<i_closeable_open_file>> open_files{};
for (const auto &api_path : closeable_list) {
auto closeable_file = open_file_lookup_.at(api_path);
open_file_lookup_.erase(api_path);
open_files.emplace_back(closeable_file);
}
closeable_list.clear();
file_lock.unlock();
for (auto &closeable_file : open_files) {
closeable_file->close();
event_system::instance().raise<download_timeout>(
closeable_file->get_api_path());
}
}
auto file_manager::create(const std::string &api_path, api_meta_map &meta,
open_file_data ofd, std::uint64_t &handle,
std::shared_ptr<i_open_file> &f) -> api_error {
recur_mutex_lock file_lock(open_file_mtx_);
auto res = provider_.create_file(api_path, meta);
if (res != api_error::success && res != api_error::item_exists) {
return res;
}
return open(api_path, false, ofd, handle, f);
}
auto file_manager::evict_file(const std::string &api_path) -> bool {
if (provider_.is_direct_only()) {
return false;
}
recur_mutex_lock open_lock(open_file_mtx_);
if (is_processing(api_path)) {
return false;
}
if (get_open_file_count(api_path) != 0U) {
return false;
}
std::string pinned;
auto res = provider_.get_item_meta(api_path, META_PINNED, pinned);
if (res != api_error::success && res != api_error::item_not_found) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, res,
"failed to get pinned status");
return false;
}
if (not pinned.empty() && utils::string::to_bool(pinned)) {
return false;
}
std::string source_path{};
res = provider_.get_item_meta(api_path, META_SOURCE, source_path);
if (res != api_error::success) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, res,
"failed to get source path");
return false;
}
if (source_path.empty()) {
return false;
}
auto removed = utils::file::retry_delete_file(source_path);
if (removed) {
event_system::instance().raise<filesystem_item_evicted>(api_path,
source_path);
}
return removed;
}
auto file_manager::get_directory_items(const std::string &api_path) const
-> directory_item_list {
directory_item_list list{};
auto res = provider_.get_directory_items(api_path, list);
if (res != api_error::success) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, res,
"failed to get directory list");
}
return list;
}
auto file_manager::get_next_handle() -> std::uint64_t {
if (++next_handle_ == 0u) {
next_handle_++;
}
return next_handle_;
}
auto file_manager::get_open_file_count(const std::string &api_path) const
-> std::size_t {
recur_mutex_lock open_lock(open_file_mtx_);
auto it = open_file_lookup_.find(api_path);
if (it != open_file_lookup_.end()) {
return it->second->get_open_file_count();
}
return 0u;
}
auto file_manager::get_open_file(std::uint64_t handle, bool write_supported,
std::shared_ptr<i_open_file> &f) -> bool {
recur_mutex_lock open_lock(open_file_mtx_);
auto it = open_handle_lookup_.find(handle);
if (it == open_handle_lookup_.end()) {
return false;
}
auto of = open_file_lookup_.at(it->second->get_api_path());
if (write_supported && not of->is_write_supported()) {
auto new_f = std::make_shared<open_file>(
utils::encryption::encrypting_reader::get_data_chunk_size(),
config_.get_enable_chunk_download_timeout()
? config_.get_chunk_downloader_timeout_secs()
: 0U,
of->get_filesystem_item(), of->get_open_data(), provider_, *this);
open_file_lookup_[of->get_api_path()] = new_f;
f = new_f;
return true;
}
f = of;
return true;
}
auto file_manager::get_open_file_count() const -> std::size_t {
recur_mutex_lock open_lock(open_file_mtx_);
return open_file_lookup_.size();
}
auto file_manager::get_open_files() const
-> std::unordered_map<std::string, std::size_t> {
std::unordered_map<std::string, std::size_t> ret;
recur_mutex_lock open_lock(open_file_mtx_);
for (const auto &kv : open_file_lookup_) {
ret[kv.first] = kv.second->get_open_file_count();
}
return ret;
}
auto file_manager::get_open_handle_count() const -> std::size_t {
recur_mutex_lock open_lock(open_file_mtx_);
return open_handle_lookup_.size();
}
auto file_manager::get_stored_downloads() const -> std::vector<json> {
std::vector<json> ret;
if (not provider_.is_direct_only()) {
auto iterator = std::unique_ptr<rocksdb::Iterator>(
db_->NewIterator(rocksdb::ReadOptions(), default_family_));
for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
ret.emplace_back(json::parse(iterator->value().ToString()));
}
}
return ret;
}
auto file_manager::handle_file_rename(const std::string &from_api_path,
const std::string &to_api_path)
-> api_error {
auto ret = api_error::file_in_use;
if ((ret = provider_.rename_file(from_api_path, to_api_path)) ==
api_error::success) {
swap_renamed_items(from_api_path, to_api_path);
}
return ret;
}
auto file_manager::has_no_open_file_handles() const -> bool {
recur_mutex_lock open_lock(open_file_mtx_);
return open_handle_lookup_.empty();
}
auto file_manager::is_processing(const std::string &api_path) const -> bool {
if (provider_.is_direct_only()) {
return false;
}
recur_mutex_lock open_lock(open_file_mtx_);
mutex_lock upload_lock(upload_mtx_);
if (upload_lookup_.find(api_path) != upload_lookup_.end()) {
return true;
}
{
auto iterator = std::unique_ptr<rocksdb::Iterator>(
db_->NewIterator(rocksdb::ReadOptions(), upload_family_));
for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
const auto parts = utils::string::split(iterator->key().ToString(), ':');
if (parts[1u] == api_path) {
return true;
}
}
}
auto it = open_file_lookup_.find(api_path);
if (it != open_file_lookup_.end()) {
return it->second->is_modified() || not it->second->is_complete();
}
return false;
}
auto file_manager::open(const std::string &api_path, bool directory,
const open_file_data &ofd, std::uint64_t &handle,
std::shared_ptr<i_open_file> &f) -> api_error {
return open(api_path, directory, ofd, handle, f, nullptr);
}
auto file_manager::open(const std::string &api_path, bool directory,
const open_file_data &ofd, std::uint64_t &handle,
std::shared_ptr<i_open_file> &f,
std::shared_ptr<i_closeable_open_file> of)
-> api_error {
const auto create_and_add_handle =
[&](std::shared_ptr<i_closeable_open_file> cur_file) {
handle = get_next_handle();
cur_file->add(handle, ofd);
open_handle_lookup_[handle] = cur_file.get();
f = cur_file;
};
recur_mutex_lock open_lock(open_file_mtx_);
auto it = open_file_lookup_.find(api_path);
if (it != open_file_lookup_.end()) {
create_and_add_handle(it->second);
return api_error::success;
}
filesystem_item fsi{};
auto res = provider_.get_filesystem_item(api_path, directory, fsi);
if (res != api_error::success) {
return res;
}
if (fsi.source_path.empty()) {
fsi.source_path = utils::path::combine(config_.get_cache_directory(),
{utils::create_uuid_string()});
if ((res = provider_.set_item_meta(fsi.api_path, META_SOURCE,
fsi.source_path)) !=
api_error::success) {
return res;
}
}
if (not of) {
of = std::make_shared<open_file>(
utils::encryption::encrypting_reader::get_data_chunk_size(),
config_.get_enable_chunk_download_timeout()
? config_.get_chunk_downloader_timeout_secs()
: 0U,
fsi, provider_, *this);
}
open_file_lookup_[api_path] = of;
create_and_add_handle(of);
return api_error::success;
}
auto file_manager::perform_locked_operation(
locked_operation_callback locked_operation) -> bool {
recur_mutex_lock open_lock(open_file_mtx_);
return locked_operation(provider_);
}
void file_manager::queue_upload(const i_open_file &o) {
return queue_upload(o.get_api_path(), o.get_source_path(), false);
}
void file_manager::queue_upload(const std::string &api_path,
const std::string &source_path, bool no_lock) {
if (provider_.is_direct_only()) {
return;
}
std::unique_ptr<mutex_lock> l;
if (not no_lock) {
l = std::make_unique<mutex_lock>(upload_mtx_);
}
remove_upload(api_path, true);
auto res = db_->Put(
rocksdb::WriteOptions(), upload_family_,
std::to_string(utils::get_file_time_now()) + ":" + api_path, source_path);
if (res.ok()) {
remove_resume(api_path, source_path);
event_system::instance().raise<file_upload_queued>(api_path, source_path);
} else {
event_system::instance().raise<file_upload_failed>(api_path, source_path,
res.ToString());
}
if (not no_lock) {
upload_notify_.notify_all();
}
}
auto file_manager::remove_file(const std::string &api_path) -> api_error {
recur_mutex_lock open_lock(open_file_mtx_);
auto it = open_file_lookup_.find(api_path);
if (it != open_file_lookup_.end() && it->second->is_modified()) {
return api_error::file_in_use;
}
filesystem_item fsi{};
auto res = provider_.get_filesystem_item(api_path, false, fsi);
if (res != api_error::success) {
return res;
}
if ((res = provider_.remove_file(api_path)) != api_error::success) {
return res;
}
remove_upload(api_path);
close_all(api_path);
if (not utils::file::retry_delete_file(fsi.source_path)) {
utils::error::raise_api_path_error(
__FUNCTION__, fsi.api_path, fsi.source_path,
utils::get_last_error_code(), "failed to delete source");
return api_error::success;
}
return api_error::success;
}
void file_manager::remove_resume(const std::string &api_path,
const std::string &source_path) {
auto res = db_->Delete(rocksdb::WriteOptions(), default_family_, api_path);
if (res.ok()) {
event_system::instance().raise<download_stored_removed>(api_path,
source_path);
} else {
utils::error::raise_api_path_error(__FUNCTION__, api_path, source_path,
res.code(),
"failed to remove resume entry");
}
}
void file_manager::remove_upload(const std::string &api_path) {
remove_upload(api_path, false);
}
void file_manager::remove_upload(const std::string &api_path, bool no_lock) {
if (provider_.is_direct_only()) {
return;
}
std::unique_ptr<mutex_lock> l;
if (not no_lock) {
l = std::make_unique<mutex_lock>(upload_mtx_);
}
auto it = upload_lookup_.find(api_path);
if (it == upload_lookup_.end()) {
auto iterator = std::unique_ptr<rocksdb::Iterator>(
db_->NewIterator(rocksdb::ReadOptions(), upload_family_));
for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
const auto parts = utils::string::split(iterator->key().ToString(), ':');
if (parts[1U] == api_path) {
if (db_->Delete(rocksdb::WriteOptions(), upload_family_,
iterator->key())
.ok()) {
db_->Delete(rocksdb::WriteOptions(), upload_active_family_,
iterator->key());
}
event_system::instance().raise<file_upload_removed>(
api_path, iterator->value().ToString());
}
}
} else {
auto iterator = std::unique_ptr<rocksdb::Iterator>(
db_->NewIterator(rocksdb::ReadOptions(), upload_active_family_));
for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
const auto parts = utils::string::split(iterator->key().ToString(), ':');
if (parts[1U] == api_path) {
db_->Delete(rocksdb::WriteOptions(), upload_active_family_,
iterator->key());
}
}
event_system::instance().raise<file_upload_removed>(
api_path, it->second->get_source_path());
it->second->cancel();
upload_lookup_.erase(api_path);
}
if (not no_lock) {
upload_notify_.notify_all();
}
}
void file_manager::swap_renamed_items(std::string from_api_path,
std::string to_api_path) {
const auto it = open_file_lookup_.find(from_api_path);
if (it != open_file_lookup_.end()) {
open_file_lookup_[to_api_path] = open_file_lookup_[from_api_path];
open_file_lookup_.erase(from_api_path);
open_file_lookup_[to_api_path]->set_api_path(to_api_path);
}
}
auto file_manager::rename_directory(const std::string &from_api_path,
const std::string &to_api_path)
-> api_error {
if (not provider_.is_rename_supported()) {
return api_error::not_implemented;
}
unique_recur_mutex_lock l(open_file_mtx_);
// Ensure source directory exists
bool exists{};
auto res = provider_.is_directory(from_api_path, exists);
if (res != api_error::success) {
return res;
}
if (not exists) {
return api_error::directory_not_found;
}
// Ensure destination directory does not exist
res = provider_.is_directory(to_api_path, exists);
if (res != api_error::success) {
return res;
}
if (exists) {
return api_error::directory_exists;
}
// Ensure destination is not a file
res = provider_.is_file(from_api_path, exists);
if (res != api_error::success) {
return res;
}
if (exists) {
return api_error::item_exists;
}
// Create destination directory
res =
provider_.create_directory_clone_source_meta(from_api_path, to_api_path);
if (res != api_error::success) {
return res;
}
directory_item_list list{};
res = provider_.get_directory_items(from_api_path, list);
if (res != api_error::success) {
return res;
}
// Rename all items - directories MUST BE returned first
for (std::size_t i = 0U; (res == api_error::success) && (i < list.size());
i++) {
const auto &api_path = list[i].api_path;
if ((api_path != ".") && (api_path != "..")) {
const auto old_api_path = api_path;
const auto new_api_path =
utils::path::create_api_path(utils::path::combine(
to_api_path, {old_api_path.substr(from_api_path.size())}));
res = list[i].directory ? rename_directory(old_api_path, new_api_path)
: rename_file(old_api_path, new_api_path, false);
}
}
if (res != api_error::success) {
return res;
}
res = provider_.remove_directory(from_api_path);
if (res != api_error::success) {
return res;
}
swap_renamed_items(from_api_path, to_api_path);
return api_error::success;
}
auto file_manager::rename_file(const std::string &from_api_path,
const std::string &to_api_path, bool overwrite)
-> api_error {
if (not provider_.is_rename_supported()) {
return api_error::not_implemented;
}
// Don't rename if paths are the same
if (from_api_path == to_api_path) {
return api_error::item_exists;
}
unique_recur_mutex_lock l(open_file_mtx_);
// Don't rename if source is directory
bool exists{};
auto res = provider_.is_directory(from_api_path, exists);
if (res != api_error::success) {
return res;
}
if (exists) {
return api_error::directory_exists;
}
// Don't rename if source does not exist
res = provider_.is_file(from_api_path, exists);
if (res != api_error::success) {
return res;
}
if (not exists) {
return api_error::item_not_found;
}
// Don't rename if destination is directory
res = provider_.is_directory(to_api_path, exists);
if (res != api_error::success) {
return res;
}
if (exists) {
res = api_error::directory_exists;
}
// Check allow overwrite if file exists
bool dest_exists{};
res = provider_.is_file(to_api_path, dest_exists);
if (res != api_error::success) {
return res;
}
if (not overwrite && dest_exists) {
return api_error::item_exists;
}
// Don't rename if destination file has open handles
if (get_open_file_count(to_api_path) != 0U) {
return api_error::file_in_use;
}
// Don't rename if destination file is uploading or downloading
if (is_processing(to_api_path)) {
return api_error::file_in_use;
}
// Handle destination file exists (should overwrite)
if (dest_exists) {
filesystem_item fsi{};
res = provider_.get_filesystem_item(to_api_path, false, fsi);
if (res != api_error::success) {
return res;
}
std::uint64_t file_size{};
if (not utils::file::get_file_size(fsi.source_path, file_size)) {
return api_error::os_error;
}
res = provider_.remove_file(to_api_path);
if ((res == api_error::success) || (res == api_error::item_not_found)) {
if (not utils::file::retry_delete_file(fsi.source_path)) {
utils::error::raise_api_path_error(
__FUNCTION__, fsi.api_path, fsi.source_path,
utils::get_last_error_code(), "failed to delete source path");
}
return handle_file_rename(from_api_path, to_api_path);
}
return res;
}
// Check destination parent directory exists
res = provider_.is_directory(utils::path::get_parent_api_path(to_api_path),
exists);
if (res != api_error::success) {
return res;
}
if (not exists) {
return api_error::directory_not_found;
}
return handle_file_rename(from_api_path, to_api_path);
}
void file_manager::start() {
if (provider_.is_direct_only()) {
stop_requested_ = false;
return;
}
if (not upload_thread_) {
stop_requested_ = false;
struct active_item {
std::string api_path;
std::string source_path;
};
std::vector<active_item> active_items{};
auto iterator = std::unique_ptr<rocksdb::Iterator>(
db_->NewIterator(rocksdb::ReadOptions(), upload_active_family_));
for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
const auto parts = utils::string::split(iterator->key().ToString(), ':');
active_items.emplace_back(
active_item{parts[1U], iterator->value().ToString()});
}
for (const auto &active_item : active_items) {
queue_upload(active_item.api_path, active_item.source_path, false);
}
active_items.clear();
iterator = std::unique_ptr<rocksdb::Iterator>(
db_->NewIterator(rocksdb::ReadOptions(), default_family_));
for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
std::string api_path;
std::string source_path;
std::size_t chunk_size{};
boost::dynamic_bitset<> read_state;
restore_resume_entry(json::parse(iterator->value().ToString()), api_path,
chunk_size, read_state, source_path);
filesystem_item fsi{};
auto res = provider_.get_filesystem_item(api_path, false, fsi);
if (res == api_error::success) {
if (source_path == fsi.source_path) {
std::uint64_t file_size{};
if (utils::file::get_file_size(fsi.source_path, file_size)) {
if (file_size == fsi.size) {
auto f = std::make_shared<open_file>(
chunk_size,
config_.get_enable_chunk_download_timeout()
? config_.get_chunk_downloader_timeout_secs()
: 0U,
fsi, provider_, read_state, *this);
open_file_lookup_[api_path] = f;
event_system::instance().raise<download_restored>(
fsi.api_path, fsi.source_path);
} else {
event_system::instance().raise<download_restore_failed>(
fsi.api_path, fsi.source_path,
"file size mismatch|expected|" + std::to_string(fsi.size) +
"|actual|" + std::to_string(file_size));
}
} else {
event_system::instance().raise<download_restore_failed>(
fsi.api_path, fsi.source_path,
"failed to get file size: " +
std::to_string(utils::get_last_error_code()));
}
} else {
event_system::instance().raise<download_restore_failed>(
fsi.api_path, fsi.source_path,
"source path mismatch|expected|" + source_path + "|actual|" +
fsi.source_path);
}
} else {
event_system::instance().raise<download_restore_failed>(
api_path, source_path,
"failed to get filesystem item|" + api_error_to_string(res));
}
}
upload_thread_ =
std::make_unique<std::thread>([this] { upload_handler(); });
polling::instance().set_callback(
{"timed_out_close", polling::frequency::second,
[this]() { this->close_timed_out_files(); }});
event_system::instance().raise<service_started>("file_manager");
}
}
void file_manager::stop() {
if (not stop_requested_) {
event_system::instance().raise<service_shutdown_begin>("file_manager");
polling::instance().remove_callback("timed_out_close");
stop_requested_ = true;
unique_mutex_lock upload_lock(upload_mtx_);
upload_notify_.notify_all();
upload_lock.unlock();
if (upload_thread_) {
upload_thread_->join();
upload_thread_.reset();
}
open_file_lookup_.clear();
open_handle_lookup_.clear();
upload_lock.lock();
for (auto &kv : upload_lookup_) {
kv.second->stop();
}
upload_notify_.notify_all();
upload_lock.unlock();
while (not upload_lookup_.empty()) {
upload_lock.lock();
if (not upload_lookup_.empty()) {
upload_notify_.wait_for(upload_lock, 1s);
}
upload_notify_.notify_all();
upload_lock.unlock();
}
event_system::instance().raise<service_shutdown_end>("file_manager");
}
}
void file_manager::store_resume(const i_open_file &o) {
if (provider_.is_direct_only()) {
return;
}
recur_mutex_lock open_lock(open_file_mtx_);
const auto res = db_->Put(rocksdb::WriteOptions(), default_family_,
o.get_api_path(), create_resume_entry(o).dump());
if (res.ok()) {
event_system::instance().raise<download_stored>(o.get_api_path(),
o.get_source_path());
} else {
event_system::instance().raise<download_stored_failed>(
o.get_api_path(), o.get_source_path(), res.ToString());
}
}
void file_manager::upload_completed(const file_upload_completed &e) {
unique_mutex_lock upload_lock(upload_mtx_);
if (not utils::string::to_bool(e.get_cancelled().get<std::string>())) {
const auto err = api_error_from_string(e.get_result().get<std::string>());
switch (err) {
case api_error::success: {
auto iterator = std::unique_ptr<rocksdb::Iterator>(
db_->NewIterator(rocksdb::ReadOptions(), upload_active_family_));
for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
const auto parts =
utils::string::split(iterator->key().ToString(), ':');
if (parts[1U] == e.get_api_path().get<std::string>()) {
db_->Delete(rocksdb::WriteOptions(), upload_active_family_,
iterator->key());
break;
}
}
} break;
case api_error::upload_stopped: {
event_system::instance().raise<file_upload_retry>(e.get_api_path(),
e.get_source(), err);
queue_upload(e.get_api_path(), e.get_source(), true);
upload_notify_.wait_for(upload_lock, 5s);
} break;
default: {
bool exists{};
auto res = provider_.is_file(e.get_api_path(), exists);
if ((res == api_error::success && not exists) ||
not utils::file::is_file(e.get_source())) {
event_system::instance().raise<file_upload_not_found>(e.get_api_path(),
e.get_source());
remove_upload(e.get_api_path(), true);
return;
}
event_system::instance().raise<file_upload_retry>(e.get_api_path(),
e.get_source(), err);
queue_upload(e.get_api_path(), e.get_source(), true);
upload_notify_.wait_for(upload_lock, 5s);
} break;
}
upload_lookup_.erase(e.get_api_path());
}
upload_notify_.notify_all();
}
void file_manager::upload_handler() {
while (not stop_requested_) {
unique_mutex_lock upload_lock(upload_mtx_);
if (not stop_requested_) {
if (upload_lookup_.size() < config_.get_max_upload_count()) {
auto iterator = std::unique_ptr<rocksdb::Iterator>(
db_->NewIterator(rocksdb::ReadOptions(), upload_family_));
iterator->SeekToFirst();
if (iterator->Valid()) {
const auto parts =
utils::string::split(iterator->key().ToString(), ':');
const auto api_path = parts[1u];
const auto source_path = iterator->value().ToString();
filesystem_item fsi{};
auto res = provider_.get_filesystem_item(api_path, false, fsi);
switch (res) {
case api_error::item_not_found: {
event_system::instance().raise<file_upload_not_found>(api_path,
source_path);
remove_upload(parts[1u], true);
} break;
case api_error::success: {
upload_lookup_[fsi.api_path] =
std::make_unique<upload>(fsi, provider_);
if (db_->Delete(rocksdb::WriteOptions(), upload_family_,
iterator->key())
.ok()) {
db_->Put(rocksdb::WriteOptions(), upload_active_family_,
iterator->key(), iterator->value());
}
} break;
default: {
event_system::instance().raise<file_upload_retry>(api_path,
source_path, res);
queue_upload(api_path, source_path, true);
upload_notify_.wait_for(upload_lock, 5s);
} break;
}
} else {
iterator.release();
upload_notify_.wait(upload_lock);
}
} else {
upload_notify_.wait(upload_lock);
}
}
upload_notify_.notify_all();
}
}
void file_manager::update_used_space(std::uint64_t &used_space) const {
recur_mutex_lock open_lock(open_file_mtx_);
for (const auto &of : open_file_lookup_) {
std::uint64_t file_size{};
auto res = provider_.get_file_size(of.second->get_api_path(), file_size);
if ((res == api_error::success) &&
(file_size != of.second->get_file_size()) &&
(used_space >= file_size)) {
used_space -= file_size;
used_space += of.second->get_file_size();
}
}
}
} // namespace repertory

596
src/file_manager/file_manager_open_file.cpp Normal file
View File

@@ -0,0 +1,596 @@
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "file_manager/file_manager.hpp"
#include "file_manager/events.hpp"
#include "providers/i_provider.hpp"
#include "types/repertory.hpp"
#include "types/startup_exception.hpp"
#include "utils/error_utils.hpp"
#include "utils/file_utils.hpp"
#include "utils/path_utils.hpp"
#include "utils/unix/unix_utils.hpp"
#include "utils/utils.hpp"
namespace repertory {
file_manager::open_file::open_file(std::uint64_t chunk_size,
std::uint8_t chunk_timeout,
filesystem_item fsi, i_provider &provider,
i_upload_manager &um)
: open_file(chunk_size, chunk_timeout, fsi, {}, provider, std::nullopt,
um) {}
file_manager::open_file::open_file(
std::uint64_t chunk_size, std::uint8_t chunk_timeout, filesystem_item fsi,
std::map<std::uint64_t, open_file_data> open_data, i_provider &provider,
i_upload_manager &um)
: open_file(chunk_size, chunk_timeout, fsi, open_data, provider,
std::nullopt, um) {}
file_manager::open_file::open_file(
std::uint64_t chunk_size, std::uint8_t chunk_timeout, filesystem_item fsi,
i_provider &provider, std::optional<boost::dynamic_bitset<>> read_state,
i_upload_manager &um)
: open_file(chunk_size, chunk_timeout, fsi, {}, provider, read_state, um) {}
file_manager::open_file::open_file(
std::uint64_t chunk_size, std::uint8_t chunk_timeout, filesystem_item fsi,
std::map<std::uint64_t, open_file_data> open_data, i_provider &provider,
std::optional<boost::dynamic_bitset<>> read_state, i_upload_manager &um)
: open_file_base(chunk_size, chunk_timeout, fsi, open_data, provider),
um_(um) {
if (fsi_.directory && read_state.has_value()) {
throw startup_exception("cannot resume a directory|" + fsi.api_path);
}
if (not fsi.directory) {
set_api_error(native_file::create_or_open(
fsi.source_path, not provider_.is_direct_only(), nf_));
if (get_api_error() == api_error::success) {
if (read_state.has_value()) {
modified_ = true;
um_.store_resume(*this);
read_state_ = read_state.value();
} else if (fsi_.size > 0u) {
read_state_.resize(static_cast<std::size_t>(utils::divide_with_ceiling(
fsi_.size, chunk_size)),
false);
std::uint64_t file_size{};
if (nf_->get_file_size(file_size)) {
if (provider_.is_direct_only() || file_size == fsi.size) {
read_state_.set(0u, read_state_.size(), true);
} else if (not nf_->truncate(fsi.size)) {
set_api_error(api_error::os_error);
}
} else {
set_api_error(api_error::os_error);
}
}
if (get_api_error() != api_error::success && nf_) {
nf_->close();
}
}
}
}
file_manager::open_file::~open_file() { close(); }
void file_manager::open_file::download_chunk(std::size_t chunk,
bool skip_active,
bool should_reset) {
if (should_reset) {
reset_timeout();
}
unique_recur_mutex_lock file_lock(file_mtx_);
if ((get_api_error() == api_error::success) && (chunk < read_state_.size()) &&
not read_state_[chunk]) {
if (active_downloads_.find(chunk) != active_downloads_.end()) {
if (not skip_active) {
auto active_download = active_downloads_.at(chunk);
file_lock.unlock();
active_download->wait();
}
return;
}
const auto data_offset = chunk * chunk_size_;
const auto data_size =
(chunk == read_state_.size() - 1u) ? last_chunk_size_ : chunk_size_;
if (active_downloads_.empty() && (read_state_.count() == 0u)) {
event_system::instance().raise<download_begin>(fsi_.api_path,
fsi_.source_path);
}
event_system::instance().raise<download_chunk_begin>(
fsi_.api_path, fsi_.source_path, chunk, read_state_.size(),
read_state_.count());
active_downloads_[chunk] = std::make_shared<download>();
file_lock.unlock();
if (should_reset) {
reset_timeout();
}
std::async(std::launch::async, [this, chunk, data_size, data_offset,
should_reset]() {
const auto notify_complete = [this, chunk, should_reset]() {
unique_recur_mutex_lock file_lock(file_mtx_);
auto active_download = active_downloads_.at(chunk);
active_downloads_.erase(chunk);
event_system::instance().raise<download_chunk_end>(
fsi_.api_path, fsi_.source_path, chunk, read_state_.size(),
read_state_.count(), get_api_error());
if (get_api_error() == api_error::success) {
auto progress = (static_cast<double>(read_state_.count()) /
static_cast<double>(read_state_.size()) * 100.0);
event_system::instance().raise<download_progress>(
fsi_.api_path, fsi_.source_path, progress);
if (read_state_.all() && not notified_) {
notified_ = true;
event_system::instance().raise<download_end>(
fsi_.api_path, fsi_.source_path, get_api_error());
}
} else if (not notified_) {
notified_ = true;
event_system::instance().raise<download_end>(
fsi_.api_path, fsi_.source_path, get_api_error());
}
file_lock.unlock();
active_download->notify(get_api_error());
if (should_reset) {
reset_timeout();
}
};
data_buffer data;
auto res = provider_.read_file_bytes(get_api_path(), data_size,
data_offset, data, stop_requested_);
if (res != api_error::success) {
set_api_error(res);
notify_complete();
return;
}
if (should_reset) {
reset_timeout();
}
res = do_io([&]() -> api_error {
std::size_t bytes_written{};
if (not nf_->write_bytes(data.data(), data.size(), data_offset,
bytes_written)) {
return api_error::os_error;
}
if (should_reset) {
reset_timeout();
}
return api_error::success;
});
if (res != api_error::success) {
set_api_error(res);
notify_complete();
return;
}
unique_recur_mutex_lock file_lock(file_mtx_);
read_state_.set(chunk);
file_lock.unlock();
notify_complete();
}).wait();
}
}
void file_manager::open_file::download_range(
std::size_t start_chunk_index, std::size_t end_chunk_index_inclusive,
bool should_reset) {
for (std::size_t chunk = start_chunk_index;
chunk <= end_chunk_index_inclusive; chunk++) {
download_chunk(chunk, false, should_reset);
if (get_api_error() != api_error::success) {
return;
}
}
}
auto file_manager::open_file::get_read_state() const
-> boost::dynamic_bitset<> {
recur_mutex_lock file_lock(file_mtx_);
return read_state_;
}
auto file_manager::open_file::get_read_state(std::size_t chunk) const -> bool {
recur_mutex_lock file_lock(file_mtx_);
return read_state_[chunk];
}
auto file_manager::open_file::is_complete() const -> bool {
recur_mutex_lock file_lock(file_mtx_);
return read_state_.all();
}
auto file_manager::open_file::native_operation(
const i_open_file::native_operation_callback &operation) -> api_error {
unique_recur_mutex_lock file_lock(file_mtx_);
if (stop_requested_) {
return api_error::download_stopped;
}
file_lock.unlock();
return do_io([&]() -> api_error { return operation(nf_->get_handle()); });
}
auto file_manager::open_file::native_operation(
std::uint64_t new_file_size,
const i_open_file::native_operation_callback &operation) -> api_error {
if (fsi_.directory) {
return api_error::invalid_operation;
}
unique_recur_mutex_lock file_lock(file_mtx_);
if (stop_requested_) {
return api_error::download_stopped;
}
file_lock.unlock();
const auto is_empty_file = new_file_size == 0u;
const auto last_chunk =
is_empty_file ? std::size_t(0u)
: static_cast<std::size_t>(utils::divide_with_ceiling(
new_file_size, chunk_size_)) -
1u;
file_lock.lock();
if (not is_empty_file && (last_chunk < read_state_.size())) {
file_lock.unlock();
update_background_reader(0u);
download_chunk(last_chunk, false, true);
if (get_api_error() != api_error::success) {
return get_api_error();
}
file_lock.lock();
}
const auto original_file_size = get_file_size();
auto res = do_io([&]() -> api_error { return operation(nf_->get_handle()); });
if (res != api_error::success) {
utils::error::raise_api_path_error(__FUNCTION__, get_api_path(),
utils::get_last_error_code(),
"failed to allocate file");
return res;
}
{
std::uint64_t file_size{};
if (not nf_->get_file_size(file_size)) {
utils::error::raise_api_path_error(__FUNCTION__, get_api_path(),
utils::get_last_error_code(),
"failed to get file size");
return set_api_error(api_error::os_error);
}
if (file_size != new_file_size) {
utils::error::raise_api_path_error(
__FUNCTION__, get_api_path(), api_error::file_size_mismatch,
"allocated file size mismatch|expected|" +
std::to_string(new_file_size) + "|actual|" +
std::to_string(file_size));
return set_api_error(api_error::error);
}
}
if (is_empty_file || (read_state_.size() != (last_chunk + 1u))) {
read_state_.resize(is_empty_file ? 0u : last_chunk + 1u);
if (not is_empty_file) {
read_state_[last_chunk] = true;
}
last_chunk_size_ = static_cast<std::size_t>(
new_file_size <= chunk_size_ ? new_file_size
: new_file_size % chunk_size_ ? new_file_size % chunk_size_
: chunk_size_);
}
if (original_file_size != new_file_size) {
if (not modified_) {
um_.store_resume(*this);
}
modified_ = true;
um_.remove_upload(get_api_path());
fsi_.size = new_file_size;
const auto now = std::to_string(utils::get_file_time_now());
res = provider_.set_item_meta(
fsi_.api_path, {
{META_CHANGED, now},
{META_MODIFIED, now},
{META_SIZE, std::to_string(new_file_size)},
{META_WRITTEN, now},
});
if (res != api_error::success) {
utils::error::raise_api_path_error(__FUNCTION__, get_api_path(), res,
"failed to set file meta");
return set_api_error(res);
}
}
return res;
}
auto file_manager::open_file::read(std::size_t read_size,
std::uint64_t read_offset, data_buffer &data)
-> api_error {
if (fsi_.directory) {
return api_error::invalid_operation;
}
read_size =
utils::calculate_read_size(get_file_size(), read_size, read_offset);
if (read_size == 0u) {
return api_error::success;
}
const auto read_from_source = [this, &data, &read_offset,
&read_size]() -> api_error {
return do_io([this, &data, &read_offset, &read_size]() -> api_error {
if (provider_.is_direct_only()) {
return provider_.read_file_bytes(fsi_.api_path, read_size, read_offset,
data, stop_requested_);
}
data.resize(read_size);
std::size_t bytes_read{};
return nf_->read_bytes(data.data(), read_size, read_offset, bytes_read)
? api_error::success
: api_error::os_error;
});
};
unique_recur_mutex_lock file_lock(file_mtx_);
if (read_state_.all()) {
reset_timeout();
return read_from_source();
}
file_lock.unlock();
const auto start_chunk_index =
static_cast<std::size_t>(read_offset / chunk_size_);
const auto end_chunk_index =
static_cast<std::size_t>((read_size + read_offset) / chunk_size_);
update_background_reader(start_chunk_index);
download_range(start_chunk_index, end_chunk_index, true);
if (get_api_error() != api_error::success) {
return get_api_error();
}
file_lock.lock();
return get_api_error() == api_error::success ? read_from_source()
: get_api_error();
}
void file_manager::open_file::remove(std::uint64_t handle) {
recur_mutex_lock file_lock(file_mtx_);
open_file_base::remove(handle);
if (modified_ && read_state_.all() &&
(get_api_error() == api_error::success)) {
um_.queue_upload(*this);
modified_ = false;
}
}
auto file_manager::open_file::resize(std::uint64_t new_file_size) -> api_error {
if (fsi_.directory) {
return api_error::invalid_operation;
}
return native_operation(
new_file_size, [this, &new_file_size](native_handle) -> api_error {
return nf_->truncate(new_file_size) ? api_error::success
: api_error::os_error;
});
}
auto file_manager::open_file::close() -> bool {
if (not fsi_.directory && not stop_requested_) {
stop_requested_ = true;
unique_mutex_lock reader_lock(io_thread_mtx_);
io_thread_notify_.notify_all();
reader_lock.unlock();
if (reader_thread_) {
reader_thread_->join();
reader_thread_.reset();
}
if (open_file_base::close()) {
{
const auto err = get_api_error();
if (err == api_error::success ||
err == api_error::download_incomplete ||
err == api_error::download_stopped) {
if (modified_ && not read_state_.all()) {
set_api_error(api_error::download_incomplete);
} else if (not modified_ && (fsi_.size > 0u) &&
not read_state_.all()) {
set_api_error(api_error::download_stopped);
}
}
}
nf_->close();
nf_.reset();
if (modified_ && (get_api_error() == api_error::success)) {
um_.queue_upload(*this);
} else if (modified_ &&
(get_api_error() == api_error::download_incomplete)) {
um_.store_resume(*this);
} else if (get_api_error() != api_error::success) {
um_.remove_resume(get_api_path(), get_source_path());
if (not utils::file::retry_delete_file(fsi_.source_path)) {
utils::error::raise_api_path_error(
__FUNCTION__, get_api_path(), fsi_.source_path,
utils::get_last_error_code(), "failed to delete file");
}
auto parent = utils::path::remove_file_name(fsi_.source_path);
fsi_.source_path =
utils::path::combine(parent, {utils::create_uuid_string()});
const auto res = provider_.set_item_meta(fsi_.api_path, META_SOURCE,
fsi_.source_path);
if (res != api_error::success) {
utils::error::raise_api_path_error(__FUNCTION__, get_api_path(),
fsi_.source_path, res,
"failed to set file meta");
}
}
}
return true;
}
return false;
}
void file_manager::open_file::update_background_reader(std::size_t read_chunk) {
recur_mutex_lock file_lock(file_mtx_);
read_chunk_index_ = read_chunk;
if (not reader_thread_ && not stop_requested_) {
reader_thread_ = std::make_unique<std::thread>([this]() {
auto next_chunk = 0u;
while (not stop_requested_) {
unique_recur_mutex_lock file_lock(file_mtx_);
if ((fsi_.size == 0u) || read_state_.all()) {
file_lock.unlock();
unique_mutex_lock io_lock(io_thread_mtx_);
if (not stop_requested_ && io_thread_queue_.empty()) {
io_thread_notify_.wait(io_lock);
}
io_thread_notify_.notify_all();
io_lock.unlock();
} else {
do {
next_chunk = read_chunk_index_ =
((read_chunk_index_ + 1u) >= read_state_.size())
? 0u
: read_chunk_index_ + 1u;
} while ((next_chunk != 0u) && (active_downloads_.find(next_chunk) !=
active_downloads_.end()));
file_lock.unlock();
download_chunk(next_chunk, true, false);
}
}
});
}
}
auto file_manager::open_file::write(std::uint64_t write_offset,
const data_buffer &data,
std::size_t &bytes_written) -> api_error {
bytes_written = 0u;
if (fsi_.directory || provider_.is_direct_only()) {
return api_error::invalid_operation;
}
if (data.empty()) {
return api_error::success;
}
unique_recur_mutex_lock file_lock(file_mtx_);
if (stop_requested_) {
return api_error::download_stopped;
}
file_lock.unlock();
const auto start_chunk_index =
static_cast<std::size_t>(write_offset / chunk_size_);
const auto end_chunk_index =
static_cast<std::size_t>((write_offset + data.size()) / chunk_size_);
update_background_reader(start_chunk_index);
download_range(start_chunk_index,
std::min(read_state_.size() - 1u, end_chunk_index), true);
if (get_api_error() != api_error::success) {
return get_api_error();
}
file_lock.lock();
if ((write_offset + data.size()) > fsi_.size) {
auto res = resize(write_offset + data.size());
if (res != api_error::success) {
return res;
}
}
auto res = do_io([&]() -> api_error {
if (not nf_->write_bytes(data.data(), data.size(), write_offset,
bytes_written)) {
return api_error::os_error;
}
reset_timeout();
return api_error::success;
});
if (res != api_error::success) {
return set_api_error(res);
}
const auto now = std::to_string(utils::get_file_time_now());
res = provider_.set_item_meta(fsi_.api_path, {
{META_CHANGED, now},
{META_MODIFIED, now},
{META_WRITTEN, now},
});
if (res != api_error::success) {
utils::error::raise_api_path_error(__FUNCTION__, get_api_path(), res,
"failed to set file meta");
return set_api_error(res);
}
if (not modified_) {
um_.store_resume(*this);
}
modified_ = true;
um_.remove_upload(get_api_path());
return api_error::success;
}
} // namespace repertory

242
src/file_manager/file_manager_open_file_base.cpp Normal file
View File

@@ -0,0 +1,242 @@
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "file_manager/file_manager.hpp"
#include "utils/path_utils.hpp"
namespace repertory {
file_manager::open_file_base::open_file_base(std::uint64_t chunk_size,
std::uint8_t chunk_timeout,
filesystem_item fsi,
i_provider &provider)
: open_file_base(chunk_size, chunk_timeout, fsi, {}, provider) {}
file_manager::open_file_base::open_file_base(
std::uint64_t chunk_size, std::uint8_t chunk_timeout, filesystem_item fsi,
std::map<std::uint64_t, open_file_data> open_data, i_provider &provider)
: chunk_size_(chunk_size),
chunk_timeout_(chunk_timeout),
fsi_(std::move(fsi)),
last_chunk_size_(static_cast<std::size_t>(
fsi.size <= chunk_size ? fsi.size
: fsi.size % chunk_size ? fsi.size % chunk_size
: chunk_size)),
open_data_(std::move(open_data)),
provider_(provider) {
if (not fsi.directory) {
io_thread_ = std::make_unique<std::thread>([this] { file_io_thread(); });
}
}
void file_manager::open_file_base::add(std::uint64_t handle,
open_file_data ofd) {
recur_mutex_lock file_lock(file_mtx_);
open_data_[handle] = ofd;
if (open_data_.size() == 1u) {
event_system::instance().raise<filesystem_item_opened>(
fsi_.api_path, fsi_.source_path, fsi_.directory);
}
event_system::instance().raise<filesystem_item_handle_opened>(
fsi_.api_path, handle, fsi_.source_path, fsi_.directory);
}
auto file_manager::open_file_base::can_close() const -> bool {
recur_mutex_lock file_lock(file_mtx_);
if (fsi_.directory) {
return true;
}
if (not open_data_.empty()) {
return false;
}
if (modified_) {
return false;
}
if (get_api_error() != api_error::success) {
return true;
}
if (is_download_complete()) {
return true;
}
const std::chrono::system_clock::time_point last_access = last_access_;
const auto duration = std::chrono::duration_cast<std::chrono::seconds>(
std::chrono::system_clock::now() - last_access);
return (duration.count() >= chunk_timeout_);
}
auto file_manager::open_file_base::do_io(std::function<api_error()> action)
-> api_error {
unique_mutex_lock io_lock(io_thread_mtx_);
auto item = std::make_shared<io_item>(action);
io_thread_queue_.emplace_back(item);
io_thread_notify_.notify_all();
io_lock.unlock();
return item->get_result();
}
void file_manager::open_file_base::file_io_thread() {
unique_mutex_lock io_lock(io_thread_mtx_);
io_thread_notify_.notify_all();
io_lock.unlock();
const auto process_queue = [&]() {
io_lock.lock();
if (not io_stop_requested_ && io_thread_queue_.empty()) {
io_thread_notify_.wait(io_lock);
}
while (not io_thread_queue_.empty()) {
auto *item = io_thread_queue_.front().get();
io_thread_notify_.notify_all();
io_lock.unlock();
item->action();
io_lock.lock();
io_thread_queue_.pop_front();
}
io_thread_notify_.notify_all();
io_lock.unlock();
};
while (not io_stop_requested_) {
process_queue();
}
process_queue();
}
auto file_manager::open_file_base::get_api_error() const -> api_error {
mutex_lock error_lock(error_mtx_);
return error_;
}
auto file_manager::open_file_base::get_api_path() const -> std::string {
recur_mutex_lock file_lock(file_mtx_);
return fsi_.api_path;
}
auto file_manager::open_file_base::get_file_size() const -> std::uint64_t {
recur_mutex_lock file_lock(file_mtx_);
return fsi_.size;
}
auto file_manager::open_file_base::get_filesystem_item() const
-> filesystem_item {
recur_mutex_lock file_lock(file_mtx_);
return fsi_;
}
auto file_manager::open_file_base::get_handles() const
-> std::vector<std::uint64_t> {
recur_mutex_lock file_lock(file_mtx_);
std::vector<std::uint64_t> ret;
for (const auto &kv : open_data_) {
ret.emplace_back(kv.first);
}
return ret;
}
auto file_manager::open_file_base::get_open_data() const
-> std::map<std::uint64_t, open_file_data> {
recur_mutex_lock file_lock(file_mtx_);
return open_data_;
}
auto file_manager::open_file_base::get_open_data(std::uint64_t handle) const
-> open_file_data {
recur_mutex_lock file_lock(file_mtx_);
return open_data_.at(handle);
}
auto file_manager::open_file_base::get_open_file_count() const -> std::size_t {
recur_mutex_lock file_lock(file_mtx_);
return open_data_.size();
}
auto file_manager::open_file_base::is_modified() const -> bool {
recur_mutex_lock file_lock(file_mtx_);
return modified_;
}
void file_manager::open_file_base::remove(std::uint64_t handle) {
recur_mutex_lock file_lock(file_mtx_);
open_data_.erase(handle);
event_system::instance().raise<filesystem_item_handle_closed>(
fsi_.api_path, handle, fsi_.source_path, fsi_.directory, modified_);
if (open_data_.empty()) {
event_system::instance().raise<filesystem_item_closed>(
fsi_.api_path, fsi_.source_path, fsi_.directory, modified_);
}
}
void file_manager::open_file_base::reset_timeout() {
last_access_ = std::chrono::system_clock::now();
}
auto file_manager::open_file_base::set_api_error(const api_error &e)
-> api_error {
mutex_lock error_lock(error_mtx_);
if (error_ != e) {
return ((error_ = (error_ == api_error::success ||
error_ == api_error::download_incomplete ||
error_ == api_error::download_stopped
? e
: error_)));
}
return error_;
}
void file_manager::open_file_base::set_api_path(const std::string &api_path) {
recur_mutex_lock file_lock(file_mtx_);
fsi_.api_path = api_path;
fsi_.api_parent = utils::path::get_parent_api_path(api_path);
}
auto file_manager::open_file_base::close() -> bool {
unique_mutex_lock io_lock(io_thread_mtx_);
if (not fsi_.directory && not io_stop_requested_) {
io_stop_requested_ = true;
io_thread_notify_.notify_all();
io_lock.unlock();
if (io_thread_) {
io_thread_->join();
io_thread_.reset();
return true;
}
}
io_thread_notify_.notify_all();
io_lock.unlock();
return false;
}
} // namespace repertory

43
src/file_manager/file_manager_open_file_base_download.cpp Normal file
View File

@@ -0,0 +1,43 @@
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "file_manager/file_manager.hpp"
namespace repertory {
void file_manager::open_file_base::download::notify(const api_error &e) {
complete_ = true;
error_ = e;
unique_mutex_lock lock(mtx_);
notify_.notify_all();
}
auto file_manager::open_file_base::download::wait() -> api_error {
if (not complete_) {
unique_mutex_lock lock(mtx_);
if (not complete_) {
notify_.wait(lock);
}
notify_.notify_all();
}
return error_;
}
} // namespace repertory

41
src/file_manager/file_manager_open_file_base_io_item.cpp Normal file
View File

@@ -0,0 +1,41 @@
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "file_manager/file_manager.hpp"
namespace repertory {
void file_manager::open_file_base::io_item::action() {
result_ = action_();
mutex_lock lock(mtx_);
notify_.notify_all();
}
auto file_manager::open_file_base::io_item::get_result() -> api_error {
unique_mutex_lock lock(mtx_);
if (result_.has_value()) {
return result_.value();
}
notify_.wait(lock);
return result_.value();
}
} // namespace repertory

317
src/file_manager/file_manager_ring_buffer_open_file.cpp Normal file
View File

@@ -0,0 +1,317 @@
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "file_manager/file_manager.hpp"
#include "app_config.hpp"
#include "file_manager/events.hpp"
#include "providers/i_provider.hpp"
#include "types/repertory.hpp"
#include "utils/encrypting_reader.hpp"
#include "utils/file_utils.hpp"
#include "utils/path_utils.hpp"
#include "utils/unix/unix_utils.hpp"
#include "utils/utils.hpp"
namespace repertory {
file_manager::ring_buffer_open_file::ring_buffer_open_file(
std::string buffer_directory, std::uint64_t chunk_size,
std::uint8_t chunk_timeout, filesystem_item fsi, i_provider &provider)
: ring_buffer_open_file(std::move(buffer_directory), chunk_size,
chunk_timeout, std::move(fsi), provider,
(1024ull * 1024ull * 1024ull) / chunk_size) {}
file_manager::ring_buffer_open_file::ring_buffer_open_file(
std::string buffer_directory, std::uint64_t chunk_size,
std::uint8_t chunk_timeout, filesystem_item fsi, i_provider &provider,
std::size_t ring_size)
: open_file_base(chunk_size, chunk_timeout, fsi, provider),
ring_state_(ring_size),
total_chunks_(static_cast<std::size_t>(
utils::divide_with_ceiling(fsi.size, chunk_size_))) {
if (ring_size % 2u) {
throw std::runtime_error("ring size must be a multiple of 2");
}
if (ring_size < 4u) {
throw std::runtime_error("ring size must be greater than or equal to 4");
}
if (fsi.size < (ring_state_.size() * chunk_size)) {
throw std::runtime_error("file size is less than ring buffer size");
}
last_chunk_ = ring_state_.size() - 1u;
ring_state_.set(0u, ring_state_.size(), true);
buffer_directory = utils::path::absolute(buffer_directory);
if (not utils::file::create_full_directory_path(buffer_directory)) {
throw std::runtime_error("failed to create buffer directory|path|" +
buffer_directory + "|err|" +
std::to_string(utils::get_last_error_code()));
}
fsi_.source_path =
utils::path::combine(buffer_directory, {utils::create_uuid_string()});
auto res = native_file::create_or_open(fsi_.source_path, nf_);
if (res != api_error::success) {
throw std::runtime_error("failed to create buffer file|err|" +
std::to_string(utils::get_last_error_code()));
}
if (not nf_->truncate(ring_state_.size() * chunk_size)) {
nf_->close();
throw std::runtime_error("failed to resize buffer file|err|" +
std::to_string(utils::get_last_error_code()));
}
}
file_manager::ring_buffer_open_file::~ring_buffer_open_file() {
close();
nf_->close();
if (not utils::file::retry_delete_file(fsi_.source_path)) {
utils::error::raise_api_path_error(
__FUNCTION__, fsi_.api_path, fsi_.source_path,
utils::get_last_error_code(), "failed to delete file");
}
}
auto file_manager::file_manager::ring_buffer_open_file::download_chunk(
std::size_t chunk) -> api_error {
unique_mutex_lock chunk_lock(chunk_mtx_);
if (active_downloads_.find(chunk) != active_downloads_.end()) {
auto active_download = active_downloads_.at(chunk);
chunk_notify_.notify_all();
chunk_lock.unlock();
return active_download->wait();
}
if (ring_state_[chunk % ring_state_.size()]) {
auto active_download = std::make_shared<download>();
active_downloads_[chunk] = active_download;
ring_state_[chunk % ring_state_.size()] = false;
chunk_notify_.notify_all();
chunk_lock.unlock();
data_buffer buffer((chunk == (total_chunks_ - 1u)) ? last_chunk_size_
: chunk_size_);
stop_type stop_requested = !!ring_state_[chunk % ring_state_.size()];
auto res =
provider_.read_file_bytes(fsi_.api_path, buffer.size(),
chunk * chunk_size_, buffer, stop_requested);
if (res == api_error::success) {
res = do_io([&]() -> api_error {
std::size_t bytes_written{};
if (not nf_->write_bytes(buffer.data(), buffer.size(),
(chunk % ring_state_.size()) * chunk_size_,
bytes_written)) {
return api_error::os_error;
}
return api_error::success;
});
}
active_download->notify(res);
chunk_lock.lock();
active_downloads_.erase(chunk);
chunk_notify_.notify_all();
return res;
}
chunk_notify_.notify_all();
chunk_lock.unlock();
return api_error::success;
}
void file_manager::ring_buffer_open_file::forward(std::size_t count) {
mutex_lock chunk_lock(chunk_mtx_);
if ((current_chunk_ + count) > (total_chunks_ - 1u)) {
count = (total_chunks_ - 1u) - current_chunk_;
}
if ((current_chunk_ + count) <= last_chunk_) {
current_chunk_ += count;
} else {
const auto added = count - (last_chunk_ - current_chunk_);
if (added >= ring_state_.size()) {
ring_state_.set(0u, ring_state_.size(), true);
current_chunk_ += count;
first_chunk_ += added;
last_chunk_ =
std::min(total_chunks_ - 1u, first_chunk_ + ring_state_.size() - 1u);
} else {
for (std::size_t i = 0u; i < added; i++) {
ring_state_[(first_chunk_ + i) % ring_state_.size()] = true;
}
first_chunk_ += added;
current_chunk_ += count;
last_chunk_ =
std::min(total_chunks_ - 1u, first_chunk_ + ring_state_.size() - 1u);
}
}
chunk_notify_.notify_all();
}
auto file_manager::ring_buffer_open_file::get_read_state() const
-> boost::dynamic_bitset<> {
recur_mutex_lock file_lock(file_mtx_);
auto read_state = ring_state_;
return read_state.flip();
}
auto file_manager::ring_buffer_open_file::get_read_state(
std::size_t chunk) const -> bool {
recur_mutex_lock file_lock(file_mtx_);
return not ring_state_[chunk % ring_state_.size()];
}
auto file_manager::ring_buffer_open_file::is_download_complete() const -> bool {
return false;
}
auto file_manager::ring_buffer_open_file::native_operation(
const i_open_file::native_operation_callback &operation) -> api_error {
return do_io([&]() -> api_error { return operation(nf_->get_handle()); });
}
void file_manager::ring_buffer_open_file::reverse(std::size_t count) {
mutex_lock chunk_lock(chunk_mtx_);
if (current_chunk_ < count) {
count = current_chunk_;
}
if ((current_chunk_ - count) >= first_chunk_) {
current_chunk_ -= count;
} else {
const auto removed = count - (current_chunk_ - first_chunk_);
if (removed >= ring_state_.size()) {
ring_state_.set(0u, ring_state_.size(), true);
current_chunk_ -= count;
first_chunk_ = current_chunk_;
last_chunk_ =
std::min(total_chunks_ - 1u, first_chunk_ + ring_state_.size() - 1u);
} else {
for (std::size_t i = 0u; i < removed; i++) {
ring_state_[(last_chunk_ - i) % ring_state_.size()] = true;
}
first_chunk_ -= removed;
current_chunk_ -= count;
last_chunk_ =
std::min(total_chunks_ - 1u, first_chunk_ + ring_state_.size() - 1u);
}
}
chunk_notify_.notify_all();
}
auto file_manager::ring_buffer_open_file::read(std::size_t read_size,
std::uint64_t read_offset,
data_buffer &data) -> api_error {
if (fsi_.directory) {
return api_error::invalid_operation;
}
reset_timeout();
read_size = utils::calculate_read_size(fsi_.size, read_size, read_offset);
if (read_size == 0u) {
return api_error::success;
}
const auto start_chunk_index =
static_cast<std::size_t>(read_offset / chunk_size_);
read_offset = read_offset - (start_chunk_index * chunk_size_);
data_buffer buffer(chunk_size_);
auto res = api_error::success;
for (std::size_t chunk = start_chunk_index;
(res == api_error::success) && (read_size > 0u); chunk++) {
if (chunk > current_chunk_) {
forward(chunk - current_chunk_);
} else if (chunk < current_chunk_) {
reverse(current_chunk_ - chunk);
}
reset_timeout();
if ((res = download_chunk(chunk)) == api_error::success) {
const auto to_read = std::min(
static_cast<std::size_t>(chunk_size_ - read_offset), read_size);
res = do_io([this, &buffer, &chunk, &data, read_offset,
&to_read]() -> api_error {
std::size_t bytes_read{};
auto res = nf_->read_bytes(buffer.data(), buffer.size(),
((chunk % ring_state_.size()) * chunk_size_),
bytes_read)
? api_error::success
: api_error::os_error;
if (res == api_error::success) {
data.insert(data.end(), buffer.begin() + read_offset,
buffer.begin() + read_offset + to_read);
reset_timeout();
}
return res;
});
read_offset = 0u;
read_size -= to_read;
}
}
return res;
}
void file_manager::ring_buffer_open_file::set(std::size_t first_chunk,
std::size_t current_chunk) {
mutex_lock chunk_lock(chunk_mtx_);
if (first_chunk >= total_chunks_) {
chunk_notify_.notify_all();
throw std::runtime_error("first chunk must be less than total chunks");
}
first_chunk_ = first_chunk;
last_chunk_ = first_chunk_ + ring_state_.size() - 1u;
if (current_chunk > last_chunk_) {
chunk_notify_.notify_all();
throw std::runtime_error(
"current chunk must be less than or equal to last chunk");
}
current_chunk_ = current_chunk;
ring_state_.set(0u, ring_state_.size(), false);
chunk_notify_.notify_all();
}
void file_manager::ring_buffer_open_file::set_api_path(
const std::string &api_path) {
mutex_lock chunk_lock(chunk_mtx_);
open_file_base::set_api_path(api_path);
chunk_notify_.notify_all();
}
} // namespace repertory

62
src/file_manager/file_manager_upload.cpp Normal file
View File

@@ -0,0 +1,62 @@
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "file_manager/file_manager.hpp"
#include "providers/i_provider.hpp"
#include "utils/error_utils.hpp"
#include "utils/file_utils.hpp"
#include "utils/unix/unix_utils.hpp"
namespace repertory {
file_manager::upload::upload(filesystem_item fsi, i_provider &provider)
: fsi_(fsi), provider_(provider) {
thread_ =
std::make_unique<std::thread>(std::bind(&upload::upload_thread, this));
}
file_manager::upload::~upload() {
stop();
thread_->join();
thread_.reset();
}
void file_manager::upload::cancel() {
cancelled_ = true;
stop();
}
void file_manager::upload::stop() { stop_requested_ = true; }
void file_manager::upload::upload_thread() {
error_ = provider_.upload_file(fsi_.api_path, fsi_.source_path,
fsi_.encryption_token, stop_requested_);
if (not utils::file::reset_modified_time(fsi_.source_path)) {
utils::error::raise_api_path_error(
__FUNCTION__, fsi_.api_path, fsi_.source_path,
utils::get_last_error_code(), "failed to reset modified time");
}
event_system::instance().raise<file_upload_completed>(
get_api_path(), get_source_path(), get_api_error(), cancelled_);
}
} // namespace repertory

166
src/main.cpp
View File

@@ -1,135 +1,165 @@
#include "common.hpp"
#include "utils/cli_utils.hpp"
#include "utils/global_data.hpp"
#include "utils/polling.hpp"
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#ifdef REPERTORY_TESTING
#include "test_common.hpp"
#include "events/event_system.hpp"
#include "test_common.hpp"
#include "utils/string_utils.hpp"
#else
#include "cli/actions.hpp"
#include "types/repertory.hpp"
#endif
#include "utils/cli_utils.hpp"
#include "utils/polling.hpp"
using namespace repertory;
#ifdef _WIN32
std::size_t PROVIDER_INDEX = 0u;
#endif // _WIN32
#if defined(REPERTORY_TESTING) && defined(_WIN32)
std::size_t PROVIDER_INDEX{0U};
#endif
int main(int argc, char **argv) {
auto main(int argc, char **argv) -> int {
repertory_init();
#ifdef REPERTORY_TESTING
#ifdef _WIN32
if (utils::cli::has_option(argc, argv, "--provider_index")) {
PROVIDER_INDEX = static_cast<std::size_t>(
utils::string::to_uint64(utils::cli::parse_option(argc, argv, "--provider_index", 1u)[0u]) +
1u);
utils::string::to_uint64(
utils::cli::parse_option(argc, argv, "--provider_index", 1U)[0U]) +
1U);
}
#endif // _WIN32
curl_global_init(CURL_GLOBAL_DEFAULT);
InitGoogleTest(&argc, argv);
const auto ret = RUN_ALL_TESTS();
curl_global_cleanup();
return ret;
}
#else // REPERTORY_TESTING
#include "cli/actions.hpp"
#include "types/repertory.hpp"
using namespace repertory;
int main(int argc, char **argv) {
auto ret = RUN_ALL_TESTS();
delete_generated_files();
#else
if (argc == 1) {
argc++;
static std::string cmd(argv[0u]);
static std::vector<const char *> v({&cmd[0u], "-h"});
argv = (char **)&v[0u];
static std::string cmd(argv[0U]);
static std::vector<const char *> v({&cmd[0U], "-h"});
argv = (char **)&v[0U];
}
auto pt = utils::cli::get_provider_type_from_args(argc, argv);
std::string data_directory;
auto ret = utils::cli::parse_string_option(argc, argv, utils::cli::options::data_directory_option,
data_directory);
auto res = utils::cli::parse_string_option(
argc, argv, utils::cli::options::data_directory_option, data_directory);
std::string password;
ret = (ret == exit_code::success)
? utils::cli::parse_string_option(argc, argv, utils::cli::options::password_option,
password)
: ret;
res = (res == exit_code::success)
? utils::cli::parse_string_option(
argc, argv, utils::cli::options::password_option, password)
: res;
std::string user;
ret = (ret == exit_code::success)
? utils::cli::parse_string_option(argc, argv, utils::cli::options::user_option, user)
: ret;
res = (res == exit_code::success)
? utils::cli::parse_string_option(
argc, argv, utils::cli::options::user_option, user)
: res;
std::string remote_host;
std::uint16_t remote_port = 0u;
std::uint16_t remote_port{};
std::string unique_id;
if ((ret == exit_code::success) && (pt == provider_type::remote)) {
if ((res == exit_code::success) && (pt == provider_type::remote)) {
std::string data;
if ((ret = utils::cli::parse_string_option(argc, argv, utils::cli::options::remote_mount_option,
data)) == exit_code::success) {
if ((res = utils::cli::parse_string_option(
argc, argv, utils::cli::options::remote_mount_option, data)) ==
exit_code::success) {
const auto parts = utils::string::split(data, ':');
if (parts.size() != 2) {
std::cerr << "Invalid syntax for host/port '-rm host:port,--remote_mount host:port'"
std::cerr << "Invalid syntax for host/port '-rm "
"host:port,--remote_mount host:port'"
<< std::endl;
ret = exit_code::invalid_syntax;
res = exit_code::invalid_syntax;
} else {
unique_id = parts[0u] + ':' + parts[1u];
remote_host = parts[0u];
unique_id = parts[0U] + ':' + parts[1U];
remote_host = parts[0U];
try {
remote_port = utils::string::to_uint16(parts[1u]);
remote_port = utils::string::to_uint16(parts[1U]);
data_directory = utils::path::combine(
data_directory.empty() ? app_config::default_data_directory(pt) : data_directory,
data_directory.empty() ? app_config::default_data_directory(pt)
: data_directory,
{utils::string::replace_copy(unique_id, ':', '_')});
} catch (const std::exception &e) {
std::cerr << (e.what() ? e.what() : "Unable to parse port") << std::endl;
ret = exit_code::invalid_syntax;
std::cerr << (e.what() ? e.what() : "Unable to parse port")
<< std::endl;
res = exit_code::invalid_syntax;
}
}
}
}
#ifdef REPERTORY_ENABLE_S3
if ((ret == exit_code::success) && (pt == provider_type::s3)) {
if ((res == exit_code::success) && (pt == provider_type::s3)) {
std::string data;
if ((ret = utils::cli::parse_string_option(argc, argv, utils::cli::options::name_option,
if ((res = utils::cli::parse_string_option(argc, argv,
utils::cli::options::name_option,
data)) == exit_code::success) {
unique_id = utils::string::trim(data);
if (unique_id.empty()) {
std::cerr << "Name of S3 instance not provided" << std::endl;
ret = exit_code::invalid_syntax;
res = exit_code::invalid_syntax;
} else {
data_directory = utils::path::combine(
data_directory.empty() ? app_config::default_data_directory(pt) : data_directory,
data_directory.empty() ? app_config::default_data_directory(pt)
: data_directory,
{unique_id});
}
}
}
#endif // REPERTORY_ENABLE_S3
int mount_result = 0;
if (ret == exit_code::success) {
int mount_result{};
if (res == exit_code::success) {
if (utils::cli::has_option(argc, argv, utils::cli::options::help_option)) {
cli::actions::help<repertory_drive>(argc, argv);
} else if (utils::cli::has_option(argc, argv, utils::cli::options::version_option)) {
} else if (utils::cli::has_option(argc, argv,
utils::cli::options::version_option)) {
cli::actions::version<repertory_drive>(argc, argv);
} else {
ret = exit_code::option_not_found;
for (std::size_t i = 0;
(ret == exit_code::option_not_found) && (i < utils::cli::options::option_list.size());
i++) {
ret = cli::actions::perform_action(utils::cli::options::option_list[i], argc, argv,
data_directory, pt, unique_id, user, password);
res = exit_code::option_not_found;
for (std::size_t idx = 0U;
(res == exit_code::option_not_found) &&
(idx < utils::cli::options::option_list.size());
idx++) {
res = cli::actions::perform_action(
utils::cli::options::option_list[idx], argc, argv, data_directory,
pt, unique_id, user, password);
}
if (ret == exit_code::option_not_found) {
ret = cli::actions::mount(argc, argv, data_directory, mount_result, pt, remote_host,
remote_port, unique_id);
if (res == exit_code::option_not_found) {
res = cli::actions::mount(argc, argv, data_directory, mount_result, pt,
remote_host, remote_port, unique_id);
}
}
}
return ((ret == exit_code::mount_result) ? mount_result : static_cast<int>(ret));
}
auto ret =
((res == exit_code::mount_result) ? mount_result
: static_cast<std::int32_t>(res));
#endif // REPERTORY_TESTING
#endif
repertory_shutdown();
return ret;
}

168
src/platform/unix_platform.cpp
View File

@@ -1,36 +1,47 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#ifndef _WIN32
#include "platform/unix_platform.hpp"
#include "app_config.hpp"
#include "providers/i_provider.hpp"
#include "types/startup_exception.hpp"
#include "utils/error_utils.hpp"
#include "utils/file_utils.hpp"
#include "utils/path_utils.hpp"
#include "utils/unix/unix_utils.hpp"
namespace repertory {
lock_data::lock_data(const provider_type &pt, std::string unique_id /*= ""*/)
: pt_(pt),
unique_id_(std::move(unique_id)),
mutex_id_("repertory2_" + app_config::get_provider_name(pt) + "_" + unique_id_) {
mutex_id_("repertory2_" + app_config::get_provider_name(pt) + "_" +
unique_id_) {
lock_fd_ = open(get_lock_file().c_str(), O_CREAT | O_RDWR, S_IWUSR | S_IRUSR);
}
lock_data::lock_data() : pt_(provider_type::sia), unique_id_(""), mutex_id_(""), lock_fd_(-1) {}
lock_data::lock_data()
: pt_(provider_type::sia), unique_id_(""), mutex_id_(""), lock_fd_(-1) {}
lock_data::~lock_data() {
if (lock_fd_ != -1) {
@@ -43,21 +54,31 @@ lock_data::~lock_data() {
}
}
std::string lock_data::get_lock_data_file() {
auto lock_data::get_lock_data_file() -> std::string {
const auto dir = get_state_directory();
utils::file::create_full_directory_path(dir);
return utils::path::combine(dir, {"mountstate_" + std::to_string(getuid()) + ".json"});
if (not utils::file::create_full_directory_path(dir)) {
throw startup_exception("failed to create directory|sp|" + dir + "|err|" +
std::to_string(utils::get_last_error_code()));
}
return utils::path::combine(
dir, {"mountstate_" + std::to_string(getuid()) + ".json"});
}
std::string lock_data::get_lock_file() {
auto lock_data::get_lock_file() -> std::string {
const auto dir = get_state_directory();
utils::file::create_full_directory_path(dir);
return utils::path::combine(dir, {mutex_id_ + "_" + std::to_string(getuid())});
if (not utils::file::create_full_directory_path(dir)) {
throw startup_exception("failed to create directory|sp|" + dir + "|err|" +
std::to_string(utils::get_last_error_code()));
}
return utils::path::combine(dir,
{mutex_id_ + "_" + std::to_string(getuid())});
}
bool lock_data::get_mount_state(json &mount_state) {
auto lock_data::get_mount_state(json &mount_state) -> bool {
auto ret = false;
auto fd = open(get_lock_data_file().c_str(), O_CREAT | O_RDWR, S_IWUSR | S_IRUSR);
auto fd =
open(get_lock_data_file().c_str(), O_CREAT | O_RDWR, S_IWUSR | S_IRUSR);
if (fd != -1) {
if (wait_for_lock(fd) == 0) {
ret = utils::file::read_json_file(get_lock_data_file(), mount_state);
@@ -69,16 +90,17 @@ bool lock_data::get_mount_state(json &mount_state) {
return ret;
}
std::string lock_data::get_state_directory() {
auto lock_data::get_state_directory() -> std::string {
#ifdef __APPLE__
return utils::path::resolve("~/Library/Application Support/" + std::string(REPERTORY_DATA_NAME) +
"/state");
return utils::path::resolve("~/Library/Application Support/" +
std::string(REPERTORY_DATA_NAME) + "/state");
#else
return utils::path::resolve("~/.local/" + std::string(REPERTORY_DATA_NAME) + "/state");
return utils::path::resolve("~/.local/" + std::string(REPERTORY_DATA_NAME) +
"/state");
#endif
}
lock_result lock_data::grab_lock(const std::uint8_t &retry_count) {
auto lock_data::grab_lock(std::uint8_t retry_count) -> lock_result {
if (lock_fd_ == -1) {
return lock_result::failure;
}
@@ -86,7 +108,9 @@ lock_result lock_data::grab_lock(const std::uint8_t &retry_count) {
lock_status_ = wait_for_lock(lock_fd_, retry_count);
switch (lock_status_) {
case 0:
set_mount_state(false, "", -1);
if (not set_mount_state(false, "", -1)) {
utils::error::raise_error(__FUNCTION__, "failed to set mount state");
}
return lock_result::success;
case EWOULDBLOCK:
return lock_result::locked;
@@ -95,30 +119,44 @@ lock_result lock_data::grab_lock(const std::uint8_t &retry_count) {
}
}
bool lock_data::set_mount_state(const bool &active, const std::string &mount_location,
const int &pid) {
auto lock_data::set_mount_state(bool active, const std::string &mount_location,
int pid) -> bool {
auto ret = false;
auto fd = open(get_lock_data_file().c_str(), O_CREAT | O_RDWR, S_IWUSR | S_IRUSR);
auto fd =
open(get_lock_data_file().c_str(), O_CREAT | O_RDWR, S_IWUSR | S_IRUSR);
if (fd != -1) {
if (wait_for_lock(fd) == 0) {
const auto mount_id = app_config::get_provider_display_name(pt_) + unique_id_;
const auto mount_id =
app_config::get_provider_display_name(pt_) + unique_id_;
json mount_state;
utils::file::read_json_file(get_lock_data_file(), mount_state);
if (not utils::file::read_json_file(get_lock_data_file(), mount_state)) {
utils::error::raise_error(__FUNCTION__,
"failed to read mount state file|sp|" +
get_lock_file());
}
if ((mount_state.find(mount_id) == mount_state.end()) ||
(mount_state[mount_id].find("Active") == mount_state[mount_id].end()) ||
(mount_state[mount_id].find("Active") ==
mount_state[mount_id].end()) ||
(mount_state[mount_id]["Active"].get<bool>() != active) ||
(active && ((mount_state[mount_id].find("Location") == mount_state[mount_id].end()) ||
(mount_state[mount_id]["Location"].get<std::string>() != mount_location)))) {
(active && ((mount_state[mount_id].find("Location") ==
mount_state[mount_id].end()) ||
(mount_state[mount_id]["Location"].get<std::string>() !=
mount_location)))) {
const auto lines = utils::file::read_file_lines(get_lock_data_file());
const auto txt =
std::accumulate(lines.begin(), lines.end(), std::string(),
[](std::string s, const std::string &s2) { return std::move(s) + s2; });
[](std::string s, const std::string &s2) {
return std::move(s) + s2;
});
auto json = json::parse(txt.empty() ? "{}" : txt);
json[mount_id] = {{"Active", active},
{"Location", active ? mount_location : ""},
{"PID", active ? pid : -1}};
ret = utils::file::write_json_file(get_lock_data_file(), json);
} else {
ret = true;
}
flock(fd, LOCK_UN);
}
@@ -127,14 +165,14 @@ bool lock_data::set_mount_state(const bool &active, const std::string &mount_loc
return ret;
}
int lock_data::wait_for_lock(const int &fd, const std::uint8_t &retry_count) {
auto lock_data::wait_for_lock(int fd, std::uint8_t retry_count) -> int {
auto lock_status = EWOULDBLOCK;
std::int16_t remain = retry_count * 100u;
while ((remain > 0) && (lock_status == EWOULDBLOCK)) {
if ((lock_status = flock(fd, LOCK_EX | LOCK_NB)) == -1) {
if ((lock_status = errno) == EWOULDBLOCK) {
const auto sleep_ms =
utils::random_between(std::int16_t(1), std::min(remain, std::int16_t(100)));
const auto sleep_ms = utils::random_between(
std::int16_t(1), std::min(remain, std::int16_t(100)));
std::this_thread::sleep_for(std::chrono::milliseconds(sleep_ms));
remain -= sleep_ms;
}
@@ -143,6 +181,56 @@ int lock_data::wait_for_lock(const int &fd, const std::uint8_t &retry_count) {
return lock_status;
}
auto create_meta_attributes(
std::uint64_t accessed_date, std::uint32_t attributes,
std::uint64_t changed_date, std::uint64_t creation_date, bool directory,
const std::string &encryption_token, std::uint32_t gid,
const std::string &key, std::uint32_t mode, std::uint64_t modified_date,
std::uint32_t osx_backup, std::uint32_t osx_flags, std::uint64_t size,
const std::string &source_path, std::uint32_t uid,
std::uint64_t written_date) -> api_meta_map {
return {
{META_ACCESSED, std::to_string(accessed_date)},
{META_ATTRIBUTES, std::to_string(attributes)},
{META_BACKUP, std::to_string(osx_backup)},
{META_CHANGED, std::to_string(changed_date)},
{META_CREATION, std::to_string(creation_date)},
{META_DIRECTORY, utils::string::from_bool(directory)},
{META_ENCRYPTION_TOKEN, encryption_token},
{META_GID, std::to_string(gid)},
{META_KEY, key},
{META_MODE, std::to_string(mode)},
{META_MODIFIED, std::to_string(modified_date)},
{META_OSXFLAGS, std::to_string(osx_flags)},
{META_PINNED, "0"},
{META_SOURCE, source_path},
{META_SIZE, std::to_string(size)},
{META_UID, std::to_string(uid)},
{META_WRITTEN, std::to_string(written_date)},
};
}
auto provider_meta_handler(i_provider &provider, bool directory,
const api_file &file) -> api_error {
const auto meta = create_meta_attributes(
file.accessed_date,
directory ? FILE_ATTRIBUTE_DIRECTORY
: FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_ARCHIVE,
file.changed_date, file.creation_date, directory, file.encryption_token,
getgid(), file.key,
directory ? S_IFDIR | S_IRUSR | S_IWUSR | S_IXUSR
: S_IFREG | S_IRUSR | S_IWUSR,
file.modified_date, 0u, 0u, file.file_size, file.source_path, getuid(),
file.modified_date);
auto res = provider.set_item_meta(file.api_path, meta);
if (res == api_error::success) {
event_system::instance().raise<filesystem_item_added>(
file.api_path, file.api_parent, directory);
}
return res;
}
} // namespace repertory
#endif //_WIN32

143
src/platform/win32_platform.cpp
View File

@@ -1,29 +1,38 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#ifdef _WIN32
#include "platform/win32_platform.hpp"
#include "providers/i_provider.hpp"
#include "utils/error_utils.hpp"
namespace repertory {
bool lock_data::get_mount_state(const provider_type &pt, json &mount_state) {
auto lock_data::get_mount_state(const provider_type & /*pt*/, json &mount_state)
-> bool {
const auto ret = get_mount_state(mount_state);
if (ret) {
const auto mount_id = app_config::get_provider_display_name(pt_) + unique_id_;
const auto mount_id =
app_config::get_provider_display_name(pt_) + unique_id_;
mount_state = mount_state[mount_id].empty()
? json({{"Active", false}, {"Location", ""}, {"PID", -1}})
: mount_state[mount_id];
@@ -32,10 +41,11 @@ bool lock_data::get_mount_state(const provider_type &pt, json &mount_state) {
return ret;
}
bool lock_data::get_mount_state(json &mount_state) {
auto lock_data::get_mount_state(json &mount_state) -> bool {
HKEY key;
auto ret = !::RegCreateKeyEx(
HKEY_CURRENT_USER, ("SOFTWARE\\" + std::string(REPERTORY_DATA_NAME) + "\\Mounts").c_str(), 0,
HKEY_CURRENT_USER,
("SOFTWARE\\" + std::string(REPERTORY_DATA_NAME) + "\\Mounts").c_str(), 0,
nullptr, 0, KEY_ALL_ACCESS, nullptr, &key, nullptr);
if (ret) {
DWORD i = 0u;
@@ -43,13 +53,15 @@ bool lock_data::get_mount_state(json &mount_state) {
std::string name;
name.resize(32767u);
auto name_size = static_cast<DWORD>(name.size());
while (ret && (::RegEnumValue(key, i, &name[0], &name_size, nullptr, nullptr, nullptr,
&data_size) == ERROR_SUCCESS)) {
while (ret &&
(::RegEnumValue(key, i, &name[0], &name_size, nullptr, nullptr,
nullptr, &data_size) == ERROR_SUCCESS)) {
std::string data;
data.resize(data_size);
name_size++;
if ((ret = !::RegEnumValue(key, i++, &name[0], &name_size, nullptr, nullptr,
reinterpret_cast<LPBYTE>(&data[0]), &data_size))) {
if ((ret = !::RegEnumValue(key, i++, &name[0], &name_size, nullptr,
nullptr, reinterpret_cast<LPBYTE>(&data[0]),
&data_size))) {
mount_state[name.c_str()] = json::parse(data);
name_size = static_cast<DWORD>(name.size());
data_size = 0u;
@@ -60,13 +72,14 @@ bool lock_data::get_mount_state(json &mount_state) {
return ret;
}
lock_result lock_data::grab_lock(const std::uint8_t &retryCount) {
auto lock_data::grab_lock(std::uint8_t retry_count) -> lock_result {
auto ret = lock_result::success;
if (mutex_handle_ == INVALID_HANDLE_VALUE) {
ret = lock_result::failure;
} else {
for (auto i = 0; (i <= retryCount) &&
((mutex_state_ = ::WaitForSingleObject(mutex_handle_, 100)) == WAIT_TIMEOUT);
for (auto i = 0;
(i <= retry_count) && ((mutex_state_ = ::WaitForSingleObject(
mutex_handle_, 100)) == WAIT_TIMEOUT);
i++) {
}
@@ -76,13 +89,16 @@ lock_result lock_data::grab_lock(const std::uint8_t &retryCount) {
auto should_reset = true;
json mount_state;
if (get_mount_state(pt_, mount_state)) {
if (mount_state["Active"].get<bool>() && mount_state["Location"] == "elevating") {
if (mount_state["Active"].get<bool>() &&
mount_state["Location"] == "elevating") {
should_reset = false;
}
}
if (should_reset) {
set_mount_state(false, "", -1);
if (not set_mount_state(false, "", -1)) {
utils::error::raise_error(__FUNCTION__, "failed to set mount state");
}
}
} break;
@@ -109,37 +125,90 @@ void lock_data::release() {
}
}
bool lock_data::set_mount_state(const bool &active, const std::string &mountLocation,
const std::int64_t &pid) {
auto lock_data::set_mount_state(bool active, const std::string &mount_location,
const std::int64_t &pid) -> bool {
auto ret = false;
if (mutex_handle_ != INVALID_HANDLE_VALUE) {
const auto mount_id = app_config::get_provider_display_name(pt_) + unique_id_;
const auto mount_id =
app_config::get_provider_display_name(pt_) + unique_id_;
json mount_state;
get_mount_state(mount_state);
[[maybe_unused]] auto success = get_mount_state(mount_state);
if ((mount_state.find(mount_id) == mount_state.end()) ||
(mount_state[mount_id].find("Active") == mount_state[mount_id].end()) ||
(mount_state[mount_id]["Active"].get<bool>() != active) ||
(active && ((mount_state[mount_id].find("Location") == mount_state[mount_id].end()) ||
(mount_state[mount_id]["Location"].get<std::string>() != mountLocation)))) {
(active && ((mount_state[mount_id].find("Location") ==
mount_state[mount_id].end()) ||
(mount_state[mount_id]["Location"].get<std::string>() !=
mount_location)))) {
HKEY key;
if ((ret = !::RegCreateKeyEx(
HKEY_CURRENT_USER,
("SOFTWARE\\" + std::string(REPERTORY_DATA_NAME) + "\\Mounts").c_str(), 0, nullptr,
0, KEY_ALL_ACCESS, nullptr, &key, nullptr))) {
("SOFTWARE\\" + std::string(REPERTORY_DATA_NAME) + "\\Mounts")
.c_str(),
0, nullptr, 0, KEY_ALL_ACCESS, nullptr, &key, nullptr))) {
const auto str = json({{"Active", active},
{"Location", active ? mountLocation : ""},
{"Location", active ? mount_location : ""},
{"PID", active ? pid : -1}})
.dump(0);
ret =
!::RegSetValueEx(key, &mount_id[0], 0, REG_SZ, reinterpret_cast<const BYTE *>(&str[0]),
static_cast<DWORD>(str.size()));
ret = !::RegSetValueEx(key, &mount_id[0], 0, REG_SZ,
reinterpret_cast<const BYTE *>(&str[0]),
static_cast<DWORD>(str.size()));
::RegCloseKey(key);
}
} else {
ret = true;
}
}
return ret;
}
auto create_meta_attributes(
std::uint64_t accessed_date, std::uint32_t attributes,
std::uint64_t changed_date, std::uint64_t creation_date, bool directory,
const std::string &encryption_token, std::uint32_t gid,
const std::string &key, std::uint32_t mode, std::uint64_t modified_date,
std::uint32_t osx_backup, std::uint32_t osx_flags, std::uint64_t size,
const std::string &source_path, std::uint32_t uid,
std::uint64_t written_date) -> api_meta_map {
return {
{META_ACCESSED, std::to_string(accessed_date)},
{META_ATTRIBUTES, std::to_string(attributes)},
{META_BACKUP, std::to_string(osx_backup)},
{META_CHANGED, std::to_string(changed_date)},
{META_CREATION, std::to_string(creation_date)},
{META_DIRECTORY, utils::string::from_bool(directory)},
{META_ENCRYPTION_TOKEN, encryption_token},
{META_GID, std::to_string(gid)},
{META_KEY, key},
{META_MODE, std::to_string(mode)},
{META_MODIFIED, std::to_string(modified_date)},
{META_OSXFLAGS, std::to_string(osx_flags)},
{META_PINNED, "0"},
{META_SIZE, std::to_string(size)},
{META_SOURCE, source_path},
{META_UID, std::to_string(uid)},
{META_WRITTEN, std::to_string(written_date)},
};
}
auto provider_meta_handler(i_provider &provider, bool directory,
const api_file &file) -> api_error {
const auto meta = create_meta_attributes(
file.accessed_date,
directory ? FILE_ATTRIBUTE_DIRECTORY
: FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_ARCHIVE,
file.changed_date, file.creation_date, directory, file.encryption_token,
0u, file.key, directory ? S_IFDIR : S_IFREG, file.modified_date, 0u, 0u,
file.file_size, file.source_path, 0u, file.modified_date);
auto res = provider.set_item_meta(file.api_path, meta);
if (res == api_error::success) {
event_system::instance().raise<filesystem_item_added>(
file.api_path, file.api_parent, directory);
}
return res;
}
} // namespace repertory
#endif //_WIN32

554
src/providers/base_provider.cpp
View File

@@ -1,105 +1,196 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "providers/base_provider.hpp"
#include "app_config.hpp"
#include "file_manager/i_file_manager.hpp"
#include "types/repertory.hpp"
#include "types/startup_exception.hpp"
#include "utils/error_utils.hpp"
#include "utils/file_utils.hpp"
#include "utils/global_data.hpp"
#include "utils/native_file.hpp"
#include "utils/path_utils.hpp"
namespace repertory {
base_provider::base_provider(app_config &config) : config_(config), meta_db_(config) {}
base_provider::base_provider(app_config &config) : config_(config) {}
void base_provider::notify_directory_added(const std::string &api_path,
const std::string &api_parent) {
recur_mutex_lock l(notify_added_mutex_);
const auto now = utils::get_file_time_now();
api_item_added_(api_path, api_parent, "", true, now, now, now, now);
}
void base_provider::update_filesystem_item(const bool &directory, const api_error &error,
const std::string &api_path,
filesystem_item &fsi) const {
if (error == api_error::success) {
fsi.directory = directory;
fsi.lock = fsi.lock ? fsi.lock : std::make_shared<std::recursive_mutex>();
fsi.api_path = api_path;
fsi.api_parent = utils::path::get_parent_api_path(api_path);
} else {
event_system::instance().raise<filesystem_item_get_failed>(
api_path, std::to_string(static_cast<int>(error)));
void base_provider::calculate_used_drive_space(bool add_missing) {
api_file_list list{};
if (get_file_list(list) != api_error::success) {
return;
}
used_space_ = std::accumulate(
list.begin(), list.end(), std::uint64_t(0U),
[this, &add_missing](std::uint64_t total_size, const auto &file) {
if (add_missing && not meta_db_->get_item_meta_exists(file.api_path)) {
[[maybe_unused]] auto res = this->notify_file_added(
file.api_path, utils::path::get_parent_api_path(file.api_path),
0);
}
return total_size + file.file_size;
});
}
api_error base_provider::create_directory_clone_source_meta(const std::string &source_api_path,
const std::string &api_path) {
void base_provider::cleanup() {
remove_deleted_files();
remove_unknown_source_files();
remove_expired_orphaned_files();
}
auto base_provider::create_directory_clone_source_meta(
const std::string &source_api_path, const std::string &api_path)
-> api_error {
api_meta_map meta{};
auto ret = get_item_meta(source_api_path, meta);
if (ret == api_error::success) {
ret = create_directory(api_path, meta);
}
return ret;
return ret == api_error::item_not_found ? api_error::directory_not_found
: ret;
}
api_error base_provider::create_file(const std::string &api_path, api_meta_map &meta) {
const auto isDir = is_directory(api_path);
const auto isFile = is_file(api_path);
auto ret = isDir ? api_error::directory_exists
: isFile ? api_error::file_exists
: api_error::success;
if (ret == api_error::success) {
const auto source =
utils::path::combine(get_config().get_cache_directory(), {utils::create_uuid_string()});
auto base_provider::create_file(const std::string &api_path, api_meta_map &meta)
-> api_error {
bool exists{};
auto res = is_directory(api_path, exists);
if (res != api_error::success) {
return res;
}
if (exists) {
return api_error::directory_exists;
}
res = is_file(api_path, exists);
if (res != api_error::success) {
return res;
}
if (exists) {
return api_error::item_exists;
}
if ((res = meta_db_->set_item_meta(api_path, meta)) != api_error::success) {
return res;
}
{
native_file_ptr nf;
if ((ret = native_file::create_or_open(source, nf)) == api_error::success) {
nf->close();
res = native_file::create_or_open(meta[META_SOURCE], nf);
if (res != api_error::success) {
return res;
}
nf->close();
}
stop_type stop_requested = false;
return upload_file(api_path, meta[META_SOURCE], meta[META_ENCRYPTION_TOKEN],
stop_requested);
}
auto base_provider::get_api_path_from_source(const std::string &source_path,
std::string &api_path) const
-> api_error {
return meta_db_->get_api_path_from_source(source_path, api_path);
}
auto base_provider::get_directory_items(const std::string &api_path,
directory_item_list &list) const
-> api_error {
auto res = populate_directory_items(api_path, list);
if (res != api_error::success) {
return res;
}
std::sort(list.begin(), list.end(), [](const auto &a, const auto &b) -> bool {
return (a.directory && not b.directory) ||
(not(b.directory && not a.directory) &&
(a.api_path.compare(b.api_path) < 0));
});
list.insert(list.begin(), directory_item{
"..",
"",
true,
});
list.insert(list.begin(), directory_item{
".",
"",
true,
});
return api_error::success;
}
auto base_provider::get_file(const std::string &api_path, api_file &file) const
-> api_error {
auto ret = api_error::success;
try {
if ((ret = populate_file(api_path, file)) != api_error::success) {
event_system::instance().raise<file_get_failed>(api_path,
api_error_to_string(ret));
}
if (ret == api_error::success) {
if (((ret = set_item_meta(api_path, meta)) != api_error::success) ||
((ret = set_source_path(api_path, source)) != api_error::success) ||
((ret = upload_file(api_path, source, meta[META_ENCRYPTION_TOKEN])) !=
api_error::success)) {
meta_db_.remove_item_meta(format_api_path(api_path));
utils::file::delete_file(source);
}
std::string sz;
if ((ret = get_item_meta(api_path, META_SIZE, sz)) != api_error::success) {
return ret;
}
file.file_size = utils::string::to_uint64(sz);
return ret;
} catch (const std::exception &e) {
event_system::instance().raise<file_get_failed>(
api_path, e.what() ? e.what() : "failed to get file");
}
return api_error::error;
}
auto base_provider::get_file_size(const std::string &api_path,
std::uint64_t &file_size) const -> api_error {
api_file file{};
const auto ret = get_file(api_path, file);
if (ret == api_error::success) {
file_size = file.file_size;
} else {
event_system::instance().raise<file_get_size_failed>(
api_path, api_error_to_string(ret));
}
return ret;
}
api_error base_provider::get_api_path_from_source(const std::string &source_path,
std::string &api_path) const {
const auto ret = meta_db_.get_api_path_from_source(source_path, api_path);
restore_api_path(api_path);
return ret;
}
api_error base_provider::get_filesystem_item(const std::string &api_path, const bool &directory,
filesystem_item &fsi) const {
auto base_provider::get_filesystem_item(const std::string &api_path,
bool directory,
filesystem_item &fsi) const
-> api_error {
auto ret = api_error::error;
if (directory) {
ret = is_directory(api_path) ? api_error::success : api_error::item_not_found;
bool exists{};
ret = is_directory(api_path, exists);
if (ret != api_error::success) {
return ret;
}
ret = exists ? api_error::success : api_error::item_not_found;
update_filesystem_item(true, ret, api_path, fsi);
} else {
api_file file{};
@@ -109,16 +200,25 @@ api_error base_provider::get_filesystem_item(const std::string &api_path, const
return ret;
}
api_error base_provider::get_filesystem_item_and_file(const std::string &api_path, api_file &file,
filesystem_item &fsi) const {
auto base_provider::get_filesystem_item_and_file(const std::string &api_path,
api_file &file,
filesystem_item &fsi) const
-> api_error {
auto ret = get_item_meta(api_path, META_SOURCE, fsi.source_path);
if (ret == api_error::success) {
ret = get_file(api_path, file);
if (ret == api_error::success) {
fsi.encryption_token = file.encryption_token;
fsi.size = file.file_size;
} else if (not is_file(api_path)) {
ret = api_error::item_not_found;
} else {
bool exists{};
ret = is_file(api_path, exists);
if (ret != api_error::success) {
return ret;
}
if (not exists) {
ret = api_error::item_not_found;
}
}
}
@@ -126,12 +226,13 @@ api_error base_provider::get_filesystem_item_and_file(const std::string &api_pat
return ret;
}
api_error base_provider::get_filesystem_item_from_source_path(const std::string &source_path,
filesystem_item &fsi) const {
auto base_provider::get_filesystem_item_from_source_path(
const std::string &source_path, filesystem_item &fsi) const -> api_error {
auto ret = api_error::item_not_found;
if (not source_path.empty()) {
std::string api_path;
if ((ret = get_api_path_from_source(source_path, api_path)) == api_error::success) {
if ((ret = get_api_path_from_source(source_path, api_path)) ==
api_error::success) {
ret = get_filesystem_item(api_path, false, fsi);
}
}
@@ -139,71 +240,308 @@ api_error base_provider::get_filesystem_item_from_source_path(const std::string
return ret;
}
api_error base_provider::get_item_meta(const std::string &api_path, api_meta_map &meta) const {
auto ret = meta_db_.get_item_meta(format_api_path(api_path), meta);
auto base_provider::get_item_meta(const std::string &api_path,
api_meta_map &meta) const -> api_error {
auto ret = meta_db_->get_item_meta(api_path, meta);
if (ret == api_error::item_not_found) {
auto get_meta = false;
if (is_directory(api_path)) {
notify_directory_added(api_path, utils::path::get_parent_api_path(api_path));
get_meta = true;
} else if (is_file(api_path)) {
std::uint64_t file_size = 0u;
if ((ret = get_file_size(api_path, file_size)) == api_error::success) {
get_meta = ((ret = notify_file_added(api_path, utils::path::get_parent_api_path(api_path),
file_size)) == api_error::success);
bool exists{};
ret = is_directory(api_path, exists);
if (ret != api_error::success) {
return ret;
}
if (exists) {
ret = notify_directory_added(api_path,
utils::path::get_parent_api_path(api_path));
if (ret == api_error::success) {
get_meta = true;
}
} else {
ret = is_file(api_path, exists);
if (ret != api_error::success) {
return ret;
}
if (exists) {
std::uint64_t file_size{};
if ((ret = get_file_size(api_path, file_size)) == api_error::success) {
get_meta = ((ret = notify_file_added(
api_path, utils::path::get_parent_api_path(api_path),
file_size)) == api_error::success);
}
}
}
if (get_meta) {
ret = meta_db_.get_item_meta(format_api_path(api_path), meta);
}
ret = get_meta ? meta_db_->get_item_meta(api_path, meta)
: api_error::item_not_found;
}
return ret;
}
api_error base_provider::get_item_meta(const std::string &api_path, const std::string &key,
std::string &value) const {
auto ret = meta_db_.get_item_meta(format_api_path(api_path), key, value);
auto base_provider::get_item_meta(const std::string &api_path,
const std::string &key,
std::string &value) const -> api_error {
auto ret = meta_db_->get_item_meta(api_path, key, value);
if (ret == api_error::item_not_found) {
auto get_meta = false;
if (is_directory(api_path)) {
notify_directory_added(api_path, utils::path::get_parent_api_path(api_path));
get_meta = true;
} else if (is_file(api_path)) {
std::uint64_t file_size = 0u;
if ((ret = get_file_size(api_path, file_size)) == api_error::success) {
get_meta = ((ret = notify_file_added(api_path, utils::path::get_parent_api_path(api_path),
file_size)) == api_error::success);
bool exists{};
ret = is_directory(api_path, exists);
if (ret != api_error::success) {
return ret;
}
if (exists) {
ret = notify_directory_added(api_path,
utils::path::get_parent_api_path(api_path));
if (ret == api_error::success) {
get_meta = true;
}
} else {
ret = is_file(api_path, exists);
if (ret != api_error::success) {
return ret;
}
if (exists) {
std::uint64_t file_size{};
if ((ret = get_file_size(api_path, file_size)) == api_error::success) {
get_meta = ((ret = notify_file_added(
api_path, utils::path::get_parent_api_path(api_path),
file_size)) == api_error::success);
}
}
}
if (get_meta) {
ret = meta_db_.get_item_meta(format_api_path(api_path), key, value);
}
ret = get_meta ? meta_db_->get_item_meta(api_path, key, value)
: api_error::item_not_found;
}
return ret;
}
std::uint64_t base_provider::get_used_drive_space() const {
return global_data::instance().get_used_drive_space();
auto base_provider::get_used_drive_space() const -> std::uint64_t {
std::uint64_t used_space = used_space_;
fm_->update_used_space(used_space);
return used_space;
}
bool base_provider::start(api_item_added_callback api_item_added, i_open_file_table *oft) {
auto base_provider::notify_directory_added(const std::string &api_path,
const std::string &api_parent)
-> api_error {
recur_mutex_lock l(notify_added_mutex_);
const auto now = utils::get_file_time_now();
api_file file{};
file.api_path = api_path;
file.api_parent = api_parent;
file.accessed_date = now;
file.changed_date = now;
file.creation_date = now;
file.file_size = 0U;
file.modified_date = now;
return api_item_added_(true, file);
}
auto base_provider::processed_orphaned_file(const std::string &source_path,
const std::string &api_path) const
-> bool {
const auto orphaned_directory =
utils::path::combine(get_config().get_data_directory(), {"orphaned"});
if (utils::file::create_full_directory_path(orphaned_directory)) {
event_system::instance().raise<orphaned_file_detected>(source_path);
const auto parts = utils::string::split(api_path, '/', false);
const auto orphaned_file = utils::path::combine(
orphaned_directory, {utils::path::strip_to_file_name(source_path) +
'_' + parts[parts.size() - 1U]});
if (utils::file::reset_modified_time(source_path) &&
utils::file::move_file(source_path, orphaned_file)) {
event_system::instance().raise<orphaned_file_processed>(source_path,
orphaned_file);
return true;
}
event_system::instance().raise<orphaned_file_processing_failed>(
source_path, orphaned_file,
std::to_string(utils::get_last_error_code()));
return false;
}
utils::error::raise_error(
__FUNCTION__, std::to_string(utils::get_last_error_code()),
"failed to create orphaned director|sp|" + orphaned_directory);
return false;
}
void base_provider::remove_deleted_files() {
std::vector<std::string> removed_files{};
api_file_list list{};
if (get_file_list(list) == api_error::success) {
if (not list.empty()) {
auto iterator = meta_db_->create_iterator(false);
for (iterator->SeekToFirst(); not stop_requested_ && iterator->Valid();
iterator->Next()) {
const auto meta_api_path = iterator->key().ToString();
if (meta_api_path.empty()) {
const auto res = meta_db_->remove_item_meta(meta_api_path);
if (res != api_error::success) {
utils::error::raise_api_path_error(__FUNCTION__, meta_api_path, res,
"failed to remove item meta");
}
} else {
auto api_path = meta_api_path;
const auto it = std::find_if(list.begin(), list.end(),
[&api_path](const auto &file) -> bool {
return file.api_path == api_path;
});
if (it == list.end()) {
removed_files.emplace_back(api_path);
}
}
}
}
}
while (not stop_requested_ && not removed_files.empty()) {
const auto api_path = removed_files.back();
removed_files.pop_back();
bool exists{};
auto res = is_directory(api_path, exists);
if (res != api_error::success) {
continue;
}
std::string source_path;
if (not exists &&
(check_file_exists(api_path) == api_error::item_not_found) &&
(meta_db_->get_item_meta(api_path, META_SOURCE, source_path) ==
api_error::success)) {
if (not source_path.empty()) {
fm_->perform_locked_operation(
[this, &api_path, &source_path](i_provider &) -> bool {
if (fm_->has_no_open_file_handles()) {
const auto res = meta_db_->remove_item_meta(api_path);
if (res == api_error::success) {
event_system::instance().raise<file_removed_externally>(
api_path, source_path);
processed_orphaned_file(source_path, api_path);
} else {
utils::error::raise_api_path_error(
__FUNCTION__, api_path, source_path, res,
"failed to remove item meta for externally removed file");
}
}
return true;
});
}
}
}
}
void base_provider::remove_expired_orphaned_files() {
const auto orphaned_directory =
utils::path::combine(get_config().get_data_directory(), {"orphaned"});
const auto files = utils::file::get_directory_files(orphaned_directory, true);
for (const auto &file : files) {
if (utils::file::is_modified_date_older_than(
file, std::chrono::hours(
get_config().get_orphaned_file_retention_days() * 24))) {
if (utils::file::retry_delete_file(file)) {
event_system::instance().raise<orphaned_file_deleted>(file);
}
}
if (stop_requested_) {
break;
}
}
}
void base_provider::remove_unknown_source_files() {
auto files = utils::file::get_directory_files(
get_config().get_cache_directory(), true);
while (not stop_requested_ && not files.empty()) {
const auto file = files.front();
files.pop_front();
std::string api_path;
if (not meta_db_->get_source_path_exists(file)) {
processed_orphaned_file(file);
}
}
}
auto base_provider::rename_file(const std::string &from_api_path,
const std::string &to_api_path) -> api_error {
std::string source_path;
auto ret = get_item_meta(from_api_path, META_SOURCE, source_path);
if (ret != api_error::success) {
return ret;
}
std::string encryption_token;
ret = get_item_meta(from_api_path, META_ENCRYPTION_TOKEN, encryption_token);
if (ret != api_error::success) {
return ret;
}
ret = handle_rename_file(from_api_path, to_api_path, source_path);
return ret;
}
auto base_provider::start(api_item_added_callback api_item_added,
i_file_manager *fm) -> bool {
meta_db_ = std::make_unique<meta_db>(config_);
api_item_added_ = api_item_added;
oft_ = oft;
fm_ = fm;
auto unmount_requested = false;
{
repertory::event_consumer ec("unmount_requested",
[&unmount_requested](const event &) { unmount_requested = true; });
for (std::uint16_t i = 0u; not unmount_requested && not is_online() &&
repertory::event_consumer ec(
"unmount_requested",
[&unmount_requested](const event &) { unmount_requested = true; });
for (std::uint16_t i = 0U; not unmount_requested && not is_online() &&
(i < get_config().get_online_check_retry_secs());
i++) {
event_system::instance().raise<provider_offline>(
get_config().get_host_config().host_name_or_ip, get_config().get_host_config().api_port);
get_config().get_host_config().host_name_or_ip,
get_config().get_host_config().api_port);
std::this_thread::sleep_for(1s);
}
}
return unmount_requested;
auto ret = not unmount_requested && is_online();
if (ret) {
// Force root creation
api_meta_map meta{};
auto res = get_item_meta("/", meta);
if (res != api_error::success) {
throw startup_exception("failed to create root|err|" +
api_error_to_string(res));
}
calculate_used_drive_space(false);
}
return ret;
}
void base_provider::stop() { meta_db_.reset(); }
void base_provider::update_filesystem_item(bool directory,
const api_error &error,
const std::string &api_path,
filesystem_item &fsi) const {
if (error == api_error::success) {
fsi.directory = directory;
fsi.api_path = api_path;
fsi.api_parent = utils::path::get_parent_api_path(api_path);
} else {
event_system::instance().raise<filesystem_item_get_failed>(
api_path, std::to_string(static_cast<int>(error)));
}
}
} // namespace repertory

813
src/providers/encrypt/encrypt_provider.cpp Normal file
View File

@@ -0,0 +1,813 @@
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "providers/encrypt/encrypt_provider.hpp"
#include "events/event_system.hpp"
#include "events/events.hpp"
#include "platform/win32_platform.hpp"
#include "types/repertory.hpp"
#include "utils/encrypting_reader.hpp"
#include "utils/path_utils.hpp"
#include "utils/polling.hpp"
#include "utils/rocksdb_utils.hpp"
namespace repertory {
encrypt_provider::encrypt_provider(app_config &config) : config_(config) {}
auto encrypt_provider::create_api_file(const std::string api_path,
bool directory,
const std::string &source_path)
-> api_file {
#ifdef _WIN32
struct _stat64 buf {};
_stat64(source_path.c_str(), &buf);
#else
struct stat buf {};
stat(source_path.c_str(), &buf);
#endif
api_file file{};
file.api_path = api_path;
file.api_parent = utils::path::get_parent_api_path(api_path);
file.file_size =
directory
? 0U
: utils::encryption::encrypting_reader::calculate_encrypted_size(
source_path);
file.source_path = source_path;
#ifdef __APPLE__
file.changed_date =
buf.st_ctimespec.tv_nsec + (buf.st_ctimespec.tv_sec * NANOS_PER_SECOND);
file.accessed_date =
buf.st_atimespec.tv_nsec + (buf.st_atimespec.tv_sec * NANOS_PER_SECOND);
file.creation_date = buf.st_birthtimespec.tv_nsec +
(buf.st_birthtimespec.tv_sec * NANOS_PER_SECOND);
file.modified_date =
buf.st_mtimespec.tv_nsec + (buf.st_mtimespec.tv_sec * NANOS_PER_SECOND);
#elif _WIN32
FILETIME ft{};
utils::unix_time_to_filetime(utils::time64_to_unix_time(buf.st_atime), ft);
file.accessed_date =
(static_cast<std::uint64_t>(ft.dwHighDateTime) << 32U) | ft.dwLowDateTime;
utils::unix_time_to_filetime(utils::time64_to_unix_time(buf.st_mtime), ft);
file.changed_date =
(static_cast<std::uint64_t>(ft.dwHighDateTime) << 32U) | ft.dwLowDateTime;
utils::unix_time_to_filetime(utils::time64_to_unix_time(buf.st_ctime), ft);
file.creation_date =
(static_cast<std::uint64_t>(ft.dwHighDateTime) << 32U) | ft.dwLowDateTime;
utils::unix_time_to_filetime(utils::time64_to_unix_time(buf.st_mtime), ft);
file.modified_date =
(static_cast<std::uint64_t>(ft.dwHighDateTime) << 32U) | ft.dwLowDateTime;
#else
file.changed_date =
buf.st_mtim.tv_nsec + (buf.st_mtim.tv_sec * NANOS_PER_SECOND);
file.accessed_date =
buf.st_atim.tv_nsec + (buf.st_atim.tv_sec * NANOS_PER_SECOND);
file.creation_date =
buf.st_ctim.tv_nsec + (buf.st_ctim.tv_sec * NANOS_PER_SECOND);
file.modified_date =
buf.st_mtim.tv_nsec + (buf.st_mtim.tv_sec * NANOS_PER_SECOND);
#endif
return file;
}
void encrypt_provider::create_item_meta(api_meta_map &meta, bool directory,
const api_file &file) {
#ifdef _WIN32
struct _stat64 buf {};
_stat64(file.source_path.c_str(), &buf);
#else
struct stat buf {};
stat(file.source_path.c_str(), &buf);
#endif
meta[META_ACCESSED] = std::to_string(file.accessed_date);
#ifdef _WIN32
meta[META_ATTRIBUTES] =
std::to_string(::GetFileAttributesA(file.source_path.c_str()));
#endif
#ifdef __APPLE__
meta[META_BACKUP];
#endif
meta[META_CHANGED] = std::to_string(file.changed_date);
meta[META_CREATION] = std::to_string(file.creation_date);
meta[META_DIRECTORY] = utils::string::from_bool(directory);
meta[META_GID] = std::to_string(buf.st_gid);
meta[META_MODE] = std::to_string(buf.st_mode);
meta[META_MODIFIED] = std::to_string(file.modified_date);
#ifdef __APPLE__
meta[META_OSXFLAGS];
#endif
meta[META_SIZE] = std::to_string(file.file_size);
meta[META_SOURCE] = file.source_path;
meta[META_UID] = std::to_string(buf.st_uid);
meta[META_WRITTEN] = std::to_string(file.modified_date);
}
auto encrypt_provider::create_directory(const std::string &api_path,
api_meta_map & /*meta*/) -> api_error {
if (api_path == "/") {
return api_error::success;
}
return api_error::not_implemented;
}
auto encrypt_provider::get_api_path_from_source(const std::string &source_path,
std::string &api_path) const
-> api_error {
try {
std::string api_path_data{};
db_->Get(rocksdb::ReadOptions(), file_family_, source_path, &api_path_data);
if (not api_path_data.empty()) {
api_path = json::parse(api_path_data).at("api_path").get<std::string>();
return api_error::success;
}
std::string dir_api_path{};
db_->Get(rocksdb::ReadOptions(), dir_family_, source_path, &dir_api_path);
if (dir_api_path.empty()) {
return api_error::item_not_found;
}
api_path = dir_api_path;
return api_error::success;
} catch (const std::exception &ex) {
utils::error::raise_error(__FUNCTION__, ex, source_path,
"failed to get api path from source path");
}
return api_error::error;
}
auto encrypt_provider::get_directory_item_count(
const std::string &api_path) const -> std::uint64_t {
std::string source_path{};
db_->Get(rocksdb::ReadOptions(), source_family_, api_path, &source_path);
if (source_path.empty()) {
return 0U;
}
std::string dir_api_path{};
db_->Get(rocksdb::ReadOptions(), dir_family_, source_path, &dir_api_path);
if (dir_api_path.empty()) {
return 0U;
}
const auto cfg = config_.get_encrypt_config();
std::uint64_t count{};
try {
for ([[maybe_unused]] const auto &dir_entry :
std::filesystem::directory_iterator(source_path)) {
count++;
}
} catch (const std::exception &ex) {
utils::error::raise_error(__FUNCTION__, ex, cfg.path,
"failed to get directory item count");
return 0U;
}
return count;
}
auto encrypt_provider::get_directory_items(const std::string &api_path,
directory_item_list &list) const
-> api_error {
bool exists{};
auto res = is_file(api_path, exists);
if (res != api_error::success) {
return res;
}
if (exists) {
return api_error::item_exists;
}
std::string source_path{};
db_->Get(rocksdb::ReadOptions(), source_family_, api_path, &source_path);
if (source_path.empty()) {
return api_error::directory_not_found;
}
std::string dir_api_path{};
db_->Get(rocksdb::ReadOptions(), dir_family_, source_path, &dir_api_path);
if (dir_api_path.empty()) {
return api_error::directory_not_found;
}
try {
for (const auto &dir_entry :
std::filesystem::directory_iterator(source_path)) {
try {
std::string api_path{};
if (dir_entry.is_directory()) {
db_->Get(rocksdb::ReadOptions(), dir_family_,
dir_entry.path().string(), &api_path);
if (api_path.empty()) {
const auto cfg = config_.get_encrypt_config();
for (const auto &child_dir_entry :
std::filesystem::directory_iterator(dir_entry.path())) {
if (process_directory_entry(child_dir_entry, cfg, api_path)) {
api_path = utils::path::get_parent_api_path(api_path);
break;
}
}
if (api_path.empty()) {
continue;
}
}
} else {
std::string api_path_data{};
db_->Get(rocksdb::ReadOptions(), file_family_,
dir_entry.path().string(), &api_path_data);
if (api_path_data.empty()) {
const auto cfg = config_.get_encrypt_config();
if (not process_directory_entry(dir_entry, cfg, api_path)) {
continue;
}
} else {
api_path =
json::parse(api_path_data).at("api_path").get<std::string>();
}
}
auto file = create_api_file(api_path, dir_entry.is_directory(),
dir_entry.path().string());
directory_item di{};
di.api_parent = file.api_parent;
di.api_path = file.api_path;
di.directory = dir_entry.is_directory();
di.resolved = true;
di.size = file.file_size;
create_item_meta(di.meta, di.directory, file);
list.emplace_back(std::move(di));
} catch (const std::exception &ex) {
utils::error::raise_error(__FUNCTION__, ex, dir_entry.path().string(),
"failed to process directory item");
}
}
} catch (const std::exception &ex) {
utils::error::raise_error(__FUNCTION__, ex, source_path,
"failed to get directory items");
return api_error::error;
}
std::sort(list.begin(), list.end(), [](const auto &a, const auto &b) -> bool {
return (a.directory && not b.directory) ||
(not(b.directory && not a.directory) &&
(a.api_path.compare(b.api_path) < 0));
});
list.insert(list.begin(), directory_item{
"..",
"",
true,
});
list.insert(list.begin(), directory_item{
".",
"",
true,
});
return api_error::success;
}
auto encrypt_provider::get_file(const std::string &api_path,
api_file &file) const -> api_error {
bool exists{};
auto res = is_directory(api_path, exists);
if (res != api_error::success) {
return res;
}
if (exists) {
return api_error::directory_exists;
}
std::string source_path{};
db_->Get(rocksdb::ReadOptions(), source_family_, api_path, &source_path);
if (source_path.empty()) {
return api_error::item_not_found;
}
file = create_api_file(api_path, false, source_path);
return api_error::success;
}
auto encrypt_provider::process_directory_entry(
const std::filesystem::directory_entry &dir_entry,
const encrypt_config &cfg, std::string &api_path) const -> bool {
if (dir_entry.is_regular_file() && not dir_entry.is_symlink() &&
not dir_entry.is_directory()) {
const auto relative_path = dir_entry.path().lexically_relative(cfg.path);
std::string api_path_data{};
db_->Get(rocksdb::ReadOptions(), file_family_, dir_entry.path().string(),
&api_path_data);
std::string api_parent{};
db_->Get(rocksdb::ReadOptions(), dir_family_,
dir_entry.path().parent_path().string(), &api_parent);
if (api_path_data.empty() || api_parent.empty()) {
stop_type stop_requested = false;
utils::encryption::encrypting_reader reader(
relative_path.filename().string(), dir_entry.path().string(),
stop_requested, cfg.encryption_token,
relative_path.parent_path().string());
if (api_parent.empty()) {
auto encrypted_parts =
utils::string::split(reader.get_encrypted_file_path(), '/', false);
std::size_t idx{1U};
std::string current_source_path{cfg.path};
std::string current_encrypted_path{};
for (const auto &part : relative_path.parent_path()) {
if (part.string() == "/") {
continue;
}
current_source_path =
utils::path::combine(current_source_path, {part.string()});
std::string parent_api_path{};
db_->Get(rocksdb::ReadOptions(), dir_family_, current_source_path,
&parent_api_path);
if (parent_api_path.empty()) {
parent_api_path = utils::path::create_api_path(
current_encrypted_path + '/' + encrypted_parts[idx]);
db_->Put(rocksdb::WriteOptions(), dir_family_, current_source_path,
parent_api_path);
db_->Put(rocksdb::WriteOptions(), source_family_, parent_api_path,
current_source_path);
event_system::instance().raise<filesystem_item_added>(
parent_api_path,
utils::path::get_parent_api_path(parent_api_path), true);
} else {
encrypted_parts[idx] =
utils::string::split(parent_api_path, '/', false)[idx];
}
current_encrypted_path = utils::path::create_api_path(
current_encrypted_path + '/' + encrypted_parts[idx++]);
}
api_parent = current_encrypted_path;
}
if (api_path_data.empty()) {
api_path = utils::path::create_api_path(
api_parent + "/" + reader.get_encrypted_file_name());
auto iv_list = reader.get_iv_list();
json data = {
{"api_path", api_path},
{"iv_list", iv_list},
{"original_file_size", dir_entry.file_size()},
};
db_->Put(rocksdb::WriteOptions(), file_family_,
dir_entry.path().string(), data.dump());
db_->Put(rocksdb::WriteOptions(), source_family_, api_path,
dir_entry.path().string());
event_system::instance().raise<filesystem_item_added>(
api_path, api_parent, false);
} else {
api_path = json::parse(api_path_data)["api_path"].get<std::string>();
}
} else {
api_path = json::parse(api_path_data)["api_path"].get<std::string>();
}
return true;
}
return false;
}
auto encrypt_provider::get_file_list(api_file_list &list) const -> api_error {
const auto cfg = config_.get_encrypt_config();
try {
for (const auto &dir_entry :
std::filesystem::recursive_directory_iterator(cfg.path)) {
std::string api_path{};
if (process_directory_entry(dir_entry, cfg, api_path)) {
list.emplace_back(create_api_file(api_path, dir_entry.is_directory(),
dir_entry.path().string()));
}
}
return api_error::success;
} catch (const std::exception &ex) {
utils::error::raise_error(__FUNCTION__, ex, cfg.path,
"failed to get file list");
}
return api_error::error;
}
auto encrypt_provider::get_file_size(const std::string &api_path,
std::uint64_t &file_size) const
-> api_error {
std::string source_path{};
db_->Get(rocksdb::ReadOptions(), source_family_, api_path, &source_path);
if (source_path.empty()) {
return api_error::item_not_found;
}
try {
file_size = utils::encryption::encrypting_reader::calculate_encrypted_size(
source_path);
return api_error::success;
} catch (const std::exception &ex) {
utils::error::raise_error(__FUNCTION__, ex, api_path,
"failed to get file size");
}
return api_error::error;
}
auto encrypt_provider::get_filesystem_item(const std::string &api_path,
bool directory,
filesystem_item &fsi) const
-> api_error {
std::string source_path{};
db_->Get(rocksdb::ReadOptions(), source_family_, api_path, &source_path);
if (source_path.empty()) {
return api_error::item_not_found;
}
if (directory) {
std::string api_path{};
db_->Get(rocksdb::ReadOptions(), dir_family_, source_path, &api_path);
if (api_path.empty()) {
return api_error::item_not_found;
}
fsi.api_parent = utils::path::get_parent_api_path(api_path);
fsi.api_path = api_path;
fsi.directory = true;
fsi.size = 0U;
fsi.source_path = source_path;
return api_error::success;
}
std::string api_path_data{};
db_->Get(rocksdb::ReadOptions(), file_family_, source_path, &api_path_data);
if (api_path_data.empty()) {
return api_error::item_not_found;
}
auto data = json::parse(api_path_data);
fsi.api_path = data["api_path"].get<std::string>();
fsi.api_parent = utils::path::get_parent_api_path(fsi.api_path);
fsi.directory = false;
fsi.size = utils::encryption::encrypting_reader::calculate_encrypted_size(
source_path);
fsi.source_path = source_path;
return api_error::success;
}
auto encrypt_provider::get_filesystem_item_from_source_path(
const std::string &source_path, filesystem_item &fsi) const -> api_error {
std::string api_path{};
auto res = get_api_path_from_source(source_path, api_path);
if (res != api_error::success) {
return res;
}
bool exists{};
res = is_directory(api_path, exists);
if (res != api_error::success) {
return res;
}
if (exists) {
return api_error::directory_exists;
}
return get_filesystem_item(api_path, false, fsi);
}
auto encrypt_provider::get_filesystem_item_and_file(const std::string &api_path,
api_file &file,
filesystem_item &fsi) const
-> api_error {
bool exists{};
auto res = is_directory(api_path, exists);
if (res != api_error::success) {
return res;
}
if (exists) {
return api_error::directory_exists;
}
auto ret = get_filesystem_item(api_path, exists, fsi);
if (ret != api_error::success) {
return ret;
}
file = create_api_file(api_path, false, fsi.source_path);
return api_error::success;
}
auto encrypt_provider::get_pinned_files() const -> std::vector<std::string> {
return {};
}
auto encrypt_provider::get_item_meta(const std::string &api_path,
api_meta_map &meta) const -> api_error {
std::string source_path{};
db_->Get(rocksdb::ReadOptions(), source_family_, api_path, &source_path);
if (source_path.empty()) {
return api_error::item_not_found;
}
bool exists{};
auto res = is_directory(api_path, exists);
if (res != api_error::success) {
return res;
}
auto file = create_api_file(api_path, exists, source_path);
create_item_meta(meta, exists, file);
return api_error::success;
}
auto encrypt_provider::get_item_meta(const std::string &api_path,
const std::string &key,
std::string &value) const -> api_error {
api_meta_map meta{};
auto ret = get_item_meta(api_path, meta);
if (ret != api_error::success) {
return ret;
}
value = meta[key];
return api_error::success;
}
auto encrypt_provider::get_total_drive_space() const -> std::uint64_t {
const auto cfg = config_.get_encrypt_config();
return utils::file::get_total_drive_space(cfg.path);
}
auto encrypt_provider::get_total_item_count() const -> std::uint64_t {
std::uint64_t ret{};
auto iterator = std::unique_ptr<rocksdb::Iterator>(
db_->NewIterator(rocksdb::ReadOptions(), source_family_));
for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
ret++;
}
return ret;
}
auto encrypt_provider::get_used_drive_space() const -> std::uint64_t {
const auto cfg = config_.get_encrypt_config();
return get_total_drive_space() - utils::file::get_free_drive_space(cfg.path);
}
auto encrypt_provider::is_directory(const std::string &api_path,
bool &exists) const -> api_error {
std::string source_path{};
db_->Get(rocksdb::ReadOptions(), source_family_, api_path, &source_path);
if (source_path.empty()) {
exists = false;
return api_error::success;
}
exists = utils::file::is_directory(source_path);
return api_error::success;
}
auto encrypt_provider::is_file(const std::string &api_path, bool &exists) const
-> api_error {
std::string source_path{};
db_->Get(rocksdb::ReadOptions(), source_family_, api_path, &source_path);
if (source_path.empty()) {
exists = false;
return api_error::success;
}
exists = utils::file::is_file(source_path);
return api_error::success;
}
auto encrypt_provider::is_file_writeable(const std::string & /*api_path*/) const
-> bool {
return false;
}
auto encrypt_provider::is_online() const -> bool {
return std::filesystem::exists(config_.get_encrypt_config().path);
}
auto encrypt_provider::is_rename_supported() const -> bool { return false; }
auto encrypt_provider::read_file_bytes(const std::string &api_path,
std::size_t size, std::uint64_t offset,
data_buffer &data,
stop_type &stop_requested) -> api_error {
std::string source_path{};
db_->Get(rocksdb::ReadOptions(), source_family_, api_path, &source_path);
if (source_path.empty()) {
return api_error::item_not_found;
}
std::string api_path_data{};
db_->Get(rocksdb::ReadOptions(), file_family_, source_path, &api_path_data);
if (api_path_data.empty()) {
return api_error::item_not_found;
}
std::uint64_t file_size{};
if (not utils::file::get_file_size(source_path, file_size)) {
return api_error::os_error;
}
std::vector<
std::array<unsigned char, crypto_aead_xchacha20poly1305_IETF_NPUBBYTES>>
iv_list{};
const auto cfg = config_.get_encrypt_config();
unique_recur_mutex_lock reader_lookup_lock(reader_lookup_mtx_);
auto file_data = json::parse(api_path_data);
if (file_data.at("original_file_size").get<std::uint64_t>() != file_size) {
const auto relative_path =
std::filesystem::path(source_path).lexically_relative(cfg.path);
auto ri = std::make_shared<reader_info>();
ri->reader = std::make_unique<utils::encryption::encrypting_reader>(
relative_path.filename().string(), source_path, stop_requested,
cfg.encryption_token, relative_path.parent_path().string());
reader_lookup_[source_path] = ri;
iv_list = ri->reader->get_iv_list();
file_data["original_file_size"] = file_size;
file_data["iv_list"] = iv_list;
auto res = db_->Put(rocksdb::WriteOptions(), file_family_, source_path,
file_data.dump());
if (not res.ok()) {
utils::error::raise_error(__FUNCTION__, res.code(), source_path,
"failed to update meta db");
return api_error::error;
}
} else {
iv_list =
file_data["iv_list"]
.get<std::vector<
std::array<unsigned char,
crypto_aead_xchacha20poly1305_IETF_NPUBBYTES>>>();
if (reader_lookup_.find(source_path) == reader_lookup_.end()) {
auto ri = std::make_shared<reader_info>();
ri->reader = std::make_unique<utils::encryption::encrypting_reader>(
api_path, source_path, stop_requested, cfg.encryption_token,
std::move(iv_list));
reader_lookup_[source_path] = ri;
}
}
if (file_size == 0U || size == 0U) {
return api_error::success;
}
auto ri = reader_lookup_.at(source_path);
ri->last_access_time = std::chrono::system_clock::now();
reader_lookup_lock.unlock();
mutex_lock reader_lock(ri->reader_mtx);
ri->reader->set_read_position(offset);
data.resize(size);
const auto res = ri->reader->reader_function(data.data(), 1u, data.size(),
ri->reader.get());
if (res == 0) {
return api_error::os_error;
}
return api_error::success;
}
void encrypt_provider::remove_deleted_files() {
struct removed_item {
std::string api_path{};
bool directory{};
std::string source_path{};
};
std::vector<removed_item> removed_list{};
auto iterator = std::unique_ptr<rocksdb::Iterator>(
db_->NewIterator(rocksdb::ReadOptions(), source_family_));
for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
auto source_path = iterator->value().ToString();
if (not std::filesystem::exists(source_path)) {
auto api_path =
utils::string::split(iterator->key().ToString(), '|', false)[1U];
std::string value{};
db_->Get(rocksdb::ReadOptions(), file_family_, source_path, &value);
removed_list.emplace_back(
removed_item{api_path, value.empty(), source_path});
}
}
for (const auto &item : removed_list) {
if (not item.directory) {
db_->Delete(rocksdb::WriteOptions(), source_family_, item.api_path);
db_->Delete(rocksdb::WriteOptions(), file_family_, item.source_path);
event_system::instance().raise<file_removed_externally>(item.api_path,
item.source_path);
}
}
for (const auto &item : removed_list) {
if (item.directory) {
db_->Delete(rocksdb::WriteOptions(), source_family_, item.api_path);
db_->Delete(rocksdb::WriteOptions(), dir_family_, item.source_path);
event_system::instance().raise<directory_removed_externally>(
item.api_path, item.source_path);
}
}
}
auto encrypt_provider::start(api_item_added_callback /*api_item_added*/,
i_file_manager * /*fm*/) -> bool {
if (not is_online()) {
return false;
}
auto families = std::vector<rocksdb::ColumnFamilyDescriptor>();
families.emplace_back(rocksdb::kDefaultColumnFamilyName,
rocksdb::ColumnFamilyOptions());
families.emplace_back("dir", rocksdb::ColumnFamilyOptions());
families.emplace_back("file", rocksdb::ColumnFamilyOptions());
families.emplace_back("source", rocksdb::ColumnFamilyOptions());
auto handles = std::vector<rocksdb::ColumnFamilyHandle *>();
utils::db::create_rocksdb(config_, DB_NAME, families, handles, db_);
std::size_t idx{};
dir_family_ = handles[idx++];
file_family_ = handles[idx++];
source_family_ = handles[idx++];
const auto cfg = config_.get_encrypt_config();
std::string source_path{};
db_->Get(rocksdb::ReadOptions(), source_family_, "/", &source_path);
if (source_path.empty()) {
db_->Put(rocksdb::WriteOptions(), source_family_, "/", cfg.path);
source_path = cfg.path;
}
std::string dir_api_path{};
db_->Get(rocksdb::ReadOptions(), dir_family_, source_path, &dir_api_path);
if (dir_api_path.empty()) {
db_->Put(rocksdb::WriteOptions(), dir_family_, source_path, "/");
}
polling::instance().set_callback({"check_deleted", polling::frequency::low,
[this]() { remove_deleted_files(); }});
event_system::instance().raise<service_started>("encrypt_provider");
return true;
}
void encrypt_provider::stop() {
event_system::instance().raise<service_shutdown_begin>("encrypt_provider");
polling::instance().remove_callback("check_deleted");
db_.reset();
event_system::instance().raise<service_shutdown_end>("encrypt_provider");
}
} // namespace repertory

192
src/providers/passthrough/passthroughprovider.cpp
View File

@@ -1,192 +0,0 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#if defined(REPERTORY_TESTING_NEW)
#include "providers/passthrough/passthroughprovider.hpp"
#include "utils/file_utils.hpp"
#include "utils/path_utils.hpp"
namespace repertory {
std::string CPassthroughProvider::ConstructFullPath(const std::string &api_path) const {
return utils::path::combine(passthroughLocation_, {api_path});
}
api_error CPassthroughProvider::CreateDirectory(const std::string &api_path,
const api_meta_map &meta) {
const auto fullPath = ConstructFullPath(api_path);
auto ret = utils::file::create_full_directory_path(fullPath) ? api_error::OSErrorCode
: api_error::Success;
if (ret == api_error::Success) {
ret = set_item_meta(api_path, meta);
}
return ret;
}
api_error CPassthroughProvider::get_file_list(ApiFileList &fileList) const {
const auto fullPath = ConstructFullPath("/");
const auto fl = utils::file::get_directory_files(fullPath, false, true);
for (const auto &file : fl) {
const auto api_path = utils::path::create_api_path(file.substr(fullPath.length()));
/*
struct ApiFile {
std::string ApiFilePath{};
std::string ApiParent{};
std::uint64_t AccessedDate = 0;
std::uint64_t ChangedDate = 0;
std::uint64_t CreationDate = 0;
std::string EncryptionToken{};
std::uint64_t FileSize = 0;
std::uint64_t ModifiedDate = 0;
bool Recoverable = false;
double Redundancy = 0.0;
std::string SourceFilePath{};
};
*/
ApiFile apiFile{
api_path,
utils::path::get_parent_api_path(api_path),
};
// apiFile.Recoverable = not IsProcessing(api_path);
apiFile.Redundancy = 3.0;
apiFile.SourceFilePath = file;
// utils::file::UpdateApiFileInfo(apiFile);
fileList.emplace_back(apiFile);
}
return api_error::Success;
}
std::uint64_t CPassthroughProvider::get_directory_item_count(const std::string &api_path) const {
const auto fullPath = ConstructFullPath(api_path);
return 0;
}
api_error CPassthroughProvider::get_directory_items(const std::string &api_path,
directory_item_list &list) const {
return api_error::NotImplemented;
}
api_error CPassthroughProvider::GetFile(const std::string &api_path, ApiFile &file) const {
return api_error::Error;
}
api_error CPassthroughProvider::GetFileSize(const std::string &api_path,
std::uint64_t &fileSize) const {
return api_error::Error;
}
api_error CPassthroughProvider::get_filesystem_item(const std::string &api_path,
const bool &directory,
FileSystemItem &fileSystemItem) const {
return api_error::NotImplemented;
}
api_error
CPassthroughProvider::get_filesystem_item_from_source_path(const std::string &sourceFilePath,
FileSystemItem &fileSystemItem) const {
return api_error::NotImplemented;
}
api_error CPassthroughProvider::get_item_meta(const std::string &api_path,
api_meta_map &meta) const {
return api_error::NotImplemented;
}
api_error CPassthroughProvider::get_item_meta(const std::string &api_path,
const std::string &key, std::string &value) const {
return api_error::NotImplemented;
}
std::uint64_t CPassthroughProvider::get_total_drive_space() const { return 0; }
std::uint64_t CPassthroughProvider::get_total_item_count() const { return 0; }
std::uint64_t CPassthroughProvider::get_used_drive_space() const { return 0; }
bool CPassthroughProvider::IsDirectory(const std::string &api_path) const {
const auto fullPath = ConstructFullPath(api_path);
return utils::file::is_directory(fullPath);
}
bool CPassthroughProvider::IsFile(const std::string &api_path) const {
const auto fullPath = ConstructFullPath(api_path);
return utils::file::is_file(fullPath);
}
api_error CPassthroughProvider::notify_file_added(const std::string &api_path,
const std::string &api_parent,
const std::uint64_t &size) {
return api_error::NotImplemented;
}
api_error CPassthroughProvider::read_file_bytes(const std::string &apiFilepath,
const std::size_t &size,
const std::uint64_t &offset,
std::vector<char> &data,
const bool &stop_requested) {
return api_error::NotImplemented;
}
api_error CPassthroughProvider::RemoveDirectory(const std::string &api_path) {
return api_error::NotImplemented;
}
api_error CPassthroughProvider::RemoveFile(const std::string &api_path) {
return api_error::NotImplemented;
}
api_error CPassthroughProvider::RenameFile(const std::string &fromApiPath,
const std::string &toApiPath) {
return api_error::NotImplemented;
}
api_error CPassthroughProvider::remove_item_meta(const std::string &api_path,
const std::string &key) {
return api_error::NotImplemented;
}
api_error CPassthroughProvider::set_item_meta(const std::string &api_path,
const std::string &key, const std::string &value) {
return api_error::NotImplemented;
}
api_error CPassthroughProvider::set_item_meta(const std::string &api_path,
const api_meta_map &meta) {
return api_error::NotImplemented;
}
api_error CPassthroughProvider::set_source_path(const std::string &api_path,
const std::string &sourcePath) {
return api_error::NotImplemented;
}
bool CPassthroughProvider::Start(ApiItemAdded apiItemAdded, i_open_file_table *openFileTable) {
return false;
}
void CPassthroughProvider::Stop() {}
api_error CPassthroughProvider::upload_file(const std::string &api_path,
const std::string &sourcePath,
const std::string &encryptionToken) {
return api_error::NotImplemented;
}
} // namespace repertory
#endif // defined(REPERTORY_TESTING_NEW)

94
src/providers/provider.cpp
View File

@@ -1,76 +1,82 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "providers/provider.hpp"
#include "comm/aws_s3/aws_s3_comm.hpp"
#include "comm/curl/curl_comm.hpp"
#include "comm/i_comm.hpp"
#include "comm/i_s3_comm.hpp"
#include "app_config.hpp"
#include "comm/curl/curl_comm.hpp"
#include "comm/i_http_comm.hpp"
#include "comm/i_s3_comm.hpp"
#include "comm/s3/s3_comm.hpp"
#include "events/events.hpp"
#include "providers/passthrough/passthroughprovider.hpp"
#include "providers/encrypt/encrypt_provider.hpp"
#include "providers/s3/s3_provider.hpp"
#include "providers/sia/sia_provider.hpp"
#include "providers/skynet/skynet_provider.hpp"
#include "types/startup_exception.hpp"
namespace repertory {
template <typename i, typename t>
static void create_comm(std::unique_ptr<i> &comm, app_config &config) {
namespace {
template <typename intf_t, typename comm_t, typename config_t>
inline void create_comm(std::unique_ptr<intf_t> &comm, const config_t &config) {
if (comm) {
throw startup_exception("'create_provider' should only be called once");
throw repertory::startup_exception(
"'create_provider' should only be called once");
}
comm = std::make_unique<t>(config);
}
std::unique_ptr<i_provider> create_provider(const provider_type &pt, app_config &config) {
comm = std::make_unique<comm_t>(config);
}
} // namespace
namespace repertory {
auto create_provider(const provider_type &pt, app_config &config)
-> std::unique_ptr<i_provider> {
static std::mutex mutex;
mutex_lock lock(mutex);
static std::unique_ptr<i_comm> comm;
static std::unique_ptr<i_http_comm> comm;
#if defined(REPERTORY_ENABLE_S3)
static std::unique_ptr<i_s3_comm> s3_comm;
static std::unique_ptr<i_s3_comm> s3_comm_;
#endif // defined(REPERTORY_ENABLE_S3)
switch (pt) {
case provider_type::sia:
create_comm<i_comm, curl_comm>(comm, config);
return std::unique_ptr<i_provider>(dynamic_cast<i_provider *>(new sia_provider(config, *comm)));
case provider_type::sia: {
create_comm<i_http_comm, curl_comm, host_config>(comm,
config.get_host_config());
return std::unique_ptr<i_provider>(
dynamic_cast<i_provider *>(new sia_provider(config, *comm)));
}
#if defined(REPERTORY_ENABLE_S3)
case provider_type::s3:
create_comm<i_s3_comm, aws_s3_comm>(s3_comm, config);
case provider_type::s3: {
create_comm<i_s3_comm, s3_comm, app_config>(s3_comm_, config);
return std::unique_ptr<i_provider>(
dynamic_cast<i_provider *>(new s3_provider(config, *s3_comm)));
dynamic_cast<i_provider *>(new s3_provider(config, *s3_comm_)));
}
#endif // defined(REPERTORY_ENABLE_S3)
#if defined(REPERTORY_ENABLE_SKYNET)
case provider_type::skynet:
create_comm<i_comm, curl_comm>(comm, config);
case provider_type::encrypt: {
return std::unique_ptr<i_provider>(
dynamic_cast<i_provider *>(new skynet_provider(config, *comm)));
#endif // defined(REPERTORY_ENABLE_SKYNET)
#if defined(REPERTORY_TESTING_NEW)
case provider_type::passthrough:
return std::unique_ptr<i_provider>(
dynamic_cast<i_provider *>(new CPassthroughProvider(config)));
#endif // defined(REPERTORY_TESTING_NEW)
dynamic_cast<i_provider *>(new encrypt_provider(config)));
}
case provider_type::unknown:
default:
throw startup_exception("provider not supported: " + app_config::get_provider_display_name(pt));
throw startup_exception("provider not supported: " +
app_config::get_provider_display_name(pt));
}
}
} // namespace repertory

956
src/providers/s3/s3_provider.cpp
View File

File diff suppressed because it is too large Load Diff

1826
src/providers/sia/sia_provider.cpp
View File

File diff suppressed because it is too large Load Diff

649
src/providers/skynet/skynet_provider.cpp
View File

@@ -1,649 +0,0 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#if defined(REPERTORY_ENABLE_SKYNET)
#include "providers/skynet/skynet_provider.hpp"
#include "comm/i_comm.hpp"
#include "app_config.hpp"
#include "drives/i_open_file_table.hpp"
#include "events/events.hpp"
#include "types/repertory.hpp"
#include "types/skynet.hpp"
#include "types/startup_exception.hpp"
#include "utils/encryption.hpp"
#include "utils/encrypting_reader.hpp"
#include "utils/file_utils.hpp"
#include "utils/global_data.hpp"
#include "utils/path_utils.hpp"
namespace repertory {
skynet_provider::skynet_provider(app_config &config, i_comm &comm)
: base_provider(config),
comm_(comm),
directory_db_(config),
upload_manager_(
config, [this](const std::string &api_path) -> bool { return this->is_file(api_path); },
[this](const upload_manager::upload &upload, json &data, json &error) -> api_error {
return this->upload_handler(upload, data, error);
},
[this](const std::string &api_path, const std::string &source_path, const json &data) {
return this->upload_completed(api_path, source_path, data);
}) {
next_download_index_ = 0u;
next_upload_index_ = 0u;
update_portal_list();
E_SUBSCRIBE_EXACT(skynet_portal_list_changed,
[this](const skynet_portal_list_changed &) { this->update_portal_list(); });
// Remove legacy encrypted files
api_file_list list;
get_file_list(list);
for (const auto &file : list) {
std::string token;
get_item_meta(file.api_path, "token", token);
if (not token.empty()) {
remove_file(file.api_path);
}
}
}
api_error skynet_provider::create_directory(const std::string &api_path, const api_meta_map &meta) {
auto ret = api_error::success;
if (utils::path::is_trash_directory(api_path)) {
ret = api_error::access_denied;
} else {
#ifdef _WIN32
ret = is_directory(api_path) ? api_error::directory_exists
: is_file(api_path) ? api_error::file_exists
: api_error::success;
if (ret != api_error::success) {
return ret;
}
#endif
if ((ret = directory_db_.create_directory(api_path)) != api_error::success) {
return ret;
}
set_item_meta(api_path, meta);
}
return ret;
}
api_error skynet_provider::create_file(const std::string &api_path, api_meta_map &meta) {
if (meta[META_SIZE].empty()) {
meta[META_SIZE] = "0";
}
// When META_ID is present, an external import is occurring.
// Need to skip the encryption token in this scenario.
if (meta[META_ID].empty() && meta[META_ENCRYPTION_TOKEN].empty()) {
meta[META_ENCRYPTION_TOKEN] = get_config().get_skynet_config().encryption_token;
}
auto ret = base_provider::create_file(api_path, meta);
if (ret == api_error::success) {
ret = directory_db_.create_file(api_path);
}
return ret;
}
json skynet_provider::export_all() const {
json ret = {{"success", std::vector<json>()}, {"failed", std::vector<std::string>()}};
api_file_list list;
get_file_list(list);
for (const auto &file : list) {
process_export(ret, file.api_path);
}
return ret;
}
json skynet_provider::export_list(const std::vector<std::string> &api_path_list) const {
json ret = {{"success", std::vector<json>()}, {"failed", std::vector<std::string>()}};
for (const auto &api_path : api_path_list) {
process_export(ret, api_path);
}
return ret;
}
std::uint64_t skynet_provider::get_directory_item_count(const std::string &api_path) const {
return is_directory(api_path) ? directory_db_.get_directory_item_count(api_path) : 0u;
}
api_error skynet_provider::get_directory_items(const std::string &api_path,
directory_item_list &list) const {
if (is_file(api_path)) {
return api_error::item_is_file;
}
if (not is_directory(api_path)) {
const_cast<skynet_provider *>(this)->remove_item_meta(api_path);
return api_error::directory_not_found;
}
directory_db_.populate_sub_directories(
api_path,
[this](directory_item &di, const bool &) {
this->get_item_meta(di.api_path, di.meta);
this->oft_->update_directory_item(di);
},
list);
directory_db_.populate_directory_files(
api_path,
[this](directory_item &di, const bool &) {
di.api_parent = utils::path::get_parent_api_path(di.api_path);
this->get_item_meta(di.api_path, di.meta);
this->oft_->update_directory_item(di);
},
list);
std::sort(list.begin(), list.end(), [](const auto &a, const auto &b) -> bool {
return (a.directory && not b.directory) ? true
: (b.directory && not a.directory) ? false
: (a.api_path.compare(b.api_path) < 0);
});
list.insert(list.begin(), directory_item{
"..",
"",
true,
});
list.insert(list.begin(), directory_item{
".",
"",
true,
});
return api_error::success;
}
api_error skynet_provider::get_file(const std::string &api_path, api_file &file) const {
const auto ret =
directory_db_.get_file(api_path, file, [this](api_file &file) { populate_api_file(file); });
if (ret != api_error::success) {
if (not is_directory(api_path)) {
const_cast<skynet_provider *>(this)->remove_item_meta(api_path);
}
event_system::instance().raise<file_get_failed>(api_path,
std::to_string(static_cast<int>(ret)));
}
return ret;
}
api_error skynet_provider::get_file_list(api_file_list &list) const {
const auto ret =
directory_db_.get_file_list(list, [this](api_file &file) { populate_api_file(file); });
if (ret != api_error::success) {
event_system::instance().raise<file_get_api_list_failed>(std::to_string(static_cast<int>(ret)));
}
return ret;
}
api_error skynet_provider::get_file_size(const std::string &api_path,
std::uint64_t &file_size) const {
api_file file{};
const auto ret = get_file(api_path, file);
if (ret == api_error::success) {
file_size = file.file_size;
} else {
event_system::instance().raise<file_get_size_failed>(api_path,
std::to_string(static_cast<int>(ret)));
}
return ret;
}
host_config skynet_provider::get_host_config(const bool &upload) {
const auto format_host_config = [&](host_config hc) {
hc.path = upload ? "/skynet/skyfile" : "/";
return hc;
};
unique_mutex_lock portal_lock(portal_mutex_);
const auto portal_list = upload ? upload_list_ : download_list_;
portal_lock.unlock();
auto &next_upload_index = upload ? next_upload_index_ : next_download_index_;
auto idx = next_upload_index++;
if (idx >= portal_list->size()) {
idx = next_upload_index = 0u;
}
return format_host_config((*portal_list)[idx]);
}
std::size_t skynet_provider::get_retry_count() const {
mutex_lock portal_lock(portal_mutex_);
return std::max(download_list_->size(),
static_cast<std::size_t>(get_config().get_retry_read_count()));
}
api_error skynet_provider::get_skynet_metadata(const std::string &skylink, json &json_meta) {
auto ret = api_error::error;
http_headers headers;
const auto retry_count = get_retry_count();
for (std::size_t i = 0u; (ret != api_error::success) && (i < retry_count); i++) {
json data, error;
const auto hc = get_host_config(false);
if (comm_.get(hc, "/skynet/metadata/" + skylink, data, error) == api_error::success) {
headers["skynet-file-metadata"] = data.dump();
ret = api_error::success;
} else {
std::vector<char> buffer;
if (comm_.get_range_and_headers(hc, utils::path::create_api_path(skylink), 0u,
{{"format", "concat"}}, "", buffer, {{0, 0}}, error, headers,
stop_requested_) == api_error::success) {
ret = api_error::success;
} else if (not error.empty()) {
event_system::instance().raise<repertory_exception>(__FUNCTION__, error.dump(2));
}
}
}
if (ret == api_error::success) {
json_meta = json::parse(headers["skynet-file-metadata"]);
if (json_meta["subfiles"].empty()) {
auto sub_file = json_meta;
sub_file["len"] =
utils::string::to_uint64(utils::string::split(headers["content-range"], '/')[1]);
json sub_files = {{json_meta["filename"], sub_file}};
json_meta["subfiles"] = sub_files;
}
}
return ret;
}
api_error skynet_provider::import_skylink(const skylink_import &si) {
json json_meta;
auto ret = get_skynet_metadata(si.skylink, json_meta);
if (ret == api_error::success) {
const auto encrypted = not si.token.empty();
for (auto sub_file : json_meta["subfiles"]) {
const auto meta_file_name = sub_file["filename"].get<std::string>();
auto file_name = meta_file_name;
if (encrypted) {
if ((ret = utils::encryption::decrypt_file_name(si.token, file_name)) !=
api_error::success) {
event_system::instance().raise<skynet_import_decryption_failed>(
si.skylink, sub_file["filename"], ret);
}
}
if (ret == api_error::success) {
const auto api_path =
utils::path::create_api_path(utils::path::combine(si.directory, {file_name}));
const auto api_parent = utils::path::get_parent_api_path(api_path);
const auto parts = utils::string::split(api_parent, '/', false);
std::string sub_directory = "/";
for (std::size_t i = 0u; (ret == api_error::success) && (i < parts.size()); i++) {
sub_directory =
utils::path::create_api_path(utils::path::combine(sub_directory, {parts[i]}));
if (not is_directory(sub_directory)) {
if ((ret = directory_db_.create_directory(sub_directory)) == api_error::success) {
base_provider::notify_directory_added(
sub_directory, utils::path::get_parent_api_path(sub_directory));
} else {
event_system::instance().raise<skynet_import_directory_failed>(si.skylink,
sub_directory, ret);
}
}
}
if (ret == api_error::success) {
auto file_size = sub_file["len"].get<std::uint64_t>();
if (encrypted) {
file_size = utils::encryption::encrypting_reader::calculate_decrypted_size(file_size);
}
const auto skylink =
si.skylink + ((json_meta["filename"].get<std::string>() == meta_file_name)
? ""
: "/" + meta_file_name);
api_meta_map meta{};
meta[META_ID] = json({{"skylink", skylink}}).dump();
meta[META_ENCRYPTION_TOKEN] = si.token;
if ((ret = create_file(api_path, meta)) == api_error::success) {
const auto now = utils::get_file_time_now();
api_item_added_(api_path, api_parent, "", false, now, now, now, now);
if (file_size > 0u) {
set_item_meta(api_path, META_SIZE, std::to_string(file_size));
global_data::instance().increment_used_drive_space(file_size);
}
} else {
event_system::instance().raise<skynet_import_file_failed>(si.skylink, api_path, ret);
}
}
}
}
}
return ret;
}
bool skynet_provider::is_directory(const std::string &api_path) const {
return (api_path == "/") || directory_db_.is_directory(api_path);
}
bool skynet_provider::is_file(const std::string &api_path) const {
return (api_path != "/") && directory_db_.is_file(api_path);
}
bool skynet_provider::is_file_writeable(const std::string &api_path) const {
auto ret = true;
std::string id;
get_item_meta(api_path, META_ID, id);
if (not id.empty()) {
try {
const auto skynet_data = json::parse(id);
const auto skylink = skynet_data["skylink"].get<std::string>();
ret = not utils::string::contains(skylink, "/");
} catch (const std::exception &e) {
event_system::instance().raise<repertory_exception>(
__FUNCTION__, e.what() ? e.what() : "exception occurred");
ret = false;
}
}
return ret;
}
bool skynet_provider::is_processing(const std::string &api_path) const {
return upload_manager_.is_processing(api_path);
}
void skynet_provider::notify_directory_added(const std::string &api_path,
const std::string &api_parent) {
if (api_path == "/") {
if (directory_db_.create_directory("/") == api_error::success) {
base_provider::notify_directory_added(api_path, api_parent);
}
}
}
api_error skynet_provider::notify_file_added(const std::string &, const std::string &,
const std::uint64_t &) {
return api_error::not_implemented;
}
void skynet_provider::populate_api_file(api_file &file) const {
api_meta_map meta{};
this->get_item_meta(file.api_path, meta);
file.api_parent = utils::path::get_parent_api_path(file.api_path);
file.accessed_date = utils::string::to_uint64(meta[META_ACCESSED]);
file.changed_date = utils::string::to_uint64(meta[META_MODIFIED]);
file.created_date = utils::string::to_uint64(meta[META_CREATION]);
file.encryption_token = meta[META_ENCRYPTION_TOKEN].empty() ? "" : meta[META_ENCRYPTION_TOKEN];
file.file_size = utils::string::to_uint64(meta[META_SIZE]);
file.modified_date = utils::string::to_uint64(meta[META_MODIFIED]);
file.recoverable = not is_processing(file.api_path);
file.redundancy = 3.0;
file.source_path = meta[META_SOURCE];
}
void skynet_provider::process_export(json &result, const std::string &api_path) const {
try {
std::string id;
std::string token;
if (is_file(api_path) && (get_item_meta(api_path, META_ID, id) == api_error::success) &&
(get_item_meta(api_path, META_ENCRYPTION_TOKEN, token) == api_error::success)) {
auto directory = utils::path::get_parent_api_path(api_path);
const auto skylink = json::parse(id)["skylink"].get<std::string>();
if (utils::string::contains(skylink, "/")) {
const auto pos = skylink.find('/');
const auto path =
utils::path::create_api_path(utils::path::remove_file_name(skylink.substr(pos)));
if (path != "/") {
directory =
utils::path::create_api_path(directory.substr(0, directory.length() - path.length()));
}
}
result["success"].emplace_back(
json({{"skylink", skylink},
{"token", token},
{"directory", directory},
{"filename", utils::path::strip_to_file_name(api_path)}}));
} else {
result["failed"].emplace_back(api_path);
}
} catch (const std::exception &e) {
result["failed"].emplace_back(api_path);
event_system::instance().raise<repertory_exception>(
__FUNCTION__, e.what() ? e.what() : "export failed: " + api_path);
}
}
api_error skynet_provider::read_file_bytes(const std::string &api_path, const std::size_t &size,
const std::uint64_t &offset, std::vector<char> &data,
const bool &stop_requested) {
if (size == 0u) {
return api_error::success;
}
std::string id;
auto ret = get_item_meta(api_path, META_ID, id);
if (ret == api_error::success) {
ret = api_error::download_failed;
const auto skynet_data = json::parse(id);
const auto path = utils::path::create_api_path(skynet_data["skylink"].get<std::string>());
const auto ranges = http_ranges({{offset, offset + size - 1}});
std::string encryption_token;
get_item_meta(api_path, META_ENCRYPTION_TOKEN, encryption_token);
std::uint64_t file_size{};
{
std::string temp;
get_item_meta(api_path, META_SIZE, temp);
file_size = utils::string::to_uint64(temp);
}
const auto retry_count = get_retry_count();
for (std::size_t i = 0u; not stop_requested && (ret != api_error::success) && (i < retry_count);
i++) {
json error;
ret = (comm_.get_range(get_host_config(false), path, file_size, {{"format", "concat"}},
encryption_token, data, ranges, error,
stop_requested) == api_error::success)
? api_error::success
: api_error::download_failed;
if (ret != api_error::success) {
event_system::instance().raise<file_read_bytes_failed>(api_path, error.dump(2), i + 1);
}
}
}
return ret;
}
api_error skynet_provider::remove_directory(const std::string &api_path) {
const auto ret = directory_db_.remove_directory(api_path);
if (ret == api_error::success) {
remove_item_meta(api_path);
event_system::instance().raise<directory_removed>(api_path);
} else {
event_system::instance().raise<directory_remove_failed>(api_path,
std::to_string(static_cast<int>(ret)));
}
return ret;
}
api_error skynet_provider::remove_file(const std::string &api_path) {
upload_manager_.remove_upload(api_path);
const auto ret =
directory_db_.remove_file(api_path) ? api_error::success : api_error::item_not_found;
if (ret == api_error::success) {
remove_item_meta(api_path);
event_system::instance().raise<file_removed>(api_path);
} else {
event_system::instance().raise<file_remove_failed>(api_path,
std::to_string(static_cast<int>(ret)));
}
return ret;
}
api_error skynet_provider::rename_file(const std::string &from_api_path,
const std::string &to_api_path) {
std::string id;
auto ret = get_item_meta(from_api_path, META_ID, id);
if (ret == api_error::success) {
ret = api_error::access_denied;
const auto skynet_data = json::parse(id.empty() ? R"({"skylink":""})" : id);
const auto skylink = skynet_data["skylink"].get<std::string>();
if (utils::string::contains(skylink, "/")) {
const auto pos = skylink.find('/');
const auto len = skylink.size() - pos;
if (to_api_path.size() >= len) {
const auto comp1 = to_api_path.substr(to_api_path.size() - len);
const auto comp2 = skylink.substr(pos);
ret = (comp1 == comp2) ? api_error::success : ret;
}
} else {
ret = (skylink.empty() || (utils::path::strip_to_file_name(from_api_path) ==
utils::path::strip_to_file_name(to_api_path)))
? api_error::success
: ret;
}
if (ret == api_error::success) {
std::string current_source;
if ((ret = get_item_meta(from_api_path, META_SOURCE, current_source)) == api_error::success) {
if (not upload_manager_.execute_if_not_processing({from_api_path, to_api_path}, [&]() {
if ((ret = directory_db_.rename_file(from_api_path, to_api_path)) ==
api_error::success) {
meta_db_.rename_item_meta(current_source, from_api_path, to_api_path);
}
})) {
ret = api_error::file_in_use;
}
}
}
}
return ret;
}
bool skynet_provider::start(api_item_added_callback api_item_added, i_open_file_table *oft) {
const auto ret = base_provider::start(api_item_added, oft);
if (not ret) {
api_file_list list;
if (get_file_list(list) != api_error::success) {
throw startup_exception("failed to determine used space");
}
const auto total_size =
std::accumulate(list.begin(), list.end(), std::uint64_t(0),
[](std::uint64_t t, const api_file &file) { return t + file.file_size; });
global_data::instance().initialize_used_drive_space(total_size);
upload_manager_.start();
}
return ret;
}
void skynet_provider::stop() {
stop_requested_ = true;
upload_manager_.stop();
}
bool skynet_provider::update_portal_list() {
auto portal_list = get_config().get_skynet_config().portal_list;
auto download_portal_list = std::make_shared<std::vector<host_config>>();
auto upload_portal_list = std::make_shared<std::vector<host_config>>();
std::copy_if(portal_list.begin(), portal_list.end(), std::back_inserter(*upload_portal_list),
[](const auto &portal) -> bool {
return not portal.auth_url.empty() && not portal.auth_user.empty();
});
for (const auto &portal : portal_list) {
if (upload_portal_list->empty()) {
upload_portal_list->emplace_back(portal);
}
download_portal_list->emplace_back(portal);
}
unique_mutex_lock portal_lock(portal_mutex_);
download_list_ = std::move(download_portal_list);
upload_list_ = std::move(upload_portal_list);
portal_lock.unlock();
return true;
}
void skynet_provider::upload_completed(const std::string &api_path, const std::string &,
const json &data) {
set_item_meta(api_path, META_ID, data.dump());
}
api_error skynet_provider::upload_file(const std::string &api_path, const std::string &source_path,
const std::string &encryption_token) {
std::uint64_t file_size = 0u;
utils::file::get_file_size(source_path, file_size);
auto ret = set_source_path(api_path, source_path);
if (ret == api_error::success) {
if (((ret = set_item_meta(api_path, META_SIZE, std::to_string(file_size))) ==
api_error::success) &&
((ret = set_item_meta(api_path, META_ENCRYPTION_TOKEN, encryption_token)) ==
api_error::success) &&
(file_size != 0u)) {
ret = upload_manager_.queue_upload(api_path, source_path, encryption_token);
}
}
return ret;
}
api_error skynet_provider::upload_handler(const upload_manager::upload &upload, json &data,
json &error) {
const auto file_name = utils::path::strip_to_file_name(upload.api_path);
auto ret = api_error::upload_failed;
if (not upload.cancel) {
event_system::instance().raise<file_upload_begin>(upload.api_path, upload.source_path);
ret = (comm_.post_multipart_file(get_host_config(true), "", file_name, upload.source_path,
upload.encryption_token, data, error,
upload.cancel) == api_error::success)
? api_error::success
: api_error::upload_failed;
}
event_system::instance().raise<file_upload_end>(upload.api_path, upload.source_path, ret);
return ret;
}
} // namespace repertory
#endif // defined(REPERTORY_ENABLE_SKYNET)

385
src/rpc/client/client.cpp
View File

@@ -1,240 +1,267 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "rpc/client/client.hpp"
#include "comm/curl/curl_resolver.hpp"
#include "types/repertory.hpp"
#include "utils/Base64.hpp"
#include "utils/utils.hpp"
namespace repertory {
client::client(rpc_host_info host_info) : host_info_(std::move(host_info)) { request_id_ = 0u; }
rpc_response client::export_list(const std::vector<std::string> &paths) {
auto ret = make_request(rpc_method::export_links, {paths});
if (ret.response_type == rpc_response_type::success) {
ret.data = ret.data["result"];
}
return ret;
client::client(rpc_host_info host_info) : host_info_(std::move(host_info)) {
request_id_ = 0u;
}
rpc_response client::export_all() {
auto ret = make_request(rpc_method::export_links, {}, 60000u);
if (ret.response_type == rpc_response_type::success) {
ret.data = ret.data["result"];
auto client::get_drive_information() -> rpc_response {
const auto base_url =
"http://" + host_info_.host + ":" + std::to_string(host_info_.port);
httplib::Client cli{base_url};
cli.set_basic_auth(host_info_.user, host_info_.password);
auto resp = cli.Get("/api/v1/" + rpc_method::get_drive_information);
if (resp.error() != httplib::Error::Success) {
return rpc_response{rpc_response_type::http_error,
{{"error", httplib::to_string(resp.error())}}};
}
if (resp->status != 200) {
return rpc_response{rpc_response_type::http_error,
{{"error", std::to_string(resp->status)}}};
}
return ret;
return rpc_response{rpc_response_type::success, json::parse(resp->body)};
}
rpc_response client::get_drive_information() {
auto ret = make_request(rpc_method::get_drive_information, {});
if (ret.response_type == rpc_response_type::success) {
ret.data = ret.data["result"];
auto client::get_config() -> rpc_response {
const auto base_url =
"http://" + host_info_.host + ":" + std::to_string(host_info_.port);
httplib::Client cli{base_url};
cli.set_basic_auth(host_info_.user, host_info_.password);
auto resp = cli.Get("/api/v1/" + rpc_method::get_config);
if (resp.error() != httplib::Error::Success) {
return rpc_response{rpc_response_type::http_error,
{{"error", httplib::to_string(resp.error())}}};
}
if (resp->status != 200) {
return rpc_response{rpc_response_type::http_error,
{{"error", std::to_string(resp->status)}}};
}
return ret;
return rpc_response{rpc_response_type::success, json::parse(resp->body)};
}
rpc_response client::get_config() {
auto ret = make_request(rpc_method::get_config, {});
if (ret.response_type == rpc_response_type::success) {
ret.data = ret.data["result"];
auto client::get_config_value_by_name(const std::string &name) -> rpc_response {
const auto base_url =
"http://" + host_info_.host + ":" + std::to_string(host_info_.port);
httplib::Params params{{"name", name}};
httplib::Client cli{base_url};
cli.set_basic_auth(host_info_.user, host_info_.password);
auto resp =
cli.Get("/api/v1/" + rpc_method::get_config_value_by_name, params, {});
if (resp.error() != httplib::Error::Success) {
return rpc_response{rpc_response_type::http_error,
{{"error", httplib::to_string(resp.error())}}};
}
if (resp->status != 200) {
return rpc_response{rpc_response_type::http_error,
{{"error", std::to_string(resp->status)}}};
}
return ret;
return rpc_response{rpc_response_type::success, json::parse(resp->body)};
}
rpc_response client::get_config_value_by_name(const std::string &name) {
auto ret = make_request(rpc_method::get_config_value_by_name, {name});
if (ret.response_type == rpc_response_type::success) {
if (ret.data["result"]["value"].get<std::string>().empty()) {
ret.response_type = rpc_response_type::config_value_not_found;
} else {
ret.data = ret.data["result"];
}
auto client::get_directory_items(const std::string &api_path) -> rpc_response {
const auto base_url =
"http://" + host_info_.host + ":" + std::to_string(host_info_.port);
httplib::Params params{{"api_path", api_path}};
httplib::Client cli{base_url};
cli.set_basic_auth(host_info_.user, host_info_.password);
auto resp = cli.Get("/api/v1/" + rpc_method::get_directory_items, params, {});
if (resp.error() != httplib::Error::Success) {
return rpc_response{rpc_response_type::http_error,
{{"error", httplib::to_string(resp.error())}}};
}
if (resp->status != 200) {
return rpc_response{rpc_response_type::http_error,
{{"error", std::to_string(resp->status)}}};
}
return ret;
return rpc_response{rpc_response_type::success, json::parse(resp->body)};
}
rpc_response client::get_directory_items(const std::string &api_path) {
auto ret = make_request(rpc_method::get_directory_items, {api_path});
if (ret.response_type == rpc_response_type::success) {
ret.data = ret.data["result"];
auto client::get_open_files() -> rpc_response {
const auto base_url =
"http://" + host_info_.host + ":" + std::to_string(host_info_.port);
httplib::Client cli{base_url};
cli.set_basic_auth(host_info_.user, host_info_.password);
auto resp = cli.Get("/api/v1/" + rpc_method::get_open_files);
if (resp.error() != httplib::Error::Success) {
return rpc_response{rpc_response_type::http_error,
{{"error", httplib::to_string(resp.error())}}};
}
if (resp->status != 200) {
return rpc_response{rpc_response_type::http_error,
{{"error", std::to_string(resp->status)}}};
}
return ret;
return rpc_response{rpc_response_type::success, json::parse(resp->body)};
}
rpc_response client::get_open_files() {
auto ret = make_request(rpc_method::get_open_files, {});
if (ret.response_type == rpc_response_type::success) {
ret.data = ret.data["result"];
auto client::get_pinned_files() -> rpc_response {
const auto base_url =
"http://" + host_info_.host + ":" + std::to_string(host_info_.port);
httplib::Client cli{base_url};
cli.set_basic_auth(host_info_.user, host_info_.password);
auto resp = cli.Get("/api/v1/" + rpc_method::get_pinned_files);
if (resp.error() != httplib::Error::Success) {
return rpc_response{rpc_response_type::http_error,
{{"error", httplib::to_string(resp.error())}}};
}
if (resp->status != 200) {
return rpc_response{rpc_response_type::http_error,
{{"error", std::to_string(resp->status)}}};
}
return ret;
return rpc_response{rpc_response_type::success, json::parse(resp->body)};
}
rpc_response client::get_pinned_files() {
auto ret = make_request(rpc_method::get_pinned_files, {});
if (ret.response_type == rpc_response_type::success) {
ret.data = ret.data["result"];
auto client::pin_file(const std::string &api_path) -> rpc_response {
const auto base_url =
"http://" + host_info_.host + ":" + std::to_string(host_info_.port);
httplib::Params params{{"api_path", api_path}};
httplib::Client cli{base_url};
cli.set_basic_auth(host_info_.user, host_info_.password);
auto resp = cli.Post("/api/v1/" + rpc_method::pin_file, params);
if (resp.error() != httplib::Error::Success) {
return rpc_response{rpc_response_type::http_error,
{{"error", httplib::to_string(resp.error())}}};
}
if (resp->status != 200) {
return rpc_response{rpc_response_type::http_error,
{{"error", std::to_string(resp->status)}}};
}
return ret;
return rpc_response{rpc_response_type::success, {}};
}
rpc_response client::import_skylink(const skylink_import_list &list) {
std::vector<json> json_list;
for (const auto &skynet_import : list) {
json_list.emplace_back(skynet_import.to_json());
auto client::pinned_status(const std::string &api_path) -> rpc_response {
const auto base_url =
"http://" + host_info_.host + ":" + std::to_string(host_info_.port);
httplib::Params params{{"api_path", api_path}};
httplib::Client cli{base_url};
cli.set_basic_auth(host_info_.user, host_info_.password);
auto resp = cli.Get("/api/v1/" + rpc_method::pinned_status, params, {});
if (resp.error() != httplib::Error::Success) {
return rpc_response{rpc_response_type::http_error,
{{"error", httplib::to_string(resp.error())}}};
}
if (resp->status != 200) {
return rpc_response{rpc_response_type::http_error,
{{"error", std::to_string(resp->status)}}};
}
auto ret = make_request(rpc_method::import, {json(json_list)}, 60000u);
if (ret.response_type == rpc_response_type::success) {
ret.data = ret.data["result"];
}
return ret;
return rpc_response{rpc_response_type::success, json::parse(resp->body)};
}
rpc_response client::make_request(const std::string &command, const std::vector<json> &args,
std::uint32_t timeout_ms) {
auto error = rpc_response_type::success;
auto client::set_config_value_by_name(const std::string &name,
const std::string &value)
-> rpc_response {
const auto base_url =
"http://" + host_info_.host + ":" + std::to_string(host_info_.port);
auto *curl_handle = utils::create_curl();
httplib::Params params{
{"name", name},
{"value", value},
};
const auto port = static_cast<std::uint16_t>(host_info_.port);
const auto url = "http://" + host_info_.host + ":" + std::to_string(port) + "/api";
const auto request = json({{"jsonrpc", "2.0"},
{"id", std::to_string(++request_id_)},
{"method", command},
{"params", args}})
.dump();
httplib::Client cli{base_url};
cli.set_basic_auth(host_info_.user, host_info_.password);
struct curl_slist *hs = nullptr;
hs = curl_slist_append(hs, "Content-Type: application/json;");
if (not(host_info_.password.empty() && host_info_.user.empty())) {
curl_easy_setopt(curl_handle, CURLOPT_USERNAME, &host_info_.user[0]);
curl_easy_setopt(curl_handle, CURLOPT_PASSWORD, &host_info_.password[0]);
auto resp =
cli.Post("/api/v1/" + rpc_method::set_config_value_by_name, params);
if (resp.error() != httplib::Error::Success) {
return rpc_response{rpc_response_type::http_error,
{{"error", httplib::to_string(resp.error())}}};
}
#ifndef __APPLE__
curl_resolver resolver(curl_handle,
{"localhost:" + std::to_string(host_info_.port) + ":127.0.0.1"}, true);
#endif
if (timeout_ms > 0) {
curl_easy_setopt(curl_handle, CURLOPT_TIMEOUT_MS, timeout_ms);
}
curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, hs);
curl_easy_setopt(curl_handle, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, request.c_str());
curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDSIZE, request.size());
curl_easy_setopt(curl_handle, CURLOPT_CONNECTTIMEOUT, 5L);
curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION,
static_cast<size_t (*)(char *, size_t, size_t, void *)>(
[](char *buffer, size_t size, size_t nitems, void *outstream) -> size_t {
(*reinterpret_cast<std::string *>(outstream)) +=
std::string(buffer, size * nitems);
return size * nitems;
}));
if (resp->status != 200) {
return rpc_response{rpc_response_type::http_error,
{{"error", std::to_string(resp->status)}}};
};
std::string response;
curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, &response);
json json_data;
const auto res = curl_easy_perform(curl_handle);
if (res == CURLE_OK) {
long httpErrorCode;
curl_easy_getinfo(curl_handle, CURLINFO_RESPONSE_CODE, &httpErrorCode);
if (httpErrorCode == 200) {
json_data = json::parse(response.begin(), response.end());
} else {
json parsed;
try {
parsed = json::parse(response.begin(), response.end());
} catch (...) {
}
error = rpc_response_type::http_error;
json_data = {{"error", {{"code", std::to_string(httpErrorCode)}}}};
if (parsed.empty()) {
json_data["error"]["response"] = response;
} else {
json_data["error"]["response"] = parsed;
}
}
} else {
error = rpc_response_type::curl_error;
json_data = {{"error", {{"message", curl_easy_strerror(res)}}}};
}
curl_easy_cleanup(curl_handle);
curl_slist_free_all(hs);
return rpc_response({error, json_data});
return rpc_response{rpc_response_type::success,
nlohmann::json::parse(resp->body)};
}
rpc_response client::pin_file(const std::string &api_path) {
auto ret = make_request(rpc_method::pin_file, {api_path});
if (ret.response_type == rpc_response_type::success) {
ret.data = ret.data["result"];
auto client::unmount() -> rpc_response {
const auto base_url =
"http://" + host_info_.host + ":" + std::to_string(host_info_.port);
httplib::Client cli{base_url};
cli.set_basic_auth(host_info_.user, host_info_.password);
auto resp = cli.Post("/api/v1/" + rpc_method::unmount);
if (resp.error() != httplib::Error::Success) {
return rpc_response{rpc_response_type::http_error,
{{"error", httplib::to_string(resp.error())}}};
}
if (resp->status != 200) {
return rpc_response{rpc_response_type::http_error,
{{"error", std::to_string(resp->status)}}};
}
return ret;
return rpc_response{rpc_response_type::success, {}};
}
rpc_response client::pinned_status(const std::string &api_path) {
auto ret = make_request(rpc_method::pinned_status, {api_path});
if (ret.response_type == rpc_response_type::success) {
ret.data = ret.data["result"];
auto client::unpin_file(const std::string &api_path) -> rpc_response {
const auto base_url =
"http://" + host_info_.host + ":" + std::to_string(host_info_.port);
httplib::Params params{{"api_path", api_path}};
httplib::Client cli{base_url};
cli.set_basic_auth(host_info_.user, host_info_.password);
auto resp = cli.Post("/api/v1/" + rpc_method::unpin_file, params);
if (resp.error() != httplib::Error::Success) {
return rpc_response{rpc_response_type::http_error,
{{"error", httplib::to_string(resp.error())}}};
}
if (resp->status != 200) {
return rpc_response{rpc_response_type::http_error,
{{"error", std::to_string(resp->status)}}};
}
return ret;
}
rpc_response client::set_config_value_by_name(const std::string &name, const std::string &value) {
auto ret = make_request(rpc_method::set_config_value_by_name, {name, value});
if (ret.response_type == rpc_response_type::success) {
if (ret.data["result"]["value"].get<std::string>().empty()) {
ret.response_type = rpc_response_type::config_value_not_found;
} else {
ret.data = ret.data["result"];
}
}
return ret;
}
rpc_response client::unmount() {
auto ret = make_request(rpc_method::unmount, {});
if (ret.response_type == rpc_response_type::success) {
ret.data = ret.data["result"];
}
return ret;
}
rpc_response client::unpin_file(const std::string &api_path) {
auto ret = make_request(rpc_method::unpin_file, {api_path});
if (ret.response_type == rpc_response_type::success) {
ret.data = ret.data["result"];
}
return ret;
return rpc_response{rpc_response_type::success, {}};
}
} // namespace repertory

329
src/rpc/server/full_server.cpp
View File

@@ -1,165 +1,204 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "rpc/server/full_server.hpp"
#include "app_config.hpp"
#include "drives/directory_iterator.hpp"
#include "drives/i_open_file_table.hpp"
#include "file_manager/i_file_manager.hpp"
#include "providers/i_provider.hpp"
#include "providers/skynet/skynet_provider.hpp"
#include "types/repertory.hpp"
#include "types/rpc.hpp"
#include "types/skynet.hpp"
#include "utils/global_data.hpp"
#include "utils/file_utils.hpp"
#include "utils/path_utils.hpp"
namespace repertory {
full_server::full_server(app_config &config, i_provider &provider, i_open_file_table &oft)
: server(config), provider_(provider), oft_(oft) {}
full_server::full_server(app_config &config, i_provider &provider,
i_file_manager &fm)
: server(config), provider_(provider), fm_(fm) {}
bool full_server::handle_request(jsonrpcpp::request_ptr &request,
std::unique_ptr<jsonrpcpp::Response> &response) {
auto handled = true;
if (request->method == rpc_method::get_drive_information) {
if (request->params.param_array.empty()) {
response = std::make_unique<jsonrpcpp::Response>(
*request, json({{"cache_space_used", global_data::instance().get_used_cache_space()},
{"drive_space_total", provider_.get_total_drive_space()},
{"drive_space_used", global_data::instance().get_used_drive_space()},
{"item_count", provider_.get_total_item_count()}}));
} else {
throw jsonrpcpp::InvalidParamsException(*request);
}
} else if (request->method == rpc_method::get_pinned_files) {
if (request->params.param_array.empty()) {
response = std::make_unique<jsonrpcpp::Response>(*request, provider_.get_pinned_files());
} else {
throw jsonrpcpp::InvalidParamsException(*request);
}
} else if (request->method == rpc_method::get_directory_items) {
if (request->params.param_array.size() == 1u) {
const auto api_path = utils::path::create_api_path(request->params.param_array[0]);
auto directoryItems = oft_.get_directory_items(api_path);
std::vector<json> items;
for (const auto &item : directoryItems) {
items.emplace_back(item.to_json());
}
void full_server::handle_get_directory_items(const httplib::Request &req,
httplib::Response &res) {
const auto api_path =
utils::path::create_api_path(req.get_param_value("api_path"));
const auto list = fm_.get_directory_items(api_path);
response = std::make_unique<jsonrpcpp::Response>(*request, json({{"items", items}}));
} else {
throw jsonrpcpp::InvalidParamsException(*request);
}
} else if (request->method == rpc_method::pin_file) {
if (request->params.param_array.size() == 1u) {
const auto api_path = utils::path::create_api_path(request->params.param_array[0]);
auto success = provider_.is_file(api_path);
if (success) {
success = api_error::success ==
provider_.set_item_meta(api_path, META_PINNED, utils::string::from_bool(true));
}
if (success) {
event_system::instance().raise<file_pinned>(api_path);
}
response = std::make_unique<jsonrpcpp::Response>(*request, json({{"success", success}}));
} else {
throw jsonrpcpp::InvalidParamsException(*request);
}
} else if (request->method == rpc_method::pinned_status) {
if (request->params.param_array.size() == 1u) {
const auto api_path = utils::path::create_api_path(request->params.param_array[0]);
std::string pinned;
const auto success =
api_error::success == provider_.get_item_meta(api_path, META_PINNED, pinned);
response = std::make_unique<jsonrpcpp::Response>(
*request, json({{"success", success},
{"pinned", pinned.empty() ? false : utils::string::to_bool(pinned)}}));
} else {
throw jsonrpcpp::InvalidParamsException(*request);
}
} else if (request->method == rpc_method::unpin_file) {
if (request->params.param_array.size() == 1u) {
const auto api_path = utils::path::create_api_path(request->params.param_array[0]);
auto success = provider_.is_file(api_path);
if (success) {
success = api_error::success ==
provider_.set_item_meta(api_path, META_PINNED, utils::string::from_bool(false));
}
if (success) {
event_system::instance().raise<file_unpinned>(api_path);
}
response = std::make_unique<jsonrpcpp::Response>(*request, json({{"success", success}}));
} else {
throw jsonrpcpp::InvalidParamsException(*request);
}
} else if (request->method == rpc_method::get_open_files) {
if (request->params.param_array.empty()) {
const auto list = oft_.get_open_files();
json open_files = {{"file_list", std::vector<json>()}};
for (const auto &kv : list) {
open_files["file_list"].emplace_back(json({{"path", kv.first}, {"count", kv.second}}));
}
response = std::make_unique<jsonrpcpp::Response>(*request, open_files);
} else {
throw jsonrpcpp::InvalidParamsException(*request);
}
#if defined(REPERTORY_ENABLE_SKYNET)
} else if (get_config().get_provider_type() == provider_type::skynet) {
if (request->method == rpc_method::import) {
if (request->params.param_array.size() == 1) {
json results = {{"success", std::vector<json>()}, {"failed", std::vector<std::string>()}};
for (const auto &link : request->params.param_array[0]) {
const auto si = skylink_import::from_json(link);
auto &provider = dynamic_cast<skynet_provider &>(provider_);
const auto res = provider.import_skylink(si);
if (res == api_error::success) {
results["success"].emplace_back(link);
} else {
results["failed"].emplace_back(link["skylink"].get<std::string>());
}
}
response = std::make_unique<jsonrpcpp::Response>(*request, results);
} else {
throw jsonrpcpp::InvalidParamsException(*request);
}
} else if (request->method == rpc_method::export_links) {
if (request->params.param_array.empty()) {
auto &provider = dynamic_cast<skynet_provider &>(provider_);
response = std::make_unique<jsonrpcpp::Response>(*request, provider.export_all());
} else if (request->params.param_array.size() == 1) {
auto &provider = dynamic_cast<skynet_provider &>(provider_);
response = std::make_unique<jsonrpcpp::Response>(
*request,
provider.export_list(request->params.param_array[0].get<std::vector<std::string>>()));
} else {
throw jsonrpcpp::InvalidParamsException(*request);
}
} else {
handled = server::handle_request(request, response);
}
#endif
} else {
handled = server::handle_request(request, response);
json items = {{"items", std::vector<json>()}};
for (const auto &item : list) {
items["items"].emplace_back(item.to_json());
}
return handled;
res.set_content(items.dump(), "application/json");
res.status = 200;
}
void full_server::handle_get_drive_information(const httplib::Request & /*req*/,
httplib::Response &res) {
res.set_content(
json({
{"cache_space_used",
utils::file::calculate_used_space(
get_config().get_cache_directory(), false)},
{"drive_space_total", provider_.get_total_drive_space()},
{"drive_space_used", provider_.get_used_drive_space()},
{"item_count", provider_.get_total_item_count()},
})
.dump(),
"application/json");
res.status = 200;
}
void full_server::handle_get_open_files(const httplib::Request & /*req*/,
httplib::Response &res) {
const auto list = fm_.get_open_files();
json open_files = {{"items", std::vector<json>()}};
for (const auto &kv : list) {
open_files["items"].emplace_back(json({
{"path", kv.first},
{"count", kv.second},
}));
}
res.set_content(open_files.dump(), "application/json");
res.status = 200;
}
void full_server::handle_get_pinned_files(const httplib::Request & /*req*/,
httplib::Response &res) {
res.set_content(json({{"items", provider_.get_pinned_files()}}).dump(),
"application/json");
res.status = 200;
}
void full_server::handle_get_pinned_status(const httplib::Request &req,
httplib::Response &res) {
const auto api_path =
utils::path::create_api_path(req.get_param_value("api_path"));
std::string pinned;
const auto result = provider_.get_item_meta(api_path, META_PINNED, pinned);
if (result != api_error::success) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, result,
"failed to get pinned status");
res.status = 500;
return;
}
res.set_content(
json(
{{"pinned", pinned.empty() ? false : utils::string::to_bool(pinned)}})
.dump(),
"application/json");
res.status = 200;
}
void full_server::handle_pin_file(const httplib::Request &req,
httplib::Response &res) {
const auto api_path =
utils::path::create_api_path(req.get_param_value("api_path"));
bool exists{};
auto result = provider_.is_file(api_path, exists);
if (result != api_error::success) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, result,
"failed to pin file");
res.status = 500;
return;
}
if (exists) {
exists = api_error::success ==
provider_.set_item_meta(api_path, META_PINNED,
utils::string::from_bool(true));
}
if (exists) {
event_system::instance().raise<file_pinned>(api_path);
}
res.status = exists ? 200 : 404;
}
void full_server::handle_unpin_file(const httplib::Request &req,
httplib::Response &res) {
const auto api_path =
utils::path::create_api_path(req.get_param_value("api_path"));
bool exists{};
auto result = provider_.is_file(api_path, exists);
if (result != api_error::success) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, result,
"failed to unpin file");
res.status = 500;
return;
}
if (exists) {
exists = api_error::success ==
provider_.set_item_meta(api_path, META_PINNED,
utils::string::from_bool(false));
}
if (exists) {
event_system::instance().raise<file_unpinned>(api_path);
}
res.status = exists ? 200 : 404;
}
void full_server::initialize(httplib::Server &inst) {
server::initialize(inst);
inst.Get("/api/v1/" + rpc_method::get_directory_items,
[this](auto &&req, auto &&res) {
handle_get_directory_items(std::forward<decltype(req)>(req),
std::forward<decltype(res)>(res));
});
inst.Get("/api/v1/" + rpc_method::get_drive_information,
[this](auto &&req, auto &&res) {
handle_get_drive_information(std::forward<decltype(req)>(req),
std::forward<decltype(res)>(res));
});
inst.Get("/api/v1/" + rpc_method::get_open_files,
[this](auto &&req, auto &&res) {
handle_get_open_files(std::forward<decltype(req)>(req),
std::forward<decltype(res)>(res));
});
inst.Get("/api/v1/" + rpc_method::get_pinned_files,
[this](auto &&req, auto &&res) {
handle_get_pinned_files(std::forward<decltype(req)>(req),
std::forward<decltype(res)>(res));
});
inst.Get("/api/v1/" + rpc_method::pinned_status,
[this](auto &&req, auto &&res) {
handle_get_pinned_status(std::forward<decltype(req)>(req),
std::forward<decltype(res)>(res));
});
inst.Post("/api/v1/" + rpc_method::pin_file, [this](auto &&req, auto &&res) {
handle_pin_file(std::forward<decltype(req)>(req),
std::forward<decltype(res)>(res));
});
inst.Post("/api/v1/" + rpc_method::unpin_file,
[this](auto &&req, auto &&res) {
handle_unpin_file(std::forward<decltype(req)>(req),
std::forward<decltype(res)>(res));
});
}
} // namespace repertory

276
src/rpc/server/server.cpp
View File

@@ -1,163 +1,181 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "rpc/server/server.hpp"
#include "app_config.hpp"
#include "utils/Base64.hpp"
#include "utils/error_utils.hpp"
namespace repertory {
server::rpc_resource::rpc_resource(server &owner) : httpserver::http_resource(), owner_(owner) {
disallow_all();
set_allowing("POST", true);
}
server::server(app_config &config) : config_(config) {}
const std::shared_ptr<httpserver::http_response>
server::rpc_resource::render(const httpserver::http_request &request) {
std::shared_ptr<json_response> ret;
try {
if (owner_.check_authorization(request)) {
std::unique_ptr<jsonrpcpp::Response> response;
auto entity = jsonrpcpp::Parser::do_parse(utils::string::to_utf8(request.get_content()));
if (entity->is_request()) {
auto request = std::dynamic_pointer_cast<jsonrpcpp::Request>(entity);
if (not owner_.handle_request(request, response)) {
throw jsonrpcpp::MethodNotFoundException(*request);
}
ret = std::make_shared<json_response>(response->to_json());
}
} else {
ret = std::make_shared<json_response>(json({{"error", "unauthorized"}}),
httpserver::http::http_utils::http_unauthorized);
}
} catch (const jsonrpcpp::RequestException &e) {
ret = std::make_shared<json_response>(e.to_json(),
httpserver::http::http_utils::http_bad_request);
event_system::instance().raise<rpc_server_exception>(e.to_json().dump());
} catch (const std::exception &e2) {
ret = std::make_shared<json_response>(json({{"exception", e2.what()}}),
httpserver::http::http_utils::http_internal_server_error);
event_system::instance().raise<rpc_server_exception>(e2.what());
auto server::check_authorization(const httplib::Request &req) -> bool {
if (config_.get_api_auth().empty() || config_.get_api_user().empty()) {
utils::error::raise_error(__FUNCTION__,
"authorization user or password is not set");
return false;
}
return ret;
}
server::server(app_config &config) : config_(config), resource_(*this) {}
bool server::check_authorization(const httpserver::http_request &request) {
auto ret = (config_.get_api_auth().empty() && config_.get_api_user().empty());
if (not ret) {
const auto authorization = request.get_header("Authorization");
if (not authorization.empty()) {
const auto auth_parts = utils::string::split(authorization, ' ');
if (not auth_parts.empty()) {
const auto auth_type = auth_parts[0];
if (auth_type == "Basic") {
const auto data = macaron::Base64::Decode(authorization.substr(6));
const auto auth = utils::string::split(std::string(data.begin(), data.end()), ':');
if (auth.size() == 2) {
const auto &user = auth[0];
const auto &pwd = auth[1];
ret = (user == config_.get_api_user()) && (pwd == config_.get_api_auth());
}
}
}
}
const auto authorization = req.get_header_value("Authorization");
if (authorization.empty()) {
return false;
}
return ret;
const auto auth_parts = utils::string::split(authorization, ' ');
if (auth_parts.empty()) {
return false;
}
const auto auth_type = auth_parts[0U];
if (auth_type != "Basic") {
return false;
}
const auto data = macaron::Base64::Decode(authorization.substr(6U));
const auto auth =
utils::string::split(std::string(data.begin(), data.end()), ':');
if (auth.size() != 2U) {
return false;
}
const auto &user = auth[0U];
const auto &pwd = auth[1U];
return (user == config_.get_api_user()) && (pwd == config_.get_api_auth());
}
bool server::handle_request(jsonrpcpp::request_ptr &request,
std::unique_ptr<jsonrpcpp::Response> &response) {
auto handled = true;
if (request->method == rpc_method::get_config) {
if (request->params.param_array.empty()) {
response = std::make_unique<jsonrpcpp::Response>(*request, config_.get_json());
} else {
throw jsonrpcpp::InvalidParamsException(*request);
}
} else if (request->method == rpc_method::get_config_value_by_name) {
if (request->params.param_array.size() == 1) {
response = std::make_unique<jsonrpcpp::Response>(
*request, json({{"value", config_.get_value_by_name(request->params.param_array[0])}}));
} else {
throw jsonrpcpp::InvalidParamsException(*request);
}
} else if (request->method == rpc_method::set_config_value_by_name) {
if (request->params.param_array.size() == 2) {
response = std::make_unique<jsonrpcpp::Response>(
*request, json({{"value", config_.set_value_by_name(request->params.param_array[0],
request->params.param_array[1])}}));
} else {
throw jsonrpcpp::InvalidParamsException(*request);
}
} else if (request->method == rpc_method::unmount) {
if (request->params.param_array.empty()) {
event_system::instance().raise<unmount_requested>();
response = std::make_unique<jsonrpcpp::Response>(*request, json({{"success", true}}));
} else {
throw jsonrpcpp::InvalidParamsException(*request);
}
} else {
handled = false;
}
return handled;
void server::handle_get_config(const httplib::Request & /*req*/,
httplib::Response &res) {
auto data = config_.get_json();
res.set_content(data.dump(), "application/json");
res.status = 200;
}
/*void server::Start() {
mutex_lock l(startStopMutex_);
if (not started_) {
struct sockaddr_in localHost{};
inet_pton(AF_INET, "127.0.0.1", &localHost.sin_addr);
void server::handle_get_config_value_by_name(const httplib::Request &req,
httplib::Response &res) {
auto name = req.get_param_value("name");
auto data = json({{"value", config_.get_value_by_name(name)}});
res.set_content(data.dump(), "application/json");
res.status = 200;
}
ws_ = std::make_unique<httpserver::webserver>(
httpserver::create_webserver(config_.GetAPIPort())
.bind_address((sockaddr*)&localHost)
.default_policy(httpserver::http::http_utils::REJECT));
ws_->allow_ip("127.0.0.1");
ws_->register_resource("/api", &rpcResource_);
ws_->start(false);
started_ = true;
}
}*/
void server::handle_set_config_value_by_name(const httplib::Request &req,
httplib::Response &res) {
auto name = req.get_param_value("name");
auto value = req.get_param_value("value");
json data = {{"value", config_.set_value_by_name(name, value)}};
res.set_content(data.dump(), "application/json");
res.status = 200;
}
void server::handle_unmount(const httplib::Request & /*req*/,
httplib::Response &res) {
event_system::instance().raise<unmount_requested>();
res.status = 200;
}
void server::initialize(httplib::Server &inst) {
inst.Get("/api/v1/" + rpc_method::get_config, [this](auto &&req, auto &&res) {
handle_get_config(std::forward<decltype(req)>(req),
std::forward<decltype(res)>(res));
});
inst.Get("/api/v1/" + rpc_method::get_config_value_by_name,
[this](auto &&req, auto &&res) {
handle_get_config_value_by_name(std::forward<decltype(req)>(req),
std::forward<decltype(res)>(res));
});
inst.Post("/api/v1/" + rpc_method::set_config_value_by_name,
[this](auto &&req, auto &&res) {
handle_set_config_value_by_name(std::forward<decltype(req)>(req),
std::forward<decltype(res)>(res));
});
inst.Post("/api/v1/" + rpc_method::unmount, [this](auto &&req, auto &&res) {
handle_unmount(std::forward<decltype(req)>(req),
std::forward<decltype(res)>(res));
});
}
void server::start() {
mutex_lock l(start_stop_mutex_);
if (not started_) {
ws_ = std::make_unique<httpserver::webserver>(
httpserver::create_webserver(config_.get_api_port())
.default_policy(httpserver::http::http_utils::REJECT));
ws_->allow_ip("127.0.0.1");
ws_->register_resource("/api", &resource_);
ws_->start(false);
event_system::instance().raise<service_started>("server");
server_ = std::make_unique<httplib::Server>();
server_->set_exception_handler([](const httplib::Request &req,
httplib::Response &res,
std::exception_ptr ep) {
json data = {{"path", req.path}};
try {
std::rethrow_exception(ep);
} catch (std::exception &e) {
data["error"] = e.what() ? e.what() : "unknown error";
utils::error::raise_error(__FUNCTION__, e,
"failed request: " + req.path);
} catch (...) {
data["error"] = "unknown error";
utils::error::raise_error(__FUNCTION__, "unknown error",
"failed request: " + req.path);
}
res.set_content(data.dump(), "application/json");
res.status = 500;
});
server_->set_pre_routing_handler(
[this](auto &&req, auto &&res) -> httplib::Server::HandlerResponse {
if (check_authorization(req)) {
return httplib::Server::HandlerResponse::Unhandled;
}
res.status = 401;
return httplib::Server::HandlerResponse::Handled;
});
initialize(*server_);
server_thread_ = std::make_unique<std::thread>(
[this]() { server_->listen("127.0.0.1", config_.get_api_port()); });
started_ = true;
}
}
void server::stop() {
mutex_lock l(start_stop_mutex_);
if (started_) {
event_system::instance().raise<service_shutdown>("server");
ws_->stop();
ws_.reset();
started_ = false;
mutex_lock l(start_stop_mutex_);
if (started_) {
event_system::instance().raise<service_shutdown_begin>("server");
server_->stop();
server_thread_->join();
server_thread_.reset();
started_ = false;
event_system::instance().raise<service_shutdown_end>("server");
}
}
}
} // namespace repertory

33
src/types/remote.cpp
View File

@@ -1,26 +1,29 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
of the Software, and to permit persons to whom the Software is furnished to do
so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "types/remote.hpp"
namespace repertory::remote {
#ifndef _WIN32
open_flags create_open_flags(const std::uint32_t &flags) {
auto create_open_flags(std::uint32_t flags) -> open_flags {
open_flags ret{};
{
const auto f = (flags & 3u);
@@ -99,7 +102,7 @@ open_flags create_open_flags(const std::uint32_t &flags) {
return ret;
}
std::uint32_t create_os_open_flags(const open_flags &flags) {
auto create_os_open_flags(const open_flags &flags) -> std::uint32_t {
std::uint32_t ret = 0u;
if ((flags & open_flags::read_write) == open_flags::read_write) {
ret |= static_cast<std::uint32_t>(O_RDWR);

126
src/types/repertory.cpp
View File

@@ -1,69 +1,85 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "types/repertory.hpp"
#include "types/startup_exception.hpp"
namespace repertory {
const std::string &api_error_to_string(const api_error &error) {
static const std::unordered_map<api_error, std::string> LOOKUP = {
{api_error::success, "success"},
{api_error::access_denied, "access_denied"},
{api_error::bad_address, "bad_address"},
{api_error::buffer_overflow, "buffer_overflow"},
{api_error::buffer_too_small, "buffer_too_small"},
{api_error::comm_error, "comm_error"},
{api_error::decryption_error, "decryption_error"},
{api_error::directory_end_of_files, "directory_end_of_files"},
{api_error::directory_exists, "directory_exists"},
{api_error::directory_not_empty, "directory_not_empty"},
{api_error::directory_not_found, "directory_not_found"},
{api_error::download_failed, "download_failed"},
{api_error::download_incomplete, "download_incomplete"},
{api_error::download_stopped, "download_stopped"},
{api_error::download_timeout, "download_timeout"},
{api_error::empty_ring_buffer_chunk_size, "empty_ring_buffer_chunk_size"},
{api_error::empty_ring_buffer_size, "empty_ring_buffer_size"},
{api_error::error, "error"},
{api_error::file_exists, "file_exists"},
{api_error::file_in_use, "file_in_use"},
{api_error::incompatible_version, "incompatible_version"},
{api_error::invalid_handle, "invalid_handle"},
{api_error::invalid_operation, "invalid_operation"},
{api_error::invalid_ring_buffer_multiple, "invalid_ring_buffer_multiple"},
{api_error::invalid_ring_buffer_size, "invalid_ring_buffer_size"},
{api_error::invalid_version, "invalid_version"},
{api_error::item_is_file, "item_is_file"},
{api_error::item_not_found, "item_not_found"},
{api_error::not_implemented, "not_implemented"},
{api_error::not_supported, "not_supported"},
{api_error::os_error, "os_error"},
{api_error::permission_denied, "permission_denied"},
{api_error::upload_failed, "upload_failed"},
{api_error::upload_stopped, "upload_stopped"},
{api_error::xattr_buffer_small, "xattr_buffer_small"},
{api_error::xattr_exists, "xattr_exists"},
{api_error::xattr_invalid_namespace, "xattr_invalid_namespace"},
{api_error::xattr_not_found, "xattr_not_found"},
{api_error::xattr_osx_invalid, "xattr_osx_invalid"},
{api_error::xattr_too_big, "xattr_too_big"},
};
static const std::unordered_map<api_error, std::string> LOOKUP = {
{api_error::success, "success"},
{api_error::access_denied, "access_denied"},
{api_error::bad_address, "bad_address"},
{api_error::buffer_overflow, "buffer_overflow"},
{api_error::buffer_too_small, "buffer_too_small"},
{api_error::comm_error, "comm_error"},
{api_error::decryption_error, "decryption_error"},
{api_error::directory_end_of_files, "directory_end_of_files"},
{api_error::directory_exists, "directory_exists"},
{api_error::directory_not_empty, "directory_not_empty"},
{api_error::directory_not_found, "directory_not_found"},
{api_error::download_failed, "download_failed"},
{api_error::download_incomplete, "download_incomplete"},
{api_error::download_stopped, "download_stopped"},
{api_error::empty_ring_buffer_chunk_size, "empty_ring_buffer_chunk_size"},
{api_error::empty_ring_buffer_size, "empty_ring_buffer_size"},
{api_error::error, "error"},
{api_error::file_in_use, "file_in_use"},
{api_error::file_size_mismatch, "file_size_mismatch"},
{api_error::incompatible_version, "incompatible_version"},
{api_error::invalid_handle, "invalid_handle"},
{api_error::invalid_operation, "invalid_operation"},
{api_error::invalid_ring_buffer_multiple, "invalid_ring_buffer_multiple"},
{api_error::invalid_ring_buffer_size, "invalid_ring_buffer_size"},
{api_error::invalid_version, "invalid_version"},
{api_error::item_exists, "item_exists"},
{api_error::item_not_found, "item_not_found"},
{api_error::no_disk_space, "no_disk_space"},
{api_error::not_implemented, "not_implemented"},
{api_error::not_supported, "not_supported"},
{api_error::os_error, "os_error"},
{api_error::out_of_memory, "out_of_memory"},
{api_error::permission_denied, "permission_denied"},
{api_error::upload_failed, "upload_failed"},
{api_error::upload_stopped, "upload_stopped"},
{api_error::xattr_buffer_small, "xattr_buffer_small"},
{api_error::xattr_exists, "xattr_exists"},
{api_error::xattr_not_found, "xattr_not_found"},
{api_error::xattr_too_big, "xattr_too_big"},
};
auto api_error_from_string(std::string_view s) -> api_error {
if (LOOKUP.size() != static_cast<std::size_t>(api_error::ERROR_COUNT)) {
throw startup_exception("undefined api_error strings");
}
const auto it =
std::find_if(LOOKUP.begin(), LOOKUP.end(),
[&s](const std::pair<api_error, std::string> &kv) -> bool {
return kv.second == s;
});
return it == LOOKUP.end() ? api_error::error : it->first;
}
auto api_error_to_string(const api_error &error) -> const std::string & {
if (LOOKUP.size() != static_cast<std::size_t>(api_error::ERROR_COUNT)) {
throw startup_exception("undefined api_error strings");
}

111
src/types/skynet.cpp
View File

@@ -1,111 +0,0 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#include "types/skynet.hpp"
#include "utils/string_utils.hpp"
namespace repertory {
std::vector<host_config> skynet_config::from_string(const std::string &list) {
const auto obj = json::parse(list);
std::vector<host_config> ret;
std::transform(obj.begin(), obj.end(), std::back_inserter(ret),
[](const json &portal) -> host_config {
host_config hc{};
hc.agent_string = portal["AgentString"].get<std::string>();
hc.api_password = portal["ApiPassword"].get<std::string>();
hc.api_port = portal["ApiPort"].get<std::uint16_t>();
hc.host_name_or_ip = portal["HostNameOrIp"].get<std::string>();
hc.path = portal["Path"].get<std::string>();
hc.protocol = portal["Protocol"].get<std::string>();
hc.timeout_ms = portal["TimeoutMs"].get<std::uint32_t>();
hc.auth_url = portal["AuthURL"].get<std::string>();
hc.auth_user = portal["AuthUser"].get<std::string>();
hc.auth_password = portal["AuthPassword"].get<std::string>();
return hc;
});
return ret;
}
std::string skynet_config::to_string(const std::vector<host_config> &list) {
json j;
std::transform(list.begin(), list.end(), std::back_inserter(j), [](const auto &hc) -> json {
return {
{"AgentString", hc.agent_string},
{"ApiPassword", hc.api_password},
{"ApiPort", hc.api_port},
{"HostNameOrIp", hc.host_name_or_ip},
{"Path", hc.path},
{"Protocol", hc.protocol},
{"TimeoutMs", hc.timeout_ms},
{"AuthURL", hc.auth_url},
{"AuthUser", hc.auth_user},
{"AuthPassword", hc.auth_password},
};
});
return j.dump();
}
skylink_import skylink_import::from_json(const json &j) {
return skylink_import{
j.contains("directory") ? j["directory"].get<std::string>() : "/",
j.contains("filename") ? j["filename"].get<std::string>() : "",
j["skylink"].get<std::string>(),
j.contains("token") ? j["token"].get<std::string>() : "",
};
}
skylink_import skylink_import::from_string(const std::string &str) {
auto parts = utils::string::split(str, ',');
if (parts.empty() || (parts.size() > 4)) {
throw std::runtime_error("unable to parse skylink from string: " + str);
}
std::unordered_map<std::string, std::string> linkData;
for (auto &part : parts) {
utils::string::replace(part, "@comma@", ",");
auto parts2 = utils::string::split(part, '=', false);
if (parts2.size() != 2) {
throw std::runtime_error("unable to parse skylink from string: " + str);
}
for (auto &part2 : parts2) {
utils::string::replace(part2, "@equal@", "=");
}
linkData[parts2[0]] = parts2[1];
}
return skylink_import{
linkData["directory"],
linkData["filename"],
linkData["skylink"],
linkData["token"],
};
}
json skylink_import::to_json() const {
return {
{"directory", directory},
{"filename", file_name},
{"skylink", skylink},
{"token", token},
};
}
} // namespace repertory

259
src/upload/upload_manager.cpp
View File

@@ -1,259 +0,0 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#include "upload/upload_manager.hpp"
#include "comm/i_comm.hpp"
#include "app_config.hpp"
#include "events/events.hpp"
#include "events/event_system.hpp"
#include "utils/file_utils.hpp"
#include "utils/rocksdb_utils.hpp"
namespace repertory {
upload_manager::upload_manager(const app_config &config, api_file_exists_callback api_file_exists,
upload_handler_callback upload_handler,
upload_completed_callback upload_completed)
: config_(config),
api_file_exists_(std::move(api_file_exists)),
upload_handler_(std::move(upload_handler)),
upload_completed_(std::move(upload_completed)) {
utils::db::create_rocksdb(config, ROCKS_DB_NAME, upload_db_);
auto iterator =
std::unique_ptr<rocksdb::Iterator>(upload_db_->NewIterator(rocksdb::ReadOptions()));
for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
auto data = json::parse(iterator->value().ToString());
auto u = std::make_shared<upload>(upload{
data["path"].get<std::string>(),
false,
data["token"].get<std::string>(),
data["source"].get<std::string>(),
});
upload_lookup_[iterator->key().ToString()] = u;
upload_queue_.emplace_back(u.get());
}
}
upload_manager::~upload_manager() {
stop();
upload_db_.reset();
}
bool upload_manager::execute_if_not_processing(const std::vector<std::string> &api_paths,
const std::function<void()> &action) const {
mutex_lock upload_lock(upload_mutex_);
const auto ret =
(std::find_if(api_paths.begin(), api_paths.end(), [this](const auto &path) -> bool {
return upload_lookup_.find(path) != upload_lookup_.end();
}) == api_paths.end());
if (ret) {
action();
}
return ret;
}
bool upload_manager::is_processing(const std::string &api_path) const {
mutex_lock upload_lock(upload_mutex_);
return upload_lookup_.find(api_path) != upload_lookup_.end();
}
api_error upload_manager::queue_upload(const std::string &api_path, const std::string &source_path,
const std::string &encryption_token) {
auto ret = remove_upload(api_path);
if ((ret == api_error::success) || (ret == api_error::item_not_found)) {
ret = api_error::success;
auto u = std::make_shared<upload>(upload{api_path, false, encryption_token, source_path});
unique_mutex_lock upload_lock(upload_mutex_);
upload_db_->Put(rocksdb::WriteOptions(), api_path,
json({
{"path", api_path},
{"source", source_path},
{"token", encryption_token},
})
.dump());
if (not stop_requested_) {
upload_lookup_[api_path] = u;
upload_queue_.emplace_back(u.get());
}
upload_notify_.notify_all();
event_system::instance().raise<file_upload_queued>(api_path, source_path);
upload_lock.unlock();
}
return ret;
}
api_error upload_manager::remove_upload(const std::string &api_path) {
auto ret = api_error::item_not_found;
unique_mutex_lock upload_lock(upload_mutex_);
if (upload_lookup_.find(api_path) != upload_lookup_.end()) {
auto upload = upload_lookup_[api_path];
utils::remove_element_from(upload_queue_, upload.get());
upload_lookup_.erase(api_path);
upload_db_->Delete(rocksdb::WriteOptions(), api_path);
upload->cancel = true;
upload_lock.unlock();
upload->wait();
event_system::instance().raise<file_upload_removed>(api_path, upload->source_path);
ret = api_error::success;
}
return ret;
}
void upload_manager::start() {
mutex_lock upload_lock(upload_mutex_);
if (not upload_thread_) {
stop_requested_ = false;
upload_thread_ = std::make_unique<std::thread>([this]() { this->upload_thread(); });
}
}
void upload_manager::stop() {
unique_mutex_lock upload_lock(upload_mutex_);
if (upload_thread_) {
stop_requested_ = true;
upload_notify_.notify_all();
upload_lock.unlock();
upload_thread_->join();
upload_thread_.reset();
stop_requested_ = false;
event_system::instance().raise<service_shutdown>("upload_manager");
}
}
void upload_manager::upload_thread() {
unique_mutex_lock upload_lock(upload_mutex_);
upload_lock.unlock();
std::deque<std::shared_ptr<upload>> completed;
const auto process_completed = [&]() {
while (not completed.empty()) {
upload_lock.lock();
auto u = completed.front();
completed.pop_front();
upload_lock.unlock();
u->thread->join();
u->thread.reset();
u->completed = true;
}
};
const auto process_next = [this, &completed]() {
if (not upload_queue_.empty()) {
auto *next_upload = upload_queue_.front();
if (not next_upload->thread) {
upload_queue_.pop_front();
next_upload->completed = false;
active_uploads_.emplace_back(next_upload);
auto u = upload_lookup_[next_upload->api_path];
auto upload_handler = [this, &completed, u]() {
json data, err;
const auto ret = this->upload_handler_(*u, data, err);
unique_mutex_lock upload_lock(upload_mutex_);
if (ret == api_error::success) {
upload_lookup_.erase(u->api_path);
upload_db_->Delete(rocksdb::WriteOptions(), u->api_path);
upload_completed_(u->api_path, u->source_path, data);
event_system::instance().raise<file_upload_completed>(u->api_path, u->source_path);
} else {
event_system::instance().raise<file_upload_failed>(u->api_path, u->source_path,
err.dump(2));
if (u->cancel) {
upload_lookup_.erase(u->api_path);
} else if (not api_file_exists_(u->api_path) ||
not utils::file::is_file(u->source_path)) {
upload_lookup_.erase(u->api_path);
upload_db_->Delete(rocksdb::WriteOptions(), u->api_path);
event_system::instance().raise<file_upload_not_found>(u->api_path, u->source_path);
} else {
upload_queue_.emplace_back(u.get());
u->retry = true;
}
}
utils::remove_element_from(active_uploads_, u.get());
completed.emplace_back(u);
upload_notify_.notify_all();
upload_lock.unlock();
};
u->thread = std::make_unique<std::thread>(std::move(upload_handler));
}
}
};
const auto has_only_retries = [this]() -> bool {
return std::find_if(upload_queue_.begin(), upload_queue_.end(), [](const auto &upload) -> bool {
return not upload->retry;
}) == upload_queue_.end();
};
const auto cancel_all_active = [this, &process_completed, &upload_lock]() {
upload_lock.lock();
for (auto &upload : upload_lookup_) {
upload.second->cancel = true;
}
upload_lock.unlock();
process_completed();
};
const auto drain_queue = [this, &process_completed, &process_next, &upload_lock]() {
while (not upload_lookup_.empty()) {
upload_lock.lock();
process_next();
upload_lock.unlock();
process_completed();
}
};
const auto run_queue_loop = [this, &has_only_retries, &process_completed, &process_next,
&upload_lock]() {
while (not stop_requested_) {
upload_lock.lock();
if ((active_uploads_.size() >= config_.get_max_upload_count()) || upload_queue_.empty()) {
upload_notify_.wait_for(upload_lock, 5s);
if (not stop_requested_ && has_only_retries()) {
upload_notify_.wait_for(upload_lock, 5s);
}
} else {
process_next();
}
upload_lock.unlock();
process_completed();
}
};
run_queue_loop();
cancel_all_active();
drain_queue();
}
} // namespace repertory

21
src/utils/action_queue.cpp Normal file
View File

@@ -0,0 +1,21 @@
#include "utils/action_queue.hpp"
#include "types/repertory.hpp"
namespace repertory::utils::action_queue {
action_queue::action_queue(const std::string &id,
std::uint8_t max_concurrent_actions)
: single_thread_service_base("action_queue_" + id),
id_(id),
max_concurrent_actions_(max_concurrent_actions) {}
void action_queue::service_function() {
//
}
void action_queue::push(std::function<void()> action) {
unique_mutex_lock queue_lock(queue_mtx_);
queue_.emplace_back(action);
queue_notify_.notify_all();
}
} // namespace repertory::utils::action_queue

158
src/utils/cli_utils.cpp
View File

@@ -1,22 +1,26 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "utils/cli_utils.hpp"
#include "app_config.hpp"
#include "utils/file_utils.hpp"
#include "utils/path_utils.hpp"
@@ -24,18 +28,18 @@
#include "utils/utils.hpp"
namespace repertory::utils::cli {
void get_api_authentication_data(std::string &user, std::string &password, std::uint16_t &port,
const provider_type &pt, const std::string &data_directory) {
void get_api_authentication_data(std::string &user, std::string &password,
std::uint16_t &port, const provider_type &pt,
const std::string &data_directory) {
const auto cfg_file_path = utils::path::combine(
data_directory.empty() ? app_config::default_data_directory(pt) : data_directory,
data_directory.empty() ? app_config::default_data_directory(pt)
: data_directory,
{"config.json"});
json data;
auto success = false;
for (auto i = 0; not(success = utils::file::read_json_file(cfg_file_path, data)) && (i < 20);
i++) {
std::this_thread::sleep_for(100ms);
}
const auto success = utils::retryable_action([&]() -> bool {
return utils::file::read_json_file(cfg_file_path, data);
});
if (success) {
if (user.empty() && password.empty()) {
@@ -46,29 +50,24 @@ void get_api_authentication_data(std::string &user, std::string &password, std::
}
}
provider_type get_provider_type_from_args(const int &argc, char *argv[]) {
auto pt = provider_type::unknown;
auto get_provider_type_from_args(int argc, char *argv[]) -> provider_type {
#if defined(REPERTORY_ENABLE_S3)
if ((pt == provider_type::unknown) && (has_option(argc, argv, options::s3_option))) {
pt = provider_type::s3;
if (has_option(argc, argv, options::s3_option)) {
return provider_type::s3;
}
#endif // defined(REPERTORY_ENABLE_S3)
#if defined(REPERTORY_ENABLE_SKYNET)
if ((pt == provider_type::unknown) && (has_option(argc, argv, options::skynet_option))) {
pt = provider_type::skynet;
if (has_option(argc, argv, options::remote_mount_option)) {
return provider_type::remote;
}
#endif // defined(REPERTORY_ENABLE_SKYNET)
if ((pt == provider_type::unknown) && (has_option(argc, argv, options::remote_mount_option))) {
pt = provider_type::remote;
if (has_option(argc, argv, options::encrypt_option)) {
return provider_type::encrypt;
}
if (pt == provider_type::unknown) {
pt = provider_type::sia;
}
return pt;
return provider_type::sia;
}
bool has_option(const int &argc, char *argv[], const std::string &option_name) {
auto has_option(int argc, char *argv[], const std::string &option_name)
-> bool {
auto ret = false;
for (int i = 0; not ret && (i < argc); i++) {
ret = (option_name == argv[i]);
@@ -76,12 +75,12 @@ bool has_option(const int &argc, char *argv[], const std::string &option_name) {
return ret;
}
bool has_option(const int &argc, char *argv[], const option &opt) {
auto has_option(int argc, char *argv[], const option &opt) -> bool {
return has_option(argc, argv, opt[0u]) || has_option(argc, argv, opt[1u]);
}
std::vector<std::string> parse_option(const int &argc, char *argv[], const std::string &option_name,
std::uint8_t count) {
auto parse_option(int argc, char *argv[], const std::string &option_name,
std::uint8_t count) -> std::vector<std::string> {
std::vector<std::string> ret;
auto found = false;
for (auto i = 0; not found && (i < argc); i++) {
@@ -97,14 +96,16 @@ std::vector<std::string> parse_option(const int &argc, char *argv[], const std::
return ret;
}
exit_code parse_string_option(const int &argc, char **argv, const option &opt, std::string &value) {
auto parse_string_option(int argc, char **argv, const option &opt,
std::string &value) -> exit_code {
auto ret = exit_code::success;
if (has_option(argc, argv, opt[0u]) || has_option(argc, argv, opt[1u])) {
auto data = parse_option(argc, argv, opt[0u], 1u);
if (data.empty()) {
data = parse_option(argc, argv, opt[1u], 1u);
if (data.empty()) {
std::cerr << "Invalid syntax for '" << opt[0u] << "," << opt[1u] << '\'' << std::endl;
std::cerr << "Invalid syntax for '" << opt[0u] << "," << opt[1u] << '\''
<< std::endl;
ret = exit_code::invalid_syntax;
}
}
@@ -117,16 +118,19 @@ exit_code parse_string_option(const int &argc, char **argv, const option &opt, s
return ret;
}
std::vector<std::string> parse_drive_options(const int &argc, char **argv, provider_type &pt,
std::string &data_directory) {
auto parse_drive_options(int argc, char **argv,
[[maybe_unused]] provider_type &pt,
[[maybe_unused]] std::string &data_directory)
-> std::vector<std::string> {
// Strip out options from command line
const auto &option_list = options::option_list;
std::vector<std::string> drive_args;
for (int i = 0; i < argc; i++) {
const auto &a = argv[i];
if (std::find_if(option_list.begin(), option_list.end(), [&a](const auto &pair) -> bool {
return ((pair[0] == a) || (pair[1] == a));
}) == option_list.end()) {
if (std::find_if(option_list.begin(), option_list.end(),
[&a](const auto &pair) -> bool {
return ((pair[0] == a) || (pair[1] == a));
}) == option_list.end()) {
drive_args.emplace_back(argv[i]);
} else if ((std::string(argv[i]) == options::remote_mount_option[0]) ||
(std::string(argv[i]) == options::remote_mount_option[1])) {
@@ -160,17 +164,14 @@ std::vector<std::string> parse_drive_options(const int &argc, char **argv, provi
pt = provider_type::s3;
}
#endif // defined(REPERTORY_ENABLE_S3)
#if defined(REPERTORY_ENABLE_SKYNET)
if ((fuse_option.find("sk") == 0) || (fuse_option.find("skynet") == 0)) {
pt = provider_type::skynet;
}
#endif // defined(REPERTORY_ENABLE_SKYNET)
if ((fuse_option.find("dd") == 0) || (fuse_option.find("data_directory") == 0)) {
if ((fuse_option.find("dd") == 0) ||
(fuse_option.find("data_directory") == 0)) {
const auto data = utils::string::split(fuse_option, '=');
if (data.size() == 2) {
data_directory = data[1];
} else {
std::cerr << "Invalid syntax for '-dd,--data_directory'" << std::endl;
std::cerr << "Invalid syntax for '-dd,--data_directory'"
<< std::endl;
exit(3);
}
} else {
@@ -197,25 +198,44 @@ std::vector<std::string> parse_drive_options(const int &argc, char **argv, provi
}
}
#if FUSE_USE_VERSION < 30
{
auto it = std::remove_if(
fuse_flags_list.begin(), fuse_flags_list.end(),
[](const auto &opt) -> bool { return opt.find("hard_remove") == 0; });
if (it == fuse_flags_list.end()) {
fuse_flags_list.emplace_back("hard_remove");
}
}
#endif
#ifdef __APPLE__
auto it = std::remove_if(fuse_flags_list.begin(), fuse_flags_list.end(),
[](const auto &opt) -> bool { return opt.find("volname") == 0; });
if (it != fuse_flags_list.end()) {
fuse_flags_list.erase(it, fuse_flags_list.end());
}
fuse_flags_list.emplace_back("volname=Repertory" + app_config::get_provider_display_name(pt));
{
auto it = std::remove_if(
fuse_flags_list.begin(), fuse_flags_list.end(),
[](const auto &opt) -> bool { return opt.find("volname") == 0; });
if (it != fuse_flags_list.end()) {
fuse_flags_list.erase(it, fuse_flags_list.end());
}
fuse_flags_list.emplace_back("volname=Repertory" +
app_config::get_provider_display_name(pt));
it = std::remove_if(fuse_flags_list.begin(), fuse_flags_list.end(),
[](const auto &opt) -> bool { return opt.find("daemon_timeout") == 0; });
if (it != fuse_flags_list.end()) {
fuse_flags_list.erase(it, fuse_flags_list.end());
}
fuse_flags_list.emplace_back("daemon_timeout=300");
it = std::remove_if(fuse_flags_list.begin(), fuse_flags_list.end(),
[](const auto &opt) -> bool {
return opt.find("daemon_timeout") == 0;
});
if (it != fuse_flags_list.end()) {
fuse_flags_list.erase(it, fuse_flags_list.end());
}
fuse_flags_list.emplace_back("daemon_timeout=300");
it = std::remove_if(fuse_flags_list.begin(), fuse_flags_list.end(),
[](const auto &opt) -> bool { return opt.find("noappledouble") == 0; });
if (it == fuse_flags_list.end()) {
fuse_flags_list.emplace_back("noappledouble");
it = std::remove_if(fuse_flags_list.begin(), fuse_flags_list.end(),
[](const auto &opt) -> bool {
return opt.find("noappledouble") == 0;
});
if (it == fuse_flags_list.end()) {
fuse_flags_list.emplace_back("noappledouble");
}
}
#endif // __APPLE__

222
src/utils/encrypting_reader.cpp
View File

@@ -1,31 +1,38 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "utils/encrypting_reader.hpp"
#include "events/events.hpp"
#include "events/event_system.hpp"
#include "events/events.hpp"
#include "types/repertory.hpp"
#include "utils/encryption.hpp"
#include "utils/error_utils.hpp"
#include "utils/utils.hpp"
namespace repertory::utils::encryption {
class encrypting_streambuf final : public encrypting_reader::streambuf {
public:
explicit encrypting_streambuf(const encrypting_reader &reader) : reader_(reader) {
explicit encrypting_streambuf(const encrypting_reader &reader)
: reader_(reader) {
setg(reinterpret_cast<char *>(0), reinterpret_cast<char *>(0),
reinterpret_cast<char *>(reader_.get_total_size()));
}
@@ -36,9 +43,10 @@ private:
encrypting_reader reader_;
protected:
pos_type seekoff(off_type off, std::ios_base::seekdir dir,
std::ios_base::openmode which = std::ios_base::out |
std::ios_base::in) override {
auto seekoff(off_type off, std::ios_base::seekdir dir,
std::ios_base::openmode which = std::ios_base::out |
std::ios_base::in)
-> pos_type override {
if ((which & std::ios_base::in) != std::ios_base::in) {
throw std::runtime_error("output is not supported");
}
@@ -46,9 +54,11 @@ protected:
const auto set_position = [this](char *next) -> pos_type {
if ((next <= egptr()) && (next >= eback())) {
setg(eback(), next, reinterpret_cast<char *>(reader_.get_total_size()));
return pos_type(reinterpret_cast<std::uintptr_t>(gptr()));
return static_cast<std::streamoff>(
reinterpret_cast<std::uintptr_t>(gptr()));
}
return pos_type(traits_type::eof());
return {traits_type::eof()};
};
switch (dir) {
@@ -65,12 +75,14 @@ protected:
return encrypting_reader::streambuf::seekoff(off, dir, which);
}
pos_type seekpos(pos_type pos, std::ios_base::openmode which = std::ios_base::out |
std::ios_base::in) override {
auto seekpos(pos_type pos,
std::ios_base::openmode which = std::ios_base::out |
std::ios_base::in)
-> pos_type override {
return seekoff(pos, std::ios_base::beg, which);
}
int_type uflow() override {
auto uflow() -> int_type override {
auto ret = underflow();
if (ret == traits_type::eof()) {
return ret;
@@ -81,12 +93,13 @@ protected:
return ret;
}
int_type underflow() override {
auto underflow() -> int_type override {
if (gptr() == egptr()) {
return traits_type::eof();
}
reader_.set_read_position(static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(gptr())));
reader_.set_read_position(
static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(gptr())));
char c{};
const auto res = reader_.reader_function(&c, 1u, 1u, &reader_);
@@ -97,12 +110,13 @@ protected:
return c;
}
std::streamsize xsgetn(char *ptr, std::streamsize count) override {
auto xsgetn(char *ptr, std::streamsize count) -> std::streamsize override {
if (gptr() == egptr()) {
return traits_type::eof();
}
reader_.set_read_position(static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(gptr())));
reader_.set_read_position(
static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(gptr())));
const auto res = reader_.reader_function(ptr, 1u, count, &reader_);
if ((res == reader_.get_error_return()) ||
@@ -110,14 +124,16 @@ protected:
return traits_type::eof();
}
setg(eback(), gptr() + res, reinterpret_cast<char *>(reader_.get_total_size()));
setg(eback(), gptr() + res,
reinterpret_cast<char *>(reader_.get_total_size()));
return res;
}
};
class encrypting_reader_iostream final : public encrypting_reader::iostream {
public:
explicit encrypting_reader_iostream(std::unique_ptr<encrypting_streambuf> buffer)
explicit encrypting_reader_iostream(
std::unique_ptr<encrypting_streambuf> buffer)
: encrypting_reader::iostream(buffer.get()), buffer_(std::move(buffer)) {}
~encrypting_reader_iostream() override = default;
@@ -127,42 +143,101 @@ private:
};
const std::size_t encrypting_reader::header_size_ = ([]() {
CryptoPP::XChaCha20Poly1305::Encryption enc;
return enc.IVSize() + enc.DigestSize();
return crypto_aead_xchacha20poly1305_IETF_NPUBBYTES +
crypto_aead_xchacha20poly1305_IETF_ABYTES;
})();
const std::size_t encrypting_reader::data_chunk_size_ = (8u * 1024u * 1024u);
const std::size_t encrypting_reader::encrypted_chunk_size_ = data_chunk_size_ + header_size_;
const std::size_t encrypting_reader::encrypted_chunk_size_ =
data_chunk_size_ + header_size_;
encrypting_reader::encrypting_reader(const std::string &file_name, const std::string &source_path,
const bool &stop_requested, const std::string &token,
const size_t error_return)
encrypting_reader::encrypting_reader(
const std::string &file_name, const std::string &source_path,
stop_type &stop_requested, const std::string &token,
std::optional<std::string> relative_parent_path, const size_t error_return)
: key_(utils::encryption::generate_key(token)),
stop_requested_(stop_requested),
error_return_(error_return) {
const auto res = native_file::open(source_path, source_file_);
const auto res = native_file::open(
source_path, not relative_parent_path.has_value(), source_file_);
if (res != api_error::success) {
throw std::runtime_error("file open failed: " + source_path + ':' + api_error_to_string(res));
throw std::runtime_error("file open failed|src|" + source_path + '|' +
api_error_to_string(res));
}
std::vector<char> result;
data_buffer result;
utils::encryption::encrypt_data(key_, file_name.c_str(),
strnlen(file_name.c_str(), file_name.size()), result);
strnlen(file_name.c_str(), file_name.size()),
result);
encrypted_file_name_ = utils::to_hex_string(result);
if (relative_parent_path.has_value()) {
for (const auto &part :
std::filesystem::path(relative_parent_path.value())) {
utils::encryption::encrypt_data(
key_, part.string().c_str(),
strnlen(part.string().c_str(), part.string().size()), result);
encrypted_file_path_ += '/' + utils::to_hex_string(result);
}
encrypted_file_path_ += '/' + encrypted_file_name_;
}
std::uint64_t file_size{};
if (not utils::file::get_file_size(source_path, file_size)) {
throw std::runtime_error("get file size failed: " + source_path + ':' +
throw std::runtime_error("get file size failed|src|" + source_path + '|' +
std::to_string(utils::get_last_error_code()));
}
const auto total_chunks = static_cast<std::size_t>(
utils::divide_with_ceiling(file_size, static_cast<std::uint64_t>(data_chunk_size_)));
total_size_ = file_size + (total_chunks * encrypting_reader::get_header_size());
const auto total_chunks = static_cast<std::size_t>(utils::divide_with_ceiling(
file_size, static_cast<std::uint64_t>(data_chunk_size_)));
total_size_ =
file_size + (total_chunks * encrypting_reader::get_header_size());
last_data_chunk_ = total_chunks - 1u;
last_data_chunk_size_ =
static_cast<std::size_t>((file_size <= data_chunk_size_) ? file_size
: (file_size % data_chunk_size_) ? file_size % data_chunk_size_
: data_chunk_size_);
last_data_chunk_size_ = static_cast<std::size_t>(
(file_size <= data_chunk_size_) ? file_size
: (file_size % data_chunk_size_) ? file_size % data_chunk_size_
: data_chunk_size_);
iv_list_.resize(total_chunks);
for (auto &iv : iv_list_) {
randombytes_buf(iv.data(), iv.size());
}
}
encrypting_reader::encrypting_reader(
const std::string &encrypted_file_path, const std::string &source_path,
stop_type &stop_requested, const std::string &token,
std::vector<
std::array<unsigned char, crypto_aead_xchacha20poly1305_IETF_NPUBBYTES>>
iv_list,
const size_t error_return)
: key_(utils::encryption::generate_key(token)),
stop_requested_(stop_requested),
error_return_(error_return) {
const auto res = native_file::open(source_path, false, source_file_);
if (res != api_error::success) {
throw std::runtime_error("file open failed|src|" + source_path + '|' +
api_error_to_string(res));
}
encrypted_file_path_ = encrypted_file_path;
encrypted_file_name_ =
std::filesystem::path(encrypted_file_path_).filename().string();
std::uint64_t file_size{};
if (not utils::file::get_file_size(source_path, file_size)) {
throw std::runtime_error("get file size failed|src|" + source_path + '|' +
std::to_string(utils::get_last_error_code()));
}
const auto total_chunks = static_cast<std::size_t>(utils::divide_with_ceiling(
file_size, static_cast<std::uint64_t>(data_chunk_size_)));
total_size_ =
file_size + (total_chunks * encrypting_reader::get_header_size());
last_data_chunk_ = total_chunks - 1u;
last_data_chunk_size_ = static_cast<std::size_t>(
(file_size <= data_chunk_size_) ? file_size
: (file_size % data_chunk_size_) ? file_size % data_chunk_size_
: data_chunk_size_);
iv_list_ = std::move(iv_list);
}
encrypting_reader::encrypting_reader(const encrypting_reader &r)
@@ -171,6 +246,8 @@ encrypting_reader::encrypting_reader(const encrypting_reader &r)
error_return_(r.error_return_),
chunk_buffers_(r.chunk_buffers_),
encrypted_file_name_(r.encrypted_file_name_),
encrypted_file_path_(r.encrypted_file_path_),
iv_list_(r.iv_list_),
last_data_chunk_(r.last_data_chunk_),
last_data_chunk_size_(r.last_data_chunk_size_),
read_offset_(r.read_offset_),
@@ -183,23 +260,41 @@ encrypting_reader::~encrypting_reader() {
}
}
std::uint64_t encrypting_reader::calculate_decrypted_size(const std::uint64_t &total_size) {
auto encrypting_reader::calculate_decrypted_size(std::uint64_t total_size)
-> std::uint64_t {
return total_size - (utils::divide_with_ceiling(
total_size, static_cast<std::uint64_t>(get_encrypted_chunk_size())) *
total_size, static_cast<std::uint64_t>(
get_encrypted_chunk_size())) *
get_header_size());
}
std::shared_ptr<encrypting_reader::iostream> encrypting_reader::create_iostream() const {
auto encrypting_reader::calculate_encrypted_size(const std::string &source_path)
-> std::uint64_t {
std::uint64_t file_size{};
if (not utils::file::get_file_size(source_path, file_size)) {
throw std::runtime_error("get file size failed|src|" + source_path + '|' +
std::to_string(utils::get_last_error_code()));
}
const auto total_chunks = static_cast<std::size_t>(utils::divide_with_ceiling(
file_size, static_cast<std::uint64_t>(data_chunk_size_)));
return file_size + (total_chunks * encrypting_reader::get_header_size());
}
auto encrypting_reader::create_iostream() const
-> std::shared_ptr<encrypting_reader::iostream> {
return std::make_shared<encrypting_reader_iostream>(
std::make_unique<encrypting_streambuf>(*this));
}
size_t encrypting_reader::reader_function(char *buffer, size_t size, size_t nitems) {
const auto read_size = static_cast<std::size_t>(
std::min(static_cast<std::uint64_t>(size * nitems), total_size_ - read_offset_));
auto encrypting_reader::reader_function(char *buffer, size_t size,
size_t nitems) -> size_t {
const auto read_size = static_cast<std::size_t>(std::min(
static_cast<std::uint64_t>(size * nitems), total_size_ - read_offset_));
auto chunk = static_cast<std::size_t>(read_offset_ / encrypted_chunk_size_);
auto chunk_offset = static_cast<std::size_t>(read_offset_ % encrypted_chunk_size_);
auto chunk_offset =
static_cast<std::size_t>(read_offset_ % encrypted_chunk_size_);
std::size_t total_read = 0u;
auto ret = false;
@@ -210,21 +305,25 @@ size_t encrypting_reader::reader_function(char *buffer, size_t size, size_t nite
while (not stop_requested_ && ret && remain) {
if (chunk_buffers_.find(chunk) == chunk_buffers_.end()) {
auto &chunk_buffer = chunk_buffers_[chunk];
std::vector<char> file_data(chunk == last_data_chunk_ ? last_data_chunk_size_
: data_chunk_size_);
data_buffer file_data(chunk == last_data_chunk_
? last_data_chunk_size_
: data_chunk_size_);
chunk_buffer.resize(file_data.size() + get_header_size());
std::size_t bytes_read{};
if ((ret = source_file_->read_bytes(&file_data[0u], file_data.size(),
chunk * data_chunk_size_, bytes_read))) {
utils::encryption::encrypt_data(key_, file_data, chunk_buffer);
chunk * data_chunk_size_,
bytes_read))) {
utils::encryption::encrypt_data(iv_list_.at(chunk), key_, file_data,
chunk_buffer);
}
} else if (chunk) {
chunk_buffers_.erase(chunk - 1u);
}
auto &chunk_buffer = chunk_buffers_[chunk];
const auto to_read = std::min(chunk_buffer.size() - chunk_offset, remain);
const auto to_read =
std::min(chunk_buffer.size() - chunk_offset, remain);
std::memcpy(buffer + total_read, &chunk_buffer[chunk_offset], to_read);
total_read += to_read;
remain -= to_read;
@@ -233,12 +332,13 @@ size_t encrypting_reader::reader_function(char *buffer, size_t size, size_t nite
read_offset_ += to_read;
}
} catch (const std::exception &e) {
event_system::instance().raise<repertory_exception>(__FUNCTION__,
e.what() ? e.what() : "unkown error");
utils::error::raise_error(__FUNCTION__, e, "exception occurred");
ret = false;
}
}
return stop_requested_ ? CURL_READFUNC_ABORT : ret ? total_read : error_return_;
return stop_requested_ ? CURL_READFUNC_ABORT
: ret ? total_read
: error_return_;
}
} // namespace repertory::utils::encryption

125
src/utils/encryption.cpp
View File

@@ -1,91 +1,136 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "utils/encryption.hpp"
#include "events/events.hpp"
#include "events/event_system.hpp"
#include "events/events.hpp"
#include "types/repertory.hpp"
#include "utils/encrypting_reader.hpp"
#include "utils/utils.hpp"
namespace repertory::utils::encryption {
api_error decrypt_file_name(const std::string &encryption_token, std::string &file_name) {
std::vector<char> buffer;
auto decrypt_file_path(const std::string &encryption_token,
std::string &file_path) -> api_error {
std::string decrypted_file_path{};
for (const auto &part : std::filesystem::path(file_path)) {
auto file_name = part.string();
if (file_name == "/") {
continue;
}
auto res = decrypt_file_name(encryption_token, file_name);
if (res != api_error::success) {
return res;
}
decrypted_file_path += '/' + file_name;
}
file_path = decrypted_file_path;
return api_error::success;
}
auto decrypt_file_name(const std::string &encryption_token,
std::string &file_name) -> api_error {
data_buffer buffer;
if (not utils::from_hex_string(file_name, buffer)) {
return api_error::error;
}
file_name.clear();
if (not utils::encryption::decrypt_data(encryption_token, buffer, file_name)) {
if (not utils::encryption::decrypt_data(encryption_token, buffer,
file_name)) {
return api_error::decryption_error;
}
return api_error::success;
}
CryptoPP::SecByteBlock generate_key(const std::string &encryption_token) {
CryptoPP::SecByteBlock key(CryptoPP::SHA256::DIGESTSIZE);
CryptoPP::SHA256().CalculateDigest(
reinterpret_cast<CryptoPP::byte *>(&key[0u]),
reinterpret_cast<const CryptoPP::byte *>(encryption_token.c_str()),
strnlen(encryption_token.c_str(), encryption_token.size()));
auto generate_key(const std::string &encryption_token) -> key_type {
crypto_hash_sha256_state state{};
auto res = crypto_hash_sha256_init(&state);
if (res != 0) {
throw std::runtime_error("failed to initialize sha256|" +
std::to_string(res));
}
return key;
if ((res = crypto_hash_sha256_update(
&state,
reinterpret_cast<const unsigned char *>(encryption_token.c_str()),
strnlen(encryption_token.c_str(), encryption_token.size()))) != 0) {
throw std::runtime_error("failed to update sha256|" + std::to_string(res));
}
key_type ret{};
if ((res = crypto_hash_sha256_final(&state, ret.data())) != 0) {
throw std::runtime_error("failed to finalize sha256|" +
std::to_string(res));
}
return ret;
}
api_error read_encrypted_range(
const http_range &range, const CryptoPP::SecByteBlock &key,
const std::function<api_error(std::vector<char> &ct, const std::uint64_t &start_offset,
const std::uint64_t &end_offset)> &reader,
const std::uint64_t &total_size, std::vector<char> &data) {
static constexpr const auto &encrypted_chunk_size =
auto read_encrypted_range(
const http_range &range, const key_type &key,
const std::function<api_error(data_buffer &ct, std::uint64_t start_offset,
std::uint64_t end_offset)> &reader,
std::uint64_t total_size, data_buffer &data) -> api_error {
const auto encrypted_chunk_size =
utils::encryption::encrypting_reader::get_encrypted_chunk_size();
static constexpr const auto &data_chunk_size =
const auto data_chunk_size =
utils::encryption::encrypting_reader::get_data_chunk_size();
static constexpr const auto &header_size =
const auto header_size =
utils::encryption::encrypting_reader::get_header_size();
const auto start_chunk = static_cast<std::size_t>(range.begin / data_chunk_size);
const auto start_chunk =
static_cast<std::size_t>(range.begin / data_chunk_size);
const auto end_chunk = static_cast<std::size_t>(range.end / data_chunk_size);
auto remain = range.end - range.begin + 1u;
auto source_offset = static_cast<std::size_t>(range.begin % data_chunk_size);
for (std::size_t chunk = start_chunk; chunk <= end_chunk; chunk++) {
std::vector<char> ct;
data_buffer ct;
const auto start_offset = chunk * encrypted_chunk_size;
const auto end_offset =
std::min(start_offset + (total_size - (chunk * data_chunk_size)) + header_size - 1u,
static_cast<std::uint64_t>(start_offset + encrypted_chunk_size - 1u));
const auto end_offset = std::min(
start_offset + (total_size - (chunk * data_chunk_size)) + header_size -
1u,
static_cast<std::uint64_t>(start_offset + encrypted_chunk_size - 1u));
const auto result = reader(ct, start_offset, end_offset);
if (result != api_error::success) {
return result;
}
std::vector<char> source_buffer;
data_buffer source_buffer;
if (not utils::encryption::decrypt_data(key, ct, source_buffer)) {
return api_error::decryption_error;
}
ct.clear();
const auto data_size = static_cast<std::size_t>(
std::min(remain, static_cast<std::uint64_t>(data_chunk_size - source_offset)));
const auto data_size = static_cast<std::size_t>(std::min(
remain, static_cast<std::uint64_t>(data_chunk_size - source_offset)));
std::copy(source_buffer.begin() + source_offset,
source_buffer.begin() + source_offset + data_size, std::back_inserter(data));
source_buffer.begin() + source_offset + data_size,
std::back_inserter(data));
remain -= data_size;
source_offset = 0u;
}

172
src/utils/error_utils.cpp Normal file
View File

@@ -0,0 +1,172 @@
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "utils/error_utils.hpp"
#include "events/event_system.hpp"
#include "events/events.hpp"
#include "types/repertory.hpp"
namespace repertory::utils::error {
void raise_error(std::string_view function, std::string_view msg) {
event_system::instance().raise<repertory_exception>(
static_cast<std::string>(function), static_cast<std::string>(msg));
}
void raise_error(std::string_view function, const api_error &e,
std::string_view msg) {
event_system::instance().raise<repertory_exception>(
static_cast<std::string>(function),
static_cast<std::string>(msg) + "|err|" + api_error_to_string(e));
}
void raise_error(std::string_view function, const std::exception &e,
std::string_view msg) {
event_system::instance().raise<repertory_exception>(
static_cast<std::string>(function),
static_cast<std::string>(msg) + "|err|" +
(e.what() ? e.what() : "unknown error"));
}
void raise_error(std::string_view function, const json &e,
std::string_view msg) {
event_system::instance().raise<repertory_exception>(
static_cast<std::string>(function),
static_cast<std::string>(msg) + "|err|" + e.dump(2));
}
void raise_error(std::string_view function, std::int64_t e,
std::string_view msg) {
event_system::instance().raise<repertory_exception>(
static_cast<std::string>(function),
static_cast<std::string>(msg) + "|err|" + std::to_string(e));
}
void raise_error(std::string_view function, const api_error &e,
std::string_view file_path, std::string_view msg) {
event_system::instance().raise<repertory_exception>(
static_cast<std::string>(function),
static_cast<std::string>(msg) + "|sp|" +
static_cast<std::string>(file_path) + "|err|" +
api_error_to_string(e));
}
void raise_error(std::string_view function, std::int64_t e,
std::string_view file_path, std::string_view msg) {
event_system::instance().raise<repertory_exception>(
static_cast<std::string>(function),
static_cast<std::string>(msg) + "|sp|" +
static_cast<std::string>(file_path) + "|err|" + std::to_string(e));
}
void raise_error(std::string_view function, const std::exception &e,
std::string_view file_path, std::string_view msg) {
event_system::instance().raise<repertory_exception>(
static_cast<std::string>(function),
static_cast<std::string>(msg) + "|sp|" +
static_cast<std::string>(file_path) + "|err|" +
(e.what() ? e.what() : "unknown error"));
}
void raise_api_path_error(std::string_view function, std::string_view api_path,
const api_error &e, std::string_view msg) {
event_system::instance().raise<repertory_exception>(
static_cast<std::string>(function),
static_cast<std::string>(msg) + "|ap|" +
static_cast<std::string>(api_path) + "|err|" +
api_error_to_string(e));
}
void raise_api_path_error(std::string_view function, std::string_view api_path,
std::int64_t e, std::string_view msg) {
event_system::instance().raise<repertory_exception>(
static_cast<std::string>(function),
static_cast<std::string>(msg) + "|ap|" +
static_cast<std::string>(api_path) + "|err|" + std::to_string(e));
}
void raise_api_path_error(std::string_view function, std::string_view api_path,
const std::exception &e, std::string_view msg) {
event_system::instance().raise<repertory_exception>(
static_cast<std::string>(function),
static_cast<std::string>(msg) + "|ap|" +
static_cast<std::string>(api_path) + "|err|" +
(e.what() ? e.what() : "unknown error"));
}
void raise_api_path_error(std::string_view function, std::string_view api_path,
std::string_view source_path, const api_error &e,
std::string_view msg) {
event_system::instance().raise<repertory_exception>(
static_cast<std::string>(function),
static_cast<std::string>(msg) + "|ap|" +
static_cast<std::string>(api_path) + "|sp|" +
static_cast<std::string>(source_path) + "|err|" +
api_error_to_string(e));
}
void raise_api_path_error(std::string_view function, std::string_view api_path,
std::string_view source_path, std::int64_t e,
std::string_view msg) {
event_system::instance().raise<repertory_exception>(
static_cast<std::string>(function),
static_cast<std::string>(msg) + "|ap|" +
static_cast<std::string>(api_path) + "|sp|" +
static_cast<std::string>(source_path) + "|err|" + std::to_string(e));
}
void raise_api_path_error(std::string_view function, std::string_view api_path,
const json &e, std::string_view msg) {
event_system::instance().raise<repertory_exception>(
static_cast<std::string>(function),
static_cast<std::string>(msg) + "|ap|" +
static_cast<std::string>(api_path) + "|err|" + e.dump(2));
}
void raise_api_path_error(std::string_view function, std::string_view api_path,
std::string_view source_path, const std::exception &e,
std::string_view msg) {
event_system::instance().raise<repertory_exception>(
static_cast<std::string>(function),
static_cast<std::string>(msg) + "|ap|" +
static_cast<std::string>(api_path) + "|sp|" +
static_cast<std::string>(source_path) + "|err|" +
(e.what() ? e.what() : "unknown error"));
}
void raise_url_error(std::string_view function, std::string_view url,
CURLcode e, std::string_view msg) {
event_system::instance().raise<repertory_exception>(
static_cast<std::string>(function),
static_cast<std::string>(msg) + "|url|" + static_cast<std::string>(url) +
"|err|" + curl_easy_strerror(e));
}
void raise_url_error(std::string_view function, std::string_view url,
std::string_view source_path, const std::exception &e,
std::string_view msg) {
event_system::instance().raise<repertory_exception>(
static_cast<std::string>(function),
static_cast<std::string>(msg) + "|url|" + static_cast<std::string>(url) +
"|sp|" + static_cast<std::string>(source_path) + "|err|" +
(e.what() ? e.what() : "unknown error"));
}
} // namespace repertory::utils::error

410
src/utils/file_utils.cpp
View File

@@ -1,44 +1,34 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "utils/file_utils.hpp"
#include "types/repertory.hpp"
#include "utils/error_utils.hpp"
#include "utils/path_utils.hpp"
#include "utils/string_utils.hpp"
#include "utils/utils.hpp"
namespace repertory::utils::file {
api_error assign_and_get_native_file(filesystem_item &fi, native_file_ptr &nf) {
recur_mutex_lock l(*fi.lock);
if (fi.handle == REPERTORY_INVALID_HANDLE) {
const auto res = native_file::create_or_open(fi.source_path, nf);
if (res != api_error::success) {
return res;
}
fi.handle = nf->get_handle();
} else {
nf = native_file::attach(fi.handle);
}
return api_error::success;
}
std::uint64_t calculate_used_space(std::string path, const bool &recursive) {
auto calculate_used_space(std::string path, bool recursive) -> std::uint64_t {
path = utils::path::absolute(path);
std::uint64_t ret = 0u;
#ifdef _WIN32
@@ -50,7 +40,8 @@ std::uint64_t calculate_used_space(std::string path, const bool &recursive) {
const auto file_name = std::string(fd.cFileName);
if (fd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) {
if (recursive && (file_name != ".") && (file_name != "..")) {
ret += calculate_used_space(utils::path::combine(path, {file_name}), recursive);
ret += calculate_used_space(utils::path::combine(path, {file_name}),
recursive);
}
} else {
std::uint64_t file_size{};
@@ -67,12 +58,15 @@ std::uint64_t calculate_used_space(std::string path, const bool &recursive) {
struct dirent *de{};
while ((de = readdir(root)) != nullptr) {
if (de->d_type == DT_DIR) {
if (recursive && (strcmp(de->d_name, ".") != 0) && (strcmp(de->d_name, "..") != 0)) {
ret += calculate_used_space(utils::path::combine(path, {de->d_name}), recursive);
if (recursive && (strcmp(de->d_name, ".") != 0) &&
(strcmp(de->d_name, "..") != 0)) {
ret += calculate_used_space(utils::path::combine(path, {de->d_name}),
recursive);
}
} else {
std::uint64_t file_size{};
if (get_file_size(utils::path::combine(path, {de->d_name}), file_size)) {
if (get_file_size(utils::path::combine(path, {de->d_name}),
file_size)) {
ret += file_size;
}
}
@@ -88,7 +82,8 @@ void change_to_process_directory() {
#ifdef _WIN32
std::string file_name;
file_name.resize(MAX_PATH);
::GetModuleFileNameA(nullptr, &file_name[0u], static_cast<DWORD>(file_name.size()));
::GetModuleFileNameA(nullptr, &file_name[0u],
static_cast<DWORD>(file_name.size()));
std::string path = file_name.c_str();
::PathRemoveFileSpecA(&path[0u]);
@@ -106,19 +101,18 @@ void change_to_process_directory() {
#endif
}
bool copy_file(std::string from_path, std::string to_path) {
auto copy_file(std::string from_path, std::string to_path) -> bool {
from_path = utils::path::absolute(from_path);
to_path = utils::path::absolute(to_path);
if (is_file(from_path) && not is_directory(to_path)) {
boost::system::error_code ec{};
boost::filesystem::copy_file(from_path, to_path, ec);
return not ec.failed();
return std::filesystem::copy_file(from_path, to_path);
}
return false;
}
bool copy_directory_recursively(std::string from_path, std::string to_path) {
auto copy_directory_recursively(std::string from_path, std::string to_path)
-> bool {
from_path = utils::path::absolute(from_path);
to_path = utils::path::absolute(to_path);
auto ret = create_full_directory_path(to_path);
@@ -131,9 +125,11 @@ bool copy_directory_recursively(std::string from_path, std::string to_path) {
ret = true;
do {
if (fd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) {
if ((std::string(fd.cFileName) != ".") && (std::string(fd.cFileName) != "..")) {
ret = copy_directory_recursively(utils::path::combine(from_path, {fd.cFileName}),
utils::path::combine(to_path, {fd.cFileName}));
if ((std::string(fd.cFileName) != ".") &&
(std::string(fd.cFileName) != "..")) {
ret = copy_directory_recursively(
utils::path::combine(from_path, {fd.cFileName}),
utils::path::combine(to_path, {fd.cFileName}));
}
} else {
ret = copy_file(utils::path::combine(from_path, {fd.cFileName}),
@@ -149,9 +145,11 @@ bool copy_directory_recursively(std::string from_path, std::string to_path) {
struct dirent *de{};
while (ret && (de = readdir(root))) {
if (de->d_type == DT_DIR) {
if ((strcmp(de->d_name, ".") != 0) && (strcmp(de->d_name, "..") != 0)) {
ret = copy_directory_recursively(utils::path::combine(from_path, {de->d_name}),
utils::path::combine(to_path, {de->d_name}));
if ((strcmp(de->d_name, ".") != 0) &&
(strcmp(de->d_name, "..") != 0)) {
ret = copy_directory_recursively(
utils::path::combine(from_path, {de->d_name}),
utils::path::combine(to_path, {de->d_name}));
}
} else {
ret = copy_file(utils::path::combine(from_path, {de->d_name}),
@@ -167,15 +165,16 @@ bool copy_directory_recursively(std::string from_path, std::string to_path) {
return ret;
}
bool create_full_directory_path(std::string path) {
auto create_full_directory_path(std::string path) -> bool {
#ifdef _WIN32
const auto unicode_path = utils::string::from_utf8(utils::path::absolute(path));
const auto unicode_path =
utils::string::from_utf8(utils::path::absolute(path));
return is_directory(path) ||
(::SHCreateDirectory(nullptr, unicode_path.c_str()) == ERROR_SUCCESS);
#else
auto ret = true;
const auto paths =
utils::string::split(utils::path::absolute(path), utils::path::directory_seperator[0u]);
const auto paths = utils::string::split(utils::path::absolute(path),
utils::path::directory_seperator[0u]);
std::string current_path;
for (std::size_t i = 0u; ret && (i < paths.size()); i++) {
if (paths[i].empty()) { // Skip root
@@ -191,21 +190,22 @@ bool create_full_directory_path(std::string path) {
#endif
}
bool delete_directory(std::string path, const bool &recursive) {
auto delete_directory(std::string path, bool recursive) -> bool {
if (recursive) {
return delete_directory_recursively(path);
}
path = utils::path::absolute(path);
#ifdef _WIN32
return (not is_directory(path) ||
utils::retryable_action([&]() -> bool { return !!::RemoveDirectoryA(path.c_str()); }));
return (not is_directory(path) || utils::retryable_action([&]() -> bool {
return !!::RemoveDirectoryA(path.c_str());
}));
#else
return not is_directory(path) || (rmdir(path.c_str()) == 0);
#endif
}
bool delete_directory_recursively(std::string path) {
auto delete_directory_recursively(std::string path) -> bool {
path = utils::path::absolute(path);
#ifdef _WIN32
@@ -213,14 +213,16 @@ bool delete_directory_recursively(std::string path) {
const auto search = utils::path::combine(path, {"*.*"});
auto find = ::FindFirstFile(search.c_str(), &fd);
if (find != INVALID_HANDLE_VALUE) {
auto res = false;
auto res = true;
do {
if (fd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) {
if ((std::string(fd.cFileName) != ".") && (std::string(fd.cFileName) != "..")) {
res = delete_directory_recursively(utils::path::combine(path, {fd.cFileName}));
if ((std::string(fd.cFileName) != ".") &&
(std::string(fd.cFileName) != "..")) {
res = delete_directory_recursively(
utils::path::combine(path, {fd.cFileName}));
}
} else {
res = delete_file(utils::path::combine(path, {fd.cFileName}));
res = retry_delete_file(utils::path::combine(path, {fd.cFileName}));
}
} while (res && (::FindNextFile(find, &fd) != 0));
@@ -234,10 +236,11 @@ bool delete_directory_recursively(std::string path) {
while (res && (de = readdir(root))) {
if (de->d_type == DT_DIR) {
if ((strcmp(de->d_name, ".") != 0) && (strcmp(de->d_name, "..") != 0)) {
res = delete_directory_recursively(utils::path::combine(path, {de->d_name}));
res = delete_directory_recursively(
utils::path::combine(path, {de->d_name}));
}
} else {
res = delete_file(utils::path::combine(path, {de->d_name}));
res = retry_delete_file(utils::path::combine(path, {de->d_name}));
}
}
@@ -248,7 +251,7 @@ bool delete_directory_recursively(std::string path) {
return delete_directory(path, false);
}
bool delete_file(std::string path) {
auto delete_file(std::string path) -> bool {
path = utils::path::absolute(path);
#ifdef _WIN32
return (not is_file(path) || utils::retryable_action([&]() -> bool {
@@ -264,18 +267,55 @@ bool delete_file(std::string path) {
#endif
}
std::string generate_sha256(const std::string &file_path) {
std::string value;
CryptoPP::SHA256 hash;
CryptoPP::FileSource(
file_path.c_str(), true,
new CryptoPP::HashFilter(hash, new CryptoPP::HexEncoder(new CryptoPP::StringSink(value))));
return value;
auto generate_sha256(const std::string &file_path) -> std::string {
crypto_hash_sha256_state state{};
auto res = crypto_hash_sha256_init(&state);
if (res != 0) {
throw std::runtime_error("failed to initialize sha256|" +
std::to_string(res));
}
native_file_ptr nf;
if (native_file::open(file_path, nf) != api_error::success) {
throw std::runtime_error("failed to open file|" + file_path);
}
{
data_buffer buffer(1048576u);
std::uint64_t read_offset = 0u;
std::size_t bytes_read = 0u;
while (
nf->read_bytes(buffer.data(), buffer.size(), read_offset, bytes_read)) {
if (not bytes_read) {
break;
}
read_offset += bytes_read;
res = crypto_hash_sha256_update(
&state, reinterpret_cast<const unsigned char *>(buffer.data()),
bytes_read);
if (res != 0) {
nf->close();
throw std::runtime_error("failed to update sha256|" +
std::to_string(res));
}
}
nf->close();
}
std::array<unsigned char, crypto_hash_sha256_BYTES> out{};
res = crypto_hash_sha256_final(&state, out.data());
if (res != 0) {
throw std::runtime_error("failed to finalize sha256|" +
std::to_string(res));
}
return utils::to_hex_string(out);
}
std::uint64_t get_available_drive_space(const std::string &path) {
auto get_free_drive_space(const std::string &path) -> std::uint64_t {
#ifdef _WIN32
ULARGE_INTEGER li = {0};
ULARGE_INTEGER li{};
::GetDiskFreeSpaceEx(path.c_str(), &li, nullptr, nullptr);
return li.QuadPart;
#endif
@@ -290,12 +330,33 @@ std::uint64_t get_available_drive_space(const std::string &path) {
#if __APPLE__
struct statvfs st {};
statvfs(path.c_str(), &st);
return st.f_bavail * st.f_frsize;
return st.f_bfree * st.f_frsize;
#endif
}
std::deque<std::string> get_directory_files(std::string path, const bool &oldest_first,
const bool &recursive) {
auto get_total_drive_space(const std::string &path) -> std::uint64_t {
#ifdef _WIN32
ULARGE_INTEGER li{};
::GetDiskFreeSpaceEx(path.c_str(), nullptr, &li, nullptr);
return li.QuadPart;
#endif
#ifdef __linux__
std::uint64_t ret = 0;
struct statfs64 st {};
if (statfs64(path.c_str(), &st) == 0) {
ret = st.f_blocks * st.f_bsize;
}
return ret;
#endif
#if __APPLE__
struct statvfs st {};
statvfs(path.c_str(), &st);
return st.f_blocks * st.f_frsize;
#endif
}
auto get_directory_files(std::string path, bool oldest_first, bool recursive)
-> std::deque<std::string> {
path = utils::path::absolute(path);
std::deque<std::string> ret;
std::unordered_map<std::string, std::uint64_t> lookup;
@@ -305,20 +366,28 @@ std::deque<std::string> get_directory_files(std::string path, const bool &oldest
auto find = ::FindFirstFile(search.c_str(), &fd);
if (find != INVALID_HANDLE_VALUE) {
do {
if ((fd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != FILE_ATTRIBUTE_DIRECTORY) {
ULARGE_INTEGER li{};
li.HighPart = fd.ftLastWriteTime.dwHighDateTime;
li.LowPart = fd.ftLastWriteTime.dwLowDateTime;
const auto full_path = utils::path::combine(path, {fd.cFileName});
lookup[full_path] = li.QuadPart;
ret.emplace_back(full_path);
const auto full_path = utils::path::combine(path, {fd.cFileName});
if ((fd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) ==
FILE_ATTRIBUTE_DIRECTORY) {
if (recursive) {
const auto sub_files =
get_directory_files(full_path, oldest_first, recursive);
ret.insert(ret.end(), sub_files.begin(), sub_files.end());
} else {
ULARGE_INTEGER li{};
li.HighPart = fd.ftLastWriteTime.dwHighDateTime;
li.LowPart = fd.ftLastWriteTime.dwLowDateTime;
lookup[full_path] = li.QuadPart;
ret.emplace_back(full_path);
}
}
} while (::FindNextFile(find, &fd) != 0);
::FindClose(find);
std::sort(ret.begin(), ret.end(), [&](const auto &p1, const auto &p2) -> bool {
return (oldest_first != 0) == (lookup[p1] < lookup[p2]);
});
std::sort(ret.begin(), ret.end(),
[&](const auto &p1, const auto &p2) -> bool {
return (oldest_first != 0) == (lookup[p1] < lookup[p2]);
});
}
#else
auto *root = opendir(path.c_str());
@@ -327,9 +396,10 @@ std::deque<std::string> get_directory_files(std::string path, const bool &oldest
while ((de = readdir(root)) != nullptr) {
if (de->d_type == DT_DIR) {
if (recursive) {
const auto subFiles = get_directory_files(utils::path::combine(path, {de->d_name}),
oldest_first, recursive);
ret.insert(ret.end(), subFiles.begin(), subFiles.end());
const auto sub_files =
get_directory_files(utils::path::combine(path, {de->d_name}),
oldest_first, recursive);
ret.insert(ret.end(), sub_files.begin(), sub_files.end());
}
} else {
ret.emplace_back(utils::path::combine(path, {de->d_name}));
@@ -343,25 +413,28 @@ std::deque<std::string> get_directory_files(std::string path, const bool &oldest
stat(lookup_path.c_str(), &st);
#ifdef __APPLE__
lookup[lookup_path] = static_cast<std::uint64_t>(
(st.st_mtimespec.tv_sec * NANOS_PER_SECOND) + st.st_mtimespec.tv_nsec);
(st.st_mtimespec.tv_sec * NANOS_PER_SECOND) +
st.st_mtimespec.tv_nsec);
#else
lookup[lookup_path] =
static_cast<std::uint64_t>((st.st_mtim.tv_sec * NANOS_PER_SECOND) + st.st_mtim.tv_nsec);
lookup[lookup_path] = static_cast<std::uint64_t>(
(st.st_mtim.tv_sec * NANOS_PER_SECOND) + st.st_mtim.tv_nsec);
#endif
}
};
std::sort(ret.begin(), ret.end(), [&](const auto &p1, const auto &p2) -> bool {
add_to_lookup(p1);
add_to_lookup(p2);
return (oldest_first != 0) == (lookup[p1] < lookup[p2]);
});
std::sort(ret.begin(), ret.end(),
[&](const auto &p1, const auto &p2) -> bool {
add_to_lookup(p1);
add_to_lookup(p2);
return (oldest_first != 0) == (lookup[p1] < lookup[p2]);
});
}
#endif
return ret;
}
bool get_accessed_time(const std::string &path, std::uint64_t &accessed) {
auto get_accessed_time(const std::string &path, std::uint64_t &accessed)
-> bool {
auto ret = false;
accessed = 0;
#ifdef _WIN32
@@ -372,10 +445,11 @@ bool get_accessed_time(const std::string &path, std::uint64_t &accessed) {
struct stat st {};
if (stat(path.c_str(), &st) != -1) {
#ifdef __APPLE__
accessed = static_cast<uint64_t>(st.st_atimespec.tv_nsec +
(st.st_atimespec.tv_sec * NANOS_PER_SECOND));
accessed = static_cast<uint64_t>(
st.st_atimespec.tv_nsec + (st.st_atimespec.tv_sec * NANOS_PER_SECOND));
#else
accessed = static_cast<uint64_t>(st.st_atim.tv_nsec + (st.st_atim.tv_sec * NANOS_PER_SECOND));
accessed = static_cast<uint64_t>(st.st_atim.tv_nsec +
(st.st_atim.tv_sec * NANOS_PER_SECOND));
#endif
#endif
ret = true;
@@ -384,7 +458,8 @@ bool get_accessed_time(const std::string &path, std::uint64_t &accessed) {
return ret;
}
bool get_modified_time(const std::string &path, std::uint64_t &modified) {
auto get_modified_time(const std::string &path, std::uint64_t &modified)
-> bool {
auto ret = false;
modified = 0u;
#ifdef _WIN32
@@ -395,10 +470,11 @@ bool get_modified_time(const std::string &path, std::uint64_t &modified) {
struct stat st {};
if (stat(path.c_str(), &st) != -1) {
#ifdef __APPLE__
modified = static_cast<uint64_t>(st.st_mtimespec.tv_nsec +
(st.st_mtimespec.tv_sec * NANOS_PER_SECOND));
modified = static_cast<uint64_t>(
st.st_mtimespec.tv_nsec + (st.st_mtimespec.tv_sec * NANOS_PER_SECOND));
#else
modified = static_cast<uint64_t>(st.st_mtim.tv_nsec + (st.st_mtim.tv_sec * NANOS_PER_SECOND));
modified = static_cast<uint64_t>(st.st_mtim.tv_nsec +
(st.st_mtim.tv_sec * NANOS_PER_SECOND));
#endif
#endif
ret = true;
@@ -407,7 +483,7 @@ bool get_modified_time(const std::string &path, std::uint64_t &modified) {
return ret;
}
bool get_file_size(std::string path, std::uint64_t &file_size) {
auto get_file_size(std::string path, std::uint64_t &file_size) -> bool {
file_size = 0u;
path = utils::path::finalize(path);
@@ -433,7 +509,7 @@ bool get_file_size(std::string path, std::uint64_t &file_size) {
return (st.st_size >= 0);
}
bool is_directory(const std::string &path) {
auto is_directory(const std::string &path) -> bool {
#ifdef _WIN32
return ::PathIsDirectory(path.c_str()) != 0;
#else
@@ -442,36 +518,43 @@ bool is_directory(const std::string &path) {
#endif
}
bool is_file(const std::string &path) {
auto is_file(const std::string &path) -> bool {
#ifdef _WIN32
return (::PathFileExists(path.c_str()) && not ::PathIsDirectory(path.c_str()));
return (::PathFileExists(path.c_str()) &&
not ::PathIsDirectory(path.c_str()));
#else
struct stat st {};
return (not stat(path.c_str(), &st) && not S_ISDIR(st.st_mode));
#endif
}
bool is_modified_date_older_than(const std::string &path, const std::chrono::hours &hours) {
auto is_modified_date_older_than(const std::string &path,
const std::chrono::hours &hours) -> bool {
auto ret = false;
std::uint64_t modified = 0;
if (get_modified_time(path, modified)) {
const auto seconds = std::chrono::duration_cast<std::chrono::seconds>(hours);
const auto seconds =
std::chrono::duration_cast<std::chrono::seconds>(hours);
#ifdef _WIN32
return (std::chrono::system_clock::from_time_t(modified) + seconds) <
std::chrono::system_clock::now();
#else
return (modified + (seconds.count() * NANOS_PER_SECOND)) < utils::get_time_now();
return (modified + (seconds.count() * NANOS_PER_SECOND)) <
utils::get_time_now();
#endif
}
return ret;
}
bool move_file(std::string from, std::string to) {
auto move_file(std::string from, std::string to) -> bool {
from = utils::path::finalize(from);
to = utils::path::finalize(to);
const auto directory = utils::path::remove_file_name(to);
create_full_directory_path(directory);
if (not create_full_directory_path(directory)) {
return false;
}
#ifdef _WIN32
const bool ret = ::MoveFile(from.c_str(), to.c_str()) != 0;
#else
@@ -481,24 +564,7 @@ bool move_file(std::string from, std::string to) {
return ret;
}
api_error read_from_source(filesystem_item &fi, const std::size_t &read_size,
const std::uint64_t &read_offset, std::vector<char> &data) {
native_file_ptr nf;
const auto res = assign_and_get_native_file(fi, nf);
if (res != api_error::success) {
return res;
}
data.resize(read_size);
std::size_t bytes_read = 0u;
if (not nf->read_bytes(&data[0u], data.size(), read_offset, bytes_read)) {
return api_error::os_error;
}
return api_error::success;
}
std::vector<std::string> read_file_lines(const std::string &path) {
auto read_file_lines(const std::string &path) -> std::vector<std::string> {
std::vector<std::string> ret;
if (is_file(path)) {
std::ifstream fs(path);
@@ -512,30 +578,39 @@ std::vector<std::string> read_file_lines(const std::string &path) {
return ret;
}
bool read_json_file(const std::string &path, json &data) {
auto ret = false;
auto read_json_file(const std::string &path, json &data) -> bool {
if (not utils::file::is_file(path)) {
return true;
}
try {
std::ifstream file_stream(path.c_str());
if (file_stream.is_open()) {
std::stringstream ss;
ss << file_stream.rdbuf();
std::string json_text = ss.str();
if (json_text.empty()) {
try {
std::stringstream ss;
ss << file_stream.rdbuf();
std::string json_text = ss.str();
if (not json_text.empty()) {
data = json::parse(json_text.c_str());
}
file_stream.close();
} else {
data = json::parse(json_text.c_str());
ret = true;
return true;
} catch (const std::exception &e) {
file_stream.close();
throw e;
}
}
} catch (const std::exception &) {
} catch (const std::exception &e) {
utils::error::raise_error(__FUNCTION__, e, path,
"failed to read json file");
}
return ret;
return false;
}
bool reset_modified_time(const std::string &path) {
auto reset_modified_time(const std::string &path) -> bool {
auto ret = false;
#ifdef _WIN32
SYSTEMTIME st{};
@@ -543,9 +618,10 @@ bool reset_modified_time(const std::string &path) {
FILETIME ft{};
if ((ret = !!::SystemTimeToFileTime(&st, &ft))) {
auto handle = ::CreateFileA(path.c_str(), FILE_READ_ATTRIBUTES | FILE_WRITE_ATTRIBUTES,
FILE_SHARE_DELETE | FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr,
OPEN_EXISTING, 0, nullptr);
auto handle = ::CreateFileA(
path.c_str(), FILE_READ_ATTRIBUTES | FILE_WRITE_ATTRIBUTES,
FILE_SHARE_DELETE | FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr,
OPEN_EXISTING, 0, nullptr);
if ((ret = (handle != INVALID_HANDLE_VALUE))) {
ret = !!::SetFileTime(handle, nullptr, &ft, &ft);
::CloseHandle(handle);
@@ -561,21 +637,27 @@ bool reset_modified_time(const std::string &path) {
return ret;
}
api_error truncate_source(filesystem_item &fi, const std::uint64_t &size) {
native_file_ptr nf;
auto res = assign_and_get_native_file(fi, nf);
if (res != api_error::success) {
return res;
auto retry_delete_directory(const std::string &dir) -> bool {
auto deleted = false;
for (std::uint8_t i = 0u; not(deleted = delete_directory(dir)) && (i < 200u);
i++) {
std::this_thread::sleep_for(10ms);
}
if (not nf->truncate(size)) {
return api_error::os_error;
}
return api_error::success;
return deleted;
}
bool write_json_file(const std::string &path, const json &j) {
auto retry_delete_file(const std::string &file) -> bool {
auto deleted = false;
for (std::uint8_t i = 0u; not(deleted = delete_file(file)) && (i < 200u);
i++) {
std::this_thread::sleep_for(10ms);
}
return deleted;
}
auto write_json_file(const std::string &path, const json &j) -> bool {
std::string data;
{
std::stringstream ss;
@@ -586,27 +668,11 @@ bool write_json_file(const std::string &path, const json &j) {
auto ret = (native_file::create_or_open(path, nf) == api_error::success);
if (ret) {
std::size_t bytes_written = 0u;
ret = nf->truncate(0) && nf->write_bytes(&data[0u], data.size(), 0u, bytes_written);
ret = nf->truncate(0) &&
nf->write_bytes(&data[0u], data.size(), 0u, bytes_written);
nf->close();
}
return ret;
}
api_error write_to_source(filesystem_item &fi, const std::uint64_t &write_offset,
const std::vector<char> &data, std::size_t &bytes_written) {
bytes_written = 0u;
native_file_ptr nf;
auto res = assign_and_get_native_file(fi, nf);
if (res != api_error::success) {
return res;
}
if (not nf->write_bytes(&data[0u], data.size(), write_offset, bytes_written)) {
return api_error::os_error;
}
return api_error::success;
}
} // namespace repertory::utils::file

40
src/utils/global_data.cpp
View File

@@ -1,40 +0,0 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#include "utils/global_data.hpp"
namespace repertory {
global_data global_data::instance_;
void global_data::update_used_space(const std::uint64_t &file_size,
const std::uint64_t &new_file_size, const bool &cache_only) {
if (file_size > new_file_size) {
const auto diff = (file_size - new_file_size);
used_cache_space_ -= diff;
if (not cache_only) {
used_drive_space_ -= diff;
}
} else if (file_size < new_file_size) {
const auto diff = (new_file_size - file_size);
used_cache_space_ += diff;
if (not cache_only) {
used_drive_space_ += diff;
}
}
}
} // namespace repertory

153
src/utils/native_file.cpp
View File

@@ -1,26 +1,41 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "utils/native_file.hpp"
#include "types/repertory.hpp"
#include "utils/string_utils.hpp"
#include "utils/unix/unix_utils.hpp"
#include "utils/utils.hpp"
namespace repertory {
native_file_ptr native_file::clone(const native_file_ptr &nf) {
auto native_file::get_handle() -> native_handle { return handle_; }
native_file::~native_file() {
if (auto_close) {
close();
}
}
auto native_file::clone(const native_file_ptr &nf) -> native_file_ptr {
std::string source_path;
#ifdef _WIN32
@@ -31,53 +46,79 @@ native_file_ptr native_file::clone(const native_file_ptr &nf) {
source_path.resize(PATH_MAX + 1);
#ifdef __APPLE__
fcntl(nf->get_handle(), F_GETPATH, &source_path[0u]);
a
#else
readlink(("/proc/self/fd/" + utils::string::from_int64(nf->get_handle())).c_str(),
readlink(("/proc/self/fd/" + std::to_string(nf->get_handle())).c_str(),
&source_path[0u], source_path.size());
#endif
#endif
source_path = source_path.c_str();
native_file_ptr clone;
if (native_file::open(source_path, clone) != api_error::success) {
throw std::runtime_error("unable to clone file " + source_path);
auto res = native_file::open(source_path, clone);
if (res != api_error::success) {
throw std::runtime_error("unable to open file|sp|" + source_path + "|err|" +
api_error_to_string(res));
}
return clone;
}
api_error native_file::create_or_open(const std::string &source_path, native_file_ptr &nf) {
auto native_file::create_or_open(const std::string &source_path,
[[maybe_unused]] bool should_chmod,
native_file_ptr &nf) -> api_error {
#ifdef _WIN32
auto handle = ::CreateFileA(source_path.c_str(), GENERIC_READ | GENERIC_WRITE,
FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, OPEN_ALWAYS,
FILE_FLAG_RANDOM_ACCESS, nullptr);
FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr,
OPEN_ALWAYS, FILE_FLAG_RANDOM_ACCESS, nullptr);
#else
auto handle = ::open(source_path.c_str(), O_CREAT | O_RDWR | O_CLOEXEC, 0600u);
chmod(source_path.c_str(), 0600u);
auto handle = should_chmod ? ::open(source_path.c_str(),
O_CREAT | O_RDWR | O_CLOEXEC, 0600u)
: ::open(source_path.c_str(), O_RDWR | O_CLOEXEC);
if (should_chmod) {
chmod(source_path.c_str(), 0600u);
}
#endif
nf = native_file::attach(handle);
return ((handle == REPERTORY_INVALID_HANDLE) ? api_error::os_error : api_error::success);
return ((handle == REPERTORY_INVALID_HANDLE) ? api_error::os_error
: api_error::success);
}
api_error native_file::open(const std::string &source_path, native_file_ptr &nf) {
auto native_file::create_or_open(const std::string &source_path,
native_file_ptr &nf) -> api_error {
return create_or_open(source_path, true, nf);
}
auto native_file::open(const std::string &source_path, native_file_ptr &nf)
-> api_error {
return open(source_path, true, nf);
}
auto native_file::open(const std::string &source_path,
[[maybe_unused]] bool should_chmod, native_file_ptr &nf)
-> api_error {
#ifdef _WIN32
auto handle = ::CreateFileA(source_path.c_str(), GENERIC_READ | GENERIC_WRITE,
FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, OPEN_EXISTING,
FILE_FLAG_RANDOM_ACCESS, nullptr);
FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr,
OPEN_EXISTING, FILE_FLAG_RANDOM_ACCESS, nullptr);
#else
auto handle = ::open(source_path.c_str(), O_RDWR | O_CLOEXEC, 0600u);
chmod(source_path.c_str(), 0600u);
auto handle = should_chmod
? ::open(source_path.c_str(), O_RDWR | O_CLOEXEC, 0600u)
: ::open(source_path.c_str(), O_RDONLY | O_CLOEXEC);
if (should_chmod) {
chmod(source_path.c_str(), 0600u);
}
#endif
nf = native_file::attach(handle);
return ((handle == REPERTORY_INVALID_HANDLE) ? api_error::os_error : api_error::success);
return ((handle == REPERTORY_INVALID_HANDLE) ? api_error::os_error
: api_error::success);
}
bool native_file::allocate(const std::uint64_t &file_size) {
auto native_file::allocate(std::uint64_t file_size) -> bool {
#ifdef _WIN32
LARGE_INTEGER li{};
li.QuadPart = file_size;
return (::SetFilePointerEx(handle_, li, nullptr, FILE_BEGIN) && ::SetEndOfFile(handle_));
return (::SetFilePointerEx(handle_, li, nullptr, FILE_BEGIN) &&
::SetEndOfFile(handle_));
#endif
#ifdef __linux__
return (fallocate(handle_, 0, 0, file_size) >= 0);
@@ -98,16 +139,17 @@ void native_file::close() {
}
}
bool native_file::copy_from(const native_file_ptr &nf) {
auto native_file::copy_from(const native_file_ptr &nf) -> bool {
auto ret = false;
std::uint64_t file_size = 0u;
if ((ret = nf->get_file_size(file_size))) {
std::vector<char> buffer;
data_buffer buffer;
buffer.resize(65536u * 2u);
std::uint64_t offset = 0u;
while (ret && (file_size > 0u)) {
std::size_t bytes_read{};
if ((ret = nf->read_bytes(&buffer[0u], buffer.size(), offset, bytes_read))) {
if ((ret = nf->read_bytes(&buffer[0u], buffer.size(), offset,
bytes_read))) {
std::size_t bytes_written = 0u;
ret = write_bytes(&buffer[0u], bytes_read, offset, bytes_written);
file_size -= bytes_read;
@@ -120,7 +162,7 @@ bool native_file::copy_from(const native_file_ptr &nf) {
return ret;
}
bool native_file::copy_from(const std::string &path) {
auto native_file::copy_from(const std::string &path) -> bool {
auto ret = false;
native_file_ptr nf;
if (native_file::create_or_open(path, nf) == api_error ::success) {
@@ -140,7 +182,7 @@ void native_file::flush() {
#endif
}
bool native_file::get_file_size(std::uint64_t &file_size) {
auto native_file::get_file_size(std::uint64_t &file_size) -> bool {
auto ret = false;
#ifdef _WIN32
LARGE_INTEGER li{};
@@ -164,8 +206,9 @@ bool native_file::get_file_size(std::uint64_t &file_size) {
}
#ifdef _WIN32
bool native_file::read_bytes(char *buffer, const std::size_t &read_size,
const std::uint64_t &read_offset, std::size_t &bytes_read) {
auto native_file::read_bytes(char *buffer, std::size_t read_size,
std::uint64_t read_offset, std::size_t &bytes_read)
-> bool {
recur_mutex_lock l(read_write_mutex_);
auto ret = false;
@@ -176,7 +219,8 @@ bool native_file::read_bytes(char *buffer, const std::size_t &read_size,
DWORD current_read = 0u;
do {
current_read = 0u;
ret = !!::ReadFile(handle_, &buffer[bytes_read], static_cast<DWORD>(read_size - bytes_read),
ret = !!::ReadFile(handle_, &buffer[bytes_read],
static_cast<DWORD>(read_size - bytes_read),
&current_read, nullptr);
bytes_read += current_read;
} while (ret && (bytes_read < read_size) && (current_read != 0));
@@ -189,13 +233,14 @@ bool native_file::read_bytes(char *buffer, const std::size_t &read_size,
return ret;
}
#else
bool native_file::read_bytes(char *buffer, const std::size_t &read_size,
const std::uint64_t &read_offset, std::size_t &bytes_read) {
auto native_file::read_bytes(char *buffer, std::size_t read_size,
std::uint64_t read_offset, std::size_t &bytes_read)
-> bool {
bytes_read = 0u;
ssize_t result = 0;
do {
result =
pread64(handle_, &buffer[bytes_read], read_size - bytes_read, read_offset + bytes_read);
result = pread64(handle_, &buffer[bytes_read], read_size - bytes_read,
read_offset + bytes_read);
if (result > 0) {
bytes_read += static_cast<size_t>(result);
}
@@ -204,20 +249,22 @@ bool native_file::read_bytes(char *buffer, const std::size_t &read_size,
return (result >= 0);
}
#endif
bool native_file::truncate(const std::uint64_t &file_size) {
auto native_file::truncate(std::uint64_t file_size) -> bool {
#ifdef _WIN32
recur_mutex_lock l(read_write_mutex_);
LARGE_INTEGER li{};
li.QuadPart = file_size;
return (::SetFilePointerEx(handle_, li, nullptr, FILE_BEGIN) && ::SetEndOfFile(handle_));
return (::SetFilePointerEx(handle_, li, nullptr, FILE_BEGIN) &&
::SetEndOfFile(handle_));
#else
return (ftruncate(handle_, file_size) >= 0);
#endif
}
#ifdef _WIN32
bool native_file::write_bytes(const char *buffer, const std::size_t &write_size,
const std::uint64_t &write_offset, std::size_t &bytes_written) {
auto native_file::write_bytes(const char *buffer, std::size_t write_size,
std::uint64_t write_offset,
std::size_t &bytes_written) -> bool {
recur_mutex_lock l(read_write_mutex_);
bytes_written = 0u;
@@ -229,7 +276,8 @@ bool native_file::write_bytes(const char *buffer, const std::size_t &write_size,
do {
DWORD current_write = 0u;
ret = !!::WriteFile(handle_, &buffer[bytes_written],
static_cast<DWORD>(write_size - bytes_written), &current_write, nullptr);
static_cast<DWORD>(write_size - bytes_written),
&current_write, nullptr);
bytes_written += current_write;
} while (ret && (bytes_written < write_size));
}
@@ -237,13 +285,14 @@ bool native_file::write_bytes(const char *buffer, const std::size_t &write_size,
return ret;
}
#else
bool native_file::write_bytes(const char *buffer, const std::size_t &write_size,
const std::uint64_t &write_offset, std::size_t &bytes_written) {
auto native_file::write_bytes(const char *buffer, std::size_t write_size,
std::uint64_t write_offset,
std::size_t &bytes_written) -> bool {
bytes_written = 0;
ssize_t result = 0;
do {
result = pwrite64(handle_, &buffer[bytes_written], write_size - bytes_written,
write_offset + bytes_written);
result = pwrite64(handle_, &buffer[bytes_written],
write_size - bytes_written, write_offset + bytes_written);
if (result > 0) {
bytes_written += static_cast<size_t>(result);
}

133
src/utils/path_utils.cpp
View File

@@ -1,31 +1,33 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "utils/path_utils.hpp"
#include "types/repertory.hpp"
#include "utils/error_utils.hpp"
#include "utils/string_utils.hpp"
#include "utils/utils.hpp"
namespace repertory::utils::path {
static std::string &format_path(std::string &path, const std::string &sep,
const std::string &not_sep);
std::string absolute(std::string path) {
auto absolute(std::string path) -> std::string {
#ifdef _WIN32
if (not path.empty() && ::PathIsRelative(&path[0u])) {
std::string temp;
@@ -51,35 +53,61 @@ std::string absolute(std::string path) {
}
#endif
return path;
return finalize(path);
}
std::string combine(std::string path, const std::vector<std::string> &paths) {
auto combine(std::string path, const std::vector<std::string> &paths)
-> std::string {
return finalize(
std::accumulate(paths.begin(), paths.end(), path, [](std::string path, const auto &pathPart) {
path += (directory_seperator + pathPart);
return path;
}));
std::accumulate(paths.begin(), paths.end(), path,
[](std::string next_path, const auto &path_part) {
if (not next_path.empty()) {
next_path += (directory_seperator + path_part);
}
return next_path;
}));
}
std::string create_api_path(std::string path) {
if (path.empty() || (path == ".")) {
path = "/";
} else {
format_path(path, "/", "\\");
if (path.find("./") == 0) {
path = path.substr(1);
}
if (path[0u] != '/') {
path = "/" + path;
}
auto create_api_path(std::string path) -> std::string {
if (path.empty() || (path == ".") || (path == "/")) {
return "/";
}
format_path(path, "/", "\\");
if (path.find("./") == 0) {
path = path.substr(1);
}
if (path[0u] != '/') {
path = "/" + path;
}
if ((path != "/") && utils::string::ends_with(path, "/")) {
utils::string::right_trim(path, '/');
}
return path;
}
static std::string &format_path(std::string &path, const std::string &sep,
const std::string &not_sep) {
auto finalize(std::string path) -> std::string {
format_path(path, not_directory_seperator, directory_seperator);
format_path(path, directory_seperator, not_directory_seperator);
if ((path.size() > 1u) &&
(path[path.size() - 1u] == directory_seperator[0u])) {
path = path.substr(0u, path.size() - 1u);
}
#ifdef _WIN32
if ((path.size() >= 2u) && (path[1u] == ':')) {
path[0u] = utils::string::to_lower(std::string(1u, path[0u]))[0u];
}
#endif
return path;
}
auto format_path(std::string &path, const std::string &sep,
const std::string &not_sep) -> std::string & {
std::replace(path.begin(), path.end(), not_sep[0u], sep[0u]);
while (utils::string::contains(path, sep + sep)) {
@@ -89,16 +117,7 @@ static std::string &format_path(std::string &path, const std::string &sep,
return path;
}
std::string finalize(std::string path) {
format_path(path, directory_seperator, not_directory_seperator);
if ((path.size() > 1) && (path[path.size() - 1] == directory_seperator[0u])) {
path = path.substr(0, path.size() - 1);
}
return path;
}
std::string get_parent_api_path(const std::string &path) {
auto get_parent_api_path(const std::string &path) -> std::string {
std::string ret;
if (path != "/") {
ret = path.substr(0, path.rfind('/') + 1);
@@ -111,7 +130,7 @@ std::string get_parent_api_path(const std::string &path) {
}
#ifndef _WIN32
std::string get_parent_directory(std::string path) {
auto get_parent_directory(std::string path) -> std::string {
auto ret = std::string(dirname(&path[0u]));
if (ret == ".") {
ret = "/";
@@ -121,7 +140,7 @@ std::string get_parent_directory(std::string path) {
}
#endif
bool is_ads_file_path(const std::string &path) {
auto is_ads_file_path([[maybe_unused]] const std::string &path) -> bool {
#ifdef _WIN32
return utils::string::contains(path, ":");
#else
@@ -129,7 +148,7 @@ bool is_ads_file_path(const std::string &path) {
#endif
}
bool is_trash_directory(std::string path) {
auto is_trash_directory(std::string path) -> bool {
path = create_api_path(utils::string::to_lower(path));
if (utils::string::begins_with(path, "/.trash-") ||
utils::string::begins_with(path, "/.trashes") ||
@@ -139,7 +158,7 @@ bool is_trash_directory(std::string path) {
return false;
}
std::string remove_file_name(std::string path) {
auto remove_file_name(std::string path) -> std::string {
path = finalize(path);
#ifdef _WIN32
@@ -160,18 +179,20 @@ std::string remove_file_name(std::string path) {
}
#ifndef _WIN32
std::string resolve(std::string path) {
struct passwd *pw = getpwuid(getuid());
std::string home = (pw->pw_dir ? pw->pw_dir : "");
if (home.empty() || ((home == "/") && (getuid() != 0))) {
home = combine("/home", {pw->pw_name});
}
auto resolve(std::string path) -> std::string {
std::string home{};
use_getpwuid(getuid(), [&home](struct passwd *pw) {
home = (pw->pw_dir ? pw->pw_dir : "");
if (home.empty() || ((home == "/") && (getuid() != 0))) {
home = combine("/home", {pw->pw_name});
}
});
return finalize(utils::string::replace(path, "~", home));
}
#endif
std::string strip_to_file_name(std::string path) {
auto strip_to_file_name(std::string path) -> std::string {
#ifdef _WIN32
return ::PathFindFileName(&path[0u]);
#else

104
src/utils/polling.cpp
View File

@@ -1,44 +1,57 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "utils/polling.hpp"
#include "app_config.hpp"
namespace repertory {
polling polling::instance_;
void polling::frequency_thread(std::function<std::uint32_t()> get_frequency_seconds,
bool low_frequency) {
void polling::frequency_thread(
std::function<std::uint32_t()> get_frequency_seconds, frequency freq) {
while (not stop_requested_) {
std::deque<std::future<void>> futures;
unique_mutex_lock l(mutex_);
if (not stop_requested_ && notify_.wait_for(l, std::chrono::seconds(get_frequency_seconds())) ==
std::cv_status::timeout) {
for (auto &kv : items_) {
if (kv.second.low_frequency == low_frequency) {
futures.emplace_back(std::async(std::launch::async, [kv]() -> void {
event_system::instance().raise<polling_item_begin>(kv.first);
kv.second.action();
event_system::instance().raise<polling_item_end>(kv.first);
}));
if (not stop_requested_ &&
notify_.wait_for(l, std::chrono::seconds(get_frequency_seconds())) ==
std::cv_status::timeout) {
for (const auto &kv : items_) {
if (kv.second.freq == freq) {
futures.emplace_back(
std::async(std::launch::async, [this, &freq, kv]() -> void {
if (config_->get_event_level() == event_level::verbose ||
freq != frequency::second) {
event_system::instance().raise<polling_item_begin>(kv.first);
}
kv.second.action();
if (config_->get_event_level() == event_level::verbose ||
freq != frequency::second) {
event_system::instance().raise<polling_item_end>(kv.first);
}
}));
}
}
l.unlock();
while (not futures.empty()) {
futures.front().wait();
futures.pop_front();
@@ -52,40 +65,57 @@ void polling::remove_callback(const std::string &name) {
items_.erase(name);
}
void polling::set_callback(const polling_item &pollingItem) {
void polling::set_callback(const polling_item &pi) {
mutex_lock l(mutex_);
items_[pollingItem.name] = pollingItem;
items_[pi.name] = pi;
}
void polling::start(app_config *config) {
mutex_lock l(start_stop_mutex_);
if (not high_frequency_thread_) {
event_system::instance().raise<service_started>("polling");
config_ = config;
stop_requested_ = false;
high_frequency_thread_ = std::make_unique<std::thread>([this]() -> void {
this->frequency_thread(
[this]() -> std::uint32_t { return config_->get_high_frequency_interval_secs(); }, false);
[this]() -> std::uint32_t {
return config_->get_high_frequency_interval_secs();
},
frequency::high);
});
low_frequency_thread_ = std::make_unique<std::thread>([this]() -> void {
this->frequency_thread(
[this]() -> std::uint32_t { return config_->get_low_frequency_interval_secs(); }, true);
[this]() -> std::uint32_t {
return config_->get_low_frequency_interval_secs();
},
frequency::low);
});
second_frequency_thread_ = std::make_unique<std::thread>([this]() -> void {
this->frequency_thread([]() -> std::uint32_t { return 1U; },
frequency::second);
});
}
}
void polling::stop() {
mutex_lock l(start_stop_mutex_);
if (high_frequency_thread_) {
event_system::instance().raise<service_shutdown>("polling");
{
mutex_lock l2(mutex_);
stop_requested_ = true;
notify_.notify_all();
event_system::instance().raise<service_shutdown_begin>("polling");
mutex_lock l(start_stop_mutex_);
if (high_frequency_thread_) {
{
stop_requested_ = true;
mutex_lock l2(mutex_);
notify_.notify_all();
}
high_frequency_thread_->join();
low_frequency_thread_->join();
second_frequency_thread_->join();
high_frequency_thread_.reset();
low_frequency_thread_.reset();
second_frequency_thread_.reset();
}
high_frequency_thread_->join();
low_frequency_thread_->join();
high_frequency_thread_.reset();
low_frequency_thread_.reset();
event_system::instance().raise<service_shutdown_end>("polling");
}
}
} // namespace repertory

55
src/utils/rocksdb_utils.cpp
View File

@@ -1,25 +1,30 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "utils/rocksdb_utils.hpp"
#include "app_config.hpp"
#include "events/event_system.hpp"
#include "types/startup_exception.hpp"
#include "utils/error_utils.hpp"
#include "utils/path_utils.hpp"
namespace repertory::utils::db {
@@ -30,35 +35,37 @@ void create_rocksdb(const app_config &config, const std::string &name,
options.db_log_dir = config.get_log_directory();
options.keep_log_file_num = 10;
rocksdb::DB *db_ptr = nullptr;
rocksdb::DB *db_ptr{};
const auto status = rocksdb::DB::Open(
options, utils::path::combine(config.get_data_directory(), {name}), &db_ptr);
options, utils::path::combine(config.get_data_directory(), {name}),
&db_ptr);
if (status.ok()) {
db.reset(db_ptr);
} else {
event_system::instance().raise<repertory_exception>(__FUNCTION__, status.ToString());
utils::error::raise_error(__FUNCTION__, status.ToString());
throw startup_exception(status.ToString());
}
}
void create_rocksdb(const app_config &config, const std::string &name,
const std::vector<rocksdb::ColumnFamilyDescriptor> &families,
std::vector<rocksdb::ColumnFamilyHandle *> &handles,
std::unique_ptr<rocksdb::DB> &db) {
void create_rocksdb(
const app_config &config, const std::string &name,
const std::vector<rocksdb::ColumnFamilyDescriptor> &families,
std::vector<rocksdb::ColumnFamilyHandle *> &handles,
std::unique_ptr<rocksdb::DB> &db) {
rocksdb::Options options{};
options.create_if_missing = true;
options.create_missing_column_families = true;
options.db_log_dir = config.get_log_directory();
options.keep_log_file_num = 10;
rocksdb::DB *db_ptr = nullptr;
const auto status =
rocksdb::DB::Open(options, utils::path::combine(config.get_data_directory(), {name}),
families, &handles, &db_ptr);
rocksdb::DB *db_ptr{};
const auto status = rocksdb::DB::Open(
options, utils::path::combine(config.get_data_directory(), {name}),
families, &handles, &db_ptr);
if (status.ok()) {
db.reset(db_ptr);
} else {
event_system::instance().raise<repertory_exception>(__FUNCTION__, status.ToString());
utils::error::raise_error(__FUNCTION__, status.ToString());
throw startup_exception(status.ToString());
}
}

65
src/utils/single_thread_service_base.cpp Normal file
View File

@@ -0,0 +1,65 @@
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "utils/single_thread_service_base.hpp"
#include "events/event_system.hpp"
#include "events/events.hpp"
#include "types/repertory.hpp"
namespace repertory {
void single_thread_service_base::notify_all() const {
mutex_lock lock(get_mutex());
notify_.notify_all();
}
void single_thread_service_base::start() {
mutex_lock lock(mtx_);
if (not thread_) {
stop_requested_ = false;
on_start();
thread_ = std::make_unique<std::thread>([this]() {
event_system::instance().raise<service_started>(service_name_);
while (not stop_requested_) {
service_function();
}
});
}
}
void single_thread_service_base::stop() {
if (thread_) {
event_system::instance().raise<service_shutdown_begin>(service_name_);
unique_mutex_lock lock(mtx_);
if (thread_) {
stop_requested_ = true;
notify_.notify_all();
lock.unlock();
thread_->join();
thread_.reset();
on_stop();
}
event_system::instance().raise<service_shutdown_end>(service_name_);
}
}
} // namespace repertory

188
src/utils/string_utils.cpp
View File

@@ -1,121 +1,129 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "utils/string_utils.hpp"
#include "common.hpp"
namespace repertory::utils::string {
bool begins_with(const std::string &str, const std::string &val) { return (str.find(val) == 0); }
bool contains(const std::string &str, const std::string &search) {
return (str.find(search) != std::string::npos);
}
bool ends_with(const std::string &str, const std::string &val) {
/* constexpr c++20 */ auto ends_with(std::string_view str, std::string_view val)
-> bool {
if (val.size() > str.size()) {
return false;
}
return std::equal(val.rbegin(), val.rend(), str.rbegin());
}
std::string from_bool(const bool &val) { return std::to_string(val); }
auto from_bool(bool val) -> std::string { return std::to_string(val); }
std::string from_double(const double &value) { return std::to_string(value); }
std::string from_dynamic_bitset(const boost::dynamic_bitset<> &bitset) {
auto from_dynamic_bitset(const boost::dynamic_bitset<> &bitset) -> std::string {
std::stringstream ss;
boost::archive::text_oarchive archive(ss);
archive << bitset;
return ss.str();
}
std::string from_int32(const std::int32_t &val) { return std::to_string(val); }
std::string from_int64(const std::int64_t &val) { return std::to_string(val); }
std::string from_uint8(const std::uint8_t &val) { return std::to_string(val); }
std::string from_uint16(const std::uint16_t &val) { return std::to_string(val); }
std::string from_uint32(const std::uint32_t &val) { return std::to_string(val); }
std::string from_uint64(const std::uint64_t &val) { return std::to_string(val); }
std::wstring from_utf8(const std::string &str) {
return str.empty() ? L""
: std::wstring_convert<std::codecvt_utf8<wchar_t>, wchar_t>().from_bytes(str);
auto from_utf8(const std::string &str) -> std::wstring {
return str.empty()
? L""
: std::wstring_convert<std::codecvt_utf8<wchar_t>, wchar_t>()
.from_bytes(str);
}
bool is_numeric(const std::string &s) {
static const auto r = std::regex(R"(^(\+|\-)?(([0-9]*)|(([0-9]*)\.([0-9]*)))$)");
return std::regex_match(s, r);
/* constexpr c++20 */ auto is_numeric(std::string_view s) -> bool {
if ((s.length() > 1u) && (s[0u] == '+' || s[0u] == '-')) {
s = s.substr(1u);
}
if (s.empty()) {
return false;
}
auto has_decimal = false;
return std::find_if(
s.begin(), s.end(),
[&has_decimal](const std::string_view::value_type &c) -> bool {
if (has_decimal && c == '.') {
return true;
}
if ((has_decimal = has_decimal || c == '.')) {
return false;
}
return not std::isdigit(c);
}) == s.end();
}
std::string join(const std::vector<std::string> &arr, const char &delim) {
auto join(const std::vector<std::string> &arr, const char &delim)
-> std::string {
if (arr.empty()) {
return "";
}
return std::accumulate(std::next(arr.begin()), arr.end(), arr[0u],
[&delim](auto s, const auto &v) { return s + delim + v; });
return std::accumulate(
std::next(arr.begin()), arr.end(), arr[0u],
[&delim](auto s, const auto &v) { return s + delim + v; });
}
std::string &left_trim(std::string &s) { return left_trim(s, ' '); }
auto left_trim(std::string &s) -> std::string & { return left_trim(s, ' '); }
std::string &left_trim(std::string &s, const char &c) {
auto left_trim(std::string &s, const char &c) -> std::string & {
s.erase(0, s.find_first_not_of(c));
return s;
}
std::string &replace(std::string &src, const char &character, const char &with) {
auto replace(std::string &src, const char &character, const char &with)
-> std::string & {
std::replace(src.begin(), src.end(), character, with);
return src;
}
std::string &replace(std::string &src, const std::string &find, const std::string &with,
size_t startPos) {
if (!src.empty() && (startPos < src.size())) {
while ((startPos = src.find(find, startPos)) != std::string::npos) {
src.replace(startPos, find.size(), with);
startPos += with.size();
auto replace(std::string &src, const std::string &find, const std::string &with,
size_t start_pos) -> std::string & {
if (!src.empty() && (start_pos < src.size())) {
while ((start_pos = src.find(find, start_pos)) != std::string::npos) {
src.replace(start_pos, find.size(), with);
start_pos += with.size();
}
}
return src;
}
std::string replace_copy(std::string src, const char &character, const char &with) {
auto replace_copy(std::string src, const char &character, const char &with)
-> std::string {
std::replace(src.begin(), src.end(), character, with);
return src;
}
std::string replace_copy(std::string src, const std::string &find, const std::string &with,
size_t startPos) {
return replace(src, find, with, startPos);
auto replace_copy(std::string src, const std::string &find,
const std::string &with, size_t start_pos) -> std::string {
return replace(src, find, with, start_pos);
}
std::string &right_trim(std::string &s) { return right_trim(s, ' '); }
auto right_trim(std::string &s) -> std::string & { return right_trim(s, ' '); }
std::string &right_trim(std::string &s, const char &c) {
auto right_trim(std::string &s, const char &c) -> std::string & {
s.erase(s.find_last_not_of(c) + 1);
return s;
}
std::vector<std::string> split(const std::string &str, const char &delim, const bool &should_trim) {
auto split(const std::string &str, const char &delim, bool should_trim)
-> std::vector<std::string> {
std::vector<std::string> ret;
std::stringstream ss(str);
std::string item;
@@ -125,7 +133,7 @@ std::vector<std::string> split(const std::string &str, const char &delim, const
return ret;
}
bool to_bool(std::string val) {
auto to_bool(std::string val) -> bool {
auto b = false;
trim(val);
@@ -142,9 +150,9 @@ bool to_bool(std::string val) {
return b;
}
double to_double(const std::string &str) { return std::stod(str); }
auto to_double(const std::string &str) -> double { return std::stod(str); }
boost::dynamic_bitset<> to_dynamic_bitset(const std::string &val) {
auto to_dynamic_bitset(const std::string &val) -> boost::dynamic_bitset<> {
std::stringstream ss(val);
boost::dynamic_bitset<> bitset;
boost::archive::text_iarchive archive(ss);
@@ -152,44 +160,64 @@ boost::dynamic_bitset<> to_dynamic_bitset(const std::string &val) {
return bitset;
}
std::int32_t to_int32(const std::string &val) { return std::stoi(val); }
auto to_int32(const std::string &val) -> std::int32_t { return std::stoi(val); }
std::int64_t to_int64(const std::string &val) { return std::stoll(val); }
auto to_int64(const std::string &val) -> std::int64_t {
return std::stoll(val);
}
std::string to_lower(std::string str) {
auto to_lower(std::string str) -> std::string {
std::transform(str.begin(), str.end(), str.begin(), ::tolower);
return str;
}
std::uint8_t to_uint8(const std::string &val) { return static_cast<std::uint8_t>(std::stoul(val)); }
auto to_size_t(const std::string &val) -> std::size_t {
return static_cast<std::size_t>(std::stoull(val));
}
std::uint16_t to_uint16(const std::string &val) {
auto to_uint8(const std::string &val) -> std::uint8_t {
return static_cast<std::uint8_t>(std::stoul(val));
}
auto to_uint16(const std::string &val) -> std::uint16_t {
return static_cast<std::uint16_t>(std::stoul(val));
}
std::uint32_t to_uint32(const std::string &val) {
auto to_uint32(const std::string &val) -> std::uint32_t {
return static_cast<std::uint32_t>(std::stoul(val));
}
std::uint64_t to_uint64(const std::string &val) { return std::stoull(val); }
auto to_uint64(const std::string &val) -> std::uint64_t {
return std::stoull(val);
}
std::string to_upper(std::string str) {
auto to_upper(std::string str) -> std::string {
std::transform(str.begin(), str.end(), str.begin(), ::toupper);
return str;
}
const std::string &to_utf8(const std::string &str) { return str; }
auto to_utf8(std::string str) -> std::string { return str; }
std::string to_utf8(const std::wstring &str) {
return str.empty() ? ""
: std::wstring_convert<std::codecvt_utf8<wchar_t>, wchar_t>().to_bytes(str);
auto to_utf8(const std::wstring &str) -> std::string {
return str.empty()
? ""
: std::wstring_convert<std::codecvt_utf8<wchar_t>, wchar_t>()
.to_bytes(str);
}
std::string &trim(std::string &str) { return right_trim(left_trim(str)); }
auto trim(std::string &str) -> std::string & {
return right_trim(left_trim(str));
}
std::string &trim(std::string &str, const char &c) { return right_trim(left_trim(str, c), c); }
auto trim(std::string &str, const char &c) -> std::string & {
return right_trim(left_trim(str, c), c);
}
std::string trim_copy(std::string str) { return right_trim(left_trim(str)); }
auto trim_copy(std::string str) -> std::string {
return right_trim(left_trim(str));
}
std::string trim_copy(std::string str, const char &c) { return right_trim(left_trim(str, c), c); }
auto trim_copy(std::string str, const char &c) -> std::string {
return right_trim(left_trim(str, c), c);
}
} // namespace repertory::utils::string

28
src/utils/throttle.cpp
View File

@@ -1,22 +1,26 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "utils/throttle.hpp"
#include "types/repertory.hpp"
namespace repertory {

47
src/utils/timeout.cpp
View File

@@ -1,22 +1,26 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "utils/timeout.hpp"
#include "types/repertory.hpp"
namespace repertory {
@@ -24,15 +28,16 @@ timeout::timeout(std::function<void()> timeout_callback,
const std::chrono::system_clock::duration &duration)
: timeout_killed_(duration == 0s) {
if (not timeout_killed_) {
timeout_thread_ = std::make_unique<std::thread>([this, duration, timeout_callback]() {
unique_mutex_lock lock(timeout_mutex_);
if (not timeout_killed_) {
timeout_notify_.wait_for(lock, duration);
if (not timeout_killed_) {
timeout_callback();
}
}
});
timeout_thread_ =
std::make_unique<std::thread>([this, duration, timeout_callback]() {
unique_mutex_lock lock(timeout_mutex_);
if (not timeout_killed_) {
timeout_notify_.wait_for(lock, duration);
if (not timeout_killed_) {
timeout_callback();
}
}
});
}
}

202
src/utils/unix/unix_utils.cpp
View File

@@ -1,53 +1,39 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#ifndef _WIN32
#include "utils/unix/unix_utils.hpp"
#include "utils/error_utils.hpp"
#include "utils/utils.hpp"
namespace repertory::utils {
#ifndef __APPLE__
std::uint64_t convert_to_uint64(const pthread_t &t) { return static_cast<std::uint64_t>(t); }
#endif
int get_last_error_code() { return errno; }
std::uint64_t get_thread_id() { return convert_to_uint64(pthread_self()); }
bool is_uid_member_of_group(const uid_t &uid, const gid_t &gid) {
auto *pw = getpwuid(uid);
std::vector<gid_t> groups;
int group_count = 0;
if (getgrouplist(pw->pw_name, pw->pw_gid, nullptr, &group_count) < 0) {
groups.resize(static_cast<unsigned long>(group_count));
#ifdef __APPLE__
getgrouplist(pw->pw_name, pw->pw_gid, reinterpret_cast<int *>(&groups[0]), &group_count);
#else
getgrouplist(pw->pw_name, pw->pw_gid, &groups[0], &group_count);
#endif
}
return collection_includes(groups, gid);
auto convert_to_uint64(const pthread_t &t) -> std::uint64_t {
return static_cast<std::uint64_t>(t);
}
#endif
int translate_api_error(const api_error &e) {
auto from_api_error(const api_error &e) -> int {
switch (e) {
case api_error::access_denied:
return -EACCES;
@@ -62,14 +48,14 @@ int translate_api_error(const api_error &e) {
case api_error::directory_not_found:
return -ENOTDIR;
case api_error::download_failed:
#if __APPLE__
#ifdef __APPLE__
return -EBADMSG;
#else
return -EREMOTEIO;
#endif
case api_error::error:
return -EIO;
case api_error::file_exists:
case api_error::item_exists:
return -EEXIST;
case api_error::file_in_use:
return -EBUSY;
@@ -77,8 +63,10 @@ int translate_api_error(const api_error &e) {
return -EINVAL;
case api_error::item_not_found:
return -ENOENT;
case api_error::item_is_file:
return -ENOTDIR;
case api_error::out_of_memory:
return -ENOMEM;
case api_error::no_disk_space:
return -ENOSPC;
case api_error::os_error:
return -errno;
case api_error::permission_denied:
@@ -90,17 +78,11 @@ int translate_api_error(const api_error &e) {
case api_error::not_implemented:
return -ENOSYS;
case api_error::upload_failed:
#if __APPLE__
return -EBADMSG;
#else
return -EREMOTEIO;
#endif
return -ENETDOWN;
case api_error::xattr_buffer_small:
return -ERANGE;
case api_error::xattr_exists:
return -EEXIST;
case api_error::xattr_invalid_namespace:
return -ENOTSUP;
case api_error::xattr_not_found:
#ifdef __APPLE__
return -ENOATTR;
@@ -112,19 +94,109 @@ int translate_api_error(const api_error &e) {
return -ENAMETOOLONG;
#else
return -E2BIG;
#endif
#ifdef __APPLE__
case api_error::XAttrOSXInvalid:
return -EINVAL;
#endif
default:
return -EIO;
}
}
auto get_last_error_code() -> int { return errno; }
auto get_thread_id() -> std::uint64_t {
return convert_to_uint64(pthread_self());
}
auto is_uid_member_of_group(const uid_t &uid, const gid_t &gid) -> bool {
static const auto function_name = __FUNCTION__;
std::vector<gid_t> groups{};
use_getpwuid(uid, [&groups](struct passwd *pw) {
int group_count{};
if (getgrouplist(pw->pw_name, pw->pw_gid, nullptr, &group_count) < 0) {
groups.resize(static_cast<std::size_t>(group_count));
#ifdef __APPLE__
getgrouplist(pw->pw_name, pw->pw_gid,
reinterpret_cast<int *>(groups.data()), &group_count);
#else
getgrouplist(pw->pw_name, pw->pw_gid, groups.data(), &group_count);
#endif
}
});
return collection_includes(groups, gid);
}
auto to_api_error(int e) -> api_error {
switch (abs(e)) {
case 0:
return api_error::success;
case EBADF:
return api_error::invalid_handle;
case EACCES:
return api_error::access_denied;
case EFAULT:
return api_error::bad_address;
case EOF:
return api_error::directory_end_of_files;
case EISDIR:
return api_error::directory_exists;
case ENOTEMPTY:
return api_error::directory_not_empty;
case ENOTDIR:
return api_error::directory_not_found;
#ifdef __APPLE__
case EBADMSG:
return api_error::download_failed;
#else
case EREMOTEIO:
return api_error::download_failed;
#endif
case EIO:
return api_error::error;
case EEXIST:
return api_error::item_exists;
case EBUSY:
return api_error::file_in_use;
case EINVAL:
return api_error::invalid_operation;
case ENOENT:
return api_error::item_not_found;
case ENOMEM:
return api_error::out_of_memory;
case EPERM:
return api_error::permission_denied;
case ENOSPC:
return api_error::no_disk_space;
case ENOTSUP:
return api_error::not_supported;
case ENOSYS:
return api_error::not_implemented;
case ENETDOWN:
return api_error::upload_failed;
case ERANGE:
return api_error::xattr_buffer_small;
#ifdef __APPLE__
case ENOATTR:
return api_error::xattr_not_found;
#else
case ENODATA:
return api_error::xattr_not_found;
#endif
#ifdef __APPLE__
case ENAMETOOLONG:
return api_error::xattr_too_big;
#else
case E2BIG:
return api_error::xattr_too_big;
#endif
default:
return api_error::error;
}
}
void set_last_error_code(int error_code) { errno = error_code; }
std::int32_t unix_error_to_windows(const int &e) {
auto unix_error_to_windows(int e) -> std::int32_t {
switch (e) {
case 0:
return STATUS_SUCCESS;
@@ -146,7 +218,7 @@ std::int32_t unix_error_to_windows(const int &e) {
case EIO:
return STATUS_UNEXPECTED_IO_ERROR;
case EISDIR:
return STATUS_FILE_IS_A_DIRECTORY;
return STATUS_OBJECT_NAME_EXISTS;
case EMFILE:
return STATUS_INSUFFICIENT_RESOURCES;
case ENOENT:
@@ -158,18 +230,31 @@ std::int32_t unix_error_to_windows(const int &e) {
case ENOSPC:
return STATUS_DEVICE_INSUFFICIENT_RESOURCES;
case ENOTDIR:
return STATUS_OBJECT_PATH_INVALID;
return STATUS_OBJECT_NAME_NOT_FOUND;
default:
return STATUS_INTERNAL_ERROR;
}
}
UINT64 unix_time_to_windows_time(const remote::file_time &ts) {
auto unix_time_to_windows_time(const remote::file_time &ts) -> UINT64 {
return (ts / 100ull) + 116444736000000000ull;
}
void windows_create_to_unix(const UINT32 &create_options, const UINT32 &granted_access,
std::uint32_t &flags, remote::file_mode &mode) {
void use_getpwuid(uid_t uid, std::function<void(struct passwd *pw)> fn) {
static std::mutex mtx{};
mutex_lock lock{mtx};
auto *pw = getpwuid(uid);
if (not pw) {
utils::error::raise_error(__FUNCTION__, "'getpwuid' returned nullptr");
return;
}
fn(pw);
}
void windows_create_to_unix(const UINT32 &create_options,
const UINT32 &granted_access, std::uint32_t &flags,
remote::file_mode &mode) {
mode = S_IRUSR | S_IWUSR;
flags = O_CREAT | O_RDWR;
if (create_options & FILE_DIRECTORY_FILE) {
@@ -177,13 +262,14 @@ void windows_create_to_unix(const UINT32 &create_options, const UINT32 &granted_
flags = O_DIRECTORY;
}
if ((granted_access & GENERIC_EXECUTE) || (granted_access & FILE_GENERIC_EXECUTE) ||
if ((granted_access & GENERIC_EXECUTE) ||
(granted_access & FILE_GENERIC_EXECUTE) ||
(granted_access & FILE_EXECUTE)) {
mode |= (S_IXUSR);
}
}
remote::file_time windows_time_to_unix_time(const std::uint64_t &t) {
auto windows_time_to_unix_time(std::uint64_t t) -> remote::file_time {
return (t - 116444736000000000ull) * 100ull;
}
} // namespace repertory::utils

206
src/utils/utils.cpp
View File

@@ -1,27 +1,30 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "utils/utils.hpp"
#include "app_config.hpp"
#include "events/events.hpp"
#include "events/event_system.hpp"
#include "events/events.hpp"
#include "providers/i_provider.hpp"
#include "types/skynet.hpp"
#include "types/startup_exception.hpp"
#include "utils/com_init_wrapper.hpp"
#include "utils/native_file.hpp"
@@ -29,18 +32,9 @@
#include "utils/string_utils.hpp"
namespace repertory::utils {
hastings api_currency_to_hastings(const api_currency &currency) {
ttmath::Parser<api_currency> parser;
parser.Parse(currency.ToString() + " * (10 ^ 24)");
ttmath::Conv conv;
conv.scient_from = 256;
conv.base = 10u;
conv.round = 0;
return parser.stack[0u].value.ToString(conv);
}
void calculate_allocation_size(const bool &directory, const std::uint64_t &file_size,
UINT64 allocation_size, std::string &allocation_meta_size) {
void calculate_allocation_size(bool directory, std::uint64_t file_size,
UINT64 allocation_size,
std::string &allocation_meta_size) {
if (directory) {
allocation_meta_size = "0";
return;
@@ -49,20 +43,24 @@ void calculate_allocation_size(const bool &directory, const std::uint64_t &file_
if (file_size > allocation_size) {
allocation_size = file_size;
}
allocation_size = ((allocation_size == 0u) ? WINFSP_ALLOCATION_UNIT : allocation_size);
allocation_size =
utils::divide_with_ceiling(allocation_size, WINFSP_ALLOCATION_UNIT) * WINFSP_ALLOCATION_UNIT;
((allocation_size == 0u) ? WINFSP_ALLOCATION_UNIT : allocation_size);
allocation_size =
utils::divide_with_ceiling(allocation_size, WINFSP_ALLOCATION_UNIT) *
WINFSP_ALLOCATION_UNIT;
allocation_meta_size = std::to_string(allocation_size);
}
std::size_t calculate_read_size(const uint64_t &total_size, const std::size_t &read_size,
const uint64_t &offset) {
return static_cast<std::size_t>(((offset + read_size) > total_size)
? ((offset < total_size) ? total_size - offset : 0u)
: read_size);
auto calculate_read_size(const uint64_t &total_size, std::size_t read_size,
const uint64_t &offset) -> std::size_t {
return static_cast<std::size_t>(
((offset + read_size) > total_size)
? ((offset < total_size) ? total_size - offset : 0u)
: read_size);
}
int compare_version_strings(std::string version1, std::string version2) {
auto compare_version_strings(std::string version1, std::string version2)
-> int {
if (utils::string::contains(version1, "-")) {
version1 = utils::string::split(version1, '-')[0u];
}
@@ -94,44 +92,33 @@ int compare_version_strings(std::string version1, std::string version2) {
return 0;
}
std::uint64_t convert_api_date(const std::string &date) {
//"2019-02-21T02:24:37.653091916-06:00"
const auto parts = utils::string::split(date, '.');
const auto dt = parts[0];
const auto nanos = utils::string::to_uint64(utils::string::split(parts[1u], '-')[0u]);
auto convert_api_date(const std::string &date) -> std::uint64_t {
// 2009-10-12T17:50:30.000Z
const auto date_parts = utils::string::split(date, '.');
const auto date_time = date_parts[0U];
const auto nanos =
utils::string::to_uint64(utils::string::split(date_parts[1U], 'Z')[0U]);
struct tm tm1 {};
#ifdef _WIN32
auto convert_time = [](time_t t) -> std::uint64_t {
const auto ll = Int32x32To64(t, 10000000) + 116444736000000000ull;
ULARGE_INTEGER ft{};
ft.LowPart = static_cast<DWORD>(ll);
ft.HighPart = static_cast<DWORD>(ll >> 32);
return ft.QuadPart;
};
const auto parts2 = utils::string::split(dt, 'T');
const auto date_parts = utils::string::split(parts2[0u], '-');
const auto time_parts = utils::string::split(parts2[1u], ':');
tm1.tm_year = utils::string::to_int32(date_parts[0u]) - 1900;
tm1.tm_mon = utils::string::to_int32(date_parts[1u]) - 1;
tm1.tm_mday = utils::string::to_int32(date_parts[2u]);
tm1.tm_hour = utils::string::to_uint32(time_parts[0u]);
tm1.tm_min = utils::string::to_uint32(time_parts[1u]);
tm1.tm_sec = utils::string::to_uint32(time_parts[2u]);
tm1.tm_wday = -1;
tm1.tm_yday = -1;
tm1.tm_isdst = -1;
return (nanos / 100) + convert_time(mktime(&tm1));
utils::strptime(date_time.c_str(), "%Y-%m-%dT%T", &tm1);
#else
strptime(&dt[0], "%Y-%m-%dT%T", &tm1);
return nanos + (mktime(&tm1) * NANOS_PER_SECOND);
strptime(date_time.c_str(), "%Y-%m-%dT%T", &tm1);
#endif
return nanos + (mktime(&tm1) * NANOS_PER_SECOND);
}
CURL *create_curl() { return reset_curl(curl_easy_init()); }
auto create_curl() -> CURL * {
static std::recursive_mutex mtx;
std::string create_uuid_string() {
unique_recur_mutex_lock l(mtx);
curl_global_init(CURL_GLOBAL_DEFAULT);
l.unlock();
return reset_curl(curl_easy_init());
}
auto create_uuid_string() -> std::string {
#ifdef _WIN32
UUID guid{};
UuidCreate(&guid);
@@ -161,11 +148,13 @@ std::string create_uuid_string() {
#endif
}
std::string create_volume_label(const provider_type &pt) {
auto create_volume_label(const provider_type &pt) -> std::string {
return "repertory_" + app_config::get_provider_name(pt);
}
download_type download_type_from_string(std::string type, const download_type &default_type) {
auto download_type_from_string(std::string type,
const download_type &default_type)
-> download_type {
type = utils::string::to_lower(utils::string::trim(type));
if (type == "direct") {
return download_type::direct;
@@ -178,7 +167,7 @@ download_type download_type_from_string(std::string type, const download_type &d
return default_type;
}
std::string download_type_to_string(const download_type &type) {
auto download_type_to_string(const download_type &type) -> std::string {
switch (type) {
case download_type::direct:
return "direct";
@@ -193,7 +182,7 @@ std::string download_type_to_string(const download_type &type) {
#ifdef _WIN32
// https://www.frenk.com/2009/12/convert-filetime-to-unix-timestamp/
remote::file_time filetime_to_unix_time(const FILETIME &ft) {
auto filetime_to_unix_time(const FILETIME &ft) -> remote::file_time {
LARGE_INTEGER date{};
date.HighPart = ft.dwHighDateTime;
date.LowPart = ft.dwLowDateTime;
@@ -209,7 +198,7 @@ void unix_time_to_filetime(const remote::file_time &ts, FILETIME &ft) {
}
#endif
std::string generate_random_string(const std::uint16_t &length) {
auto generate_random_string(std::uint16_t length) -> std::string {
srand(static_cast<unsigned int>(get_time_now()));
std::string ret;
@@ -217,13 +206,18 @@ std::string generate_random_string(const std::uint16_t &length) {
for (std::uint16_t i = 0u; i < length; i++) {
do {
ret[i] = static_cast<char>(rand() % 74 + 48);
} while (((ret[i] >= 91) && (ret[i] <= 96)) || ((ret[i] >= 58) && (ret[i] <= 64)));
} while (((ret[i] >= 91) && (ret[i] <= 96)) ||
((ret[i] >= 58) && (ret[i] <= 64)));
}
return ret;
}
std::string get_environment_variable(const std::string &variable) {
auto get_attributes_from_meta(const api_meta_map &meta) -> DWORD {
return static_cast<DWORD>(utils::string::to_uint32(meta.at(META_ATTRIBUTES)));
}
auto get_environment_variable(const std::string &variable) -> std::string {
#ifdef _WIN32
std::string value;
auto sz = ::GetEnvironmentVariable(&variable[0], nullptr, 0);
@@ -239,7 +233,7 @@ std::string get_environment_variable(const std::string &variable) {
#endif
}
std::uint64_t get_file_time_now() {
auto get_file_time_now() -> std::uint64_t {
#ifdef _WIN32
SYSTEMTIME st{};
::GetSystemTime(&st);
@@ -254,7 +248,8 @@ std::uint64_t get_file_time_now() {
void get_local_time_now(struct tm &local_time) {
memset(&local_time, 0, sizeof(local_time));
const auto now = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now());
const auto now =
std::chrono::system_clock::to_time_t(std::chrono::system_clock::now());
#ifdef _WIN32
localtime_s(&local_time, &now);
#else
@@ -265,10 +260,8 @@ void get_local_time_now(struct tm &local_time) {
#endif
}
#ifdef _WIN32
#endif
bool get_next_available_port(std::uint16_t first_port, std::uint16_t &available_port) {
auto get_next_available_port(std::uint16_t first_port,
std::uint16_t &available_port) -> bool {
using namespace boost::asio;
using ip::tcp;
boost::system::error_code ec;
@@ -285,28 +278,25 @@ bool get_next_available_port(std::uint16_t first_port, std::uint16_t &available_
return not ec;
}
std::uint64_t get_time_now() {
auto get_time_now() -> std::uint64_t {
#ifdef _WIN32
return static_cast<std::uint64_t>(
std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()));
#else
#if __APPLE__
return std::chrono::nanoseconds(std::chrono::system_clock::now().time_since_epoch()).count();
return std::chrono::nanoseconds(
std::chrono::system_clock::now().time_since_epoch())
.count();
#else
return static_cast<std::uint64_t>(
std::chrono::nanoseconds(std::chrono::high_resolution_clock::now().time_since_epoch())
std::chrono::nanoseconds(
std::chrono::high_resolution_clock::now().time_since_epoch())
.count());
#endif
#endif
}
api_currency hastings_string_to_api_currency(const std::string &amount) {
ttmath::Parser<api_currency> parser;
parser.Parse(amount + " / (10 ^ 24)");
return parser.stack[0].value;
}
CURL *reset_curl(CURL *curl_handle) {
auto reset_curl(CURL *curl_handle) -> CURL * {
curl_easy_reset(curl_handle);
#if __APPLE__
curl_easy_setopt(curl_handle, CURLOPT_NOSIGNAL, 1);
@@ -314,56 +304,40 @@ CURL *reset_curl(CURL *curl_handle) {
return curl_handle;
}
bool retryable_action(const std::function<bool()> &action) {
auto retryable_action(const std::function<bool()> &action) -> bool {
auto succeeded = false;
for (auto i = 0; not(succeeded = action()) && (i < 10); i++) {
for (std::uint8_t i = 0u; not(succeeded = action()) && (i < 20u); i++) {
std::this_thread::sleep_for(100ms);
}
return succeeded;
}
/* bool parse_url(const std::string &url, HostConfig &hc) { */
/* static const auto pathRegex = std::regex( */
/* R"(^((\w+):)?(\/\/((\w+)?(:(\w+))?@)?([^\/\?:]+)(:(\d+))?)?(\/?([^\/\?#][^\?#]*)?)?(\?([^#]+))?(#(\w*))?$)");
*/
/* */
/* std::smatch results; */
/* if (std::regex_search(url, results, pathRegex)) { */
/* hc.HostNameOrIp = results[8u].str(); */
/* hc.Path = utils::path::create_api_path(results[11u].str()); */
/* hc.Protocol = utils::string::to_lower(results[2u].str()); */
/* hc.ApiPort = results[10u].str().empty() ? ((hc.Protocol == "https") ? 443u : 80u) */
/* : utils::string::to_uint16(results[10u].str()); */
/* return not hc.HostNameOrIp.empty() && not hc.Protocol.empty() && */
/* ((hc.Protocol == "http") || (hc.Protocol == "https")); */
/* } */
/* */
/* return false; */
/* } */
void spin_wait_for_mutex(std::function<bool()> complete, std::condition_variable &cv,
std::mutex &mtx, const std::string &text) {
void spin_wait_for_mutex(std::function<bool()> complete,
std::condition_variable &cv, std::mutex &mtx,
const std::string &text) {
while (not complete()) {
unique_mutex_lock l(mtx);
if (not complete()) {
if (not text.empty()) {
/* event_system::instance().raise<DebugLog>(__FUNCTION__, "spin_wait_for_mutex", text); */
/* event_system::instance().raise<DebugLog>(__FUNCTION__,
* "spin_wait_for_mutex", text); */
}
cv.wait_for(l, 5s);
cv.wait_for(l, 1s);
}
l.unlock();
}
}
void spin_wait_for_mutex(bool &complete, std::condition_variable &cv, std::mutex &mtx,
const std::string &text) {
void spin_wait_for_mutex(bool &complete, std::condition_variable &cv,
std::mutex &mtx, const std::string &text) {
while (not complete) {
unique_mutex_lock l(mtx);
if (not complete) {
if (not text.empty()) {
/* event_system::instance().raise<DebugLog>(__FUNCTION__, "spin_wait_for_mutex", text); */
/* event_system::instance().raise<DebugLog>(__FUNCTION__,
* "spin_wait_for_mutex", text); */
}
cv.wait_for(l, 5s);
cv.wait_for(l, 1s);
}
l.unlock();
}

194
src/utils/windows/windows_utils.cpp
View File

@@ -1,73 +1,43 @@
/*
Copyright <2018-2022> <scott.e.graves@protonmail.com>
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#ifdef _WIN32
#include "utils/windows/windows_utils.hpp"
#include "types/startup_exception.hpp"
#include "utils/com_init_wrapper.hpp"
#include "utils/string_utils.hpp"
#ifndef STATUS_DEVICE_INSUFFICIENT_RESOURCES
#define STATUS_DEVICE_INSUFFICIENT_RESOURCES ((NTSTATUS)0xC0000468L)
#endif
namespace repertory::utils {
DWORD get_last_error_code() { return ::GetLastError(); }
const std::string &get_local_app_data_directory() {
static std::string app_data = ([]() -> std::string {
com_init_wrapper cw;
PWSTR local_app_data{};
if (SUCCEEDED(::SHGetKnownFolderPath(FOLDERID_LocalAppData, 0, nullptr, &local_app_data))) {
auto app_data = utils::string::to_utf8(local_app_data);
::CoTaskMemFree(local_app_data);
return app_data;
}
throw startup_exception("unable to detect local application data folder");
})();
return app_data;
}
std::uint64_t get_accessed_time_from_meta(const api_meta_map &meta) {
return utils::string::to_uint64(meta.at(META_ACCESSED));
}
DWORD get_attributes_from_meta(const api_meta_map &meta) {
return static_cast<DWORD>(utils::string::to_uint32(meta.at(META_ATTRIBUTES)));
}
std::uint64_t get_changed_time_from_meta(const api_meta_map &meta) {
return utils::string::to_uint64(meta.at(META_MODIFIED));
}
std::uint64_t get_creation_time_from_meta(const api_meta_map &meta) {
return utils::string::to_uint64(meta.at(META_CREATION));
}
std::uint64_t get_written_time_from_meta(const api_meta_map &meta) {
return utils::string::to_uint64(meta.at(META_WRITTEN));
}
std::uint64_t get_thread_id() { return static_cast<std::uint64_t>(::GetCurrentThreadId()); }
NTSTATUS translate_api_error(const api_error &e) {
auto from_api_error(const api_error &e) -> NTSTATUS {
switch (e) {
case api_error::access_denied:
return FspNtStatusFromWin32(ERROR_ACCESS_DENIED);
return STATUS_ACCESS_DENIED;
case api_error::bad_address:
return STATUS_INVALID_ADDRESS;
case api_error::buffer_too_small:
return STATUS_BUFFER_TOO_SMALL;
case api_error::buffer_overflow:
@@ -84,20 +54,24 @@ NTSTATUS translate_api_error(const api_error &e) {
return FspNtStatusFromWin32(ERROR_INTERNAL_ERROR);
case api_error::error:
return FspNtStatusFromWin32(ERROR_INTERNAL_ERROR);
case api_error::file_exists:
case api_error::invalid_handle:
return STATUS_INVALID_HANDLE;
case api_error::invalid_operation:
return STATUS_INVALID_PARAMETER;
case api_error::item_exists:
return FspNtStatusFromWin32(ERROR_FILE_EXISTS);
case api_error::file_in_use:
return FspNtStatusFromWin32(ERROR_BUSY);
return STATUS_DEVICE_BUSY;
case api_error::incompatible_version:
return STATUS_CLIENT_SERVER_PARAMETERS_INVALID;
case api_error::invalid_operation:
return FspNtStatusFromWin32(ERROR_INTERNAL_ERROR);
case api_error::item_not_found:
return STATUS_OBJECT_NAME_NOT_FOUND;
case api_error::item_is_file:
return STATUS_ACCESS_DENIED;
case api_error::no_disk_space:
return STATUS_DEVICE_INSUFFICIENT_RESOURCES;
case api_error::os_error:
return FspNtStatusFromWin32(::GetLastError());
case api_error::out_of_memory:
return STATUS_NO_MEMORY;
case api_error::permission_denied:
return FspNtStatusFromWin32(ERROR_ACCESS_DENIED);
case api_error::success:
@@ -112,7 +86,47 @@ NTSTATUS translate_api_error(const api_error &e) {
return FspNtStatusFromWin32(ERROR_INTERNAL_ERROR);
}
}
bool is_process_elevated() {
auto get_last_error_code() -> DWORD { return ::GetLastError(); }
auto get_local_app_data_directory() -> const std::string & {
static std::string app_data = ([]() -> std::string {
com_init_wrapper cw;
PWSTR local_app_data{};
if (SUCCEEDED(::SHGetKnownFolderPath(FOLDERID_LocalAppData, 0, nullptr,
&local_app_data))) {
auto app_data = utils::string::to_utf8(local_app_data);
::CoTaskMemFree(local_app_data);
return app_data;
}
throw startup_exception("unable to detect local application data folder");
})();
return app_data;
}
auto get_accessed_time_from_meta(const api_meta_map &meta) -> std::uint64_t {
return utils::string::to_uint64(meta.at(META_ACCESSED));
}
auto get_changed_time_from_meta(const api_meta_map &meta) -> std::uint64_t {
return utils::string::to_uint64(meta.at(META_MODIFIED));
}
auto get_creation_time_from_meta(const api_meta_map &meta) -> std::uint64_t {
return utils::string::to_uint64(meta.at(META_CREATION));
}
auto get_written_time_from_meta(const api_meta_map &meta) -> std::uint64_t {
return utils::string::to_uint64(meta.at(META_WRITTEN));
}
auto get_thread_id() -> std::uint64_t {
return static_cast<std::uint64_t>(::GetCurrentThreadId());
}
auto is_process_elevated() -> bool {
auto ret = false;
HANDLE token = INVALID_HANDLE_VALUE;
if (::OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY, &token)) {
@@ -130,7 +144,7 @@ bool is_process_elevated() {
return ret;
}
int run_process_elevated(int argc, char *argv[]) {
auto run_process_elevated(int argc, char *argv[]) -> int {
std::cout << "Elevating Process" << std::endl;
std::string parameters = "-hidden";
for (int i = 1; i < argc; i++) {
@@ -161,7 +175,19 @@ int run_process_elevated(int argc, char *argv[]) {
void set_last_error_code(DWORD error_code) { ::SetLastError(error_code); }
int unix_access_mask_to_windows(std::int32_t mask) {
// https://stackoverflow.com/questions/321849/strptime-equivalent-on-windows
auto strptime(const char *s, const char *f, struct tm *tm) -> const char * {
std::istringstream input(s);
input.imbue(std::locale(setlocale(LC_ALL, nullptr)));
input >> std::get_time(tm, f);
if (input.fail()) {
return nullptr;
}
return reinterpret_cast<const char *>(s + input.tellg());
}
auto unix_access_mask_to_windows(std::int32_t mask) -> int {
if (mask & 1) {
mask -= 1;
if (not mask) {
@@ -172,25 +198,43 @@ int unix_access_mask_to_windows(std::int32_t mask) {
return mask & 6;
}
int unix_open_flags_to_flags_and_perms(const remote::file_mode &mode,
const remote::open_flags &flags, std::int32_t &perms) {
auto unix_open_flags_to_flags_and_perms(const remote::file_mode & /*mode*/,
const remote::open_flags &flags,
std::int32_t &perms) -> int {
auto ret = _O_BINARY | _O_RANDOM;
ret |= (((flags & remote::open_flags::create) == remote::open_flags::create) ? _O_CREAT : 0);
ret |= (((flags & remote::open_flags::excl) == remote::open_flags::excl) ? _O_EXCL : 0);
ret |= (((flags & remote::open_flags::truncate) == remote::open_flags::truncate) ? _O_TRUNC : 0);
ret |= (((flags & remote::open_flags::append) == remote::open_flags::append) ? _O_APPEND : 0);
ret |= (((flags & remote::open_flags::temp_file) == remote::open_flags::temp_file) ? _O_TEMPORARY
: 0);
ret |= ((flags & remote::open_flags::write_only) == remote::open_flags::write_only) ? _O_WRONLY
: ((flags & remote::open_flags::read_write) == remote::open_flags::read_write) ? _O_RDWR
: _O_RDONLY;
ret |= (((flags & remote::open_flags::create) == remote::open_flags::create)
? _O_CREAT
: 0);
ret |= (((flags & remote::open_flags::excl) == remote::open_flags::excl)
? _O_EXCL
: 0);
ret |=
(((flags & remote::open_flags::truncate) == remote::open_flags::truncate)
? _O_TRUNC
: 0);
ret |= (((flags & remote::open_flags::append) == remote::open_flags::append)
? _O_APPEND
: 0);
ret |= (((flags & remote::open_flags::temp_file) ==
remote::open_flags::temp_file)
? _O_TEMPORARY
: 0);
ret |= ((flags & remote::open_flags::write_only) ==
remote::open_flags::write_only)
? _O_WRONLY
: ((flags & remote::open_flags::read_write) ==
remote::open_flags::read_write)
? _O_RDWR
: _O_RDONLY;
perms = _S_IREAD | _S_IWRITE;
return ret;
}
remote::file_time time64_to_unix_time(const __time64_t &t) { return t * NANOS_PER_SECOND; }
auto time64_to_unix_time(const __time64_t &t) -> remote::file_time {
return t * NANOS_PER_SECOND;
}
} // namespace repertory::utils
#endif // _WIN32