initial commit
This commit is contained in:
733
src/app_config.cpp
Normal file
733
src/app_config.cpp
Normal file
@@ -0,0 +1,733 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "app_config.hpp"
|
||||
#include "types/startup_exception.hpp"
|
||||
#include "utils/file_utils.hpp"
|
||||
#include "utils/path_utils.hpp"
|
||||
#include "utils/utils.hpp"
|
||||
|
||||
namespace repertory {
|
||||
app_config::app_config(const provider_type &pt, const std::string &data_directory)
|
||||
: pt_(pt),
|
||||
api_auth_(utils::generate_random_string(48u)),
|
||||
api_port_(default_rpc_port(pt)),
|
||||
api_user_("repertory"),
|
||||
config_changed_(false),
|
||||
data_directory_(data_directory.empty() ? default_data_directory(pt)
|
||||
: ((pt == provider_type::remote) || (pt == provider_type::s3))
|
||||
? utils::path::absolute(data_directory)
|
||||
: utils::path::absolute(
|
||||
utils::path::combine(data_directory, {get_provider_name(pt)}))),
|
||||
download_timeout_secs_(30),
|
||||
enable_chunk_downloader_timeout_(true),
|
||||
enable_comm_duration_events_(false),
|
||||
enable_drive_events_(false),
|
||||
enable_max_cache_size_(true),
|
||||
#ifdef _WIN32
|
||||
enable_mount_manager_(false),
|
||||
#endif
|
||||
enable_remote_mount_(false),
|
||||
event_level_(event_level::normal),
|
||||
eviction_delay_mins_(30),
|
||||
eviction_uses_accessed_time_(false),
|
||||
high_freq_interval_secs_(30),
|
||||
is_remote_mount_(false),
|
||||
low_freq_interval_secs_(60 * 60),
|
||||
max_cache_size_bytes_(20 * 1024 * 1024 * 1024ULL),
|
||||
max_upload_count_(5u),
|
||||
min_download_timeout_secs_(5),
|
||||
minimum_redundancy_(2.5),
|
||||
online_check_retry_secs_(60),
|
||||
orphaned_file_retention_days_(15),
|
||||
preferred_download_type_(utils::download_type_to_string(download_type::fallback)),
|
||||
read_ahead_count_(4),
|
||||
remote_client_pool_size_(10),
|
||||
remote_host_name_or_ip_(""),
|
||||
remote_max_connections_(20),
|
||||
remote_port_((pt == provider_type::sia) ? 20000
|
||||
: (pt == provider_type::s3) ? 20002
|
||||
: (pt == provider_type::skynet) ? 20003
|
||||
: 20001),
|
||||
remote_receive_timeout_secs_(120),
|
||||
remote_send_timeout_secs_(30),
|
||||
remote_token_(""),
|
||||
retry_read_count_(6),
|
||||
ring_buffer_file_size_(512),
|
||||
storage_byte_month_("0") {
|
||||
cache_directory_ = utils::path::combine(data_directory_, {"cache"});
|
||||
log_directory_ = utils::path::combine(data_directory_, {"logs"});
|
||||
|
||||
hc_.agent_string = default_agent_name(pt_);
|
||||
hc_.api_password = get_provider_api_password(pt_);
|
||||
hc_.api_port = default_api_port(pt_);
|
||||
|
||||
if (not utils::file::create_full_directory_path(data_directory_))
|
||||
throw startup_exception("unable to create: " + data_directory_);
|
||||
|
||||
if (not utils::file::create_full_directory_path(cache_directory_))
|
||||
throw startup_exception("unable to create: " + cache_directory_);
|
||||
|
||||
if (not utils::file::create_full_directory_path(log_directory_))
|
||||
throw startup_exception("unable to create: " + log_directory_);
|
||||
|
||||
if (not load()) {
|
||||
save();
|
||||
}
|
||||
}
|
||||
|
||||
std::string app_config::get_config_file_path() const {
|
||||
const auto configFilePath = utils::path::combine(data_directory_, {"config.json"});
|
||||
return configFilePath;
|
||||
}
|
||||
|
||||
std::string app_config::default_agent_name(const provider_type &pt) {
|
||||
static const std::vector<std::string> PROVIDER_AGENT_NAMES = {"Sia-Agent", "", "", "", ""};
|
||||
|
||||
return PROVIDER_AGENT_NAMES[static_cast<int>(pt)];
|
||||
}
|
||||
|
||||
std::uint16_t app_config::default_api_port(const provider_type &pt) {
|
||||
static const std::vector<std::uint16_t> PROVIDER_API_PORTS = {9980u, 0u, 0u, 0u, 0u};
|
||||
return PROVIDER_API_PORTS[static_cast<int>(pt)];
|
||||
}
|
||||
|
||||
std::string app_config::default_data_directory(const provider_type &pt) {
|
||||
#ifdef _WIN32
|
||||
auto data_directory =
|
||||
utils::path::combine(utils::get_local_app_data_directory(),
|
||||
{REPERTORY_DATA_NAME, app_config::get_provider_name(pt)});
|
||||
#else
|
||||
#ifdef __APPLE__
|
||||
auto data_directory =
|
||||
utils::path::resolve(std::string("~/Library/Application Support/") + REPERTORY_DATA_NAME +
|
||||
'/' + app_config::get_provider_name(pt));
|
||||
#else
|
||||
auto data_directory = utils::path::resolve(std::string("~/.local/") + REPERTORY_DATA_NAME + '/' +
|
||||
app_config::get_provider_name(pt));
|
||||
#endif
|
||||
#endif
|
||||
return data_directory;
|
||||
}
|
||||
|
||||
std::uint16_t app_config::default_rpc_port(const provider_type &pt) {
|
||||
static const std::vector<std::uint16_t> PROVIDER_RPC_PORTS = {
|
||||
11101u, 11105u, 11103u, 11104u, 11105u,
|
||||
};
|
||||
return PROVIDER_RPC_PORTS[static_cast<int>(pt)];
|
||||
}
|
||||
|
||||
json app_config::get_json() const {
|
||||
json ret = {{"ApiAuth", api_auth_},
|
||||
{"ApiPort", api_port_},
|
||||
{"ApiUser", api_user_},
|
||||
{"ChunkDownloaderTimeoutSeconds", download_timeout_secs_},
|
||||
{"EnableChunkDownloaderTimeout", enable_chunk_downloader_timeout_},
|
||||
{"EnableCommDurationEvents", enable_comm_duration_events_},
|
||||
{"EnableDriveEvents", enable_drive_events_},
|
||||
#ifdef _WIN32
|
||||
{"EnableMountManager", enable_mount_manager_},
|
||||
#endif
|
||||
{"EnableMaxCacheSize", enable_max_cache_size_},
|
||||
{"EventLevel", event_level_to_string(event_level_)},
|
||||
{"EvictionDelayMinutes", eviction_delay_mins_},
|
||||
{"EvictionUsesAccessedTime", eviction_uses_accessed_time_},
|
||||
{"HighFreqIntervalSeconds", high_freq_interval_secs_},
|
||||
{"HostConfig",
|
||||
{{"AgentString", hc_.agent_string},
|
||||
{"ApiPassword", hc_.api_password},
|
||||
{"ApiPort", hc_.api_port},
|
||||
{"HostNameOrIp", hc_.host_name_or_ip},
|
||||
{"TimeoutMs", hc_.timeout_ms}}},
|
||||
{"LowFreqIntervalSeconds", low_freq_interval_secs_},
|
||||
{"MaxCacheSizeBytes", max_cache_size_bytes_},
|
||||
{"MaxUploadCount", max_upload_count_},
|
||||
{"MinimumRedundancy", minimum_redundancy_},
|
||||
{"OnlineCheckRetrySeconds", online_check_retry_secs_},
|
||||
{"OrphanedFileRetentionDays", orphaned_file_retention_days_},
|
||||
{"PreferredDownloadType", preferred_download_type_},
|
||||
{"ReadAheadCount", read_ahead_count_},
|
||||
{
|
||||
"RemoteMount",
|
||||
{{"EnableRemoteMount", enable_remote_mount_},
|
||||
{"IsRemoteMount", is_remote_mount_},
|
||||
{"RemoteClientPoolSize", remote_client_pool_size_},
|
||||
{"RemoteMaxConnections", remote_max_connections_},
|
||||
{"RemoteHostNameOrIp", remote_host_name_or_ip_},
|
||||
{"RemotePort", remote_port_},
|
||||
{"RemoteReceiveTimeoutSeconds", remote_receive_timeout_secs_},
|
||||
{"RemoteSendTimeoutSeconds", remote_send_timeout_secs_},
|
||||
{"RemoteToken", remote_token_}},
|
||||
},
|
||||
{"RetryReadCount", retry_read_count_},
|
||||
{"RingBufferFileSize", ring_buffer_file_size_},
|
||||
{"S3Config",
|
||||
{{"AccessKey", s3_config_.access_key},
|
||||
{"Bucket", s3_config_.bucket},
|
||||
{"CacheTimeoutSeconds", s3_config_.cache_timeout_secs},
|
||||
{"EncryptionToken", s3_config_.encryption_token},
|
||||
{"Region", s3_config_.region},
|
||||
{"SecretKey", s3_config_.secret_key},
|
||||
{"TimeoutMs", s3_config_.timeout_ms},
|
||||
{"URL", s3_config_.url}}},
|
||||
{"SkynetConfig",
|
||||
{{"EncryptionToken", skynet_config_.encryption_token},
|
||||
{"PortalList", skynet_config_.portal_list}}},
|
||||
{"StorageByteMonth", storage_byte_month_.ToString()},
|
||||
{"Version", version_}};
|
||||
|
||||
if (pt_ == provider_type::s3) {
|
||||
ret.erase("HostConfig");
|
||||
ret.erase("SkynetConfig");
|
||||
ret.erase("StorageByteMonth");
|
||||
} else if ((pt_ == provider_type::sia)) {
|
||||
ret.erase("S3Config");
|
||||
ret.erase("SkynetConfig");
|
||||
ret.erase("MaxUploadCount");
|
||||
} else if (pt_ == provider_type::skynet) {
|
||||
ret.erase("S3Config");
|
||||
ret.erase("HostConfig");
|
||||
ret.erase("StorageByteMonth");
|
||||
ret.erase("OrphanedFileRetentionDays");
|
||||
} else if (pt_ == provider_type::remote) {
|
||||
ret.erase("MaxUploadCount");
|
||||
ret.erase("S3Config");
|
||||
ret.erase("HostConfig");
|
||||
ret.erase("SkynetConfig");
|
||||
ret.erase("ChunkDownloaderTimeoutSeconds");
|
||||
ret.erase("EnableChunkDownloaderTimeout");
|
||||
ret.erase("EnableChunkDownloaderTimeout");
|
||||
ret.erase("EnableMountManager");
|
||||
ret.erase("EnableMaxCacheSize");
|
||||
ret.erase("EvictionDelayMinutes");
|
||||
ret.erase("HighFreqIntervalSeconds");
|
||||
ret.erase("LowFreqIntervalSeconds");
|
||||
ret.erase("MaxCacheSizeBytes");
|
||||
ret.erase("MinimumRedundancy");
|
||||
ret.erase("OnlineCheckRetrySeconds");
|
||||
ret.erase("OrphanedFileRetentionDays");
|
||||
ret.erase("PreferredDownloadType");
|
||||
ret.erase("ReadAheadCount");
|
||||
ret.erase("RetryReadCount");
|
||||
ret.erase("RingBufferFileSize");
|
||||
ret.erase("StorageByteMonth");
|
||||
}
|
||||
|
||||
if ((pt_ == provider_type::s3) || (pt_ == provider_type::skynet)) {
|
||||
ret.erase("MinimumRedundancy");
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
std::uint64_t app_config::get_max_cache_size_bytes() const {
|
||||
const auto maxSpace = std::max((std::uint64_t)100 * 1024 * 1024, max_cache_size_bytes_);
|
||||
return std::min(utils::file::get_available_drive_space(get_cache_directory()), maxSpace);
|
||||
}
|
||||
|
||||
std::string app_config::get_provider_api_password(const provider_type &pt) {
|
||||
#ifdef _WIN32
|
||||
auto api_file = utils::path::combine(utils::get_local_app_data_directory(),
|
||||
{get_provider_display_name(pt), "apipassword"});
|
||||
#else
|
||||
#ifdef __APPLE__
|
||||
auto api_file = utils::path::combine(
|
||||
utils::path::resolve("~"),
|
||||
{"/Library/Application Support", get_provider_display_name(pt), "apipassword"});
|
||||
#else
|
||||
auto api_file =
|
||||
utils::path::combine(utils::path::resolve("~/."), {get_provider_name(pt), "apipassword"});
|
||||
#endif
|
||||
#endif
|
||||
auto lines = utils::file::read_file_lines(api_file);
|
||||
return lines.empty() ? "" : utils::string::trim(lines[0]);
|
||||
}
|
||||
|
||||
std::string app_config::get_provider_display_name(const provider_type &pt) {
|
||||
static const std::vector<std::string> PROVIDER_DISPLAY_NAMES = {
|
||||
"Sia", "Remote", "S3", "Skynet", "Passthrough",
|
||||
};
|
||||
return PROVIDER_DISPLAY_NAMES[static_cast<int>(pt)];
|
||||
}
|
||||
|
||||
std::string app_config::get_provider_minimum_version(const provider_type &pt) {
|
||||
static const std::vector<std::string> PROVIDER_MIN_VERSIONS = {
|
||||
MIN_SIA_VERSION, MIN_SP_VERSION, MIN_REMOTE_VERSION, "", "", ""};
|
||||
return PROVIDER_MIN_VERSIONS[static_cast<int>(pt)];
|
||||
}
|
||||
|
||||
std::string app_config::get_provider_name(const provider_type &pt) {
|
||||
static const std::vector<std::string> PROVIDER_NAMES = {
|
||||
"sia", "remote", "s3", "skynet", "passthrough",
|
||||
};
|
||||
return PROVIDER_NAMES[static_cast<int>(pt)];
|
||||
}
|
||||
|
||||
std::string app_config::get_provider_path_name(const provider_type &pt) {
|
||||
static const std::vector<std::string> PROVIDER_PATH_NAMES = {
|
||||
"siapath", "", "", "", "",
|
||||
};
|
||||
return PROVIDER_PATH_NAMES[static_cast<int>(pt)];
|
||||
}
|
||||
|
||||
std::string app_config::get_value_by_name(const std::string &name) {
|
||||
try {
|
||||
if (name == "ApiAuth") {
|
||||
return api_auth_;
|
||||
} else if (name == "ApiPort") {
|
||||
return std::to_string(get_api_port());
|
||||
} else if (name == "ApiUser") {
|
||||
return api_user_;
|
||||
} else if (name == "ChunkDownloaderTimeoutSeconds") {
|
||||
return std::to_string(get_chunk_downloader_timeout_secs());
|
||||
} else if (name == "EnableChunkDownloaderTimeout") {
|
||||
return std::to_string(get_enable_chunk_download_timeout());
|
||||
} else if (name == "GetEnableCommDurationEvents") {
|
||||
return std::to_string(get_enable_comm_duration_events());
|
||||
} else if (name == "EnableDriveEvents") {
|
||||
return std::to_string(get_enable_drive_events());
|
||||
} else if (name == "EnableMaxCacheSize") {
|
||||
return std::to_string(get_enable_max_cache_size());
|
||||
#ifdef _WIN32
|
||||
} else if (name == "EnableMountManager") {
|
||||
return std::to_string(get_enable_mount_manager());
|
||||
#endif
|
||||
} else if (name == "EventLevel") {
|
||||
return event_level_to_string(get_event_level());
|
||||
} else if (name == "EvictionDelayMinutes") {
|
||||
return std::to_string(get_eviction_delay_mins());
|
||||
} else if (name == "EvictionUsesAccessedTime") {
|
||||
return std::to_string(get_eviction_uses_accessed_time());
|
||||
} else if (name == "HighFreqIntervalSeconds") {
|
||||
return std::to_string(get_high_frequency_interval_secs());
|
||||
} else if (name == "HostConfig.AgentString") {
|
||||
return hc_.agent_string;
|
||||
} else if (name == "HostConfig.ApiPassword") {
|
||||
return hc_.api_password;
|
||||
} else if (name == "HostConfig.ApiPort") {
|
||||
return std::to_string(hc_.api_port);
|
||||
} else if (name == "HostConfig.HostNameOrIp") {
|
||||
return hc_.host_name_or_ip;
|
||||
} else if (name == "HostConfig.TimeoutMs") {
|
||||
return std::to_string(hc_.timeout_ms);
|
||||
} else if (name == "LowFreqIntervalSeconds") {
|
||||
return std::to_string(get_low_frequency_interval_secs());
|
||||
} else if (name == "MaxCacheSizeBytes") {
|
||||
return std::to_string(get_max_cache_size_bytes());
|
||||
} else if (name == "MaxUploadCount") {
|
||||
return std::to_string(get_max_upload_count());
|
||||
} else if (name == "MinimumRedundancy") {
|
||||
return std::to_string(get_minimum_redundancy());
|
||||
} else if (name == "OnlineCheckRetrySeconds") {
|
||||
return std::to_string(get_online_check_retry_secs());
|
||||
} else if (name == "OrphanedFileRetentionDays") {
|
||||
return std::to_string(get_orphaned_file_retention_days());
|
||||
} else if (name == "PreferredDownloadType") {
|
||||
return utils::download_type_to_string(
|
||||
utils::download_type_from_string(preferred_download_type_, download_type::fallback));
|
||||
} else if (name == "ReadAheadCount") {
|
||||
return std::to_string(get_read_ahead_count());
|
||||
} else if (name == "RemoteMount.EnableRemoteMount") {
|
||||
return std::to_string(get_enable_remote_mount());
|
||||
} else if (name == "RemoteMount.IsRemoteMount") {
|
||||
return std::to_string(get_is_remote_mount());
|
||||
} else if (name == "RemoteMount.RemoteClientPoolSize") {
|
||||
return std::to_string(get_remote_client_pool_size());
|
||||
} else if (name == "RemoteMount.RemoteHostNameOrIp") {
|
||||
return get_remote_host_name_or_ip();
|
||||
} else if (name == "RemoteMount.RemoteMaxConnections") {
|
||||
return std::to_string(get_remote_max_connections());
|
||||
} else if (name == "RemoteMount.RemotePort") {
|
||||
return std::to_string(get_remote_port());
|
||||
} else if (name == "RemoteMount.RemoteReceiveTimeoutSeconds") {
|
||||
return std::to_string(get_remote_receive_timeout_secs());
|
||||
} else if (name == "RemoteMount.RemoteSendTimeoutSeconds") {
|
||||
return std::to_string(get_remote_send_timeout_secs());
|
||||
} else if (name == "RemoteMount.RemoteToken") {
|
||||
return get_remote_token();
|
||||
} else if (name == "RetryReadCount") {
|
||||
return std::to_string(get_retry_read_count());
|
||||
} else if (name == "RingBufferFileSize") {
|
||||
return std::to_string(get_ring_buffer_file_size());
|
||||
} else if (name == "S3Config.AccessKey") {
|
||||
return s3_config_.access_key;
|
||||
} else if (name == "S3Config.Bucket") {
|
||||
return s3_config_.bucket;
|
||||
} else if (name == "S3Config.EncryptionToken") {
|
||||
return s3_config_.encryption_token;
|
||||
} else if (name == "S3Config.CacheTimeoutSeconds") {
|
||||
return std::to_string(s3_config_.cache_timeout_secs);
|
||||
} else if (name == "S3Config.Region") {
|
||||
return s3_config_.region;
|
||||
} else if (name == "S3Config.SecretKey") {
|
||||
return s3_config_.secret_key;
|
||||
} else if (name == "S3Config.URL") {
|
||||
return s3_config_.url;
|
||||
} else if (name == "S3Config.TimeoutMs") {
|
||||
return std::to_string(s3_config_.timeout_ms);
|
||||
} else if (name == "S3Config.EncryptionToken") {
|
||||
return s3_config_.encryption_token;
|
||||
} else if (name == "SkynetConfig.EncryptionToken") {
|
||||
return skynet_config_.encryption_token;
|
||||
} else if (name == "SkynetConfig.PortalList") {
|
||||
return skynet_config_.to_string();
|
||||
}
|
||||
} catch (const std::exception &e) {
|
||||
event_system::instance().raise<repertory_exception>(__FUNCTION__, e.what());
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
bool app_config::load() {
|
||||
auto ret = false;
|
||||
|
||||
const auto config_file_path = get_config_file_path();
|
||||
recur_mutex_lock l(read_write_mutex_);
|
||||
if (utils::file::is_file(config_file_path)) {
|
||||
try {
|
||||
std::ifstream config_file(&config_file_path[0]);
|
||||
if (config_file.is_open()) {
|
||||
std::stringstream ss;
|
||||
ss << config_file.rdbuf();
|
||||
const auto json_text = ss.str();
|
||||
config_file.close();
|
||||
if ((ret = not json_text.empty())) {
|
||||
const auto json_document = json::parse(json_text);
|
||||
|
||||
get_value(json_document, "ApiAuth", api_auth_, ret);
|
||||
get_value(json_document, "ApiPort", api_port_, ret);
|
||||
get_value(json_document, "ApiUser", api_user_, ret);
|
||||
get_value(json_document, "ChunkDownloaderTimeoutSeconds", download_timeout_secs_, ret);
|
||||
get_value(json_document, "EvictionDelayMinutes", eviction_delay_mins_, ret);
|
||||
get_value(json_document, "EvictionUsesAccessedTime", eviction_uses_accessed_time_, ret);
|
||||
get_value(json_document, "EnableChunkDownloaderTimeout", enable_chunk_downloader_timeout_,
|
||||
ret);
|
||||
get_value(json_document, "EnableCommDurationEvents", enable_comm_duration_events_, ret);
|
||||
get_value(json_document, "EnableDriveEvents", enable_drive_events_, ret);
|
||||
|
||||
std::string level;
|
||||
if (get_value(json_document, "EventLevel", level, ret)) {
|
||||
event_level_ = event_level_from_string(level);
|
||||
}
|
||||
|
||||
if (json_document.find("HostConfig") != json_document.end()) {
|
||||
auto host_config_json = json_document["HostConfig"];
|
||||
auto hc = hc_;
|
||||
get_value(host_config_json, "AgentString", hc.agent_string, ret);
|
||||
get_value(host_config_json, "ApiPassword", hc.api_password, ret);
|
||||
get_value(host_config_json, "ApiPort", hc.api_port, ret);
|
||||
get_value(host_config_json, "HostNameOrIp", hc.host_name_or_ip, ret);
|
||||
get_value(host_config_json, "TimeoutMs", hc.timeout_ms, ret);
|
||||
hc_ = hc;
|
||||
} else {
|
||||
ret = false;
|
||||
}
|
||||
|
||||
if (hc_.api_password.empty()) {
|
||||
hc_.api_password = get_provider_api_password(pt_);
|
||||
if (hc_.api_password.empty()) {
|
||||
ret = false;
|
||||
}
|
||||
}
|
||||
|
||||
if (json_document.find("S3Config") != json_document.end()) {
|
||||
auto s3_config_json = json_document["S3Config"];
|
||||
auto s3 = s3_config_;
|
||||
get_value(s3_config_json, "AccessKey", s3.access_key, ret);
|
||||
get_value(s3_config_json, "Bucket", s3.bucket, ret);
|
||||
get_value(s3_config_json, "CacheTimeoutSeconds", s3.cache_timeout_secs, ret);
|
||||
get_value(s3_config_json, "EncryptionToken", s3.encryption_token, ret);
|
||||
get_value(s3_config_json, "Region", s3.region, ret);
|
||||
get_value(s3_config_json, "SecretKey", s3.secret_key, ret);
|
||||
get_value(s3_config_json, "TimeoutMs", s3.timeout_ms, ret);
|
||||
get_value(s3_config_json, "URL", s3.url, ret);
|
||||
s3_config_ = s3;
|
||||
} else {
|
||||
ret = false;
|
||||
}
|
||||
|
||||
if (json_document.find("SkynetConfig") != json_document.end()) {
|
||||
auto skynet_config_json = json_document["SkynetConfig"];
|
||||
auto skynet = skynet_config_;
|
||||
get_value(skynet_config_json, "EncryptionToken", skynet.encryption_token, ret);
|
||||
get_value(skynet_config_json, "PortalList", skynet.portal_list, ret);
|
||||
skynet_config_ = skynet;
|
||||
} else {
|
||||
ret = false;
|
||||
}
|
||||
|
||||
get_value(json_document, "ReadAheadCount", read_ahead_count_, ret);
|
||||
get_value(json_document, "RingBufferFileSize", ring_buffer_file_size_, ret);
|
||||
get_value(json_document, "MinimumRedundancy", minimum_redundancy_, ret);
|
||||
get_value(json_document, "EnableMaxCacheSize", enable_max_cache_size_, ret);
|
||||
#ifdef _WIN32
|
||||
get_value(json_document, "EnableMountManager", enable_mount_manager_, ret);
|
||||
#endif
|
||||
get_value(json_document, "MaxCacheSizeBytes", max_cache_size_bytes_, ret);
|
||||
get_value(json_document, "MaxUploadCount", max_upload_count_, ret);
|
||||
get_value(json_document, "OnlineCheckRetrySeconds", online_check_retry_secs_, ret);
|
||||
get_value(json_document, "HighFreqIntervalSeconds", high_freq_interval_secs_, ret);
|
||||
get_value(json_document, "LowFreqIntervalSeconds", low_freq_interval_secs_, ret);
|
||||
get_value(json_document, "OrphanedFileRetentionDays", orphaned_file_retention_days_, ret);
|
||||
get_value(json_document, "PreferredDownloadType", preferred_download_type_, ret);
|
||||
get_value(json_document, "RetryReadCount", retry_read_count_, ret);
|
||||
if (json_document.find("RemoteMount") != json_document.end()) {
|
||||
auto remoteMount = json_document["RemoteMount"];
|
||||
get_value(remoteMount, "EnableRemoteMount", enable_remote_mount_, ret);
|
||||
get_value(remoteMount, "IsRemoteMount", is_remote_mount_, ret);
|
||||
get_value(remoteMount, "RemoteClientPoolSize", remote_client_pool_size_, ret);
|
||||
get_value(remoteMount, "RemoteHostNameOrIp", remote_host_name_or_ip_, ret);
|
||||
get_value(remoteMount, "RemoteMaxConnections", remote_max_connections_, ret);
|
||||
get_value(remoteMount, "RemotePort", remote_port_, ret);
|
||||
get_value(remoteMount, "RemoteReceiveTimeoutSeconds", remote_receive_timeout_secs_,
|
||||
ret);
|
||||
get_value(remoteMount, "RemoteSendTimeoutSeconds", remote_send_timeout_secs_, ret);
|
||||
get_value(remoteMount, "RemoteToken", remote_token_, ret);
|
||||
} else {
|
||||
ret = false;
|
||||
}
|
||||
std::string storage_byte_month;
|
||||
if (get_value(json_document, "StorageByteMonth", storage_byte_month, ret)) {
|
||||
storage_byte_month_ = storage_byte_month;
|
||||
}
|
||||
|
||||
std::uint64_t version = 0u;
|
||||
get_value(json_document, "Version", version, ret);
|
||||
|
||||
// Handle configuration defaults for new config versions
|
||||
if (version != REPERTORY_CONFIG_VERSION) {
|
||||
if (version > REPERTORY_CONFIG_VERSION) {
|
||||
version = 0u;
|
||||
}
|
||||
|
||||
version_ = version;
|
||||
ret = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (not ret) {
|
||||
config_changed_ = true;
|
||||
}
|
||||
} catch (const std::exception &ex) {
|
||||
event_system::instance().raise<repertory_exception>(__FUNCTION__, ex.what());
|
||||
ret = false;
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
void app_config::save() {
|
||||
const auto configFilePath = get_config_file_path();
|
||||
recur_mutex_lock l(read_write_mutex_);
|
||||
if (config_changed_ || not utils::file::is_file(configFilePath)) {
|
||||
if (not utils::file::is_directory(data_directory_)) {
|
||||
utils::file::create_full_directory_path(data_directory_);
|
||||
}
|
||||
config_changed_ = false;
|
||||
json data = get_json();
|
||||
auto success = false;
|
||||
for (auto i = 0; not success && (i < 5); i++) {
|
||||
if (not(success = utils::file::write_json_file(configFilePath, data))) {
|
||||
std::this_thread::sleep_for(1s);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void app_config::set_enable_remote_mount(const bool &enable_remote_mount) {
|
||||
recur_mutex_lock remote_lock(remote_mount_mutex_);
|
||||
if (get_is_remote_mount()) {
|
||||
set_value(enable_remote_mount_, false);
|
||||
} else {
|
||||
set_value(enable_remote_mount_, enable_remote_mount);
|
||||
}
|
||||
}
|
||||
|
||||
void app_config::set_is_remote_mount(const bool &is_remote_mount) {
|
||||
recur_mutex_lock remote_lock(remote_mount_mutex_);
|
||||
|
||||
if (get_enable_remote_mount()) {
|
||||
set_value(is_remote_mount_, false);
|
||||
return;
|
||||
}
|
||||
|
||||
set_value(is_remote_mount_, is_remote_mount);
|
||||
}
|
||||
|
||||
void app_config::set_storage_byte_month(const api_currency &storage_byte_month) {
|
||||
if ((storage_byte_month > 0) && (storage_byte_month_ != storage_byte_month)) {
|
||||
storage_byte_month_ = storage_byte_month;
|
||||
config_changed_ = true;
|
||||
}
|
||||
}
|
||||
|
||||
std::string app_config::set_value_by_name(const std::string &name, const std::string &value) {
|
||||
try {
|
||||
if (name == "ApiAuth") {
|
||||
set_api_auth(value);
|
||||
return get_api_auth();
|
||||
} else if (name == "ApiPort") {
|
||||
set_api_port(utils::string::to_uint16(value));
|
||||
return std::to_string(get_api_port());
|
||||
} else if (name == "ApiUser") {
|
||||
set_api_user(value);
|
||||
return get_api_user();
|
||||
} else if (name == "ChunkDownloaderTimeoutSeconds") {
|
||||
set_chunk_downloader_timeout_secs(utils::string::to_uint8(value));
|
||||
return std::to_string(get_chunk_downloader_timeout_secs());
|
||||
} else if (name == "EnableChunkDownloaderTimeout") {
|
||||
set_enable_chunk_downloader_timeout(utils::string::to_bool(value));
|
||||
return std::to_string(get_enable_chunk_download_timeout());
|
||||
} else if (name == "EnableCommDurationEvents") {
|
||||
set_enable_comm_duration_events(utils::string::to_bool(value));
|
||||
return std::to_string(get_enable_comm_duration_events());
|
||||
} else if (name == "EnableDriveEvents") {
|
||||
set_enable_drive_events(utils::string::to_bool(value));
|
||||
return std::to_string(get_enable_drive_events());
|
||||
} else if (name == "EnableMaxCacheSize") {
|
||||
set_enable_max_cache_size(utils::string::to_bool(value));
|
||||
return std::to_string(get_enable_max_cache_size());
|
||||
#ifdef _WIN32
|
||||
} else if (name == "EnableMountManager") {
|
||||
set_enable_mount_manager(utils::string::to_bool(value));
|
||||
return std::to_string(get_enable_mount_manager());
|
||||
#endif
|
||||
} else if (name == "EventLevel") {
|
||||
set_event_level(event_level_from_string(value));
|
||||
return event_level_to_string(get_event_level());
|
||||
} else if (name == "EvictionDelayMinutes") {
|
||||
set_eviction_delay_mins(utils::string::to_uint32(value));
|
||||
return std::to_string(get_eviction_delay_mins());
|
||||
} else if (name == "EvictionUsesAccessedTime") {
|
||||
set_eviction_uses_accessed_time(utils::string::to_bool(value));
|
||||
return std::to_string(get_eviction_uses_accessed_time());
|
||||
} else if (name == "HighFreqIntervalSeconds") {
|
||||
set_high_frequency_interval_secs(utils::string::to_uint8(value));
|
||||
return std::to_string(get_high_frequency_interval_secs());
|
||||
} else if (name == "HostConfig.AgentString") {
|
||||
set_value(hc_.agent_string, value);
|
||||
return hc_.agent_string;
|
||||
} else if (name == "HostConfig.ApiPassword") {
|
||||
set_value(hc_.api_password, value);
|
||||
return hc_.api_password;
|
||||
} else if (name == "HostConfig.ApiPort") {
|
||||
set_value(hc_.api_port, utils::string::to_uint16(value));
|
||||
return std::to_string(hc_.api_port);
|
||||
} else if (name == "HostConfig.HostNameOrIp") {
|
||||
set_value(hc_.host_name_or_ip, value);
|
||||
return hc_.host_name_or_ip;
|
||||
} else if (name == "HostConfig.TimeoutMs") {
|
||||
set_value(hc_.timeout_ms, utils::string::to_uint32(value));
|
||||
return std::to_string(hc_.timeout_ms);
|
||||
} else if (name == "LowFreqIntervalSeconds") {
|
||||
set_low_frequency_interval_secs(utils::string::to_uint32(value));
|
||||
return std::to_string(get_low_frequency_interval_secs());
|
||||
} else if (name == "MaxCacheSizeBytes") {
|
||||
set_max_cache_size_bytes(utils::string::to_uint64(value));
|
||||
return std::to_string(get_max_cache_size_bytes());
|
||||
} else if (name == "MaxUploadCount") {
|
||||
set_max_upload_count(utils::string::to_uint8(value));
|
||||
return std::to_string(get_max_upload_count());
|
||||
} else if (name == "MinimumRedundancy") {
|
||||
set_minimum_redundancy(utils::string::to_double(value));
|
||||
return std::to_string(get_minimum_redundancy());
|
||||
} else if (name == "OnlineCheckRetrySeconds") {
|
||||
set_online_check_retry_secs(utils::string::to_uint16(value));
|
||||
return std::to_string(get_online_check_retry_secs());
|
||||
} else if (name == "OrphanedFileRetentionDays") {
|
||||
set_orphaned_file_retention_days(utils::string::to_uint16(value));
|
||||
return std::to_string(get_orphaned_file_retention_days());
|
||||
} else if (name == "PreferredDownloadType") {
|
||||
set_preferred_download_type(utils::download_type_from_string(value, download_type::fallback));
|
||||
return utils::download_type_to_string(get_preferred_download_type());
|
||||
} else if (name == "ReadAheadCount") {
|
||||
set_read_ahead_count(utils::string::to_uint8(value));
|
||||
return std::to_string(get_read_ahead_count());
|
||||
} else if (name == "RemoteMount.EnableRemoteMount") {
|
||||
set_enable_remote_mount(utils::string::to_bool(value));
|
||||
return std::to_string(get_enable_remote_mount());
|
||||
} else if (name == "RemoteMount.IsRemoteMount") {
|
||||
set_is_remote_mount(utils::string::to_bool(value));
|
||||
return std::to_string(get_is_remote_mount());
|
||||
} else if (name == "RemoteMount.RemoteClientPoolSize") {
|
||||
set_remote_client_pool_size(utils::string::to_uint8(value));
|
||||
return std::to_string(get_remote_client_pool_size());
|
||||
} else if (name == "RemoteMount.RemoteHostNameOrIp") {
|
||||
set_remote_host_name_or_ip(value);
|
||||
return get_remote_host_name_or_ip();
|
||||
} else if (name == "RemoteMount.RemoteMaxConnections") {
|
||||
set_remote_max_connections(utils::string::to_uint8(value));
|
||||
return std::to_string(get_remote_max_connections());
|
||||
} else if (name == "RemoteMount.RemotePort") {
|
||||
set_remote_port(utils::string::to_uint16(value));
|
||||
return std::to_string(get_remote_port());
|
||||
} else if (name == "RemoteMount.RemoteReceiveTimeoutSeconds") {
|
||||
set_remote_receive_timeout_secs(utils::string::to_uint16(value));
|
||||
return std::to_string(get_remote_receive_timeout_secs());
|
||||
} else if (name == "RemoteMount.RemoteSendTimeoutSeconds") {
|
||||
set_remote_send_timeout_secs(utils::string::to_uint16(value));
|
||||
return std::to_string(get_remote_send_timeout_secs());
|
||||
} else if (name == "RemoteMount.RemoteToken") {
|
||||
set_remote_token(value);
|
||||
return get_remote_token();
|
||||
} else if (name == "RetryReadCount") {
|
||||
set_retry_read_count(utils::string::to_uint16(value));
|
||||
return std::to_string(get_retry_read_count());
|
||||
} else if (name == "RingBufferFileSize") {
|
||||
set_ring_buffer_file_size(utils::string::to_uint16(value));
|
||||
return std::to_string(get_ring_buffer_file_size());
|
||||
} else if (name == "S3Config.AccessKey") {
|
||||
set_value(s3_config_.access_key, value);
|
||||
return s3_config_.access_key;
|
||||
} else if (name == "S3Config.Bucket") {
|
||||
set_value(s3_config_.bucket, value);
|
||||
return s3_config_.bucket;
|
||||
} else if (name == "S3Config.CacheTimeoutSeconds") {
|
||||
const auto timeout = std::max(std::uint16_t(5u), utils::string::to_uint16(value));
|
||||
set_value(s3_config_.cache_timeout_secs, timeout);
|
||||
return std::to_string(s3_config_.cache_timeout_secs);
|
||||
} else if (name == "S3Config.Region") {
|
||||
set_value(s3_config_.region, value);
|
||||
return s3_config_.region;
|
||||
} else if (name == "S3Config.SecretKey") {
|
||||
set_value(s3_config_.secret_key, value);
|
||||
return s3_config_.secret_key;
|
||||
} else if (name == "S3Config.URL") {
|
||||
set_value(s3_config_.url, value);
|
||||
return s3_config_.url;
|
||||
} else if (name == "S3Config.TimeoutMs") {
|
||||
set_value(s3_config_.timeout_ms, utils::string::to_uint32(value));
|
||||
return std::to_string(s3_config_.timeout_ms);
|
||||
} else if (name == "S3Config.EncryptionToken") {
|
||||
set_value(s3_config_.encryption_token, value);
|
||||
return s3_config_.encryption_token;
|
||||
} else if (name == "SkynetConfig.EncryptionToken") {
|
||||
set_value(skynet_config_.encryption_token, value);
|
||||
return skynet_config_.encryption_token;
|
||||
} else if (name == "SkynetConfig.PortalList") {
|
||||
set_value(skynet_config_.portal_list, skynet_config::from_string(value));
|
||||
return skynet_config_.to_string();
|
||||
}
|
||||
} catch (const std::exception &e) {
|
||||
event_system::instance().raise<repertory_exception>(__FUNCTION__, e.what());
|
||||
}
|
||||
return "";
|
||||
}
|
||||
} // namespace repertory
|
||||
677
src/comm/aws_s3/aws_s3_comm.cpp
Normal file
677
src/comm/aws_s3/aws_s3_comm.cpp
Normal file
@@ -0,0 +1,677 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#if defined(REPERTORY_ENABLE_S3)
|
||||
#include "comm/aws_s3/aws_s3_comm.hpp"
|
||||
#include "app_config.hpp"
|
||||
#include "events/events.hpp"
|
||||
#include "events/event_system.hpp"
|
||||
#include "providers/i_provider.hpp"
|
||||
#include "types/repertory.hpp"
|
||||
#include "utils/encryption.hpp"
|
||||
#include "utils/path_utils.hpp"
|
||||
#include "utils/polling.hpp"
|
||||
#include "utils/string_utils.hpp"
|
||||
|
||||
namespace repertory {
|
||||
static const i_s3_comm::get_key_callback empty_key = []() { return ""; };
|
||||
|
||||
aws_s3_comm::aws_s3_comm(const app_config &config)
|
||||
: config_(config), s3_config_(config.get_s3_config()) {
|
||||
s3_config_.bucket = utils::string::trim(s3_config_.bucket);
|
||||
Aws::InitAPI(sdk_options_);
|
||||
|
||||
if (config_.get_event_level() >= event_level::debug) {
|
||||
Aws::Utils::Logging::InitializeAWSLogging(
|
||||
Aws::MakeShared<Aws::Utils::Logging::DefaultLogSystem>(
|
||||
"Repertory", Aws::Utils::Logging::LogLevel::Trace,
|
||||
utils::path::combine(config_.get_log_directory(), {"aws_sdk_"})));
|
||||
}
|
||||
|
||||
Aws::Auth::AWSCredentials credentials;
|
||||
credentials.SetAWSAccessKeyId(s3_config_.access_key);
|
||||
credentials.SetAWSSecretKey(s3_config_.secret_key);
|
||||
|
||||
Aws::Client::ClientConfiguration configuration;
|
||||
configuration.endpointOverride = s3_config_.url;
|
||||
configuration.httpRequestTimeoutMs = s3_config_.timeout_ms;
|
||||
configuration.region = s3_config_.region;
|
||||
configuration.requestTimeoutMs = s3_config_.timeout_ms;
|
||||
|
||||
configuration.connectTimeoutMs = s3_config_.timeout_ms;
|
||||
// TODO make configurable
|
||||
const auto enable_path_style = utils::string::begins_with(s3_config_.url, "http://localhost") ||
|
||||
utils::string::begins_with(s3_config_.url, "https://localhost") ||
|
||||
utils::string::begins_with(s3_config_.url, "http://127.0.0.1") ||
|
||||
utils::string::begins_with(s3_config_.url, "https://127.0.0.1");
|
||||
|
||||
s3_client_ = std::make_unique<Aws::S3::S3Client>(
|
||||
credentials, configuration, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
|
||||
not enable_path_style);
|
||||
polling::instance().set_callback(
|
||||
{"s3_directory_cache", false, [this]() { this->clear_expired_directories(); }});
|
||||
}
|
||||
|
||||
aws_s3_comm::~aws_s3_comm() {
|
||||
polling::instance().remove_callback("s3_directory_cache");
|
||||
Aws::ShutdownAPI(sdk_options_);
|
||||
Aws::Utils::Logging::ShutdownAWSLogging();
|
||||
s3_client_.reset();
|
||||
}
|
||||
|
||||
void aws_s3_comm::clear_expired_directories() {
|
||||
recur_mutex_lock l(cached_directories_mutex_);
|
||||
std::vector<std::string> expired_list;
|
||||
for (const auto &kv : cached_directories_) {
|
||||
if (kv.second.expiration <= std::chrono::system_clock::now()) {
|
||||
expired_list.emplace_back(kv.first);
|
||||
}
|
||||
}
|
||||
|
||||
for (const auto &expired : expired_list) {
|
||||
event_system::instance().raise<debug_log>(__FUNCTION__, expired, "expired");
|
||||
cached_directories_.erase(expired);
|
||||
}
|
||||
}
|
||||
|
||||
api_error aws_s3_comm::create_bucket(const std::string &api_path) {
|
||||
std::string bucket_name, object_name;
|
||||
get_bucket_name_and_object_name(api_path, empty_key, bucket_name, object_name);
|
||||
|
||||
if (config_.get_event_level() >= event_level::debug) {
|
||||
event_system::instance().raise<debug_log>(__FUNCTION__, bucket_name, "begin");
|
||||
}
|
||||
|
||||
auto ret = api_error::access_denied;
|
||||
if (s3_config_.bucket.empty()) {
|
||||
Aws::S3::Model::CreateBucketRequest request{};
|
||||
request.SetBucket(bucket_name);
|
||||
const auto outcome = s3_client_->CreateBucket(request);
|
||||
if (outcome.IsSuccess()) {
|
||||
remove_cached_directory(utils::path::get_parent_api_path(api_path));
|
||||
ret = api_error::success;
|
||||
} else {
|
||||
const auto &error = outcome.GetError();
|
||||
event_system::instance().raise<repertory_exception>(
|
||||
__FUNCTION__, error.GetExceptionName() + "|" + error.GetMessage());
|
||||
ret = api_error::comm_error;
|
||||
}
|
||||
}
|
||||
|
||||
if (config_.get_event_level() >= event_level::debug) {
|
||||
event_system::instance().raise<debug_log>(__FUNCTION__, bucket_name,
|
||||
"end|" + std::to_string(static_cast<int>(ret)));
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool aws_s3_comm::exists(const std::string &api_path, const get_key_callback &get_key) const {
|
||||
if (config_.get_event_level() >= event_level::debug) {
|
||||
event_system::instance().raise<debug_log>(__FUNCTION__, api_path, "begin");
|
||||
}
|
||||
if (api_path == "/") {
|
||||
return false;
|
||||
}
|
||||
|
||||
auto ret = false;
|
||||
if (not get_cached_file_exists(api_path, ret)) {
|
||||
std::string bucket_name, object_name;
|
||||
get_bucket_name_and_object_name(api_path, get_key, bucket_name, object_name);
|
||||
|
||||
Aws::S3::Model::HeadObjectRequest request{};
|
||||
request.SetBucket(bucket_name);
|
||||
request.SetKey(object_name);
|
||||
ret = s3_client_->HeadObject(request).IsSuccess();
|
||||
}
|
||||
|
||||
if (config_.get_event_level() >= event_level::debug) {
|
||||
event_system::instance().raise<debug_log>(__FUNCTION__, api_path, "end|" + std::to_string(ret));
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
void aws_s3_comm::get_bucket_name_and_object_name(const std::string &api_path,
|
||||
const get_key_callback &get_key,
|
||||
std::string &bucket_name,
|
||||
std::string &object_name) const {
|
||||
bucket_name = s3_config_.bucket;
|
||||
object_name = api_path.substr(1);
|
||||
|
||||
if (bucket_name.empty()) {
|
||||
bucket_name = utils::string::split(api_path, '/')[1];
|
||||
object_name = object_name.substr(bucket_name.size());
|
||||
if (not object_name.empty() && (object_name[0] == '/')) {
|
||||
object_name = object_name.substr(1);
|
||||
}
|
||||
}
|
||||
|
||||
const auto key = get_key();
|
||||
if (not key.empty()) {
|
||||
auto parts = utils::string::split(object_name, '/', false);
|
||||
parts[parts.size() - 1u] = key;
|
||||
object_name = utils::string::join(parts, '/');
|
||||
}
|
||||
}
|
||||
|
||||
bool aws_s3_comm::get_cached_directory_item_count(const std::string &api_path,
|
||||
std::size_t &count) const {
|
||||
recur_mutex_lock l(cached_directories_mutex_);
|
||||
if (cached_directories_.find(api_path) != cached_directories_.end()) {
|
||||
count = cached_directories_.at(api_path).items.size();
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
bool aws_s3_comm::get_cached_directory_items(const std::string &api_path,
|
||||
const meta_provider_callback &meta_provider,
|
||||
directory_item_list &list) const {
|
||||
unique_recur_mutex_lock l(cached_directories_mutex_);
|
||||
if (cached_directories_.find(api_path) != cached_directories_.end()) {
|
||||
auto &cachedEntry = cached_directories_.at(api_path);
|
||||
list = cachedEntry.items;
|
||||
cached_directories_[api_path].reset_timeout(
|
||||
std::chrono::seconds(config_.get_s3_config().cache_timeout_secs));
|
||||
l.unlock();
|
||||
|
||||
for (auto &item : list) {
|
||||
meta_provider(item, false);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool aws_s3_comm::get_cached_file_exists(const std::string &api_path, bool &exists) const {
|
||||
exists = false;
|
||||
|
||||
unique_recur_mutex_lock l(cached_directories_mutex_);
|
||||
const auto parent_api_path = utils::path::get_parent_api_path(api_path);
|
||||
if (cached_directories_.find(parent_api_path) != cached_directories_.end()) {
|
||||
auto &entry = cached_directories_.at(parent_api_path);
|
||||
exists =
|
||||
std::find_if(entry.items.begin(), entry.items.end(), [&api_path](const auto &item) -> bool {
|
||||
return not item.directory && (api_path == item.api_path);
|
||||
}) != entry.items.end();
|
||||
if (exists) {
|
||||
cached_directories_[api_path].reset_timeout(
|
||||
std::chrono::seconds(config_.get_s3_config().cache_timeout_secs));
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
std::size_t
|
||||
aws_s3_comm::get_directory_item_count(const std::string &api_path,
|
||||
const meta_provider_callback &meta_provider) const {
|
||||
if (config_.get_event_level() >= event_level::debug) {
|
||||
event_system::instance().raise<debug_log>(__FUNCTION__, api_path, "begin");
|
||||
}
|
||||
|
||||
std::size_t ret = 0u;
|
||||
if (not(s3_config_.bucket.empty() && (api_path == "/"))) {
|
||||
if (not get_cached_directory_item_count(api_path, ret)) {
|
||||
directory_item_list list;
|
||||
grab_directory_items(api_path, meta_provider, list);
|
||||
return list.size();
|
||||
}
|
||||
}
|
||||
|
||||
if (config_.get_event_level() >= event_level::debug) {
|
||||
event_system::instance().raise<debug_log>(__FUNCTION__, api_path, "end|" + std::to_string(ret));
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error aws_s3_comm::get_directory_items(const std::string &api_path,
|
||||
const meta_provider_callback &meta_provider,
|
||||
directory_item_list &list) const {
|
||||
if (config_.get_event_level() >= event_level::debug) {
|
||||
event_system::instance().raise<debug_log>(__FUNCTION__, api_path, "begin");
|
||||
}
|
||||
|
||||
auto ret = api_error::success;
|
||||
if (not get_cached_directory_items(api_path, meta_provider, list)) {
|
||||
ret = grab_directory_items(api_path, meta_provider, list);
|
||||
}
|
||||
|
||||
if (config_.get_event_level() >= event_level::debug) {
|
||||
event_system::instance().raise<debug_log>(__FUNCTION__, api_path,
|
||||
"end|" + std::to_string(std::uint8_t(ret)));
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error aws_s3_comm::get_file(const std::string &api_path, const get_key_callback &get_key,
|
||||
const get_name_callback &get_name,
|
||||
const get_token_callback &get_token, api_file &file) const {
|
||||
if (config_.get_event_level() >= event_level::debug) {
|
||||
event_system::instance().raise<debug_log>(__FUNCTION__, api_path, "begin");
|
||||
}
|
||||
auto ret = api_error::success;
|
||||
|
||||
std::string bucket_name, object_name;
|
||||
get_bucket_name_and_object_name(api_path, get_key, bucket_name, object_name);
|
||||
|
||||
Aws::S3::Model::HeadObjectRequest request{};
|
||||
request.SetBucket(bucket_name);
|
||||
request.SetKey(object_name);
|
||||
|
||||
const auto outcome = s3_client_->HeadObject(request);
|
||||
if (outcome.IsSuccess()) {
|
||||
const auto key = get_key();
|
||||
auto object = outcome.GetResult();
|
||||
object_name = get_name(key, object_name);
|
||||
file.accessed_date = utils::get_file_time_now();
|
||||
file.api_path = utils::path::create_api_path(utils::path::combine(bucket_name, {object_name}));
|
||||
file.api_parent = utils::path::get_parent_api_path(file.api_path);
|
||||
file.changed_date = object.GetLastModified().Millis() * 1000u * 1000u;
|
||||
file.created_date = object.GetLastModified().Millis() * 1000u * 1000u;
|
||||
file.encryption_token = get_token();
|
||||
if (file.encryption_token.empty()) {
|
||||
file.file_size = object.GetContentLength();
|
||||
} else {
|
||||
file.file_size =
|
||||
utils::encryption::encrypting_reader::calculate_decrypted_size(object.GetContentLength());
|
||||
}
|
||||
file.modified_date = object.GetLastModified().Millis() * 1000u * 1000u;
|
||||
file.recoverable = true;
|
||||
file.redundancy = 3.0;
|
||||
} else {
|
||||
const auto &error = outcome.GetError();
|
||||
event_system::instance().raise<repertory_exception>(__FUNCTION__, error.GetExceptionName() +
|
||||
"|" + error.GetMessage());
|
||||
ret = api_error::comm_error;
|
||||
}
|
||||
|
||||
if (config_.get_event_level() >= event_level::debug) {
|
||||
event_system::instance().raise<debug_log>(__FUNCTION__, api_path,
|
||||
"end|" + std::to_string(std::uint8_t(ret)));
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error aws_s3_comm::get_file_list(const get_api_file_token_callback &get_api_file_token,
|
||||
const get_name_callback &get_name, api_file_list &list) const {
|
||||
if (config_.get_event_level() >= event_level::debug) {
|
||||
event_system::instance().raise<debug_log>(__FUNCTION__, "/", "begin");
|
||||
}
|
||||
list.clear();
|
||||
auto ret = api_error::success;
|
||||
|
||||
const auto bucket_name = s3_config_.bucket;
|
||||
if (bucket_name.empty()) {
|
||||
const auto outcome = s3_client_->ListBuckets();
|
||||
if (outcome.IsSuccess()) {
|
||||
const auto &bucket_list = outcome.GetResult().GetBuckets();
|
||||
for (const auto &bucket : bucket_list) {
|
||||
get_file_list(bucket.GetName(), get_api_file_token, get_name, list);
|
||||
}
|
||||
} else {
|
||||
const auto &error = outcome.GetError();
|
||||
event_system::instance().raise<repertory_exception>(
|
||||
__FUNCTION__, error.GetExceptionName() + "|" + error.GetMessage());
|
||||
ret = api_error::comm_error;
|
||||
}
|
||||
} else {
|
||||
ret = get_file_list("", get_api_file_token, get_name, list);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error aws_s3_comm::get_file_list(const std::string &bucket_name,
|
||||
const get_api_file_token_callback &get_api_file_token,
|
||||
const get_name_callback &get_name, api_file_list &list) const {
|
||||
auto ret = api_error::success;
|
||||
|
||||
Aws::S3::Model::ListObjectsRequest request{};
|
||||
request.SetBucket(bucket_name.empty() ? s3_config_.bucket : bucket_name);
|
||||
|
||||
const auto outcome = s3_client_->ListObjects(request);
|
||||
if (outcome.IsSuccess()) {
|
||||
const auto &object_list = outcome.GetResult().GetContents();
|
||||
for (auto const &object : object_list) {
|
||||
api_file file{};
|
||||
file.accessed_date = utils::get_file_time_now();
|
||||
|
||||
std::string object_name = object.GetKey();
|
||||
object_name =
|
||||
get_name(*(utils::string::split(object_name, '/', false).end() - 1u), object_name);
|
||||
file.api_path = utils::path::create_api_path(utils::path::combine(
|
||||
bucket_name.empty() ? s3_config_.bucket : bucket_name, {object_name}));
|
||||
file.api_parent = utils::path::get_parent_api_path(file.api_path);
|
||||
file.changed_date = object.GetLastModified().Millis() * 1000u * 1000u;
|
||||
file.created_date = object.GetLastModified().Millis() * 1000u * 1000u;
|
||||
file.encryption_token = get_api_file_token(file.api_path);
|
||||
if (file.encryption_token.empty()) {
|
||||
file.file_size = object.GetSize();
|
||||
} else {
|
||||
file.file_size =
|
||||
object.GetSize() -
|
||||
(utils::divide_with_ceiling(
|
||||
static_cast<std::uint64_t>(object.GetSize()),
|
||||
static_cast<std::uint64_t>(
|
||||
utils::encryption::encrypting_reader::get_encrypted_chunk_size())) *
|
||||
utils::encryption::encrypting_reader::get_header_size());
|
||||
}
|
||||
file.modified_date = object.GetLastModified().Millis() * 1000u * 1000u;
|
||||
file.recoverable = true;
|
||||
file.redundancy = 3.0;
|
||||
list.emplace_back(std::move(file));
|
||||
}
|
||||
} else {
|
||||
const auto &error = outcome.GetError();
|
||||
event_system::instance().raise<repertory_exception>(__FUNCTION__, error.GetExceptionName() +
|
||||
"|" + error.GetMessage());
|
||||
ret = api_error::comm_error;
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error aws_s3_comm::grab_directory_items(const std::string &api_path,
|
||||
const meta_provider_callback &meta_provider,
|
||||
directory_item_list &list) const {
|
||||
auto ret = api_error::success;
|
||||
if (s3_config_.bucket.empty() && (api_path == "/")) {
|
||||
const auto outcome = s3_client_->ListBuckets();
|
||||
if (outcome.IsSuccess()) {
|
||||
const auto &bucket_list = outcome.GetResult().GetBuckets();
|
||||
for (const auto &bucket : bucket_list) {
|
||||
directory_item di{};
|
||||
di.api_path = utils::path::create_api_path(utils::path::combine("/", {bucket.GetName()}));
|
||||
di.api_parent = utils::path::get_parent_api_path(di.api_path);
|
||||
di.directory = true;
|
||||
di.size = get_directory_item_count(di.api_path, meta_provider);
|
||||
meta_provider(di, true);
|
||||
list.emplace_back(std::move(di));
|
||||
}
|
||||
|
||||
set_cached_directory_items(api_path, list);
|
||||
} else {
|
||||
const auto &error = outcome.GetError();
|
||||
event_system::instance().raise<repertory_exception>(
|
||||
__FUNCTION__, error.GetExceptionName() + "|" + error.GetMessage());
|
||||
ret = api_error::comm_error;
|
||||
}
|
||||
} else {
|
||||
std::string bucket_name, object_name;
|
||||
get_bucket_name_and_object_name(api_path, empty_key, bucket_name, object_name);
|
||||
|
||||
Aws::S3::Model::ListObjectsRequest request{};
|
||||
request.SetBucket(bucket_name);
|
||||
request.SetDelimiter("/");
|
||||
request.SetPrefix(object_name.empty() ? object_name : object_name + "/");
|
||||
|
||||
const auto outcome = s3_client_->ListObjects(request);
|
||||
if (outcome.IsSuccess()) {
|
||||
const auto &object_list = outcome.GetResult().GetContents();
|
||||
for (auto const &object : object_list) {
|
||||
directory_item item{};
|
||||
item.api_path =
|
||||
utils::path::create_api_path(utils::path::combine(bucket_name, {object.GetKey()}));
|
||||
item.api_parent = utils::path::get_parent_api_path(item.api_path);
|
||||
item.directory = false;
|
||||
item.size = object.GetSize();
|
||||
meta_provider(item, true);
|
||||
list.emplace_back(std::move(item));
|
||||
}
|
||||
|
||||
set_cached_directory_items(api_path, list);
|
||||
} else {
|
||||
const auto &error = outcome.GetError();
|
||||
event_system::instance().raise<repertory_exception>(
|
||||
__FUNCTION__, error.GetExceptionName() + "|" + error.GetMessage());
|
||||
ret = api_error::comm_error;
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error aws_s3_comm::read_file_bytes(const std::string &api_path, const std::size_t &size,
|
||||
const std::uint64_t &offset, std::vector<char> &data,
|
||||
const get_key_callback &get_key,
|
||||
const get_size_callback &get_size,
|
||||
const get_token_callback &get_token,
|
||||
const bool &stop_requested) const {
|
||||
auto ret = api_error::success;
|
||||
data.clear();
|
||||
|
||||
std::string bucket_name, object_name;
|
||||
get_bucket_name_and_object_name(api_path, get_key, bucket_name, object_name);
|
||||
|
||||
const auto encryption_token = get_token();
|
||||
const auto data_size = get_size();
|
||||
if (encryption_token.empty()) {
|
||||
Aws::S3::Model::GetObjectRequest request{};
|
||||
request.SetBucket(bucket_name);
|
||||
request.SetKey(object_name);
|
||||
request.SetResponseContentType("application/octet-stream");
|
||||
request.SetRange("bytes=" + utils::string::from_uint64(offset) + "-" +
|
||||
utils::string::from_uint64(offset + size - 1u));
|
||||
request.SetContinueRequestHandler(
|
||||
[&stop_requested](const Aws::Http::HttpRequest *) { return not stop_requested; });
|
||||
|
||||
auto outcome = s3_client_->GetObject(request);
|
||||
if (outcome.IsSuccess()) {
|
||||
auto result = outcome.GetResultWithOwnership();
|
||||
const auto len = result.GetContentLength();
|
||||
data.resize(len);
|
||||
result.GetBody().read(&data[0], len);
|
||||
} else {
|
||||
const auto &error = outcome.GetError();
|
||||
event_system::instance().raise<repertory_exception>(
|
||||
__FUNCTION__, error.GetExceptionName() + "|" + error.GetMessage());
|
||||
ret = api_error::comm_error;
|
||||
}
|
||||
} else {
|
||||
const auto key = utils::encryption::generate_key(encryption_token);
|
||||
ret = utils::encryption::read_encrypted_range(
|
||||
{offset, offset + size - 1}, key,
|
||||
[&](std::vector<char> &ct, const std::uint64_t &start_offset,
|
||||
const std::uint64_t &end_offset) -> api_error {
|
||||
return read_file_bytes(
|
||||
api_path, (end_offset - start_offset + 1u), start_offset, ct, get_key, get_size,
|
||||
[]() -> std::string { return ""; }, stop_requested);
|
||||
},
|
||||
data_size, data);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error aws_s3_comm::remove_bucket(const std::string &api_path) {
|
||||
std::string bucket_name, object_name;
|
||||
get_bucket_name_and_object_name(api_path, empty_key, bucket_name, object_name);
|
||||
|
||||
if (config_.get_event_level() >= event_level::debug) {
|
||||
event_system::instance().raise<debug_log>(__FUNCTION__, bucket_name, "begin");
|
||||
}
|
||||
|
||||
auto ret = api_error::access_denied;
|
||||
if (s3_config_.bucket.empty()) {
|
||||
Aws::S3::Model::DeleteBucketRequest request{};
|
||||
request.SetBucket(bucket_name);
|
||||
const auto outcome = s3_client_->DeleteBucket(request);
|
||||
if (outcome.IsSuccess()) {
|
||||
remove_cached_directory(api_path);
|
||||
ret = api_error::success;
|
||||
} else {
|
||||
const auto &error = outcome.GetError();
|
||||
event_system::instance().raise<repertory_exception>(
|
||||
__FUNCTION__, error.GetExceptionName() + "|" + error.GetMessage());
|
||||
ret = api_error::comm_error;
|
||||
}
|
||||
}
|
||||
|
||||
if (config_.get_event_level() >= event_level::debug) {
|
||||
event_system::instance().raise<debug_log>(__FUNCTION__, bucket_name,
|
||||
"end|" + std::to_string(static_cast<int>(ret)));
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
void aws_s3_comm::remove_cached_directory(const std::string &api_path) {
|
||||
recur_mutex_lock l(cached_directories_mutex_);
|
||||
cached_directories_.erase(api_path);
|
||||
}
|
||||
|
||||
api_error aws_s3_comm::remove_file(const std::string &api_path, const get_key_callback &get_key) {
|
||||
if (config_.get_event_level() >= event_level::debug) {
|
||||
event_system::instance().raise<debug_log>(__FUNCTION__, api_path, "begin");
|
||||
}
|
||||
auto ret = api_error::success;
|
||||
|
||||
std::string bucket_name, object_name;
|
||||
get_bucket_name_and_object_name(api_path, get_key, bucket_name, object_name);
|
||||
|
||||
Aws::S3::Model::DeleteObjectRequest request{};
|
||||
request.SetBucket(bucket_name);
|
||||
request.SetKey(object_name);
|
||||
const auto outcome = s3_client_->DeleteObject(request);
|
||||
if (outcome.IsSuccess()) {
|
||||
remove_cached_directory(utils::path::get_parent_api_path(api_path));
|
||||
} else {
|
||||
const auto &error = outcome.GetError();
|
||||
event_system::instance().raise<repertory_exception>(__FUNCTION__, error.GetExceptionName() +
|
||||
"|" + error.GetMessage());
|
||||
ret = api_error::comm_error;
|
||||
}
|
||||
|
||||
if (config_.get_event_level() >= event_level::debug) {
|
||||
event_system::instance().raise<debug_log>(__FUNCTION__, api_path,
|
||||
"end|" + std::to_string(std::uint8_t(ret)));
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error aws_s3_comm::rename_file(const std::string & /*api_path*/,
|
||||
const std::string & /*new_api_path*/) {
|
||||
return api_error::not_implemented;
|
||||
/* if (config_.get_event_level() >= event_level::debug) { */
|
||||
/* event_system::instance().raise<debug_log>(__FUNCTION__, api_path, "begin"); */
|
||||
/* } */
|
||||
/* auto ret = api_error::success; */
|
||||
/* */
|
||||
/* std::string bucket_name, object_name; */
|
||||
/* get_bucket_name_and_object_name(api_path, bucket_name, object_name); */
|
||||
/* */
|
||||
/* std::string new_object_name; */
|
||||
/* get_bucket_name_and_object_name(new_api_path, bucket_name, new_object_name); */
|
||||
/* */
|
||||
/* Aws::S3::Model::CopyObjectRequest request{}; */
|
||||
/* request.SetBucket(bucket_name); */
|
||||
/* request.SetCopySource(bucket_name + '/' + object_name); */
|
||||
/* request.SetKey(new_object_name); */
|
||||
/* */
|
||||
/* const auto outcome = s3_client_->CopyObject(request); */
|
||||
/* if (outcome.IsSuccess()) { */
|
||||
/* ret = remove_file(api_path); */
|
||||
/* } else { */
|
||||
/* const auto &error = outcome.GetError(); */
|
||||
/* event_system::instance().raise<repertory_exception>(__FUNCTION__, error.GetExceptionName()
|
||||
* +
|
||||
* "|" + */
|
||||
/* error.GetMessage()); */
|
||||
/* ret = api_error::comm_error; */
|
||||
/* } */
|
||||
/* */
|
||||
/* if (config_.get_event_level() >= event_level::debug) { */
|
||||
/* event_system::instance().raise<debug_log>(__FUNCTION__, api_path, */
|
||||
/* "end|" + std::to_string(std::uint8_t(ret))); */
|
||||
/* } */
|
||||
/* return ret; */
|
||||
}
|
||||
|
||||
void aws_s3_comm::set_cached_directory_items(const std::string &api_path,
|
||||
directory_item_list list) const {
|
||||
recur_mutex_lock l(cached_directories_mutex_);
|
||||
cached_directories_[api_path].items = std::move(list);
|
||||
cached_directories_[api_path].reset_timeout(
|
||||
std::chrono::seconds(config_.get_s3_config().cache_timeout_secs));
|
||||
}
|
||||
|
||||
api_error aws_s3_comm::upload_file(const std::string &api_path, const std::string &source_path,
|
||||
const std::string &encryption_token,
|
||||
const get_key_callback &get_key, const set_key_callback &set_key,
|
||||
const bool &stop_requested) {
|
||||
static const auto no_stop = false;
|
||||
if (config_.get_event_level() >= event_level::debug) {
|
||||
event_system::instance().raise<debug_log>(__FUNCTION__, api_path, "begin");
|
||||
}
|
||||
auto ret = api_error::success;
|
||||
|
||||
std::string bucket_name, object_name;
|
||||
get_bucket_name_and_object_name(api_path, get_key, bucket_name, object_name);
|
||||
|
||||
std::shared_ptr<Aws::IOStream> file_stream;
|
||||
if (encryption_token.empty()) {
|
||||
file_stream = Aws::MakeShared<Aws::FStream>(&source_path[0], &source_path[0],
|
||||
std::ios_base::in | std::ios_base::binary);
|
||||
} else {
|
||||
const auto file_name = ([&api_path]() -> std::string {
|
||||
return *(utils::string::split(api_path, '/', false).end() - 1u);
|
||||
})();
|
||||
|
||||
const auto reader =
|
||||
utils::encryption::encrypting_reader(file_name, source_path, no_stop, encryption_token, -1);
|
||||
auto key = get_key();
|
||||
if (key.empty()) {
|
||||
key = reader.get_encrypted_file_name();
|
||||
set_key(key);
|
||||
|
||||
auto parts = utils::string::split(object_name, '/', false);
|
||||
parts[parts.size() - 1u] = key;
|
||||
object_name = utils::string::join(parts, '/');
|
||||
}
|
||||
|
||||
file_stream = reader.create_iostream();
|
||||
}
|
||||
file_stream->seekg(0);
|
||||
|
||||
Aws::S3::Model::PutObjectRequest request{};
|
||||
request.SetBucket(bucket_name);
|
||||
request.SetKey(object_name);
|
||||
request.SetBody(file_stream);
|
||||
request.SetContinueRequestHandler(
|
||||
[&stop_requested](const Aws::Http::HttpRequest *) { return not stop_requested; });
|
||||
|
||||
const auto outcome = s3_client_->PutObject(request);
|
||||
if (outcome.IsSuccess()) {
|
||||
remove_cached_directory(utils::path::get_parent_api_path(api_path));
|
||||
} else {
|
||||
const auto &error = outcome.GetError();
|
||||
event_system::instance().raise<repertory_exception>(__FUNCTION__, error.GetExceptionName() +
|
||||
"|" + error.GetMessage());
|
||||
ret = api_error::upload_failed;
|
||||
}
|
||||
|
||||
if (config_.get_event_level() >= event_level::debug) {
|
||||
event_system::instance().raise<debug_log>(__FUNCTION__, api_path,
|
||||
"end|" + std::to_string(std::uint8_t(ret)));
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
} // namespace repertory
|
||||
|
||||
#endif // REPERTORY_ENABLE_S3
|
||||
849
src/comm/curl/curl_comm.cpp
Normal file
849
src/comm/curl/curl_comm.cpp
Normal file
@@ -0,0 +1,849 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "comm/curl/curl_comm.hpp"
|
||||
#include "comm/curl/curl_resolver.hpp"
|
||||
#include "curl/curl.h"
|
||||
#include "types/repertory.hpp"
|
||||
#include "utils/Base64.hpp"
|
||||
#include "utils/encryption.hpp"
|
||||
#include "utils/encrypting_reader.hpp"
|
||||
#include "utils/file_utils.hpp"
|
||||
#include "utils/path_utils.hpp"
|
||||
|
||||
namespace repertory {
|
||||
struct curl_setup {
|
||||
bool allow_timeout = false;
|
||||
const host_config &hc;
|
||||
const http_parameters *parameters = nullptr;
|
||||
http_headers *headers = nullptr;
|
||||
bool post = false;
|
||||
raw_write_data *write_data = nullptr;
|
||||
std::string *write_string = nullptr;
|
||||
std::unique_ptr<curl_resolver> resolver;
|
||||
};
|
||||
|
||||
struct raw_write_data {
|
||||
std::vector<char> *buffer;
|
||||
const bool &stop_requested;
|
||||
};
|
||||
|
||||
struct read_data {
|
||||
const bool *stop_requested = nullptr;
|
||||
native_file *nf = nullptr;
|
||||
std::uint64_t offset = 0u;
|
||||
};
|
||||
|
||||
CURL *curl_comm::common_curl_setup(const std::string &path, curl_setup &setup, std::string &url,
|
||||
std::string &fields) {
|
||||
return common_curl_setup(utils::create_curl(), path, setup, url, fields);
|
||||
}
|
||||
|
||||
CURL *curl_comm::common_curl_setup(CURL *curl_handle, const std::string &path, curl_setup &setup,
|
||||
std::string &url, std::string &fields) {
|
||||
url = construct_url(curl_handle, path, setup.hc);
|
||||
if (setup.post) {
|
||||
curl_easy_setopt(curl_handle, CURLOPT_HTTPGET, 0);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_POST, 1L);
|
||||
if (setup.parameters && not setup.parameters->empty()) {
|
||||
for (const auto ¶m : *setup.parameters) {
|
||||
if (not fields.empty()) {
|
||||
fields += "&";
|
||||
}
|
||||
if (param.first ==
|
||||
"new" + app_config::get_provider_path_name(config_.get_provider_type())) {
|
||||
fields += (param.first + "=" + url_encode(curl_handle, param.second));
|
||||
} else {
|
||||
fields += (param.first + "=" + param.second);
|
||||
}
|
||||
}
|
||||
curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, &fields[0]);
|
||||
} else {
|
||||
curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, "");
|
||||
}
|
||||
} else {
|
||||
curl_easy_setopt(curl_handle, CURLOPT_POST, 0);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_HTTPGET, 1L);
|
||||
if (setup.parameters && not setup.parameters->empty()) {
|
||||
url += "?";
|
||||
for (const auto ¶m : *setup.parameters) {
|
||||
if (url[url.size() - 1] != '?') {
|
||||
url += "&";
|
||||
}
|
||||
url += (param.first + "=" + url_encode(curl_handle, param.second));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (not setup.hc.agent_string.empty()) {
|
||||
curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, &setup.hc.agent_string[0]);
|
||||
}
|
||||
|
||||
if (setup.write_data) {
|
||||
setup.write_data->buffer->clear();
|
||||
curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, setup.write_data);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, write_data_callback_);
|
||||
} else if (setup.write_string) {
|
||||
setup.write_string->clear();
|
||||
curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, setup.write_string);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, write_string_callback_);
|
||||
}
|
||||
|
||||
curl_easy_setopt(curl_handle, CURLOPT_URL, url.c_str());
|
||||
|
||||
if (setup.allow_timeout) {
|
||||
curl_easy_setopt(curl_handle, CURLOPT_TIMEOUT_MS, setup.hc.timeout_ms);
|
||||
}
|
||||
|
||||
if (not setup.hc.api_password.empty()) {
|
||||
curl_easy_setopt(curl_handle, CURLOPT_USERNAME, "");
|
||||
curl_easy_setopt(curl_handle, CURLOPT_PASSWORD, &setup.hc.api_password[0]);
|
||||
}
|
||||
|
||||
if (setup.headers) {
|
||||
curl_easy_setopt(curl_handle, CURLOPT_HEADERDATA, setup.headers);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, write_header_callback_);
|
||||
}
|
||||
|
||||
std::vector<std::string> items = {"localhost:" + std::to_string(setup.hc.api_port) +
|
||||
":127.0.0.1"};
|
||||
setup.resolver = std::make_unique<curl_resolver>(curl_handle, items);
|
||||
|
||||
return curl_handle;
|
||||
}
|
||||
|
||||
std::string curl_comm::construct_url(CURL *curl_handle, const std::string &relative_path,
|
||||
const host_config &hc) {
|
||||
auto custom_port = (((hc.protocol == "http") && (hc.api_port == 80u)) ||
|
||||
((hc.protocol == "https") && (hc.api_port == 443u)))
|
||||
? ""
|
||||
: ":" + std::to_string(hc.api_port);
|
||||
auto ret = hc.protocol + "://" + utils::string::trim_copy(hc.host_name_or_ip) + custom_port;
|
||||
auto path = utils::path::combine("/", {hc.path});
|
||||
if (relative_path.empty()) {
|
||||
ret += utils::path::create_api_path(path);
|
||||
if (utils::string::ends_with(hc.path, "/")) {
|
||||
ret += '/';
|
||||
}
|
||||
} else {
|
||||
path = utils::path::combine(path, {url_encode(curl_handle, relative_path, true)});
|
||||
ret += utils::path::create_api_path(path);
|
||||
if (utils::string::ends_with(relative_path, "/")) {
|
||||
ret += '/';
|
||||
}
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool curl_comm::create_auth_session(CURL *&curl_handle, const app_config &config, host_config hc,
|
||||
std::string &session) {
|
||||
auto ret = true;
|
||||
if (not curl_handle) {
|
||||
curl_handle = utils::create_curl();
|
||||
}
|
||||
|
||||
if (not hc.auth_url.empty() && not hc.auth_user.empty()) {
|
||||
event_system::instance().raise<comm_auth_begin>(hc.auth_url, hc.auth_user);
|
||||
ret = false;
|
||||
session = utils::create_uuid_string();
|
||||
const auto cookie_path = utils::path::combine(config.get_data_directory(), {session + ".txt"});
|
||||
http_headers headers{};
|
||||
auto url = utils::string::right_trim(utils::string::trim(hc.auth_url), '/') + "/api/login";
|
||||
|
||||
if (not hc.agent_string.empty()) {
|
||||
curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, &hc.agent_string[0]);
|
||||
}
|
||||
|
||||
if (not hc.api_password.empty()) {
|
||||
curl_easy_setopt(curl_handle, CURLOPT_USERNAME, "");
|
||||
curl_easy_setopt(curl_handle, CURLOPT_PASSWORD, &hc.api_password[0]);
|
||||
}
|
||||
|
||||
struct curl_slist *hs = nullptr;
|
||||
hs = curl_slist_append(hs, "Content-Type: application/json");
|
||||
curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, hs);
|
||||
|
||||
const auto payload = json({
|
||||
{"email", hc.auth_user},
|
||||
{"password", hc.auth_password},
|
||||
})
|
||||
.dump(2);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_COOKIEFILE, cookie_path.c_str());
|
||||
curl_easy_setopt(curl_handle, CURLOPT_COOKIEJAR, cookie_path.c_str());
|
||||
curl_easy_setopt(curl_handle, CURLOPT_HEADERDATA, &headers);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, write_header_callback_);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_HTTPGET, 0L);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_POST, 1L);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_TIMEOUT_MS, hc.timeout_ms);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_URL, url.c_str());
|
||||
curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, payload.c_str());
|
||||
|
||||
long code{};
|
||||
auto res = curl_easy_perform(curl_handle);
|
||||
curl_easy_getinfo(curl_handle, CURLINFO_HTTP_CODE, &code);
|
||||
if ((res == CURLE_OK) && ((code >= 200) && (code < 300))) {
|
||||
auto cookie = headers["set-cookie"];
|
||||
if (cookie.empty()) {
|
||||
code = 400;
|
||||
} else {
|
||||
cookie = headers["set-cookie"];
|
||||
if (cookie.empty() || not utils::string::contains(cookie, "skynet-jwt=")) {
|
||||
code = 401;
|
||||
} else {
|
||||
ret = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (ret) {
|
||||
curl_easy_setopt(curl_handle, CURLOPT_COOKIELIST, "FLUSH");
|
||||
update_auth_session(utils::reset_curl(curl_handle), config, session);
|
||||
} else {
|
||||
curl_easy_cleanup(curl_handle);
|
||||
release_auth_session(config, hc, session);
|
||||
}
|
||||
curl_slist_free_all(hs);
|
||||
|
||||
event_system::instance().raise<comm_auth_end>(hc.auth_url, hc.auth_user, res, code);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error curl_comm::get_or_post(const host_config &hc, const bool &post, const std::string &path,
|
||||
const http_parameters ¶meters, json &data, json &error,
|
||||
http_headers *headers, std::function<void(CURL *curl_handle)> cb) {
|
||||
std::string result;
|
||||
auto setup = curl_setup{not post, hc, ¶meters, headers, post, nullptr, &result, nullptr};
|
||||
|
||||
std::string fields;
|
||||
std::string url;
|
||||
auto *curl_handle = common_curl_setup(path, setup, url, fields);
|
||||
if (cb) {
|
||||
cb(curl_handle);
|
||||
}
|
||||
|
||||
if (config_.get_event_level() >= event_level::verbose) {
|
||||
if (post) {
|
||||
event_system::instance().raise<comm_post_begin>(url, fields);
|
||||
} else {
|
||||
event_system::instance().raise<comm_get_begin>(url);
|
||||
}
|
||||
}
|
||||
|
||||
std::unique_ptr<std::chrono::system_clock::time_point> tp(nullptr);
|
||||
if (config_.get_enable_comm_duration_events()) {
|
||||
tp = std::make_unique<std::chrono::system_clock::time_point>(std::chrono::system_clock::now());
|
||||
}
|
||||
|
||||
const auto curl_code = curl_easy_perform(curl_handle);
|
||||
if (curl_code == CURLE_OPERATION_TIMEDOUT) {
|
||||
event_system::instance().raise<repertory_exception>(__FUNCTION__, "CURL timeout: " + path);
|
||||
}
|
||||
|
||||
long http_code = -1;
|
||||
curl_easy_getinfo(curl_handle, CURLINFO_RESPONSE_CODE, &http_code);
|
||||
if (tp != nullptr) {
|
||||
const auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(
|
||||
std::chrono::system_clock::now() - *tp);
|
||||
event_system::instance().raise<comm_duration>(url, std::to_string(duration.count()));
|
||||
}
|
||||
|
||||
const auto ret = process_json_response(url, curl_code, http_code, result, data, error);
|
||||
if (config_.get_event_level() >= event_level::verbose) {
|
||||
if (post) {
|
||||
event_system::instance().raise<comm_post_end>(
|
||||
url, curl_code, http_code, ((ret == api_error::success) ? data.dump(2) : error.dump(2)));
|
||||
} else {
|
||||
event_system::instance().raise<comm_get_end>(
|
||||
url, curl_code, http_code, ((ret == api_error::success) ? data.dump(2) : error.dump(2)));
|
||||
}
|
||||
}
|
||||
|
||||
curl_easy_cleanup(curl_handle);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error curl_comm::get_range(const host_config &hc, const std::string &path,
|
||||
const std::uint64_t &data_size, const http_parameters ¶meters,
|
||||
const std::string &encryption_token, std::vector<char> &data,
|
||||
const http_ranges &ranges, json &error, http_headers *headers,
|
||||
const bool &stop_requested) {
|
||||
if (encryption_token.empty()) {
|
||||
return get_range_unencrypted(hc, path, parameters, data, ranges, error, headers,
|
||||
stop_requested);
|
||||
}
|
||||
|
||||
if (ranges.empty()) {
|
||||
return api_error::error;
|
||||
}
|
||||
|
||||
const auto key = utils::encryption::generate_key(encryption_token);
|
||||
for (const auto &range : ranges) {
|
||||
const auto result = utils::encryption::read_encrypted_range(
|
||||
range, key,
|
||||
[&](std::vector<char> &ct, const std::uint64_t &start_offset,
|
||||
const std::uint64_t &end_offset) -> api_error {
|
||||
const auto ret =
|
||||
get_range_unencrypted(hc, path, parameters, ct, {{start_offset, end_offset}}, error,
|
||||
headers, stop_requested);
|
||||
headers = nullptr;
|
||||
return ret;
|
||||
},
|
||||
data_size, data);
|
||||
if (result != api_error::success) {
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
api_error curl_comm::get_range_unencrypted(const host_config &hc, const std::string &path,
|
||||
const http_parameters ¶meters,
|
||||
std::vector<char> &data, const http_ranges &ranges,
|
||||
json &error, http_headers *headers,
|
||||
const bool &stop_requested) {
|
||||
raw_write_data wd = {&data, stop_requested};
|
||||
|
||||
auto setup = curl_setup{false, hc, ¶meters, headers, false, &wd, nullptr, nullptr};
|
||||
|
||||
std::string fields;
|
||||
std::string url;
|
||||
auto *curl_handle = common_curl_setup(path, setup, url, fields);
|
||||
|
||||
std::string range_list;
|
||||
if (not ranges.empty()) {
|
||||
range_list = std::accumulate(
|
||||
std::next(ranges.begin()), ranges.end(), http_range_to_string(ranges[0]),
|
||||
[](const auto &l, const auto &r) { return l + ',' + http_range_to_string(r); });
|
||||
curl_easy_setopt(curl_handle, CURLOPT_RANGE, &range_list[0]);
|
||||
}
|
||||
|
||||
return execute_binary_operation<comm_get_range_begin, comm_get_range_end>(curl_handle, url, data,
|
||||
error, stop_requested);
|
||||
}
|
||||
|
||||
api_error curl_comm::get_raw(const host_config &hc, const std::string &path,
|
||||
const http_parameters ¶meters, std::vector<char> &data,
|
||||
json &error, const bool &stop_requested) {
|
||||
raw_write_data wd = {&data, stop_requested};
|
||||
|
||||
auto setup = curl_setup{false, hc, ¶meters, nullptr, false, &wd, nullptr, nullptr};
|
||||
|
||||
std::string fields;
|
||||
std::string url;
|
||||
return execute_binary_operation<comm_get_range_begin, comm_get_range_end>(
|
||||
common_curl_setup(path, setup, url, fields), url, data, error, stop_requested);
|
||||
}
|
||||
|
||||
std::string curl_comm::http_range_to_string(const http_range &range) {
|
||||
return std::to_string(range.begin) + '-' + std::to_string(range.end);
|
||||
}
|
||||
|
||||
api_error curl_comm::post_file(const host_config &hc, const std::string &path,
|
||||
const std::string &source_path, const http_parameters ¶meters,
|
||||
json &data, json &error, const bool &stop_requested) {
|
||||
auto ret = api_error::os_error;
|
||||
std::uint64_t file_size{};
|
||||
if (utils::file::get_file_size(source_path, file_size)) {
|
||||
std::string result;
|
||||
auto setup = curl_setup{false, hc, ¶meters, nullptr, true, nullptr, &result, nullptr};
|
||||
|
||||
std::string fields;
|
||||
std::string url;
|
||||
auto *curl_handle = common_curl_setup(path, setup, url, fields);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_UPLOAD, 1L);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_INFILESIZE_LARGE, file_size);
|
||||
|
||||
native_file::native_file_ptr nf;
|
||||
native_file::create_or_open(source_path, nf);
|
||||
if (nf) {
|
||||
read_data rd = {&stop_requested};
|
||||
rd.nf = nf.get();
|
||||
curl_easy_setopt(curl_handle, CURLOPT_READDATA, &rd);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_READFUNCTION, read_data_callback_);
|
||||
|
||||
ret = execute_json_operation<comm_post_file_begin, comm_post_file_end>(
|
||||
curl_handle, url, result, data, error, stop_requested);
|
||||
nf->close();
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error curl_comm::post_multipart_file(const host_config &hc, const std::string &path,
|
||||
const std::string &file_name,
|
||||
const std::string &source_path,
|
||||
const std::string &encryption_token, json &data,
|
||||
json &error, const bool &stop_requested) {
|
||||
std::string result;
|
||||
std::uint64_t file_size{};
|
||||
std::unique_ptr<utils::encryption::encrypting_reader> reader;
|
||||
if (encryption_token.empty()) {
|
||||
if (not utils::file::get_file_size(source_path, file_size)) {
|
||||
return api_error::os_error;
|
||||
}
|
||||
} else {
|
||||
try {
|
||||
reader = std::make_unique<utils::encryption::encrypting_reader>(
|
||||
file_name, source_path, stop_requested, encryption_token);
|
||||
file_size = reader->get_total_size();
|
||||
} catch (const std::exception &e) {
|
||||
event_system::instance().raise<repertory_exception>(__FUNCTION__, e.what());
|
||||
return api_error::error;
|
||||
}
|
||||
}
|
||||
|
||||
std::string session;
|
||||
CURL *curl_handle = nullptr;
|
||||
if (not session_manager_.create_auth_session(curl_handle, config_, hc, session)) {
|
||||
return api_error::access_denied;
|
||||
}
|
||||
|
||||
if (config_.get_provider_type() == provider_type::skynet) {
|
||||
curl_easy_cleanup(curl_handle);
|
||||
|
||||
const auto fn = reader ? reader->get_encrypted_file_name() : file_name;
|
||||
const auto fs = reader ? reader->get_total_size() : file_size;
|
||||
|
||||
std::string location;
|
||||
if (tus_upload_create(hc, fn, fs, location)) {
|
||||
std::string skylink;
|
||||
if (tus_upload(hc, source_path, fn, fs, location, skylink, stop_requested, reader.get())) {
|
||||
data["skylink"] = skylink;
|
||||
return api_error::success;
|
||||
}
|
||||
}
|
||||
|
||||
return api_error::comm_error;
|
||||
}
|
||||
|
||||
auto setup = curl_setup{false, hc, nullptr, nullptr, true, nullptr, &result, nullptr};
|
||||
|
||||
std::string fields;
|
||||
std::string url;
|
||||
common_curl_setup(curl_handle, path, setup, url, fields);
|
||||
|
||||
auto *curl_mime = curl_mime_init(curl_handle);
|
||||
auto *mime_part = curl_mime_addpart(curl_mime);
|
||||
curl_mime_name(mime_part, "file");
|
||||
|
||||
auto ret = api_error::success;
|
||||
if (encryption_token.empty()) {
|
||||
curl_mime_filename(mime_part, &file_name[0]);
|
||||
curl_mime_filedata(mime_part, &source_path[0]);
|
||||
} else {
|
||||
try {
|
||||
curl_mime_filename(mime_part, reader->get_encrypted_file_name().c_str());
|
||||
curl_mime_data_cb(
|
||||
mime_part, reader->get_total_size(),
|
||||
static_cast<curl_read_callback>(utils::encryption::encrypting_reader::reader_function),
|
||||
nullptr, nullptr, reader.get());
|
||||
} catch (const std::exception &e) {
|
||||
event_system::instance().raise<repertory_exception>(__FUNCTION__, e.what());
|
||||
ret = api_error::error;
|
||||
}
|
||||
}
|
||||
|
||||
if (ret == api_error::success) {
|
||||
curl_easy_setopt(curl_handle, CURLOPT_MIMEPOST, curl_mime);
|
||||
|
||||
ret = execute_json_operation<comm_post_multi_part_file_begin, comm_post_multi_part_file_end>(
|
||||
curl_handle, url, result, data, error, stop_requested,
|
||||
(ret == api_error::success) ? CURLE_OK : CURLE_SEND_ERROR);
|
||||
}
|
||||
|
||||
curl_mime_free(curl_mime);
|
||||
curl_easy_cleanup(curl_handle);
|
||||
|
||||
/* if (not session.empty() && (ret == api_error::success)) { */
|
||||
/* auto pin_hc = hc; */
|
||||
/* */
|
||||
/* const auto skylink = data["skylink"].get<std::string>(); */
|
||||
/* utils::string::replace(pin_hc.path, "/skyfile", "/pin/" + skylink); */
|
||||
/* */
|
||||
/* ret = api_error::comm_error; */
|
||||
/* for (std::uint8_t i = 0u; not stop_requested && (i < 30u) && (ret != api_error::success);
|
||||
* i++) { */
|
||||
/* if (i) { */
|
||||
/* event_system::instance().raise<repertory_exception>( */
|
||||
/* __FUNCTION__, "RETRY [" + std::to_string(i) + "] Pin failed for file: " +
|
||||
* file_name); */
|
||||
/* std::this_thread::sleep_for(1s); */
|
||||
/* } */
|
||||
/* */
|
||||
/* json response; */
|
||||
/* http_headers headers{}; */
|
||||
/* ret = get_or_post(pin_hc, true, "", {}, response, error, &headers, [&](CURL *curl_handle)
|
||||
* { */
|
||||
/* session_manager_.update_auth_session(curl_handle, config_, hc); */
|
||||
/* }); */
|
||||
/* } */
|
||||
/* } */
|
||||
|
||||
session_manager_.release_auth_session(config_, hc, session);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error curl_comm::process_binary_response(const std::string &url, const CURLcode &res,
|
||||
const long &http_code, std::vector<char> data,
|
||||
json &error) {
|
||||
const auto ret = process_response(
|
||||
url, res, http_code, data.size(),
|
||||
[&]() -> std::string { return (data.empty() ? "" : std::string(&data[0], data.size())); },
|
||||
nullptr, error);
|
||||
|
||||
if (ret != api_error::success) {
|
||||
data.clear();
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error curl_comm::process_json_response(const std::string &url, const CURLcode &res,
|
||||
const long &http_code, const std::string &result,
|
||||
json &data, json &error) {
|
||||
const auto ret = process_response(
|
||||
url, res, http_code, result.size(), [&]() -> std::string { return result; },
|
||||
[&]() {
|
||||
if (result.length()) {
|
||||
data = json::parse(result.c_str());
|
||||
}
|
||||
},
|
||||
error);
|
||||
|
||||
if (ret != api_error::success) {
|
||||
data.clear();
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error curl_comm::process_response(const std::string &url, const CURLcode &res,
|
||||
const long &http_code, const std::size_t &data_size,
|
||||
const std::function<std::string()> &to_string_convertor,
|
||||
const std::function<void()> &success_handler,
|
||||
json &error) const {
|
||||
auto ret = api_error::success;
|
||||
|
||||
auto construct_error = [&]() {
|
||||
ret = api_error::comm_error;
|
||||
const auto *curl_string = curl_easy_strerror(res);
|
||||
std::string error_string(curl_string ? curl_string : "");
|
||||
error["message"] = error_string + ":" + std::to_string(http_code);
|
||||
error["url"] = url;
|
||||
};
|
||||
|
||||
if ((res == CURLE_OK) && ((http_code >= 200) && (http_code < 300))) {
|
||||
if (success_handler) {
|
||||
success_handler();
|
||||
}
|
||||
} else if (data_size == 0u) {
|
||||
construct_error();
|
||||
} else {
|
||||
try {
|
||||
const auto tmp = json::parse(to_string_convertor().c_str());
|
||||
if (tmp.find("message") != tmp.end()) {
|
||||
ret = api_error::comm_error;
|
||||
error = tmp;
|
||||
error["url"] = url;
|
||||
} else {
|
||||
construct_error();
|
||||
}
|
||||
} catch (...) {
|
||||
construct_error();
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
void curl_comm::release_auth_session(const app_config &config, host_config hc,
|
||||
const std::string &session) {
|
||||
if (not hc.auth_url.empty() && not hc.auth_user.empty()) {
|
||||
event_system::instance().raise<comm_auth_logout_begin>(hc.auth_url, hc.auth_user);
|
||||
const auto cookie_path = utils::path::combine(config.get_data_directory(), {session + ".txt"});
|
||||
const auto url =
|
||||
utils::string::right_trim(utils::string::trim(hc.auth_url), '/') + "/api/logout";
|
||||
|
||||
auto *curl_handle = utils::create_curl();
|
||||
if (not hc.agent_string.empty()) {
|
||||
curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, &hc.agent_string[0]);
|
||||
}
|
||||
|
||||
if (not hc.api_password.empty()) {
|
||||
curl_easy_setopt(curl_handle, CURLOPT_USERNAME, "");
|
||||
curl_easy_setopt(curl_handle, CURLOPT_PASSWORD, &hc.api_password[0]);
|
||||
}
|
||||
|
||||
curl_easy_setopt(curl_handle, CURLOPT_COOKIEFILE, cookie_path.c_str());
|
||||
curl_easy_setopt(curl_handle, CURLOPT_COOKIEJAR, cookie_path.c_str());
|
||||
curl_easy_setopt(curl_handle, CURLOPT_HTTPGET, 0L);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_POST, 1L);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, "");
|
||||
curl_easy_setopt(curl_handle, CURLOPT_URL, url.c_str());
|
||||
curl_easy_setopt(curl_handle, CURLOPT_TIMEOUT_MS, hc.timeout_ms);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, write_null_callback_);
|
||||
|
||||
const auto curl_code = curl_easy_perform(curl_handle);
|
||||
|
||||
long http_code = -1;
|
||||
curl_easy_getinfo(curl_handle, CURLINFO_HTTP_CODE, &http_code);
|
||||
curl_easy_cleanup(curl_handle);
|
||||
|
||||
event_system::instance().raise<comm_auth_logout_end>(hc.auth_url, hc.auth_user, curl_code,
|
||||
http_code);
|
||||
|
||||
utils::file::delete_file(cookie_path);
|
||||
}
|
||||
}
|
||||
|
||||
bool curl_comm::tus_upload(host_config hc, const std::string &source_path,
|
||||
const std::string &file_name, std::uint64_t file_size,
|
||||
const std::string &location, std::string &skylink,
|
||||
const bool &stop_requested,
|
||||
utils::encryption::encrypting_reader *reader) {
|
||||
static const constexpr std::uint64_t max_skynet_file_size = (1ull << 22ull) * 10ull;
|
||||
|
||||
auto tus_hc = hc;
|
||||
utils::string::replace(tus_hc.path, "/skyfile", "/tus");
|
||||
|
||||
auto ret = true;
|
||||
std::uint64_t offset = 0u;
|
||||
native_file::native_file_ptr nf;
|
||||
while (ret && file_size) {
|
||||
const auto chunk_size = std::min(max_skynet_file_size, file_size);
|
||||
|
||||
auto *curl_handle = utils::create_curl();
|
||||
curl_easy_setopt(curl_handle, CURLOPT_CUSTOMREQUEST, "PATCH");
|
||||
curl_easy_setopt(curl_handle, CURLOPT_HTTPGET, 0);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_NOBODY, 0);
|
||||
const auto upload_offset = "Upload-Offset: " + std::to_string(offset);
|
||||
const auto content_length = "Content-Length: " + std::to_string(chunk_size);
|
||||
|
||||
struct curl_slist *hs = nullptr;
|
||||
hs = curl_slist_append(hs, "tus-resumable: 1.0.0");
|
||||
hs = curl_slist_append(hs, "Content-Type: application/offset+octet-stream");
|
||||
hs = curl_slist_append(hs, content_length.c_str());
|
||||
hs = curl_slist_append(hs, upload_offset.c_str());
|
||||
curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, hs);
|
||||
|
||||
curl_easy_setopt(curl_handle, CURLOPT_INFILESIZE_LARGE, chunk_size);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_UPLOAD, 1L);
|
||||
|
||||
session_manager_.update_auth_session(curl_handle, config_, hc);
|
||||
|
||||
read_data rd = {&stop_requested};
|
||||
if (reader) {
|
||||
curl_easy_setopt(curl_handle, CURLOPT_READDATA, reader);
|
||||
curl_easy_setopt(
|
||||
curl_handle, CURLOPT_READFUNCTION,
|
||||
static_cast<curl_read_callback>(utils::encryption::encrypting_reader::reader_function));
|
||||
} else {
|
||||
if (not nf) {
|
||||
native_file::create_or_open(source_path, nf);
|
||||
}
|
||||
|
||||
if (nf) {
|
||||
rd.nf = nf.get();
|
||||
rd.offset = offset;
|
||||
curl_easy_setopt(curl_handle, CURLOPT_READDATA, &rd);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_READFUNCTION, read_data_callback_);
|
||||
} else {
|
||||
ret = false;
|
||||
}
|
||||
}
|
||||
|
||||
if (ret) {
|
||||
event_system::instance().raise<comm_tus_upload_begin>(file_name, location, file_size, offset);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_URL, location.c_str());
|
||||
|
||||
const auto curl_code = curl_easy_perform(curl_handle);
|
||||
|
||||
long http_code = -1;
|
||||
curl_easy_getinfo(curl_handle, CURLINFO_HTTP_CODE, &http_code);
|
||||
|
||||
event_system::instance().raise<comm_tus_upload_end>(file_name, location, file_size, offset,
|
||||
curl_code, http_code);
|
||||
if ((ret = ((curl_code == CURLE_OK) && (http_code >= 200) && (http_code < 300)))) {
|
||||
file_size -= chunk_size;
|
||||
offset += chunk_size;
|
||||
}
|
||||
}
|
||||
|
||||
curl_easy_cleanup(curl_handle);
|
||||
curl_slist_free_all(hs);
|
||||
}
|
||||
|
||||
if (nf) {
|
||||
nf->close();
|
||||
}
|
||||
|
||||
if (ret) {
|
||||
auto *curl_handle = utils::create_curl();
|
||||
|
||||
http_headers headers{};
|
||||
curl_easy_setopt(curl_handle, CURLOPT_CUSTOMREQUEST, "HEAD");
|
||||
curl_easy_setopt(curl_handle, CURLOPT_NOBODY, 1L);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_HEADERDATA, &headers);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, write_header_callback_);
|
||||
|
||||
session_manager_.update_auth_session(curl_handle, config_, hc);
|
||||
|
||||
struct curl_slist *hs = nullptr;
|
||||
hs = curl_slist_append(hs, "tus-resumable: 1.0.0");
|
||||
curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, hs);
|
||||
|
||||
curl_easy_setopt(curl_handle, CURLOPT_URL, location.c_str());
|
||||
|
||||
const auto res = curl_easy_perform(curl_handle);
|
||||
long http_code = -1;
|
||||
curl_easy_getinfo(curl_handle, CURLINFO_HTTP_CODE, &http_code);
|
||||
|
||||
ret = ((res == CURLE_OK) && (http_code >= 200) && (http_code < 300));
|
||||
|
||||
skylink = headers["skynet-skylink"];
|
||||
|
||||
curl_easy_cleanup(curl_handle);
|
||||
curl_slist_free_all(hs);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool curl_comm::tus_upload_create(host_config hc, const std::string &fileName,
|
||||
const std::uint64_t &fileSize, std::string &location) {
|
||||
auto tus_hc = hc;
|
||||
utils::string::replace(tus_hc.path, "/skyfile", "/tus");
|
||||
|
||||
auto *curl_handle = utils::create_curl();
|
||||
|
||||
const auto url = construct_url(curl_handle, "", tus_hc);
|
||||
event_system::instance().raise<comm_tus_upload_create_begin>(fileName, url);
|
||||
|
||||
http_headers headers{};
|
||||
curl_easy_setopt(curl_handle, CURLOPT_HTTPGET, 0);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_POST, 1L);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, "");
|
||||
curl_easy_setopt(curl_handle, CURLOPT_HEADERDATA, &headers);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, write_header_callback_);
|
||||
|
||||
const auto upload_length = "upload-length: " + utils::string::from_uint64(fileSize);
|
||||
const auto upload_metadata = "upload-metadata: filename " + macaron::Base64::Encode(fileName) +
|
||||
",filetype " + macaron::Base64::Encode("application/octet-stream");
|
||||
|
||||
struct curl_slist *hs = nullptr;
|
||||
hs = curl_slist_append(hs, "tus-resumable: 1.0.0");
|
||||
hs = curl_slist_append(hs, "Content-Length: 0");
|
||||
hs = curl_slist_append(hs, upload_length.c_str());
|
||||
hs = curl_slist_append(hs, upload_metadata.c_str());
|
||||
curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, hs);
|
||||
|
||||
session_manager_.update_auth_session(curl_handle, config_, hc);
|
||||
|
||||
curl_easy_setopt(curl_handle, CURLOPT_URL, url.c_str());
|
||||
|
||||
const auto res = curl_easy_perform(curl_handle);
|
||||
|
||||
long http_code = -1;
|
||||
curl_easy_getinfo(curl_handle, CURLINFO_HTTP_CODE, &http_code);
|
||||
|
||||
const auto ret = (res == CURLE_OK && http_code == 201);
|
||||
if (ret) {
|
||||
location = headers["location"];
|
||||
}
|
||||
|
||||
curl_easy_cleanup(curl_handle);
|
||||
curl_slist_free_all(hs);
|
||||
|
||||
event_system::instance().raise<comm_tus_upload_create_end>(fileName, url, res, http_code);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
void curl_comm::update_auth_session(CURL *curl_handle, const app_config &config,
|
||||
const std::string &session) {
|
||||
const auto cookie_path = utils::path::combine(config.get_data_directory(), {session + ".txt"});
|
||||
curl_easy_setopt(curl_handle, CURLOPT_COOKIEFILE, cookie_path.c_str());
|
||||
curl_easy_setopt(curl_handle, CURLOPT_COOKIEJAR, cookie_path.c_str());
|
||||
}
|
||||
|
||||
std::string curl_comm::url_encode(CURL *curl_handle, const std::string &data,
|
||||
const bool &allow_slash) {
|
||||
auto *value = curl_easy_escape(curl_handle, data.c_str(), 0);
|
||||
std::string ret = value;
|
||||
curl_free(value);
|
||||
|
||||
if (allow_slash) {
|
||||
utils::string::replace(ret, "%2F", "/");
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
curl_comm::curl_read_callback curl_comm::read_data_callback_ =
|
||||
static_cast<curl_comm::curl_read_callback>(
|
||||
[](char *buffer, size_t size, size_t nitems, void *instream) -> size_t {
|
||||
auto *rd = reinterpret_cast<read_data *>(instream);
|
||||
std::size_t bytes_read{};
|
||||
const auto ret = rd->nf->read_bytes(buffer, size * nitems, rd->offset, bytes_read);
|
||||
if (ret) {
|
||||
rd->offset += bytes_read;
|
||||
}
|
||||
return ret && not *rd->stop_requested ? bytes_read : CURL_READFUNC_ABORT;
|
||||
});
|
||||
|
||||
curl_comm::curl_write_callback curl_comm::write_data_callback_ =
|
||||
static_cast<curl_comm::curl_write_callback>(
|
||||
[](char *buffer, size_t size, size_t nitems, void *outstream) -> size_t {
|
||||
auto *wd = reinterpret_cast<raw_write_data *>(outstream);
|
||||
std::copy(buffer, buffer + (size * nitems), std::back_inserter(*wd->buffer));
|
||||
return wd->stop_requested ? 0 : size * nitems;
|
||||
});
|
||||
|
||||
curl_comm::curl_write_callback curl_comm::write_header_callback_ =
|
||||
static_cast<curl_comm::curl_write_callback>(
|
||||
[](char *buffer, size_t size, size_t nitems, void *outstream) -> size_t {
|
||||
auto &headers = *reinterpret_cast<http_headers *>(outstream);
|
||||
const auto header = std::string(buffer, size * nitems);
|
||||
const auto parts = utils::string::split(header, ':');
|
||||
if (parts.size() > 1u) {
|
||||
auto data = header.substr(parts[0u].size() + 1u);
|
||||
utils::string::left_trim(data);
|
||||
utils::string::right_trim(data, '\r');
|
||||
utils::string::right_trim(data, '\n');
|
||||
utils::string::right_trim(data, '\r');
|
||||
headers[utils::string::to_lower(parts[0u])] = data;
|
||||
}
|
||||
return size * nitems;
|
||||
});
|
||||
|
||||
curl_comm::curl_write_callback curl_comm::write_null_callback_ =
|
||||
static_cast<curl_comm::curl_write_callback>(
|
||||
[](char *, size_t size, size_t nitems, void *) -> size_t { return size * nitems; });
|
||||
|
||||
curl_comm::curl_write_callback curl_comm::write_string_callback_ =
|
||||
static_cast<curl_comm::curl_write_callback>(
|
||||
[](char *buffer, size_t size, size_t nitems, void *outstream) -> size_t {
|
||||
(*reinterpret_cast<std::string *>(outstream)) += std::string(buffer, size * nitems);
|
||||
return size * nitems;
|
||||
});
|
||||
} // namespace repertory
|
||||
43
src/comm/curl/curl_resolver.cpp
Normal file
43
src/comm/curl/curl_resolver.cpp
Normal file
@@ -0,0 +1,43 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "comm/curl/curl_resolver.hpp"
|
||||
|
||||
namespace repertory {
|
||||
curl_resolver::curl_resolver(CURL *handle, std::vector<std::string> items, const bool &ignore_root)
|
||||
: items_(std::move(items)) {
|
||||
#ifndef _WIN32
|
||||
if (ignore_root && (getuid() == 0)) {
|
||||
items_.clear();
|
||||
}
|
||||
#endif
|
||||
for (const auto &item : items_) {
|
||||
host_list_ = curl_slist_append(host_list_, &item[0u]);
|
||||
}
|
||||
|
||||
if (host_list_) {
|
||||
curl_easy_setopt(handle, CURLOPT_RESOLVE, host_list_);
|
||||
}
|
||||
}
|
||||
|
||||
curl_resolver::~curl_resolver() {
|
||||
if (host_list_) {
|
||||
curl_slist_free_all(host_list_);
|
||||
}
|
||||
}
|
||||
} // namespace repertory
|
||||
58
src/comm/curl/multi_request.cpp
Normal file
58
src/comm/curl/multi_request.cpp
Normal file
@@ -0,0 +1,58 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "comm/curl/multi_request.hpp"
|
||||
#include "utils/utils.hpp"
|
||||
|
||||
namespace repertory {
|
||||
multi_request::multi_request(CURL *curl_handle, const bool &stop_requested)
|
||||
: curl_handle_(curl_handle), stop_requested_(stop_requested), multi_handle_(curl_multi_init()) {
|
||||
curl_multi_add_handle(multi_handle_, curl_handle);
|
||||
}
|
||||
|
||||
multi_request::~multi_request() {
|
||||
curl_multi_remove_handle(multi_handle_, curl_handle_);
|
||||
curl_easy_cleanup(curl_handle_);
|
||||
curl_multi_cleanup(multi_handle_);
|
||||
}
|
||||
|
||||
void multi_request::get_result(CURLcode &curl_code, long &http_code) {
|
||||
curl_code = CURLcode::CURLE_ABORTED_BY_CALLBACK;
|
||||
http_code = -1;
|
||||
|
||||
auto error = false;
|
||||
int running_handles = 0;
|
||||
curl_multi_perform(multi_handle_, &running_handles);
|
||||
while (not error && (running_handles > 0) && not stop_requested_) {
|
||||
int ignored;
|
||||
curl_multi_wait(multi_handle_, nullptr, 0, 100, &ignored);
|
||||
|
||||
const auto ret = curl_multi_perform(multi_handle_, &running_handles);
|
||||
error = (ret != CURLM_CALL_MULTI_PERFORM) && (ret != CURLM_OK);
|
||||
}
|
||||
|
||||
if (not stop_requested_) {
|
||||
int remaining_messages = 0;
|
||||
auto *multi_result = curl_multi_info_read(multi_handle_, &remaining_messages);
|
||||
if (multi_result && (multi_result->msg == CURLMSG_DONE)) {
|
||||
curl_easy_getinfo(multi_result->easy_handle, CURLINFO_RESPONSE_CODE, &http_code);
|
||||
curl_code = multi_result->data.result;
|
||||
}
|
||||
}
|
||||
}
|
||||
} // namespace repertory
|
||||
73
src/comm/curl/session_manager.cpp
Normal file
73
src/comm/curl/session_manager.cpp
Normal file
@@ -0,0 +1,73 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "comm/curl/session_manager.hpp"
|
||||
#include "app_config.hpp"
|
||||
#include "comm/curl/curl_comm.hpp"
|
||||
#include "utils/utils.hpp"
|
||||
|
||||
namespace repertory {
|
||||
bool session_manager::create_auth_session(CURL *&curl_handle, const app_config &config,
|
||||
host_config hc, std::string &session) {
|
||||
auto ret = true;
|
||||
if (not curl_handle) {
|
||||
curl_handle = utils::create_curl();
|
||||
}
|
||||
|
||||
if (not hc.auth_url.empty() && not hc.auth_user.empty()) {
|
||||
mutex_lock l(session_mutex_);
|
||||
if (session_.empty()) {
|
||||
if ((ret = curl_comm::create_auth_session(curl_handle, config, hc, session))) {
|
||||
session_ = session;
|
||||
}
|
||||
} else {
|
||||
session = session_;
|
||||
curl_comm::update_auth_session(curl_handle, config, session_);
|
||||
}
|
||||
|
||||
if (ret) {
|
||||
session_count_++;
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
void session_manager::release_auth_session(const app_config &config, host_config hc,
|
||||
const std::string &session) {
|
||||
if (not hc.auth_url.empty() && not hc.auth_user.empty()) {
|
||||
mutex_lock l(session_mutex_);
|
||||
if (not session_.empty() && (session == session_) && session_count_) {
|
||||
if (not --session_count_) {
|
||||
curl_comm::release_auth_session(config, hc, session_);
|
||||
session_.clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void session_manager::update_auth_session(CURL *curl_handle, const app_config &config,
|
||||
const host_config &hc) {
|
||||
mutex_lock l(session_mutex_);
|
||||
if (hc.auth_url.empty() || hc.auth_user.empty() || session_.empty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
curl_comm::update_auth_session(curl_handle, config, session_);
|
||||
}
|
||||
} // namespace repertory
|
||||
150
src/comm/packet/client_pool.cpp
Normal file
150
src/comm/packet/client_pool.cpp
Normal file
@@ -0,0 +1,150 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "comm/packet/client_pool.hpp"
|
||||
#include "events/events.hpp"
|
||||
#include "events/event_system.hpp"
|
||||
|
||||
namespace repertory {
|
||||
void client_pool::pool::execute(const std::uint64_t &thread_id, const worker_callback &worker,
|
||||
const worker_complete_callback &worker_complete) {
|
||||
const auto index = static_cast<std::size_t>(thread_id % pool_queues_.size());
|
||||
auto wi = std::make_shared<work_item>(worker, worker_complete);
|
||||
auto &pool_queue = pool_queues_[index];
|
||||
|
||||
unique_mutex_lock queue_lock(pool_queue->mutex);
|
||||
pool_queue->queue.emplace_back(wi);
|
||||
pool_queue->notify.notify_all();
|
||||
queue_lock.unlock();
|
||||
}
|
||||
|
||||
client_pool::pool::pool(const std::uint8_t &pool_size) {
|
||||
thread_index_ = 0u;
|
||||
for (std::uint8_t i = 0u; i < pool_size; i++) {
|
||||
pool_queues_.emplace_back(std::make_unique<work_queue>());
|
||||
}
|
||||
|
||||
for (std::size_t i = 0u; i < pool_queues_.size(); i++) {
|
||||
pool_threads_.emplace_back(std::thread([this]() {
|
||||
const auto thread_index = thread_index_++;
|
||||
|
||||
auto &pool_queue = pool_queues_[thread_index];
|
||||
auto &queue = pool_queue->queue;
|
||||
auto &queue_mutex = pool_queue->mutex;
|
||||
auto &queue_notify = pool_queue->notify;
|
||||
|
||||
unique_mutex_lock queue_lock(queue_mutex);
|
||||
queue_notify.notify_all();
|
||||
queue_lock.unlock();
|
||||
while (not shutdown_) {
|
||||
queue_lock.lock();
|
||||
if (queue.empty()) {
|
||||
queue_notify.wait(queue_lock);
|
||||
}
|
||||
|
||||
while (not queue.empty()) {
|
||||
auto workItem = queue.front();
|
||||
queue.pop_front();
|
||||
queue_notify.notify_all();
|
||||
queue_lock.unlock();
|
||||
|
||||
try {
|
||||
const auto result = workItem->work();
|
||||
workItem->work_complete(result);
|
||||
} catch (const std::exception &e) {
|
||||
workItem->work_complete(utils::translate_api_error(api_error::error));
|
||||
event_system::instance().raise<repertory_exception>(__FUNCTION__,
|
||||
e.what() ? e.what() : "unknown");
|
||||
}
|
||||
|
||||
queue_lock.lock();
|
||||
}
|
||||
|
||||
queue_notify.notify_all();
|
||||
queue_lock.unlock();
|
||||
}
|
||||
|
||||
queue_lock.lock();
|
||||
while (not queue.empty()) {
|
||||
auto wi = queue.front();
|
||||
queue.pop_front();
|
||||
queue_notify.notify_all();
|
||||
queue_lock.unlock();
|
||||
|
||||
wi->work_complete(utils::translate_api_error(api_error::download_stopped));
|
||||
|
||||
queue_lock.lock();
|
||||
}
|
||||
|
||||
queue_notify.notify_all();
|
||||
queue_lock.unlock();
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
void client_pool::pool::shutdown() {
|
||||
shutdown_ = true;
|
||||
|
||||
for (auto &pool_queue : pool_queues_) {
|
||||
unique_mutex_lock l(pool_queue->mutex);
|
||||
pool_queue->notify.notify_all();
|
||||
}
|
||||
|
||||
for (auto &thread : pool_threads_) {
|
||||
thread.join();
|
||||
}
|
||||
|
||||
pool_queues_.clear();
|
||||
pool_threads_.clear();
|
||||
}
|
||||
|
||||
void client_pool::execute(const std::string &client_id, const std::uint64_t &thread_id,
|
||||
const worker_callback &worker,
|
||||
const worker_complete_callback &worker_complete) {
|
||||
unique_mutex_lock pool_lock(pool_mutex_);
|
||||
if (shutdown_) {
|
||||
pool_lock.unlock();
|
||||
throw std::runtime_error("Client pool is shutdown");
|
||||
}
|
||||
|
||||
if (not pool_lookup_[client_id]) {
|
||||
pool_lookup_[client_id] = std::make_shared<pool>(pool_size_);
|
||||
}
|
||||
|
||||
pool_lookup_[client_id]->execute(thread_id, worker, worker_complete);
|
||||
pool_lock.unlock();
|
||||
}
|
||||
|
||||
void client_pool::remove_client(const std::string &client_id) {
|
||||
mutex_lock pool_lock(pool_mutex_);
|
||||
pool_lookup_.erase(client_id);
|
||||
}
|
||||
|
||||
void client_pool::shutdown() {
|
||||
unique_mutex_lock pool_lock(pool_mutex_);
|
||||
if (not shutdown_) {
|
||||
shutdown_ = true;
|
||||
event_system::instance().raise<service_shutdown>("client_pool");
|
||||
for (auto &kv : pool_lookup_) {
|
||||
kv.second->shutdown();
|
||||
}
|
||||
pool_lookup_.clear();
|
||||
}
|
||||
pool_lock.unlock();
|
||||
}
|
||||
} // namespace repertory
|
||||
555
src/comm/packet/packet.cpp
Normal file
555
src/comm/packet/packet.cpp
Normal file
@@ -0,0 +1,555 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "comm/packet/packet.hpp"
|
||||
#include "events/events.hpp"
|
||||
#include "events/event_system.hpp"
|
||||
#include "types/remote.hpp"
|
||||
#include "types/repertory.hpp"
|
||||
#include "utils/encryption.hpp"
|
||||
#include "utils/utils.hpp"
|
||||
|
||||
namespace repertory {
|
||||
void packet::clear() {
|
||||
buffer_.clear();
|
||||
decode_offset_ = 0u;
|
||||
}
|
||||
|
||||
packet::error_type packet::decode(std::string &data) {
|
||||
const auto *str = &buffer_[decode_offset_];
|
||||
const auto length = strnlen(str, buffer_.size() - decode_offset_);
|
||||
data = std::string(str, length);
|
||||
decode_offset_ += (length + 1);
|
||||
|
||||
return utils::translate_api_error(api_error::success);
|
||||
}
|
||||
|
||||
packet::error_type packet::decode(std::wstring &data) {
|
||||
std::string utf8_string;
|
||||
const auto ret = decode(utf8_string);
|
||||
if (ret == 0) {
|
||||
data = utils::string::from_utf8(utf8_string);
|
||||
}
|
||||
|
||||
return utils::translate_api_error(api_error::success);
|
||||
}
|
||||
|
||||
packet::error_type packet::decode(void *&ptr) {
|
||||
return decode(reinterpret_cast<std::uint64_t &>(ptr));
|
||||
}
|
||||
|
||||
packet::error_type packet::decode(void *buffer, const size_t &size) {
|
||||
if (size) {
|
||||
const auto read_size = utils::calculate_read_size(buffer_.size(), size, decode_offset_);
|
||||
if (read_size == size) {
|
||||
memcpy(buffer, &buffer_[decode_offset_], size);
|
||||
decode_offset_ += size;
|
||||
return utils::translate_api_error(api_error::success);
|
||||
}
|
||||
|
||||
return ((decode_offset_ + size) > buffer_.size())
|
||||
? utils::translate_api_error(api_error::buffer_overflow)
|
||||
: utils::translate_api_error(api_error::buffer_too_small);
|
||||
}
|
||||
return utils::translate_api_error(api_error::success);
|
||||
}
|
||||
|
||||
packet::error_type packet::decode(std::int8_t &i) {
|
||||
const auto ret = decode(&i, sizeof(i));
|
||||
if (ret == 0) {
|
||||
boost::endian::big_to_native_inplace(i);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type packet::decode(std::uint8_t &i) {
|
||||
const auto ret = decode(&i, sizeof(i));
|
||||
if (ret == 0) {
|
||||
boost::endian::big_to_native_inplace(i);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type packet::decode(std::int16_t &i) {
|
||||
const auto ret = decode(&i, sizeof(i));
|
||||
if (ret == 0) {
|
||||
boost::endian::big_to_native_inplace(i);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type packet::decode(std::uint16_t &i) {
|
||||
const auto ret = decode(&i, sizeof(i));
|
||||
if (ret == 0) {
|
||||
boost::endian::big_to_native_inplace(i);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type packet::decode(std::int32_t &i) {
|
||||
const auto ret = decode(&i, sizeof(i));
|
||||
if (ret == 0) {
|
||||
boost::endian::big_to_native_inplace(i);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type packet::decode(std::uint32_t &i) {
|
||||
const auto ret = decode(&i, sizeof(i));
|
||||
if (ret == 0) {
|
||||
boost::endian::big_to_native_inplace(i);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type packet::decode(std::int64_t &i) {
|
||||
const auto ret = decode(&i, sizeof(i));
|
||||
if (ret == 0) {
|
||||
boost::endian::big_to_native_inplace(i);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type packet::decode(std::uint64_t &i) {
|
||||
const auto ret = decode(&i, sizeof(i));
|
||||
if (ret == 0) {
|
||||
boost::endian::big_to_native_inplace(i);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type packet::decode(remote::setattr_x &i) {
|
||||
const auto ret = decode(&i, sizeof(i));
|
||||
if (ret == 0) {
|
||||
boost::endian::big_to_native_inplace(i.acctime);
|
||||
boost::endian::big_to_native_inplace(i.bkuptime);
|
||||
boost::endian::big_to_native_inplace(i.chgtime);
|
||||
boost::endian::big_to_native_inplace(i.crtime);
|
||||
boost::endian::big_to_native_inplace(i.flags);
|
||||
boost::endian::big_to_native_inplace(i.gid);
|
||||
boost::endian::big_to_native_inplace(i.mode);
|
||||
boost::endian::big_to_native_inplace(i.modtime);
|
||||
boost::endian::big_to_native_inplace(i.size);
|
||||
boost::endian::big_to_native_inplace(i.uid);
|
||||
boost::endian::big_to_native_inplace(i.valid);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type packet::decode(remote::stat &i) {
|
||||
const auto ret = decode(&i, sizeof(i));
|
||||
if (ret == 0) {
|
||||
boost::endian::big_to_native_inplace(i.st_mode);
|
||||
boost::endian::big_to_native_inplace(i.st_nlink);
|
||||
boost::endian::big_to_native_inplace(i.st_uid);
|
||||
boost::endian::big_to_native_inplace(i.st_gid);
|
||||
boost::endian::big_to_native_inplace(i.st_atimespec);
|
||||
boost::endian::big_to_native_inplace(i.st_mtimespec);
|
||||
boost::endian::big_to_native_inplace(i.st_ctimespec);
|
||||
boost::endian::big_to_native_inplace(i.st_birthtimespec);
|
||||
boost::endian::big_to_native_inplace(i.st_size);
|
||||
boost::endian::big_to_native_inplace(i.st_blocks);
|
||||
boost::endian::big_to_native_inplace(i.st_blksize);
|
||||
boost::endian::big_to_native_inplace(i.st_flags);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type packet::decode(remote::statfs &i) {
|
||||
const auto ret = decode(&i, sizeof(i));
|
||||
if (ret == 0) {
|
||||
boost::endian::big_to_native_inplace(i.f_bavail);
|
||||
boost::endian::big_to_native_inplace(i.f_bfree);
|
||||
boost::endian::big_to_native_inplace(i.f_blocks);
|
||||
boost::endian::big_to_native_inplace(i.f_favail);
|
||||
boost::endian::big_to_native_inplace(i.f_ffree);
|
||||
boost::endian::big_to_native_inplace(i.f_files);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type packet::decode(remote::statfs_x &i) {
|
||||
auto ret = decode(*dynamic_cast<remote::statfs *>(&i));
|
||||
if (ret == 0) {
|
||||
ret = decode(&i.f_mntfromname[0], 1024);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type packet::decode(remote::file_info &i) {
|
||||
const auto ret = decode(&i, sizeof(i));
|
||||
if (ret == 0) {
|
||||
boost::endian::big_to_native_inplace(i.AllocationSize);
|
||||
boost::endian::big_to_native_inplace(i.ChangeTime);
|
||||
boost::endian::big_to_native_inplace(i.CreationTime);
|
||||
boost::endian::big_to_native_inplace(i.EaSize);
|
||||
boost::endian::big_to_native_inplace(i.FileAttributes);
|
||||
boost::endian::big_to_native_inplace(i.FileSize);
|
||||
boost::endian::big_to_native_inplace(i.HardLinks);
|
||||
boost::endian::big_to_native_inplace(i.IndexNumber);
|
||||
boost::endian::big_to_native_inplace(i.LastAccessTime);
|
||||
boost::endian::big_to_native_inplace(i.LastWriteTime);
|
||||
boost::endian::big_to_native_inplace(i.ReparseTag);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
int packet::decode_json(packet &response, json &json_data) {
|
||||
int ret = 0;
|
||||
std::string data;
|
||||
if ((ret = response.decode(data)) == 0) {
|
||||
try {
|
||||
json_data = json::parse(data);
|
||||
} catch (const std::exception &e) {
|
||||
event_system::instance().raise<repertory_exception>(
|
||||
__FUNCTION__, e.what() ? e.what() : "Failed to parse JSON string");
|
||||
ret = -EIO;
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type packet::decrypt(const std::string &token) {
|
||||
auto ret = utils::translate_api_error(api_error::success);
|
||||
try {
|
||||
std::vector<char> result;
|
||||
if (not utils::encryption::decrypt_data(token, &buffer_[decode_offset_],
|
||||
buffer_.size() - decode_offset_, result)) {
|
||||
throw std::runtime_error("Decryption failed");
|
||||
}
|
||||
buffer_ = std::move(result);
|
||||
decode_offset_ = 0;
|
||||
} catch (const std::exception &e) {
|
||||
event_system::instance().raise<repertory_exception>(__FUNCTION__, e.what());
|
||||
ret = utils::translate_api_error(api_error::error);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
void packet::encode(const void *buffer, const std::size_t &size, bool should_reserve) {
|
||||
if (size) {
|
||||
if (should_reserve) {
|
||||
buffer_.reserve(buffer_.size() + size);
|
||||
}
|
||||
const auto *char_buffer = reinterpret_cast<const char *>(buffer);
|
||||
buffer_.insert(buffer_.end(), char_buffer, char_buffer + size);
|
||||
}
|
||||
}
|
||||
|
||||
void packet::encode(const std::string &str) {
|
||||
const auto len = strnlen(&str[0], str.size());
|
||||
buffer_.reserve(len + 1 + buffer_.size());
|
||||
encode(&str[0], len, false);
|
||||
buffer_.emplace_back(0);
|
||||
}
|
||||
|
||||
void packet::encode(wchar_t *str) { encode(utils::string::to_utf8(str ? str : L"")); }
|
||||
|
||||
void packet::encode(const wchar_t *str) { encode(utils::string::to_utf8(str ? str : L"")); }
|
||||
|
||||
void packet::encode(const std::wstring &str) { encode(utils::string::to_utf8(str)); }
|
||||
|
||||
void packet::encode(std::int8_t i) {
|
||||
boost::endian::native_to_big_inplace(i);
|
||||
encode(&i, sizeof(i), true);
|
||||
}
|
||||
|
||||
void packet::encode(std::uint8_t i) {
|
||||
boost::endian::native_to_big_inplace(i);
|
||||
encode(&i, sizeof(i), true);
|
||||
}
|
||||
|
||||
void packet::encode(std::int16_t i) {
|
||||
boost::endian::native_to_big_inplace(i);
|
||||
encode(&i, sizeof(i), true);
|
||||
}
|
||||
|
||||
void packet::encode(std::uint16_t i) {
|
||||
boost::endian::native_to_big_inplace(i);
|
||||
encode(&i, sizeof(i), true);
|
||||
}
|
||||
|
||||
void packet::encode(std::int32_t i) {
|
||||
boost::endian::native_to_big_inplace(i);
|
||||
encode(&i, sizeof(i), true);
|
||||
}
|
||||
|
||||
void packet::encode(std::uint32_t i) {
|
||||
boost::endian::native_to_big_inplace(i);
|
||||
encode(&i, sizeof(i), true);
|
||||
}
|
||||
|
||||
void packet::encode(std::int64_t i) {
|
||||
boost::endian::native_to_big_inplace(i);
|
||||
encode(&i, sizeof(i), true);
|
||||
}
|
||||
|
||||
void packet::encode(std::uint64_t i) {
|
||||
boost::endian::native_to_big_inplace(i);
|
||||
encode(&i, sizeof(i), true);
|
||||
}
|
||||
|
||||
void packet::encode(remote::setattr_x i) {
|
||||
boost::endian::native_to_big_inplace(i.acctime);
|
||||
boost::endian::native_to_big_inplace(i.bkuptime);
|
||||
boost::endian::native_to_big_inplace(i.chgtime);
|
||||
boost::endian::native_to_big_inplace(i.crtime);
|
||||
boost::endian::native_to_big_inplace(i.flags);
|
||||
boost::endian::native_to_big_inplace(i.gid);
|
||||
boost::endian::native_to_big_inplace(i.mode);
|
||||
boost::endian::native_to_big_inplace(i.modtime);
|
||||
boost::endian::native_to_big_inplace(i.size);
|
||||
boost::endian::native_to_big_inplace(i.uid);
|
||||
boost::endian::native_to_big_inplace(i.valid);
|
||||
|
||||
encode(&i, sizeof(i), true);
|
||||
}
|
||||
|
||||
void packet::encode(remote::stat i) {
|
||||
boost::endian::native_to_big_inplace(i.st_mode);
|
||||
boost::endian::native_to_big_inplace(i.st_nlink);
|
||||
boost::endian::native_to_big_inplace(i.st_uid);
|
||||
boost::endian::native_to_big_inplace(i.st_gid);
|
||||
boost::endian::native_to_big_inplace(i.st_atimespec);
|
||||
boost::endian::native_to_big_inplace(i.st_mtimespec);
|
||||
boost::endian::native_to_big_inplace(i.st_ctimespec);
|
||||
boost::endian::native_to_big_inplace(i.st_birthtimespec);
|
||||
boost::endian::native_to_big_inplace(i.st_size);
|
||||
boost::endian::native_to_big_inplace(i.st_blocks);
|
||||
boost::endian::native_to_big_inplace(i.st_blksize);
|
||||
boost::endian::native_to_big_inplace(i.st_flags);
|
||||
|
||||
encode(&i, sizeof(i), true);
|
||||
}
|
||||
|
||||
void packet::encode(remote::statfs i, bool should_reserve) {
|
||||
boost::endian::native_to_big_inplace(i.f_bavail);
|
||||
boost::endian::native_to_big_inplace(i.f_bfree);
|
||||
boost::endian::native_to_big_inplace(i.f_blocks);
|
||||
boost::endian::native_to_big_inplace(i.f_favail);
|
||||
boost::endian::native_to_big_inplace(i.f_ffree);
|
||||
boost::endian::native_to_big_inplace(i.f_files);
|
||||
|
||||
encode(&i, sizeof(remote::statfs), should_reserve);
|
||||
}
|
||||
|
||||
void packet::encode(remote::statfs_x i) {
|
||||
buffer_.reserve(buffer_.size() + sizeof(remote::statfs) + 1024);
|
||||
encode(*dynamic_cast<remote::statfs *>(&i), false);
|
||||
encode(&i.f_mntfromname[0], 1024, false);
|
||||
}
|
||||
|
||||
void packet::encode(remote::file_info i) {
|
||||
boost::endian::native_to_big_inplace(i.FileAttributes);
|
||||
boost::endian::native_to_big_inplace(i.ReparseTag);
|
||||
boost::endian::native_to_big_inplace(i.AllocationSize);
|
||||
boost::endian::native_to_big_inplace(i.FileSize);
|
||||
boost::endian::native_to_big_inplace(i.CreationTime);
|
||||
boost::endian::native_to_big_inplace(i.LastAccessTime);
|
||||
boost::endian::native_to_big_inplace(i.LastWriteTime);
|
||||
boost::endian::native_to_big_inplace(i.ChangeTime);
|
||||
boost::endian::native_to_big_inplace(i.IndexNumber);
|
||||
boost::endian::native_to_big_inplace(i.HardLinks);
|
||||
boost::endian::native_to_big_inplace(i.EaSize);
|
||||
|
||||
encode(&i, sizeof(i), true);
|
||||
}
|
||||
|
||||
void packet::encode_top(const void *buffer, const std::size_t &size, bool should_reserve) {
|
||||
if (size) {
|
||||
if (should_reserve) {
|
||||
buffer_.reserve(buffer_.size() + size);
|
||||
}
|
||||
const auto *char_buffer = reinterpret_cast<const char *>(buffer);
|
||||
buffer_.insert(buffer_.begin(), char_buffer, char_buffer + size);
|
||||
}
|
||||
}
|
||||
|
||||
void packet::encode_top(const std::string &str) {
|
||||
const auto len = strnlen(&str[0], str.size());
|
||||
buffer_.reserve(len + 1 + buffer_.size());
|
||||
encode_top(&str[0], len, false);
|
||||
buffer_.insert(buffer_.begin() + len, 0);
|
||||
}
|
||||
|
||||
void packet::encode_top(const std::wstring &str) { encode_top(utils::string::to_utf8(str)); }
|
||||
|
||||
void packet::encode_top(std::int8_t i) {
|
||||
boost::endian::native_to_big_inplace(i);
|
||||
encode_top(&i, sizeof(i), true);
|
||||
}
|
||||
|
||||
void packet::encode_top(std::uint8_t i) {
|
||||
boost::endian::native_to_big_inplace(i);
|
||||
encode_top(&i, sizeof(i), true);
|
||||
}
|
||||
|
||||
void packet::encode_top(std::int16_t i) {
|
||||
boost::endian::native_to_big_inplace(i);
|
||||
encode_top(&i, sizeof(i), true);
|
||||
}
|
||||
|
||||
void packet::encode_top(std::uint16_t i) {
|
||||
boost::endian::native_to_big_inplace(i);
|
||||
encode_top(&i, sizeof(i), true);
|
||||
}
|
||||
|
||||
void packet::encode_top(std::int32_t i) {
|
||||
boost::endian::native_to_big_inplace(i);
|
||||
encode_top(&i, sizeof(i), true);
|
||||
}
|
||||
|
||||
void packet::encode_top(std::uint32_t i) {
|
||||
boost::endian::native_to_big_inplace(i);
|
||||
encode_top(&i, sizeof(i), true);
|
||||
}
|
||||
|
||||
void packet::encode_top(std::int64_t i) {
|
||||
boost::endian::native_to_big_inplace(i);
|
||||
encode_top(&i, sizeof(i), true);
|
||||
}
|
||||
|
||||
void packet::encode_top(std::uint64_t i) {
|
||||
boost::endian::native_to_big_inplace(i);
|
||||
encode_top(&i, sizeof(i), true);
|
||||
}
|
||||
|
||||
void packet::encode_top(remote::setattr_x i) {
|
||||
boost::endian::native_to_big_inplace(i.acctime);
|
||||
boost::endian::native_to_big_inplace(i.bkuptime);
|
||||
boost::endian::native_to_big_inplace(i.chgtime);
|
||||
boost::endian::native_to_big_inplace(i.crtime);
|
||||
boost::endian::native_to_big_inplace(i.flags);
|
||||
boost::endian::native_to_big_inplace(i.gid);
|
||||
boost::endian::native_to_big_inplace(i.mode);
|
||||
boost::endian::native_to_big_inplace(i.modtime);
|
||||
boost::endian::native_to_big_inplace(i.size);
|
||||
boost::endian::native_to_big_inplace(i.uid);
|
||||
boost::endian::native_to_big_inplace(i.valid);
|
||||
|
||||
encode_top(&i, sizeof(i), true);
|
||||
}
|
||||
|
||||
void packet::encode_top(remote::stat i) {
|
||||
boost::endian::native_to_big_inplace(i.st_mode);
|
||||
boost::endian::native_to_big_inplace(i.st_nlink);
|
||||
boost::endian::native_to_big_inplace(i.st_uid);
|
||||
boost::endian::native_to_big_inplace(i.st_gid);
|
||||
boost::endian::native_to_big_inplace(i.st_atimespec);
|
||||
boost::endian::native_to_big_inplace(i.st_mtimespec);
|
||||
boost::endian::native_to_big_inplace(i.st_ctimespec);
|
||||
boost::endian::native_to_big_inplace(i.st_birthtimespec);
|
||||
boost::endian::native_to_big_inplace(i.st_size);
|
||||
boost::endian::native_to_big_inplace(i.st_blocks);
|
||||
boost::endian::native_to_big_inplace(i.st_blksize);
|
||||
boost::endian::native_to_big_inplace(i.st_flags);
|
||||
|
||||
encode_top(&i, sizeof(i), true);
|
||||
}
|
||||
|
||||
void packet::encode_top(remote::statfs i, bool should_reserve) {
|
||||
boost::endian::native_to_big_inplace(i.f_bavail);
|
||||
boost::endian::native_to_big_inplace(i.f_bfree);
|
||||
boost::endian::native_to_big_inplace(i.f_blocks);
|
||||
boost::endian::native_to_big_inplace(i.f_favail);
|
||||
boost::endian::native_to_big_inplace(i.f_ffree);
|
||||
boost::endian::native_to_big_inplace(i.f_files);
|
||||
|
||||
encode_top(&i, sizeof(remote::statfs), should_reserve);
|
||||
}
|
||||
|
||||
void packet::encode_top(remote::statfs_x i) {
|
||||
buffer_.reserve(buffer_.size() + sizeof(remote::statfs) + 1024);
|
||||
encode_top(&i.f_mntfromname[0], 1024, false);
|
||||
encode_top(*dynamic_cast<remote::statfs *>(&i), false);
|
||||
}
|
||||
|
||||
void packet::encode_top(remote::file_info i) {
|
||||
boost::endian::native_to_big_inplace(i.FileAttributes);
|
||||
boost::endian::native_to_big_inplace(i.ReparseTag);
|
||||
boost::endian::native_to_big_inplace(i.AllocationSize);
|
||||
boost::endian::native_to_big_inplace(i.FileSize);
|
||||
boost::endian::native_to_big_inplace(i.CreationTime);
|
||||
boost::endian::native_to_big_inplace(i.LastAccessTime);
|
||||
boost::endian::native_to_big_inplace(i.LastWriteTime);
|
||||
boost::endian::native_to_big_inplace(i.ChangeTime);
|
||||
boost::endian::native_to_big_inplace(i.IndexNumber);
|
||||
boost::endian::native_to_big_inplace(i.HardLinks);
|
||||
boost::endian::native_to_big_inplace(i.EaSize);
|
||||
|
||||
encode_top(&i, sizeof(i), true);
|
||||
}
|
||||
|
||||
void packet::encrypt(const std::string &token) {
|
||||
try {
|
||||
std::vector<char> result;
|
||||
utils::encryption::encrypt_data(token, buffer_, result);
|
||||
buffer_ = std::move(result);
|
||||
encode_top(static_cast<std::uint32_t>(buffer_.size()));
|
||||
} catch (const std::exception &e) {
|
||||
event_system::instance().raise<repertory_exception>(__FUNCTION__, e.what());
|
||||
}
|
||||
}
|
||||
|
||||
void packet::transfer_into(std::vector<char> &buffer) {
|
||||
buffer = std::move(buffer_);
|
||||
buffer_ = std::vector<char>();
|
||||
decode_offset_ = 0;
|
||||
}
|
||||
|
||||
packet &packet::operator=(const std::vector<char> &buffer) noexcept {
|
||||
if (&buffer_ != &buffer) {
|
||||
buffer_ = buffer;
|
||||
decode_offset_ = 0;
|
||||
}
|
||||
|
||||
return *this;
|
||||
}
|
||||
|
||||
packet &packet::operator=(std::vector<char> &&buffer) noexcept {
|
||||
if (&buffer_ != &buffer) {
|
||||
buffer_ = std::move(buffer);
|
||||
decode_offset_ = 0;
|
||||
}
|
||||
|
||||
return *this;
|
||||
}
|
||||
|
||||
packet &packet::operator=(const packet &p) noexcept {
|
||||
if (this != &p) {
|
||||
buffer_ = p.buffer_;
|
||||
decode_offset_ = p.decode_offset_;
|
||||
}
|
||||
|
||||
return *this;
|
||||
}
|
||||
|
||||
packet &packet::operator=(packet &&p) noexcept {
|
||||
if (this != &p) {
|
||||
buffer_ = std::move(p.buffer_);
|
||||
decode_offset_ = p.decode_offset_;
|
||||
}
|
||||
|
||||
return *this;
|
||||
}
|
||||
} // namespace repertory
|
||||
227
src/comm/packet/packet_client.cpp
Normal file
227
src/comm/packet/packet_client.cpp
Normal file
@@ -0,0 +1,227 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "comm/packet/packet_client.hpp"
|
||||
#include "events/events.hpp"
|
||||
#include "types/repertory.hpp"
|
||||
#include "utils/timeout.hpp"
|
||||
|
||||
namespace repertory {
|
||||
// clang-format off
|
||||
E_SIMPLE2(packet_client_timeout, error, true,
|
||||
std::string, event_name, en, E_STRING,
|
||||
std::string, message, msg, E_STRING
|
||||
);
|
||||
// clang-format on
|
||||
|
||||
packet_client::packet_client(std::string host_name_or_ip, const std::uint8_t &max_connections,
|
||||
const std::uint16_t &port, const std::uint16_t &receive_timeout,
|
||||
const std::uint16_t &send_timeout, std::string encryption_token)
|
||||
: io_context_(),
|
||||
host_name_or_ip_(std::move(host_name_or_ip)),
|
||||
max_connections_(max_connections ? max_connections : 20u),
|
||||
port_(port),
|
||||
receive_timeout_(receive_timeout),
|
||||
send_timeout_(send_timeout),
|
||||
encryption_token_(std::move(encryption_token)),
|
||||
unique_id_(utils::create_uuid_string()) {}
|
||||
|
||||
packet_client::~packet_client() {
|
||||
allow_connections_ = false;
|
||||
close_all();
|
||||
io_context_.stop();
|
||||
}
|
||||
|
||||
void packet_client::close(client &client) const {
|
||||
try {
|
||||
boost::system::error_code ec;
|
||||
client.socket.close(ec);
|
||||
} catch (...) {
|
||||
}
|
||||
}
|
||||
|
||||
void packet_client::close_all() {
|
||||
unique_mutex_lock clients_lock(clients_mutex_);
|
||||
for (auto &c : clients_) {
|
||||
close(*c.get());
|
||||
}
|
||||
|
||||
clients_.clear();
|
||||
unique_id_ = utils::create_uuid_string();
|
||||
}
|
||||
|
||||
bool packet_client::connect(client &c) {
|
||||
auto ret = false;
|
||||
try {
|
||||
resolve();
|
||||
boost::asio::connect(c.socket, resolve_results_);
|
||||
c.socket.set_option(boost::asio::ip::tcp::no_delay(true));
|
||||
c.socket.set_option(boost::asio::socket_base::linger(false, 0));
|
||||
|
||||
packet response;
|
||||
read_packet(c, response);
|
||||
|
||||
ret = true;
|
||||
} catch (const std::exception &e) {
|
||||
event_system::instance().raise<repertory_exception>(__FUNCTION__, e.what());
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
std::shared_ptr<packet_client::client> packet_client::get_client() {
|
||||
std::shared_ptr<client> ret;
|
||||
|
||||
unique_mutex_lock clients_lock(clients_mutex_);
|
||||
if (allow_connections_) {
|
||||
if (clients_.empty()) {
|
||||
clients_lock.unlock();
|
||||
ret = std::make_shared<client>(io_context_);
|
||||
connect(*ret);
|
||||
} else {
|
||||
ret = clients_[0u];
|
||||
utils::remove_element_from(clients_, ret);
|
||||
clients_lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
void packet_client::put_client(std::shared_ptr<client> &c) {
|
||||
mutex_lock clientsLock(clients_mutex_);
|
||||
if (clients_.size() < max_connections_) {
|
||||
clients_.emplace_back(c);
|
||||
}
|
||||
}
|
||||
|
||||
packet::error_type packet_client::read_packet(client &c, packet &response) {
|
||||
std::vector<char> buffer(sizeof(std::uint32_t));
|
||||
const auto read_buffer = [&]() {
|
||||
std::uint32_t offset = 0u;
|
||||
while (offset < buffer.size()) {
|
||||
const auto bytes_read =
|
||||
boost::asio::read(c.socket, boost::asio::buffer(&buffer[offset], buffer.size() - offset));
|
||||
if (bytes_read <= 0) {
|
||||
throw std::runtime_error("Read failed: " + std::to_string(bytes_read));
|
||||
}
|
||||
offset += static_cast<std::uint32_t>(bytes_read);
|
||||
}
|
||||
};
|
||||
read_buffer();
|
||||
|
||||
const auto size = boost::endian::big_to_native(*reinterpret_cast<std::uint32_t *>(&buffer[0u]));
|
||||
buffer.resize(size);
|
||||
|
||||
read_buffer();
|
||||
response = std::move(buffer);
|
||||
|
||||
auto ret = response.decrypt(encryption_token_);
|
||||
if (ret == 0) {
|
||||
ret = response.decode(c.nonce);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
void packet_client::resolve() {
|
||||
if (resolve_results_.empty()) {
|
||||
resolve_results_ =
|
||||
tcp::resolver(io_context_).resolve({host_name_or_ip_, std::to_string(port_)});
|
||||
}
|
||||
}
|
||||
|
||||
packet::error_type packet_client::send(const std::string &method, std::uint32_t &service_flags) {
|
||||
packet request;
|
||||
return send(method, request, service_flags);
|
||||
}
|
||||
|
||||
packet::error_type packet_client::send(const std::string &method, packet &request,
|
||||
std::uint32_t &service_flags) {
|
||||
packet response;
|
||||
return send(method, request, response, service_flags);
|
||||
}
|
||||
|
||||
packet::error_type packet_client::send(const std::string &method, packet &request, packet &response,
|
||||
std::uint32_t &service_flags) {
|
||||
auto success = false;
|
||||
packet::error_type ret = utils::translate_api_error(api_error::error);
|
||||
request.encode_top(method);
|
||||
request.encode_top(utils::get_thread_id());
|
||||
request.encode_top(unique_id_);
|
||||
request.encode_top(PACKET_SERVICE_FLAGS);
|
||||
request.encode_top(get_repertory_version());
|
||||
|
||||
static const auto max_attempts = 2;
|
||||
for (auto i = 1; allow_connections_ && not success && (i <= max_attempts); i++) {
|
||||
auto c = get_client();
|
||||
if (c) {
|
||||
try {
|
||||
request.encode_top(c->nonce);
|
||||
request.encrypt(encryption_token_);
|
||||
|
||||
timeout request_timeout(
|
||||
[this, method, c]() {
|
||||
event_system::instance().raise<packet_client_timeout>("Request", method);
|
||||
close(*c.get());
|
||||
},
|
||||
std::chrono::seconds(send_timeout_));
|
||||
|
||||
std::uint32_t offset = 0u;
|
||||
while (offset < request.get_size()) {
|
||||
const auto bytes_written = boost::asio::write(
|
||||
c->socket, boost::asio::buffer(&request[offset], request.get_size() - offset));
|
||||
if (bytes_written <= 0) {
|
||||
throw std::runtime_error("Write failed: " + std::to_string(bytes_written));
|
||||
}
|
||||
offset += static_cast<std::uint32_t>(bytes_written);
|
||||
}
|
||||
request_timeout.disable();
|
||||
|
||||
timeout response_timeout(
|
||||
[this, method, c]() {
|
||||
event_system::instance().raise<packet_client_timeout>("Response", method);
|
||||
close(*c.get());
|
||||
},
|
||||
std::chrono::seconds(receive_timeout_));
|
||||
|
||||
ret = read_packet(*c, response);
|
||||
response_timeout.disable();
|
||||
if (ret == 0) {
|
||||
response.decode(service_flags);
|
||||
response.decode(ret);
|
||||
success = true;
|
||||
put_client(c);
|
||||
}
|
||||
} catch (const std::exception &e) {
|
||||
event_system::instance().raise<repertory_exception>(__FUNCTION__, e.what());
|
||||
close_all();
|
||||
if (allow_connections_ && (i < max_attempts)) {
|
||||
std::this_thread::sleep_for(1s);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (not allow_connections_) {
|
||||
ret = utils::translate_api_error(api_error::error);
|
||||
success = true;
|
||||
}
|
||||
}
|
||||
|
||||
return CONVERT_STATUS_NOT_IMPLEMENTED(ret);
|
||||
}
|
||||
} // namespace repertory
|
||||
225
src/comm/packet/packet_server.cpp
Normal file
225
src/comm/packet/packet_server.cpp
Normal file
@@ -0,0 +1,225 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "comm/packet/packet_server.hpp"
|
||||
#include "events/events.hpp"
|
||||
#include "events/event_system.hpp"
|
||||
#include "comm/packet/packet.hpp"
|
||||
#include "types/repertory.hpp"
|
||||
#include "utils/utils.hpp"
|
||||
|
||||
namespace repertory {
|
||||
packet_server::packet_server(const std::uint16_t &port, std::string token, std::uint8_t pool_size,
|
||||
closed_callback closed, message_handler_callback message_handler)
|
||||
: encryption_token_(std::move(token)), closed_(closed), message_handler_(message_handler) {
|
||||
initialize(port, pool_size);
|
||||
}
|
||||
|
||||
packet_server::~packet_server() {
|
||||
event_system::instance().raise<service_shutdown>("packet_server");
|
||||
std::thread([this]() {
|
||||
for (std::size_t i = 0u; i < service_threads_.size(); i++) {
|
||||
io_context_.stop();
|
||||
}
|
||||
}).detach();
|
||||
|
||||
server_thread_->join();
|
||||
server_thread_.reset();
|
||||
}
|
||||
|
||||
void packet_server::add_client(connection &c, const std::string &client_id) {
|
||||
c.client_id = client_id;
|
||||
|
||||
recur_mutex_lock connection_lock(connection_mutex_);
|
||||
if (connection_lookup_.find(client_id) == connection_lookup_.end()) {
|
||||
connection_lookup_[client_id] = 1u;
|
||||
} else {
|
||||
connection_lookup_[client_id]++;
|
||||
}
|
||||
}
|
||||
|
||||
void packet_server::initialize(const uint16_t &port, uint8_t pool_size) {
|
||||
pool_size = std::max(uint8_t(1u), pool_size);
|
||||
server_thread_ = std::make_unique<std::thread>([this, port, pool_size]() {
|
||||
tcp::acceptor acceptor(io_context_);
|
||||
try {
|
||||
const auto endpoint = tcp::endpoint(tcp::v4(), port);
|
||||
acceptor.open(endpoint.protocol());
|
||||
acceptor.set_option(socket_base::reuse_address(true));
|
||||
acceptor.bind(endpoint);
|
||||
acceptor.listen();
|
||||
} catch (const std::exception &e) {
|
||||
event_system::instance().raise<repertory_exception>(__FUNCTION__, e.what());
|
||||
}
|
||||
listen_for_connection(acceptor);
|
||||
|
||||
for (std::uint8_t i = 0u; i < pool_size; i++) {
|
||||
service_threads_.emplace_back(std::thread([this]() { io_context_.run(); }));
|
||||
}
|
||||
|
||||
for (auto &th : service_threads_) {
|
||||
th.join();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
void packet_server::listen_for_connection(tcp::acceptor &acceptor) {
|
||||
auto c = std::make_shared<packet_server::connection>(io_context_, acceptor);
|
||||
acceptor.async_accept(
|
||||
c->socket, boost::bind(&packet_server::on_accept, this, c, boost::asio::placeholders::error));
|
||||
}
|
||||
|
||||
void packet_server::on_accept(std::shared_ptr<connection> c, boost::system::error_code ec) {
|
||||
listen_for_connection(c->acceptor);
|
||||
if (ec) {
|
||||
event_system::instance().raise<repertory_exception>(__FUNCTION__, ec.message());
|
||||
std::this_thread::sleep_for(1s);
|
||||
} else {
|
||||
c->socket.set_option(boost::asio::ip::tcp::no_delay(true));
|
||||
c->socket.set_option(boost::asio::socket_base::linger(false, 0));
|
||||
|
||||
c->generate_nonce();
|
||||
|
||||
packet response;
|
||||
send_response(c, 0, response);
|
||||
}
|
||||
}
|
||||
|
||||
void packet_server::read_header(std::shared_ptr<connection> c) {
|
||||
c->buffer.resize(sizeof(std::uint32_t));
|
||||
boost::asio::async_read(c->socket, boost::asio::buffer(&c->buffer[0u], c->buffer.size()),
|
||||
[this, c](boost::system::error_code ec, std::size_t) {
|
||||
if (ec) {
|
||||
remove_client(*c);
|
||||
event_system::instance().raise<repertory_exception>(__FUNCTION__,
|
||||
ec.message());
|
||||
} else {
|
||||
auto to_read = *reinterpret_cast<std::uint32_t *>(&c->buffer[0u]);
|
||||
boost::endian::big_to_native_inplace(to_read);
|
||||
read_packet(c, to_read);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
void packet_server::read_packet(std::shared_ptr<connection> c, const std::uint32_t &data_size) {
|
||||
try {
|
||||
const auto read_buffer = [&]() {
|
||||
std::uint32_t offset = 0u;
|
||||
while (offset < c->buffer.size()) {
|
||||
const auto bytes_read = boost::asio::read(
|
||||
c->socket, boost::asio::buffer(&c->buffer[offset], c->buffer.size() - offset));
|
||||
if (bytes_read <= 0) {
|
||||
throw std::runtime_error("read failed: " + std::to_string(bytes_read));
|
||||
}
|
||||
offset += static_cast<std::uint32_t>(bytes_read);
|
||||
}
|
||||
};
|
||||
|
||||
auto should_send_response = true;
|
||||
auto response = std::make_shared<packet>();
|
||||
c->buffer.resize(data_size);
|
||||
read_buffer();
|
||||
|
||||
packet::error_type ret;
|
||||
auto request = std::make_shared<packet>(c->buffer);
|
||||
if (request->decrypt(encryption_token_) == 0) {
|
||||
std::string nonce;
|
||||
if ((ret = request->decode(nonce)) == 0) {
|
||||
if (nonce != c->nonce) {
|
||||
throw std::runtime_error("invalid nonce");
|
||||
}
|
||||
c->generate_nonce();
|
||||
|
||||
std::string version;
|
||||
if ((ret = request->decode(version)) == 0) {
|
||||
if (utils::compare_version_strings(version, MIN_REMOTE_VERSION) >= 0) {
|
||||
std::uint32_t service_flags = 0u;
|
||||
DECODE_OR_IGNORE(request, service_flags);
|
||||
|
||||
std::string client_id;
|
||||
DECODE_OR_IGNORE(request, client_id);
|
||||
|
||||
std::uint64_t thread_id = 0u;
|
||||
DECODE_OR_IGNORE(request, thread_id);
|
||||
|
||||
std::string method;
|
||||
DECODE_OR_IGNORE(request, method);
|
||||
|
||||
if (ret == 0) {
|
||||
if (c->client_id.empty()) {
|
||||
add_client(*c, client_id);
|
||||
}
|
||||
|
||||
should_send_response = false;
|
||||
message_handler_(service_flags, client_id, thread_id, method, request.get(),
|
||||
*response,
|
||||
[this, c, request, response](const packet::error_type &result) {
|
||||
this->send_response(c, result, *response);
|
||||
});
|
||||
}
|
||||
} else {
|
||||
ret = utils::translate_api_error(api_error::incompatible_version);
|
||||
}
|
||||
} else {
|
||||
ret = utils::translate_api_error(api_error::invalid_version);
|
||||
}
|
||||
} else {
|
||||
throw std::runtime_error("invalid nonce");
|
||||
}
|
||||
} else {
|
||||
throw std::runtime_error("decryption failed");
|
||||
}
|
||||
if (should_send_response) {
|
||||
send_response(c, ret, *response);
|
||||
}
|
||||
} catch (const std::exception &e) {
|
||||
remove_client(*c);
|
||||
event_system::instance().raise<repertory_exception>(__FUNCTION__, e.what());
|
||||
}
|
||||
}
|
||||
|
||||
void packet_server::remove_client(connection &c) {
|
||||
if (not c.client_id.empty()) {
|
||||
recur_mutex_lock connection_lock(connection_mutex_);
|
||||
if (not --connection_lookup_[c.client_id]) {
|
||||
connection_lookup_.erase(c.client_id);
|
||||
closed_(c.client_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void packet_server::send_response(std::shared_ptr<connection> c, const packet::error_type &result,
|
||||
packet &response) {
|
||||
response.encode_top(result);
|
||||
response.encode_top(PACKET_SERVICE_FLAGS);
|
||||
response.encode_top(c->nonce);
|
||||
response.encrypt(encryption_token_);
|
||||
response.transfer_into(c->buffer);
|
||||
|
||||
boost::asio::async_write(c->socket, boost::asio::buffer(c->buffer),
|
||||
[this, c](boost::system::error_code ec, std::size_t /*length*/) {
|
||||
if (ec) {
|
||||
remove_client(*c);
|
||||
event_system::instance().raise<repertory_exception>(__FUNCTION__,
|
||||
ec.message());
|
||||
} else {
|
||||
read_header(c);
|
||||
}
|
||||
});
|
||||
}
|
||||
} // namespace repertory
|
||||
29
src/common.cpp.in
Normal file
29
src/common.cpp.in
Normal file
@@ -0,0 +1,29 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "common.hpp"
|
||||
|
||||
const std::string &get_repertory_version() {
|
||||
static const std::string version = "@REPERTORY_VERSION@";
|
||||
return version;
|
||||
}
|
||||
|
||||
const std::string &get_repertory_git_revision() {
|
||||
static const std::string git_revision = "@REPERTORY_GIT_REV@";
|
||||
return git_revision;
|
||||
}
|
||||
350
src/db/directory_db.cpp
Normal file
350
src/db/directory_db.cpp
Normal file
@@ -0,0 +1,350 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "db/directory_db.hpp"
|
||||
#include "utils/path_utils.hpp"
|
||||
|
||||
namespace repertory {
|
||||
void directory_db::directory_tree::add_path(const std::string &api_path,
|
||||
const std::vector<std::string> &files,
|
||||
rocksdb::DB &db) {
|
||||
const auto create_not_found = [&](const auto &create_path) {
|
||||
std::string value;
|
||||
if (not db.Get(rocksdb::ReadOptions(), create_path, &value).ok()) {
|
||||
json directoryData = {
|
||||
{"path", api_path},
|
||||
{"files", files},
|
||||
};
|
||||
db.Put(rocksdb::WriteOptions(), create_path, directoryData.dump());
|
||||
}
|
||||
};
|
||||
|
||||
const auto parts = utils::string::split(api_path, '/', false);
|
||||
std::string previous_directory;
|
||||
for (const auto &directory_part : parts) {
|
||||
if (directory_part.empty()) {
|
||||
sub_directory_lookup_["/"];
|
||||
previous_directory = "/";
|
||||
create_not_found("/");
|
||||
} else {
|
||||
auto &sub_directories = sub_directory_lookup_[previous_directory];
|
||||
if (std::find(sub_directories.begin(), sub_directories.end(), directory_part) ==
|
||||
sub_directories.end()) {
|
||||
sub_directories.emplace_back(directory_part);
|
||||
}
|
||||
previous_directory =
|
||||
utils::path::create_api_path(utils::path::combine(previous_directory, {directory_part}));
|
||||
sub_directory_lookup_[previous_directory];
|
||||
create_not_found(previous_directory);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
std::size_t directory_db::directory_tree::get_count(const std::string &api_path) const {
|
||||
return (sub_directory_lookup_.find(api_path) == sub_directory_lookup_.end())
|
||||
? 0
|
||||
: sub_directory_lookup_.at(api_path).size();
|
||||
}
|
||||
|
||||
std::vector<std::string> directory_db::directory_tree::get_directories() const {
|
||||
std::vector<std::string> ret;
|
||||
std::transform(sub_directory_lookup_.begin(), sub_directory_lookup_.end(),
|
||||
std::back_inserter(ret), [](const auto &kv) { return kv.first; });
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
std::vector<std::string>
|
||||
directory_db::directory_tree::get_sub_directories(const std::string &api_path) const {
|
||||
std::vector<std::string> ret;
|
||||
if (sub_directory_lookup_.find(api_path) != sub_directory_lookup_.end()) {
|
||||
const auto &lookup = sub_directory_lookup_.at(api_path);
|
||||
std::transform(
|
||||
lookup.begin(), lookup.end(), std::back_inserter(ret), [&api_path](const auto &directory) {
|
||||
return utils::path::create_api_path(utils::path::combine(api_path, {directory}));
|
||||
});
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool directory_db::directory_tree::is_directory(const std::string &api_path) const {
|
||||
return sub_directory_lookup_.find(api_path) != sub_directory_lookup_.end();
|
||||
}
|
||||
|
||||
bool directory_db::directory_tree::remove_directory(const std::string &api_path, rocksdb::DB &db,
|
||||
const bool &allow_remove_root) {
|
||||
if ((allow_remove_root || (api_path != "/")) && is_directory(api_path)) {
|
||||
sub_directory_lookup_.erase(api_path);
|
||||
db.Delete(rocksdb::WriteOptions(), api_path);
|
||||
|
||||
const auto parent_api_path = utils::path::get_parent_api_path(api_path);
|
||||
const auto parts = utils::string::split(api_path, '/', false);
|
||||
utils::remove_element_from(sub_directory_lookup_[parent_api_path], parts[parts.size() - 1]);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
directory_db::directory_db(const app_config &config) {
|
||||
utils::db::create_rocksdb(config, DIRDB_NAME, db_);
|
||||
auto iterator = std::unique_ptr<rocksdb::Iterator>(db_->NewIterator(rocksdb::ReadOptions()));
|
||||
for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
|
||||
auto directory_data = json::parse(iterator->value().ToString());
|
||||
tree_.add_path(directory_data["path"].get<std::string>(),
|
||||
directory_data["files"].get<std::vector<std::string>>(), *db_);
|
||||
}
|
||||
}
|
||||
|
||||
directory_db::~directory_db() { db_.reset(); }
|
||||
|
||||
api_error directory_db::create_directory(const std::string &api_path, const bool &create_always) {
|
||||
recur_mutex_lock directory_lock(directory_mutex_);
|
||||
auto ret = api_error::directory_exists;
|
||||
if (not is_directory(api_path)) {
|
||||
ret = api_error::directory_not_found;
|
||||
if (create_always || (api_path == "/") ||
|
||||
is_directory(utils::path::get_parent_api_path(api_path))) {
|
||||
ret = api_error::file_exists;
|
||||
if (not is_file(api_path)) {
|
||||
tree_.add_path(api_path, {}, *db_);
|
||||
ret = api_error::success;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error directory_db::create_file(const std::string &api_path) {
|
||||
recur_mutex_lock directory_lock(directory_mutex_);
|
||||
if (is_directory(api_path)) {
|
||||
return api_error::directory_exists;
|
||||
}
|
||||
|
||||
const auto parent_api_path = utils::path::get_parent_api_path(api_path);
|
||||
auto directory_data = get_directory_data(parent_api_path);
|
||||
if (directory_data.empty()) {
|
||||
return api_error::directory_not_found;
|
||||
}
|
||||
|
||||
const auto file_name = utils::path::strip_to_file_name(api_path);
|
||||
if (utils::collection_includes(directory_data["files"], file_name)) {
|
||||
return api_error::file_exists;
|
||||
}
|
||||
|
||||
directory_data["files"].emplace_back(file_name);
|
||||
db_->Put(rocksdb::WriteOptions(), parent_api_path, directory_data.dump());
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
json directory_db::get_directory_data(const std::string &api_path) const {
|
||||
std::string data;
|
||||
db_->Get(rocksdb::ReadOptions(), api_path, &data);
|
||||
if (data.empty()) {
|
||||
return json();
|
||||
}
|
||||
|
||||
return json::parse(data);
|
||||
}
|
||||
|
||||
std::uint64_t directory_db::get_directory_item_count(const std::string &api_path) const {
|
||||
auto directory_data = get_directory_data(api_path);
|
||||
const auto sub_directory_count = get_sub_directory_count(api_path);
|
||||
const auto file_count = (directory_data.empty() ? 0 : directory_data["files"].size());
|
||||
return sub_directory_count + file_count;
|
||||
}
|
||||
|
||||
api_error directory_db::get_file(const std::string &api_path, api_file &file,
|
||||
api_file_provider_callback api_file_provider) const {
|
||||
const auto parent_api_path = utils::path::get_parent_api_path(api_path);
|
||||
auto directory_data = get_directory_data(parent_api_path);
|
||||
if (not directory_data.empty()) {
|
||||
const auto file_name = utils::path::strip_to_file_name(api_path);
|
||||
if (utils::collection_includes(directory_data["files"], file_name)) {
|
||||
file.api_path =
|
||||
utils::path::create_api_path(utils::path::combine(parent_api_path, {file_name})),
|
||||
api_file_provider(file);
|
||||
return api_error::success;
|
||||
}
|
||||
}
|
||||
|
||||
return api_error::item_not_found;
|
||||
}
|
||||
|
||||
api_error directory_db::get_file_list(api_file_list &list,
|
||||
api_file_provider_callback api_file_provider) const {
|
||||
auto iterator = std::unique_ptr<rocksdb::Iterator>(db_->NewIterator(rocksdb::ReadOptions()));
|
||||
for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
|
||||
auto directory_data = json::parse(iterator->value().ToString());
|
||||
for (const auto &directory_file : directory_data["files"]) {
|
||||
api_file file{
|
||||
utils::path::create_api_path(utils::path::combine(iterator->key().ToString(),
|
||||
{directory_file.get<std::string>()})),
|
||||
};
|
||||
api_file_provider(file);
|
||||
list.emplace_back(file);
|
||||
}
|
||||
}
|
||||
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
std::size_t directory_db::get_sub_directory_count(const std::string &api_path) const {
|
||||
recur_mutex_lock directoryLock(directory_mutex_);
|
||||
return tree_.get_count(api_path);
|
||||
}
|
||||
|
||||
std::uint64_t directory_db::get_total_item_count() const {
|
||||
unique_recur_mutex_lock directory_lock(directory_mutex_);
|
||||
const auto directories = tree_.get_directories();
|
||||
directory_lock.unlock();
|
||||
|
||||
return std::accumulate(directories.begin(), directories.end(), std::uint64_t(directories.size()),
|
||||
[this](std::uint64_t c, const std::string &directory) {
|
||||
const auto dirData = this->get_directory_data(directory);
|
||||
return c + (dirData.empty() ? 0 : dirData["files"].size());
|
||||
});
|
||||
}
|
||||
|
||||
bool directory_db::is_directory(const std::string &api_path) const {
|
||||
recur_mutex_lock directory_lock(directory_mutex_);
|
||||
return tree_.is_directory(api_path);
|
||||
}
|
||||
|
||||
bool directory_db::is_file(const std::string &api_path) const {
|
||||
auto directory_data = get_directory_data(utils::path::get_parent_api_path(api_path));
|
||||
if (directory_data.empty()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const auto file_name = utils::path::strip_to_file_name(api_path);
|
||||
return utils::collection_includes(directory_data["files"], file_name);
|
||||
}
|
||||
|
||||
void directory_db::populate_directory_files(const std::string &api_path,
|
||||
const meta_provider_callback &meta_provider,
|
||||
directory_item_list &list) const {
|
||||
auto directory_data = get_directory_data(api_path);
|
||||
if (not directory_data.empty()) {
|
||||
for (const auto &directory_file : directory_data["files"]) {
|
||||
directory_item di{};
|
||||
di.api_path = utils::path::create_api_path(
|
||||
utils::path::combine(api_path, {directory_file.get<std::string>()}));
|
||||
di.directory = false;
|
||||
meta_provider(di, true);
|
||||
di.size = utils::string::to_uint64(di.meta[META_SIZE]);
|
||||
list.emplace_back(std::move(di));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void directory_db::populate_sub_directories(const std::string &api_path,
|
||||
const meta_provider_callback &meta_provider,
|
||||
directory_item_list &list) const {
|
||||
unique_recur_mutex_lock directory_lock(directory_mutex_);
|
||||
const auto directories = tree_.get_sub_directories(api_path);
|
||||
directory_lock.unlock();
|
||||
|
||||
std::size_t offset = 0u;
|
||||
for (const auto &directory : directories) {
|
||||
if (std::find_if(list.begin(), list.end(), [&directory](const auto &di) -> bool {
|
||||
return directory == di.api_path;
|
||||
}) == list.end()) {
|
||||
directory_item di{};
|
||||
di.api_path = directory;
|
||||
di.api_parent = utils::path::get_parent_api_path(directory);
|
||||
di.directory = true;
|
||||
di.size = get_sub_directory_count(directory);
|
||||
meta_provider(di, true);
|
||||
list.insert(list.begin() + offset++, std::move(di));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
api_error directory_db::remove_directory(const std::string &api_path,
|
||||
const bool &allow_remove_root) {
|
||||
recur_mutex_lock directory_lock(directory_mutex_);
|
||||
if ((api_path == "/") && not allow_remove_root) {
|
||||
return api_error::access_denied;
|
||||
}
|
||||
|
||||
if (is_file(api_path)) {
|
||||
return api_error::item_is_file;
|
||||
}
|
||||
|
||||
if (not is_directory(api_path)) {
|
||||
return api_error::directory_not_found;
|
||||
}
|
||||
|
||||
if (tree_.get_count(api_path) == 0) {
|
||||
auto directory_data = get_directory_data(api_path);
|
||||
if (directory_data.empty() || directory_data["files"].empty()) {
|
||||
tree_.remove_directory(api_path, *db_, allow_remove_root);
|
||||
return api_error::success;
|
||||
}
|
||||
}
|
||||
|
||||
return api_error::directory_not_empty;
|
||||
}
|
||||
|
||||
bool directory_db::remove_file(const std::string &api_path) {
|
||||
recur_mutex_lock directory_lock(directory_mutex_);
|
||||
|
||||
if (is_directory(api_path)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const auto parent_api_path = utils::path::get_parent_api_path(api_path);
|
||||
auto directory_data = get_directory_data(parent_api_path);
|
||||
if (directory_data.empty()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const auto file_name = utils::path::strip_to_file_name(api_path);
|
||||
if (utils::collection_excludes(directory_data["files"], file_name)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
utils::remove_element_from(directory_data["files"], file_name);
|
||||
db_->Put(rocksdb::WriteOptions(), parent_api_path, directory_data.dump());
|
||||
return true;
|
||||
}
|
||||
|
||||
api_error directory_db::rename_file(const std::string &from_api_path,
|
||||
const std::string &to_api_path) {
|
||||
recur_mutex_lock directory_lock(directory_mutex_);
|
||||
|
||||
if (is_directory(from_api_path) || is_directory(to_api_path)) {
|
||||
return api_error::directory_exists;
|
||||
}
|
||||
|
||||
if (not is_directory(utils::path::get_parent_api_path(to_api_path))) {
|
||||
return api_error::directory_not_found;
|
||||
}
|
||||
|
||||
if (is_file(to_api_path)) {
|
||||
return api_error::file_exists;
|
||||
}
|
||||
|
||||
if (not remove_file(from_api_path)) {
|
||||
return api_error::item_not_found;
|
||||
}
|
||||
|
||||
return create_file(to_api_path);
|
||||
}
|
||||
} // namespace repertory
|
||||
251
src/db/meta_db.cpp
Normal file
251
src/db/meta_db.cpp
Normal file
@@ -0,0 +1,251 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "db/meta_db.hpp"
|
||||
#include "types/repertory.hpp"
|
||||
#include "types/startup_exception.hpp"
|
||||
#include "utils/file_utils.hpp"
|
||||
#include "utils/path_utils.hpp"
|
||||
#include "utils/utils.hpp"
|
||||
|
||||
namespace repertory {
|
||||
meta_db::meta_db(const app_config &config) {
|
||||
const auto create_resources = [this, &config](const std::string &name) {
|
||||
auto families = std::vector<rocksdb::ColumnFamilyDescriptor>();
|
||||
families.emplace_back(rocksdb::ColumnFamilyDescriptor{rocksdb::kDefaultColumnFamilyName,
|
||||
rocksdb::ColumnFamilyOptions()});
|
||||
families.emplace_back(
|
||||
rocksdb::ColumnFamilyDescriptor{"source", rocksdb::ColumnFamilyOptions()});
|
||||
families.emplace_back(rocksdb::ColumnFamilyDescriptor{"keys", rocksdb::ColumnFamilyOptions()});
|
||||
|
||||
auto handles = std::vector<rocksdb::ColumnFamilyHandle *>();
|
||||
utils::db::create_rocksdb(config, name, families, handles, db_);
|
||||
default_family_.reset(handles[0u]);
|
||||
source_family_.reset(handles[1u]);
|
||||
keys_family_.reset(handles[2u]);
|
||||
};
|
||||
create_resources(METADB_NAME);
|
||||
}
|
||||
|
||||
meta_db::~meta_db() { release_resources(); }
|
||||
|
||||
void meta_db::release_resources() {
|
||||
default_family_.reset();
|
||||
source_family_.reset();
|
||||
keys_family_.reset();
|
||||
db_.reset();
|
||||
}
|
||||
|
||||
std::shared_ptr<rocksdb::Iterator> meta_db::create_iterator(const bool &source_family) {
|
||||
return std::shared_ptr<rocksdb::Iterator>(db_->NewIterator(
|
||||
rocksdb::ReadOptions(), source_family ? source_family_.get() : default_family_.get()));
|
||||
}
|
||||
|
||||
api_error meta_db::get_api_path_from_key(const std::string &key, std::string &api_path) const {
|
||||
auto ret = api_error::item_not_found;
|
||||
if (not key.empty()) {
|
||||
if (db_->Get(rocksdb::ReadOptions(), keys_family_.get(), key, &api_path).ok()) {
|
||||
ret = api_error::success;
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error meta_db::get_api_path_from_source(const std::string &source_path,
|
||||
std::string &api_path) const {
|
||||
auto ret = api_error::item_not_found;
|
||||
if (not source_path.empty()) {
|
||||
if (db_->Get(rocksdb::ReadOptions(), source_family_.get(), source_path, &api_path).ok()) {
|
||||
ret = api_error::success;
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error meta_db::get_item_meta_json(const std::string &api_path, json &json_data) const {
|
||||
auto ret = api_error::item_not_found;
|
||||
|
||||
std::string value;
|
||||
if (db_->Get(rocksdb::ReadOptions(), default_family_.get(), api_path, &value).ok()) {
|
||||
json_data = json::parse(value);
|
||||
ret = api_error::success;
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error meta_db::get_item_meta(const std::string &api_path, api_meta_map &meta) const {
|
||||
json json_data;
|
||||
const auto ret = get_item_meta_json(api_path, json_data);
|
||||
if (ret == api_error::success) {
|
||||
for (auto it = json_data.begin(); it != json_data.end(); it++) {
|
||||
meta.insert({it.key(), it.value().get<std::string>()});
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error meta_db::get_item_meta(const std::string &api_path, const std::string &key,
|
||||
std::string &value) const {
|
||||
json json_data;
|
||||
const auto ret = get_item_meta_json(api_path, json_data);
|
||||
if (ret == api_error::success) {
|
||||
if (json_data.find(key) != json_data.end()) {
|
||||
value = json_data[key].get<std::string>();
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool meta_db::get_item_meta_exists(const std::string &api_path) const {
|
||||
std::string value;
|
||||
return db_->Get(rocksdb::ReadOptions(), api_path, &value).ok();
|
||||
}
|
||||
|
||||
std::vector<std::string> meta_db::get_pinned_files() const {
|
||||
std::vector<std::string> ret;
|
||||
|
||||
auto iterator = const_cast<meta_db *>(this)->create_iterator(false);
|
||||
for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
|
||||
auto api_path = iterator->key().ToString();
|
||||
std::string pinned;
|
||||
get_item_meta(api_path, META_PINNED, pinned);
|
||||
if (not pinned.empty() && utils::string::to_bool(pinned)) {
|
||||
ret.emplace_back(api_path);
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool meta_db::get_source_path_exists(const std::string &sourceFilePath) const {
|
||||
std::string value;
|
||||
return db_->Get(rocksdb::ReadOptions(), source_family_.get(), sourceFilePath, &value).ok();
|
||||
}
|
||||
|
||||
void meta_db::remove_item_meta(const std::string &api_path) {
|
||||
std::string source;
|
||||
get_item_meta(api_path, META_SOURCE, source);
|
||||
|
||||
std::string key;
|
||||
get_item_meta(api_path, META_SOURCE, key);
|
||||
|
||||
db_->Delete(rocksdb::WriteOptions(), source_family_.get(), source);
|
||||
db_->Delete(rocksdb::WriteOptions(), default_family_.get(), api_path);
|
||||
db_->Delete(rocksdb::WriteOptions(), keys_family_.get(), key);
|
||||
}
|
||||
|
||||
api_error meta_db::remove_item_meta(const std::string &api_path, const std::string &key) {
|
||||
json json_data;
|
||||
auto ret = get_item_meta_json(api_path, json_data);
|
||||
|
||||
if (ret == api_error::success) {
|
||||
if ((key == META_KEY) && not json_data[META_KEY].empty()) {
|
||||
db_->Delete(rocksdb::WriteOptions(), keys_family_.get(),
|
||||
json_data[META_KEY].get<std::string>());
|
||||
}
|
||||
|
||||
json_data.erase(key);
|
||||
ret = db_->Put(rocksdb::WriteOptions(), default_family_.get(), api_path, json_data.dump()).ok()
|
||||
? api_error::success
|
||||
: api_error::error;
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error meta_db::rename_item_meta(const std::string &source_path,
|
||||
const std::string &from_api_path,
|
||||
const std::string &to_api_path) {
|
||||
auto ret = api_error::success;
|
||||
|
||||
std::string key;
|
||||
get_item_meta(from_api_path, META_KEY, key);
|
||||
|
||||
if (not key.empty()) {
|
||||
remove_item_meta(from_api_path, META_KEY);
|
||||
}
|
||||
|
||||
std::string value;
|
||||
if (db_->Get(rocksdb::ReadOptions(), default_family_.get(), from_api_path, &value).ok()) {
|
||||
db_->Delete(rocksdb::WriteOptions(), default_family_.get(), from_api_path);
|
||||
db_->Put(rocksdb::WriteOptions(), default_family_.get(), to_api_path, value);
|
||||
}
|
||||
|
||||
if (not key.empty()) {
|
||||
set_item_meta(to_api_path, META_KEY, key);
|
||||
}
|
||||
|
||||
db_->Put(rocksdb::WriteOptions(), source_family_.get(), source_path, to_api_path);
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error meta_db::set_item_meta(const std::string &api_path, const std::string &key,
|
||||
const std::string &value) {
|
||||
if (key == META_KEY) {
|
||||
remove_item_meta(api_path, META_KEY);
|
||||
}
|
||||
|
||||
json json_data;
|
||||
get_item_meta_json(api_path, json_data);
|
||||
|
||||
json_data[key] = value;
|
||||
|
||||
auto ret =
|
||||
db_->Put(rocksdb::WriteOptions(), default_family_.get(), api_path, json_data.dump()).ok()
|
||||
? api_error::success
|
||||
: api_error::error;
|
||||
if ((key == META_KEY) && (ret == api_error::success)) {
|
||||
ret = db_->Put(rocksdb::WriteOptions(), keys_family_.get(), value, api_path).ok()
|
||||
? api_error::success
|
||||
: api_error::error;
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error meta_db::set_item_meta(const std::string &api_path, const api_meta_map &meta) {
|
||||
auto ret = api_error::success;
|
||||
auto it = meta.begin();
|
||||
for (std::size_t i = 0u; (ret == api_error::success) && (i < meta.size()); i++) {
|
||||
ret = set_item_meta(api_path, it->first, it->second);
|
||||
it++;
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error meta_db::set_source_path(const std::string &api_path, const std::string &source_path) {
|
||||
std::string current_source_path;
|
||||
get_item_meta(api_path, META_SOURCE, current_source_path);
|
||||
if (not current_source_path.empty()) {
|
||||
db_->Delete(rocksdb::WriteOptions(), source_family_.get(), current_source_path);
|
||||
}
|
||||
|
||||
auto ret = set_item_meta(api_path, META_SOURCE, source_path);
|
||||
if (ret == api_error::success) {
|
||||
db_->Put(rocksdb::WriteOptions(), source_family_.get(), source_path, api_path);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
} // namespace repertory
|
||||
81
src/db/retry_db.cpp
Normal file
81
src/db/retry_db.cpp
Normal file
@@ -0,0 +1,81 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "db/retry_db.hpp"
|
||||
|
||||
namespace repertory {
|
||||
retry_db::retry_db(const app_config &config) {
|
||||
utils::db::create_rocksdb(config, ROCKS_DB_NAME, db_);
|
||||
}
|
||||
|
||||
retry_db::~retry_db() { db_.reset(); }
|
||||
|
||||
bool retry_db::exists(const std::string &api_path) const {
|
||||
std::string value;
|
||||
return db_->Get(rocksdb::ReadOptions(), api_path, &value).ok();
|
||||
}
|
||||
|
||||
void retry_db::pause() {
|
||||
mutex_lock processing_lock(processing_mutex_);
|
||||
paused_ = true;
|
||||
}
|
||||
|
||||
bool retry_db::process_all(const process_callback &process) {
|
||||
auto processed = false;
|
||||
if (not paused_) {
|
||||
unique_mutex_lock processing_lock(processing_mutex_);
|
||||
if (not paused_) {
|
||||
auto iterator = std::unique_ptr<rocksdb::Iterator>(db_->NewIterator(rocksdb::ReadOptions()));
|
||||
for (iterator->SeekToFirst(); not paused_ && iterator->Valid(); iterator->Next()) {
|
||||
const auto api_path = iterator->value().ToString();
|
||||
if (process(api_path)) {
|
||||
remove(api_path);
|
||||
processed = true;
|
||||
}
|
||||
|
||||
processing_lock.unlock();
|
||||
std::this_thread::sleep_for(1ms);
|
||||
processing_lock.lock();
|
||||
}
|
||||
}
|
||||
processing_lock.unlock();
|
||||
}
|
||||
|
||||
return processed;
|
||||
}
|
||||
|
||||
void retry_db::remove(const std::string &api_path) {
|
||||
db_->Delete(rocksdb::WriteOptions(), api_path);
|
||||
}
|
||||
|
||||
void retry_db::rename(const std::string &from_api_path, const std::string &to_api_path) {
|
||||
if (exists(from_api_path)) {
|
||||
remove(from_api_path);
|
||||
set(to_api_path);
|
||||
}
|
||||
}
|
||||
|
||||
void retry_db::resume() {
|
||||
mutex_lock processing_lock(processing_mutex_);
|
||||
paused_ = false;
|
||||
}
|
||||
|
||||
void retry_db::set(const std::string &api_path) {
|
||||
db_->Put(rocksdb::WriteOptions(), api_path, api_path);
|
||||
}
|
||||
} // namespace repertory
|
||||
204
src/download/buffered_reader.cpp
Normal file
204
src/download/buffered_reader.cpp
Normal file
@@ -0,0 +1,204 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "download/buffered_reader.hpp"
|
||||
#include "app_config.hpp"
|
||||
#include "download/reader_pool.hpp"
|
||||
|
||||
namespace repertory {
|
||||
buffered_reader::buffered_reader(const app_config &config, const filesystem_item &fsi,
|
||||
const api_reader_callback &api_reader,
|
||||
const std::size_t &chunk_size, const std::size_t &total_chunks,
|
||||
const std::size_t &start_chunk)
|
||||
: fsi_(fsi),
|
||||
api_reader_(api_reader),
|
||||
chunk_size_(chunk_size),
|
||||
total_chunks_(total_chunks),
|
||||
ring_state_(config.get_read_ahead_count()) {
|
||||
ring_data_.resize(ring_state_.size());
|
||||
const auto read_chunk = [this](const std::size_t &read_offset,
|
||||
std::unique_ptr<std::vector<char>> &chunk_ptr) -> api_error {
|
||||
auto ret = api_error::success;
|
||||
const auto read_size = utils::calculate_read_size(fsi_.size, chunk_size_, read_offset);
|
||||
if (read_size) {
|
||||
std::vector<char> data;
|
||||
if ((ret = api_reader_(fsi_.api_path, read_size, read_offset + read_offset_, data,
|
||||
stop_requested_)) == api_error::success) {
|
||||
chunk_ptr = std::make_unique<std::vector<char>>(std::move(data));
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
};
|
||||
|
||||
auto first = std::async(std::launch::async, [this, &read_chunk]() -> api_error {
|
||||
return read_chunk(0u, first_chunk_data_);
|
||||
});
|
||||
|
||||
if (total_chunks_ > 1u) {
|
||||
auto last = std::async(std::launch::async, [this, &read_chunk]() -> api_error {
|
||||
return read_chunk(((total_chunks_ - 1u) * chunk_size_), last_chunk_data_);
|
||||
});
|
||||
error_ = last.get();
|
||||
}
|
||||
|
||||
if (error_ == api_error::success) {
|
||||
if ((error_ = first.get()) == api_error::success) {
|
||||
read_chunk_index_ = write_chunk_index_ =
|
||||
(first_chunk_data_ ? ((start_chunk == 0u) ? 1u : start_chunk) : start_chunk);
|
||||
if (ring_data_.size() > 1u) {
|
||||
reader_thread_ = std::make_unique<std::thread>([this]() { this->reader_thread(); });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
buffered_reader::~buffered_reader() {
|
||||
notify_stop_requested();
|
||||
|
||||
unique_mutex_lock write_lock(write_mutex_);
|
||||
read_notify_.notify_all();
|
||||
write_lock.unlock();
|
||||
|
||||
if (reader_thread_) {
|
||||
reader_thread_->join();
|
||||
reader_thread_.reset();
|
||||
}
|
||||
}
|
||||
|
||||
void buffered_reader::notify_stop_requested() {
|
||||
error_ = api_error::download_stopped;
|
||||
stop_requested_ = true;
|
||||
}
|
||||
|
||||
api_error buffered_reader::read_chunk(const std::size_t &chunk_index, std::vector<char> &data) {
|
||||
if (error_ == api_error::success) {
|
||||
data.clear();
|
||||
|
||||
if (last_chunk_data_ && (chunk_index == (total_chunks_ - 1u))) {
|
||||
data = *last_chunk_data_;
|
||||
} else if (first_chunk_data_ && (chunk_index == 0u)) {
|
||||
data = *first_chunk_data_;
|
||||
} else if (ring_state_.size() == 1u) {
|
||||
const auto read_offset = chunk_index * chunk_size_;
|
||||
std::size_t read_size = 0u;
|
||||
if ((read_size = utils::calculate_read_size(fsi_.size, chunk_size_, read_offset)) > 0u) {
|
||||
error_ = api_reader_(fsi_.api_path, read_size, read_offset + read_offset_, data,
|
||||
stop_requested_);
|
||||
}
|
||||
} else {
|
||||
mutex_lock read_lock(read_mutex_);
|
||||
const auto read_position = chunk_index % ring_state_.size();
|
||||
unique_mutex_lock write_lock(write_mutex_);
|
||||
while ((reset_reader_ ||
|
||||
((chunk_index < read_chunk_index_) || (chunk_index > write_chunk_index_))) &&
|
||||
is_active()) {
|
||||
reset_reader_ = true;
|
||||
read_chunk_index_ = write_chunk_index_ = chunk_index;
|
||||
read_notify_.notify_all();
|
||||
read_notify_.wait(write_lock);
|
||||
}
|
||||
|
||||
if (is_active()) {
|
||||
read_chunk_index_ = chunk_index;
|
||||
read_notify_.notify_all();
|
||||
while (not ring_state_[read_position] && is_active()) {
|
||||
read_notify_.wait(write_lock);
|
||||
}
|
||||
data = ring_data_[read_position];
|
||||
}
|
||||
|
||||
read_notify_.notify_all();
|
||||
write_lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
return error_;
|
||||
}
|
||||
|
||||
void buffered_reader::reader_thread() {
|
||||
reader_pool pool(ring_state_.size(), api_reader_);
|
||||
|
||||
unique_mutex_lock write_lock(write_mutex_);
|
||||
read_notify_.notify_all();
|
||||
write_lock.unlock();
|
||||
|
||||
while (is_active()) {
|
||||
write_lock.lock();
|
||||
while ((write_chunk_index_ > read_chunk_index_) &&
|
||||
(write_chunk_index_ > (read_chunk_index_ + ring_state_.size() - 1u)) &&
|
||||
not reset_reader_ && is_active()) {
|
||||
read_notify_.wait(write_lock);
|
||||
}
|
||||
|
||||
if (is_active()) {
|
||||
if (reset_reader_) {
|
||||
read_notify_.notify_all();
|
||||
write_lock.unlock();
|
||||
|
||||
pool.restart();
|
||||
|
||||
write_lock.lock();
|
||||
write_chunk_index_ = read_chunk_index_;
|
||||
ring_state_.reset();
|
||||
reset_reader_ = false;
|
||||
}
|
||||
|
||||
const auto file_offset = write_chunk_index_ * chunk_size_;
|
||||
const auto write_chunk_index = write_chunk_index_;
|
||||
const auto write_position = write_chunk_index_ % ring_state_.size();
|
||||
ring_state_[write_position] = false;
|
||||
read_notify_.notify_all();
|
||||
write_lock.unlock();
|
||||
|
||||
const auto read_size = utils::calculate_read_size(fsi_.size, chunk_size_, file_offset);
|
||||
if (read_size > 0u) {
|
||||
if ((first_chunk_data_ && (write_chunk_index == 0u)) ||
|
||||
(last_chunk_data_ && (write_chunk_index == (total_chunks_ - 1u)))) {
|
||||
write_lock.lock();
|
||||
write_chunk_index_++;
|
||||
} else {
|
||||
const auto completed = [this, write_position](api_error error) {
|
||||
mutex_lock write_lock(write_mutex_);
|
||||
if (error == api_error::success) {
|
||||
ring_state_[write_position] = true;
|
||||
} else if (not reset_reader_) {
|
||||
error_ = error;
|
||||
}
|
||||
read_notify_.notify_all();
|
||||
};
|
||||
pool.queue_read_bytes(fsi_.api_path, read_size,
|
||||
((write_chunk_index * chunk_size_) + read_offset_),
|
||||
ring_data_[write_position], completed);
|
||||
|
||||
write_lock.lock();
|
||||
write_chunk_index_++;
|
||||
}
|
||||
} else {
|
||||
write_lock.lock();
|
||||
while (not reset_reader_ && ((write_chunk_index_ * chunk_size_) >= fsi_.size) &&
|
||||
is_active()) {
|
||||
read_notify_.wait(write_lock);
|
||||
}
|
||||
}
|
||||
}
|
||||
read_notify_.notify_all();
|
||||
write_lock.unlock();
|
||||
}
|
||||
}
|
||||
} // namespace repertory
|
||||
116
src/download/direct_download.cpp
Normal file
116
src/download/direct_download.cpp
Normal file
@@ -0,0 +1,116 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "download/direct_download.hpp"
|
||||
#include "app_config.hpp"
|
||||
#include "download/buffered_reader.hpp"
|
||||
#include "download/events.hpp"
|
||||
#include "download/utils.hpp"
|
||||
#include "events/event_system.hpp"
|
||||
#include "utils/encrypting_reader.hpp"
|
||||
|
||||
namespace repertory {
|
||||
direct_download::direct_download(const app_config &config, filesystem_item fsi,
|
||||
const api_reader_callback &api_reader, const std::uint64_t &handle)
|
||||
: config_(config), fsi_(std::move(fsi)), api_reader_(api_reader), handle_(handle) {}
|
||||
|
||||
direct_download::~direct_download() {
|
||||
stop_requested_ = true;
|
||||
|
||||
unique_mutex_lock read_lock(read_mutex_);
|
||||
if (buffered_reader_) {
|
||||
buffered_reader_->notify_stop_requested();
|
||||
}
|
||||
read_lock.unlock();
|
||||
|
||||
if (buffered_reader_) {
|
||||
buffered_reader_.reset();
|
||||
}
|
||||
}
|
||||
|
||||
void direct_download::notify_download_end() {
|
||||
unique_mutex_lock read_lock(read_mutex_);
|
||||
if (not disable_download_end_ && not download_end_notified_) {
|
||||
download_end_notified_ = true;
|
||||
read_lock.unlock();
|
||||
event_system::instance().raise<download_end>(fsi_.api_path, "direct", handle_, error_);
|
||||
}
|
||||
}
|
||||
|
||||
void direct_download::notify_stop_requested() {
|
||||
set_api_error(api_error::download_stopped);
|
||||
stop_requested_ = true;
|
||||
notify_download_end();
|
||||
}
|
||||
|
||||
api_error direct_download::read_bytes(const std::uint64_t &, std::size_t read_size,
|
||||
const std::uint64_t &read_offset, std::vector<char> &data) {
|
||||
data.clear();
|
||||
|
||||
if ((read_size = utils::calculate_read_size(fsi_.size, read_size, read_offset)) > 0u) {
|
||||
unique_mutex_lock read_lock(read_mutex_);
|
||||
if (is_active()) {
|
||||
if (not buffered_reader_) {
|
||||
const auto chunk_size = utils::encryption::encrypting_reader::get_data_chunk_size();
|
||||
const auto read_chunk = static_cast<std::size_t>(read_offset / chunk_size);
|
||||
event_system::instance().raise<download_begin>(fsi_.api_path, "direct");
|
||||
buffered_reader_ = std::make_unique<buffered_reader>(
|
||||
config_, fsi_, api_reader_, chunk_size,
|
||||
static_cast<std::size_t>(
|
||||
utils::divide_with_ceiling(fsi_.size, static_cast<std::uint64_t>(chunk_size))),
|
||||
read_chunk);
|
||||
}
|
||||
|
||||
auto read_chunk = static_cast<std::size_t>(read_offset / buffered_reader_->get_chunk_size());
|
||||
auto offset = static_cast<std::size_t>(read_offset % buffered_reader_->get_chunk_size());
|
||||
while (is_active() && (read_size > 0u)) {
|
||||
std::vector<char> buffer;
|
||||
auto ret = api_error::success;
|
||||
if ((ret = buffered_reader_->read_chunk(read_chunk, buffer)) == api_error::success) {
|
||||
const auto total_read = std::min(buffered_reader_->get_chunk_size() - offset, read_size);
|
||||
data.insert(data.end(), buffer.begin() + offset, buffer.begin() + offset + total_read);
|
||||
buffer.clear();
|
||||
read_chunk++;
|
||||
offset = 0u;
|
||||
read_size -= total_read;
|
||||
} else {
|
||||
set_api_error(ret);
|
||||
|
||||
read_lock.unlock();
|
||||
notify_download_end();
|
||||
read_lock.lock();
|
||||
}
|
||||
}
|
||||
|
||||
if (is_active()) {
|
||||
utils::download::notify_progress<download_progress>(
|
||||
config_, fsi_.api_path, "direct", static_cast<double>(data.size() + read_offset),
|
||||
static_cast<double>(fsi_.size), progress_);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return error_;
|
||||
}
|
||||
|
||||
void direct_download::set_api_error(const api_error &error) {
|
||||
if ((error_ == api_error::success) && (error_ != error)) {
|
||||
error_ = error;
|
||||
}
|
||||
}
|
||||
} // namespace repertory
|
||||
855
src/download/download.cpp
Normal file
855
src/download/download.cpp
Normal file
@@ -0,0 +1,855 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "download/download.hpp"
|
||||
#include "download/events.hpp"
|
||||
#include "download/utils.hpp"
|
||||
#include "drives/i_open_file_table.hpp"
|
||||
#include "events/event_system.hpp"
|
||||
#include "providers/i_provider.hpp"
|
||||
#include "utils/file_utils.hpp"
|
||||
#include "utils/global_data.hpp"
|
||||
#include "utils/path_utils.hpp"
|
||||
#include "utils/utils.hpp"
|
||||
|
||||
namespace repertory {
|
||||
download::download(const app_config &config, filesystem_item &fsi,
|
||||
const api_reader_callback &api_reader, const std::size_t &chunk_size,
|
||||
i_open_file_table &oft)
|
||||
: config_(config),
|
||||
fsi_(fsi),
|
||||
api_reader_(api_reader),
|
||||
oft_(oft),
|
||||
chunk_size_(chunk_size),
|
||||
read_chunk_state_(static_cast<std::size_t>(
|
||||
utils::divide_with_ceiling(fsi.size, static_cast<std::uint64_t>(chunk_size)))),
|
||||
last_chunk_size_(static_cast<std::size_t>(fsi.size <= chunk_size ? fsi.size
|
||||
: fsi.size % chunk_size ? fsi.size % chunk_size
|
||||
: chunk_size)),
|
||||
write_chunk_state_(static_cast<std::size_t>(
|
||||
utils::divide_with_ceiling(fsi.size, static_cast<std::uint64_t>(chunk_size)))) {
|
||||
/* std::cout << "size:" << fsi.size << ":last:" << last_chunk_size_ << std::endl; */
|
||||
initialize_download(fsi, true);
|
||||
}
|
||||
|
||||
download::download(const app_config &config, filesystem_item &fsi,
|
||||
const api_reader_callback &api_reader, const std::size_t &chunk_size,
|
||||
std::size_t &last_chunk_size, boost::dynamic_bitset<> &read_state,
|
||||
boost::dynamic_bitset<> &write_state, i_open_file_table &oft)
|
||||
: config_(config),
|
||||
fsi_(fsi),
|
||||
api_reader_(api_reader),
|
||||
oft_(oft),
|
||||
chunk_size_(chunk_size),
|
||||
read_chunk_state_(read_state),
|
||||
last_chunk_size_(last_chunk_size),
|
||||
write_chunk_state_(write_state) {
|
||||
initialize_download(fsi, false);
|
||||
}
|
||||
|
||||
download::~download() {
|
||||
if (not get_complete()) {
|
||||
notify_stop_requested();
|
||||
}
|
||||
|
||||
if (io_thread_) {
|
||||
io_thread_->join();
|
||||
}
|
||||
|
||||
if (auto_close_) {
|
||||
native_file::attach(fsi_.handle)->close();
|
||||
fsi_.handle = REPERTORY_INVALID_HANDLE;
|
||||
}
|
||||
}
|
||||
|
||||
api_error download::allocate(const std::uint64_t &, const std::uint64_t &size,
|
||||
const allocator_callback &allocator,
|
||||
const completer_callback &completer) {
|
||||
reset_timeout(false);
|
||||
/* std::cout << "allocate called" << std::endl; */
|
||||
|
||||
if (size == fsi_.size) {
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
if (size > fsi_.size) {
|
||||
download_chunk(read_chunk_state_.size() - 1u, false);
|
||||
} else {
|
||||
download_chunk(static_cast<std::size_t>(
|
||||
utils::divide_with_ceiling(size, static_cast<std::uint64_t>(chunk_size_))),
|
||||
false);
|
||||
}
|
||||
|
||||
if (error_ == api_error::success) {
|
||||
const auto original_size = fsi_.size;
|
||||
unique_mutex_lock read_write_lock(read_write_mutex_);
|
||||
const auto allocate_local_file = [&]() -> api_error {
|
||||
if ((error_ = allocator()) == api_error::success) {
|
||||
utils::file::get_file_size(fsi_.source_path, fsi_.size);
|
||||
completer(original_size, fsi_.size, true);
|
||||
oft_.force_schedule_upload(fsi_);
|
||||
}
|
||||
read_write_notify_.notify_all();
|
||||
read_write_lock.unlock();
|
||||
|
||||
return error_;
|
||||
};
|
||||
|
||||
if (processed_) {
|
||||
return allocate_local_file();
|
||||
}
|
||||
|
||||
read_write_notify_.notify_all();
|
||||
read_write_lock.unlock();
|
||||
|
||||
pause();
|
||||
|
||||
read_write_lock.lock();
|
||||
if (processed_) {
|
||||
/* std::cout << "recursive allocate" << std::endl; */
|
||||
return allocate_local_file();
|
||||
}
|
||||
|
||||
if ((error_ = allocator()) == api_error::success) {
|
||||
utils::file::get_file_size(fsi_.source_path, fsi_.size);
|
||||
|
||||
const auto end_chunk_index = static_cast<std::size_t>(
|
||||
utils::divide_with_ceiling(fsi_.size, static_cast<std::uint64_t>(chunk_size_)));
|
||||
if (end_chunk_index >= read_chunk_state_.size()) {
|
||||
write_chunk_state_.resize(end_chunk_index + 1u);
|
||||
read_chunk_state_.resize(end_chunk_index + 1u);
|
||||
}
|
||||
|
||||
last_chunk_size_ =
|
||||
static_cast<std::size_t>(fsi_.size <= chunk_size_ ? fsi_.size
|
||||
: fsi_.size % chunk_size_ ? fsi_.size % chunk_size_
|
||||
: chunk_size_);
|
||||
|
||||
read_chunk_state_[end_chunk_index] = write_chunk_state_[end_chunk_index] = true;
|
||||
completer(original_size, fsi_.size, true);
|
||||
}
|
||||
|
||||
read_write_notify_.notify_all();
|
||||
read_write_lock.unlock();
|
||||
|
||||
resume();
|
||||
}
|
||||
|
||||
return error_;
|
||||
}
|
||||
|
||||
void download::create_active_chunk(std::size_t chunk_index) {
|
||||
auto reader = [this, chunk_index]() {
|
||||
const auto chunk_read_size =
|
||||
(chunk_index == (read_chunk_state_.size() - 1u)) ? last_chunk_size_ : chunk_size_;
|
||||
const auto chunk_read_offset = chunk_index * chunk_size_;
|
||||
|
||||
api_error ret;
|
||||
if (not get_complete()) {
|
||||
std::vector<char> buffer;
|
||||
if ((ret = api_reader_(fsi_.api_path, chunk_read_size, chunk_read_offset + read_offset_,
|
||||
buffer, stop_requested_)) == api_error::success) {
|
||||
unique_mutex_lock read_write_lock(read_write_mutex_);
|
||||
if (not get_complete()) {
|
||||
write_queue_.emplace_back(
|
||||
std::make_shared<write_data>(chunk_index, std::move(buffer), chunk_read_offset));
|
||||
}
|
||||
read_write_notify_.notify_all();
|
||||
read_write_lock.unlock();
|
||||
}
|
||||
|
||||
if (ret != api_error::success) {
|
||||
error_ = ((error_ == api_error::success) ? ret : error_);
|
||||
|
||||
mutex_lock read_write_lock(read_write_mutex_);
|
||||
read_write_notify_.notify_all();
|
||||
}
|
||||
}
|
||||
};
|
||||
active_chunks_.insert({chunk_index, std::make_shared<active_chunk>(std::thread(reader))});
|
||||
}
|
||||
|
||||
void download::download_chunk(std::size_t chunk_index, bool inactive_only) {
|
||||
unique_mutex_lock read_write_lock(read_write_mutex_);
|
||||
|
||||
const auto should_download = [&]() -> bool {
|
||||
return (chunk_index < read_chunk_state_.size()) && not read_chunk_state_[chunk_index] &&
|
||||
not get_complete();
|
||||
};
|
||||
|
||||
if (should_download()) {
|
||||
while (paused_ && not get_complete()) {
|
||||
read_write_notify_.notify_all();
|
||||
read_write_notify_.wait(read_write_lock);
|
||||
}
|
||||
|
||||
if (should_download()) {
|
||||
auto created = false;
|
||||
if (active_chunks_.find(chunk_index) == active_chunks_.end()) {
|
||||
create_active_chunk(chunk_index);
|
||||
created = true;
|
||||
}
|
||||
|
||||
if (not inactive_only || created) {
|
||||
auto chunk = active_chunks_[chunk_index];
|
||||
read_write_notify_.notify_all();
|
||||
read_write_lock.unlock();
|
||||
|
||||
if (not inactive_only) {
|
||||
reset_timeout(false);
|
||||
}
|
||||
|
||||
utils::spin_wait_for_mutex(
|
||||
[this, chunk_index]() -> bool {
|
||||
return read_chunk_state_[chunk_index] || get_complete();
|
||||
},
|
||||
chunk->notify, chunk->mutex, "download_chunk");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
api_error download::download_all() {
|
||||
const auto total_chunks = read_chunk_state_.size();
|
||||
for (std::size_t chunk = 0u; not get_complete() && (chunk < total_chunks); chunk++) {
|
||||
download_chunk(chunk, false);
|
||||
}
|
||||
utils::spin_wait_for_mutex(processed_, processed_notify_, read_write_mutex_, "download_all");
|
||||
return error_;
|
||||
}
|
||||
|
||||
bool download::get_complete() const {
|
||||
return read_chunk_state_.all() || (error_ != api_error::success) || stop_requested_ || processed_;
|
||||
}
|
||||
|
||||
bool download::get_timeout_enabled() const {
|
||||
return not paused_ && not write_chunk_state_.any() &&
|
||||
config_.get_enable_chunk_download_timeout() && (oft_.get_open_count(fsi_.api_path) == 0u);
|
||||
}
|
||||
|
||||
void download::get_state_information(filesystem_item &fsi, std::size_t &chunk_size,
|
||||
std::size_t &last_chunk_size,
|
||||
boost::dynamic_bitset<> &read_state,
|
||||
boost::dynamic_bitset<> &write_state) {
|
||||
fsi = fsi_;
|
||||
chunk_size = chunk_size_;
|
||||
last_chunk_size = last_chunk_size_;
|
||||
read_state = read_chunk_state_;
|
||||
write_state = write_chunk_state_;
|
||||
}
|
||||
|
||||
void download::handle_active_chunk_complete(std::size_t chunk_index, unique_mutex_lock &lock) {
|
||||
lock.lock();
|
||||
auto activeChunk = active_chunks_[chunk_index];
|
||||
active_chunks_.erase(chunk_index);
|
||||
lock.unlock();
|
||||
|
||||
unique_mutex_lock completed_lock(activeChunk->mutex);
|
||||
activeChunk->notify.notify_all();
|
||||
completed_lock.unlock();
|
||||
|
||||
activeChunk->worker.join();
|
||||
}
|
||||
|
||||
void download::initialize_download(filesystem_item &fsi, const bool &delete_existing) {
|
||||
const auto current_source_path = fsi.source_path;
|
||||
|
||||
if (delete_existing) {
|
||||
// Handle should always be invalid (just in case it's not - no leaking).
|
||||
unique_recur_mutex_lock item_lock(*fsi.lock);
|
||||
if (fsi.handle != REPERTORY_INVALID_HANDLE) {
|
||||
native_file::attach(fsi.handle)->close();
|
||||
fsi.handle = REPERTORY_INVALID_HANDLE;
|
||||
}
|
||||
item_lock.unlock();
|
||||
|
||||
// Update current open file information to non-existing source file and free open handle.
|
||||
fsi.source_path =
|
||||
utils::path::combine(config_.get_cache_directory(), {utils::create_uuid_string()});
|
||||
fsi.source_path_changed = true;
|
||||
|
||||
// Update cache space if existing file size is >0 and delete
|
||||
std::uint64_t file_size = 0u;
|
||||
utils::file::get_file_size(current_source_path, file_size);
|
||||
if (file_size) {
|
||||
global_data::instance().update_used_space(file_size, 0u, true);
|
||||
}
|
||||
|
||||
utils::file::delete_file(current_source_path);
|
||||
|
||||
// Update completed file information and create new lock mutex
|
||||
fsi_.source_path =
|
||||
utils::path::combine(config_.get_cache_directory(), {utils::create_uuid_string()});
|
||||
fsi_.lock = std::make_shared<std::recursive_mutex>();
|
||||
}
|
||||
// For resume-only, open file to report correct file size in fuse/winfsp operations
|
||||
else if ((error_ = oft_.open(fsi_, open_file_handle_)) != api_error::success) {
|
||||
event_system::instance().raise<download_restore_failed>(
|
||||
fsi_.api_path, fsi_.source_path, "open failed: " + api_error_to_string(error_));
|
||||
}
|
||||
|
||||
if (error_ == api_error::success) {
|
||||
event_system::instance().raise<download_begin>(fsi_.api_path, fsi_.source_path);
|
||||
// Create new file for reading and writing
|
||||
if ((error_ = native_file::create_or_open(fsi_.source_path, read_write_file_)) ==
|
||||
api_error::success) {
|
||||
// Pre-allocate full file and update used cache space
|
||||
if (read_write_file_->allocate(fsi_.size)) {
|
||||
if (not delete_existing) {
|
||||
// Update used cache space
|
||||
global_data::instance().update_used_space(0u, fsi_.size, true);
|
||||
}
|
||||
|
||||
// Launch read-ahead workers
|
||||
for (std::uint8_t i = 0u; i < config_.get_read_ahead_count(); i++) {
|
||||
background_workers_.emplace_back(std::thread([this]() { read_ahead_worker(); }));
|
||||
}
|
||||
|
||||
// Launch read-behind worker
|
||||
background_workers_.emplace_back(std::thread([this]() { read_behind_worker(); }));
|
||||
|
||||
// Launch read-end worker
|
||||
if (fsi.size > (chunk_size_ * config_.get_read_ahead_count())) {
|
||||
background_workers_.emplace_back(std::thread([this]() { read_end_worker(); }));
|
||||
}
|
||||
|
||||
// Launch read/write IO worker
|
||||
io_thread_ = std::make_unique<std::thread>([this]() { io_data_worker(); });
|
||||
} else if (delete_existing) {
|
||||
const auto temp_error_code = utils::get_last_error_code();
|
||||
utils::file::delete_file(fsi_.source_path);
|
||||
utils::set_last_error_code(temp_error_code);
|
||||
error_ = api_error::os_error;
|
||||
}
|
||||
}
|
||||
|
||||
if (error_ != api_error::success) {
|
||||
event_system::instance().raise<download_end>(fsi_.api_path, fsi_.source_path, 0u, error_);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void download::io_data_worker() {
|
||||
unique_mutex_lock read_write_lock(read_write_mutex_);
|
||||
read_write_lock.unlock();
|
||||
|
||||
do {
|
||||
process_timeout(read_write_lock);
|
||||
process_read_queue(read_write_lock);
|
||||
process_write_queue(read_write_lock);
|
||||
notify_progress();
|
||||
process_download_complete(read_write_lock);
|
||||
wait_for_io(read_write_lock);
|
||||
} while (not processed_);
|
||||
|
||||
shutdown(read_write_lock);
|
||||
}
|
||||
|
||||
void download::notify_progress() {
|
||||
if (read_chunk_state_.any()) {
|
||||
utils::download::notify_progress<download_progress>(
|
||||
config_, fsi_.api_path, fsi_.source_path, static_cast<double>(read_chunk_state_.count()),
|
||||
static_cast<double>(read_chunk_state_.size()), progress_);
|
||||
}
|
||||
}
|
||||
|
||||
void download::notify_stop_requested() {
|
||||
if (not stop_requested_) {
|
||||
unique_mutex_lock read_write_lock(read_write_mutex_);
|
||||
error_ =
|
||||
write_chunk_state_.any() ? api_error::download_incomplete : api_error::download_stopped;
|
||||
stop_requested_ = true;
|
||||
paused_ = false;
|
||||
read_write_notify_.notify_all();
|
||||
read_write_lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
bool download::pause() {
|
||||
unique_mutex_lock read_write_lock(read_write_mutex_);
|
||||
if (not paused_) {
|
||||
paused_ = true;
|
||||
|
||||
while (not active_chunks_.empty() && not stop_requested_) {
|
||||
read_write_notify_.notify_all();
|
||||
read_write_notify_.wait_for(read_write_lock, 2s);
|
||||
}
|
||||
read_write_notify_.notify_all();
|
||||
read_write_lock.unlock();
|
||||
|
||||
if (stop_requested_) {
|
||||
paused_ = false;
|
||||
} else {
|
||||
event_system::instance().raise<download_paused>(fsi_.api_path, fsi_.source_path);
|
||||
}
|
||||
|
||||
return paused_;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void download::process_download_complete(unique_mutex_lock &lock) {
|
||||
if (get_complete() && not processed_) {
|
||||
lock.lock();
|
||||
if (not processed_) {
|
||||
if ((error_ == api_error::success) && read_chunk_state_.all()) {
|
||||
const auto assign_handle_and_auto_close = [this]() {
|
||||
if (fsi_.handle != REPERTORY_INVALID_HANDLE) {
|
||||
native_file::attach(fsi_.handle)->close();
|
||||
fsi_.handle = REPERTORY_INVALID_HANDLE;
|
||||
}
|
||||
fsi_.handle = read_write_file_->get_handle();
|
||||
auto_close_ = true;
|
||||
};
|
||||
|
||||
if (not oft_.perform_locked_operation(
|
||||
[this, &assign_handle_and_auto_close](i_open_file_table &oft,
|
||||
i_provider &provider) -> bool {
|
||||
filesystem_item *fsi = nullptr;
|
||||
if (oft.get_open_file(fsi_.api_path, fsi)) {
|
||||
unique_recur_mutex_lock item_lock(*fsi->lock);
|
||||
// Handle should always be invalid (just in case it's not - no leaking).
|
||||
if (fsi->handle != REPERTORY_INVALID_HANDLE) {
|
||||
native_file::attach(fsi->handle)->close();
|
||||
fsi->handle = REPERTORY_INVALID_HANDLE;
|
||||
}
|
||||
fsi->handle = read_write_file_->get_handle();
|
||||
item_lock.unlock();
|
||||
|
||||
fsi->source_path = fsi_.source_path;
|
||||
fsi->source_path_changed = true;
|
||||
fsi->changed = false;
|
||||
|
||||
fsi_ = *fsi;
|
||||
} else {
|
||||
provider.set_source_path(fsi_.api_path, fsi_.source_path);
|
||||
assign_handle_and_auto_close();
|
||||
}
|
||||
|
||||
return true;
|
||||
})) {
|
||||
assign_handle_and_auto_close();
|
||||
}
|
||||
} else if (error_ == api_error::success) {
|
||||
error_ = api_error::download_failed;
|
||||
}
|
||||
|
||||
stop_requested_ = processed_ = true;
|
||||
read_write_notify_.notify_all();
|
||||
lock.unlock();
|
||||
|
||||
process_write_queue(lock);
|
||||
process_read_queue(lock);
|
||||
|
||||
lock.lock();
|
||||
|
||||
if (error_ != api_error::success) {
|
||||
read_write_file_->close();
|
||||
if (error_ != api_error::download_incomplete) {
|
||||
if (utils::file::delete_file(fsi_.source_path)) {
|
||||
global_data::instance().update_used_space(fsi_.size, 0u, true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
processed_notify_.notify_all();
|
||||
read_write_notify_.notify_all();
|
||||
lock.unlock();
|
||||
|
||||
if (write_chunk_state_.any() && (error_ == api_error::success)) {
|
||||
oft_.force_schedule_upload(fsi_);
|
||||
}
|
||||
|
||||
if (open_file_handle_ != static_cast<std::uint64_t>(REPERTORY_API_INVALID_HANDLE)) {
|
||||
oft_.close(open_file_handle_);
|
||||
open_file_handle_ = REPERTORY_API_INVALID_HANDLE;
|
||||
}
|
||||
|
||||
if (not disable_download_end_) {
|
||||
event_system::instance().raise<download_end>(fsi_.api_path, fsi_.source_path, 0u, error_);
|
||||
}
|
||||
} else {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void download::process_read_queue(unique_mutex_lock &lock) {
|
||||
lock.lock();
|
||||
while (not read_queue_.empty()) {
|
||||
auto rd = read_queue_.front();
|
||||
read_queue_.pop_front();
|
||||
lock.unlock();
|
||||
|
||||
if (error_ == api_error::success) {
|
||||
std::size_t bytes_read = 0u;
|
||||
if (not read_write_file_->read_bytes(&rd->data[0u], rd->data.size(), rd->offset,
|
||||
bytes_read)) {
|
||||
error_ = ((error_ == api_error::success) ? api_error::os_error : error_);
|
||||
}
|
||||
}
|
||||
|
||||
rd->complete = true;
|
||||
unique_mutex_lock read_lock(rd->mutex);
|
||||
rd->notify.notify_all();
|
||||
read_lock.unlock();
|
||||
|
||||
lock.lock();
|
||||
}
|
||||
lock.unlock();
|
||||
}
|
||||
|
||||
void download::process_timeout(unique_mutex_lock &lock) {
|
||||
if ((std::chrono::system_clock::now() >= timeout_) && not get_complete() &&
|
||||
get_timeout_enabled()) {
|
||||
error_ = api_error::download_timeout;
|
||||
stop_requested_ = true;
|
||||
lock.lock();
|
||||
read_write_notify_.notify_all();
|
||||
lock.unlock();
|
||||
event_system::instance().raise<download_timeout>(fsi_.api_path, fsi_.source_path);
|
||||
}
|
||||
}
|
||||
|
||||
void download::process_write_queue(unique_mutex_lock &lock) {
|
||||
lock.lock();
|
||||
do {
|
||||
if (not write_queue_.empty()) {
|
||||
auto wd = write_queue_.front();
|
||||
write_queue_.pop_front();
|
||||
lock.unlock();
|
||||
|
||||
if (not get_complete() || (error_ == api_error::download_incomplete)) {
|
||||
auto &bytes_written = wd->written;
|
||||
if (read_write_file_->write_bytes(&wd->data[0u], wd->data.size(), wd->offset,
|
||||
bytes_written)) {
|
||||
read_write_file_->flush();
|
||||
if (wd->from_read) {
|
||||
lock.lock();
|
||||
read_chunk_state_[wd->chunk_index] = true;
|
||||
lock.unlock();
|
||||
} else {
|
||||
const auto start_chunk_index = static_cast<std::size_t>(wd->offset / chunk_size_);
|
||||
const auto end_chunk_index =
|
||||
static_cast<std::size_t>((wd->data.size() + wd->offset) / chunk_size_);
|
||||
lock.lock();
|
||||
for (std::size_t chunk_index = start_chunk_index; (chunk_index <= end_chunk_index);
|
||||
chunk_index++) {
|
||||
write_chunk_state_[chunk_index] = true;
|
||||
}
|
||||
lock.unlock();
|
||||
}
|
||||
} else {
|
||||
error_ = ((error_ == api_error::success) ? api_error::os_error : error_);
|
||||
}
|
||||
}
|
||||
|
||||
if (wd->from_read) {
|
||||
handle_active_chunk_complete(wd->chunk_index, lock);
|
||||
} else {
|
||||
wd->complete = true;
|
||||
unique_mutex_lock write_lock(wd->mutex);
|
||||
wd->notify.notify_all();
|
||||
write_lock.unlock();
|
||||
}
|
||||
|
||||
lock.lock();
|
||||
}
|
||||
} while (not write_queue_.empty() && read_queue_.empty());
|
||||
lock.unlock();
|
||||
}
|
||||
|
||||
void download::read_ahead_worker() {
|
||||
const auto total_chunks = read_chunk_state_.size();
|
||||
|
||||
auto first_chunk_index = current_chunk_index_;
|
||||
const auto get_next_chunk =
|
||||
[&first_chunk_index](const std::size_t ¤t_chunk_index,
|
||||
const std::size_t &last_chunk_index) -> std::size_t {
|
||||
if (current_chunk_index == first_chunk_index) {
|
||||
return last_chunk_index + 1ul;
|
||||
}
|
||||
|
||||
first_chunk_index = current_chunk_index;
|
||||
return current_chunk_index + 1ul;
|
||||
};
|
||||
|
||||
while (not get_complete()) {
|
||||
for (auto chunk_index = get_next_chunk(current_chunk_index_, 0u);
|
||||
(chunk_index < total_chunks) && not get_complete();) {
|
||||
download_chunk(chunk_index, true);
|
||||
chunk_index = get_next_chunk(current_chunk_index_, chunk_index);
|
||||
}
|
||||
|
||||
utils::spin_wait_for_mutex(
|
||||
[this, &first_chunk_index]() -> bool {
|
||||
return get_complete() || (first_chunk_index != current_chunk_index_);
|
||||
},
|
||||
read_write_notify_, read_write_mutex_, "read_ahead_worker");
|
||||
}
|
||||
}
|
||||
|
||||
void download::read_behind_worker() {
|
||||
auto first_chunk_index = current_chunk_index_;
|
||||
const auto get_next_chunk =
|
||||
[&first_chunk_index](const std::size_t ¤t_chunk_index,
|
||||
const std::size_t &last_chunk_index) -> std::size_t {
|
||||
if (current_chunk_index == first_chunk_index) {
|
||||
return last_chunk_index ? last_chunk_index - 1ul : 0u;
|
||||
}
|
||||
|
||||
first_chunk_index = current_chunk_index;
|
||||
return current_chunk_index ? current_chunk_index - 1ul : 0u;
|
||||
};
|
||||
|
||||
while (not get_complete()) {
|
||||
for (auto chunk_index = get_next_chunk(current_chunk_index_, current_chunk_index_);
|
||||
not get_complete();) {
|
||||
download_chunk(chunk_index, true);
|
||||
if (not chunk_index) {
|
||||
break;
|
||||
}
|
||||
chunk_index = get_next_chunk(current_chunk_index_, chunk_index);
|
||||
}
|
||||
|
||||
utils::spin_wait_for_mutex(
|
||||
[this, &first_chunk_index]() -> bool {
|
||||
return get_complete() || (first_chunk_index != current_chunk_index_);
|
||||
},
|
||||
read_write_notify_, read_write_mutex_, "read_behind_worker");
|
||||
}
|
||||
}
|
||||
|
||||
api_error download::read_bytes(const std::uint64_t &, std::size_t read_size,
|
||||
const std::uint64_t &read_offset, std::vector<char> &data) {
|
||||
/* std::cout << "read called:" << read_size << ':' << read_offset << std::endl; */
|
||||
data.clear();
|
||||
if ((read_size > 0u) && (error_ == api_error::success)) {
|
||||
unique_mutex_lock read_write_lock(read_write_mutex_);
|
||||
const auto read_local_file = [&]() -> api_error {
|
||||
read_write_notify_.notify_all();
|
||||
read_write_lock.unlock();
|
||||
|
||||
error_ = (error_ == api_error::success)
|
||||
? utils::file::read_from_source(fsi_, read_size, read_offset, data)
|
||||
: error_;
|
||||
|
||||
return error_;
|
||||
};
|
||||
|
||||
if (processed_) {
|
||||
return read_local_file();
|
||||
}
|
||||
|
||||
read_write_notify_.notify_all();
|
||||
read_write_lock.unlock();
|
||||
|
||||
const auto start_chunk_index = current_chunk_index_ =
|
||||
static_cast<std::size_t>(read_offset / chunk_size_);
|
||||
const auto end_chunk_index = static_cast<std::size_t>((read_size + read_offset) / chunk_size_);
|
||||
for (std::size_t chunk = start_chunk_index; not get_complete() && (chunk <= end_chunk_index);
|
||||
chunk++) {
|
||||
download_chunk(chunk, false);
|
||||
}
|
||||
|
||||
if (error_ == api_error::success) {
|
||||
data.resize(read_size);
|
||||
auto rd = std::make_shared<read_data>(data, read_offset);
|
||||
|
||||
read_write_lock.lock();
|
||||
if (processed_) {
|
||||
/* std::cout << "recursive read:" << read_size << ':' << read_offset << std::endl; */
|
||||
return read_local_file();
|
||||
}
|
||||
|
||||
read_queue_.emplace_front(rd);
|
||||
read_write_notify_.notify_all();
|
||||
read_write_lock.unlock();
|
||||
|
||||
reset_timeout(false);
|
||||
utils::spin_wait_for_mutex(rd->complete, rd->notify, rd->mutex, "read_bytes");
|
||||
|
||||
reset_timeout(false);
|
||||
}
|
||||
}
|
||||
|
||||
return error_;
|
||||
}
|
||||
|
||||
void download::read_end_worker() {
|
||||
const auto total_chunks = read_chunk_state_.size();
|
||||
auto to_read = utils::divide_with_ceiling(std::size_t(262144u), chunk_size_);
|
||||
for (auto chunk_index = total_chunks ? total_chunks - 1ul : 0u;
|
||||
chunk_index && to_read && not get_complete(); chunk_index--, to_read--) {
|
||||
download_chunk(chunk_index, true);
|
||||
}
|
||||
}
|
||||
|
||||
void download::reset_timeout(const bool &) {
|
||||
mutex_lock read_write_lock(read_write_mutex_);
|
||||
timeout_ = std::chrono::system_clock::now() +
|
||||
std::chrono::seconds(config_.get_chunk_downloader_timeout_secs());
|
||||
read_write_notify_.notify_all();
|
||||
}
|
||||
|
||||
void download::resume() {
|
||||
reset_timeout(false);
|
||||
|
||||
mutex_lock read_write_lock(read_write_mutex_);
|
||||
if (paused_) {
|
||||
paused_ = false;
|
||||
event_system::instance().raise<download_resumed>(fsi_.api_path, fsi_.source_path);
|
||||
}
|
||||
read_write_notify_.notify_all();
|
||||
}
|
||||
|
||||
void download::set_api_path(const std::string &api_path) {
|
||||
mutex_lock read_write_lock(read_write_mutex_);
|
||||
fsi_.api_path = api_path;
|
||||
fsi_.api_parent = utils::path::get_parent_api_path(api_path);
|
||||
read_write_notify_.notify_all();
|
||||
}
|
||||
|
||||
void download::shutdown(unique_mutex_lock &lock) {
|
||||
lock.lock();
|
||||
while (not active_chunks_.empty()) {
|
||||
const auto chunk_index = active_chunks_.begin()->first;
|
||||
lock.unlock();
|
||||
handle_active_chunk_complete(chunk_index, lock);
|
||||
lock.lock();
|
||||
}
|
||||
lock.unlock();
|
||||
|
||||
for (auto &worker : background_workers_) {
|
||||
worker.join();
|
||||
}
|
||||
background_workers_.clear();
|
||||
}
|
||||
|
||||
void download::wait_for_io(unique_mutex_lock &lock) {
|
||||
lock.lock();
|
||||
if (not get_complete() && read_queue_.empty() && write_queue_.empty()) {
|
||||
if (get_timeout_enabled()) {
|
||||
read_write_notify_.wait_until(lock, timeout_);
|
||||
} else {
|
||||
read_write_notify_.wait(lock);
|
||||
}
|
||||
}
|
||||
lock.unlock();
|
||||
}
|
||||
|
||||
api_error download::write_bytes(const std::uint64_t &, const std::uint64_t &write_offset,
|
||||
std::vector<char> data, std::size_t &bytes_written,
|
||||
const completer_callback &completer) {
|
||||
/* std::cout << "write called:" << write_offset << std::endl; */
|
||||
bytes_written = 0u;
|
||||
|
||||
if (not data.empty() && (error_ == api_error::success)) {
|
||||
const auto start_chunk_index = current_chunk_index_ =
|
||||
static_cast<std::size_t>(write_offset / chunk_size_);
|
||||
const auto end_chunk_index =
|
||||
static_cast<std::size_t>((data.size() + write_offset) / chunk_size_);
|
||||
for (std::size_t chunk_index = start_chunk_index;
|
||||
not get_complete() &&
|
||||
(chunk_index <= std::min(read_chunk_state_.size() - 1u, end_chunk_index));
|
||||
chunk_index++) {
|
||||
download_chunk(chunk_index, false);
|
||||
}
|
||||
|
||||
if (error_ == api_error::success) {
|
||||
const auto original_size = fsi_.size;
|
||||
|
||||
unique_mutex_lock read_write_lock(read_write_mutex_);
|
||||
const auto write_local_file = [&]() -> api_error {
|
||||
error_ = (error_ == api_error::success)
|
||||
? utils::file::write_to_source(fsi_, write_offset, data, bytes_written)
|
||||
: error_;
|
||||
|
||||
if (error_ == api_error::success) {
|
||||
utils::file::get_file_size(fsi_.source_path, fsi_.size);
|
||||
completer(original_size, fsi_.size, true);
|
||||
oft_.force_schedule_upload(fsi_);
|
||||
}
|
||||
|
||||
read_write_notify_.notify_all();
|
||||
read_write_lock.unlock();
|
||||
|
||||
return error_;
|
||||
};
|
||||
|
||||
if (processed_) {
|
||||
return write_local_file();
|
||||
}
|
||||
|
||||
read_write_notify_.notify_all();
|
||||
read_write_lock.unlock();
|
||||
|
||||
const auto size = (write_offset + data.size());
|
||||
if (size > fsi_.size) {
|
||||
pause();
|
||||
read_write_lock.lock();
|
||||
|
||||
if (end_chunk_index >= write_chunk_state_.size()) {
|
||||
const auto first_new_chunk_index = write_chunk_state_.size();
|
||||
write_chunk_state_.resize(end_chunk_index + 1u);
|
||||
read_chunk_state_.resize(end_chunk_index + 1u);
|
||||
for (std::size_t chunk_index = first_new_chunk_index; chunk_index <= end_chunk_index;
|
||||
chunk_index++) {
|
||||
read_chunk_state_[chunk_index] = true;
|
||||
}
|
||||
}
|
||||
|
||||
fsi_.size = size;
|
||||
last_chunk_size_ =
|
||||
static_cast<std::size_t>(fsi_.size <= chunk_size_ ? fsi_.size
|
||||
: fsi_.size % chunk_size_ ? fsi_.size % chunk_size_
|
||||
: chunk_size_);
|
||||
read_write_lock.unlock();
|
||||
resume();
|
||||
}
|
||||
|
||||
auto wd = std::make_shared<write_data>(write_offset, std::move(data));
|
||||
|
||||
read_write_lock.lock();
|
||||
if (processed_) {
|
||||
/* std::cout << "recursive write:" << write_offset << std::endl; */
|
||||
return write_local_file();
|
||||
}
|
||||
|
||||
write_queue_.emplace_back(wd);
|
||||
read_write_notify_.notify_all();
|
||||
read_write_lock.unlock();
|
||||
|
||||
reset_timeout(false);
|
||||
|
||||
utils::spin_wait_for_mutex(wd->complete, wd->notify, wd->mutex, "write_bytes");
|
||||
|
||||
reset_timeout(false);
|
||||
|
||||
bytes_written = wd->written;
|
||||
if (bytes_written && (size > fsi_.size)) {
|
||||
fsi_.size = size;
|
||||
completer(original_size, fsi_.size, false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return error_;
|
||||
}
|
||||
} // namespace repertory
|
||||
502
src/download/download_manager.cpp
Normal file
502
src/download/download_manager.cpp
Normal file
@@ -0,0 +1,502 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "download/download_manager.hpp"
|
||||
#include "download/direct_download.hpp"
|
||||
#include "download/download.hpp"
|
||||
#include "download/events.hpp"
|
||||
#include "download/ring_download.hpp"
|
||||
#include "drives/i_open_file_table.hpp"
|
||||
#include "events/events.hpp"
|
||||
#include "types/repertory.hpp"
|
||||
#include "utils/encrypting_reader.hpp"
|
||||
#include "utils/file_utils.hpp"
|
||||
#include "utils/global_data.hpp"
|
||||
#include "utils/path_utils.hpp"
|
||||
#include "utils/rocksdb_utils.hpp"
|
||||
#include "utils/utils.hpp"
|
||||
|
||||
namespace repertory {
|
||||
static json create_resume_entry(i_download &d) {
|
||||
filesystem_item fsi;
|
||||
std::size_t chunk_size;
|
||||
std::size_t last_chunk_size;
|
||||
boost::dynamic_bitset<> read_state, write_state;
|
||||
d.get_state_information(fsi, chunk_size, last_chunk_size, read_state, write_state);
|
||||
|
||||
return {
|
||||
{"chunk_size", chunk_size},
|
||||
{"encryption_token", fsi.encryption_token},
|
||||
{"last_chunk_size", last_chunk_size},
|
||||
{"path", fsi.api_path},
|
||||
{"read_state", utils::string::from_dynamic_bitset(read_state)},
|
||||
{"source", fsi.source_path},
|
||||
{"write_state", utils::string::from_dynamic_bitset(write_state)},
|
||||
};
|
||||
}
|
||||
|
||||
static void restore_resume_entry(const json &resume_entry, filesystem_item &fsi,
|
||||
std::size_t &chunk_size, std::size_t &last_chunk_size,
|
||||
boost::dynamic_bitset<> &read_state,
|
||||
boost::dynamic_bitset<> &write_state) {
|
||||
chunk_size = resume_entry["chunk_size"].get<std::size_t>();
|
||||
fsi.encryption_token = resume_entry["encryption_token"].get<std::string>();
|
||||
last_chunk_size = resume_entry["last_chunk_size"].get<std::size_t>();
|
||||
fsi.api_path = resume_entry["path"].get<std::string>();
|
||||
read_state = utils::string::to_dynamic_bitset(resume_entry["read_state"].get<std::string>());
|
||||
fsi.source_path = resume_entry["source"].get<std::string>();
|
||||
write_state = utils::string::to_dynamic_bitset(resume_entry["write_state"].get<std::string>());
|
||||
fsi.api_path = utils::path::get_parent_api_path(fsi.api_path);
|
||||
}
|
||||
|
||||
static void create_source_file_if_empty(filesystem_item &fsi, const app_config &config) {
|
||||
unique_recur_mutex_lock item_lock(*fsi.lock);
|
||||
if (fsi.source_path.empty()) {
|
||||
fsi.source_path =
|
||||
utils::path::combine(config.get_cache_directory(), {utils::create_uuid_string()});
|
||||
fsi.source_path_changed = true;
|
||||
}
|
||||
item_lock.unlock();
|
||||
}
|
||||
|
||||
download_manager::download_manager(const app_config &config, api_reader_callback api_reader,
|
||||
const bool &force_download)
|
||||
: config_(config), api_reader_(api_reader), force_download_(force_download) {
|
||||
utils::db::create_rocksdb(config, "resume_db", restore_db_);
|
||||
|
||||
E_SUBSCRIBE_EXACT(download_end, [this](const download_end &downloadEnd) {
|
||||
this->handle_download_end(downloadEnd);
|
||||
});
|
||||
|
||||
E_SUBSCRIBE_EXACT(filesystem_item_handle_closed,
|
||||
[this](const filesystem_item_handle_closed &fileHandleClosed) {
|
||||
this->on_handle_closed(fileHandleClosed);
|
||||
});
|
||||
|
||||
E_SUBSCRIBE_EXACT(filesystem_item_closed, [this](const filesystem_item_closed &fileClosed) {
|
||||
reset_timeout(fileClosed.get_api_path(), true);
|
||||
});
|
||||
|
||||
E_SUBSCRIBE_EXACT(filesystem_item_opened, [this](const filesystem_item_opened &fileOpened) {
|
||||
reset_timeout(fileOpened.get_api_path(), false);
|
||||
});
|
||||
}
|
||||
|
||||
download_manager::~download_manager() {
|
||||
E_CONSUMER_RELEASE();
|
||||
stop();
|
||||
}
|
||||
|
||||
api_error download_manager::allocate(const std::uint64_t &handle, filesystem_item &fsi,
|
||||
const std::uint64_t &size,
|
||||
const i_download::allocator_callback &allocator) {
|
||||
create_source_file_if_empty(fsi, config_);
|
||||
|
||||
if (stop_requested_) {
|
||||
return api_error::download_stopped;
|
||||
}
|
||||
|
||||
const auto completer = [&](std::uint64_t old_size, std::uint64_t new_size, bool changed) {
|
||||
fsi.changed = changed;
|
||||
if (old_size != new_size) {
|
||||
fsi.size = new_size;
|
||||
global_data::instance().update_used_space(old_size, new_size, false);
|
||||
}
|
||||
|
||||
const auto modified = std::to_string(utils::get_file_time_now());
|
||||
oft_->set_item_meta(fsi.api_path, META_MODIFIED, modified);
|
||||
oft_->set_item_meta(fsi.api_path, META_WRITTEN, modified);
|
||||
};
|
||||
|
||||
if (utils::file::is_file(fsi.source_path)) {
|
||||
std::uint64_t file_size = 0u;
|
||||
if (utils::file::get_file_size(fsi.source_path, file_size)) {
|
||||
if (fsi.size == file_size) {
|
||||
native_file_ptr nf;
|
||||
auto ret = utils::file::assign_and_get_native_file(fsi, nf);
|
||||
if (ret != api_error::success) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
if ((ret = allocator()) == api_error::success) {
|
||||
std::uint64_t new_file_size{};
|
||||
utils::file::get_file_size(fsi.source_path, new_file_size);
|
||||
completer(fsi.size, new_file_size, true);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
} else {
|
||||
return api_error::os_error;
|
||||
}
|
||||
}
|
||||
|
||||
auto d = get_download(handle, fsi, true);
|
||||
const auto result = d->get_result();
|
||||
return ((result == api_error::success) ? d->allocate(handle, size, allocator, completer)
|
||||
: result);
|
||||
}
|
||||
|
||||
bool download_manager::contains_handle(const std::string &api_path,
|
||||
const std::uint64_t &handle) const {
|
||||
return ((download_lookup_.find(api_path) != download_lookup_.end()) &&
|
||||
(download_lookup_.at(api_path).find(handle) != download_lookup_.at(api_path).end()));
|
||||
}
|
||||
|
||||
bool download_manager::contains_restore(const std::string &api_path) const {
|
||||
std::string value;
|
||||
restore_db_->Get(rocksdb::ReadOptions(), api_path, &value);
|
||||
return not value.empty();
|
||||
}
|
||||
|
||||
api_error download_manager::download_file(const std::uint64_t &handle, filesystem_item &fsi) {
|
||||
create_source_file_if_empty(fsi, config_);
|
||||
|
||||
if (stop_requested_) {
|
||||
return api_error::download_stopped;
|
||||
} else if (fsi.size > 0u) {
|
||||
std::uint64_t file_size = 0u;
|
||||
if (utils::file::get_file_size(fsi.source_path, file_size)) {
|
||||
if (file_size == fsi.size) {
|
||||
native_file_ptr nf;
|
||||
return utils::file::assign_and_get_native_file(fsi, nf);
|
||||
}
|
||||
}
|
||||
|
||||
auto d = get_download(handle, fsi, true);
|
||||
return ((d->get_result() == api_error::success) ? d->download_all() : d->get_result());
|
||||
} else if (fsi.handle == REPERTORY_INVALID_HANDLE) {
|
||||
native_file_ptr nf;
|
||||
return utils::file::assign_and_get_native_file(fsi, nf);
|
||||
} else {
|
||||
return api_error::success;
|
||||
}
|
||||
}
|
||||
|
||||
download_ptr download_manager::get_download(std::uint64_t handle, filesystem_item &fsi,
|
||||
const bool &write_supported) {
|
||||
const auto create_download = [this, &handle, &fsi, &write_supported]() -> download_ptr {
|
||||
const auto chunk_size = utils::encryption::encrypting_reader::get_data_chunk_size();
|
||||
if (write_supported || force_download_) {
|
||||
handle = 0u;
|
||||
return std::dynamic_pointer_cast<i_download>(
|
||||
std::make_shared<download>(config_, fsi, api_reader_, chunk_size, *oft_));
|
||||
} else {
|
||||
const auto dt = config_.get_preferred_download_type();
|
||||
const std::size_t ring_buffer_size = config_.get_ring_buffer_file_size() * 1024ull * 1024ull;
|
||||
const auto ring_allowed = (fsi.size > ring_buffer_size);
|
||||
|
||||
const auto no_space_remaining =
|
||||
(utils::file::get_available_drive_space(config_.get_cache_directory()) < fsi.size);
|
||||
if (ring_allowed && (dt != download_type::direct) &&
|
||||
(no_space_remaining || (dt == download_type::ring_buffer))) {
|
||||
return std::dynamic_pointer_cast<i_download>(std::make_shared<ring_download>(
|
||||
config_, fsi, api_reader_, handle, chunk_size, ring_buffer_size));
|
||||
}
|
||||
|
||||
if (no_space_remaining || (dt == download_type::direct)) {
|
||||
return std::dynamic_pointer_cast<i_download>(
|
||||
std::make_shared<direct_download>(config_, fsi, api_reader_, handle));
|
||||
} else {
|
||||
handle = 0u;
|
||||
return std::dynamic_pointer_cast<i_download>(
|
||||
std::make_shared<download>(config_, fsi, api_reader_, chunk_size, *oft_));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
download_ptr d;
|
||||
{
|
||||
recur_mutex_lock download_lock(download_mutex_);
|
||||
if (contains_handle(fsi.api_path, 0u)) {
|
||||
d = download_lookup_[fsi.api_path][0u];
|
||||
} else if (contains_handle(fsi.api_path, handle)) {
|
||||
d = download_lookup_[fsi.api_path][handle];
|
||||
} else {
|
||||
d = create_download();
|
||||
download_lookup_[fsi.api_path][handle] = d;
|
||||
}
|
||||
if (write_supported && not d->get_write_supported()) {
|
||||
for (auto &kv : download_lookup_[fsi.api_path]) {
|
||||
kv.second->set_disable_download_end(true);
|
||||
}
|
||||
d = create_download();
|
||||
download_lookup_[fsi.api_path] = {{0u, d}};
|
||||
}
|
||||
}
|
||||
|
||||
return d;
|
||||
}
|
||||
|
||||
std::string download_manager::get_source_path(const std::string &api_path) const {
|
||||
recur_mutex_lock download_lock(download_mutex_);
|
||||
if (contains_handle(api_path, 0u)) {
|
||||
return download_lookup_.at(api_path).at(0u)->get_source_path();
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
void download_manager::handle_download_end(const download_end &de) {
|
||||
download_ptr d;
|
||||
const auto api_path = de.get_api_path();
|
||||
const auto handle = utils::string::to_uint64(de.get_handle());
|
||||
unique_recur_mutex_lock download_lock(download_mutex_);
|
||||
if (contains_handle(api_path, handle)) {
|
||||
d = download_lookup_[api_path][handle];
|
||||
}
|
||||
download_lock.unlock();
|
||||
if (d) {
|
||||
download_lock.lock();
|
||||
download_lookup_[api_path].erase(handle);
|
||||
if (download_lookup_[api_path].empty()) {
|
||||
download_lookup_.erase(api_path);
|
||||
}
|
||||
|
||||
if (d->get_result() == api_error::download_incomplete) {
|
||||
const auto res = restore_db_->Put(rocksdb::WriteOptions(), api_path.get<std::string>(),
|
||||
create_resume_entry(*d).dump());
|
||||
if (res.ok()) {
|
||||
event_system::instance().raise<download_stored>(api_path, d->get_source_path());
|
||||
} else {
|
||||
event_system::instance().raise<download_store_failed>(api_path.get<std::string>(),
|
||||
d->get_source_path(), res.ToString());
|
||||
}
|
||||
}
|
||||
|
||||
download_lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
void download_manager::on_handle_closed(const filesystem_item_handle_closed &handle_closed) {
|
||||
download_ptr d;
|
||||
const auto api_path = handle_closed.get_api_path();
|
||||
const auto handle = utils::string::to_uint64(handle_closed.get_handle());
|
||||
unique_recur_mutex_lock download_lock(download_mutex_);
|
||||
if (contains_handle(api_path, handle)) {
|
||||
d = download_lookup_[api_path][handle];
|
||||
}
|
||||
download_lock.unlock();
|
||||
if (d) {
|
||||
d->notify_stop_requested();
|
||||
}
|
||||
}
|
||||
|
||||
bool download_manager::is_processing(const std::string &api_path) const {
|
||||
recur_mutex_lock download_lock(download_mutex_);
|
||||
return download_lookup_.find(api_path) != download_lookup_.end();
|
||||
}
|
||||
|
||||
bool download_manager::pause_download(const std::string &api_path) {
|
||||
auto ret = true;
|
||||
|
||||
recur_mutex_lock download_lock(download_mutex_);
|
||||
if (download_lookup_.find(api_path) != download_lookup_.end()) {
|
||||
ret = false;
|
||||
if (download_lookup_[api_path].find(0u) != download_lookup_[api_path].end()) {
|
||||
if ((ret = (download_lookup_[api_path].size() == 1))) {
|
||||
ret = download_lookup_[api_path][0u]->pause();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error download_manager::read_bytes(const std::uint64_t &handle, filesystem_item &fsi,
|
||||
std::size_t read_size, const std::uint64_t &read_offset,
|
||||
std::vector<char> &data) {
|
||||
create_source_file_if_empty(fsi, config_);
|
||||
|
||||
if (stop_requested_) {
|
||||
return api_error::download_stopped;
|
||||
}
|
||||
|
||||
read_size = utils::calculate_read_size(fsi.size, read_size, read_offset);
|
||||
if (read_size == 0u) {
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
if (utils::file::is_file(fsi.source_path)) {
|
||||
std::uint64_t file_size = 0u;
|
||||
if (utils::file::get_file_size(fsi.source_path, file_size)) {
|
||||
if ((read_offset + read_size) <= file_size) {
|
||||
return utils::file::read_from_source(fsi, read_size, read_offset, data);
|
||||
}
|
||||
} else {
|
||||
return api_error::os_error;
|
||||
}
|
||||
}
|
||||
|
||||
auto d = get_download(handle, fsi, false);
|
||||
const auto result = d->get_result();
|
||||
return ((result == api_error::success) ? d->read_bytes(handle, read_size, read_offset, data)
|
||||
: result);
|
||||
}
|
||||
|
||||
void download_manager::rename_download(const std::string &from_api_path,
|
||||
const std::string &to_api_path) {
|
||||
recur_mutex_lock download_lock(download_mutex_);
|
||||
if (download_lookup_.find(from_api_path) != download_lookup_.end()) {
|
||||
if (download_lookup_[from_api_path].find(0u) != download_lookup_[from_api_path].end()) {
|
||||
download_lookup_[from_api_path][0u]->set_api_path(to_api_path);
|
||||
download_lookup_[to_api_path][0u] = download_lookup_[from_api_path][0u];
|
||||
download_lookup_.erase(from_api_path);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void download_manager::reset_timeout(const std::string &api_path, const bool &file_closed) {
|
||||
download_ptr d;
|
||||
unique_recur_mutex_lock download_lock(download_mutex_);
|
||||
if (contains_handle(api_path, 0u)) {
|
||||
d = download_lookup_[api_path][0u];
|
||||
}
|
||||
download_lock.unlock();
|
||||
if (d) {
|
||||
d->reset_timeout(file_closed);
|
||||
}
|
||||
}
|
||||
|
||||
api_error download_manager::resize(const std::uint64_t &handle, filesystem_item &fsi,
|
||||
const std::uint64_t &size) {
|
||||
return allocate(handle, fsi, size,
|
||||
[&]() -> api_error { return utils::file::truncate_source(fsi, size); });
|
||||
}
|
||||
|
||||
void download_manager::resume_download(const std::string &api_path) {
|
||||
recur_mutex_lock download_lock(download_mutex_);
|
||||
if (download_lookup_.find(api_path) != download_lookup_.end()) {
|
||||
if (download_lookup_[api_path].find(0u) != download_lookup_[api_path].end()) {
|
||||
download_lookup_[api_path][0u]->resume();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void download_manager::start(i_open_file_table *oft) {
|
||||
recur_mutex_lock start_stop_lock(start_stop_mutex_);
|
||||
if (stop_requested_) {
|
||||
stop_requested_ = false;
|
||||
oft_ = oft;
|
||||
start_incomplete();
|
||||
}
|
||||
}
|
||||
|
||||
void download_manager::start_incomplete() {
|
||||
auto iterator =
|
||||
std::shared_ptr<rocksdb::Iterator>(restore_db_->NewIterator(rocksdb::ReadOptions()));
|
||||
for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
|
||||
filesystem_item fsi{};
|
||||
fsi.lock = std::make_shared<std::recursive_mutex>();
|
||||
|
||||
std::size_t chunk_size, last_chunk_size;
|
||||
boost::dynamic_bitset<> readState, writeState;
|
||||
restore_resume_entry(json::parse(iterator->value().ToString()), fsi, chunk_size,
|
||||
last_chunk_size, readState, writeState);
|
||||
if (utils::file::get_file_size(fsi.source_path, fsi.size)) {
|
||||
download_lookup_[fsi.api_path][0u] = std::make_shared<download>(
|
||||
config_, fsi, api_reader_, chunk_size, last_chunk_size, readState, writeState, *oft_);
|
||||
event_system::instance().raise<download_restored>(fsi.api_path, fsi.source_path);
|
||||
} else {
|
||||
event_system::instance().raise<download_restore_failed>(
|
||||
fsi.api_path, fsi.source_path,
|
||||
"failed to get file size: " + utils::string::from_uint64(utils::get_last_error_code()));
|
||||
}
|
||||
restore_db_->Delete(rocksdb::WriteOptions(), iterator->key());
|
||||
}
|
||||
}
|
||||
|
||||
void download_manager::stop() {
|
||||
unique_recur_mutex_lock start_stop_lock(start_stop_mutex_);
|
||||
if (not stop_requested_) {
|
||||
event_system::instance().raise<service_shutdown>("download_manager");
|
||||
stop_requested_ = true;
|
||||
start_stop_lock.unlock();
|
||||
|
||||
std::vector<download_ptr> downloads;
|
||||
unique_recur_mutex_lock download_lock(download_mutex_);
|
||||
for (auto &d : download_lookup_) {
|
||||
for (auto &kv : d.second) {
|
||||
auto download = kv.second;
|
||||
if (download) {
|
||||
downloads.emplace_back(download);
|
||||
}
|
||||
}
|
||||
}
|
||||
download_lock.unlock();
|
||||
|
||||
for (auto &d : downloads) {
|
||||
d->notify_stop_requested();
|
||||
}
|
||||
|
||||
while (not download_lookup_.empty()) {
|
||||
std::this_thread::sleep_for(1ms);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
api_error download_manager::write_bytes(const std::uint64_t &handle, filesystem_item &fsi,
|
||||
const std::uint64_t &write_offset, std::vector<char> data,
|
||||
std::size_t &bytes_written) {
|
||||
bytes_written = 0u;
|
||||
create_source_file_if_empty(fsi, config_);
|
||||
|
||||
if (stop_requested_) {
|
||||
return api_error::download_stopped;
|
||||
}
|
||||
|
||||
if (data.empty()) {
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
const auto completer = [&](std::uint64_t old_size, std::uint64_t new_size, bool changed) {
|
||||
fsi.changed = changed;
|
||||
if (old_size != new_size) {
|
||||
fsi.size = new_size;
|
||||
global_data::instance().update_used_space(old_size, new_size, false);
|
||||
}
|
||||
|
||||
const auto modified = std::to_string(utils::get_file_time_now());
|
||||
oft_->set_item_meta(fsi.api_path, META_MODIFIED, modified);
|
||||
oft_->set_item_meta(fsi.api_path, META_WRITTEN, modified);
|
||||
};
|
||||
|
||||
if (utils::file::is_file(fsi.source_path)) {
|
||||
std::uint64_t file_size = 0u;
|
||||
if (utils::file::get_file_size(fsi.source_path, file_size)) {
|
||||
if (fsi.size == file_size) {
|
||||
auto ret = utils::file::write_to_source(fsi, write_offset, data, bytes_written);
|
||||
if (ret == api_error::success) {
|
||||
std::uint64_t new_size{};
|
||||
utils::file::get_file_size(fsi.source_path, new_size);
|
||||
completer(fsi.size, new_size, true);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
} else {
|
||||
return api_error::os_error;
|
||||
}
|
||||
}
|
||||
|
||||
auto d = get_download(handle, fsi, true);
|
||||
const auto result = d->get_result();
|
||||
return ((result == api_error::success)
|
||||
? d->write_bytes(handle, write_offset, std::move(data), bytes_written, completer)
|
||||
: result);
|
||||
}
|
||||
} // namespace repertory
|
||||
166
src/download/reader_pool.cpp
Normal file
166
src/download/reader_pool.cpp
Normal file
@@ -0,0 +1,166 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "download/reader_pool.hpp"
|
||||
|
||||
namespace repertory {
|
||||
void reader_pool::pause() {
|
||||
unique_mutex_lock work_lock(work_mutex_);
|
||||
if (not paused_) {
|
||||
paused_ = true;
|
||||
while (not stop_requested_ && not work_queue_.empty()) {
|
||||
work_notify_.notify_all();
|
||||
work_notify_.wait_for(work_lock, 2s);
|
||||
}
|
||||
}
|
||||
work_notify_.notify_all();
|
||||
}
|
||||
|
||||
void reader_pool::process_work_item(pool_work_item &work) {
|
||||
const auto result =
|
||||
api_reader_(work.api_path, work.read_size, work.read_offset, work.data, restart_active_);
|
||||
work.completed(result);
|
||||
}
|
||||
|
||||
void reader_pool::queue_read_bytes(const std::string &api_path, const std::size_t &read_size,
|
||||
const std::uint64_t &read_offset, std::vector<char> &data,
|
||||
completed_callback completed) {
|
||||
data.clear();
|
||||
|
||||
unique_mutex_lock work_lock(work_mutex_);
|
||||
wait_for_resume(work_lock);
|
||||
if (not stop_requested_) {
|
||||
if (pool_size_ == 1ull) {
|
||||
work_notify_.notify_all();
|
||||
work_lock.unlock();
|
||||
completed(api_reader_(api_path, read_size, read_offset, data, restart_active_));
|
||||
} else {
|
||||
auto work =
|
||||
std::make_shared<pool_work_item>(api_path, read_size, read_offset, data, completed);
|
||||
if (not restart_active_) {
|
||||
while ((work_queue_.size() >= pool_size_) && not restart_active_) {
|
||||
work_notify_.wait(work_lock);
|
||||
}
|
||||
|
||||
if (not restart_active_) {
|
||||
work_queue_.emplace_back(work);
|
||||
work_notify_.notify_all();
|
||||
}
|
||||
|
||||
work_lock.unlock();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void reader_pool::restart() {
|
||||
restart_active_ = true;
|
||||
|
||||
unique_mutex_lock work_lock(work_mutex_);
|
||||
wait_for_resume(work_lock);
|
||||
work_notify_.notify_all();
|
||||
work_lock.unlock();
|
||||
|
||||
while (not work_queue_.empty() || active_count_) {
|
||||
work_lock.lock();
|
||||
if (not work_queue_.empty() || active_count_) {
|
||||
work_notify_.wait(work_lock);
|
||||
}
|
||||
work_lock.unlock();
|
||||
}
|
||||
|
||||
work_lock.lock();
|
||||
restart_active_ = false;
|
||||
work_notify_.notify_all();
|
||||
work_lock.unlock();
|
||||
}
|
||||
|
||||
void reader_pool::resume() {
|
||||
unique_mutex_lock work_lock(work_mutex_);
|
||||
if (paused_) {
|
||||
paused_ = false;
|
||||
}
|
||||
work_notify_.notify_all();
|
||||
}
|
||||
|
||||
void reader_pool::start() {
|
||||
restart_active_ = stop_requested_ = false;
|
||||
active_count_ = 0u;
|
||||
if (pool_size_ > 1u) {
|
||||
for (std::size_t i = 0u; i < pool_size_; i++) {
|
||||
work_threads_.emplace_back(std::thread([this]() {
|
||||
unique_mutex_lock work_lock(work_mutex_);
|
||||
work_notify_.notify_all();
|
||||
work_lock.unlock();
|
||||
|
||||
while (not stop_requested_) {
|
||||
work_lock.lock();
|
||||
if (not stop_requested_) {
|
||||
if (work_queue_.empty()) {
|
||||
work_notify_.wait(work_lock);
|
||||
}
|
||||
|
||||
if (not work_queue_.empty()) {
|
||||
active_count_++;
|
||||
auto work = work_queue_.front();
|
||||
work_queue_.pop_front();
|
||||
work_notify_.notify_all();
|
||||
work_lock.unlock();
|
||||
|
||||
process_work_item(*work);
|
||||
|
||||
work_lock.lock();
|
||||
active_count_--;
|
||||
}
|
||||
}
|
||||
|
||||
work_notify_.notify_all();
|
||||
work_lock.unlock();
|
||||
}
|
||||
|
||||
work_lock.lock();
|
||||
work_notify_.notify_all();
|
||||
work_lock.unlock();
|
||||
}));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void reader_pool::stop() {
|
||||
stop_requested_ = true;
|
||||
paused_ = false;
|
||||
restart();
|
||||
restart_active_ = true;
|
||||
|
||||
unique_mutex_lock work_lock(work_mutex_);
|
||||
work_notify_.notify_all();
|
||||
work_lock.unlock();
|
||||
|
||||
for (auto &work_thread : work_threads_) {
|
||||
work_thread.join();
|
||||
}
|
||||
work_threads_.clear();
|
||||
}
|
||||
|
||||
void reader_pool::wait_for_resume(unique_mutex_lock &lock) {
|
||||
while (paused_ && not stop_requested_) {
|
||||
work_notify_.notify_all();
|
||||
work_notify_.wait_for(lock, 2s);
|
||||
}
|
||||
}
|
||||
} // namespace repertory
|
||||
450
src/download/ring_download.cpp
Normal file
450
src/download/ring_download.cpp
Normal file
@@ -0,0 +1,450 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "download/ring_download.hpp"
|
||||
#include "app_config.hpp"
|
||||
#include "download/buffered_reader.hpp"
|
||||
#include "download/events.hpp"
|
||||
#include "download/utils.hpp"
|
||||
#include "events/event_system.hpp"
|
||||
#include "utils/file_utils.hpp"
|
||||
#include "utils/path_utils.hpp"
|
||||
#include "utils/utils.hpp"
|
||||
|
||||
namespace repertory {
|
||||
ring_download::ring_download(const app_config &config, filesystem_item fsi,
|
||||
const api_reader_callback &api_reader, const std::uint64_t &handle,
|
||||
const std::size_t &chunk_size, const std::size_t &ring_buffer_size)
|
||||
: config_(config),
|
||||
fsi_(std::move(fsi)),
|
||||
api_reader_(api_reader),
|
||||
handle_(handle),
|
||||
chunk_size_(chunk_size),
|
||||
ring_state_(ring_buffer_size / chunk_size) {
|
||||
if (not chunk_size) {
|
||||
error_ = api_error::empty_ring_buffer_chunk_size;
|
||||
} else if (not ring_buffer_size) {
|
||||
error_ = api_error::empty_ring_buffer_size;
|
||||
} else if (ring_buffer_size % chunk_size) {
|
||||
error_ = api_error::invalid_ring_buffer_multiple;
|
||||
} else if (ring_buffer_size < chunk_size) {
|
||||
error_ = api_error::invalid_ring_buffer_size;
|
||||
} else {
|
||||
*const_cast<std::size_t *>(&total_chunks_) =
|
||||
utils::divide_with_ceiling(static_cast<std::size_t>(fsi.size), chunk_size_);
|
||||
const auto buffer_directory = utils::path::combine(config_.get_data_directory(), {"buffer"});
|
||||
if (utils::file::create_full_directory_path(buffer_directory)) {
|
||||
buffer_file_path_ = utils::path::combine(buffer_directory, {utils::create_uuid_string()});
|
||||
if ((error_ = native_file::create_or_open(buffer_file_path_, buffer_file_)) ==
|
||||
api_error::success) {
|
||||
if (not buffer_file_->allocate(ring_buffer_size)) {
|
||||
error_ = api_error::os_error;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
error_ = api_error::os_error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ring_download::~ring_download() {
|
||||
stop();
|
||||
|
||||
if (buffer_file_) {
|
||||
buffer_file_->close();
|
||||
utils::file::delete_file(buffer_file_path_);
|
||||
}
|
||||
}
|
||||
|
||||
void ring_download::buffer_thread(std::size_t start_chunk_index) {
|
||||
buffered_reader reader(config_, fsi_, api_reader_, chunk_size_, total_chunks_, start_chunk_index);
|
||||
|
||||
unique_mutex_lock write_lock(write_mutex_);
|
||||
buffered_reader_ = &reader;
|
||||
read_notify_.notify_all();
|
||||
write_lock.unlock();
|
||||
|
||||
const auto ring_size = ring_state_.size();
|
||||
const auto half_ring_size = ring_size / 2;
|
||||
double progress = 0.0;
|
||||
|
||||
const auto reset_on_overflow = [&](const std::size_t &amt) -> bool {
|
||||
if (amt >= ring_size) {
|
||||
ring_state_.reset();
|
||||
head_chunk_index_ = read_chunk_;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
const auto decrement_head_chunk = [&](std::size_t amt) {
|
||||
if (not reset_on_overflow(amt)) {
|
||||
while (amt-- && head_chunk_index_) {
|
||||
head_chunk_index_--;
|
||||
ring_state_[head_chunk_index_ % ring_size] = false;
|
||||
}
|
||||
}
|
||||
write_chunk_ = read_chunk_;
|
||||
};
|
||||
|
||||
const auto increment_head_chunk = [&](std::size_t amt) {
|
||||
if (not reset_on_overflow(amt)) {
|
||||
while (amt-- && (head_chunk_index_ < (total_chunks_ - 1u))) {
|
||||
ring_state_[head_chunk_index_ % ring_size] = false;
|
||||
head_chunk_index_++;
|
||||
}
|
||||
}
|
||||
write_chunk_ = read_chunk_;
|
||||
};
|
||||
|
||||
while (is_active()) {
|
||||
write_lock.lock();
|
||||
|
||||
const auto get_overflow_chunk = [&]() { return head_chunk_index_ + ring_size; };
|
||||
while ((write_chunk_ > read_chunk_) && (write_chunk_ >= get_overflow_chunk()) && is_active()) {
|
||||
// Buffer 50% for read-ahead/read-behind
|
||||
auto buffered = false;
|
||||
while ((write_chunk_ > read_chunk_) && ((write_chunk_ - read_chunk_) > half_ring_size) &&
|
||||
is_active()) {
|
||||
buffered = true;
|
||||
read_notify_.wait(write_lock);
|
||||
}
|
||||
|
||||
if (not buffered && is_active()) {
|
||||
read_notify_.wait(write_lock);
|
||||
}
|
||||
|
||||
if ((write_chunk_ > read_chunk_) && ((write_chunk_ - read_chunk_) <= half_ring_size)) {
|
||||
increment_head_chunk(1u);
|
||||
}
|
||||
}
|
||||
|
||||
if (is_active()) {
|
||||
if (read_chunk_ >= get_overflow_chunk()) {
|
||||
increment_head_chunk(read_chunk_ - get_overflow_chunk() + 1u);
|
||||
} else if (read_chunk_ < head_chunk_index_) {
|
||||
decrement_head_chunk(head_chunk_index_ - read_chunk_);
|
||||
} else if ((write_chunk_ < head_chunk_index_) || (write_chunk_ >= get_overflow_chunk())) {
|
||||
write_chunk_ = read_chunk_;
|
||||
}
|
||||
|
||||
const auto write_position = write_chunk_ % ring_size;
|
||||
if (ring_state_[write_position]) {
|
||||
write_chunk_++;
|
||||
} else {
|
||||
const auto file_offset = write_chunk_ * chunk_size_;
|
||||
const auto write_chunk_index = write_chunk_;
|
||||
read_notify_.notify_all();
|
||||
write_lock.unlock();
|
||||
|
||||
const auto read_size = utils::calculate_read_size(fsi_.size, chunk_size_, file_offset);
|
||||
if (read_size > 0u) {
|
||||
if (((write_chunk_index == 0u) && reader.has_first_chunk()) ||
|
||||
((write_chunk_index == (total_chunks_ - 1ull)) && reader.has_last_chunk())) {
|
||||
write_lock.lock();
|
||||
write_chunk_++;
|
||||
} else {
|
||||
std::vector<char> data;
|
||||
auto error = reader.read_chunk(write_chunk_index, data);
|
||||
if (error == api_error::success) {
|
||||
std::mutex job_mutex;
|
||||
std::condition_variable job_notify;
|
||||
unique_mutex_lock job_lock(job_mutex);
|
||||
|
||||
if (queue_io_item(job_mutex, job_notify, false, [&]() {
|
||||
#ifdef _DEBUG
|
||||
write_lock.lock();
|
||||
if (write_chunk_index >= (head_chunk_index_ + ring_size)) {
|
||||
throw std::runtime_error("Invalid write: write chunk >= head");
|
||||
}
|
||||
if (write_chunk_index < head_chunk_index_) {
|
||||
throw std::runtime_error("Invalid write: write chunk < head");
|
||||
}
|
||||
#endif // _DEBUG
|
||||
std::size_t bytes_written = 0u;
|
||||
if (buffer_file_->write_bytes(&data[0u], data.size(),
|
||||
write_position * chunk_size_, bytes_written)) {
|
||||
buffer_file_->flush();
|
||||
utils::download::notify_progress<download_progress>(
|
||||
config_, fsi_.api_path, buffer_file_path_,
|
||||
static_cast<double>(write_chunk_index + 1ull),
|
||||
static_cast<double>(total_chunks_), progress);
|
||||
} else {
|
||||
error = api_error::os_error;
|
||||
}
|
||||
#ifdef _DEBUG
|
||||
write_lock.unlock();
|
||||
#endif // _DEBUG
|
||||
})) {
|
||||
job_notify.wait(job_lock);
|
||||
}
|
||||
job_notify.notify_all();
|
||||
job_lock.unlock();
|
||||
}
|
||||
|
||||
set_api_error(error);
|
||||
|
||||
write_lock.lock();
|
||||
if (error == api_error::success) {
|
||||
ring_state_[write_position] = true;
|
||||
write_chunk_++;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
write_lock.lock();
|
||||
while ((read_chunk_ <= write_chunk_) && (read_chunk_ >= head_chunk_index_) &&
|
||||
(((write_chunk_ * chunk_size_) >= fsi_.size)) && is_active()) {
|
||||
read_notify_.wait(write_lock);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
read_notify_.notify_all();
|
||||
write_lock.unlock();
|
||||
}
|
||||
|
||||
write_lock.lock();
|
||||
buffered_reader_ = nullptr;
|
||||
read_notify_.notify_all();
|
||||
write_lock.unlock();
|
||||
|
||||
if (not disable_download_end_) {
|
||||
event_system::instance().raise<download_end>(fsi_.api_path, buffer_file_path_, handle_, error_);
|
||||
}
|
||||
}
|
||||
|
||||
void ring_download::io_thread() {
|
||||
const auto process_all = [&]() {
|
||||
unique_mutex_lock io_lock(io_mutex_);
|
||||
while (not io_queue_.empty()) {
|
||||
auto &action = io_queue_.front();
|
||||
io_lock.unlock();
|
||||
|
||||
unique_mutex_lock job_lock(action->mutex);
|
||||
action->action();
|
||||
action->notify.notify_all();
|
||||
job_lock.unlock();
|
||||
|
||||
io_lock.lock();
|
||||
utils::remove_element_from(io_queue_, action);
|
||||
}
|
||||
io_lock.unlock();
|
||||
};
|
||||
|
||||
while (not stop_requested_) {
|
||||
if (io_queue_.empty()) {
|
||||
unique_mutex_lock io_lock(io_mutex_);
|
||||
if (not stop_requested_ && io_queue_.empty()) {
|
||||
io_notify_.wait(io_lock);
|
||||
}
|
||||
} else {
|
||||
process_all();
|
||||
}
|
||||
}
|
||||
|
||||
process_all();
|
||||
}
|
||||
|
||||
void ring_download::notify_stop_requested() {
|
||||
set_api_error(api_error::download_stopped);
|
||||
stop_requested_ = true;
|
||||
|
||||
unique_mutex_lock write_lock(write_mutex_);
|
||||
read_notify_.notify_all();
|
||||
write_lock.unlock();
|
||||
|
||||
unique_mutex_lock io_lock(io_mutex_);
|
||||
io_notify_.notify_all();
|
||||
io_lock.unlock();
|
||||
}
|
||||
|
||||
bool ring_download::queue_io_item(std::mutex &m, std::condition_variable &cv, const bool &is_read,
|
||||
std::function<void()> action) {
|
||||
auto ret = false;
|
||||
|
||||
mutex_lock io_lock(io_mutex_);
|
||||
if (not stop_requested_) {
|
||||
ret = true;
|
||||
if (is_read) {
|
||||
io_queue_.insert(io_queue_.begin(), std::make_unique<io_action>(m, cv, action));
|
||||
} else {
|
||||
io_queue_.emplace_back(std::make_unique<io_action>(m, cv, action));
|
||||
}
|
||||
}
|
||||
io_notify_.notify_all();
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
void ring_download::read(std::size_t read_chunk_index, std::size_t read_size,
|
||||
std::size_t read_offset, std::vector<char> &data) {
|
||||
while (is_active() && (read_size > 0u)) {
|
||||
unique_mutex_lock write_lock(write_mutex_);
|
||||
if ((read_chunk_index == (total_chunks_ - 1ull)) && buffered_reader_ &&
|
||||
buffered_reader_->has_last_chunk()) {
|
||||
std::vector<char> *buffer = nullptr;
|
||||
buffered_reader_->get_last_chunk(buffer);
|
||||
if (buffer) {
|
||||
data.insert(data.end(), buffer->begin() + read_offset,
|
||||
buffer->begin() + read_offset + read_size);
|
||||
}
|
||||
read_notify_.notify_all();
|
||||
write_lock.unlock();
|
||||
read_size = 0u;
|
||||
} else {
|
||||
const auto notify_read_location = [&]() {
|
||||
if (read_chunk_index != read_chunk_) {
|
||||
write_lock.lock();
|
||||
read_chunk_ = read_chunk_index;
|
||||
read_notify_.notify_all();
|
||||
write_lock.unlock();
|
||||
}
|
||||
};
|
||||
|
||||
if ((read_chunk_index == 0u) && buffered_reader_ && buffered_reader_->has_first_chunk()) {
|
||||
const auto to_read = std::min(chunk_size_ - read_offset, read_size);
|
||||
std::vector<char> *buffer = nullptr;
|
||||
buffered_reader_->get_first_chunk(buffer);
|
||||
if (buffer) {
|
||||
data.insert(data.end(), buffer->begin() + read_offset,
|
||||
buffer->begin() + read_offset + to_read);
|
||||
}
|
||||
read_notify_.notify_all();
|
||||
write_lock.unlock();
|
||||
|
||||
read_size -= to_read;
|
||||
read_offset = 0u;
|
||||
read_chunk_index++;
|
||||
notify_read_location();
|
||||
} else {
|
||||
const auto ring_size = ring_state_.size();
|
||||
while (((read_chunk_index > write_chunk_) || (read_chunk_index < head_chunk_index_) ||
|
||||
(read_chunk_index >= (head_chunk_index_ + ring_size))) &&
|
||||
is_active()) {
|
||||
read_chunk_ = read_chunk_index;
|
||||
read_notify_.notify_all();
|
||||
read_notify_.wait(write_lock);
|
||||
}
|
||||
read_notify_.notify_all();
|
||||
write_lock.unlock();
|
||||
|
||||
if (is_active()) {
|
||||
const auto ringPos = read_chunk_index % ring_size;
|
||||
notify_read_location();
|
||||
if (ring_state_[ringPos]) {
|
||||
const auto to_read = std::min(chunk_size_ - read_offset, read_size);
|
||||
std::mutex job_mutex;
|
||||
std::condition_variable job_notify;
|
||||
unique_mutex_lock job_lock(job_mutex);
|
||||
|
||||
if (queue_io_item(job_mutex, job_notify, true, [&]() {
|
||||
std::vector<char> buffer(to_read);
|
||||
std::size_t bytes_read = 0u;
|
||||
if (buffer_file_->read_bytes(&buffer[0u], buffer.size(),
|
||||
(ringPos * chunk_size_) + read_offset, bytes_read)) {
|
||||
data.insert(data.end(), buffer.begin(), buffer.end());
|
||||
buffer.clear();
|
||||
|
||||
read_size -= to_read;
|
||||
read_offset = 0u;
|
||||
} else {
|
||||
set_api_error(api_error::os_error);
|
||||
}
|
||||
})) {
|
||||
job_notify.wait(job_lock);
|
||||
read_chunk_index++;
|
||||
}
|
||||
job_notify.notify_all();
|
||||
job_lock.unlock();
|
||||
} else {
|
||||
write_lock.lock();
|
||||
while (not ring_state_[ringPos] && is_active()) {
|
||||
read_notify_.wait(write_lock);
|
||||
}
|
||||
read_notify_.notify_all();
|
||||
write_lock.unlock();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
api_error ring_download::read_bytes(const std::uint64_t &, std::size_t read_size,
|
||||
const std::uint64_t &read_offset, std::vector<char> &data) {
|
||||
data.clear();
|
||||
|
||||
mutex_lock read_lock(read_mutex_);
|
||||
if (is_active()) {
|
||||
if (((read_size = utils::calculate_read_size(fsi_.size, read_size, read_offset)) > 0u)) {
|
||||
const auto read_chunk_index = static_cast<std::size_t>(read_offset / chunk_size_);
|
||||
if (not buffer_thread_) {
|
||||
start(read_chunk_index);
|
||||
}
|
||||
|
||||
read(read_chunk_index, read_size, read_offset % chunk_size_, data);
|
||||
set_api_error(((error_ == api_error::success) && stop_requested_)
|
||||
? api_error::download_stopped
|
||||
: error_);
|
||||
}
|
||||
}
|
||||
|
||||
return error_;
|
||||
}
|
||||
|
||||
void ring_download::set_api_error(const api_error &error) {
|
||||
if ((error_ == api_error::success) && (error_ != error)) {
|
||||
error_ = error;
|
||||
}
|
||||
}
|
||||
|
||||
void ring_download::start(const std::size_t &startChunk) {
|
||||
event_system::instance().raise<download_begin>(fsi_.api_path, buffer_file_path_);
|
||||
|
||||
error_ = api_error::success;
|
||||
stop_requested_ = false;
|
||||
head_chunk_index_ = read_chunk_ = write_chunk_ = startChunk;
|
||||
ring_state_.reset();
|
||||
|
||||
io_thread_ = std::make_unique<std::thread>([this] { this->io_thread(); });
|
||||
|
||||
unique_mutex_lock write_lock(write_mutex_);
|
||||
buffer_thread_ =
|
||||
std::make_unique<std::thread>([this, startChunk]() { this->buffer_thread(startChunk); });
|
||||
read_notify_.wait(write_lock);
|
||||
read_notify_.notify_all();
|
||||
write_lock.unlock();
|
||||
}
|
||||
|
||||
void ring_download::stop() {
|
||||
notify_stop_requested();
|
||||
|
||||
if (buffer_thread_) {
|
||||
buffer_thread_->join();
|
||||
buffer_thread_.reset();
|
||||
}
|
||||
|
||||
if (io_thread_) {
|
||||
io_thread_->join();
|
||||
io_thread_.reset();
|
||||
|
||||
io_queue_.clear();
|
||||
}
|
||||
}
|
||||
} // namespace repertory
|
||||
110
src/drives/directory_cache.cpp
Normal file
110
src/drives/directory_cache.cpp
Normal file
@@ -0,0 +1,110 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "drives/directory_cache.hpp"
|
||||
#include "drives/directory_iterator.hpp"
|
||||
#include "events/events.hpp"
|
||||
#include "events/event_system.hpp"
|
||||
#include "types/repertory.hpp"
|
||||
|
||||
namespace repertory {
|
||||
bool directory_cache::execute_action(const std::string &api_path, const execute_callback &execute) {
|
||||
auto found = false;
|
||||
recur_mutex_lock directory_lock(directory_mutex_);
|
||||
if ((found = (directory_lookup_.find(api_path) != directory_lookup_.end()))) {
|
||||
execute(*directory_lookup_[api_path].iterator);
|
||||
}
|
||||
|
||||
return found;
|
||||
}
|
||||
|
||||
void directory_cache::refresh_thread() {
|
||||
while (not is_shutdown_) {
|
||||
unique_recur_mutex_lock directory_lock(directory_mutex_);
|
||||
auto lookup = directory_lookup_;
|
||||
directory_lock.unlock();
|
||||
|
||||
for (const auto &kv : lookup) {
|
||||
if (std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now() -
|
||||
kv.second.last_update) >= 120s) {
|
||||
directory_lock.lock();
|
||||
directory_lookup_.erase(kv.first);
|
||||
directory_lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
if (not is_shutdown_) {
|
||||
unique_mutex_lock shutdown_lock(shutdown_mutex_);
|
||||
if (not is_shutdown_) {
|
||||
shutdown_notify_.wait_for(shutdown_lock, 15s);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
directory_iterator *directory_cache::remove_directory(const std::string &api_path) {
|
||||
directory_iterator *ret = nullptr;
|
||||
|
||||
recur_mutex_lock directory_lock(directory_mutex_);
|
||||
if (directory_lookup_.find(api_path) != directory_lookup_.end()) {
|
||||
ret = directory_lookup_[api_path].iterator;
|
||||
directory_lookup_.erase(api_path);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
void directory_cache::remove_directory(directory_iterator *iterator) {
|
||||
if (iterator) {
|
||||
recur_mutex_lock directory_lock(directory_mutex_);
|
||||
const auto it = std::find_if(
|
||||
directory_lookup_.begin(), directory_lookup_.end(),
|
||||
[&iterator](const auto &kv) -> bool { return kv.second.iterator == iterator; });
|
||||
if (it != directory_lookup_.end()) {
|
||||
directory_lookup_.erase(it->first);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void directory_cache::set_directory(const std::string &api_path, directory_iterator *iterator) {
|
||||
recur_mutex_lock directory_lock(directory_mutex_);
|
||||
directory_lookup_[api_path] = {iterator};
|
||||
}
|
||||
|
||||
void directory_cache::start() {
|
||||
unique_mutex_lock shutdown_lock(shutdown_mutex_);
|
||||
if (is_shutdown_) {
|
||||
is_shutdown_ = false;
|
||||
|
||||
refresh_thread_ = std::make_unique<std::thread>([this]() { this->refresh_thread(); });
|
||||
}
|
||||
}
|
||||
|
||||
void directory_cache::stop() {
|
||||
unique_mutex_lock shutdown_lock(shutdown_mutex_);
|
||||
if (not is_shutdown_) {
|
||||
event_system::instance().raise<service_shutdown>("directory_cache");
|
||||
is_shutdown_ = true;
|
||||
shutdown_notify_.notify_all();
|
||||
shutdown_lock.unlock();
|
||||
|
||||
refresh_thread_->join();
|
||||
refresh_thread_.reset();
|
||||
}
|
||||
}
|
||||
} // namespace repertory
|
||||
129
src/drives/directory_iterator.cpp
Normal file
129
src/drives/directory_iterator.cpp
Normal file
@@ -0,0 +1,129 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "drives/directory_iterator.hpp"
|
||||
#include "utils/path_utils.hpp"
|
||||
|
||||
namespace repertory {
|
||||
#ifndef _WIN32
|
||||
int directory_iterator::fill_buffer(const remote::file_offset &offset,
|
||||
fuse_fill_dir_t filler_function, void *buffer,
|
||||
populate_stat_callback populate_stat) {
|
||||
if (offset < items_.size()) {
|
||||
std::string item_name;
|
||||
struct stat st {};
|
||||
struct stat *pst = nullptr;
|
||||
switch (offset) {
|
||||
case 0: {
|
||||
item_name = ".";
|
||||
} break;
|
||||
|
||||
case 1: {
|
||||
item_name = "..";
|
||||
} break;
|
||||
|
||||
default: {
|
||||
const auto &item = items_[offset];
|
||||
item_name = utils::path::strip_to_file_name(item.api_path);
|
||||
populate_stat(item.api_path, item.size, item.meta, item.directory, &st);
|
||||
pst = &st;
|
||||
} break;
|
||||
}
|
||||
|
||||
if (filler_function(buffer, &item_name[0], pst, offset + 1) != 0) {
|
||||
errno = ENOMEM;
|
||||
return -1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
errno = 120;
|
||||
return -1;
|
||||
}
|
||||
#endif // !_WIN32
|
||||
|
||||
int directory_iterator::get(const std::size_t &offset, std::string &item) {
|
||||
if (offset < items_.size()) {
|
||||
item = items_[offset].api_path;
|
||||
return 0;
|
||||
}
|
||||
|
||||
errno = 120;
|
||||
return -1;
|
||||
}
|
||||
|
||||
api_error directory_iterator::get_directory_item(const std::size_t &offset, directory_item &di) {
|
||||
if (offset < items_.size()) {
|
||||
di = items_[offset];
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
return api_error::directory_end_of_files;
|
||||
}
|
||||
|
||||
api_error directory_iterator::get_directory_item(const std::string &api_path, directory_item &di) {
|
||||
auto iter = std::find_if(items_.begin(), items_.end(),
|
||||
[&](const auto &item) -> bool { return api_path == item.api_path; });
|
||||
if (iter != items_.end()) {
|
||||
di = *iter;
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
return api_error::item_not_found;
|
||||
}
|
||||
|
||||
int directory_iterator::get_json(const std::size_t &offset, json &item) {
|
||||
if (offset < items_.size()) {
|
||||
item = items_[offset].to_json();
|
||||
return 0;
|
||||
}
|
||||
|
||||
errno = 120;
|
||||
return -1;
|
||||
}
|
||||
|
||||
std::size_t directory_iterator::get_next_directory_offset(const std::string &api_path) const {
|
||||
const auto it = std::find_if(items_.begin(), items_.end(), [&api_path](const auto &di) -> bool {
|
||||
return api_path == di.api_path;
|
||||
});
|
||||
|
||||
return (it == items_.end()) ? 0 : std::distance(items_.begin(), it) + std::size_t(1u);
|
||||
}
|
||||
|
||||
directory_iterator &directory_iterator::operator=(const directory_iterator &iterator) noexcept {
|
||||
if (this != &iterator) {
|
||||
items_ = iterator.items_;
|
||||
}
|
||||
return *this;
|
||||
}
|
||||
|
||||
directory_iterator &directory_iterator::operator=(directory_iterator &&iterator) noexcept {
|
||||
if (this != &iterator) {
|
||||
items_ = std::move(iterator.items_);
|
||||
}
|
||||
return *this;
|
||||
}
|
||||
|
||||
directory_iterator &directory_iterator::operator=(directory_item_list list) noexcept {
|
||||
if (&items_ != &list) {
|
||||
items_ = std::move(list);
|
||||
}
|
||||
return *this;
|
||||
}
|
||||
} // namespace repertory
|
||||
145
src/drives/eviction.cpp
Normal file
145
src/drives/eviction.cpp
Normal file
@@ -0,0 +1,145 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "drives/eviction.hpp"
|
||||
#include "app_config.hpp"
|
||||
#include "drives/i_open_file_table.hpp"
|
||||
#include "providers/i_provider.hpp"
|
||||
#include "types/repertory.hpp"
|
||||
#include "utils/file_utils.hpp"
|
||||
#include "utils/global_data.hpp"
|
||||
|
||||
namespace repertory {
|
||||
void eviction::check_items_thread() {
|
||||
while (not stop_requested_) {
|
||||
auto should_evict = true;
|
||||
|
||||
// Handle maximum cache size eviction
|
||||
auto used_bytes = global_data::instance().get_used_cache_space();
|
||||
if (config_.get_enable_max_cache_size()) {
|
||||
should_evict = (used_bytes > config_.get_max_cache_size_bytes());
|
||||
}
|
||||
|
||||
// Evict all items if minimum redundancy eviction is enabled; otherwise, evict
|
||||
// until required space is reclaimed.
|
||||
if (should_evict) {
|
||||
// Remove cached source files that don't meet minimum requirements
|
||||
auto cached_files_list = get_filtered_cached_files();
|
||||
if (not cached_files_list.empty()) {
|
||||
while (not stop_requested_ && should_evict && not cached_files_list.empty()) {
|
||||
std::string api_path;
|
||||
if (provider_.get_api_path_from_source(cached_files_list.front(), api_path) ==
|
||||
api_error::success) {
|
||||
std::string pinned;
|
||||
provider_.get_item_meta(api_path, META_PINNED, pinned);
|
||||
|
||||
api_file file{};
|
||||
filesystem_item fsi{};
|
||||
if ((pinned.empty() || not utils::string::to_bool(pinned)) &&
|
||||
provider_.get_filesystem_item_and_file(api_path, file, fsi) == api_error::success) {
|
||||
// Only evict files that match expected size
|
||||
std::uint64_t file_size = 0u;
|
||||
utils::file::get_file_size(cached_files_list.front(), file_size);
|
||||
if (file_size == fsi.size) {
|
||||
// Ensure minimum file redundancy has been met or source path is not being
|
||||
// used for local recovery
|
||||
const auto different_source = file.source_path != fsi.source_path;
|
||||
if (file.recoverable &&
|
||||
((file.redundancy >= config_.get_minimum_redundancy()) || different_source)) {
|
||||
// Try to evict file
|
||||
if (oft_.evict_file(fsi.api_path) && config_.get_enable_max_cache_size()) {
|
||||
// Restrict number of items evicted if maximum cache size is enabled
|
||||
used_bytes -= file_size;
|
||||
should_evict = (used_bytes > config_.get_max_cache_size_bytes());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
cached_files_list.pop_front();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (not stop_requested_) {
|
||||
unique_mutex_lock l(eviction_mutex_);
|
||||
if (not stop_requested_) {
|
||||
stop_notify_.wait_for(l, 30s);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool eviction::check_minimum_requirements(const std::string &file_path) {
|
||||
auto ret = false;
|
||||
// Only evict cachedFileList that are > 0
|
||||
std::uint64_t file_size = 0u;
|
||||
utils::file::get_file_size(file_path, file_size);
|
||||
if (file_size) {
|
||||
// Check modified/accessed date/time
|
||||
std::uint64_t reference_time = 0u;
|
||||
if ((ret = config_.get_eviction_uses_accessed_time()
|
||||
? utils::file::get_accessed_time(file_path, reference_time)
|
||||
: utils::file::get_modified_time(file_path, reference_time))) {
|
||||
#ifdef _WIN32
|
||||
const auto now = std::chrono::system_clock::now();
|
||||
const auto delay = std::chrono::minutes(config_.get_eviction_delay_mins());
|
||||
ret = ((std::chrono::system_clock::from_time_t(reference_time) + delay) <= now);
|
||||
#else
|
||||
const auto now = utils::get_time_now();
|
||||
const auto delay = (config_.get_eviction_delay_mins() * 60L) * NANOS_PER_SECOND;
|
||||
ret = ((reference_time + delay) <= now);
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
std::deque<std::string> eviction::get_filtered_cached_files() {
|
||||
auto list = utils::file::get_directory_files(config_.get_cache_directory(), true);
|
||||
list.erase(std::remove_if(list.begin(), list.end(),
|
||||
[this](const std::string &path) -> bool {
|
||||
return not this->check_minimum_requirements(path);
|
||||
}),
|
||||
list.end());
|
||||
return list;
|
||||
}
|
||||
|
||||
void eviction::start() {
|
||||
mutex_lock l(start_stop_mutex_);
|
||||
if (not eviction_thread_) {
|
||||
stop_requested_ = false;
|
||||
eviction_thread_ = std::make_unique<std::thread>([this] { this->check_items_thread(); });
|
||||
}
|
||||
}
|
||||
|
||||
void eviction::stop() {
|
||||
mutex_lock l(start_stop_mutex_);
|
||||
if (eviction_thread_) {
|
||||
event_system::instance().raise<service_shutdown>("eviction");
|
||||
stop_requested_ = true;
|
||||
{
|
||||
mutex_lock l2(eviction_mutex_);
|
||||
stop_notify_.notify_all();
|
||||
}
|
||||
eviction_thread_->join();
|
||||
eviction_thread_.reset();
|
||||
}
|
||||
}
|
||||
} // namespace repertory
|
||||
844
src/drives/fuse/fuse_base.cpp
Normal file
844
src/drives/fuse/fuse_base.cpp
Normal file
@@ -0,0 +1,844 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#ifndef _WIN32
|
||||
|
||||
#include "drives/fuse/fuse_base.hpp"
|
||||
#include "app_config.hpp"
|
||||
#include "drives/fuse/events.hpp"
|
||||
#include "events/events.hpp"
|
||||
#include "events/event_system.hpp"
|
||||
#include "providers/i_provider.hpp"
|
||||
#include "utils/path_utils.hpp"
|
||||
#include "utils/string_utils.hpp"
|
||||
#include "utils/utils.hpp"
|
||||
|
||||
namespace repertory {
|
||||
fuse_base &fuse_base::instance() {
|
||||
return *reinterpret_cast<fuse_base *>(fuse_get_context()->private_data);
|
||||
}
|
||||
|
||||
fuse_base::fuse_base(app_config &config) : config_(config) {
|
||||
#ifdef __APPLE__
|
||||
fuse_ops_.chflags = fuse_base::chflags_;
|
||||
fuse_ops_.fsetattr_x = fuse_base::fsetattr_x_;
|
||||
fuse_ops_.getxtimes = fuse_base::getxtimes_;
|
||||
fuse_ops_.setattr_x = fuse_base::setattr_x_;
|
||||
fuse_ops_.setbkuptime = fuse_base::setbkuptime_;
|
||||
fuse_ops_.setchgtime = fuse_base::setchgtime_;
|
||||
fuse_ops_.setcrtime = fuse_base::setcrtime_;
|
||||
fuse_ops_.setvolname = fuse_base::setvolname_;
|
||||
fuse_ops_.statfs_x = fuse_base::statfs_x_;
|
||||
#endif // __APPLE__
|
||||
|
||||
E_SUBSCRIBE_EXACT(unmount_requested, [this](const unmount_requested &) {
|
||||
std::thread([this]() { this->shutdown(); }).detach();
|
||||
});
|
||||
}
|
||||
|
||||
fuse_base::~fuse_base() { E_CONSUMER_RELEASE(); }
|
||||
|
||||
int fuse_base::access_(const char *path, int mask) {
|
||||
return instance().instance().execute_callback(__FUNCTION__, path,
|
||||
[&](const std::string &api_path) -> api_error {
|
||||
return instance().access_impl(api_path, mask);
|
||||
});
|
||||
}
|
||||
|
||||
#ifdef __APPLE__
|
||||
int fuse_base::chflags_(const char *path, uint32_t flags) {
|
||||
return instance().instance().execute_callback(__FUNCTION__, path,
|
||||
[&](const std::string &api_path) -> api_error {
|
||||
return instance().chflags_impl(api_path, flags);
|
||||
});
|
||||
}
|
||||
#endif // __APPLE__
|
||||
|
||||
api_error fuse_base::check_access(const std::string &api_path, int mask) const {
|
||||
api_meta_map meta;
|
||||
const auto res = get_item_meta(api_path, meta);
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
|
||||
// Always allow root
|
||||
if (fuse_get_context()->uid != 0) {
|
||||
// Always allow forced user
|
||||
if (not forced_uid_.has_value() || (fuse_get_context()->uid != get_effective_uid())) {
|
||||
// Always allow if checking file exists
|
||||
if (F_OK != mask) {
|
||||
const auto effective_uid =
|
||||
(forced_uid_.has_value() ? forced_uid_.value() : get_uid_from_meta(meta));
|
||||
const auto effective_gid =
|
||||
(forced_gid_.has_value() ? forced_gid_.value() : get_gid_from_meta(meta));
|
||||
|
||||
// Create file mode
|
||||
mode_t effective_mode = forced_umask_.has_value()
|
||||
? ((S_IRWXU | S_IRWXG | S_IRWXO) & (~forced_umask_.value()))
|
||||
: get_mode_from_meta(meta);
|
||||
|
||||
// Create access mask
|
||||
mode_t active_mask = S_IRWXO;
|
||||
if (fuse_get_context()->uid == effective_uid) {
|
||||
active_mask |= S_IRWXU;
|
||||
}
|
||||
if (fuse_get_context()->gid == effective_gid) {
|
||||
active_mask |= S_IRWXG;
|
||||
}
|
||||
if (utils::is_uid_member_of_group(fuse_get_context()->uid, effective_gid)) {
|
||||
active_mask |= S_IRWXG;
|
||||
}
|
||||
|
||||
// Calculate effective file mode
|
||||
effective_mode &= active_mask;
|
||||
|
||||
// Check allow execute
|
||||
if ((mask & X_OK) == X_OK) {
|
||||
if ((effective_mode & (S_IXUSR | S_IXGRP | S_IXOTH)) == 0) {
|
||||
return api_error::permission_denied;
|
||||
}
|
||||
}
|
||||
|
||||
// Check allow write
|
||||
if ((mask & W_OK) == W_OK) {
|
||||
if ((effective_mode & (S_IWUSR | S_IWGRP | S_IWOTH)) == 0) {
|
||||
return api_error::access_denied;
|
||||
}
|
||||
}
|
||||
|
||||
// Check allow read
|
||||
if ((mask & R_OK) == R_OK) {
|
||||
if ((effective_mode & (S_IRUSR | S_IRGRP | S_IROTH)) == 0) {
|
||||
return api_error::access_denied;
|
||||
}
|
||||
}
|
||||
|
||||
if (effective_mode == 0) {
|
||||
// Deny access if effective mode is 0
|
||||
return api_error::access_denied;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
api_error fuse_base::check_and_perform(const std::string &api_path, int parent_mask,
|
||||
const std::function<api_error(api_meta_map &meta)> &action) {
|
||||
api_meta_map meta;
|
||||
auto ret = get_item_meta(api_path, meta);
|
||||
if (ret != api_error::success) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
if ((ret = check_parent_access(api_path, parent_mask)) != api_error::success) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
if ((ret = check_owner(meta)) != api_error::success) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
return action(meta);
|
||||
}
|
||||
|
||||
api_error fuse_base::check_open_flags(const int &flags, const int &mask,
|
||||
const api_error &fail_error) {
|
||||
return ((flags & mask) ? fail_error : api_error::success);
|
||||
}
|
||||
|
||||
api_error fuse_base::check_owner(const api_meta_map &meta) const {
|
||||
// Always allow root
|
||||
if ((fuse_get_context()->uid != 0) &&
|
||||
// Always allow forced UID
|
||||
(not forced_uid_.has_value() || (fuse_get_context()->uid != get_effective_uid())) &&
|
||||
// Check if current uid matches file uid
|
||||
(get_uid_from_meta(meta) != get_effective_uid())) {
|
||||
return api_error::permission_denied;
|
||||
}
|
||||
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
api_error fuse_base::check_parent_access(const std::string &api_path, int mask) const {
|
||||
auto ret = api_error::success;
|
||||
|
||||
// Ignore root
|
||||
if (api_path != "/") {
|
||||
if ((mask & X_OK) == X_OK) {
|
||||
for (auto parent = utils::path::get_parent_directory(api_path);
|
||||
(ret == api_error::success) && not parent.empty();
|
||||
parent = utils::path::get_parent_directory(parent)) {
|
||||
if (((ret = check_access(parent, X_OK)) == api_error::success) && (parent == "/")) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (ret == api_error::success) {
|
||||
mask &= (~X_OK);
|
||||
if (mask != 0) {
|
||||
ret = check_access(utils::path::get_parent_directory(api_path), mask);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error fuse_base::check_readable(const int &flags, const api_error &fail_error) {
|
||||
const auto mode = (flags & O_ACCMODE);
|
||||
return ((mode == O_WRONLY) ? fail_error : api_error::success);
|
||||
}
|
||||
|
||||
api_error fuse_base::check_writeable(const int &flags, const api_error &fail_error) {
|
||||
return ((flags & O_ACCMODE) ? api_error::success : fail_error);
|
||||
}
|
||||
|
||||
int fuse_base::chmod_(const char *path, mode_t mode) {
|
||||
return instance().execute_callback(__FUNCTION__, path,
|
||||
[&](const std::string &api_path) -> api_error {
|
||||
return instance().chmod_impl(api_path, mode);
|
||||
});
|
||||
}
|
||||
|
||||
int fuse_base::chown_(const char *path, uid_t uid, gid_t gid) {
|
||||
return instance().execute_callback(__FUNCTION__, path,
|
||||
[&](const std::string &api_path) -> api_error {
|
||||
return instance().chown_impl(api_path, uid, gid);
|
||||
});
|
||||
}
|
||||
|
||||
int fuse_base::create_(const char *path, mode_t mode, struct fuse_file_info *fi) {
|
||||
return instance().execute_callback(__FUNCTION__, path,
|
||||
[&](const std::string &api_path) -> api_error {
|
||||
return instance().create_impl(api_path, mode, fi);
|
||||
});
|
||||
}
|
||||
|
||||
void fuse_base::destroy_(void *ptr) {
|
||||
execute_void_callback(__FUNCTION__, [&]() { instance().destroy_impl(ptr); });
|
||||
}
|
||||
|
||||
void fuse_base::display_options(int argc, char *argv[]) {
|
||||
struct fuse_operations fuse_ops {};
|
||||
fuse_main(argc, argv, &fuse_ops, nullptr);
|
||||
std::cout << std::endl;
|
||||
}
|
||||
|
||||
void fuse_base::display_version_information(int argc, char *argv[]) {
|
||||
struct fuse_operations fuse_ops {};
|
||||
fuse_main(argc, argv, &fuse_ops, nullptr);
|
||||
}
|
||||
|
||||
int fuse_base::execute_callback(const std::string &function_name, const char *from, const char *to,
|
||||
const std::function<api_error(const std::string &from_api_file,
|
||||
const std::string &to_api_path)> &cb,
|
||||
const bool &disable_logging) {
|
||||
const auto from_api_file = utils::path::create_api_path(from ? from : "");
|
||||
const auto to_api_file = utils::path::create_api_path(to ? to : "");
|
||||
const auto res = utils::translate_api_error(cb(from_api_file, to_api_file));
|
||||
raise_fuse_event(function_name, "from|" + from_api_file + "|to|" + to_api_file, res,
|
||||
disable_logging);
|
||||
return res;
|
||||
}
|
||||
|
||||
int fuse_base::execute_callback(const std::string &function_name, const char *path,
|
||||
const std::function<api_error(const std::string &api_path)> &cb,
|
||||
const bool &disable_logging) {
|
||||
const auto api_path = utils::path::create_api_path(path ? path : "");
|
||||
const auto res = utils::translate_api_error(cb(api_path));
|
||||
raise_fuse_event(function_name, api_path, res, disable_logging);
|
||||
return res;
|
||||
}
|
||||
|
||||
void fuse_base::execute_void_callback(const std::string &function_name,
|
||||
const std::function<void()> &cb) {
|
||||
cb();
|
||||
|
||||
instance().raise_fuse_event(function_name, "", 0, false);
|
||||
}
|
||||
|
||||
void *fuse_base::execute_void_pointer_callback(const std::string &function_name,
|
||||
const std::function<void *()> &cb) {
|
||||
auto *ret = cb();
|
||||
|
||||
instance().raise_fuse_event(function_name, "", 0, false);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
int fuse_base::fallocate_(const char *path, int mode, off_t offset, off_t length,
|
||||
struct fuse_file_info *fi) {
|
||||
return instance().execute_callback(
|
||||
__FUNCTION__, path, [&](const std::string &api_path) -> api_error {
|
||||
return instance().fallocate_impl(api_path, mode, offset, length, fi);
|
||||
});
|
||||
}
|
||||
|
||||
int fuse_base::fgetattr_(const char *path, struct stat *st, struct fuse_file_info *fi) {
|
||||
return instance().execute_callback(__FUNCTION__, path,
|
||||
[&](const std::string &api_path) -> api_error {
|
||||
return instance().fgetattr_impl(api_path, st, fi);
|
||||
});
|
||||
}
|
||||
|
||||
#ifdef __APPLE__
|
||||
int fuse_base::fsetattr_x_(const char *path, struct setattr_x *attr, struct fuse_file_info *fi) {
|
||||
return instance().execute_callback(__FUNCTION__, path,
|
||||
[&](const std::string &api_path) -> api_error {
|
||||
return instance().fsetattr_x_impl(api_path, attr, fi);
|
||||
});
|
||||
}
|
||||
#endif // __APPLE__
|
||||
|
||||
int fuse_base::fsync_(const char *path, int datasync, struct fuse_file_info *fi) {
|
||||
return instance().execute_callback(__FUNCTION__, path,
|
||||
[&](const std::string &api_path) -> api_error {
|
||||
return instance().fsync_impl(api_path, datasync, fi);
|
||||
});
|
||||
}
|
||||
|
||||
int fuse_base::ftruncate_(const char *path, off_t size, struct fuse_file_info *fi) {
|
||||
return instance().execute_callback(__FUNCTION__, path,
|
||||
[&](const std::string &api_path) -> api_error {
|
||||
return instance().ftruncate_impl(api_path, size, fi);
|
||||
});
|
||||
}
|
||||
|
||||
uid_t fuse_base::get_effective_uid() const {
|
||||
return forced_uid_.has_value() ? forced_uid_.value() : fuse_get_context()->uid;
|
||||
}
|
||||
|
||||
uid_t fuse_base::get_effective_gid() const {
|
||||
return forced_gid_.has_value() ? forced_gid_.value() : fuse_get_context()->gid;
|
||||
}
|
||||
|
||||
#ifdef __APPLE__
|
||||
__uint32_t fuse_base::get_flags_from_meta(const api_meta_map &meta) {
|
||||
return utils::string::to_uint32(meta.at(META_OSXFLAGS));
|
||||
}
|
||||
#endif // __APPLE__
|
||||
|
||||
gid_t fuse_base::get_gid_from_meta(const api_meta_map &meta) {
|
||||
return static_cast<gid_t>(utils::string::to_uint32(meta.at(META_GID)));
|
||||
}
|
||||
|
||||
mode_t fuse_base::get_mode_from_meta(const api_meta_map &meta) {
|
||||
return static_cast<mode_t>(utils::string::to_uint32(meta.at(META_MODE)));
|
||||
}
|
||||
|
||||
void fuse_base::get_timespec_from_meta(const api_meta_map &meta, const std::string &name,
|
||||
struct timespec &ts) {
|
||||
const auto t = utils::string::to_uint64(meta.at(name));
|
||||
ts.tv_nsec = t % NANOS_PER_SECOND;
|
||||
ts.tv_sec = t / NANOS_PER_SECOND;
|
||||
}
|
||||
|
||||
uid_t fuse_base::get_uid_from_meta(const api_meta_map &meta) {
|
||||
return static_cast<uid_t>(utils::string::to_uint32(meta.at(META_UID)));
|
||||
}
|
||||
|
||||
int fuse_base::getattr_(const char *path, struct stat *st) {
|
||||
return instance().execute_callback(__FUNCTION__, path,
|
||||
[&](const std::string &api_path) -> api_error {
|
||||
return instance().getattr_impl(api_path, st);
|
||||
});
|
||||
}
|
||||
|
||||
#ifdef __APPLE__
|
||||
int fuse_base::getxtimes_(const char *path, struct timespec *bkuptime, struct timespec *crtime) {
|
||||
return instance().execute_callback(__FUNCTION__, path,
|
||||
[&](const std::string &api_path) -> api_error {
|
||||
return instance().getxtimes_impl(api_path, bkuptime, crtime);
|
||||
});
|
||||
}
|
||||
#endif // __APPLE__
|
||||
|
||||
void *fuse_base::init_(struct fuse_conn_info *conn) {
|
||||
return execute_void_pointer_callback(__FUNCTION__,
|
||||
[&]() -> void * { return instance().init_impl(conn); });
|
||||
}
|
||||
|
||||
void *fuse_base::init_impl(struct fuse_conn_info *conn) {
|
||||
#ifdef __APPLE__
|
||||
conn->want |= FUSE_CAP_VOL_RENAME;
|
||||
conn->want |= FUSE_CAP_XTIMES;
|
||||
#endif // __APPLE__
|
||||
conn->want |= FUSE_CAP_BIG_WRITES;
|
||||
|
||||
return this;
|
||||
}
|
||||
|
||||
int fuse_base::mkdir_(const char *path, mode_t mode) {
|
||||
return instance().execute_callback(__FUNCTION__, path,
|
||||
[&](const std::string &api_path) -> api_error {
|
||||
return instance().mkdir_impl(api_path, mode);
|
||||
});
|
||||
}
|
||||
|
||||
int fuse_base::mount(std::vector<std::string> args) {
|
||||
auto ret = parse_args(args);
|
||||
if (ret == 0) {
|
||||
std::vector<const char *> fuse_argv(args.size());
|
||||
for (std::size_t i = 0u; i < args.size(); i++) {
|
||||
fuse_argv[i] = args[i].c_str();
|
||||
}
|
||||
|
||||
{
|
||||
struct fuse_args fa =
|
||||
FUSE_ARGS_INIT(static_cast<int>(fuse_argv.size()), (char **)&fuse_argv[0]);
|
||||
|
||||
char *mount_location = nullptr;
|
||||
fuse_parse_cmdline(&fa, &mount_location, nullptr, nullptr);
|
||||
if (mount_location) {
|
||||
mount_location_ = mount_location;
|
||||
free(mount_location);
|
||||
}
|
||||
}
|
||||
|
||||
notify_fuse_args_parsed(args);
|
||||
|
||||
umask(0);
|
||||
ret = fuse_main(static_cast<int>(fuse_argv.size()), (char **)&fuse_argv[0], &fuse_ops_, this);
|
||||
notify_fuse_main_exit(ret);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
int fuse_base::open_(const char *path, struct fuse_file_info *fi) {
|
||||
return instance().execute_callback(
|
||||
__FUNCTION__, path,
|
||||
[&](const std::string &api_path) -> api_error { return instance().open_impl(api_path, fi); });
|
||||
}
|
||||
|
||||
int fuse_base::opendir_(const char *path, struct fuse_file_info *fi) {
|
||||
return instance().execute_callback(__FUNCTION__, path,
|
||||
[&](const std::string &api_path) -> api_error {
|
||||
return instance().opendir_impl(api_path, fi);
|
||||
});
|
||||
}
|
||||
|
||||
int fuse_base::read_(const char *path, char *buffer, size_t read_size, off_t read_offset,
|
||||
struct fuse_file_info *fi) {
|
||||
std::size_t bytes_read{};
|
||||
const auto res = instance().execute_callback(
|
||||
__FUNCTION__, path,
|
||||
[&](const std::string &api_path) -> api_error {
|
||||
return instance().read_impl(api_path, buffer, read_size, read_offset, fi, bytes_read);
|
||||
},
|
||||
true);
|
||||
return (res == 0) ? static_cast<int>(bytes_read) : res;
|
||||
}
|
||||
|
||||
int fuse_base::readdir_(const char *path, void *buf, fuse_fill_dir_t fuse_fill_dir, off_t offset,
|
||||
struct fuse_file_info *fi) {
|
||||
return instance().execute_callback(
|
||||
__FUNCTION__, path, [&](const std::string &api_path) -> api_error {
|
||||
return instance().readdir_impl(api_path, buf, fuse_fill_dir, offset, fi);
|
||||
});
|
||||
}
|
||||
|
||||
int fuse_base::release_(const char *path, struct fuse_file_info *fi) {
|
||||
return instance().execute_callback(__FUNCTION__, path,
|
||||
[&](const std::string &api_path) -> api_error {
|
||||
return instance().release_impl(api_path, fi);
|
||||
});
|
||||
}
|
||||
|
||||
int fuse_base::releasedir_(const char *path, struct fuse_file_info *fi) {
|
||||
return instance().execute_callback(__FUNCTION__, path,
|
||||
[&](const std::string &api_path) -> api_error {
|
||||
return instance().releasedir_impl(api_path, fi);
|
||||
});
|
||||
}
|
||||
|
||||
int fuse_base::rename_(const char *from, const char *to) {
|
||||
return instance().execute_callback(
|
||||
__FUNCTION__, from, to,
|
||||
[&](const std::string &from_api_file, const std::string &to_api_path) -> api_error {
|
||||
return instance().rename_impl(from_api_file, to_api_path);
|
||||
});
|
||||
}
|
||||
|
||||
int fuse_base::rmdir_(const char *path) {
|
||||
return instance().execute_callback(
|
||||
__FUNCTION__, path,
|
||||
[&](const std::string &api_path) -> api_error { return instance().rmdir_impl(api_path); });
|
||||
}
|
||||
|
||||
#ifdef HAS_SETXATTR
|
||||
#ifdef __APPLE__
|
||||
int fuse_base::getxattr_(const char *path, const char *name, char *value, size_t size,
|
||||
uint32_t position) {
|
||||
int attribute_size = 0;
|
||||
const auto res = instance().execute_callback(
|
||||
__FUNCTION__, path, [&](const std::string &api_path) -> api_error {
|
||||
return instance().getxattr_impl(api_path, name, value, size, position, attribute_size);
|
||||
});
|
||||
|
||||
return res == 0 ? attribute_size : res;
|
||||
}
|
||||
#else // __APPLE__
|
||||
int fuse_base::getxattr_(const char *path, const char *name, char *value, size_t size) {
|
||||
int attribute_size = 0;
|
||||
const auto res = instance().execute_callback(
|
||||
__FUNCTION__, path, [&](const std::string &api_path) -> api_error {
|
||||
return instance().getxattr_impl(api_path, name, value, size, attribute_size);
|
||||
});
|
||||
|
||||
return res == 0 ? attribute_size : res;
|
||||
}
|
||||
#endif // __APPLE__
|
||||
|
||||
int fuse_base::listxattr_(const char *path, char *buffer, size_t size) {
|
||||
int required_size = 0;
|
||||
bool return_size = false;
|
||||
|
||||
const auto res = instance().execute_callback(
|
||||
__FUNCTION__, path, [&](const std::string &api_path) -> api_error {
|
||||
return instance().listxattr_impl(api_path, buffer, size, required_size, return_size);
|
||||
});
|
||||
|
||||
return return_size ? required_size : res;
|
||||
}
|
||||
|
||||
int fuse_base::parse_args(std::vector<std::string> &args) {
|
||||
auto force_no_console = false;
|
||||
for (std::size_t i = 1; !force_no_console && (i < args.size()); i++) {
|
||||
if (args[i] == "-nc") {
|
||||
force_no_console = true;
|
||||
}
|
||||
}
|
||||
utils::remove_element_from(args, "-nc");
|
||||
|
||||
for (std::size_t i = 1; i < args.size(); i++) {
|
||||
if (args[i] == "-f") {
|
||||
console_enabled_ = not force_no_console;
|
||||
} else if (args[i].find("-o") == 0) {
|
||||
std::string options = "";
|
||||
if (args[i].size() == 2) {
|
||||
if ((i + 1) < args.size()) {
|
||||
options = args[++i];
|
||||
}
|
||||
} else {
|
||||
options = args[i].substr(2);
|
||||
}
|
||||
|
||||
const auto option_parts = utils::string::split(options, ',');
|
||||
for (const auto &option : option_parts) {
|
||||
if (option.find("gid") == 0) {
|
||||
const auto parts = utils::string::split(option, '=');
|
||||
if (parts.size() == 2) {
|
||||
auto gid = getgrnam(parts[1].c_str());
|
||||
if (not gid) {
|
||||
gid = getgrgid(utils::string::to_uint32(parts[1]));
|
||||
}
|
||||
if ((getgid() != 0) && (gid->gr_gid == 0)) {
|
||||
std::cerr << "'gid=0' requires running as root" << std::endl;
|
||||
return -1;
|
||||
} else {
|
||||
forced_gid_ = gid->gr_gid;
|
||||
}
|
||||
}
|
||||
} else if (option.find("noatime") == 0) {
|
||||
atime_enabled_ = false;
|
||||
} else if (option.find("uid") == 0) {
|
||||
const auto parts = utils::string::split(option, '=');
|
||||
if (parts.size() == 2) {
|
||||
auto uid = getpwnam(parts[1].c_str());
|
||||
if (not uid) {
|
||||
uid = getpwuid(utils::string::to_uint32(parts[1]));
|
||||
}
|
||||
if ((getuid() != 0) && (uid->pw_uid == 0)) {
|
||||
std::cerr << "'uid=0' requires running as root" << std::endl;
|
||||
return -1;
|
||||
} else {
|
||||
forced_uid_ = uid->pw_uid;
|
||||
}
|
||||
}
|
||||
} else if (option.find("umask") == 0) {
|
||||
const auto parts = utils::string::split(option, '=');
|
||||
if (parts.size() == 2) {
|
||||
static const auto match_number_regex = std::regex("[0-9]+");
|
||||
try {
|
||||
if (not std::regex_match(parts[1], match_number_regex)) {
|
||||
throw std::runtime_error("invalid syntax");
|
||||
} else {
|
||||
forced_umask_ = utils::string::to_uint32(parts[1]);
|
||||
}
|
||||
} catch (...) {
|
||||
std::cerr << ("'" + option + "' invalid syntax") << std::endl;
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
#ifdef __APPLE__
|
||||
api_error fuse_base::parse_xattr_parameters(const char *name, const uint32_t &position,
|
||||
std::string &attribute_name,
|
||||
const std::string &api_path) {
|
||||
#else
|
||||
api_error fuse_base::parse_xattr_parameters(const char *name, std::string &attribute_name,
|
||||
const std::string &api_path) {
|
||||
#endif
|
||||
auto res = api_path.empty() ? api_error::bad_address : api_error::success;
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
|
||||
if (not name) {
|
||||
return api_error::bad_address;
|
||||
}
|
||||
|
||||
attribute_name = std::string(name);
|
||||
#ifdef __APPLE__
|
||||
if (attribute_name == A_KAUTH_FILESEC_XATTR) {
|
||||
char new_name[MAXPATHLEN] = {0};
|
||||
memcpy(new_name, A_KAUTH_FILESEC_XATTR, sizeof(A_KAUTH_FILESEC_XATTR));
|
||||
memcpy(new_name, G_PREFIX, sizeof(G_PREFIX) - 1);
|
||||
attribute_name = new_name;
|
||||
} else if (attribute_name.empty() ||
|
||||
((attribute_name != XATTR_RESOURCEFORK_NAME) && (position != 0))) {
|
||||
return api_error::xattr_osx_invalid;
|
||||
}
|
||||
#endif
|
||||
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
#ifdef __APPLE__
|
||||
api_error fuse_base::parse_xattr_parameters(const char *name, const char *value, size_t size,
|
||||
const uint32_t &position, std::string &attribute_name,
|
||||
const std::string &api_path) {
|
||||
auto res = parse_xattr_parameters(name, position, attribute_name, api_path);
|
||||
#else
|
||||
api_error fuse_base::parse_xattr_parameters(const char *name, const char *value, size_t size,
|
||||
std::string &attribute_name,
|
||||
const std::string &api_path) {
|
||||
auto res = parse_xattr_parameters(name, attribute_name, api_path);
|
||||
#endif
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
|
||||
return (value ? api_error::success : (size ? api_error::bad_address : api_error::success));
|
||||
}
|
||||
|
||||
void fuse_base::populate_stat(const std::string &api_path, const std::uint64_t &size_or_count,
|
||||
const api_meta_map &meta, const bool &directory, i_provider &provider,
|
||||
struct stat *st) {
|
||||
memset(st, 0, sizeof(struct stat));
|
||||
st->st_nlink =
|
||||
(directory ? 2 + (size_or_count ? size_or_count : provider.get_directory_item_count(api_path))
|
||||
: 1);
|
||||
if (directory) {
|
||||
st->st_blocks = 0;
|
||||
} else {
|
||||
st->st_size = size_or_count;
|
||||
static const auto blockSizeStat = static_cast<std::uint64_t>(512u);
|
||||
static const auto blockSize = static_cast<std::uint64_t>(4096u);
|
||||
const auto size =
|
||||
utils::divide_with_ceiling(static_cast<std::uint64_t>(st->st_size), blockSize) * blockSize;
|
||||
st->st_blocks =
|
||||
std::max(blockSize / blockSizeStat, utils::divide_with_ceiling(size, blockSizeStat));
|
||||
}
|
||||
st->st_gid = get_gid_from_meta(meta);
|
||||
st->st_mode = (directory ? S_IFDIR : S_IFREG) | get_mode_from_meta(meta);
|
||||
st->st_uid = get_uid_from_meta(meta);
|
||||
#ifdef __APPLE__
|
||||
st->st_blksize = 0;
|
||||
st->st_flags = get_flags_from_meta(meta);
|
||||
|
||||
set_timespec_from_meta(meta, META_MODIFIED, st->st_mtimespec);
|
||||
set_timespec_from_meta(meta, META_CREATION, st->st_birthtimespec);
|
||||
set_timespec_from_meta(meta, META_CHANGED, st->st_ctimespec);
|
||||
set_timespec_from_meta(meta, META_ACCESSED, st->st_atimespec);
|
||||
#else // __APPLE__
|
||||
st->st_blksize = 4096;
|
||||
|
||||
set_timespec_from_meta(meta, META_MODIFIED, st->st_mtim);
|
||||
set_timespec_from_meta(meta, META_CREATION, st->st_ctim);
|
||||
set_timespec_from_meta(meta, META_ACCESSED, st->st_atim);
|
||||
#endif // __APPLE__
|
||||
}
|
||||
|
||||
void fuse_base::raise_fuse_event(std::string function_name, const std::string &api_path,
|
||||
const int &ret, const bool &disable_logging) {
|
||||
if ((ret >= 0) && disable_logging) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (not config_.get_enable_drive_events()) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (((config_.get_event_level() >= fuse_event::level) && (ret != 0)) ||
|
||||
(config_.get_event_level() >= event_level::verbose)) {
|
||||
event_system::instance().raise<fuse_event>(utils::string::right_trim(function_name, '_'),
|
||||
api_path, ret);
|
||||
}
|
||||
}
|
||||
|
||||
int fuse_base::removexattr_(const char *path, const char *name) {
|
||||
return instance().execute_callback(__FUNCTION__, path,
|
||||
[&](const std::string &api_path) -> api_error {
|
||||
return instance().removexattr_impl(api_path, name);
|
||||
});
|
||||
}
|
||||
|
||||
void fuse_base::set_timespec_from_meta(const api_meta_map &meta, const std::string &name,
|
||||
struct timespec &ts) {
|
||||
const auto t = utils::string::to_uint64(meta.at(name));
|
||||
ts.tv_nsec = t % NANOS_PER_SECOND;
|
||||
ts.tv_sec = t / NANOS_PER_SECOND;
|
||||
}
|
||||
|
||||
#ifdef __APPLE__
|
||||
int fuse_base::setxattr_(const char *path, const char *name, const char *value, size_t size,
|
||||
int flags, uint32_t position) {
|
||||
const auto res = instance().execute_callback(
|
||||
__FUNCTION__, path, [&](const std::string &api_path) -> api_error {
|
||||
return instance().setxattr_impl(api_path, name, value, size, flags, position);
|
||||
});
|
||||
if (res != 0) {
|
||||
errno = std::abs(res);
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
#else // __APPLE__
|
||||
int fuse_base::setxattr_(const char *path, const char *name, const char *value, size_t size,
|
||||
int flags) {
|
||||
const auto res = instance().execute_callback(
|
||||
__FUNCTION__, path, [&](const std::string &api_path) -> api_error {
|
||||
return instance().setxattr_impl(api_path, name, value, size, flags);
|
||||
});
|
||||
if (res != 0) {
|
||||
errno = std::abs(res);
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
#endif // __APPLE__
|
||||
#endif // HAS_SETXATTR
|
||||
|
||||
int fuse_base::shutdown() {
|
||||
#if __APPLE__
|
||||
const auto unmount = "umount \"" + mount_location_ + "\" >/dev/null 2>&1";
|
||||
#else
|
||||
const auto unmount = "fusermount -u \"" + mount_location_ + "\" >/dev/null 2>&1";
|
||||
#endif
|
||||
|
||||
return system(unmount.c_str());
|
||||
}
|
||||
|
||||
#ifdef __APPLE__
|
||||
int fuse_base::setattr_x_(const char *path, struct setattr_x *attr) {
|
||||
return instance().execute_callback(__FUNCTION__, path,
|
||||
[&](const std::string &api_path) -> api_error {
|
||||
return instance().setattr_x_impl(api_path, attr);
|
||||
});
|
||||
}
|
||||
|
||||
int fuse_base::setbkuptime_(const char *path, const struct timespec *bkuptime) {
|
||||
return instance().execute_callback(__FUNCTION__, path,
|
||||
[&](const std::string &api_path) -> api_error {
|
||||
return instance().setbkuptime_impl(api_path, bkuptime);
|
||||
});
|
||||
}
|
||||
|
||||
int fuse_base::setchgtime_(const char *path, const struct timespec *chgtime) {
|
||||
return instance().execute_callback(__FUNCTION__, path,
|
||||
[&](const std::string &api_path) -> api_error {
|
||||
return instance().setchgtime_impl(api_path, chgtime);
|
||||
});
|
||||
}
|
||||
|
||||
int fuse_base::setcrtime_(const char *path, const struct timespec *crtime) {
|
||||
return instance().execute_callback(__FUNCTION__, path,
|
||||
[&](const std::string &api_path) -> api_error {
|
||||
return instance().setcrtime_impl(api_path, crtime);
|
||||
});
|
||||
}
|
||||
|
||||
int fuse_base::setvolname_(const char *volname) {
|
||||
return instance().execute_callback(__FUNCTION__, volname,
|
||||
[&](const std::string &api_path) -> api_error {
|
||||
return instance().setvolname_impl(volname);
|
||||
});
|
||||
}
|
||||
|
||||
int fuse_base::statfs_x_(const char *path, struct statfs *stbuf) {
|
||||
return instance().execute_callback(__FUNCTION__, path,
|
||||
[&](const std::string &api_path) -> api_error {
|
||||
return instance().statfs_x_impl(api_path, stbuf);
|
||||
});
|
||||
}
|
||||
#else // __APPLE__
|
||||
int fuse_base::statfs_(const char *path, struct statvfs *stbuf) {
|
||||
return instance().execute_callback(__FUNCTION__, path,
|
||||
[&](const std::string &api_path) -> api_error {
|
||||
return instance().statfs_impl(api_path, stbuf);
|
||||
});
|
||||
}
|
||||
#endif // __APPLE__
|
||||
|
||||
int fuse_base::truncate_(const char *path, off_t size) {
|
||||
return instance().execute_callback(__FUNCTION__, path,
|
||||
[&](const std::string &api_path) -> api_error {
|
||||
return instance().truncate_impl(api_path, size);
|
||||
});
|
||||
}
|
||||
|
||||
int fuse_base::unlink_(const char *path) {
|
||||
return instance().execute_callback(
|
||||
__FUNCTION__, path,
|
||||
[&](const std::string &api_path) -> api_error { return instance().unlink_impl(api_path); });
|
||||
}
|
||||
|
||||
int fuse_base::utimens_(const char *path, const struct timespec tv[2]) {
|
||||
return instance().execute_callback(__FUNCTION__, path,
|
||||
[&](const std::string &api_path) -> api_error {
|
||||
return instance().utimens_impl(api_path, tv);
|
||||
});
|
||||
}
|
||||
|
||||
int fuse_base::write_(const char *path, const char *buffer, size_t write_size, off_t write_offset,
|
||||
struct fuse_file_info *fi) {
|
||||
std::size_t bytes_written{};
|
||||
|
||||
const auto res = instance().execute_callback(
|
||||
__FUNCTION__, path,
|
||||
[&](const std::string &api_path) -> api_error {
|
||||
return instance().write_impl(api_path, buffer, write_size, write_offset, fi, bytes_written);
|
||||
},
|
||||
true);
|
||||
return (res == 0) ? static_cast<int>(bytes_written) : res;
|
||||
}
|
||||
} // namespace repertory
|
||||
|
||||
#endif // _WIN32
|
||||
1182
src/drives/fuse/fuse_drive.cpp
Normal file
1182
src/drives/fuse/fuse_drive.cpp
Normal file
File diff suppressed because it is too large
Load Diff
716
src/drives/fuse/remotefuse/remote_client.cpp
Normal file
716
src/drives/fuse/remotefuse/remote_client.cpp
Normal file
@@ -0,0 +1,716 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "drives/fuse/remotefuse/remote_client.hpp"
|
||||
#include "comm/packet/packet.hpp"
|
||||
#include "app_config.hpp"
|
||||
#include "events/events.hpp"
|
||||
#include "utils/path_utils.hpp"
|
||||
|
||||
namespace repertory::remote_fuse {
|
||||
// clang-format off
|
||||
E_SIMPLE3(remote_fuse_client_event, debug, true,
|
||||
std::string, function, func, E_STRING,
|
||||
std::string, api_path, ap, E_STRING,
|
||||
int, result, res, E_FROM_INT32
|
||||
);
|
||||
// clang-format on
|
||||
|
||||
#define RAISE_REMOTE_CLIENT_FUSE_EVENT(func, file, ret) \
|
||||
if (config_.get_enable_drive_events() && \
|
||||
(((config_.get_event_level() >= remote_fuse_client_event::level) && (ret != 0)) || \
|
||||
(config_.get_event_level() >= event_level::verbose))) \
|
||||
event_system::instance().raise<remote_fuse_client_event>(func, file, ret)
|
||||
|
||||
remote_client::remote_client(const app_config &config)
|
||||
: config_(config),
|
||||
packet_client_(config.get_remote_host_name_or_ip(), config.get_remote_max_connections(),
|
||||
config.get_remote_port(), config.get_remote_receive_timeout_secs(),
|
||||
config.get_remote_send_timeout_secs(), config.get_remote_token()) {}
|
||||
|
||||
packet::error_type remote_client::fuse_access(const char *path, const std::int32_t &mask) {
|
||||
packet request;
|
||||
request.encode(path);
|
||||
request.encode(mask);
|
||||
|
||||
std::uint32_t service_flags = 0;
|
||||
const auto ret = packet_client_.send(__FUNCTION__, request, service_flags);
|
||||
RAISE_REMOTE_CLIENT_FUSE_EVENT(__FUNCTION__, utils::path::create_api_path(path), ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::fuse_chflags(const char *path, const std::uint32_t &flags) {
|
||||
packet request;
|
||||
request.encode(path);
|
||||
request.encode(flags);
|
||||
|
||||
std::uint32_t service_flags = 0;
|
||||
const auto ret = packet_client_.send(__FUNCTION__, request, service_flags);
|
||||
RAISE_REMOTE_CLIENT_FUSE_EVENT(__FUNCTION__, utils::path::create_api_path(path), ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::fuse_chmod(const char *path, const remote::file_mode &mode) {
|
||||
packet request;
|
||||
request.encode(path);
|
||||
request.encode(mode);
|
||||
|
||||
std::uint32_t service_flags = 0;
|
||||
const auto ret = packet_client_.send(__FUNCTION__, request, service_flags);
|
||||
RAISE_REMOTE_CLIENT_FUSE_EVENT(__FUNCTION__, utils::path::create_api_path(path), ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::fuse_chown(const char *path, const remote::user_id &uid,
|
||||
const remote::group_id &gid) {
|
||||
packet request;
|
||||
request.encode(path);
|
||||
request.encode(uid);
|
||||
request.encode(gid);
|
||||
|
||||
std::uint32_t service_flags = 0;
|
||||
const auto ret = packet_client_.send(__FUNCTION__, request, service_flags);
|
||||
RAISE_REMOTE_CLIENT_FUSE_EVENT(__FUNCTION__, utils::path::create_api_path(path), ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::fuse_destroy() {
|
||||
std::uint32_t service_flags = 0;
|
||||
const auto ret = packet_client_.send(__FUNCTION__, service_flags);
|
||||
RAISE_REMOTE_CLIENT_FUSE_EVENT(__FUNCTION__, "", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/*packet::error_type remote_client::fuse_fallocate(const char *path, const std::int32_t &mode,
|
||||
const remote::file_offset &offset,
|
||||
const remote::file_offset &length,
|
||||
const remote::file_handle &handle) {
|
||||
packet request;
|
||||
request.encode(path);
|
||||
request.encode(mode);
|
||||
request.encode(offset);
|
||||
request.encode(length);
|
||||
request.encode(handle);
|
||||
|
||||
std::uint32_t service_flags = 0;
|
||||
const auto ret = packetClient_.send(__FUNCTION__, request, service_flags);
|
||||
RAISE_REMOTE_CLIENT_FUSE_EVENT(__FUNCTION__, utils::path::create_api_path(path), ret);
|
||||
return ret;
|
||||
}*/
|
||||
|
||||
packet::error_type remote_client::fuse_fgetattr(const char *path, remote::stat &st, bool &directory,
|
||||
const remote::file_handle &handle) {
|
||||
packet request;
|
||||
request.encode(path);
|
||||
request.encode(handle);
|
||||
request.encode(uid_);
|
||||
request.encode(gid_);
|
||||
|
||||
packet response;
|
||||
std::uint32_t service_flags = 0;
|
||||
auto ret = packet_client_.send(__FUNCTION__, request, response, service_flags);
|
||||
if (ret == 0) {
|
||||
if ((ret = response.decode(st)) == 0) {
|
||||
std::uint8_t d = 0;
|
||||
if ((ret = response.decode(d)) == 0) {
|
||||
directory = static_cast<bool>(d);
|
||||
}
|
||||
}
|
||||
}
|
||||
RAISE_REMOTE_CLIENT_FUSE_EVENT(__FUNCTION__, utils::path::create_api_path(path), ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::fuse_fsetattr_x(const char *path, const remote::setattr_x &attr,
|
||||
const remote::file_handle &handle) {
|
||||
packet request;
|
||||
request.encode(path);
|
||||
request.encode(attr);
|
||||
request.encode(handle);
|
||||
|
||||
std::uint32_t service_flags = 0;
|
||||
const auto ret = packet_client_.send(__FUNCTION__, request, service_flags);
|
||||
RAISE_REMOTE_CLIENT_FUSE_EVENT(__FUNCTION__, utils::path::create_api_path(path), ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::fuse_fsync(const char *path, const std::int32_t &datasync,
|
||||
const remote::file_handle &handle) {
|
||||
packet request;
|
||||
request.encode(path);
|
||||
request.encode(datasync);
|
||||
request.encode(handle);
|
||||
|
||||
std::uint32_t service_flags = 0;
|
||||
const auto ret = packet_client_.send(__FUNCTION__, request, service_flags);
|
||||
RAISE_REMOTE_CLIENT_FUSE_EVENT(__FUNCTION__, utils::path::create_api_path(path), ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::fuse_ftruncate(const char *path, const remote::file_offset &size,
|
||||
const remote::file_handle &handle) {
|
||||
packet request;
|
||||
request.encode(path);
|
||||
request.encode(size);
|
||||
request.encode(handle);
|
||||
|
||||
std::uint32_t service_flags = 0;
|
||||
const auto ret = packet_client_.send(__FUNCTION__, request, service_flags);
|
||||
RAISE_REMOTE_CLIENT_FUSE_EVENT(__FUNCTION__, utils::path::create_api_path(path), ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::fuse_getattr(const char *path, remote::stat &st,
|
||||
bool &directory) {
|
||||
packet request;
|
||||
request.encode(path);
|
||||
request.encode(uid_);
|
||||
request.encode(gid_);
|
||||
|
||||
packet response;
|
||||
std::uint32_t service_flags = 0;
|
||||
auto ret = packet_client_.send(__FUNCTION__, request, response, service_flags);
|
||||
if (ret == 0) {
|
||||
if ((ret = response.decode(st)) == 0) {
|
||||
std::uint8_t d = 0;
|
||||
if ((ret = response.decode(d)) == 0) {
|
||||
directory = static_cast<bool>(d);
|
||||
}
|
||||
}
|
||||
}
|
||||
RAISE_REMOTE_CLIENT_FUSE_EVENT(__FUNCTION__, utils::path::create_api_path(path), ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/*packet::error_type remote_client::fuse_getxattr(const char *path, const char *name, char
|
||||
*value, const remote::file_size &size) { packet::error_type ret = 0; if (size >
|
||||
std::numeric_limits<std::size_t>::max()) { ret = -ERANGE; } else { packet request;
|
||||
request.encode(path);
|
||||
request.encode(name);
|
||||
request.encode(size);
|
||||
|
||||
packet response;
|
||||
std::uint32_t service_flags = 0;
|
||||
if ((ret = packetClient_.send(__FUNCTION__, request, response, service_flags)) == 0) {
|
||||
remote::file_size size2;
|
||||
if ((ret = response.decode(size2)) == 0) {
|
||||
if (size2 > std::numeric_limits<std::size_t>::max()) {
|
||||
ret = -ERANGE;
|
||||
} else {
|
||||
memcpy(value, response.CurrentPointer(), static_cast<std::size_t>(size2));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
RAISE_REMOTE_CLIENT_FUSE_EVENT(__FUNCTION__, utils::path::create_api_path(path), ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::fuse_getxattr_osx(const char *path, const char *name, char
|
||||
*value, const remote::file_size &size, const std::uint32_t &position) { packet::error_type ret
|
||||
= 0; if (size > std::numeric_limits<std::size_t>::max()) { ret = -ERANGE; } else { packet request;
|
||||
request.encode(path);
|
||||
request.encode(name);
|
||||
request.encode(size);
|
||||
request.encode(position);
|
||||
|
||||
packet response;
|
||||
std::uint32_t service_flags = 0;
|
||||
if ((ret = packetClient_.send(__FUNCTION__, request, response, service_flags)) == 0) {
|
||||
remote::file_size size2;
|
||||
if ((ret = response.decode(size2)) == 0) {
|
||||
if (size2 > std::numeric_limits<std::size_t>::max()) {
|
||||
ret = -ERANGE;
|
||||
} else {
|
||||
memcpy(value, response.CurrentPointer(), static_cast<std::size_t>(size2));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
RAISE_REMOTE_CLIENT_FUSE_EVENT(__FUNCTION__, utils::path::create_api_path(path), ret);
|
||||
return ret;
|
||||
}*/
|
||||
|
||||
packet::error_type remote_client::fuse_getxtimes(const char *path, remote::file_time &bkuptime,
|
||||
remote::file_time &crtime) {
|
||||
packet request;
|
||||
request.encode(path);
|
||||
|
||||
packet response;
|
||||
std::uint32_t service_flags = 0;
|
||||
auto ret = packet_client_.send(__FUNCTION__, request, response, service_flags);
|
||||
if (ret == 0) {
|
||||
response.decode(bkuptime);
|
||||
response.decode(crtime);
|
||||
}
|
||||
RAISE_REMOTE_CLIENT_FUSE_EVENT(__FUNCTION__, utils::path::create_api_path(path), ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::fuse_init() {
|
||||
std::uint32_t service_flags = 0;
|
||||
auto ret = packet_client_.send(__FUNCTION__, service_flags);
|
||||
RAISE_REMOTE_CLIENT_FUSE_EVENT(__FUNCTION__, "", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/*packet::error_type remote_client::fuse_listxattr(const char *path, char *buffer,
|
||||
const remote::file_size &size) {
|
||||
packet::error_type ret = 0;
|
||||
if (size > std::numeric_limits<std::size_t>::max()) {
|
||||
ret = -ERANGE;
|
||||
} else {
|
||||
packet request;
|
||||
request.encode(path);
|
||||
request.encode(size);
|
||||
|
||||
packet response;
|
||||
std::uint32_t service_flags = 0;
|
||||
if ((ret = packetClient_.send(__FUNCTION__, request, response, service_flags)) == 0) {
|
||||
remote::file_size size2;
|
||||
if ((ret = response.decode(size2)) == 0) {
|
||||
if (size2 > std::numeric_limits<std::size_t>::max()) {
|
||||
ret = -ERANGE;
|
||||
} else {
|
||||
memcpy(buffer, response.CurrentPointer(), static_cast<std::size_t>(size2));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
RAISE_REMOTE_CLIENT_FUSE_EVENT(__FUNCTION__, utils::path::create_api_path(path), ret);
|
||||
return ret;
|
||||
}*/
|
||||
|
||||
packet::error_type remote_client::fuse_mkdir(const char *path, const remote::file_mode &mode) {
|
||||
packet request;
|
||||
request.encode(path);
|
||||
request.encode(mode);
|
||||
|
||||
std::uint32_t service_flags = 0;
|
||||
const auto ret = packet_client_.send(__FUNCTION__, request, service_flags);
|
||||
RAISE_REMOTE_CLIENT_FUSE_EVENT(__FUNCTION__, utils::path::create_api_path(path), ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::fuse_opendir(const char *path, remote::file_handle &handle) {
|
||||
packet request;
|
||||
request.encode(path);
|
||||
|
||||
packet response;
|
||||
std::uint32_t service_flags = 0;
|
||||
auto ret = packet_client_.send(__FUNCTION__, request, response, service_flags);
|
||||
if (ret == 0) {
|
||||
ret = response.decode(handle);
|
||||
}
|
||||
RAISE_REMOTE_CLIENT_FUSE_EVENT(__FUNCTION__, utils::path::create_api_path(path), ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::fuse_create(const char *path, const remote::file_mode &mode,
|
||||
const remote::open_flags &flags,
|
||||
remote::file_handle &handle) {
|
||||
packet request;
|
||||
request.encode(path);
|
||||
request.encode(mode);
|
||||
request.encode(flags);
|
||||
|
||||
packet response;
|
||||
std::uint32_t service_flags = 0;
|
||||
auto ret = packet_client_.send(__FUNCTION__, request, response, service_flags);
|
||||
if (ret == 0) {
|
||||
ret = response.decode(handle);
|
||||
}
|
||||
RAISE_REMOTE_CLIENT_FUSE_EVENT(__FUNCTION__, utils::path::create_api_path(path), ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::fuse_open(const char *path, const remote::open_flags &flags,
|
||||
remote::file_handle &handle) {
|
||||
packet request;
|
||||
request.encode(path);
|
||||
request.encode(flags);
|
||||
|
||||
packet response;
|
||||
std::uint32_t service_flags = 0;
|
||||
auto ret = packet_client_.send(__FUNCTION__, request, response, service_flags);
|
||||
if (ret == 0) {
|
||||
ret = response.decode(handle);
|
||||
}
|
||||
RAISE_REMOTE_CLIENT_FUSE_EVENT(__FUNCTION__, utils::path::create_api_path(path), ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::fuse_read(const char *path, char *buffer,
|
||||
const remote::file_size &read_size,
|
||||
const remote::file_offset &read_offset,
|
||||
const remote::file_handle &handle) {
|
||||
packet request;
|
||||
request.encode(path);
|
||||
request.encode(read_size);
|
||||
request.encode(read_offset);
|
||||
request.encode(handle);
|
||||
|
||||
packet response;
|
||||
std::uint32_t service_flags = 0;
|
||||
auto ret = packet_client_.send(__FUNCTION__, request, response, service_flags);
|
||||
if (ret > 0) {
|
||||
memcpy(buffer, response.current_pointer(), ret);
|
||||
}
|
||||
|
||||
if (ret < 0) {
|
||||
RAISE_REMOTE_CLIENT_FUSE_EVENT(__FUNCTION__, utils::path::create_api_path(path), ret);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::fuse_rename(const char *from, const char *to) {
|
||||
packet request;
|
||||
request.encode(from);
|
||||
request.encode(to);
|
||||
|
||||
std::uint32_t service_flags = 0;
|
||||
const auto ret = packet_client_.send(__FUNCTION__, request, service_flags);
|
||||
RAISE_REMOTE_CLIENT_FUSE_EVENT(
|
||||
__FUNCTION__, utils::path::create_api_path(from) + "|" + utils::path::create_api_path(to),
|
||||
ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::fuse_write(const char *path, const char *buffer,
|
||||
const remote::file_size &write_size,
|
||||
const remote::file_offset &write_offset,
|
||||
const remote::file_handle &handle) {
|
||||
packet::error_type ret = 0;
|
||||
if (write_size > std::numeric_limits<std::size_t>::max()) {
|
||||
ret = -ERANGE;
|
||||
} else {
|
||||
packet request;
|
||||
request.encode(path);
|
||||
request.encode(write_size);
|
||||
request.encode(buffer, static_cast<std::size_t>(write_size));
|
||||
request.encode(write_offset);
|
||||
request.encode(handle);
|
||||
|
||||
std::uint32_t service_flags = 0;
|
||||
ret = packet_client_.send(__FUNCTION__, request, service_flags);
|
||||
}
|
||||
|
||||
if (ret < 0) {
|
||||
RAISE_REMOTE_CLIENT_FUSE_EVENT(__FUNCTION__, utils::path::create_api_path(path), ret);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::fuse_write_base64(const char *path, const char *buffer,
|
||||
const remote::file_size &write_size,
|
||||
const remote::file_offset &write_offset,
|
||||
const remote::file_handle &handle) {
|
||||
packet::error_type ret = 0;
|
||||
if (write_size > std::numeric_limits<std::size_t>::max()) {
|
||||
ret = -ERANGE;
|
||||
} else {
|
||||
packet request;
|
||||
request.encode(path);
|
||||
request.encode(write_size);
|
||||
request.encode(buffer, static_cast<std::size_t>(write_size));
|
||||
request.encode(write_offset);
|
||||
request.encode(handle);
|
||||
|
||||
std::uint32_t service_flags = 0;
|
||||
ret = packet_client_.send(__FUNCTION__, request, service_flags);
|
||||
}
|
||||
|
||||
if (ret < 0) {
|
||||
RAISE_REMOTE_CLIENT_FUSE_EVENT(__FUNCTION__, utils::path::create_api_path(path), ret);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::fuse_readdir(const char *path, const remote::file_offset &offset,
|
||||
const remote::file_handle &handle,
|
||||
std::string &item_path) {
|
||||
packet request;
|
||||
request.encode(path);
|
||||
request.encode(offset);
|
||||
request.encode(handle);
|
||||
|
||||
packet response;
|
||||
std::uint32_t service_flags = 0;
|
||||
auto ret = packet_client_.send(__FUNCTION__, request, response, service_flags);
|
||||
if (ret == 0) {
|
||||
DECODE_OR_IGNORE(&response, item_path);
|
||||
}
|
||||
|
||||
RAISE_REMOTE_CLIENT_FUSE_EVENT(__FUNCTION__, utils::path::create_api_path(path),
|
||||
ret == -120 ? 0 : ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::fuse_release(const char *path,
|
||||
const remote::file_handle &handle) {
|
||||
packet request;
|
||||
request.encode(path);
|
||||
request.encode(handle);
|
||||
|
||||
std::uint32_t service_flags = 0;
|
||||
const auto ret = packet_client_.send(__FUNCTION__, request, service_flags);
|
||||
RAISE_REMOTE_CLIENT_FUSE_EVENT(__FUNCTION__, utils::path::create_api_path(path), ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::fuse_releasedir(const char *path,
|
||||
const remote::file_handle &handle) {
|
||||
packet request;
|
||||
request.encode(path);
|
||||
request.encode(handle);
|
||||
|
||||
std::uint32_t service_flags = 0;
|
||||
const auto ret = packet_client_.send(__FUNCTION__, request, service_flags);
|
||||
RAISE_REMOTE_CLIENT_FUSE_EVENT(__FUNCTION__, utils::path::create_api_path(path), ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/*packet::error_type remote_client::fuse_removexattr(const char *path, const char *name)
|
||||
override { packet request; request.encode(path); request.Encode(name);
|
||||
|
||||
std::uint32_t service_flags = 0;
|
||||
const auto ret = packetClient_.send(__FUNCTION__, request, service_flags);
|
||||
RAISE_REMOTE_CLIENT_FUSE_EVENT(__FUNCTION__, utils::path::create_api_path(path), ret);
|
||||
return ret;
|
||||
}*/
|
||||
|
||||
packet::error_type remote_client::fuse_rmdir(const char *path) {
|
||||
packet request;
|
||||
request.encode(path);
|
||||
|
||||
std::uint32_t service_flags = 0;
|
||||
const auto ret = packet_client_.send(__FUNCTION__, request, service_flags);
|
||||
RAISE_REMOTE_CLIENT_FUSE_EVENT(__FUNCTION__, utils::path::create_api_path(path), ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::fuse_setattr_x(const char *path, remote::setattr_x &attr) {
|
||||
packet request;
|
||||
request.encode(path);
|
||||
request.encode(attr);
|
||||
|
||||
std::uint32_t service_flags = 0;
|
||||
const auto ret = packet_client_.send(__FUNCTION__, request, service_flags);
|
||||
RAISE_REMOTE_CLIENT_FUSE_EVENT(__FUNCTION__, utils::path::create_api_path(path), ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::fuse_setbkuptime(const char *path,
|
||||
const remote::file_time &bkuptime) {
|
||||
packet request;
|
||||
request.encode(path);
|
||||
request.encode(bkuptime);
|
||||
|
||||
std::uint32_t service_flags = 0;
|
||||
const auto ret = packet_client_.send(__FUNCTION__, request, service_flags);
|
||||
RAISE_REMOTE_CLIENT_FUSE_EVENT(__FUNCTION__, utils::path::create_api_path(path), ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::fuse_setchgtime(const char *path,
|
||||
const remote::file_time &chgtime) {
|
||||
packet request;
|
||||
request.encode(path);
|
||||
request.encode(chgtime);
|
||||
|
||||
std::uint32_t service_flags = 0;
|
||||
const auto ret = packet_client_.send(__FUNCTION__, request, service_flags);
|
||||
RAISE_REMOTE_CLIENT_FUSE_EVENT(__FUNCTION__, utils::path::create_api_path(path), ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::fuse_setcrtime(const char *path,
|
||||
const remote::file_time &crtime) {
|
||||
packet request;
|
||||
request.encode(path);
|
||||
request.encode(crtime);
|
||||
|
||||
std::uint32_t service_flags = 0;
|
||||
const auto ret = packet_client_.send(__FUNCTION__, request, service_flags);
|
||||
RAISE_REMOTE_CLIENT_FUSE_EVENT(__FUNCTION__, utils::path::create_api_path(path), ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::fuse_setvolname(const char *volname) {
|
||||
packet request;
|
||||
request.encode(volname);
|
||||
|
||||
std::uint32_t service_flags = 0;
|
||||
const auto ret = packet_client_.send(__FUNCTION__, request, service_flags);
|
||||
RAISE_REMOTE_CLIENT_FUSE_EVENT(__FUNCTION__, volname ? volname : "", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/*packet::error_type remote_client::fuse_setxattr(const char *path, const char *name, const
|
||||
char *value, const remote::file_size &size, const std::int32_t &flags) { packet::error_type ret
|
||||
= 0; if (size > std::numeric_limits<std::size_t>::max()) { ret = -ERANGE; } else { packet request;
|
||||
request.encode(path);
|
||||
request.encode(name);
|
||||
request.encode(size);
|
||||
request.encode(value, static_cast<std::size_t>(size));
|
||||
request.encode(flags);
|
||||
|
||||
std::uint32_t service_flags = 0;
|
||||
ret = packetClient_.send(__FUNCTION__, request, service_flags);
|
||||
}
|
||||
|
||||
RAISE_REMOTE_CLIENT_FUSE_EVENT(__FUNCTION__, utils::path::create_api_path(path), ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::fuse_setxattr_osx(const char *path, const char *name, const
|
||||
char *value, const remote::file_size &size, const std::int32_t &flags, const std::uint32_t
|
||||
&position) override { packet::error_type ret = 0; if (size >
|
||||
std::numeric_limits<std::size_t>::max()) { ret = -ERANGE; } else { packet request;
|
||||
request.encode(path); request.Encode(name); request.Encode(size); request.encode(value,
|
||||
static_cast<std::size_t>(size)); request.encode(flags); request.encode(position);
|
||||
|
||||
std::uint32_t service_flags = 0;
|
||||
ret = packetClient_.send(__FUNCTION__, request, service_flags);
|
||||
}
|
||||
RAISE_REMOTE_CLIENT_FUSE_EVENT(__FUNCTION__, utils::path::create_api_path(path), ret);
|
||||
return ret;
|
||||
}*/
|
||||
|
||||
packet::error_type remote_client::fuse_statfs(const char *path, const std::uint64_t &frsize,
|
||||
remote::statfs &st) {
|
||||
packet request;
|
||||
request.encode(path);
|
||||
request.encode(frsize);
|
||||
|
||||
packet response;
|
||||
std::uint32_t service_flags = 0;
|
||||
const auto ret = packet_client_.send(__FUNCTION__, request, response, service_flags);
|
||||
if (ret == 0) {
|
||||
response.decode(st);
|
||||
}
|
||||
RAISE_REMOTE_CLIENT_FUSE_EVENT(__FUNCTION__, utils::path::create_api_path(path), ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::fuse_statfs_x(const char *path, const std::uint64_t &bsize,
|
||||
remote::statfs_x &st) {
|
||||
packet request;
|
||||
request.encode(path);
|
||||
request.encode(bsize);
|
||||
|
||||
packet response;
|
||||
std::uint32_t service_flags = 0;
|
||||
const auto ret = packet_client_.send(__FUNCTION__, request, response, service_flags);
|
||||
if (ret == 0) {
|
||||
response.decode(st);
|
||||
}
|
||||
RAISE_REMOTE_CLIENT_FUSE_EVENT(__FUNCTION__, utils::path::create_api_path(path), ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::fuse_truncate(const char *path, const remote::file_offset &size) {
|
||||
packet request;
|
||||
request.encode(path);
|
||||
request.encode(size);
|
||||
|
||||
std::uint32_t service_flags = 0;
|
||||
const auto ret = packet_client_.send(__FUNCTION__, request, service_flags);
|
||||
RAISE_REMOTE_CLIENT_FUSE_EVENT(__FUNCTION__, utils::path::create_api_path(path), ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::fuse_unlink(const char *path) {
|
||||
packet request;
|
||||
request.encode(path);
|
||||
|
||||
std::uint32_t service_flags = 0;
|
||||
const auto ret = packet_client_.send(__FUNCTION__, request, service_flags);
|
||||
RAISE_REMOTE_CLIENT_FUSE_EVENT(__FUNCTION__, utils::path::create_api_path(path), ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::fuse_utimens(const char *path, const remote::file_time *tv,
|
||||
const std::uint64_t &op0, const std::uint64_t &op1) {
|
||||
packet request;
|
||||
request.encode(path);
|
||||
request.encode(&tv[0], sizeof(remote::file_time) * 2);
|
||||
request.encode(op0);
|
||||
request.encode(op1);
|
||||
|
||||
std::uint32_t service_flags = 0;
|
||||
const auto ret = packet_client_.send(__FUNCTION__, request, service_flags);
|
||||
RAISE_REMOTE_CLIENT_FUSE_EVENT(__FUNCTION__, utils::path::create_api_path(path), ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::json_create_directory_snapshot(const std::string &path,
|
||||
json &json_data) {
|
||||
packet request;
|
||||
request.encode(path);
|
||||
|
||||
packet response;
|
||||
std::uint32_t service_flags = 0;
|
||||
auto ret = packet_client_.send(__FUNCTION__, request, response, service_flags);
|
||||
if (ret == 0) {
|
||||
ret = packet::decode_json(response, json_data);
|
||||
}
|
||||
|
||||
RAISE_REMOTE_CLIENT_FUSE_EVENT(__FUNCTION__, path, ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::json_read_directory_snapshot(const std::string &path,
|
||||
const remote::file_handle &handle,
|
||||
const std::uint32_t &page,
|
||||
json &json_data) {
|
||||
packet request;
|
||||
request.encode(path);
|
||||
request.encode(handle);
|
||||
request.encode(page);
|
||||
|
||||
packet response;
|
||||
std::uint32_t service_flags = 0;
|
||||
auto ret = packet_client_.send(__FUNCTION__, request, response, service_flags);
|
||||
if (ret == 0) {
|
||||
ret = packet::decode_json(response, json_data);
|
||||
}
|
||||
|
||||
RAISE_REMOTE_CLIENT_FUSE_EVENT(__FUNCTION__, path, ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type
|
||||
remote_client::json_release_directory_snapshot(const std::string &path,
|
||||
const remote::file_handle &handle) {
|
||||
packet request;
|
||||
request.encode(path);
|
||||
request.encode(handle);
|
||||
|
||||
std::uint32_t service_flags = 0;
|
||||
const auto ret = packet_client_.send(__FUNCTION__, request, service_flags);
|
||||
|
||||
RAISE_REMOTE_CLIENT_FUSE_EVENT(__FUNCTION__, path, ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
void remote_client::set_fuse_uid_gid(const remote::user_id &uid, const remote::group_id &gid) {
|
||||
uid_ = uid;
|
||||
gid_ = gid;
|
||||
}
|
||||
} // namespace repertory::remote_fuse
|
||||
617
src/drives/fuse/remotefuse/remote_fuse_drive.cpp
Normal file
617
src/drives/fuse/remotefuse/remote_fuse_drive.cpp
Normal file
@@ -0,0 +1,617 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#ifndef _WIN32
|
||||
|
||||
#include "drives/fuse/remotefuse/remote_fuse_drive.hpp"
|
||||
#include "app_config.hpp"
|
||||
#include "drives/fuse/events.hpp"
|
||||
#include "events/consumers/console_consumer.hpp"
|
||||
#include "events/consumers/logging_consumer.hpp"
|
||||
#include "events/events.hpp"
|
||||
#include "events/event_system.hpp"
|
||||
#include "platform/platform.hpp"
|
||||
#include "rpc/server/server.hpp"
|
||||
#include "types/remote.hpp"
|
||||
#include "utils/file_utils.hpp"
|
||||
#include "utils/path_utils.hpp"
|
||||
|
||||
namespace repertory::remote_fuse {
|
||||
app_config *remote_fuse_drive::remote_fuse_impl::config_ = nullptr;
|
||||
lock_data *remote_fuse_drive::remote_fuse_impl::lock_ = nullptr;
|
||||
std::string *remote_fuse_drive::remote_fuse_impl::mount_location_ = nullptr;
|
||||
bool remote_fuse_drive::remote_fuse_impl::console_enabled_ = false;
|
||||
bool remote_fuse_drive::remote_fuse_impl::was_mounted_ = false;
|
||||
std::optional<gid_t> remote_fuse_drive::remote_fuse_impl::forced_gid_;
|
||||
std::optional<uid_t> remote_fuse_drive::remote_fuse_impl::forced_uid_;
|
||||
std::optional<mode_t> remote_fuse_drive::remote_fuse_impl::forced_umask_;
|
||||
remote_instance_factory *remote_fuse_drive::remote_fuse_impl::factory_ = nullptr;
|
||||
std::unique_ptr<logging_consumer> remote_fuse_drive::remote_fuse_impl::logging_consumer_;
|
||||
std::unique_ptr<console_consumer> remote_fuse_drive::remote_fuse_impl::console_consumer_;
|
||||
std::unique_ptr<i_remote_instance> remote_fuse_drive::remote_fuse_impl::remote_instance_;
|
||||
std::unique_ptr<server> remote_fuse_drive::remote_fuse_impl::server_;
|
||||
|
||||
void remote_fuse_drive::remote_fuse_impl::tear_down(const int &ret) {
|
||||
if (was_mounted_) {
|
||||
event_system::instance().raise<drive_mount_result>(std::to_string(ret));
|
||||
event_system::instance().stop();
|
||||
logging_consumer_.reset();
|
||||
console_consumer_.reset();
|
||||
}
|
||||
}
|
||||
|
||||
void remote_fuse_drive::remote_fuse_impl::populate_stat(const remote::stat &r,
|
||||
const bool &directory, struct stat &st) {
|
||||
memset(&st, 0, sizeof(struct stat));
|
||||
|
||||
#ifdef __APPLE__
|
||||
st.st_blksize = 0;
|
||||
|
||||
st.st_atimespec.tv_nsec = r.st_atimespec % NANOS_PER_SECOND;
|
||||
st.st_atimespec.tv_sec = r.st_atimespec / NANOS_PER_SECOND;
|
||||
|
||||
st.st_birthtimespec.tv_nsec = r.st_birthtimespec % NANOS_PER_SECOND;
|
||||
st.st_birthtimespec.tv_sec = r.st_birthtimespec / NANOS_PER_SECOND;
|
||||
|
||||
st.st_ctimespec.tv_nsec = r.st_ctimespec % NANOS_PER_SECOND;
|
||||
st.st_ctimespec.tv_sec = r.st_ctimespec / NANOS_PER_SECOND;
|
||||
|
||||
st.st_mtimespec.tv_nsec = r.st_mtimespec % NANOS_PER_SECOND;
|
||||
st.st_mtimespec.tv_sec = r.st_mtimespec / NANOS_PER_SECOND;
|
||||
|
||||
st.st_flags = r.st_flags;
|
||||
#else
|
||||
st.st_blksize = 4096;
|
||||
|
||||
st.st_atim.tv_nsec = r.st_atimespec % NANOS_PER_SECOND;
|
||||
st.st_atim.tv_sec = r.st_atimespec / NANOS_PER_SECOND;
|
||||
|
||||
st.st_ctim.tv_nsec = r.st_ctimespec % NANOS_PER_SECOND;
|
||||
st.st_ctim.tv_sec = r.st_ctimespec / NANOS_PER_SECOND;
|
||||
|
||||
st.st_mtim.tv_nsec = r.st_mtimespec % NANOS_PER_SECOND;
|
||||
st.st_mtim.tv_sec = r.st_mtimespec / NANOS_PER_SECOND;
|
||||
#endif
|
||||
if (not directory) {
|
||||
const auto blockSizeStat = static_cast<std::uint64_t>(512);
|
||||
const auto blockSize = static_cast<std::uint64_t>(4096);
|
||||
const auto size =
|
||||
utils::divide_with_ceiling(static_cast<std::uint64_t>(st.st_size), blockSize) * blockSize;
|
||||
st.st_blocks =
|
||||
std::max(blockSize / blockSizeStat, utils::divide_with_ceiling(size, blockSizeStat));
|
||||
}
|
||||
|
||||
st.st_gid = r.st_gid;
|
||||
st.st_mode = (directory ? S_IFDIR : S_IFREG) | r.st_mode;
|
||||
|
||||
st.st_nlink = r.st_nlink;
|
||||
st.st_size = r.st_size;
|
||||
st.st_uid = r.st_uid;
|
||||
}
|
||||
|
||||
int remote_fuse_drive::remote_fuse_impl::repertory_access(const char *path, int mask) {
|
||||
return remote_instance_->fuse_access(path, mask);
|
||||
}
|
||||
|
||||
#ifdef __APPLE__
|
||||
int remote_fuse_drive::remote_fuse_impl::repertory_chflags(const char *path, uint32_t flags) {
|
||||
return remote_instance_->fuse_chflags(path, flags);
|
||||
}
|
||||
#endif
|
||||
|
||||
int remote_fuse_drive::remote_fuse_impl::repertory_chmod(const char *path, mode_t mode) {
|
||||
return remote_instance_->fuse_chmod(path, mode);
|
||||
}
|
||||
|
||||
int remote_fuse_drive::remote_fuse_impl::repertory_chown(const char *path, uid_t uid, gid_t gid) {
|
||||
return remote_instance_->fuse_chown(path, uid, gid);
|
||||
}
|
||||
|
||||
int remote_fuse_drive::remote_fuse_impl::repertory_create(const char *path, mode_t mode,
|
||||
struct fuse_file_info *fi) {
|
||||
return remote_instance_->fuse_create(path, mode, remote::open_flags(fi->flags), fi->fh);
|
||||
}
|
||||
|
||||
void remote_fuse_drive::remote_fuse_impl::repertory_destroy(void * /*ptr*/) {
|
||||
event_system::instance().raise<drive_unmount_pending>(*mount_location_);
|
||||
if (server_) {
|
||||
server_->stop();
|
||||
server_.reset();
|
||||
}
|
||||
|
||||
remote_instance_->fuse_destroy();
|
||||
remote_instance_.reset();
|
||||
|
||||
lock_->set_mount_state(false, "", -1);
|
||||
event_system::instance().raise<drive_mounted>(*mount_location_);
|
||||
}
|
||||
|
||||
/*int remote_fuse_drive::remote_fuse_impl::repertory_fallocate(const char *path, int mode, off_t
|
||||
offset, off_t length, struct fuse_file_info *fi) { return remote_instance_->fuse_fallocate(path,
|
||||
mode, offset, length, fi->fh);
|
||||
}*/
|
||||
|
||||
int remote_fuse_drive::remote_fuse_impl::repertory_fgetattr(const char *path, struct stat *st,
|
||||
struct fuse_file_info *fi) {
|
||||
remote::stat r{};
|
||||
|
||||
bool directory = false;
|
||||
auto ret = remote_instance_->fuse_fgetattr(path, r, directory, fi->fh);
|
||||
if (ret == 0) {
|
||||
populate_stat(r, directory, *st);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
#ifdef __APPLE__
|
||||
int remote_fuse_drive::remote_fuse_impl::repertory_fsetattr_x(const char *path,
|
||||
struct setattr_x *attr,
|
||||
struct fuse_file_info *fi) {
|
||||
remote::setattr_x attrRepertory{};
|
||||
attrRepertory.valid = attr->valid;
|
||||
attrRepertory.mode = attr->mode;
|
||||
attrRepertory.uid = attr->uid;
|
||||
attrRepertory.gid = attr->gid;
|
||||
attrRepertory.size = attr->size;
|
||||
attrRepertory.acctime = ((attr->acctime.tv_sec * NANOS_PER_SECOND) + attr->acctime.tv_nsec);
|
||||
attrRepertory.modtime = ((attr->modtime.tv_sec * NANOS_PER_SECOND) + attr->modtime.tv_nsec);
|
||||
attrRepertory.crtime = ((attr->crtime.tv_sec * NANOS_PER_SECOND) + attr->crtime.tv_nsec);
|
||||
attrRepertory.chgtime = ((attr->chgtime.tv_sec * NANOS_PER_SECOND) + attr->chgtime.tv_nsec);
|
||||
attrRepertory.bkuptime = ((attr->bkuptime.tv_sec * NANOS_PER_SECOND) + attr->bkuptime.tv_nsec);
|
||||
attrRepertory.flags = attr->flags;
|
||||
return remote_instance_->fuse_fsetattr_x(path, attrRepertory, fi->fh);
|
||||
}
|
||||
#endif
|
||||
|
||||
int remote_fuse_drive::remote_fuse_impl::repertory_fsync(const char *path, int datasync,
|
||||
struct fuse_file_info *fi) {
|
||||
return remote_instance_->fuse_fsync(path, datasync, fi->fh);
|
||||
}
|
||||
|
||||
int remote_fuse_drive::remote_fuse_impl::repertory_ftruncate(const char *path, off_t size,
|
||||
struct fuse_file_info *fi) {
|
||||
return remote_instance_->fuse_ftruncate(path, size, fi->fh);
|
||||
}
|
||||
|
||||
int remote_fuse_drive::remote_fuse_impl::repertory_getattr(const char *path, struct stat *st) {
|
||||
bool directory = false;
|
||||
remote::stat r{};
|
||||
const auto ret = remote_instance_->fuse_getattr(path, r, directory);
|
||||
if (ret == 0) {
|
||||
populate_stat(r, directory, *st);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
#ifdef __APPLE__
|
||||
int remote_fuse_drive::remote_fuse_impl::repertory_getxtimes(const char *path,
|
||||
struct timespec *bkuptime,
|
||||
struct timespec *crtime) {
|
||||
if (not(bkuptime && crtime)) {
|
||||
return -EFAULT;
|
||||
} else {
|
||||
remote::file_time repertory_bkuptime = 0;
|
||||
remote::file_time repertory_crtime = 0;
|
||||
const auto ret = remote_instance_->fuse_getxtimes(path, repertory_bkuptime, repertory_crtime);
|
||||
if (ret == 0) {
|
||||
bkuptime->tv_nsec = static_cast<long>(repertory_bkuptime % NANOS_PER_SECOND);
|
||||
bkuptime->tv_sec = repertory_bkuptime / NANOS_PER_SECOND;
|
||||
crtime->tv_nsec = static_cast<long>(repertory_crtime % NANOS_PER_SECOND);
|
||||
crtime->tv_sec = repertory_crtime / NANOS_PER_SECOND;
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
void *remote_fuse_drive::remote_fuse_impl::repertory_init(struct fuse_conn_info *conn) {
|
||||
utils::file::change_to_process_directory();
|
||||
#ifdef __APPLE__
|
||||
conn->want |= FUSE_CAP_VOL_RENAME;
|
||||
conn->want |= FUSE_CAP_XTIMES;
|
||||
#endif
|
||||
conn->want |= FUSE_CAP_BIG_WRITES;
|
||||
|
||||
if (console_enabled_) {
|
||||
console_consumer_ = std::make_unique<console_consumer>();
|
||||
}
|
||||
logging_consumer_ =
|
||||
std::make_unique<logging_consumer>(config_->get_log_directory(), config_->get_event_level());
|
||||
event_system::instance().start();
|
||||
|
||||
was_mounted_ = true;
|
||||
lock_->set_mount_state(true, *mount_location_, getpid());
|
||||
|
||||
remote_instance_ = (*factory_)();
|
||||
remote_instance_->set_fuse_uid_gid(getuid(), getgid());
|
||||
|
||||
if (remote_instance_->fuse_init() != 0) {
|
||||
event_system::instance().raise<repertory_exception>(__FUNCTION__,
|
||||
"Failed to connect to remote server");
|
||||
event_system::instance().raise<unmount_requested>();
|
||||
} else {
|
||||
server_ = std::make_unique<server>(*config_);
|
||||
server_->start();
|
||||
event_system::instance().raise<drive_mounted>(*mount_location_);
|
||||
}
|
||||
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
int remote_fuse_drive::remote_fuse_impl::repertory_mkdir(const char *path, mode_t mode) {
|
||||
return remote_instance_->fuse_mkdir(path, mode);
|
||||
}
|
||||
|
||||
int remote_fuse_drive::remote_fuse_impl::repertory_open(const char *path,
|
||||
struct fuse_file_info *fi) {
|
||||
return remote_instance_->fuse_open(path, remote::open_flags(fi->flags), fi->fh);
|
||||
}
|
||||
|
||||
int remote_fuse_drive::remote_fuse_impl::repertory_opendir(const char *path,
|
||||
struct fuse_file_info *fi) {
|
||||
return remote_instance_->fuse_opendir(path, fi->fh);
|
||||
}
|
||||
|
||||
int remote_fuse_drive::remote_fuse_impl::repertory_read(const char *path, char *buffer,
|
||||
size_t readSize, off_t readOffset,
|
||||
struct fuse_file_info *fi) {
|
||||
return remote_instance_->fuse_read(path, buffer, readSize, readOffset, fi->fh);
|
||||
}
|
||||
|
||||
int remote_fuse_drive::remote_fuse_impl::repertory_readdir(const char *path, void *buf,
|
||||
fuse_fill_dir_t fuseFillDir,
|
||||
off_t offset,
|
||||
struct fuse_file_info *fi) {
|
||||
std::string item_path;
|
||||
int ret = 0;
|
||||
while ((ret = remote_instance_->fuse_readdir(path, offset, fi->fh, item_path)) == 0) {
|
||||
if ((item_path != ".") && (item_path != "..")) {
|
||||
item_path = utils::path::strip_to_file_name(item_path);
|
||||
}
|
||||
if (fuseFillDir(buf, &item_path[0], nullptr, ++offset) != 0) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (ret == -120) {
|
||||
ret = 0;
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
int remote_fuse_drive::remote_fuse_impl::repertory_release(const char *path,
|
||||
struct fuse_file_info *fi) {
|
||||
return remote_instance_->fuse_release(path, fi->fh);
|
||||
}
|
||||
|
||||
int remote_fuse_drive::remote_fuse_impl::repertory_releasedir(const char *path,
|
||||
struct fuse_file_info *fi) {
|
||||
return remote_instance_->fuse_releasedir(path, fi->fh);
|
||||
}
|
||||
|
||||
int remote_fuse_drive::remote_fuse_impl::repertory_rename(const char *from, const char *to) {
|
||||
return remote_instance_->fuse_rename(from, to);
|
||||
}
|
||||
|
||||
int remote_fuse_drive::remote_fuse_impl::repertory_rmdir(const char *path) {
|
||||
return remote_instance_->fuse_rmdir(path);
|
||||
}
|
||||
/*
|
||||
#ifdef HAS_SETXATTR
|
||||
#ifdef __APPLE__
|
||||
int remote_fuse_drive::remote_fuse_impl::repertory_getxattr(const char *path, const char *name,
|
||||
char *value, size_t size, uint32_t position) { return remote_instance_->fuse_getxattr_osx(path,
|
||||
name, value, size, position);
|
||||
}
|
||||
#else
|
||||
int remote_fuse_drive::remote_fuse_impl::repertory_getxattr(const char *path, const char *name,
|
||||
char *value, size_t size) { return remote_instance_->fuse_getxattr(path, name, value, size);
|
||||
}
|
||||
|
||||
#endif
|
||||
int remote_fuse_drive::remote_fuse_impl::repertory_listxattr(const char *path, char *buffer,
|
||||
size_t size) { return remote_instance_->fuse_listxattr(path, buffer, size);
|
||||
}
|
||||
|
||||
remote_fuse_drive::remote_fuse_impl::int repertory_removexattr(const char *path, const char
|
||||
*name) { return remote_instance_->fuse_removexattr(path, name);
|
||||
}
|
||||
#ifdef __APPLE__
|
||||
int remote_fuse_drive::remote_fuse_impl::repertory_setxattr(const char *path, const char *name,
|
||||
const char *value, size_t size, int flags, uint32_t position) { return
|
||||
remote_instance_->fuse_setxattr_osx(path, name, value, size, flags, position);
|
||||
}
|
||||
#else
|
||||
int remote_fuse_drive::remote_fuse_impl::repertory_setxattr(const char *path, const char *name,
|
||||
const char *value, size_t size, int flags) { return remote_instance_->fuse_setxattr(path, name,
|
||||
value, size, flags);
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
*/
|
||||
#ifdef __APPLE__
|
||||
int remote_fuse_drive::remote_fuse_impl::repertory_setattr_x(const char *path,
|
||||
struct setattr_x *attr) {
|
||||
remote::setattr_x attributes{};
|
||||
attributes.valid = attr->valid;
|
||||
attributes.mode = attr->mode;
|
||||
attributes.uid = attr->uid;
|
||||
attributes.gid = attr->gid;
|
||||
attributes.size = attr->size;
|
||||
attributes.acctime = ((attr->acctime.tv_sec * NANOS_PER_SECOND) + attr->acctime.tv_nsec);
|
||||
attributes.modtime = ((attr->modtime.tv_sec * NANOS_PER_SECOND) + attr->modtime.tv_nsec);
|
||||
attributes.crtime = ((attr->crtime.tv_sec * NANOS_PER_SECOND) + attr->crtime.tv_nsec);
|
||||
attributes.chgtime = ((attr->chgtime.tv_sec * NANOS_PER_SECOND) + attr->chgtime.tv_nsec);
|
||||
attributes.bkuptime = ((attr->bkuptime.tv_sec * NANOS_PER_SECOND) + attr->bkuptime.tv_nsec);
|
||||
attributes.flags = attr->flags;
|
||||
return remote_instance_->fuse_setattr_x(path, attributes);
|
||||
}
|
||||
|
||||
int remote_fuse_drive::remote_fuse_impl::repertory_setbkuptime(const char *path,
|
||||
const struct timespec *bkuptime) {
|
||||
remote::file_time repertory_bkuptime =
|
||||
((bkuptime->tv_sec * NANOS_PER_SECOND) + bkuptime->tv_nsec);
|
||||
return remote_instance_->fuse_setbkuptime(path, repertory_bkuptime);
|
||||
}
|
||||
|
||||
int remote_fuse_drive::remote_fuse_impl::repertory_setchgtime(const char *path,
|
||||
const struct timespec *chgtime) {
|
||||
remote::file_time repertory_chgtime = ((chgtime->tv_sec * NANOS_PER_SECOND) + chgtime->tv_nsec);
|
||||
return remote_instance_->fuse_setchgtime(path, repertory_chgtime);
|
||||
}
|
||||
|
||||
int remote_fuse_drive::remote_fuse_impl::repertory_setcrtime(const char *path,
|
||||
const struct timespec *crtime) {
|
||||
remote::file_time repertory_crtime = ((crtime->tv_sec * NANOS_PER_SECOND) + crtime->tv_nsec);
|
||||
return remote_instance_->fuse_setcrtime(path, repertory_crtime);
|
||||
}
|
||||
|
||||
int remote_fuse_drive::remote_fuse_impl::repertory_setvolname(const char *volname) {
|
||||
return remote_instance_->fuse_setvolname(volname);
|
||||
}
|
||||
|
||||
int remote_fuse_drive::remote_fuse_impl::repertory_statfs_x(const char *path,
|
||||
struct statfs *stbuf) {
|
||||
auto ret = statfs(config_->get_data_directory().c_str(), stbuf);
|
||||
if (ret == 0) {
|
||||
remote::statfs_x r{};
|
||||
if ((ret = remote_instance_->fuse_statfs_x(path, stbuf->f_bsize, r)) == 0) {
|
||||
stbuf->f_blocks = r.f_blocks;
|
||||
stbuf->f_bavail = r.f_bavail;
|
||||
stbuf->f_bfree = r.f_bfree;
|
||||
stbuf->f_ffree = r.f_ffree;
|
||||
stbuf->f_files = r.f_files;
|
||||
stbuf->f_owner = getuid();
|
||||
strncpy(&stbuf->f_mntonname[0], mount_location_->c_str(), MNAMELEN);
|
||||
strncpy(&stbuf->f_mntfromname[0], &r.f_mntfromname[0], MNAMELEN);
|
||||
}
|
||||
} else {
|
||||
ret = -errno;
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
#else
|
||||
|
||||
int remote_fuse_drive::remote_fuse_impl::repertory_statfs(const char *path, struct statvfs *stbuf) {
|
||||
auto ret = statvfs(&config_->get_data_directory()[0], stbuf);
|
||||
if (ret == 0) {
|
||||
remote::statfs r{};
|
||||
if ((ret = remote_instance_->fuse_statfs(path, stbuf->f_frsize, r)) == 0) {
|
||||
stbuf->f_blocks = r.f_blocks;
|
||||
stbuf->f_bavail = r.f_bavail;
|
||||
stbuf->f_bfree = r.f_bfree;
|
||||
stbuf->f_ffree = r.f_ffree;
|
||||
stbuf->f_favail = r.f_favail;
|
||||
stbuf->f_files = r.f_files;
|
||||
}
|
||||
} else {
|
||||
ret = -errno;
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
int remote_fuse_drive::remote_fuse_impl::repertory_truncate(const char *path, off_t size) {
|
||||
return remote_instance_->fuse_truncate(path, size);
|
||||
}
|
||||
|
||||
int remote_fuse_drive::remote_fuse_impl::repertory_unlink(const char *path) {
|
||||
return remote_instance_->fuse_unlink(path);
|
||||
}
|
||||
|
||||
int remote_fuse_drive::remote_fuse_impl::repertory_utimens(const char *path,
|
||||
const struct timespec tv[2]) {
|
||||
remote::file_time rtv[2] = {0};
|
||||
if (tv) {
|
||||
rtv[0] = tv[0].tv_nsec + (tv[0].tv_sec * NANOS_PER_SECOND);
|
||||
rtv[1] = tv[1].tv_nsec + (tv[1].tv_sec * NANOS_PER_SECOND);
|
||||
}
|
||||
return remote_instance_->fuse_utimens(path, rtv, tv ? tv[0].tv_nsec : 0, tv ? tv[1].tv_nsec : 0);
|
||||
}
|
||||
|
||||
int remote_fuse_drive::remote_fuse_impl::repertory_write(const char *path, const char *buffer,
|
||||
size_t writeSize, off_t writeOffset,
|
||||
struct fuse_file_info *fi) {
|
||||
return remote_instance_->fuse_write(path, buffer, writeSize, writeOffset, fi->fh);
|
||||
}
|
||||
|
||||
remote_fuse_drive::remote_fuse_drive(app_config &config, lock_data &lock,
|
||||
remote_instance_factory factory)
|
||||
: config_(config), lock_(lock), factory_(std::move(factory)) {
|
||||
E_SUBSCRIBE_EXACT(unmount_requested, [this](const unmount_requested &) {
|
||||
std::thread([this]() { remote_fuse_drive::shutdown(mount_location_); }).detach();
|
||||
});
|
||||
}
|
||||
|
||||
int remote_fuse_drive::mount(std::vector<std::string> drive_args) {
|
||||
remote_fuse_impl::lock_ = &lock_;
|
||||
remote_fuse_impl::config_ = &config_;
|
||||
remote_fuse_impl::mount_location_ = &mount_location_;
|
||||
remote_fuse_impl::factory_ = &factory_;
|
||||
|
||||
#ifdef __APPLE__
|
||||
fuse_ops_.chflags = remote_fuse_impl::repertory_chflags;
|
||||
fuse_ops_.fsetattr_x = remote_fuse_impl::repertory_fsetattr_x;
|
||||
fuse_ops_.getxtimes = remote_fuse_impl::repertory_getxtimes;
|
||||
fuse_ops_.setattr_x = remote_fuse_impl::repertory_setattr_x;
|
||||
fuse_ops_.setbkuptime = remote_fuse_impl::repertory_setbkuptime;
|
||||
fuse_ops_.setchgtime = remote_fuse_impl::repertory_setchgtime;
|
||||
fuse_ops_.setcrtime = remote_fuse_impl::repertory_setcrtime;
|
||||
fuse_ops_.setvolname = remote_fuse_impl::repertory_setvolname;
|
||||
fuse_ops_.statfs_x = remote_fuse_impl::repertory_statfs_x;
|
||||
#endif
|
||||
auto force_no_console = false;
|
||||
for (std::size_t i = 1u; !force_no_console && (i < drive_args.size()); i++) {
|
||||
if (drive_args[i] == "-nc") {
|
||||
force_no_console = true;
|
||||
}
|
||||
}
|
||||
utils::remove_element_from(drive_args, "-nc");
|
||||
|
||||
for (std::size_t i = 1u; i < drive_args.size(); i++) {
|
||||
if (drive_args[i] == "-f") {
|
||||
remote_fuse_impl::console_enabled_ = not force_no_console;
|
||||
} else if (drive_args[i].find("-o") == 0) {
|
||||
std::string options;
|
||||
if (drive_args[i].size() == 2u) {
|
||||
if ((i + 1) < drive_args.size()) {
|
||||
options = drive_args[++i];
|
||||
}
|
||||
} else {
|
||||
options = drive_args[i].substr(2);
|
||||
}
|
||||
|
||||
const auto option_list = utils::string::split(options, ',');
|
||||
for (const auto &option : option_list) {
|
||||
if (option.find("gid") == 0) {
|
||||
const auto parts = utils::string::split(option, '=');
|
||||
if (parts.size() == 2u) {
|
||||
auto gid = getgrnam(parts[1u].c_str());
|
||||
if (not gid) {
|
||||
gid = getgrgid(utils::string::to_uint32(parts[1]));
|
||||
}
|
||||
if ((getgid() != 0) && (gid->gr_gid == 0)) {
|
||||
std::cerr << "'gid=0' requires running as root" << std::endl;
|
||||
return -1;
|
||||
} else {
|
||||
remote_fuse_impl::forced_gid_ = gid->gr_gid;
|
||||
}
|
||||
}
|
||||
} else if (option.find("uid") == 0) {
|
||||
const auto parts = utils::string::split(option, '=');
|
||||
if (parts.size() == 2u) {
|
||||
auto uid = getpwnam(parts[1u].c_str());
|
||||
if (not uid) {
|
||||
uid = getpwuid(utils::string::to_uint32(parts[1]));
|
||||
}
|
||||
if ((getuid() != 0) && (uid->pw_uid == 0)) {
|
||||
std::cerr << "'uid=0' requires running as root" << std::endl;
|
||||
return -1;
|
||||
} else {
|
||||
remote_fuse_impl::forced_uid_ = uid->pw_uid;
|
||||
}
|
||||
}
|
||||
} else if (option.find("umask") == 0) {
|
||||
const auto parts = utils::string::split(option, '=');
|
||||
if (parts.size() == 2u) {
|
||||
static const auto numeric_regex = std::regex("[0-9]+");
|
||||
try {
|
||||
if (not std::regex_match(parts[1u], numeric_regex)) {
|
||||
throw std::runtime_error("invalid syntax");
|
||||
} else {
|
||||
remote_fuse_impl::forced_umask_ = utils::string::to_uint32(parts[1]);
|
||||
}
|
||||
} catch (...) {
|
||||
std::cerr << ("'" + option + "' invalid syntax") << std::endl;
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<const char *> fuse_argv(drive_args.size());
|
||||
for (std::size_t i = 0u; i < drive_args.size(); i++) {
|
||||
fuse_argv[i] = drive_args[i].c_str();
|
||||
}
|
||||
|
||||
struct fuse_args args =
|
||||
FUSE_ARGS_INIT(static_cast<int>(fuse_argv.size()), (char **)&fuse_argv[0]);
|
||||
char *mount_location = nullptr;
|
||||
fuse_parse_cmdline(&args, &mount_location, nullptr, nullptr);
|
||||
if (mount_location) {
|
||||
mount_location_ = mount_location;
|
||||
free(mount_location);
|
||||
}
|
||||
|
||||
std::string args_string;
|
||||
for (const auto *arg : fuse_argv) {
|
||||
if (args_string.empty()) {
|
||||
args_string += arg;
|
||||
} else {
|
||||
args_string += (" " + std::string(arg));
|
||||
}
|
||||
}
|
||||
|
||||
event_system::instance().raise<fuse_args_parsed>(args_string);
|
||||
|
||||
umask(0);
|
||||
const auto ret = fuse_main(static_cast<int>(fuse_argv.size()), (char **)&fuse_argv[0], &fuse_ops_,
|
||||
&mount_location);
|
||||
remote_fuse_impl::tear_down(ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
void remote_fuse_drive::display_options(int argc, char *argv[]) {
|
||||
struct fuse_operations fuse_ops {};
|
||||
fuse_main(argc, argv, &fuse_ops, nullptr);
|
||||
std::cout << std::endl;
|
||||
}
|
||||
|
||||
void remote_fuse_drive::display_version_information(int argc, char *argv[]) {
|
||||
struct fuse_operations fuse_ops {};
|
||||
fuse_main(argc, argv, &fuse_ops, nullptr);
|
||||
}
|
||||
|
||||
void remote_fuse_drive::shutdown(std::string mount_location) {
|
||||
#if __APPLE__
|
||||
const auto unmount = "umount \"" + mount_location + "\" >/dev/null 2>&1";
|
||||
#else
|
||||
const auto unmount = "fusermount -u \"" + mount_location + "\" >/dev/null 2>&1";
|
||||
#endif
|
||||
int ret;
|
||||
for (std::uint8_t i = 0u; ((ret = system(unmount.c_str())) != 0) && (i < 10u); i++) {
|
||||
event_system::instance().raise<unmount_result>(mount_location, std::to_string(ret));
|
||||
if (i != 9u) {
|
||||
std::this_thread::sleep_for(1s);
|
||||
}
|
||||
}
|
||||
|
||||
if (ret != 0) {
|
||||
ret = kill(getpid(), SIGINT);
|
||||
}
|
||||
|
||||
event_system::instance().raise<unmount_result>(mount_location, std::to_string(ret));
|
||||
}
|
||||
} // namespace repertory::remote_fuse
|
||||
|
||||
#endif // _WIN32
|
||||
51
src/drives/fuse/remotefuse/remote_fuse_drive2.cpp
Normal file
51
src/drives/fuse/remotefuse/remote_fuse_drive2.cpp
Normal file
@@ -0,0 +1,51 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#ifndef _WIN32
|
||||
#if 0
|
||||
|
||||
#include "drives/fuse/remotefuse/remote_fuse_drive2.hpp"
|
||||
#include "app_config.hpp"
|
||||
#include "events/consumers/console_consumer.hpp"
|
||||
#include "events/consumers/logging_consumer.hpp"
|
||||
#include "events/events.hpp"
|
||||
#include "events/event_system.hpp"
|
||||
#include "platform/platform.hpp"
|
||||
#include "rpc/server/server.hpp"
|
||||
#include "types/remote.hpp"
|
||||
#include "utils/file_utils.hpp"
|
||||
#include "utils/path_utils.hpp"
|
||||
|
||||
namespace repertory::remote_fuse {
|
||||
api_error remote_fuse_drive2::access_impl(std::string api_path, int mask) {
|
||||
return utils::to_api_error(remote_instance_->fuse_access(api_path.c_str(), mask));
|
||||
}
|
||||
|
||||
#ifdef __APPLE__
|
||||
api_error remote_fuse_drive2::chflags_impl(std::string api_path, uint32_t flags) {
|
||||
return utils::to_api_error(remote_instance_->fuse_chflags(path, flags));
|
||||
}
|
||||
#endif // __APPLE__
|
||||
|
||||
api_error remote_fuse_drive2::chmod_impl(std::string api_path, mode_t mode) {
|
||||
return utils::to_api_error(remote_instance_->fuse_chmod(path, mode));
|
||||
}
|
||||
} // namespace repertory::remote_fuse
|
||||
|
||||
#endif // 0
|
||||
#endif // _WIN32
|
||||
1473
src/drives/fuse/remotefuse/remote_server.cpp
Normal file
1473
src/drives/fuse/remotefuse/remote_server.cpp
Normal file
File diff suppressed because it is too large
Load Diff
192
src/drives/remote/remote_open_file_table.cpp
Normal file
192
src/drives/remote/remote_open_file_table.cpp
Normal file
@@ -0,0 +1,192 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "drives/remote/remote_open_file_table.hpp"
|
||||
#include "utils/utils.hpp"
|
||||
|
||||
namespace repertory {
|
||||
void remote_open_file_table::add_directory(const std::string &client_id, void *dir) {
|
||||
unique_mutex_lock directory_lock(directory_mutex_);
|
||||
auto &list = directory_lookup_[client_id];
|
||||
if (utils::collection_excludes(list, dir)) {
|
||||
directory_lookup_[client_id].emplace_back(dir);
|
||||
}
|
||||
}
|
||||
|
||||
void remote_open_file_table::close_all(const std::string &client_id) {
|
||||
std::vector<remote::file_handle> compatHandles;
|
||||
unique_mutex_lock compat_lock(compat_mutex_);
|
||||
for (auto &kv : compat_lookup_) {
|
||||
if (kv.second.client_id == client_id) {
|
||||
compatHandles.emplace_back(kv.first);
|
||||
}
|
||||
}
|
||||
compat_lock.unlock();
|
||||
|
||||
for (auto &handle : compatHandles) {
|
||||
#ifdef _WIN32
|
||||
_close(static_cast<int>(handle));
|
||||
#else
|
||||
close(static_cast<int>(handle));
|
||||
#endif
|
||||
remove_compat_open_info(handle);
|
||||
}
|
||||
|
||||
std::vector<OSHandle> handles;
|
||||
unique_mutex_lock file_lock(file_mutex_);
|
||||
for (auto &kv : file_lookup_) {
|
||||
if (kv.second.client_id == client_id) {
|
||||
handles.emplace_back(kv.first);
|
||||
}
|
||||
}
|
||||
file_lock.unlock();
|
||||
|
||||
for (auto &handle : handles) {
|
||||
#ifdef _WIN32
|
||||
::CloseHandle(handle);
|
||||
#else
|
||||
close(handle);
|
||||
#endif
|
||||
remove_open_info(handle);
|
||||
}
|
||||
|
||||
std::vector<void *> dirs;
|
||||
unique_mutex_lock directory_lock(directory_mutex_);
|
||||
for (auto &kv : directory_lookup_) {
|
||||
if (kv.first == client_id) {
|
||||
dirs.insert(dirs.end(), kv.second.begin(), kv.second.end());
|
||||
}
|
||||
}
|
||||
directory_lock.unlock();
|
||||
|
||||
for (auto *dir : dirs) {
|
||||
remove_directory(client_id, dir);
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef _WIN32
|
||||
bool remote_open_file_table::get_directory_buffer(const OSHandle &handle, PVOID *&buffer) {
|
||||
mutex_lock file_lock(file_mutex_);
|
||||
if (file_lookup_.find(handle) != file_lookup_.end()) {
|
||||
buffer = &file_lookup_[handle].directory_buffer;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
#endif
|
||||
|
||||
bool remote_open_file_table::get_open_info(const OSHandle &handle, open_info &oi) {
|
||||
mutex_lock file_lock(file_mutex_);
|
||||
if (file_lookup_.find(handle) != file_lookup_.end()) {
|
||||
oi = file_lookup_[handle];
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
std::string remote_open_file_table::get_open_file_path(const OSHandle &handle) {
|
||||
mutex_lock file_lock(file_mutex_);
|
||||
if (file_lookup_.find(handle) != file_lookup_.end()) {
|
||||
return file_lookup_[handle].path;
|
||||
}
|
||||
|
||||
return "";
|
||||
}
|
||||
|
||||
bool remote_open_file_table::has_open_directory(const std::string &client_id, void *dir) {
|
||||
unique_mutex_lock directory_lock(directory_mutex_);
|
||||
auto &list = directory_lookup_[client_id];
|
||||
return (utils::collection_includes(list, dir));
|
||||
}
|
||||
|
||||
int remote_open_file_table::has_compat_open_info(const remote::file_handle &handle,
|
||||
const int &errorReturn) {
|
||||
mutex_lock compat_lock(compat_mutex_);
|
||||
const auto res = ((compat_lookup_.find(handle) == compat_lookup_.end()) ? -1 : 0);
|
||||
if (res == -1) {
|
||||
errno = errorReturn;
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
void remote_open_file_table::remove_compat_open_info(const remote::file_handle &handle) {
|
||||
mutex_lock compat_lock(compat_mutex_);
|
||||
if (compat_lookup_[handle].count > 0) {
|
||||
compat_lookup_[handle].count--;
|
||||
}
|
||||
if (not compat_lookup_[handle].count) {
|
||||
compat_lookup_.erase(handle);
|
||||
}
|
||||
}
|
||||
|
||||
bool remote_open_file_table::remove_directory(const std::string &client_id, void *dir) {
|
||||
unique_mutex_lock directory_lock(directory_mutex_);
|
||||
auto &list = directory_lookup_[client_id];
|
||||
if (utils::collection_includes(list, dir)) {
|
||||
utils::remove_element_from(list, dir);
|
||||
delete_open_directory(dir);
|
||||
if (directory_lookup_[client_id].empty()) {
|
||||
directory_lookup_.erase(client_id);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
void remote_open_file_table::remove_open_info(const OSHandle &handle) {
|
||||
mutex_lock file_lock(file_mutex_);
|
||||
if (file_lookup_[handle].count > 0) {
|
||||
file_lookup_[handle].count--;
|
||||
}
|
||||
if (not file_lookup_[handle].count) {
|
||||
#ifdef _WIN32
|
||||
if (file_lookup_[handle].directory_buffer) {
|
||||
FspFileSystemDeleteDirectoryBuffer(&file_lookup_[handle].directory_buffer);
|
||||
}
|
||||
#endif
|
||||
file_lookup_.erase(handle);
|
||||
}
|
||||
}
|
||||
|
||||
void remote_open_file_table::set_compat_client_id(const remote::file_handle &handle,
|
||||
const std::string &client_id) {
|
||||
mutex_lock compat_lock(compat_mutex_);
|
||||
compat_lookup_[handle].client_id = client_id;
|
||||
}
|
||||
|
||||
void remote_open_file_table::set_client_id(const OSHandle &handle, const std::string &client_id) {
|
||||
mutex_lock file_lock(file_mutex_);
|
||||
file_lookup_[handle].client_id = client_id;
|
||||
}
|
||||
|
||||
void remote_open_file_table::set_compat_open_info(const remote::file_handle &handle) {
|
||||
mutex_lock compat_lock(compat_mutex_);
|
||||
if (compat_lookup_.find(handle) == compat_lookup_.end()) {
|
||||
compat_lookup_[handle] = {0, ""};
|
||||
}
|
||||
compat_lookup_[handle].count++;
|
||||
}
|
||||
|
||||
void remote_open_file_table::set_open_info(const OSHandle &handle, open_info oi) {
|
||||
mutex_lock file_lock(file_mutex_);
|
||||
if (file_lookup_.find(handle) == file_lookup_.end()) {
|
||||
file_lookup_[handle] = std::move(oi);
|
||||
}
|
||||
file_lookup_[handle].count++;
|
||||
}
|
||||
} // namespace repertory
|
||||
462
src/drives/winfsp/remotewinfsp/remote_client.cpp
Normal file
462
src/drives/winfsp/remotewinfsp/remote_client.cpp
Normal file
@@ -0,0 +1,462 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "drives/winfsp/remotewinfsp/remote_client.hpp"
|
||||
#include "app_config.hpp"
|
||||
#include "drives/winfsp/remotewinfsp/i_remote_instance.hpp"
|
||||
#include "events/events.hpp"
|
||||
#include "events/event_system.hpp"
|
||||
#include "types/repertory.hpp"
|
||||
#include "utils/path_utils.hpp"
|
||||
|
||||
namespace repertory::remote_winfsp {
|
||||
#define RAISE_REMOTE_WINFSP_CLIENT_EVENT(func, file, ret) \
|
||||
if (config_.get_enable_drive_events() && \
|
||||
(((config_.get_event_level() >= remote_winfsp_client_event::level) && \
|
||||
(ret != STATUS_SUCCESS)) || \
|
||||
(config_.get_event_level() >= event_level::verbose))) \
|
||||
event_system::instance().raise<remote_winfsp_client_event>(func, file, ret)
|
||||
|
||||
// clang-format off
|
||||
E_SIMPLE3(remote_winfsp_client_event, debug, true,
|
||||
std::string, function, func, E_STRING,
|
||||
std::string, api_path, ap, E_STRING,
|
||||
packet::error_type, result, res, E_FROM_INT32
|
||||
);
|
||||
// clang-format on
|
||||
|
||||
remote_client::remote_client(const app_config &config)
|
||||
: config_(config),
|
||||
packet_client_(config.get_remote_host_name_or_ip(), config.get_remote_max_connections(),
|
||||
config.get_remote_port(), config.get_remote_receive_timeout_secs(),
|
||||
config.get_remote_send_timeout_secs(), config.get_remote_token()) {}
|
||||
|
||||
packet::error_type remote_client::winfsp_can_delete(PVOID file_desc, PWSTR file_name) {
|
||||
packet request;
|
||||
request.encode(file_desc);
|
||||
request.encode(file_name);
|
||||
|
||||
std::uint32_t service_flags = 0u;
|
||||
const auto ret = packet_client_.send(__FUNCTION__, request, service_flags);
|
||||
RAISE_REMOTE_WINFSP_CLIENT_EVENT(
|
||||
__FUNCTION__, utils::path::create_api_path(utils::string::to_utf8(file_name)), ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::json_create_directory_snapshot(const std::string &path,
|
||||
json &json_data) {
|
||||
packet request;
|
||||
request.encode(path);
|
||||
|
||||
packet response;
|
||||
std::uint32_t service_flags = 0u;
|
||||
auto ret = packet_client_.send(__FUNCTION__, request, response, service_flags);
|
||||
if (ret == 0) {
|
||||
ret = packet::decode_json(response, json_data);
|
||||
}
|
||||
|
||||
RAISE_REMOTE_WINFSP_CLIENT_EVENT(__FUNCTION__, path, ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::json_read_directory_snapshot(const std::string &path,
|
||||
const remote::file_handle &handle,
|
||||
const std::uint32_t &page,
|
||||
json &json_data) {
|
||||
packet request;
|
||||
request.encode(path);
|
||||
request.encode(handle);
|
||||
request.encode(page);
|
||||
|
||||
packet response;
|
||||
std::uint32_t service_flags = 0u;
|
||||
auto ret = packet_client_.send(__FUNCTION__, request, response, service_flags);
|
||||
if (ret == 0) {
|
||||
ret = packet::decode_json(response, json_data);
|
||||
}
|
||||
|
||||
RAISE_REMOTE_WINFSP_CLIENT_EVENT(__FUNCTION__, path, ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type
|
||||
remote_client::json_release_directory_snapshot(const std::string &path,
|
||||
const remote::file_handle &handle) {
|
||||
packet request;
|
||||
request.encode(path);
|
||||
request.encode(handle);
|
||||
|
||||
std::uint32_t service_flags = 0u;
|
||||
const auto ret = packet_client_.send(__FUNCTION__, request, service_flags);
|
||||
|
||||
RAISE_REMOTE_WINFSP_CLIENT_EVENT(__FUNCTION__, path, ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::winfsp_cleanup(PVOID file_desc, PWSTR file_name, UINT32 flags,
|
||||
BOOLEAN &was_closed) {
|
||||
const auto file_path = get_open_file_path(to_handle(file_desc));
|
||||
was_closed = 0;
|
||||
|
||||
packet request;
|
||||
request.encode(file_desc);
|
||||
request.encode(file_name);
|
||||
request.encode(flags);
|
||||
|
||||
packet response;
|
||||
std::uint32_t service_flags = 0u;
|
||||
auto ret = packet_client_.send(__FUNCTION__, request, response, service_flags);
|
||||
DECODE_OR_IGNORE(&response, was_closed);
|
||||
if (was_closed) {
|
||||
remove_open_info(to_handle(file_desc));
|
||||
}
|
||||
RAISE_REMOTE_WINFSP_CLIENT_EVENT(__FUNCTION__, file_path, ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::winfsp_close(PVOID file_desc) {
|
||||
const auto file_path = get_open_file_path(to_handle(file_desc));
|
||||
|
||||
packet request;
|
||||
request.encode(file_desc);
|
||||
|
||||
std::uint32_t service_flags = 0u;
|
||||
const auto ret = packet_client_.send(__FUNCTION__, request, service_flags);
|
||||
if (ret == STATUS_SUCCESS) {
|
||||
remove_open_info(to_handle(file_desc));
|
||||
}
|
||||
RAISE_REMOTE_WINFSP_CLIENT_EVENT(__FUNCTION__, file_path, ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::winfsp_create(PWSTR file_name, UINT32 create_options,
|
||||
UINT32 granted_access, UINT32 attributes,
|
||||
UINT64 allocation_size, PVOID *file_desc,
|
||||
remote::file_info *file_info,
|
||||
std::string &normalized_name, BOOLEAN &exists) {
|
||||
packet request;
|
||||
request.encode(file_name);
|
||||
request.encode(create_options);
|
||||
request.encode(granted_access);
|
||||
request.encode(attributes);
|
||||
request.encode(allocation_size);
|
||||
|
||||
packet response;
|
||||
std::uint32_t service_flags = 0u;
|
||||
auto ret = packet_client_.send(__FUNCTION__, request, response, service_flags);
|
||||
if (ret == STATUS_SUCCESS) {
|
||||
HANDLE handle;
|
||||
DECODE_OR_IGNORE(&response, handle);
|
||||
DECODE_OR_IGNORE(&response, *file_info);
|
||||
DECODE_OR_IGNORE(&response, normalized_name);
|
||||
DECODE_OR_IGNORE(&response, exists);
|
||||
if (ret == STATUS_SUCCESS) {
|
||||
*file_desc = reinterpret_cast<PVOID>(handle);
|
||||
set_open_info(to_handle(*file_desc),
|
||||
open_info{0, "", nullptr, utils::string::to_utf8(file_name)});
|
||||
#ifdef _WIN32
|
||||
if (exists) {
|
||||
::SetLastError(ERROR_ALREADY_EXISTS);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
RAISE_REMOTE_WINFSP_CLIENT_EVENT(__FUNCTION__, get_open_file_path(to_handle(*file_desc)), ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::winfsp_flush(PVOID file_desc, remote::file_info *file_info) {
|
||||
packet request;
|
||||
request.encode(file_desc);
|
||||
|
||||
packet response;
|
||||
std::uint32_t service_flags = 0u;
|
||||
auto ret = packet_client_.send(__FUNCTION__, request, response, service_flags);
|
||||
DECODE_OR_IGNORE(&response, *file_info);
|
||||
|
||||
RAISE_REMOTE_WINFSP_CLIENT_EVENT(__FUNCTION__, get_open_file_path(to_handle(file_desc)), ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::winfsp_get_dir_buffer(PVOID file_desc, PVOID *&ptr) {
|
||||
#ifdef _WIN32
|
||||
if (get_directory_buffer(reinterpret_cast<OSHandle>(file_desc), ptr)) {
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
#endif
|
||||
return STATUS_INVALID_HANDLE;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::winfsp_get_file_info(PVOID file_desc,
|
||||
remote::file_info *file_info) {
|
||||
packet request;
|
||||
request.encode(file_desc);
|
||||
|
||||
packet response;
|
||||
std::uint32_t service_flags = 0u;
|
||||
auto ret = packet_client_.send(__FUNCTION__, request, response, service_flags);
|
||||
DECODE_OR_IGNORE(&response, *file_info);
|
||||
|
||||
RAISE_REMOTE_WINFSP_CLIENT_EVENT(__FUNCTION__, get_open_file_path(to_handle(file_desc)), ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::winfsp_get_security_by_name(PWSTR file_name, PUINT32 attributes,
|
||||
std::uint64_t *descriptor_size,
|
||||
std::wstring &string_descriptor) {
|
||||
packet request;
|
||||
request.encode(file_name);
|
||||
request.encode(static_cast<std::uint64_t>(descriptor_size ? *descriptor_size : 0));
|
||||
request.encode(static_cast<std::uint8_t>(attributes != nullptr));
|
||||
|
||||
packet response;
|
||||
std::uint32_t service_flags = 0u;
|
||||
auto ret = packet_client_.send(__FUNCTION__, request, response, service_flags);
|
||||
|
||||
string_descriptor.clear();
|
||||
DECODE_OR_IGNORE(&response, string_descriptor);
|
||||
if (string_descriptor.empty()) {
|
||||
string_descriptor = L"O:BAG:BAD:P(A;;FA;;;SY)(A;;FA;;;BA)(A;;FA;;;WD)";
|
||||
}
|
||||
|
||||
if (attributes) {
|
||||
DECODE_OR_IGNORE(&response, *attributes);
|
||||
}
|
||||
|
||||
RAISE_REMOTE_WINFSP_CLIENT_EVENT(__FUNCTION__, utils::string::to_utf8(file_name), ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::winfsp_get_volume_info(UINT64 &total_size, UINT64 &free_size,
|
||||
std::string &volume_label) {
|
||||
packet request;
|
||||
packet response;
|
||||
std::uint32_t service_flags = 0u;
|
||||
auto ret = packet_client_.send(__FUNCTION__, request, response, service_flags);
|
||||
DECODE_OR_IGNORE(&response, total_size);
|
||||
DECODE_OR_IGNORE(&response, free_size);
|
||||
DECODE_OR_IGNORE(&response, volume_label);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::winfsp_mounted(const std::wstring &location) {
|
||||
packet request;
|
||||
request.encode(get_repertory_version());
|
||||
request.encode(location);
|
||||
|
||||
std::uint32_t service_flags = 0u;
|
||||
const auto ret = packet_client_.send(__FUNCTION__, request, service_flags);
|
||||
const auto mountLocation = utils::string::to_utf8(location);
|
||||
event_system::instance().raise<drive_mounted>(mountLocation);
|
||||
|
||||
RAISE_REMOTE_WINFSP_CLIENT_EVENT(__FUNCTION__, mountLocation, ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::winfsp_open(PWSTR file_name, UINT32 create_options,
|
||||
UINT32 granted_access, PVOID *file_desc,
|
||||
remote::file_info *file_info,
|
||||
std::string &normalized_name) {
|
||||
packet request;
|
||||
request.encode(file_name);
|
||||
request.encode(create_options);
|
||||
request.encode(granted_access);
|
||||
|
||||
packet response;
|
||||
std::uint32_t service_flags = 0u;
|
||||
auto ret = packet_client_.send(__FUNCTION__, request, response, service_flags);
|
||||
if (ret == STATUS_SUCCESS) {
|
||||
HANDLE handle;
|
||||
DECODE_OR_IGNORE(&response, handle);
|
||||
DECODE_OR_IGNORE(&response, *file_info);
|
||||
DECODE_OR_IGNORE(&response, normalized_name);
|
||||
|
||||
if (ret == STATUS_SUCCESS) {
|
||||
*file_desc = reinterpret_cast<PVOID>(handle);
|
||||
set_open_info(to_handle(*file_desc),
|
||||
open_info{0, "", nullptr, utils::string::to_utf8(file_name)});
|
||||
}
|
||||
}
|
||||
|
||||
RAISE_REMOTE_WINFSP_CLIENT_EVENT(__FUNCTION__, get_open_file_path(to_handle(*file_desc)), ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::winfsp_overwrite(PVOID file_desc, UINT32 attributes,
|
||||
BOOLEAN replace_attributes,
|
||||
UINT64 allocation_size,
|
||||
remote::file_info *file_info) {
|
||||
packet request;
|
||||
request.encode(file_desc);
|
||||
request.encode(attributes);
|
||||
request.encode(replace_attributes);
|
||||
request.encode(allocation_size);
|
||||
|
||||
packet response;
|
||||
std::uint32_t service_flags = 0u;
|
||||
auto ret = packet_client_.send(__FUNCTION__, request, response, service_flags);
|
||||
DECODE_OR_IGNORE(&response, *file_info);
|
||||
|
||||
RAISE_REMOTE_WINFSP_CLIENT_EVENT(__FUNCTION__, get_open_file_path(to_handle(file_desc)), ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::winfsp_read(PVOID file_desc, PVOID buffer, UINT64 offset,
|
||||
UINT32 length, PUINT32 bytes_transferred) {
|
||||
packet request;
|
||||
request.encode(file_desc);
|
||||
request.encode(offset);
|
||||
request.encode(length);
|
||||
|
||||
packet response;
|
||||
std::uint32_t service_flags = 0u;
|
||||
auto ret = packet_client_.send(__FUNCTION__, request, response, service_flags);
|
||||
DECODE_OR_IGNORE(&response, *bytes_transferred);
|
||||
if (ret == STATUS_SUCCESS) {
|
||||
ret = response.decode(buffer, *bytes_transferred);
|
||||
#ifdef _WIN32
|
||||
if ((ret == STATUS_SUCCESS) && (not *bytes_transferred || (*bytes_transferred != length))) {
|
||||
::SetLastError(ERROR_HANDLE_EOF);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
if (ret != STATUS_SUCCESS) {
|
||||
RAISE_REMOTE_WINFSP_CLIENT_EVENT(__FUNCTION__, get_open_file_path(to_handle(file_desc)), ret);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::winfsp_read_directory(PVOID file_desc, PWSTR pattern,
|
||||
PWSTR marker, json &item_list) {
|
||||
packet request;
|
||||
request.encode(file_desc);
|
||||
request.encode(pattern);
|
||||
request.encode(marker);
|
||||
|
||||
packet response;
|
||||
std::uint32_t service_flags = 0u;
|
||||
auto ret = packet_client_.send(__FUNCTION__, request, response, service_flags);
|
||||
if (ret == STATUS_SUCCESS) {
|
||||
ret = packet::decode_json(response, item_list);
|
||||
}
|
||||
|
||||
RAISE_REMOTE_WINFSP_CLIENT_EVENT(__FUNCTION__, get_open_file_path(to_handle(file_desc)), ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::winfsp_rename(PVOID file_desc, PWSTR file_name,
|
||||
PWSTR new_file_name, BOOLEAN replace_if_exists) {
|
||||
packet request;
|
||||
request.encode(file_desc);
|
||||
request.encode(file_name);
|
||||
request.encode(new_file_name);
|
||||
request.encode(replace_if_exists);
|
||||
|
||||
std::uint32_t service_flags = 0u;
|
||||
const auto ret = packet_client_.send(__FUNCTION__, request, service_flags);
|
||||
RAISE_REMOTE_WINFSP_CLIENT_EVENT(
|
||||
__FUNCTION__,
|
||||
utils::path::create_api_path(utils::string::to_utf8(file_name)) + "|" +
|
||||
utils::path::create_api_path(utils::string::to_utf8(new_file_name)),
|
||||
ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::winfsp_set_basic_info(PVOID file_desc, UINT32 attributes,
|
||||
UINT64 creation_time,
|
||||
UINT64 last_access_time,
|
||||
UINT64 last_write_time, UINT64 change_time,
|
||||
remote::file_info *file_info) {
|
||||
packet request;
|
||||
request.encode(file_desc);
|
||||
request.encode(attributes);
|
||||
request.encode(creation_time);
|
||||
request.encode(last_access_time);
|
||||
request.encode(last_write_time);
|
||||
request.encode(change_time);
|
||||
|
||||
packet response;
|
||||
std::uint32_t service_flags = 0u;
|
||||
auto ret = packet_client_.send(__FUNCTION__, request, response, service_flags);
|
||||
DECODE_OR_IGNORE(&response, *file_info);
|
||||
|
||||
RAISE_REMOTE_WINFSP_CLIENT_EVENT(__FUNCTION__, get_open_file_path(to_handle(file_desc)), ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::winfsp_set_file_size(PVOID file_desc, UINT64 new_size,
|
||||
BOOLEAN set_allocation_size,
|
||||
remote::file_info *file_info) {
|
||||
packet request;
|
||||
request.encode(file_desc);
|
||||
request.encode(new_size);
|
||||
request.encode(set_allocation_size);
|
||||
|
||||
packet response;
|
||||
std::uint32_t service_flags = 0u;
|
||||
auto ret = packet_client_.send(__FUNCTION__, request, response, service_flags);
|
||||
DECODE_OR_IGNORE(&response, *file_info);
|
||||
|
||||
RAISE_REMOTE_WINFSP_CLIENT_EVENT(__FUNCTION__, get_open_file_path(to_handle(file_desc)), ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::winfsp_unmounted(const std::wstring &location) {
|
||||
const auto mount_location = utils::string::to_utf8(location);
|
||||
event_system::instance().raise<drive_unmount_pending>(mount_location);
|
||||
packet request;
|
||||
request.encode(location);
|
||||
|
||||
std::uint32_t service_flags = 0u;
|
||||
const auto ret = packet_client_.send(__FUNCTION__, request, service_flags);
|
||||
event_system::instance().raise<drive_unmounted>(mount_location);
|
||||
|
||||
RAISE_REMOTE_WINFSP_CLIENT_EVENT(__FUNCTION__, mount_location, ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
packet::error_type remote_client::winfsp_write(PVOID file_desc, PVOID buffer, UINT64 offset,
|
||||
UINT32 length, BOOLEAN write_to_end,
|
||||
BOOLEAN constrained_io, PUINT32 bytes_transferred,
|
||||
remote::file_info *file_info) {
|
||||
packet request;
|
||||
request.encode(file_desc);
|
||||
request.encode(length);
|
||||
request.encode(offset);
|
||||
request.encode(write_to_end);
|
||||
request.encode(constrained_io);
|
||||
if (length) {
|
||||
request.encode(buffer, length);
|
||||
}
|
||||
|
||||
packet response;
|
||||
std::uint32_t service_flags = 0u;
|
||||
auto ret = packet_client_.send(__FUNCTION__, request, response, service_flags);
|
||||
DECODE_OR_IGNORE(&response, *bytes_transferred);
|
||||
DECODE_OR_IGNORE(&response, *file_info);
|
||||
|
||||
if (ret != STATUS_SUCCESS) {
|
||||
RAISE_REMOTE_WINFSP_CLIENT_EVENT(__FUNCTION__, get_open_file_path(to_handle(file_desc)), ret);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
} // namespace repertory::remote_winfsp
|
||||
1144
src/drives/winfsp/remotewinfsp/remote_server.cpp
Normal file
1144
src/drives/winfsp/remotewinfsp/remote_server.cpp
Normal file
File diff suppressed because it is too large
Load Diff
401
src/drives/winfsp/remotewinfsp/remote_winfsp_drive.cpp
Normal file
401
src/drives/winfsp/remotewinfsp/remote_winfsp_drive.cpp
Normal file
@@ -0,0 +1,401 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#ifdef _WIN32
|
||||
#include "drives/winfsp/remotewinfsp/remote_winfsp_drive.hpp"
|
||||
#include "app_config.hpp"
|
||||
#include "events/consumers/console_consumer.hpp"
|
||||
#include "events/consumers/logging_consumer.hpp"
|
||||
#include "events/events.hpp"
|
||||
#include "platform/platform.hpp"
|
||||
#include "rpc/server/server.hpp"
|
||||
#include "utils/file_utils.hpp"
|
||||
#include "utils/path_utils.hpp"
|
||||
#include "utils/utils.hpp"
|
||||
|
||||
namespace repertory::remote_winfsp {
|
||||
remote_winfsp_drive::winfsp_service::winfsp_service(lock_data &lock, remote_winfsp_drive &drive,
|
||||
std::vector<std::string> drive_args,
|
||||
app_config &config)
|
||||
: Service(&(L"RepertoryRemote_" + utils::string::from_utf8(lock.get_unique_id()))[0u]),
|
||||
config_(config),
|
||||
lock_(lock),
|
||||
drive_(drive),
|
||||
drive_args_(std::move(drive_args)),
|
||||
host_(drive) {}
|
||||
|
||||
NTSTATUS remote_winfsp_drive::winfsp_service::OnStart(ULONG, PWSTR *) {
|
||||
const auto mount_location =
|
||||
utils::path::absolute((drive_args_.size() > 1u) ? drive_args_[1u] : "");
|
||||
const auto drive_letter = ((mount_location.size() == 2u) ||
|
||||
((mount_location.size() == 3u) && (mount_location[2u] == '\\'))) &&
|
||||
(mount_location[1u] == ':');
|
||||
|
||||
auto ret = drive_letter ? STATUS_DEVICE_BUSY : STATUS_NOT_SUPPORTED;
|
||||
if ((drive_letter && not utils::file::is_directory(mount_location))) {
|
||||
auto unicode_mount_location = utils::string::from_utf8(mount_location);
|
||||
host_.SetFileSystemName(&unicode_mount_location[0u]);
|
||||
ret = host_.Mount(&unicode_mount_location[0u]);
|
||||
} else {
|
||||
std::cerr << (drive_letter ? "Mount location in use: " : "Mount location not supported: ")
|
||||
<< mount_location << std::endl;
|
||||
}
|
||||
|
||||
if (ret != STATUS_SUCCESS) {
|
||||
event_system::instance().raise<drive_mount_failed>(mount_location, std::to_string(ret));
|
||||
lock_.set_mount_state(false, "", -1);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
NTSTATUS remote_winfsp_drive::winfsp_service::OnStop() {
|
||||
host_.Unmount();
|
||||
lock_.set_mount_state(false, "", -1);
|
||||
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
remote_winfsp_drive::remote_winfsp_drive(app_config &config, lock_data &lockData,
|
||||
remote_instance_factory factory)
|
||||
: FileSystemBase(), config_(config), lock_(lockData), factory_(factory) {
|
||||
E_SUBSCRIBE_EXACT(unmount_requested, [this](const unmount_requested &e) {
|
||||
std::thread([this]() { this->shutdown(); }).detach();
|
||||
});
|
||||
}
|
||||
|
||||
NTSTATUS remote_winfsp_drive::CanDelete(PVOID file_node, PVOID file_desc, PWSTR file_name) {
|
||||
return remote_instance_->winfsp_can_delete(file_desc, file_name);
|
||||
}
|
||||
|
||||
VOID remote_winfsp_drive::Cleanup(PVOID file_node, PVOID file_desc, PWSTR file_name, ULONG flags) {
|
||||
BOOLEAN was_closed;
|
||||
remote_instance_->winfsp_cleanup(file_desc, file_name, flags, was_closed);
|
||||
}
|
||||
|
||||
VOID remote_winfsp_drive::Close(PVOID file_node, PVOID file_desc) {
|
||||
remote_instance_->winfsp_close(file_desc);
|
||||
}
|
||||
|
||||
NTSTATUS remote_winfsp_drive::Create(PWSTR file_name, UINT32 create_options, UINT32 granted_access,
|
||||
UINT32 attributes, PSECURITY_DESCRIPTOR descriptor,
|
||||
UINT64 allocation_size, PVOID *file_node, PVOID *file_desc,
|
||||
OpenFileInfo *ofi) {
|
||||
remote::file_info fi{};
|
||||
std::string normalized_name;
|
||||
BOOLEAN exists = 0;
|
||||
const auto ret =
|
||||
remote_instance_->winfsp_create(file_name, create_options, granted_access, attributes,
|
||||
allocation_size, file_desc, &fi, normalized_name, exists);
|
||||
if (ret == STATUS_SUCCESS) {
|
||||
SetFileInfo(ofi->FileInfo, fi);
|
||||
const auto filePath = utils::string::from_utf8(normalized_name);
|
||||
wcsncpy(ofi->NormalizedName, &filePath[0], wcslen(&filePath[0]));
|
||||
ofi->NormalizedNameSize = (UINT16)(wcslen(&filePath[0]) * sizeof(WCHAR));
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
NTSTATUS remote_winfsp_drive::Flush(PVOID file_node, PVOID file_desc, FileInfo *file_info) {
|
||||
remote::file_info fi{};
|
||||
const auto ret = remote_instance_->winfsp_flush(file_desc, &fi);
|
||||
SetFileInfo(*file_info, fi);
|
||||
return ret;
|
||||
}
|
||||
|
||||
NTSTATUS remote_winfsp_drive::GetFileInfo(PVOID file_node, PVOID file_desc, FileInfo *file_info) {
|
||||
remote::file_info fi{};
|
||||
const auto ret = remote_instance_->winfsp_get_file_info(file_desc, &fi);
|
||||
SetFileInfo(*file_info, fi);
|
||||
return ret;
|
||||
}
|
||||
|
||||
NTSTATUS remote_winfsp_drive::GetSecurityByName(PWSTR file_name, PUINT32 attributes,
|
||||
PSECURITY_DESCRIPTOR descriptor,
|
||||
SIZE_T *descriptor_size) {
|
||||
std::wstring string_descriptor;
|
||||
std::uint64_t sds = descriptor_size ? *descriptor_size : 0;
|
||||
auto ret = remote_instance_->winfsp_get_security_by_name(
|
||||
file_name, attributes, descriptor_size ? &sds : nullptr, string_descriptor);
|
||||
*descriptor_size = static_cast<SIZE_T>(sds);
|
||||
if ((ret == STATUS_SUCCESS) && *descriptor_size) {
|
||||
PSECURITY_DESCRIPTOR sd = nullptr;
|
||||
ULONG sz2 = 0u;
|
||||
if (::ConvertStringSecurityDescriptorToSecurityDescriptorW(&string_descriptor[0u],
|
||||
SDDL_REVISION_1, &sd, &sz2)) {
|
||||
if (sz2 > *descriptor_size) {
|
||||
ret = STATUS_BUFFER_TOO_SMALL;
|
||||
} else {
|
||||
::CopyMemory(descriptor, sd, sz2);
|
||||
}
|
||||
*descriptor_size = sz2;
|
||||
::LocalFree(sd);
|
||||
} else {
|
||||
ret = FspNtStatusFromWin32(::GetLastError());
|
||||
}
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
NTSTATUS remote_winfsp_drive::GetVolumeInfo(VolumeInfo *volume_info) {
|
||||
std::string volume_label;
|
||||
const auto ret = remote_instance_->winfsp_get_volume_info(volume_info->TotalSize,
|
||||
volume_info->FreeSize, volume_label);
|
||||
if (ret == STATUS_SUCCESS) {
|
||||
const auto byte_size = static_cast<UINT16>(volume_label.size() * sizeof(WCHAR));
|
||||
wcscpy_s(&volume_info->VolumeLabel[0u], 32, &utils::string::from_utf8(volume_label)[0u]);
|
||||
volume_info->VolumeLabelLength = std::min(static_cast<UINT16>(64u), byte_size);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
NTSTATUS remote_winfsp_drive::Init(PVOID host) {
|
||||
auto *file_system_host = reinterpret_cast<FileSystemHost *>(host);
|
||||
file_system_host->SetPrefix(
|
||||
&(L"\\repertory\\" + std::wstring(file_system_host->FileSystemName()).substr(0, 1))[0]);
|
||||
file_system_host->SetFileSystemName(REPERTORY_W);
|
||||
file_system_host->SetFlushAndPurgeOnCleanup(TRUE);
|
||||
file_system_host->SetReparsePoints(FALSE);
|
||||
file_system_host->SetReparsePointsAccessCheck(FALSE);
|
||||
file_system_host->SetSectorSize(WINFSP_ALLOCATION_UNIT);
|
||||
file_system_host->SetSectorsPerAllocationUnit(1);
|
||||
file_system_host->SetFileInfoTimeout(1000);
|
||||
file_system_host->SetCaseSensitiveSearch(FALSE);
|
||||
file_system_host->SetCasePreservedNames(TRUE);
|
||||
file_system_host->SetNamedStreams(FALSE);
|
||||
file_system_host->SetUnicodeOnDisk(TRUE);
|
||||
file_system_host->SetPersistentAcls(FALSE);
|
||||
file_system_host->SetPostCleanupWhenModifiedOnly(TRUE);
|
||||
file_system_host->SetPassQueryDirectoryPattern(FALSE);
|
||||
file_system_host->SetVolumeCreationTime(utils::get_file_time_now());
|
||||
file_system_host->SetVolumeSerialNumber(0);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
int remote_winfsp_drive::mount(const std::vector<std::string> &drive_args) {
|
||||
std::vector<std::string> parsed_drive_args;
|
||||
|
||||
const auto force_no_console = utils::collection_includes(drive_args, "-nc");
|
||||
|
||||
auto enable_console = false;
|
||||
for (const auto &arg : drive_args) {
|
||||
if (arg == "-f") {
|
||||
if (not force_no_console) {
|
||||
enable_console = true;
|
||||
}
|
||||
} else if (arg != "-nc") {
|
||||
parsed_drive_args.emplace_back(arg);
|
||||
}
|
||||
}
|
||||
|
||||
logging_consumer l(config_.get_log_directory(), config_.get_event_level());
|
||||
std::unique_ptr<console_consumer> c;
|
||||
if (enable_console) {
|
||||
c = std::make_unique<console_consumer>();
|
||||
}
|
||||
event_system::instance().start();
|
||||
const auto ret = winfsp_service(lock_, *this, parsed_drive_args, config_).Run();
|
||||
event_system::instance().raise<drive_mount_result>(std::to_string(ret));
|
||||
event_system::instance().stop();
|
||||
c.reset();
|
||||
return static_cast<int>(ret);
|
||||
}
|
||||
|
||||
NTSTATUS remote_winfsp_drive::Mounted(PVOID host) {
|
||||
auto *file_system_host = reinterpret_cast<FileSystemHost *>(host);
|
||||
remote_instance_ = factory_();
|
||||
server_ = std::make_unique<server>(config_);
|
||||
server_->start();
|
||||
mount_location_ = utils::string::to_utf8(file_system_host->MountPoint());
|
||||
lock_.set_mount_state(true, mount_location_, ::GetCurrentProcessId());
|
||||
return remote_instance_->winfsp_mounted(file_system_host->MountPoint());
|
||||
}
|
||||
|
||||
NTSTATUS remote_winfsp_drive::Open(PWSTR file_name, UINT32 create_options, UINT32 granted_access,
|
||||
PVOID *file_node, PVOID *file_desc, OpenFileInfo *ofi) {
|
||||
remote::file_info fi{};
|
||||
std::string normalize_name;
|
||||
const auto ret = remote_instance_->winfsp_open(file_name, create_options, granted_access,
|
||||
file_desc, &fi, normalize_name);
|
||||
if (ret == STATUS_SUCCESS) {
|
||||
SetFileInfo(ofi->FileInfo, fi);
|
||||
const auto filePath = utils::string::from_utf8(normalize_name);
|
||||
wcsncpy(ofi->NormalizedName, &filePath[0], wcslen(&filePath[0]));
|
||||
ofi->NormalizedNameSize = (UINT16)(wcslen(&filePath[0]) * sizeof(WCHAR));
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
NTSTATUS remote_winfsp_drive::Overwrite(PVOID file_node, PVOID file_desc, UINT32 attributes,
|
||||
BOOLEAN replace_attributes, UINT64 allocation_size,
|
||||
FileInfo *file_info) {
|
||||
remote::file_info fi{};
|
||||
const auto ret = remote_instance_->winfsp_overwrite(file_desc, attributes, replace_attributes,
|
||||
allocation_size, &fi);
|
||||
SetFileInfo(*file_info, fi);
|
||||
return ret;
|
||||
}
|
||||
|
||||
void remote_winfsp_drive::PopulateFileInfo(const json &item, FSP_FSCTL_FILE_INFO &file_info) {
|
||||
const auto di = directory_item::from_json(item);
|
||||
file_info.FileSize = di.directory ? 0 : di.size;
|
||||
file_info.AllocationSize =
|
||||
utils::divide_with_ceiling(file_info.FileSize, WINFSP_ALLOCATION_UNIT) *
|
||||
WINFSP_ALLOCATION_UNIT;
|
||||
file_info.ChangeTime = utils::get_changed_time_from_meta(di.meta);
|
||||
file_info.CreationTime = utils::get_creation_time_from_meta(di.meta);
|
||||
file_info.FileAttributes = utils::get_attributes_from_meta(di.meta);
|
||||
file_info.HardLinks = 0;
|
||||
file_info.IndexNumber = 0;
|
||||
file_info.LastAccessTime = utils::get_accessed_time_from_meta(di.meta);
|
||||
file_info.LastWriteTime = utils::get_written_time_from_meta(di.meta);
|
||||
file_info.ReparseTag = 0;
|
||||
file_info.EaSize = 0;
|
||||
}
|
||||
|
||||
NTSTATUS remote_winfsp_drive::Read(PVOID file_node, PVOID file_desc, PVOID buffer, UINT64 offset,
|
||||
ULONG length, PULONG bytes_transferred) {
|
||||
return remote_instance_->winfsp_read(file_desc, buffer, offset, length,
|
||||
reinterpret_cast<PUINT32>(bytes_transferred));
|
||||
}
|
||||
|
||||
NTSTATUS remote_winfsp_drive::ReadDirectory(PVOID file_node, PVOID file_desc, PWSTR pattern,
|
||||
PWSTR marker, PVOID buffer, ULONG buffer_length,
|
||||
PULONG bytes_transferred) {
|
||||
json itemList;
|
||||
NTSTATUS ret = remote_instance_->winfsp_read_directory(file_desc, pattern, marker, itemList);
|
||||
if (ret == STATUS_SUCCESS) {
|
||||
PVOID *directory_buffer = nullptr;
|
||||
if ((ret = remote_instance_->winfsp_get_dir_buffer(file_desc, directory_buffer)) ==
|
||||
STATUS_SUCCESS) {
|
||||
if (FspFileSystemAcquireDirectoryBuffer(directory_buffer,
|
||||
static_cast<BOOLEAN>(nullptr == marker), &ret)) {
|
||||
auto item_found = false;
|
||||
for (const auto &item : itemList) {
|
||||
const auto item_path = item["path"].get<std::string>();
|
||||
const auto display_name =
|
||||
utils::string::from_utf8(utils::path::strip_to_file_name(item_path));
|
||||
if (not marker || (marker && item_found)) {
|
||||
if (not utils::path::is_ads_file_path(item_path)) {
|
||||
union {
|
||||
UINT8 B[FIELD_OFFSET(FSP_FSCTL_DIR_INFO, FileNameBuf) +
|
||||
((MAX_PATH + 1) * sizeof(WCHAR))];
|
||||
FSP_FSCTL_DIR_INFO D;
|
||||
} directory_info_buffer;
|
||||
|
||||
auto *directory_info = &directory_info_buffer.D;
|
||||
::ZeroMemory(directory_info, sizeof(*directory_info));
|
||||
directory_info->Size = static_cast<UINT16>(
|
||||
FIELD_OFFSET(FSP_FSCTL_DIR_INFO, FileNameBuf) +
|
||||
(std::min((size_t)MAX_PATH, display_name.size()) * sizeof(WCHAR)));
|
||||
|
||||
if (not item["meta"].empty() || ((item_path != ".") && (item_path != ".."))) {
|
||||
PopulateFileInfo(item, directory_info->FileInfo);
|
||||
}
|
||||
if (ret == STATUS_SUCCESS) {
|
||||
::wcscpy_s(&directory_info->FileNameBuf[0], MAX_PATH, &display_name[0]);
|
||||
|
||||
FspFileSystemFillDirectoryBuffer(directory_buffer, directory_info, &ret);
|
||||
if (ret != STATUS_SUCCESS) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
item_found = display_name == std::wstring(marker);
|
||||
}
|
||||
}
|
||||
|
||||
FspFileSystemReleaseDirectoryBuffer(directory_buffer);
|
||||
}
|
||||
|
||||
if ((ret == STATUS_SUCCESS) || (ret == STATUS_NO_MORE_FILES)) {
|
||||
FspFileSystemReadDirectoryBuffer(directory_buffer, marker, buffer, buffer_length,
|
||||
reinterpret_cast<PULONG>(bytes_transferred));
|
||||
ret = STATUS_SUCCESS;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
NTSTATUS remote_winfsp_drive::Rename(PVOID file_node, PVOID file_desc, PWSTR file_name,
|
||||
PWSTR new_file_name, BOOLEAN replace_if_exists) {
|
||||
return remote_instance_->winfsp_rename(file_desc, file_name, new_file_name, replace_if_exists);
|
||||
}
|
||||
|
||||
NTSTATUS remote_winfsp_drive::SetBasicInfo(PVOID file_node, PVOID file_desc, UINT32 attributes,
|
||||
UINT64 creation_time, UINT64 last_access_time,
|
||||
UINT64 last_write_time, UINT64 change_time,
|
||||
FileInfo *file_info) {
|
||||
remote::file_info fi{};
|
||||
const auto ret = remote_instance_->winfsp_set_basic_info(
|
||||
file_desc, attributes, creation_time, last_access_time, last_write_time, change_time, &fi);
|
||||
SetFileInfo(*file_info, fi);
|
||||
return ret;
|
||||
}
|
||||
|
||||
void remote_winfsp_drive::SetFileInfo(FileInfo &dest, const remote::file_info &src) {
|
||||
dest.FileAttributes = src.FileAttributes;
|
||||
dest.ReparseTag = src.ReparseTag;
|
||||
dest.AllocationSize = src.AllocationSize;
|
||||
dest.FileSize = src.FileSize;
|
||||
dest.CreationTime = src.CreationTime;
|
||||
dest.LastAccessTime = src.LastAccessTime;
|
||||
dest.LastWriteTime = src.LastWriteTime;
|
||||
dest.ChangeTime = src.ChangeTime;
|
||||
dest.IndexNumber = src.IndexNumber;
|
||||
dest.HardLinks = src.HardLinks;
|
||||
dest.EaSize = src.EaSize;
|
||||
}
|
||||
|
||||
NTSTATUS remote_winfsp_drive::SetFileSize(PVOID file_node, PVOID file_desc, UINT64 new_size,
|
||||
BOOLEAN set_allocation_size, FileInfo *file_info) {
|
||||
remote::file_info fi{};
|
||||
const auto ret =
|
||||
remote_instance_->winfsp_set_file_size(file_desc, new_size, set_allocation_size, &fi);
|
||||
SetFileInfo(*file_info, fi);
|
||||
return ret;
|
||||
}
|
||||
|
||||
VOID remote_winfsp_drive::Unmounted(PVOID host) {
|
||||
server_->stop();
|
||||
server_.reset();
|
||||
auto *fileSystemHost = reinterpret_cast<FileSystemHost *>(host);
|
||||
lock_.set_mount_state(false, "", -1);
|
||||
remote_instance_->winfsp_unmounted(fileSystemHost->MountPoint());
|
||||
remote_instance_.reset();
|
||||
mount_location_ = "";
|
||||
}
|
||||
|
||||
NTSTATUS remote_winfsp_drive::Write(PVOID file_node, PVOID file_desc, PVOID buffer, UINT64 offset,
|
||||
ULONG length, BOOLEAN write_to_end, BOOLEAN constrained_io,
|
||||
PULONG bytes_transferred, FileInfo *file_info) {
|
||||
remote::file_info fi{};
|
||||
const auto ret = remote_instance_->winfsp_write(
|
||||
file_desc, buffer, offset, length, write_to_end, constrained_io,
|
||||
reinterpret_cast<PUINT32>(bytes_transferred), &fi);
|
||||
SetFileInfo(*file_info, fi);
|
||||
return ret;
|
||||
}
|
||||
} // namespace repertory::remote_winfsp
|
||||
|
||||
#endif // _WIN32
|
||||
1039
src/drives/winfsp/winfsp_drive.cpp
Normal file
1039
src/drives/winfsp/winfsp_drive.cpp
Normal file
File diff suppressed because it is too large
Load Diff
145
src/events/consumers/logging_consumer.cpp
Normal file
145
src/events/consumers/logging_consumer.cpp
Normal file
@@ -0,0 +1,145 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "events/consumers/logging_consumer.hpp"
|
||||
#include "events/events.hpp"
|
||||
#include "utils/file_utils.hpp"
|
||||
#include "utils/path_utils.hpp"
|
||||
|
||||
namespace repertory {
|
||||
logging_consumer::logging_consumer(const std::string &log_directory, const event_level &level)
|
||||
: event_level_(level),
|
||||
log_directory_(utils::path::finalize(log_directory)),
|
||||
log_path_(utils::path::combine(log_directory, {"repertory.log"})) {
|
||||
utils::file::create_full_directory_path(log_directory_);
|
||||
check_log_roll(0);
|
||||
reopen_log_file();
|
||||
E_SUBSCRIBE_ALL(process_event);
|
||||
E_SUBSCRIBE_EXACT(event_level_changed, [this](const event_level_changed &e) {
|
||||
event_level_ = event_level_from_string(e.get_new_event_level().get<std::string>());
|
||||
});
|
||||
logging_thread_ = std::make_unique<std::thread>([this]() { this->logging_thread(); });
|
||||
}
|
||||
|
||||
logging_consumer::~logging_consumer() {
|
||||
E_CONSUMER_RELEASE();
|
||||
{
|
||||
mutex_lock l(log_mutex_);
|
||||
logging_active_ = false;
|
||||
log_notify_.notify_all();
|
||||
}
|
||||
logging_thread_->join();
|
||||
logging_thread_.reset();
|
||||
logging_thread(true);
|
||||
close_log_file();
|
||||
}
|
||||
|
||||
void logging_consumer::check_log_roll(const size_t &count) {
|
||||
std::uint64_t file_size{};
|
||||
const auto success = utils::file::get_file_size(log_path_, file_size);
|
||||
if (success && (file_size + count) >= MAX_LOG_FILE_SIZE) {
|
||||
close_log_file();
|
||||
for (std::uint8_t i = MAX_LOG_FILES; i > 0u; i--) {
|
||||
const auto temp_log_path = utils::path::combine(
|
||||
log_directory_, {"repertory." + utils::string::from_uint8(i) + ".log"});
|
||||
if (utils::file::is_file(temp_log_path)) {
|
||||
if (i == MAX_LOG_FILES) {
|
||||
utils::file::delete_file(temp_log_path);
|
||||
} else {
|
||||
const auto next_file_path = utils::path::combine(
|
||||
log_directory_,
|
||||
{"repertory." + utils::string::from_uint8(i + std::uint8_t(1)) + ".log"});
|
||||
utils::file::move_file(temp_log_path, next_file_path);
|
||||
}
|
||||
}
|
||||
}
|
||||
utils::file::move_file(log_path_, utils::path::combine(log_directory_, {"repertory.1.log"}));
|
||||
reopen_log_file();
|
||||
}
|
||||
}
|
||||
|
||||
void logging_consumer::close_log_file() {
|
||||
if (log_file_) {
|
||||
fclose(log_file_);
|
||||
log_file_ = nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
void logging_consumer::logging_thread(const bool &drain) {
|
||||
do {
|
||||
std::deque<std::shared_ptr<event>> events;
|
||||
{
|
||||
unique_mutex_lock l(log_mutex_);
|
||||
if (event_queue_.empty() && not drain) {
|
||||
log_notify_.wait_for(l, 2s);
|
||||
} else {
|
||||
events.insert(events.end(), event_queue_.begin(), event_queue_.end());
|
||||
event_queue_.clear();
|
||||
}
|
||||
}
|
||||
|
||||
while (not events.empty()) {
|
||||
auto event = events.front();
|
||||
events.pop_front();
|
||||
|
||||
if (event->get_event_level() <= event_level_) {
|
||||
const std::string msg = ([&]() -> std::string {
|
||||
struct tm local_time {};
|
||||
utils::get_local_time_now(local_time);
|
||||
|
||||
std::stringstream ss;
|
||||
ss << std::put_time(&local_time, "%F %T") << "|"
|
||||
<< event_level_to_string(event->get_event_level()).c_str() << "|"
|
||||
<< event->get_single_line().c_str() << std::endl;
|
||||
return ss.str();
|
||||
})();
|
||||
|
||||
check_log_roll(msg.length());
|
||||
auto retry = true;
|
||||
for (int i = 0; retry && (i < 2); i++) {
|
||||
retry = (not log_file_ || (fwrite(&msg[0], 1, msg.length(), log_file_) != msg.length()));
|
||||
if (retry) {
|
||||
reopen_log_file();
|
||||
}
|
||||
}
|
||||
|
||||
if (log_file_) {
|
||||
fflush(log_file_);
|
||||
}
|
||||
}
|
||||
}
|
||||
} while (logging_active_);
|
||||
}
|
||||
|
||||
void logging_consumer::process_event(const event &event) {
|
||||
{
|
||||
mutex_lock l(log_mutex_);
|
||||
event_queue_.push_back(event.clone());
|
||||
log_notify_.notify_all();
|
||||
}
|
||||
}
|
||||
|
||||
void logging_consumer::reopen_log_file() {
|
||||
close_log_file();
|
||||
#ifdef _WIN32
|
||||
log_file_ = _fsopen(&log_path_[0], "a+", _SH_DENYWR);
|
||||
#else
|
||||
log_file_ = fopen(&log_path_[0], "a+");
|
||||
#endif
|
||||
}
|
||||
} // namespace repertory
|
||||
55
src/events/event.cpp
Normal file
55
src/events/event.cpp
Normal file
@@ -0,0 +1,55 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "events/event.hpp"
|
||||
#include "utils/string_utils.hpp"
|
||||
|
||||
namespace repertory {
|
||||
event_level event_level_from_string(std::string level) {
|
||||
level = utils::string::to_lower(level);
|
||||
if (level == "debug" || level == "event_level::debug") {
|
||||
return event_level::debug;
|
||||
} else if (level == "warn" || level == "event_level::warn") {
|
||||
return event_level::warn;
|
||||
} else if (level == "normal" || level == "event_level::normal") {
|
||||
return event_level::normal;
|
||||
} else if (level == "error" || level == "event_level::error") {
|
||||
return event_level::error;
|
||||
} else if (level == "verbose" || level == "event_level::verbose") {
|
||||
return event_level::verbose;
|
||||
}
|
||||
return event_level::normal;
|
||||
}
|
||||
|
||||
std::string event_level_to_string(const event_level &level) {
|
||||
switch (level) {
|
||||
case event_level::debug:
|
||||
return "debug";
|
||||
case event_level::error:
|
||||
return "error";
|
||||
case event_level::normal:
|
||||
return "normal";
|
||||
case event_level::warn:
|
||||
return "warn";
|
||||
case event_level::verbose:
|
||||
return "verbose";
|
||||
default:
|
||||
return "normal";
|
||||
}
|
||||
}
|
||||
} // namespace repertory
|
||||
29
src/events/t_event_system.cpp
Normal file
29
src/events/t_event_system.cpp
Normal file
@@ -0,0 +1,29 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "events/t_event_system.hpp"
|
||||
#include "events/event.hpp"
|
||||
|
||||
namespace repertory {
|
||||
template <typename event_type> t_event_system<event_type> &t_event_system<event_type>::instance() {
|
||||
return event_system_;
|
||||
}
|
||||
template <typename event_type> t_event_system<event_type> t_event_system<event_type>::event_system_;
|
||||
|
||||
template class t_event_system<event>;
|
||||
} // namespace repertory
|
||||
135
src/main.cpp
Normal file
135
src/main.cpp
Normal file
@@ -0,0 +1,135 @@
|
||||
#include "common.hpp"
|
||||
#include "utils/cli_utils.hpp"
|
||||
#include "utils/global_data.hpp"
|
||||
#include "utils/polling.hpp"
|
||||
|
||||
#ifdef REPERTORY_TESTING
|
||||
#include "test_common.hpp"
|
||||
#include "events/event_system.hpp"
|
||||
#include "utils/string_utils.hpp"
|
||||
|
||||
using namespace repertory;
|
||||
|
||||
#ifdef _WIN32
|
||||
std::size_t PROVIDER_INDEX = 0u;
|
||||
#endif // _WIN32
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
#ifdef _WIN32
|
||||
if (utils::cli::has_option(argc, argv, "--provider_index")) {
|
||||
PROVIDER_INDEX = static_cast<std::size_t>(
|
||||
utils::string::to_uint64(utils::cli::parse_option(argc, argv, "--provider_index", 1u)[0u]) +
|
||||
1u);
|
||||
}
|
||||
#endif // _WIN32
|
||||
|
||||
curl_global_init(CURL_GLOBAL_DEFAULT);
|
||||
|
||||
InitGoogleTest(&argc, argv);
|
||||
const auto ret = RUN_ALL_TESTS();
|
||||
|
||||
curl_global_cleanup();
|
||||
return ret;
|
||||
}
|
||||
#else // REPERTORY_TESTING
|
||||
#include "cli/actions.hpp"
|
||||
#include "types/repertory.hpp"
|
||||
|
||||
using namespace repertory;
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
if (argc == 1) {
|
||||
argc++;
|
||||
static std::string cmd(argv[0u]);
|
||||
static std::vector<const char *> v({&cmd[0u], "-h"});
|
||||
argv = (char **)&v[0u];
|
||||
}
|
||||
|
||||
auto pt = utils::cli::get_provider_type_from_args(argc, argv);
|
||||
|
||||
std::string data_directory;
|
||||
auto ret = utils::cli::parse_string_option(argc, argv, utils::cli::options::data_directory_option,
|
||||
data_directory);
|
||||
|
||||
std::string password;
|
||||
ret = (ret == exit_code::success)
|
||||
? utils::cli::parse_string_option(argc, argv, utils::cli::options::password_option,
|
||||
password)
|
||||
: ret;
|
||||
|
||||
std::string user;
|
||||
ret = (ret == exit_code::success)
|
||||
? utils::cli::parse_string_option(argc, argv, utils::cli::options::user_option, user)
|
||||
: ret;
|
||||
|
||||
std::string remote_host;
|
||||
std::uint16_t remote_port = 0u;
|
||||
std::string unique_id;
|
||||
if ((ret == exit_code::success) && (pt == provider_type::remote)) {
|
||||
std::string data;
|
||||
if ((ret = utils::cli::parse_string_option(argc, argv, utils::cli::options::remote_mount_option,
|
||||
data)) == exit_code::success) {
|
||||
const auto parts = utils::string::split(data, ':');
|
||||
if (parts.size() != 2) {
|
||||
std::cerr << "Invalid syntax for host/port '-rm host:port,--remote_mount host:port'"
|
||||
<< std::endl;
|
||||
ret = exit_code::invalid_syntax;
|
||||
} else {
|
||||
unique_id = parts[0u] + ':' + parts[1u];
|
||||
remote_host = parts[0u];
|
||||
try {
|
||||
remote_port = utils::string::to_uint16(parts[1u]);
|
||||
data_directory = utils::path::combine(
|
||||
data_directory.empty() ? app_config::default_data_directory(pt) : data_directory,
|
||||
{utils::string::replace_copy(unique_id, ':', '_')});
|
||||
} catch (const std::exception &e) {
|
||||
std::cerr << (e.what() ? e.what() : "Unable to parse port") << std::endl;
|
||||
ret = exit_code::invalid_syntax;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef REPERTORY_ENABLE_S3
|
||||
if ((ret == exit_code::success) && (pt == provider_type::s3)) {
|
||||
std::string data;
|
||||
if ((ret = utils::cli::parse_string_option(argc, argv, utils::cli::options::name_option,
|
||||
data)) == exit_code::success) {
|
||||
unique_id = utils::string::trim(data);
|
||||
if (unique_id.empty()) {
|
||||
std::cerr << "Name of S3 instance not provided" << std::endl;
|
||||
ret = exit_code::invalid_syntax;
|
||||
} else {
|
||||
data_directory = utils::path::combine(
|
||||
data_directory.empty() ? app_config::default_data_directory(pt) : data_directory,
|
||||
{unique_id});
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif // REPERTORY_ENABLE_S3
|
||||
|
||||
int mount_result = 0;
|
||||
if (ret == exit_code::success) {
|
||||
if (utils::cli::has_option(argc, argv, utils::cli::options::help_option)) {
|
||||
cli::actions::help<repertory_drive>(argc, argv);
|
||||
} else if (utils::cli::has_option(argc, argv, utils::cli::options::version_option)) {
|
||||
cli::actions::version<repertory_drive>(argc, argv);
|
||||
} else {
|
||||
ret = exit_code::option_not_found;
|
||||
for (std::size_t i = 0;
|
||||
(ret == exit_code::option_not_found) && (i < utils::cli::options::option_list.size());
|
||||
i++) {
|
||||
ret = cli::actions::perform_action(utils::cli::options::option_list[i], argc, argv,
|
||||
data_directory, pt, unique_id, user, password);
|
||||
}
|
||||
if (ret == exit_code::option_not_found) {
|
||||
ret = cli::actions::mount(argc, argv, data_directory, mount_result, pt, remote_host,
|
||||
remote_port, unique_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ((ret == exit_code::mount_result) ? mount_result : static_cast<int>(ret));
|
||||
}
|
||||
|
||||
#endif // REPERTORY_TESTING
|
||||
148
src/platform/unix_platform.cpp
Normal file
148
src/platform/unix_platform.cpp
Normal file
@@ -0,0 +1,148 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#ifndef _WIN32
|
||||
#include "platform/unix_platform.hpp"
|
||||
#include "app_config.hpp"
|
||||
#include "utils/file_utils.hpp"
|
||||
#include "utils/path_utils.hpp"
|
||||
|
||||
namespace repertory {
|
||||
lock_data::lock_data(const provider_type &pt, std::string unique_id /*= ""*/)
|
||||
: pt_(pt),
|
||||
unique_id_(std::move(unique_id)),
|
||||
mutex_id_("repertory2_" + app_config::get_provider_name(pt) + "_" + unique_id_) {
|
||||
lock_fd_ = open(get_lock_file().c_str(), O_CREAT | O_RDWR, S_IWUSR | S_IRUSR);
|
||||
}
|
||||
|
||||
lock_data::lock_data() : pt_(provider_type::sia), unique_id_(""), mutex_id_(""), lock_fd_(-1) {}
|
||||
|
||||
lock_data::~lock_data() {
|
||||
if (lock_fd_ != -1) {
|
||||
if (lock_status_ == 0) {
|
||||
unlink(get_lock_file().c_str());
|
||||
flock(lock_fd_, LOCK_UN);
|
||||
}
|
||||
|
||||
close(lock_fd_);
|
||||
}
|
||||
}
|
||||
|
||||
std::string lock_data::get_lock_data_file() {
|
||||
const auto dir = get_state_directory();
|
||||
utils::file::create_full_directory_path(dir);
|
||||
return utils::path::combine(dir, {"mountstate_" + std::to_string(getuid()) + ".json"});
|
||||
}
|
||||
|
||||
std::string lock_data::get_lock_file() {
|
||||
const auto dir = get_state_directory();
|
||||
utils::file::create_full_directory_path(dir);
|
||||
return utils::path::combine(dir, {mutex_id_ + "_" + std::to_string(getuid())});
|
||||
}
|
||||
|
||||
bool lock_data::get_mount_state(json &mount_state) {
|
||||
auto ret = false;
|
||||
auto fd = open(get_lock_data_file().c_str(), O_CREAT | O_RDWR, S_IWUSR | S_IRUSR);
|
||||
if (fd != -1) {
|
||||
if (wait_for_lock(fd) == 0) {
|
||||
ret = utils::file::read_json_file(get_lock_data_file(), mount_state);
|
||||
flock(fd, LOCK_UN);
|
||||
}
|
||||
|
||||
close(fd);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
std::string lock_data::get_state_directory() {
|
||||
#ifdef __APPLE__
|
||||
return utils::path::resolve("~/Library/Application Support/" + std::string(REPERTORY_DATA_NAME) +
|
||||
"/state");
|
||||
#else
|
||||
return utils::path::resolve("~/.local/" + std::string(REPERTORY_DATA_NAME) + "/state");
|
||||
#endif
|
||||
}
|
||||
|
||||
lock_result lock_data::grab_lock(const std::uint8_t &retry_count) {
|
||||
if (lock_fd_ == -1) {
|
||||
return lock_result::failure;
|
||||
}
|
||||
|
||||
lock_status_ = wait_for_lock(lock_fd_, retry_count);
|
||||
switch (lock_status_) {
|
||||
case 0:
|
||||
set_mount_state(false, "", -1);
|
||||
return lock_result::success;
|
||||
case EWOULDBLOCK:
|
||||
return lock_result::locked;
|
||||
default:
|
||||
return lock_result::failure;
|
||||
}
|
||||
}
|
||||
|
||||
bool lock_data::set_mount_state(const bool &active, const std::string &mount_location,
|
||||
const int &pid) {
|
||||
auto ret = false;
|
||||
auto fd = open(get_lock_data_file().c_str(), O_CREAT | O_RDWR, S_IWUSR | S_IRUSR);
|
||||
if (fd != -1) {
|
||||
if (wait_for_lock(fd) == 0) {
|
||||
const auto mount_id = app_config::get_provider_display_name(pt_) + unique_id_;
|
||||
json mount_state;
|
||||
utils::file::read_json_file(get_lock_data_file(), mount_state);
|
||||
if ((mount_state.find(mount_id) == mount_state.end()) ||
|
||||
(mount_state[mount_id].find("Active") == mount_state[mount_id].end()) ||
|
||||
(mount_state[mount_id]["Active"].get<bool>() != active) ||
|
||||
(active && ((mount_state[mount_id].find("Location") == mount_state[mount_id].end()) ||
|
||||
(mount_state[mount_id]["Location"].get<std::string>() != mount_location)))) {
|
||||
const auto lines = utils::file::read_file_lines(get_lock_data_file());
|
||||
const auto txt =
|
||||
std::accumulate(lines.begin(), lines.end(), std::string(),
|
||||
[](std::string s, const std::string &s2) { return std::move(s) + s2; });
|
||||
auto json = json::parse(txt.empty() ? "{}" : txt);
|
||||
json[mount_id] = {{"Active", active},
|
||||
{"Location", active ? mount_location : ""},
|
||||
{"PID", active ? pid : -1}};
|
||||
ret = utils::file::write_json_file(get_lock_data_file(), json);
|
||||
}
|
||||
flock(fd, LOCK_UN);
|
||||
}
|
||||
|
||||
close(fd);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
int lock_data::wait_for_lock(const int &fd, const std::uint8_t &retry_count) {
|
||||
auto lock_status = EWOULDBLOCK;
|
||||
std::int16_t remain = retry_count * 100u;
|
||||
while ((remain > 0) && (lock_status == EWOULDBLOCK)) {
|
||||
if ((lock_status = flock(fd, LOCK_EX | LOCK_NB)) == -1) {
|
||||
if ((lock_status = errno) == EWOULDBLOCK) {
|
||||
const auto sleep_ms =
|
||||
utils::random_between(std::int16_t(1), std::min(remain, std::int16_t(100)));
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(sleep_ms));
|
||||
remain -= sleep_ms;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return lock_status;
|
||||
}
|
||||
} // namespace repertory
|
||||
|
||||
#endif //_WIN32
|
||||
145
src/platform/win32_platform.cpp
Normal file
145
src/platform/win32_platform.cpp
Normal file
@@ -0,0 +1,145 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#ifdef _WIN32
|
||||
#include "platform/win32_platform.hpp"
|
||||
|
||||
namespace repertory {
|
||||
bool lock_data::get_mount_state(const provider_type &pt, json &mount_state) {
|
||||
const auto ret = get_mount_state(mount_state);
|
||||
if (ret) {
|
||||
const auto mount_id = app_config::get_provider_display_name(pt_) + unique_id_;
|
||||
mount_state = mount_state[mount_id].empty()
|
||||
? json({{"Active", false}, {"Location", ""}, {"PID", -1}})
|
||||
: mount_state[mount_id];
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool lock_data::get_mount_state(json &mount_state) {
|
||||
HKEY key;
|
||||
auto ret = !::RegCreateKeyEx(
|
||||
HKEY_CURRENT_USER, ("SOFTWARE\\" + std::string(REPERTORY_DATA_NAME) + "\\Mounts").c_str(), 0,
|
||||
nullptr, 0, KEY_ALL_ACCESS, nullptr, &key, nullptr);
|
||||
if (ret) {
|
||||
DWORD i = 0u;
|
||||
DWORD data_size = 0u;
|
||||
std::string name;
|
||||
name.resize(32767u);
|
||||
auto name_size = static_cast<DWORD>(name.size());
|
||||
while (ret && (::RegEnumValue(key, i, &name[0], &name_size, nullptr, nullptr, nullptr,
|
||||
&data_size) == ERROR_SUCCESS)) {
|
||||
std::string data;
|
||||
data.resize(data_size);
|
||||
name_size++;
|
||||
if ((ret = !::RegEnumValue(key, i++, &name[0], &name_size, nullptr, nullptr,
|
||||
reinterpret_cast<LPBYTE>(&data[0]), &data_size))) {
|
||||
mount_state[name.c_str()] = json::parse(data);
|
||||
name_size = static_cast<DWORD>(name.size());
|
||||
data_size = 0u;
|
||||
}
|
||||
}
|
||||
::RegCloseKey(key);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
lock_result lock_data::grab_lock(const std::uint8_t &retryCount) {
|
||||
auto ret = lock_result::success;
|
||||
if (mutex_handle_ == INVALID_HANDLE_VALUE) {
|
||||
ret = lock_result::failure;
|
||||
} else {
|
||||
for (auto i = 0; (i <= retryCount) &&
|
||||
((mutex_state_ = ::WaitForSingleObject(mutex_handle_, 100)) == WAIT_TIMEOUT);
|
||||
i++) {
|
||||
}
|
||||
|
||||
switch (mutex_state_) {
|
||||
case WAIT_OBJECT_0: {
|
||||
ret = lock_result::success;
|
||||
auto should_reset = true;
|
||||
json mount_state;
|
||||
if (get_mount_state(pt_, mount_state)) {
|
||||
if (mount_state["Active"].get<bool>() && mount_state["Location"] == "elevating") {
|
||||
should_reset = false;
|
||||
}
|
||||
}
|
||||
|
||||
if (should_reset) {
|
||||
set_mount_state(false, "", -1);
|
||||
}
|
||||
} break;
|
||||
|
||||
case WAIT_TIMEOUT:
|
||||
ret = lock_result::locked;
|
||||
break;
|
||||
|
||||
default:
|
||||
ret = lock_result::failure;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
void lock_data::release() {
|
||||
if (mutex_handle_ != INVALID_HANDLE_VALUE) {
|
||||
if ((mutex_state_ == WAIT_OBJECT_0) || (mutex_state_ == WAIT_ABANDONED)) {
|
||||
::ReleaseMutex(mutex_handle_);
|
||||
}
|
||||
::CloseHandle(mutex_handle_);
|
||||
mutex_handle_ = INVALID_HANDLE_VALUE;
|
||||
}
|
||||
}
|
||||
|
||||
bool lock_data::set_mount_state(const bool &active, const std::string &mountLocation,
|
||||
const std::int64_t &pid) {
|
||||
auto ret = false;
|
||||
if (mutex_handle_ != INVALID_HANDLE_VALUE) {
|
||||
const auto mount_id = app_config::get_provider_display_name(pt_) + unique_id_;
|
||||
json mount_state;
|
||||
get_mount_state(mount_state);
|
||||
if ((mount_state.find(mount_id) == mount_state.end()) ||
|
||||
(mount_state[mount_id].find("Active") == mount_state[mount_id].end()) ||
|
||||
(mount_state[mount_id]["Active"].get<bool>() != active) ||
|
||||
(active && ((mount_state[mount_id].find("Location") == mount_state[mount_id].end()) ||
|
||||
(mount_state[mount_id]["Location"].get<std::string>() != mountLocation)))) {
|
||||
HKEY key;
|
||||
if ((ret = !::RegCreateKeyEx(
|
||||
HKEY_CURRENT_USER,
|
||||
("SOFTWARE\\" + std::string(REPERTORY_DATA_NAME) + "\\Mounts").c_str(), 0, nullptr,
|
||||
0, KEY_ALL_ACCESS, nullptr, &key, nullptr))) {
|
||||
const auto str = json({{"Active", active},
|
||||
{"Location", active ? mountLocation : ""},
|
||||
{"PID", active ? pid : -1}})
|
||||
.dump(0);
|
||||
ret =
|
||||
!::RegSetValueEx(key, &mount_id[0], 0, REG_SZ, reinterpret_cast<const BYTE *>(&str[0]),
|
||||
static_cast<DWORD>(str.size()));
|
||||
::RegCloseKey(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
} // namespace repertory
|
||||
|
||||
#endif //_WIN32
|
||||
209
src/providers/base_provider.cpp
Normal file
209
src/providers/base_provider.cpp
Normal file
@@ -0,0 +1,209 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "providers/base_provider.hpp"
|
||||
#include "app_config.hpp"
|
||||
#include "types/repertory.hpp"
|
||||
#include "utils/file_utils.hpp"
|
||||
#include "utils/global_data.hpp"
|
||||
#include "utils/native_file.hpp"
|
||||
#include "utils/path_utils.hpp"
|
||||
|
||||
namespace repertory {
|
||||
base_provider::base_provider(app_config &config) : config_(config), meta_db_(config) {}
|
||||
|
||||
void base_provider::notify_directory_added(const std::string &api_path,
|
||||
const std::string &api_parent) {
|
||||
recur_mutex_lock l(notify_added_mutex_);
|
||||
|
||||
const auto now = utils::get_file_time_now();
|
||||
api_item_added_(api_path, api_parent, "", true, now, now, now, now);
|
||||
}
|
||||
|
||||
void base_provider::update_filesystem_item(const bool &directory, const api_error &error,
|
||||
const std::string &api_path,
|
||||
filesystem_item &fsi) const {
|
||||
if (error == api_error::success) {
|
||||
fsi.directory = directory;
|
||||
fsi.lock = fsi.lock ? fsi.lock : std::make_shared<std::recursive_mutex>();
|
||||
fsi.api_path = api_path;
|
||||
fsi.api_parent = utils::path::get_parent_api_path(api_path);
|
||||
} else {
|
||||
event_system::instance().raise<filesystem_item_get_failed>(
|
||||
api_path, std::to_string(static_cast<int>(error)));
|
||||
}
|
||||
}
|
||||
|
||||
api_error base_provider::create_directory_clone_source_meta(const std::string &source_api_path,
|
||||
const std::string &api_path) {
|
||||
api_meta_map meta{};
|
||||
auto ret = get_item_meta(source_api_path, meta);
|
||||
if (ret == api_error::success) {
|
||||
ret = create_directory(api_path, meta);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error base_provider::create_file(const std::string &api_path, api_meta_map &meta) {
|
||||
const auto isDir = is_directory(api_path);
|
||||
const auto isFile = is_file(api_path);
|
||||
auto ret = isDir ? api_error::directory_exists
|
||||
: isFile ? api_error::file_exists
|
||||
: api_error::success;
|
||||
if (ret == api_error::success) {
|
||||
const auto source =
|
||||
utils::path::combine(get_config().get_cache_directory(), {utils::create_uuid_string()});
|
||||
|
||||
native_file_ptr nf;
|
||||
if ((ret = native_file::create_or_open(source, nf)) == api_error::success) {
|
||||
nf->close();
|
||||
}
|
||||
|
||||
if (ret == api_error::success) {
|
||||
if (((ret = set_item_meta(api_path, meta)) != api_error::success) ||
|
||||
((ret = set_source_path(api_path, source)) != api_error::success) ||
|
||||
((ret = upload_file(api_path, source, meta[META_ENCRYPTION_TOKEN])) !=
|
||||
api_error::success)) {
|
||||
meta_db_.remove_item_meta(format_api_path(api_path));
|
||||
utils::file::delete_file(source);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error base_provider::get_api_path_from_source(const std::string &source_path,
|
||||
std::string &api_path) const {
|
||||
const auto ret = meta_db_.get_api_path_from_source(source_path, api_path);
|
||||
restore_api_path(api_path);
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error base_provider::get_filesystem_item(const std::string &api_path, const bool &directory,
|
||||
filesystem_item &fsi) const {
|
||||
auto ret = api_error::error;
|
||||
if (directory) {
|
||||
ret = is_directory(api_path) ? api_error::success : api_error::item_not_found;
|
||||
update_filesystem_item(true, ret, api_path, fsi);
|
||||
} else {
|
||||
api_file file{};
|
||||
ret = get_filesystem_item_and_file(api_path, file, fsi);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error base_provider::get_filesystem_item_and_file(const std::string &api_path, api_file &file,
|
||||
filesystem_item &fsi) const {
|
||||
auto ret = get_item_meta(api_path, META_SOURCE, fsi.source_path);
|
||||
if (ret == api_error::success) {
|
||||
ret = get_file(api_path, file);
|
||||
if (ret == api_error::success) {
|
||||
fsi.encryption_token = file.encryption_token;
|
||||
fsi.size = file.file_size;
|
||||
} else if (not is_file(api_path)) {
|
||||
ret = api_error::item_not_found;
|
||||
}
|
||||
}
|
||||
|
||||
update_filesystem_item(false, ret, api_path, fsi);
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error base_provider::get_filesystem_item_from_source_path(const std::string &source_path,
|
||||
filesystem_item &fsi) const {
|
||||
auto ret = api_error::item_not_found;
|
||||
if (not source_path.empty()) {
|
||||
std::string api_path;
|
||||
if ((ret = get_api_path_from_source(source_path, api_path)) == api_error::success) {
|
||||
ret = get_filesystem_item(api_path, false, fsi);
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error base_provider::get_item_meta(const std::string &api_path, api_meta_map &meta) const {
|
||||
auto ret = meta_db_.get_item_meta(format_api_path(api_path), meta);
|
||||
if (ret == api_error::item_not_found) {
|
||||
auto get_meta = false;
|
||||
if (is_directory(api_path)) {
|
||||
notify_directory_added(api_path, utils::path::get_parent_api_path(api_path));
|
||||
get_meta = true;
|
||||
} else if (is_file(api_path)) {
|
||||
std::uint64_t file_size = 0u;
|
||||
if ((ret = get_file_size(api_path, file_size)) == api_error::success) {
|
||||
get_meta = ((ret = notify_file_added(api_path, utils::path::get_parent_api_path(api_path),
|
||||
file_size)) == api_error::success);
|
||||
}
|
||||
}
|
||||
if (get_meta) {
|
||||
ret = meta_db_.get_item_meta(format_api_path(api_path), meta);
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error base_provider::get_item_meta(const std::string &api_path, const std::string &key,
|
||||
std::string &value) const {
|
||||
auto ret = meta_db_.get_item_meta(format_api_path(api_path), key, value);
|
||||
if (ret == api_error::item_not_found) {
|
||||
auto get_meta = false;
|
||||
if (is_directory(api_path)) {
|
||||
notify_directory_added(api_path, utils::path::get_parent_api_path(api_path));
|
||||
get_meta = true;
|
||||
} else if (is_file(api_path)) {
|
||||
std::uint64_t file_size = 0u;
|
||||
if ((ret = get_file_size(api_path, file_size)) == api_error::success) {
|
||||
get_meta = ((ret = notify_file_added(api_path, utils::path::get_parent_api_path(api_path),
|
||||
file_size)) == api_error::success);
|
||||
}
|
||||
}
|
||||
if (get_meta) {
|
||||
ret = meta_db_.get_item_meta(format_api_path(api_path), key, value);
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
std::uint64_t base_provider::get_used_drive_space() const {
|
||||
return global_data::instance().get_used_drive_space();
|
||||
}
|
||||
|
||||
bool base_provider::start(api_item_added_callback api_item_added, i_open_file_table *oft) {
|
||||
api_item_added_ = api_item_added;
|
||||
oft_ = oft;
|
||||
|
||||
auto unmount_requested = false;
|
||||
{
|
||||
repertory::event_consumer ec("unmount_requested",
|
||||
[&unmount_requested](const event &) { unmount_requested = true; });
|
||||
for (std::uint16_t i = 0u; not unmount_requested && not is_online() &&
|
||||
(i < get_config().get_online_check_retry_secs());
|
||||
i++) {
|
||||
event_system::instance().raise<provider_offline>(
|
||||
get_config().get_host_config().host_name_or_ip, get_config().get_host_config().api_port);
|
||||
std::this_thread::sleep_for(1s);
|
||||
}
|
||||
}
|
||||
return unmount_requested;
|
||||
}
|
||||
} // namespace repertory
|
||||
192
src/providers/passthrough/passthroughprovider.cpp
Normal file
192
src/providers/passthrough/passthroughprovider.cpp
Normal file
@@ -0,0 +1,192 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#if defined(REPERTORY_TESTING_NEW)
|
||||
#include "providers/passthrough/passthroughprovider.hpp"
|
||||
#include "utils/file_utils.hpp"
|
||||
#include "utils/path_utils.hpp"
|
||||
|
||||
namespace repertory {
|
||||
std::string CPassthroughProvider::ConstructFullPath(const std::string &api_path) const {
|
||||
return utils::path::combine(passthroughLocation_, {api_path});
|
||||
}
|
||||
|
||||
api_error CPassthroughProvider::CreateDirectory(const std::string &api_path,
|
||||
const api_meta_map &meta) {
|
||||
const auto fullPath = ConstructFullPath(api_path);
|
||||
auto ret = utils::file::create_full_directory_path(fullPath) ? api_error::OSErrorCode
|
||||
: api_error::Success;
|
||||
if (ret == api_error::Success) {
|
||||
ret = set_item_meta(api_path, meta);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error CPassthroughProvider::get_file_list(ApiFileList &fileList) const {
|
||||
const auto fullPath = ConstructFullPath("/");
|
||||
const auto fl = utils::file::get_directory_files(fullPath, false, true);
|
||||
for (const auto &file : fl) {
|
||||
const auto api_path = utils::path::create_api_path(file.substr(fullPath.length()));
|
||||
/*
|
||||
struct ApiFile {
|
||||
std::string ApiFilePath{};
|
||||
std::string ApiParent{};
|
||||
std::uint64_t AccessedDate = 0;
|
||||
std::uint64_t ChangedDate = 0;
|
||||
std::uint64_t CreationDate = 0;
|
||||
std::string EncryptionToken{};
|
||||
std::uint64_t FileSize = 0;
|
||||
std::uint64_t ModifiedDate = 0;
|
||||
bool Recoverable = false;
|
||||
double Redundancy = 0.0;
|
||||
std::string SourceFilePath{};
|
||||
};
|
||||
*/
|
||||
ApiFile apiFile{
|
||||
api_path,
|
||||
utils::path::get_parent_api_path(api_path),
|
||||
};
|
||||
// apiFile.Recoverable = not IsProcessing(api_path);
|
||||
apiFile.Redundancy = 3.0;
|
||||
apiFile.SourceFilePath = file;
|
||||
// utils::file::UpdateApiFileInfo(apiFile);
|
||||
fileList.emplace_back(apiFile);
|
||||
}
|
||||
|
||||
return api_error::Success;
|
||||
}
|
||||
|
||||
std::uint64_t CPassthroughProvider::get_directory_item_count(const std::string &api_path) const {
|
||||
const auto fullPath = ConstructFullPath(api_path);
|
||||
return 0;
|
||||
}
|
||||
|
||||
api_error CPassthroughProvider::get_directory_items(const std::string &api_path,
|
||||
directory_item_list &list) const {
|
||||
return api_error::NotImplemented;
|
||||
}
|
||||
|
||||
api_error CPassthroughProvider::GetFile(const std::string &api_path, ApiFile &file) const {
|
||||
return api_error::Error;
|
||||
}
|
||||
|
||||
api_error CPassthroughProvider::GetFileSize(const std::string &api_path,
|
||||
std::uint64_t &fileSize) const {
|
||||
return api_error::Error;
|
||||
}
|
||||
|
||||
api_error CPassthroughProvider::get_filesystem_item(const std::string &api_path,
|
||||
const bool &directory,
|
||||
FileSystemItem &fileSystemItem) const {
|
||||
return api_error::NotImplemented;
|
||||
}
|
||||
|
||||
api_error
|
||||
CPassthroughProvider::get_filesystem_item_from_source_path(const std::string &sourceFilePath,
|
||||
FileSystemItem &fileSystemItem) const {
|
||||
return api_error::NotImplemented;
|
||||
}
|
||||
|
||||
api_error CPassthroughProvider::get_item_meta(const std::string &api_path,
|
||||
api_meta_map &meta) const {
|
||||
return api_error::NotImplemented;
|
||||
}
|
||||
|
||||
api_error CPassthroughProvider::get_item_meta(const std::string &api_path,
|
||||
const std::string &key, std::string &value) const {
|
||||
return api_error::NotImplemented;
|
||||
}
|
||||
|
||||
std::uint64_t CPassthroughProvider::get_total_drive_space() const { return 0; }
|
||||
|
||||
std::uint64_t CPassthroughProvider::get_total_item_count() const { return 0; }
|
||||
|
||||
std::uint64_t CPassthroughProvider::get_used_drive_space() const { return 0; }
|
||||
|
||||
bool CPassthroughProvider::IsDirectory(const std::string &api_path) const {
|
||||
const auto fullPath = ConstructFullPath(api_path);
|
||||
return utils::file::is_directory(fullPath);
|
||||
}
|
||||
|
||||
bool CPassthroughProvider::IsFile(const std::string &api_path) const {
|
||||
const auto fullPath = ConstructFullPath(api_path);
|
||||
return utils::file::is_file(fullPath);
|
||||
}
|
||||
|
||||
api_error CPassthroughProvider::notify_file_added(const std::string &api_path,
|
||||
const std::string &api_parent,
|
||||
const std::uint64_t &size) {
|
||||
return api_error::NotImplemented;
|
||||
}
|
||||
|
||||
api_error CPassthroughProvider::read_file_bytes(const std::string &apiFilepath,
|
||||
const std::size_t &size,
|
||||
const std::uint64_t &offset,
|
||||
std::vector<char> &data,
|
||||
const bool &stop_requested) {
|
||||
return api_error::NotImplemented;
|
||||
}
|
||||
|
||||
api_error CPassthroughProvider::RemoveDirectory(const std::string &api_path) {
|
||||
return api_error::NotImplemented;
|
||||
}
|
||||
|
||||
api_error CPassthroughProvider::RemoveFile(const std::string &api_path) {
|
||||
return api_error::NotImplemented;
|
||||
}
|
||||
|
||||
api_error CPassthroughProvider::RenameFile(const std::string &fromApiPath,
|
||||
const std::string &toApiPath) {
|
||||
return api_error::NotImplemented;
|
||||
}
|
||||
|
||||
api_error CPassthroughProvider::remove_item_meta(const std::string &api_path,
|
||||
const std::string &key) {
|
||||
return api_error::NotImplemented;
|
||||
}
|
||||
|
||||
api_error CPassthroughProvider::set_item_meta(const std::string &api_path,
|
||||
const std::string &key, const std::string &value) {
|
||||
return api_error::NotImplemented;
|
||||
}
|
||||
|
||||
api_error CPassthroughProvider::set_item_meta(const std::string &api_path,
|
||||
const api_meta_map &meta) {
|
||||
return api_error::NotImplemented;
|
||||
}
|
||||
|
||||
api_error CPassthroughProvider::set_source_path(const std::string &api_path,
|
||||
const std::string &sourcePath) {
|
||||
return api_error::NotImplemented;
|
||||
}
|
||||
|
||||
bool CPassthroughProvider::Start(ApiItemAdded apiItemAdded, i_open_file_table *openFileTable) {
|
||||
return false;
|
||||
}
|
||||
|
||||
void CPassthroughProvider::Stop() {}
|
||||
|
||||
api_error CPassthroughProvider::upload_file(const std::string &api_path,
|
||||
const std::string &sourcePath,
|
||||
const std::string &encryptionToken) {
|
||||
return api_error::NotImplemented;
|
||||
}
|
||||
} // namespace repertory
|
||||
|
||||
#endif // defined(REPERTORY_TESTING_NEW)
|
||||
76
src/providers/provider.cpp
Normal file
76
src/providers/provider.cpp
Normal file
@@ -0,0 +1,76 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "providers/provider.hpp"
|
||||
#include "comm/aws_s3/aws_s3_comm.hpp"
|
||||
#include "comm/curl/curl_comm.hpp"
|
||||
#include "comm/i_comm.hpp"
|
||||
#include "comm/i_s3_comm.hpp"
|
||||
#include "app_config.hpp"
|
||||
#include "events/events.hpp"
|
||||
#include "providers/passthrough/passthroughprovider.hpp"
|
||||
#include "providers/s3/s3_provider.hpp"
|
||||
#include "providers/sia/sia_provider.hpp"
|
||||
#include "providers/skynet/skynet_provider.hpp"
|
||||
#include "types/startup_exception.hpp"
|
||||
|
||||
namespace repertory {
|
||||
template <typename i, typename t>
|
||||
static void create_comm(std::unique_ptr<i> &comm, app_config &config) {
|
||||
if (comm) {
|
||||
throw startup_exception("'create_provider' should only be called once");
|
||||
}
|
||||
comm = std::make_unique<t>(config);
|
||||
}
|
||||
|
||||
std::unique_ptr<i_provider> create_provider(const provider_type &pt, app_config &config) {
|
||||
static std::mutex mutex;
|
||||
mutex_lock lock(mutex);
|
||||
|
||||
static std::unique_ptr<i_comm> comm;
|
||||
#if defined(REPERTORY_ENABLE_S3)
|
||||
static std::unique_ptr<i_s3_comm> s3_comm;
|
||||
#endif // defined(REPERTORY_ENABLE_S3)
|
||||
|
||||
switch (pt) {
|
||||
case provider_type::sia:
|
||||
create_comm<i_comm, curl_comm>(comm, config);
|
||||
return std::unique_ptr<i_provider>(dynamic_cast<i_provider *>(new sia_provider(config, *comm)));
|
||||
#if defined(REPERTORY_ENABLE_S3)
|
||||
case provider_type::s3:
|
||||
create_comm<i_s3_comm, aws_s3_comm>(s3_comm, config);
|
||||
return std::unique_ptr<i_provider>(
|
||||
dynamic_cast<i_provider *>(new s3_provider(config, *s3_comm)));
|
||||
#endif // defined(REPERTORY_ENABLE_S3)
|
||||
#if defined(REPERTORY_ENABLE_SKYNET)
|
||||
case provider_type::skynet:
|
||||
create_comm<i_comm, curl_comm>(comm, config);
|
||||
return std::unique_ptr<i_provider>(
|
||||
dynamic_cast<i_provider *>(new skynet_provider(config, *comm)));
|
||||
#endif // defined(REPERTORY_ENABLE_SKYNET)
|
||||
#if defined(REPERTORY_TESTING_NEW)
|
||||
case provider_type::passthrough:
|
||||
return std::unique_ptr<i_provider>(
|
||||
dynamic_cast<i_provider *>(new CPassthroughProvider(config)));
|
||||
#endif // defined(REPERTORY_TESTING_NEW)
|
||||
case provider_type::unknown:
|
||||
default:
|
||||
throw startup_exception("provider not supported: " + app_config::get_provider_display_name(pt));
|
||||
}
|
||||
}
|
||||
} // namespace repertory
|
||||
562
src/providers/s3/s3_provider.cpp
Normal file
562
src/providers/s3/s3_provider.cpp
Normal file
@@ -0,0 +1,562 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#if defined(REPERTORY_ENABLE_S3)
|
||||
#include "providers/s3/s3_provider.hpp"
|
||||
#include "comm/i_s3_comm.hpp"
|
||||
#include "app_config.hpp"
|
||||
#include "db/meta_db.hpp"
|
||||
#include "drives/i_open_file_table.hpp"
|
||||
#include "types/repertory.hpp"
|
||||
#include "types/startup_exception.hpp"
|
||||
#include "utils/encryption.hpp"
|
||||
#include "utils/file_utils.hpp"
|
||||
#include "utils/global_data.hpp"
|
||||
#include "utils/path_utils.hpp"
|
||||
|
||||
namespace repertory {
|
||||
s3_provider::s3_provider(app_config &config, i_s3_comm &s3_comm)
|
||||
: base_provider(config),
|
||||
s3_comm_(s3_comm),
|
||||
directory_db_(config),
|
||||
upload_manager_(
|
||||
config, [this](const std::string &api_path) -> bool { return this->is_file(api_path); },
|
||||
[this](const upload_manager::upload &upload, json &data, json &error) -> api_error {
|
||||
return this->upload_handler(upload, data, error);
|
||||
},
|
||||
[this](const std::string &api_path, const std::string &source_path, const json &data) {
|
||||
return this->upload_completed(api_path, source_path, data);
|
||||
}) {}
|
||||
|
||||
void s3_provider::build_directories() {
|
||||
api_file_list list{};
|
||||
s3_comm_.get_file_list([](const std::string & /*api_path*/) -> std::string { return ""; },
|
||||
[](const std::string & /*key*/,
|
||||
const std::string &object_name) -> std::string { return object_name; },
|
||||
list);
|
||||
total_item_count_ = list.size();
|
||||
|
||||
std::unordered_map<std::string, std::uint8_t> directories;
|
||||
for (const auto &file : list) {
|
||||
if (directories.find(file.api_parent) == directories.end()) {
|
||||
directories[file.api_parent] = 0u;
|
||||
|
||||
const auto directory_parts = utils::string::split(file.api_parent, '/', false);
|
||||
|
||||
std::string current_directory;
|
||||
for (std::size_t i = 0u; i < (directory_parts.size() - 1u); i++) {
|
||||
current_directory = current_directory.empty()
|
||||
? utils::path::create_api_path(directory_parts[0u])
|
||||
: utils::path::combine(current_directory, {directory_parts[i]});
|
||||
directories[current_directory] = 0u;
|
||||
}
|
||||
}
|
||||
}
|
||||
list.clear();
|
||||
|
||||
for (const auto &kv : directories) {
|
||||
if (not directory_db_.is_directory(kv.first)) {
|
||||
auto api_path = kv.first;
|
||||
restore_api_path(api_path);
|
||||
this->notify_directory_added(api_path, utils::path::get_parent_api_path(api_path));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
api_error s3_provider::create_directory(const std::string &api_path, const api_meta_map &meta) {
|
||||
#ifdef _WIN32
|
||||
auto ret = is_directory(api_path) ? api_error::directory_exists
|
||||
: is_file(api_path) ? api_error::file_exists
|
||||
: api_error::success;
|
||||
if (ret != api_error::success) {
|
||||
return ret;
|
||||
}
|
||||
#else
|
||||
auto ret = api_error::success;
|
||||
#endif
|
||||
if (utils::path::is_trash_directory(api_path)) {
|
||||
ret = api_error::access_denied;
|
||||
} else {
|
||||
if ((utils::path::get_parent_api_path(api_path) == "/") &&
|
||||
s3_comm_.get_s3_config().bucket.empty()) {
|
||||
ret = s3_comm_.create_bucket(api_path);
|
||||
}
|
||||
|
||||
if (ret == api_error::success) {
|
||||
ret = directory_db_.create_directory(format_api_path(api_path));
|
||||
}
|
||||
}
|
||||
|
||||
if (ret == api_error::success) {
|
||||
set_item_meta(api_path, meta);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error s3_provider::create_file(const std::string &api_path, api_meta_map &meta) {
|
||||
if (meta[META_SIZE].empty()) {
|
||||
meta[META_SIZE] = "0";
|
||||
}
|
||||
|
||||
if (meta[META_ENCRYPTION_TOKEN].empty()) {
|
||||
meta[META_ENCRYPTION_TOKEN] = s3_comm_.get_s3_config().encryption_token;
|
||||
}
|
||||
|
||||
return base_provider::create_file(api_path, meta);
|
||||
}
|
||||
|
||||
std::string s3_provider::format_api_path(std::string api_path) const {
|
||||
return s3_comm_.get_s3_config().bucket.empty()
|
||||
? api_path
|
||||
: utils::path::create_api_path(
|
||||
utils::path::combine(s3_comm_.get_s3_config().bucket, {api_path}));
|
||||
}
|
||||
|
||||
std::uint64_t s3_provider::get_directory_item_count(const std::string &api_path) const {
|
||||
return s3_comm_.get_directory_item_count(api_path, [this](directory_item &di,
|
||||
const bool &format_path) {
|
||||
this->update_item_meta(di, format_path);
|
||||
}) + directory_db_.get_sub_directory_count(format_api_path(api_path));
|
||||
}
|
||||
|
||||
api_error s3_provider::get_directory_items(const std::string &api_path,
|
||||
directory_item_list &list) const {
|
||||
const auto res = s3_comm_.get_directory_items(
|
||||
api_path,
|
||||
[this](directory_item &di, const bool &format_path) {
|
||||
this->update_item_meta(di, format_path);
|
||||
},
|
||||
list);
|
||||
|
||||
directory_db_.populate_sub_directories(
|
||||
format_api_path(api_path),
|
||||
[this](directory_item &di, const bool &format_path) {
|
||||
if (format_path) {
|
||||
restore_api_path(di.api_path);
|
||||
di.api_parent = utils::path::get_parent_api_path(di.api_path);
|
||||
}
|
||||
this->get_item_meta(di.api_path, di.meta);
|
||||
this->oft_->update_directory_item(di);
|
||||
},
|
||||
list);
|
||||
|
||||
std::sort(list.begin(), list.end(), [](const auto &a, const auto &b) -> bool {
|
||||
return (a.directory && not b.directory) ||
|
||||
(not(b.directory && not a.directory) && (a.api_path.compare(b.api_path) < 0));
|
||||
});
|
||||
|
||||
list.insert(list.begin(), directory_item{
|
||||
"..",
|
||||
"",
|
||||
true,
|
||||
});
|
||||
list.insert(list.begin(), directory_item{
|
||||
".",
|
||||
"",
|
||||
true,
|
||||
});
|
||||
|
||||
return ((res == api_error::success) ? api_error::success : api_error::error);
|
||||
}
|
||||
|
||||
api_error s3_provider::get_file(const std::string &api_path, api_file &file) const {
|
||||
auto real_api_path = format_api_path(api_path);
|
||||
const auto ret = s3_comm_.get_file(
|
||||
api_path,
|
||||
[this, &api_path, &real_api_path]() -> std::string {
|
||||
auto key = *(utils::string::split(api_path, '/', false).end() - 1u);
|
||||
std::string meta_api_path;
|
||||
meta_db_.get_api_path_from_key(key, meta_api_path);
|
||||
if (meta_api_path.empty()) {
|
||||
meta_db_.get_item_meta(real_api_path, META_KEY, key);
|
||||
}
|
||||
return key;
|
||||
},
|
||||
[this, &real_api_path](const std::string &key,
|
||||
const std::string &object_name) -> std::string {
|
||||
if (key.empty()) {
|
||||
return object_name;
|
||||
}
|
||||
|
||||
meta_db_.get_api_path_from_key(key, real_api_path);
|
||||
|
||||
auto object_parts = utils::string::split(object_name, '/', false);
|
||||
object_parts[object_parts.size() - 1u] =
|
||||
*(utils::string::split(real_api_path, '/', false).end() - 1u);
|
||||
return utils::string::join(object_parts, '/');
|
||||
},
|
||||
[this, &real_api_path]() -> std::string {
|
||||
std::string encryption_token;
|
||||
meta_db_.get_item_meta(real_api_path, META_ENCRYPTION_TOKEN, encryption_token);
|
||||
return encryption_token;
|
||||
},
|
||||
file);
|
||||
if (ret == api_error::success) {
|
||||
restore_api_path(file.api_path);
|
||||
file.api_parent = utils::path::get_parent_api_path(file.api_path);
|
||||
} else {
|
||||
event_system::instance().raise<file_get_failed>(api_path,
|
||||
std::to_string(static_cast<int>(ret)));
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error s3_provider::get_file_list(api_file_list &list) const {
|
||||
const auto ret = s3_comm_.get_file_list(
|
||||
[this](const std::string &api_path) -> std::string {
|
||||
std::string encryption_token;
|
||||
meta_db_.get_item_meta(api_path, META_ENCRYPTION_TOKEN, encryption_token);
|
||||
return encryption_token;
|
||||
},
|
||||
[this](const std::string &key, const std::string &object_name) -> std::string {
|
||||
std::string real_api_path;
|
||||
meta_db_.get_api_path_from_key(key, real_api_path);
|
||||
if (real_api_path.empty()) {
|
||||
return object_name;
|
||||
}
|
||||
|
||||
auto object_parts = utils::string::split(object_name, '/', false);
|
||||
object_parts[object_parts.size() - 1u] =
|
||||
*(utils::string::split(real_api_path, '/', false).end() - 1u);
|
||||
return utils::string::join(object_parts, '/');
|
||||
},
|
||||
list);
|
||||
|
||||
for (auto &file : list) {
|
||||
restore_api_path(file.api_path);
|
||||
file.api_parent = utils::path::get_parent_api_path(file.api_path);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error s3_provider::get_file_size(const std::string &api_path, std::uint64_t &file_size) const {
|
||||
api_file file{};
|
||||
const auto ret = get_file(api_path, file);
|
||||
if (ret == api_error::success) {
|
||||
file_size = file.file_size;
|
||||
} else {
|
||||
event_system::instance().raise<file_get_size_failed>(api_path,
|
||||
std::to_string(static_cast<int>(ret)));
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool s3_provider::is_directory(const std::string &api_path) const {
|
||||
if (api_path == "/") {
|
||||
return true;
|
||||
}
|
||||
return directory_db_.is_directory(format_api_path(api_path));
|
||||
}
|
||||
|
||||
bool s3_provider::is_file(const std::string &api_path) const {
|
||||
if (api_path == "/") {
|
||||
return false;
|
||||
}
|
||||
|
||||
return not directory_db_.is_directory(format_api_path(api_path)) &&
|
||||
s3_comm_.exists(api_path, [&]() -> std::string {
|
||||
std::string key;
|
||||
meta_db_.get_item_meta(format_api_path(api_path), META_KEY, key);
|
||||
return key;
|
||||
});
|
||||
}
|
||||
|
||||
bool s3_provider::is_online() const { return s3_comm_.is_online(); }
|
||||
|
||||
bool s3_provider::is_processing(const std::string &api_path) const {
|
||||
return upload_manager_.is_processing(api_path);
|
||||
}
|
||||
|
||||
void s3_provider::notify_directory_added(const std::string &api_path,
|
||||
const std::string &api_parent) {
|
||||
base_provider::notify_directory_added(api_path, api_parent);
|
||||
directory_db_.create_directory(format_api_path(api_path), true);
|
||||
}
|
||||
|
||||
api_error s3_provider::notify_file_added(const std::string &api_path,
|
||||
const std::string & /*api_parent*/,
|
||||
const std::uint64_t & /*size*/) {
|
||||
recur_mutex_lock l(notify_added_mutex_);
|
||||
|
||||
api_file file{};
|
||||
auto decrypted = false;
|
||||
std::string key;
|
||||
std::string file_name;
|
||||
|
||||
auto ret = s3_comm_.get_file(
|
||||
api_path,
|
||||
[this, &api_path, &decrypted, &file_name, &key]() -> std::string {
|
||||
const auto temp_key = *(utils::string::split(api_path, '/', false).end() - 1u);
|
||||
file_name = temp_key;
|
||||
auto decrypted_file_name = file_name;
|
||||
if (utils::encryption::decrypt_file_name(s3_comm_.get_s3_config().encryption_token,
|
||||
decrypted_file_name) == api_error::success) {
|
||||
decrypted = true;
|
||||
key = temp_key;
|
||||
file_name = decrypted_file_name;
|
||||
return key;
|
||||
}
|
||||
|
||||
return "";
|
||||
},
|
||||
[&file_name](const std::string &, const std::string &object_name) -> std::string {
|
||||
auto object_parts = utils::string::split(object_name, '/', false);
|
||||
object_parts[object_parts.size() - 1u] = file_name;
|
||||
return utils::string::join(object_parts, '/');
|
||||
},
|
||||
[this, &decrypted]() -> std::string {
|
||||
return decrypted ? s3_comm_.get_s3_config().encryption_token : "";
|
||||
},
|
||||
file);
|
||||
|
||||
restore_api_path(file.api_path);
|
||||
|
||||
file.api_parent = utils::path::get_parent_api_path(file.api_path);
|
||||
if (ret == api_error::success) {
|
||||
api_item_added_(file.api_path, file.api_parent, file.source_path, false, file.created_date,
|
||||
file.accessed_date, file.modified_date, file.changed_date);
|
||||
|
||||
set_item_meta(file.api_path, META_SIZE, std::to_string(file.file_size));
|
||||
set_item_meta(file.api_path, META_KEY, key);
|
||||
set_item_meta(file.api_path, META_ENCRYPTION_TOKEN,
|
||||
decrypted ? s3_comm_.get_s3_config().encryption_token : "");
|
||||
|
||||
if (file.file_size) {
|
||||
global_data::instance().increment_used_drive_space(file.file_size);
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error s3_provider::read_file_bytes(const std::string &api_path, const std::size_t &size,
|
||||
const std::uint64_t &offset, std::vector<char> &data,
|
||||
const bool &stop_requested) {
|
||||
const auto res = s3_comm_.read_file_bytes(
|
||||
api_path, size, offset, data,
|
||||
[this, &api_path]() -> std::string {
|
||||
std::string key;
|
||||
meta_db_.get_item_meta(format_api_path(api_path), META_KEY, key);
|
||||
return key;
|
||||
},
|
||||
[this, &api_path]() -> std::uint64_t {
|
||||
std::string temp;
|
||||
meta_db_.get_item_meta(format_api_path(api_path), META_SIZE, temp);
|
||||
return utils::string::to_uint64(temp);
|
||||
},
|
||||
[this, &api_path]() -> std::string {
|
||||
std::string encryption_token;
|
||||
meta_db_.get_item_meta(format_api_path(api_path), META_ENCRYPTION_TOKEN, encryption_token);
|
||||
return encryption_token;
|
||||
},
|
||||
stop_requested);
|
||||
return ((res == api_error::success) ? api_error::success : api_error::download_failed);
|
||||
}
|
||||
|
||||
api_error s3_provider::remove_directory(const std::string &api_path) {
|
||||
auto ret = directory_db_.remove_directory(format_api_path(api_path));
|
||||
if (ret == api_error::success) {
|
||||
if ((utils::path::get_parent_api_path(api_path) == "/") &&
|
||||
s3_comm_.get_s3_config().bucket.empty()) {
|
||||
ret = s3_comm_.remove_bucket(api_path);
|
||||
}
|
||||
}
|
||||
|
||||
if (ret == api_error::success) {
|
||||
remove_item_meta(api_path);
|
||||
event_system::instance().raise<directory_removed>(api_path);
|
||||
} else {
|
||||
event_system::instance().raise<directory_remove_failed>(api_path,
|
||||
std::to_string(static_cast<int>(ret)));
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error s3_provider::remove_file(const std::string &api_path) {
|
||||
upload_manager_.remove_upload(api_path);
|
||||
const auto res = s3_comm_.remove_file(api_path, [&]() -> std::string {
|
||||
std::string key;
|
||||
meta_db_.get_item_meta(format_api_path(api_path), META_KEY, key);
|
||||
return key;
|
||||
});
|
||||
|
||||
if (res == api_error::success) {
|
||||
remove_item_meta(api_path);
|
||||
event_system::instance().raise<file_removed>(api_path);
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
event_system::instance().raise<file_remove_failed>(api_path,
|
||||
std::to_string(static_cast<int>(res)));
|
||||
return api_error::error;
|
||||
}
|
||||
|
||||
api_error s3_provider::rename_file(const std::string & /*from_api_path*/,
|
||||
const std::string & /*to_api_path*/) {
|
||||
// TODO Pending encrypted file name support
|
||||
return api_error::not_implemented;
|
||||
/* std::string curSource; */
|
||||
/* auto ret = get_item_meta(fromApiPath, META_SOURCE, curSource); */
|
||||
/* if (ret == api_error::success) { */
|
||||
/* ret = s3_comm_.RenameFile(fromApiPath, toApiPath); */
|
||||
/* if (ret == api_error::success) { */
|
||||
/* metaDb_.RenameItemMeta(curSource, FormatApiFilePath(fromApiPath), */
|
||||
/* FormatApiFilePath(toApiPath)); */
|
||||
/* } else { */
|
||||
/* event_system::instance().raise<FileRenameFailed>(fromApiPath, toApiPath, */
|
||||
/* std::to_string(static_cast<int>(ret))); */
|
||||
/* } */
|
||||
/* } */
|
||||
/* return ret; */
|
||||
}
|
||||
|
||||
std::string &s3_provider::restore_api_path(std::string &api_path) const {
|
||||
if (api_path != "/") {
|
||||
if (not s3_comm_.get_s3_config().bucket.empty()) {
|
||||
api_path = utils::path::create_api_path(
|
||||
api_path.substr(s3_comm_.get_s3_config().bucket.length() + 1u));
|
||||
}
|
||||
}
|
||||
return api_path;
|
||||
}
|
||||
|
||||
bool s3_provider::start(api_item_added_callback api_item_added, i_open_file_table *oft) {
|
||||
const auto unmount_requested = base_provider::start(api_item_added, oft);
|
||||
if (not unmount_requested && is_online()) {
|
||||
build_directories();
|
||||
upload_manager_.start();
|
||||
}
|
||||
|
||||
api_file_list list;
|
||||
if (get_file_list(list) != api_error::success) {
|
||||
throw startup_exception("failed to determine used space");
|
||||
}
|
||||
|
||||
const auto total_size =
|
||||
std::accumulate(list.begin(), list.end(), std::uint64_t(0),
|
||||
[](std::uint64_t t, const api_file &file) { return t + file.file_size; });
|
||||
global_data::instance().initialize_used_drive_space(total_size);
|
||||
|
||||
return unmount_requested;
|
||||
}
|
||||
|
||||
void s3_provider::stop() { upload_manager_.stop(); }
|
||||
|
||||
void s3_provider::update_item_meta(directory_item &di, const bool &format_path) const {
|
||||
if (format_path) {
|
||||
restore_api_path(di.api_path);
|
||||
di.api_parent = utils::path::get_parent_api_path(di.api_path);
|
||||
}
|
||||
|
||||
if (not di.directory && not di.resolved) {
|
||||
std::string real_api_path{};
|
||||
const auto get_api_path = [&]() -> api_error {
|
||||
if (meta_db_.get_api_path_from_key(
|
||||
*(utils::string::split(di.api_path, '/', false).end() - 1u), real_api_path) ==
|
||||
api_error::success) {
|
||||
di.api_path = real_api_path;
|
||||
if (format_path) {
|
||||
restore_api_path(di.api_path);
|
||||
di.api_parent = utils::path::get_parent_api_path(di.api_path);
|
||||
}
|
||||
return api_error::success;
|
||||
}
|
||||
return api_error::item_not_found;
|
||||
};
|
||||
|
||||
if (get_api_path() != api_error::success) {
|
||||
this->get_item_meta(di.api_path, di.meta);
|
||||
get_api_path();
|
||||
}
|
||||
|
||||
di.resolved = true;
|
||||
}
|
||||
|
||||
if (di.meta.empty()) {
|
||||
this->get_item_meta(di.api_path, di.meta);
|
||||
}
|
||||
|
||||
if (not di.directory) {
|
||||
if (di.meta[META_SIZE].empty()) {
|
||||
const_cast<s3_provider *>(this)->set_item_meta(di.api_path, META_SIZE,
|
||||
std::to_string(di.size));
|
||||
} else {
|
||||
di.size = utils::string::to_uint64(di.meta[META_SIZE]);
|
||||
}
|
||||
this->oft_->update_directory_item(di);
|
||||
}
|
||||
}
|
||||
|
||||
api_error s3_provider::upload_file(const std::string &api_path, const std::string &source_path,
|
||||
const std::string &encryption_token) {
|
||||
std::uint64_t file_size = 0u;
|
||||
utils::file::get_file_size(source_path, file_size);
|
||||
|
||||
auto ret = set_source_path(api_path, source_path);
|
||||
if (ret == api_error::success) {
|
||||
if (((ret = set_item_meta(api_path, META_SIZE, std::to_string(file_size))) ==
|
||||
api_error::success) &&
|
||||
((ret = set_item_meta(api_path, META_ENCRYPTION_TOKEN, encryption_token)) ==
|
||||
api_error::success)) {
|
||||
if (file_size == 0) {
|
||||
ret = s3_comm_.upload_file(
|
||||
api_path, source_path, encryption_token,
|
||||
[this, &api_path]() -> std::string {
|
||||
std::string key;
|
||||
meta_db_.get_item_meta(format_api_path(api_path), META_KEY, key);
|
||||
return key;
|
||||
},
|
||||
[this, &api_path](const std::string &key) -> api_error {
|
||||
return set_item_meta(api_path, META_KEY, key);
|
||||
},
|
||||
false);
|
||||
} else {
|
||||
ret = upload_manager_.queue_upload(api_path, source_path, encryption_token);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error s3_provider::upload_handler(const upload_manager::upload &upload, json &, json &) {
|
||||
event_system::instance().raise<file_upload_begin>(upload.api_path, upload.source_path);
|
||||
auto ret = api_error::upload_failed;
|
||||
if (not upload.cancel) {
|
||||
ret = (s3_comm_.upload_file(
|
||||
upload.api_path, upload.source_path, upload.encryption_token,
|
||||
[this, &upload]() -> std::string {
|
||||
std::string key;
|
||||
meta_db_.get_item_meta(format_api_path(upload.api_path), META_KEY, key);
|
||||
return key;
|
||||
},
|
||||
[this, &upload](const std::string &key) -> api_error {
|
||||
return set_item_meta(upload.api_path, META_KEY, key);
|
||||
},
|
||||
upload.cancel) == api_error::success)
|
||||
? api_error::success
|
||||
: api_error::upload_failed;
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
} // namespace repertory
|
||||
#endif // REPERTORY_ENABLE_S3
|
||||
700
src/providers/sia/sia_provider.cpp
Normal file
700
src/providers/sia/sia_provider.cpp
Normal file
@@ -0,0 +1,700 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "providers/sia/sia_provider.hpp"
|
||||
#include "comm/i_comm.hpp"
|
||||
#include "app_config.hpp"
|
||||
#include "db/meta_db.hpp"
|
||||
#include "drives/i_open_file_table.hpp"
|
||||
#include "events/events.hpp"
|
||||
#include "types/repertory.hpp"
|
||||
#include "types/startup_exception.hpp"
|
||||
#include "utils/file_utils.hpp"
|
||||
#include "utils/global_data.hpp"
|
||||
#include "utils/path_utils.hpp"
|
||||
#include "utils/polling.hpp"
|
||||
|
||||
namespace repertory {
|
||||
sia_provider::sia_provider(app_config &config, i_comm &comm) : base_provider(config), comm_(comm) {}
|
||||
|
||||
void sia_provider::calculate_total_drive_space() {
|
||||
std::uint64_t ret = 0u;
|
||||
json result, error;
|
||||
auto storage_cost = get_config().get_storage_byte_month();
|
||||
const auto success = (get_comm().get("/renter/prices", result, error) == api_error::success);
|
||||
if (success || (storage_cost > 0)) {
|
||||
if (success) {
|
||||
storage_cost = result["storageterabytemonth"].get<std::string>();
|
||||
get_config().set_storage_byte_month(storage_cost);
|
||||
}
|
||||
if (get_comm().get("/renter", result, error) == api_error::success) {
|
||||
const auto funds = utils::hastings_string_to_api_currency(
|
||||
result["settings"]["allowance"]["funds"].get<std::string>());
|
||||
if ((storage_cost > 0) && (funds > 0)) {
|
||||
const auto funds_in_hastings = utils::api_currency_to_hastings(funds);
|
||||
ttmath::Parser<api_currency> parser;
|
||||
parser.Parse(funds_in_hastings.ToString() + " / " + storage_cost.ToString() + " * 1e12");
|
||||
ret = not parser.stack.empty() ? parser.stack[0u].value.ToUInt() : 0u;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
total_drive_space_ = ret;
|
||||
}
|
||||
|
||||
api_error sia_provider::calculate_used_drive_space(std::uint64_t &used_space) {
|
||||
api_file_list list;
|
||||
const auto ret = get_file_list(list);
|
||||
used_space = std::accumulate(
|
||||
list.begin(), list.end(), std::uint64_t(0), [this](std::uint64_t a, const auto &v) {
|
||||
if (not meta_db_.get_item_meta_exists(v.api_path)) {
|
||||
this->notify_file_added(v.api_path, utils::path::get_parent_api_path(v.api_path), 0);
|
||||
}
|
||||
return a + v.file_size;
|
||||
});
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error sia_provider::check_file_exists(const std::string &api_path) const {
|
||||
json data, error;
|
||||
auto ret = get_comm().get("/renter/file" + api_path, data, error);
|
||||
if ((ret != api_error::success) && check_not_found(error)) {
|
||||
ret = api_error::item_not_found;
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool sia_provider::check_directory_found(const json &error) const {
|
||||
return ((error.find("message") != error.end()) &&
|
||||
(utils::string::contains(error["message"].get<std::string>(),
|
||||
"a siadir already exists at that location")));
|
||||
}
|
||||
|
||||
bool sia_provider::check_not_found(const json &error) const {
|
||||
return (
|
||||
(error.find("message") != error.end()) &&
|
||||
(utils::string::contains(error["message"].get<std::string>(), "no file known") ||
|
||||
utils::string::contains(error["message"].get<std::string>(), "path does not exist") ||
|
||||
utils::string::contains(error["message"].get<std::string>(), "no such file or directory") ||
|
||||
utils::string::contains(error["message"].get<std::string>(), "cannot find the file") ||
|
||||
utils::string::contains(error["message"].get<std::string>(),
|
||||
"no siadir known with that path") ||
|
||||
utils::string::contains(error["message"].get<std::string>(), "cannot find the path")));
|
||||
}
|
||||
|
||||
void sia_provider::cleanup() {
|
||||
remove_deleted_files();
|
||||
remove_unknown_source_files();
|
||||
remove_expired_orphaned_files();
|
||||
}
|
||||
|
||||
void sia_provider::create_api_file(const json &json_file, const std::string &path_name,
|
||||
api_file &file) const {
|
||||
file.api_path = utils::path::create_api_path(json_file[path_name].get<std::string>());
|
||||
file.api_parent = utils::path::get_parent_api_path(file.api_path);
|
||||
file.file_size = json_file["filesize"].get<std::uint64_t>();
|
||||
file.recoverable = json_file["recoverable"].get<bool>();
|
||||
file.redundancy = json_file["redundancy"].get<double>();
|
||||
file.source_path = json_file["localpath"].get<std::string>();
|
||||
|
||||
set_api_file_dates(json_file, file);
|
||||
}
|
||||
|
||||
api_error sia_provider::create_directory(const std::string &api_path, const api_meta_map &meta) {
|
||||
#ifdef _WIN32
|
||||
auto ret = is_directory(api_path) ? api_error::directory_exists
|
||||
: is_file(api_path) ? api_error::file_exists
|
||||
: api_error::success;
|
||||
#else
|
||||
auto ret = api_error::success;
|
||||
#endif
|
||||
if (ret == api_error::success) {
|
||||
json result, error;
|
||||
ret = get_comm().post("/renter/dir" + api_path, {{"action", "create"}}, result, error);
|
||||
if (ret == api_error::success) {
|
||||
ret = set_item_meta(api_path, meta);
|
||||
} else if (check_directory_found(error)) {
|
||||
ret = api_error::directory_exists;
|
||||
} else {
|
||||
event_system::instance().raise<repertory_exception>(__FUNCTION__, error.dump(2));
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
void sia_provider::drive_space_thread() {
|
||||
while (not stop_requested_) {
|
||||
unique_mutex_lock l(start_stop_mutex_);
|
||||
if (not stop_requested_) {
|
||||
start_stop_notify_.wait_for(l, 5s);
|
||||
}
|
||||
l.unlock();
|
||||
|
||||
if (not stop_requested_) {
|
||||
calculate_total_drive_space();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
api_error sia_provider::get_directory(const std::string &api_path, json &result,
|
||||
json &error) const {
|
||||
return get_comm().get("/renter/dir" + api_path, result, error);
|
||||
}
|
||||
|
||||
std::uint64_t sia_provider::get_directory_item_count(const std::string &api_path) const {
|
||||
std::uint64_t ret = 0u;
|
||||
|
||||
json result, error;
|
||||
const auto res = get_directory(api_path, result, error);
|
||||
if (res == api_error::success) {
|
||||
const auto directory_count = result["directories"].size() - 1;
|
||||
const auto file_count = result["files"].size();
|
||||
ret = file_count + directory_count;
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error sia_provider::get_directory_items(const std::string &api_path,
|
||||
directory_item_list &list) const {
|
||||
list.clear();
|
||||
|
||||
json result, error;
|
||||
auto ret = get_directory(api_path, result, error);
|
||||
if (ret == api_error::success) {
|
||||
const auto create_directory_item = [this](const std::string &item_path, const bool &directory,
|
||||
const json &item, directory_item &di) {
|
||||
di.api_path = item_path;
|
||||
di.api_parent = utils::path::get_parent_api_path(di.api_path);
|
||||
di.directory = directory;
|
||||
if (directory) {
|
||||
const auto directory_count = item["numsubdirs"].get<std::uint64_t>();
|
||||
const auto file_count = item["numfiles"].get<std::uint64_t>();
|
||||
di.size = (directory_count + file_count);
|
||||
} else {
|
||||
di.size = item["filesize"].get<std::uint64_t>();
|
||||
}
|
||||
this->get_item_meta(di.api_path, di.meta);
|
||||
this->oft_->update_directory_item(di);
|
||||
};
|
||||
|
||||
const auto process_item = [&](const bool &directory, const json &item) {
|
||||
const auto item_path = utils::path::create_api_path(
|
||||
item[app_config::get_provider_path_name(get_config().get_provider_type())]
|
||||
.get<std::string>());
|
||||
const auto is_root_path = (item_path == api_path);
|
||||
|
||||
directory_item di{};
|
||||
create_directory_item(item_path, directory, item, di);
|
||||
if (is_root_path) {
|
||||
di.api_path = ".";
|
||||
}
|
||||
list.emplace_back(di);
|
||||
if (is_root_path) {
|
||||
if (item_path == "/") {
|
||||
di.api_path = "..";
|
||||
list.emplace_back(di);
|
||||
} else {
|
||||
directory_item dot_dot_di{};
|
||||
json result, error;
|
||||
const auto res = get_directory(di.api_parent, result, error);
|
||||
if (res == api_error::success) {
|
||||
create_directory_item(di.api_parent, true, result["directories"][0u], dot_dot_di);
|
||||
}
|
||||
dot_dot_di.api_path = "..";
|
||||
list.emplace_back(dot_dot_di);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
for (const auto &dir : result["directories"]) {
|
||||
process_item(true, dir);
|
||||
}
|
||||
|
||||
for (const auto &file : result["files"]) {
|
||||
process_item(false, file);
|
||||
}
|
||||
|
||||
ret = api_error::success;
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error sia_provider::get_file(const std::string &api_path, api_file &file) const {
|
||||
json data, error;
|
||||
const auto ret = get_comm().get("/renter/file" + api_path, data, error);
|
||||
if (ret == api_error::success) {
|
||||
const std::string path_name =
|
||||
app_config::get_provider_path_name(get_config().get_provider_type());
|
||||
create_api_file(data["file"], path_name, file);
|
||||
} else {
|
||||
event_system::instance().raise<file_get_failed>(api_path, error.dump(2));
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error sia_provider::get_file_list(api_file_list &list) const {
|
||||
auto ret = api_error::success;
|
||||
json data;
|
||||
json error;
|
||||
if ((ret = get_comm().get("/renter/files", data, error)) == api_error::success) {
|
||||
for (const auto &file_data : data["files"]) {
|
||||
const std::string path_name =
|
||||
app_config::get_provider_path_name(get_config().get_provider_type());
|
||||
api_file file{};
|
||||
create_api_file(file_data, path_name, file);
|
||||
|
||||
list.emplace_back(file);
|
||||
}
|
||||
} else {
|
||||
event_system::instance().raise<file_get_api_list_failed>(error.dump(2));
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error sia_provider::get_file_size(const std::string &api_path, std::uint64_t &file_size) const {
|
||||
file_size = 0u;
|
||||
|
||||
json data, error;
|
||||
const auto ret = get_comm().get("/renter/file" + api_path, data, error);
|
||||
if (ret == api_error::success) {
|
||||
file_size = data["file"]["filesize"].get<std::uint64_t>();
|
||||
} else {
|
||||
event_system::instance().raise<file_get_size_failed>(api_path, error.dump(2));
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error sia_provider::get_filesystem_item(const std::string &api_path, const bool &directory,
|
||||
filesystem_item &fsi) const {
|
||||
auto ret = api_error::error;
|
||||
if (directory) {
|
||||
json result, error;
|
||||
ret = get_directory(api_path, result, error);
|
||||
if (ret != api_error::success) {
|
||||
if (check_not_found(error)) {
|
||||
ret = api_error::item_not_found;
|
||||
} else {
|
||||
event_system::instance().raise<filesystem_item_get_failed>(api_path, error.dump(2));
|
||||
}
|
||||
}
|
||||
|
||||
update_filesystem_item(true, ret, api_path, fsi);
|
||||
} else {
|
||||
api_file file{};
|
||||
ret = get_filesystem_item_and_file(api_path, file, fsi);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error sia_provider::get_filesystem_item_and_file(const std::string &api_path, api_file &file,
|
||||
filesystem_item &fsi) const {
|
||||
auto ret = get_item_meta(api_path, META_SOURCE, fsi.source_path);
|
||||
if (ret == api_error::success) {
|
||||
json data, error;
|
||||
ret = get_comm().get("/renter/file" + api_path, data, error);
|
||||
if (ret == api_error::success) {
|
||||
create_api_file(data["file"],
|
||||
app_config::get_provider_path_name(get_config().get_provider_type()), file);
|
||||
fsi.size = file.file_size;
|
||||
} else if (check_not_found(error)) {
|
||||
ret = api_error::item_not_found;
|
||||
} else {
|
||||
event_system::instance().raise<filesystem_item_get_failed>(api_path, error.dump(2));
|
||||
}
|
||||
}
|
||||
|
||||
update_filesystem_item(false, ret, api_path, fsi);
|
||||
return ret;
|
||||
}
|
||||
|
||||
std::uint64_t sia_provider::get_total_item_count() const {
|
||||
std::uint64_t ret = 0u;
|
||||
|
||||
json result, error;
|
||||
const auto res = get_directory("/", result, error);
|
||||
if (res == api_error::success) {
|
||||
ret = result["directories"][0u]["aggregatenumfiles"].get<std::uint64_t>();
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool sia_provider::is_directory(const std::string &api_path) const {
|
||||
auto ret = false;
|
||||
|
||||
json result, error;
|
||||
const auto res = (api_path == "/") ? api_error::success : get_directory(api_path, result, error);
|
||||
if (res == api_error::success) {
|
||||
ret = true;
|
||||
} else if (not check_not_found(error)) {
|
||||
event_system::instance().raise<repertory_exception>(__FUNCTION__, error.dump(2));
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool sia_provider::is_file(const std::string &api_path) const {
|
||||
return (api_path != "/") && (check_file_exists(api_path) == api_error::success);
|
||||
}
|
||||
|
||||
bool sia_provider::is_online() const {
|
||||
// TODO Expand here for true online detection (i.e. wallet unlocked)
|
||||
// TODO Return error string for display
|
||||
json data, error;
|
||||
return (get_comm().get("/wallet", data, error) == api_error::success);
|
||||
}
|
||||
|
||||
api_error sia_provider::notify_file_added(const std::string &api_path,
|
||||
const std::string &api_parent,
|
||||
const std::uint64_t &size) {
|
||||
recur_mutex_lock l(notify_added_mutex_);
|
||||
|
||||
json data;
|
||||
json error;
|
||||
auto ret = get_comm().get("/renter/file" + api_path, data, error);
|
||||
if (ret == api_error::success) {
|
||||
api_file file{};
|
||||
create_api_file(data["file"],
|
||||
app_config::get_provider_path_name(get_config().get_provider_type()), file);
|
||||
|
||||
get_api_item_added()(api_path, api_parent, file.source_path, false, file.created_date,
|
||||
file.accessed_date, file.modified_date, file.changed_date);
|
||||
|
||||
if (size) {
|
||||
global_data::instance().increment_used_drive_space(size);
|
||||
}
|
||||
} else {
|
||||
event_system::instance().raise<repertory_exception>(__FUNCTION__, error.dump(2));
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool sia_provider::processed_orphaned_file(const std::string &source_path,
|
||||
const std::string &api_path) const {
|
||||
auto ret = false;
|
||||
if (not oft_->contains_restore(api_path)) {
|
||||
const auto orphaned_directory =
|
||||
utils::path::combine(get_config().get_data_directory(), {"orphaned"});
|
||||
|
||||
event_system::instance().raise<orphaned_file_detected>(source_path);
|
||||
const auto orphaned_file = utils::path::combine(
|
||||
orphaned_directory,
|
||||
{utils::path::strip_to_file_name(api_path.empty() ? source_path : api_path)});
|
||||
|
||||
if (utils::file::reset_modified_time(source_path) &&
|
||||
utils::file::move_file(source_path, orphaned_file)) {
|
||||
event_system::instance().raise<orphaned_file_processed>(source_path, orphaned_file);
|
||||
ret = true;
|
||||
} else {
|
||||
event_system::instance().raise<orphaned_file_processing_failed>(
|
||||
source_path, orphaned_file, std::to_string(utils::get_last_error_code()));
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error sia_provider::read_file_bytes(const std::string &api_path, const std::size_t &size,
|
||||
const std::uint64_t &offset, std::vector<char> &buffer,
|
||||
const bool &stop_requested) {
|
||||
auto ret = api_error::download_failed;
|
||||
const auto retry_count = get_config().get_retry_read_count();
|
||||
for (std::uint16_t i = 0u; not stop_requested && (ret != api_error::success) && (i < retry_count);
|
||||
i++) {
|
||||
json error;
|
||||
ret = get_comm().get_raw("/renter/download" + api_path,
|
||||
{{"httpresp", "true"},
|
||||
{"async", "false"},
|
||||
{"offset", utils::string::from_int64(offset)},
|
||||
{"length", utils::string::from_int64(size)}},
|
||||
buffer, error, stop_requested);
|
||||
if (ret != api_error::success) {
|
||||
event_system::instance().raise<file_read_bytes_failed>(api_path, error.dump(2), i + 1u);
|
||||
if (not stop_requested && ((i + 1) < retry_count)) {
|
||||
std::this_thread::sleep_for(1s);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
void sia_provider::remove_deleted_files() {
|
||||
std::vector<std::string> removed_files{};
|
||||
|
||||
api_file_list list{};
|
||||
if (get_file_list(list) == api_error::success) {
|
||||
if (not list.empty()) {
|
||||
auto iterator = meta_db_.create_iterator(false);
|
||||
for (iterator->SeekToFirst(); not stop_requested_ && iterator->Valid(); iterator->Next()) {
|
||||
const auto api_path = iterator->key().ToString();
|
||||
if (api_path.empty()) {
|
||||
meta_db_.remove_item_meta(api_path);
|
||||
} else {
|
||||
auto it = std::find_if(list.begin(), list.end(), [&api_path](const auto &file) -> bool {
|
||||
return file.api_path == api_path;
|
||||
});
|
||||
if (it == list.end()) {
|
||||
removed_files.emplace_back(api_path);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
while (not stop_requested_ && not removed_files.empty()) {
|
||||
const auto api_path = removed_files.back();
|
||||
removed_files.pop_back();
|
||||
|
||||
std::string source_path;
|
||||
if (not is_directory(api_path) && (check_file_exists(api_path) == api_error::item_not_found) &&
|
||||
(meta_db_.get_item_meta(api_path, META_SOURCE, source_path) == api_error::success)) {
|
||||
if (not source_path.empty()) {
|
||||
oft_->perform_locked_operation(
|
||||
[this, &api_path, &source_path](i_open_file_table &, i_provider &) -> bool {
|
||||
if (oft_->has_no_open_file_handles()) {
|
||||
std::uint64_t used_space = 0u;
|
||||
if (calculate_used_drive_space(used_space) == api_error::success) {
|
||||
meta_db_.remove_item_meta(api_path);
|
||||
event_system::instance().raise<file_removed_externally>(api_path, source_path);
|
||||
|
||||
std::uint64_t fileSize = 0u;
|
||||
if (utils::file::get_file_size(source_path, fileSize) &&
|
||||
processed_orphaned_file(source_path, api_path)) {
|
||||
global_data::instance().update_used_space(fileSize, 0u, true);
|
||||
}
|
||||
|
||||
global_data::instance().initialize_used_drive_space(used_space);
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
api_error sia_provider::remove_directory(const std::string &api_path) {
|
||||
auto ret = api_error::directory_not_empty;
|
||||
if (get_directory_item_count(api_path) == 0u) {
|
||||
json result, error;
|
||||
ret = get_comm().post("/renter/dir" + api_path, {{"action", "delete"}}, result, error);
|
||||
if (ret == api_error::success) {
|
||||
meta_db_.remove_item_meta(api_path);
|
||||
event_system::instance().raise<directory_removed>(api_path);
|
||||
ret = api_error::success;
|
||||
} else if (check_not_found(error)) {
|
||||
meta_db_.remove_item_meta(api_path);
|
||||
ret = api_error::directory_not_found;
|
||||
} else {
|
||||
event_system::instance().raise<directory_remove_failed>(api_path, error.dump(2));
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
void sia_provider::remove_expired_orphaned_files() {
|
||||
const auto orphaned_directory =
|
||||
utils::path::combine(get_config().get_data_directory(), {"orphaned"});
|
||||
const auto files = utils::file::get_directory_files(orphaned_directory, true);
|
||||
for (const auto &file : files) {
|
||||
if (utils::file::is_modified_date_older_than(
|
||||
file, std::chrono::hours(get_config().get_orphaned_file_retention_days() * 24))) {
|
||||
if (utils::file::delete_file(file)) {
|
||||
event_system::instance().raise<orphaned_file_deleted>(file);
|
||||
}
|
||||
}
|
||||
if (stop_requested_) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
api_error sia_provider::remove_file(const std::string &api_path) {
|
||||
json result, error;
|
||||
auto ret = get_comm().post("/renter/delete" + api_path, {{"root", "false"}}, result, error);
|
||||
auto not_found = false;
|
||||
if ((ret == api_error::success) || (not_found = check_not_found(error))) {
|
||||
if (not not_found) {
|
||||
event_system::instance().raise<file_removed>(api_path);
|
||||
}
|
||||
ret = not_found ? api_error::item_not_found : api_error::success;
|
||||
|
||||
meta_db_.remove_item_meta(api_path);
|
||||
} else {
|
||||
event_system::instance().raise<file_remove_failed>(api_path, error.dump(2));
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
void sia_provider::remove_unknown_source_files() {
|
||||
auto files = utils::file::get_directory_files(get_config().get_cache_directory(), true);
|
||||
while (not stop_requested_ && not files.empty()) {
|
||||
const auto file = files.front();
|
||||
files.pop_front();
|
||||
|
||||
std::string api_path;
|
||||
if (not meta_db_.get_source_path_exists(file)) {
|
||||
processed_orphaned_file(file);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
api_error sia_provider::rename_file(const std::string &from_api_path,
|
||||
const std::string &to_api_path) {
|
||||
std::string current_source;
|
||||
auto ret = get_item_meta(from_api_path, META_SOURCE, current_source);
|
||||
if (ret == api_error::success) {
|
||||
json result, error;
|
||||
const auto propertyName =
|
||||
"new" + app_config::get_provider_path_name(get_config().get_provider_type());
|
||||
const auto dest_api_path = to_api_path.substr(1);
|
||||
ret = get_comm().post("/renter/rename" + from_api_path, {{propertyName, dest_api_path}}, result,
|
||||
error);
|
||||
if (ret == api_error::success) {
|
||||
meta_db_.rename_item_meta(current_source, from_api_path, to_api_path);
|
||||
} else if (check_not_found(error)) {
|
||||
ret = api_error::item_not_found;
|
||||
} else {
|
||||
event_system::instance().raise<file_rename_failed>(from_api_path, to_api_path, error.dump(2));
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
void sia_provider::set_api_file_dates(const json &file_data, api_file &file) const {
|
||||
file.accessed_date = utils::convert_api_date(file_data["accesstime"].get<std::string>());
|
||||
file.changed_date = utils::convert_api_date(file_data["changetime"].get<std::string>());
|
||||
file.created_date = utils::convert_api_date(file_data["createtime"].get<std::string>());
|
||||
file.modified_date = utils::convert_api_date(file_data["modtime"].get<std::string>());
|
||||
}
|
||||
|
||||
bool sia_provider::start(api_item_added_callback api_item_added, i_open_file_table *oft) {
|
||||
const auto unmount_requested = base_provider::start(api_item_added, oft);
|
||||
if (not unmount_requested && is_online()) {
|
||||
{
|
||||
json data, error;
|
||||
const auto res = comm_.get("/daemon/version", data, error);
|
||||
if (res == api_error::success) {
|
||||
if (utils::compare_version_strings(
|
||||
data["version"].get<std::string>(),
|
||||
app_config::get_provider_minimum_version(get_config().get_provider_type())) < 0) {
|
||||
throw startup_exception("incompatible daemon version: " +
|
||||
data["version"].get<std::string>());
|
||||
}
|
||||
} else {
|
||||
throw startup_exception("failed to get version from daemon");
|
||||
}
|
||||
}
|
||||
|
||||
mutex_lock l(start_stop_mutex_);
|
||||
if (not drive_space_thread_) {
|
||||
stop_requested_ = false;
|
||||
|
||||
calculate_total_drive_space();
|
||||
|
||||
std::uint64_t used_space = 0u;
|
||||
if (calculate_used_drive_space(used_space) != api_error::success) {
|
||||
throw startup_exception("failed to determine used space");
|
||||
}
|
||||
global_data::instance().initialize_used_drive_space(used_space);
|
||||
|
||||
drive_space_thread_ = std::make_unique<std::thread>([this] {
|
||||
cleanup();
|
||||
if (not stop_requested_) {
|
||||
polling::instance().set_callback({"check_deleted", true, [this]() {
|
||||
remove_deleted_files();
|
||||
remove_expired_orphaned_files();
|
||||
}});
|
||||
drive_space_thread();
|
||||
polling::instance().remove_callback("check_deleted");
|
||||
}
|
||||
});
|
||||
}
|
||||
} else {
|
||||
throw startup_exception("failed to connect to api");
|
||||
}
|
||||
return unmount_requested;
|
||||
}
|
||||
|
||||
void sia_provider::stop() {
|
||||
unique_mutex_lock l(start_stop_mutex_);
|
||||
if (drive_space_thread_) {
|
||||
event_system::instance().raise<service_shutdown>("sia_provider");
|
||||
stop_requested_ = true;
|
||||
|
||||
start_stop_notify_.notify_all();
|
||||
l.unlock();
|
||||
|
||||
drive_space_thread_->join();
|
||||
drive_space_thread_.reset();
|
||||
}
|
||||
}
|
||||
|
||||
api_error sia_provider::upload_file(const std::string &api_path, const std::string &source_path,
|
||||
const std::string &) {
|
||||
event_system::instance().raise<file_upload_begin>(api_path, source_path);
|
||||
|
||||
auto ret = set_source_path(api_path, source_path);
|
||||
if (ret == api_error::success) {
|
||||
std::uint64_t fileSize;
|
||||
if (utils::file::get_file_size(source_path, fileSize)) {
|
||||
json result, error;
|
||||
ret = (get_comm().post("/renter/upload" + api_path,
|
||||
{{"source", &source_path[0]}, {"force", "true"}}, result,
|
||||
error) == api_error::success)
|
||||
? api_error::success
|
||||
: api_error::upload_failed;
|
||||
if (ret != api_error::success) {
|
||||
if (check_not_found(error)) {
|
||||
ret = (get_comm().post("/renter/upload" + api_path, {{"source", &source_path[0]}}, result,
|
||||
error) == api_error::success)
|
||||
? api_error::success
|
||||
: api_error::upload_failed;
|
||||
}
|
||||
if (ret != api_error::success) {
|
||||
event_system::instance().raise<file_upload_failed>(api_path, source_path, error.dump(2));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
ret = api_error::os_error;
|
||||
event_system::instance().raise<file_upload_failed>(api_path, source_path,
|
||||
"Failed to get source file size");
|
||||
}
|
||||
} else {
|
||||
event_system::instance().raise<file_upload_failed>(api_path, source_path,
|
||||
"Failed to set source path");
|
||||
}
|
||||
|
||||
event_system::instance().raise<file_upload_end>(api_path, source_path, ret);
|
||||
return ret;
|
||||
}
|
||||
} // namespace repertory
|
||||
649
src/providers/skynet/skynet_provider.cpp
Normal file
649
src/providers/skynet/skynet_provider.cpp
Normal file
@@ -0,0 +1,649 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#if defined(REPERTORY_ENABLE_SKYNET)
|
||||
|
||||
#include "providers/skynet/skynet_provider.hpp"
|
||||
#include "comm/i_comm.hpp"
|
||||
#include "app_config.hpp"
|
||||
#include "drives/i_open_file_table.hpp"
|
||||
#include "events/events.hpp"
|
||||
#include "types/repertory.hpp"
|
||||
#include "types/skynet.hpp"
|
||||
#include "types/startup_exception.hpp"
|
||||
#include "utils/encryption.hpp"
|
||||
#include "utils/encrypting_reader.hpp"
|
||||
#include "utils/file_utils.hpp"
|
||||
#include "utils/global_data.hpp"
|
||||
#include "utils/path_utils.hpp"
|
||||
|
||||
namespace repertory {
|
||||
skynet_provider::skynet_provider(app_config &config, i_comm &comm)
|
||||
: base_provider(config),
|
||||
comm_(comm),
|
||||
directory_db_(config),
|
||||
upload_manager_(
|
||||
config, [this](const std::string &api_path) -> bool { return this->is_file(api_path); },
|
||||
[this](const upload_manager::upload &upload, json &data, json &error) -> api_error {
|
||||
return this->upload_handler(upload, data, error);
|
||||
},
|
||||
[this](const std::string &api_path, const std::string &source_path, const json &data) {
|
||||
return this->upload_completed(api_path, source_path, data);
|
||||
}) {
|
||||
next_download_index_ = 0u;
|
||||
next_upload_index_ = 0u;
|
||||
update_portal_list();
|
||||
E_SUBSCRIBE_EXACT(skynet_portal_list_changed,
|
||||
[this](const skynet_portal_list_changed &) { this->update_portal_list(); });
|
||||
|
||||
// Remove legacy encrypted files
|
||||
api_file_list list;
|
||||
get_file_list(list);
|
||||
for (const auto &file : list) {
|
||||
std::string token;
|
||||
get_item_meta(file.api_path, "token", token);
|
||||
if (not token.empty()) {
|
||||
remove_file(file.api_path);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
api_error skynet_provider::create_directory(const std::string &api_path, const api_meta_map &meta) {
|
||||
auto ret = api_error::success;
|
||||
if (utils::path::is_trash_directory(api_path)) {
|
||||
ret = api_error::access_denied;
|
||||
} else {
|
||||
#ifdef _WIN32
|
||||
ret = is_directory(api_path) ? api_error::directory_exists
|
||||
: is_file(api_path) ? api_error::file_exists
|
||||
: api_error::success;
|
||||
if (ret != api_error::success) {
|
||||
return ret;
|
||||
}
|
||||
#endif
|
||||
if ((ret = directory_db_.create_directory(api_path)) != api_error::success) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
set_item_meta(api_path, meta);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error skynet_provider::create_file(const std::string &api_path, api_meta_map &meta) {
|
||||
if (meta[META_SIZE].empty()) {
|
||||
meta[META_SIZE] = "0";
|
||||
}
|
||||
|
||||
// When META_ID is present, an external import is occurring.
|
||||
// Need to skip the encryption token in this scenario.
|
||||
if (meta[META_ID].empty() && meta[META_ENCRYPTION_TOKEN].empty()) {
|
||||
meta[META_ENCRYPTION_TOKEN] = get_config().get_skynet_config().encryption_token;
|
||||
}
|
||||
|
||||
auto ret = base_provider::create_file(api_path, meta);
|
||||
if (ret == api_error::success) {
|
||||
ret = directory_db_.create_file(api_path);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
json skynet_provider::export_all() const {
|
||||
json ret = {{"success", std::vector<json>()}, {"failed", std::vector<std::string>()}};
|
||||
|
||||
api_file_list list;
|
||||
get_file_list(list);
|
||||
for (const auto &file : list) {
|
||||
process_export(ret, file.api_path);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
json skynet_provider::export_list(const std::vector<std::string> &api_path_list) const {
|
||||
json ret = {{"success", std::vector<json>()}, {"failed", std::vector<std::string>()}};
|
||||
|
||||
for (const auto &api_path : api_path_list) {
|
||||
process_export(ret, api_path);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
std::uint64_t skynet_provider::get_directory_item_count(const std::string &api_path) const {
|
||||
return is_directory(api_path) ? directory_db_.get_directory_item_count(api_path) : 0u;
|
||||
}
|
||||
|
||||
api_error skynet_provider::get_directory_items(const std::string &api_path,
|
||||
directory_item_list &list) const {
|
||||
if (is_file(api_path)) {
|
||||
return api_error::item_is_file;
|
||||
}
|
||||
|
||||
if (not is_directory(api_path)) {
|
||||
const_cast<skynet_provider *>(this)->remove_item_meta(api_path);
|
||||
return api_error::directory_not_found;
|
||||
}
|
||||
|
||||
directory_db_.populate_sub_directories(
|
||||
api_path,
|
||||
[this](directory_item &di, const bool &) {
|
||||
this->get_item_meta(di.api_path, di.meta);
|
||||
this->oft_->update_directory_item(di);
|
||||
},
|
||||
list);
|
||||
|
||||
directory_db_.populate_directory_files(
|
||||
api_path,
|
||||
[this](directory_item &di, const bool &) {
|
||||
di.api_parent = utils::path::get_parent_api_path(di.api_path);
|
||||
this->get_item_meta(di.api_path, di.meta);
|
||||
this->oft_->update_directory_item(di);
|
||||
},
|
||||
list);
|
||||
|
||||
std::sort(list.begin(), list.end(), [](const auto &a, const auto &b) -> bool {
|
||||
return (a.directory && not b.directory) ? true
|
||||
: (b.directory && not a.directory) ? false
|
||||
: (a.api_path.compare(b.api_path) < 0);
|
||||
});
|
||||
|
||||
list.insert(list.begin(), directory_item{
|
||||
"..",
|
||||
"",
|
||||
true,
|
||||
});
|
||||
list.insert(list.begin(), directory_item{
|
||||
".",
|
||||
"",
|
||||
true,
|
||||
});
|
||||
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
api_error skynet_provider::get_file(const std::string &api_path, api_file &file) const {
|
||||
const auto ret =
|
||||
directory_db_.get_file(api_path, file, [this](api_file &file) { populate_api_file(file); });
|
||||
|
||||
if (ret != api_error::success) {
|
||||
if (not is_directory(api_path)) {
|
||||
const_cast<skynet_provider *>(this)->remove_item_meta(api_path);
|
||||
}
|
||||
event_system::instance().raise<file_get_failed>(api_path,
|
||||
std::to_string(static_cast<int>(ret)));
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error skynet_provider::get_file_list(api_file_list &list) const {
|
||||
const auto ret =
|
||||
directory_db_.get_file_list(list, [this](api_file &file) { populate_api_file(file); });
|
||||
if (ret != api_error::success) {
|
||||
event_system::instance().raise<file_get_api_list_failed>(std::to_string(static_cast<int>(ret)));
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error skynet_provider::get_file_size(const std::string &api_path,
|
||||
std::uint64_t &file_size) const {
|
||||
api_file file{};
|
||||
const auto ret = get_file(api_path, file);
|
||||
if (ret == api_error::success) {
|
||||
file_size = file.file_size;
|
||||
} else {
|
||||
event_system::instance().raise<file_get_size_failed>(api_path,
|
||||
std::to_string(static_cast<int>(ret)));
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
host_config skynet_provider::get_host_config(const bool &upload) {
|
||||
const auto format_host_config = [&](host_config hc) {
|
||||
hc.path = upload ? "/skynet/skyfile" : "/";
|
||||
return hc;
|
||||
};
|
||||
|
||||
unique_mutex_lock portal_lock(portal_mutex_);
|
||||
const auto portal_list = upload ? upload_list_ : download_list_;
|
||||
portal_lock.unlock();
|
||||
|
||||
auto &next_upload_index = upload ? next_upload_index_ : next_download_index_;
|
||||
auto idx = next_upload_index++;
|
||||
if (idx >= portal_list->size()) {
|
||||
idx = next_upload_index = 0u;
|
||||
}
|
||||
|
||||
return format_host_config((*portal_list)[idx]);
|
||||
}
|
||||
|
||||
std::size_t skynet_provider::get_retry_count() const {
|
||||
mutex_lock portal_lock(portal_mutex_);
|
||||
return std::max(download_list_->size(),
|
||||
static_cast<std::size_t>(get_config().get_retry_read_count()));
|
||||
}
|
||||
|
||||
api_error skynet_provider::get_skynet_metadata(const std::string &skylink, json &json_meta) {
|
||||
auto ret = api_error::error;
|
||||
|
||||
http_headers headers;
|
||||
const auto retry_count = get_retry_count();
|
||||
|
||||
for (std::size_t i = 0u; (ret != api_error::success) && (i < retry_count); i++) {
|
||||
json data, error;
|
||||
const auto hc = get_host_config(false);
|
||||
if (comm_.get(hc, "/skynet/metadata/" + skylink, data, error) == api_error::success) {
|
||||
headers["skynet-file-metadata"] = data.dump();
|
||||
ret = api_error::success;
|
||||
} else {
|
||||
std::vector<char> buffer;
|
||||
if (comm_.get_range_and_headers(hc, utils::path::create_api_path(skylink), 0u,
|
||||
{{"format", "concat"}}, "", buffer, {{0, 0}}, error, headers,
|
||||
stop_requested_) == api_error::success) {
|
||||
ret = api_error::success;
|
||||
} else if (not error.empty()) {
|
||||
event_system::instance().raise<repertory_exception>(__FUNCTION__, error.dump(2));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (ret == api_error::success) {
|
||||
json_meta = json::parse(headers["skynet-file-metadata"]);
|
||||
if (json_meta["subfiles"].empty()) {
|
||||
auto sub_file = json_meta;
|
||||
sub_file["len"] =
|
||||
utils::string::to_uint64(utils::string::split(headers["content-range"], '/')[1]);
|
||||
json sub_files = {{json_meta["filename"], sub_file}};
|
||||
json_meta["subfiles"] = sub_files;
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error skynet_provider::import_skylink(const skylink_import &si) {
|
||||
json json_meta;
|
||||
auto ret = get_skynet_metadata(si.skylink, json_meta);
|
||||
if (ret == api_error::success) {
|
||||
const auto encrypted = not si.token.empty();
|
||||
for (auto sub_file : json_meta["subfiles"]) {
|
||||
const auto meta_file_name = sub_file["filename"].get<std::string>();
|
||||
auto file_name = meta_file_name;
|
||||
if (encrypted) {
|
||||
if ((ret = utils::encryption::decrypt_file_name(si.token, file_name)) !=
|
||||
api_error::success) {
|
||||
event_system::instance().raise<skynet_import_decryption_failed>(
|
||||
si.skylink, sub_file["filename"], ret);
|
||||
}
|
||||
}
|
||||
|
||||
if (ret == api_error::success) {
|
||||
const auto api_path =
|
||||
utils::path::create_api_path(utils::path::combine(si.directory, {file_name}));
|
||||
const auto api_parent = utils::path::get_parent_api_path(api_path);
|
||||
const auto parts = utils::string::split(api_parent, '/', false);
|
||||
|
||||
std::string sub_directory = "/";
|
||||
for (std::size_t i = 0u; (ret == api_error::success) && (i < parts.size()); i++) {
|
||||
sub_directory =
|
||||
utils::path::create_api_path(utils::path::combine(sub_directory, {parts[i]}));
|
||||
if (not is_directory(sub_directory)) {
|
||||
if ((ret = directory_db_.create_directory(sub_directory)) == api_error::success) {
|
||||
base_provider::notify_directory_added(
|
||||
sub_directory, utils::path::get_parent_api_path(sub_directory));
|
||||
} else {
|
||||
event_system::instance().raise<skynet_import_directory_failed>(si.skylink,
|
||||
sub_directory, ret);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (ret == api_error::success) {
|
||||
auto file_size = sub_file["len"].get<std::uint64_t>();
|
||||
if (encrypted) {
|
||||
file_size = utils::encryption::encrypting_reader::calculate_decrypted_size(file_size);
|
||||
}
|
||||
|
||||
const auto skylink =
|
||||
si.skylink + ((json_meta["filename"].get<std::string>() == meta_file_name)
|
||||
? ""
|
||||
: "/" + meta_file_name);
|
||||
api_meta_map meta{};
|
||||
meta[META_ID] = json({{"skylink", skylink}}).dump();
|
||||
meta[META_ENCRYPTION_TOKEN] = si.token;
|
||||
if ((ret = create_file(api_path, meta)) == api_error::success) {
|
||||
const auto now = utils::get_file_time_now();
|
||||
api_item_added_(api_path, api_parent, "", false, now, now, now, now);
|
||||
|
||||
if (file_size > 0u) {
|
||||
set_item_meta(api_path, META_SIZE, std::to_string(file_size));
|
||||
global_data::instance().increment_used_drive_space(file_size);
|
||||
}
|
||||
} else {
|
||||
event_system::instance().raise<skynet_import_file_failed>(si.skylink, api_path, ret);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool skynet_provider::is_directory(const std::string &api_path) const {
|
||||
return (api_path == "/") || directory_db_.is_directory(api_path);
|
||||
}
|
||||
|
||||
bool skynet_provider::is_file(const std::string &api_path) const {
|
||||
return (api_path != "/") && directory_db_.is_file(api_path);
|
||||
}
|
||||
|
||||
bool skynet_provider::is_file_writeable(const std::string &api_path) const {
|
||||
auto ret = true;
|
||||
std::string id;
|
||||
get_item_meta(api_path, META_ID, id);
|
||||
if (not id.empty()) {
|
||||
try {
|
||||
const auto skynet_data = json::parse(id);
|
||||
const auto skylink = skynet_data["skylink"].get<std::string>();
|
||||
ret = not utils::string::contains(skylink, "/");
|
||||
} catch (const std::exception &e) {
|
||||
event_system::instance().raise<repertory_exception>(
|
||||
__FUNCTION__, e.what() ? e.what() : "exception occurred");
|
||||
ret = false;
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool skynet_provider::is_processing(const std::string &api_path) const {
|
||||
return upload_manager_.is_processing(api_path);
|
||||
}
|
||||
|
||||
void skynet_provider::notify_directory_added(const std::string &api_path,
|
||||
const std::string &api_parent) {
|
||||
if (api_path == "/") {
|
||||
if (directory_db_.create_directory("/") == api_error::success) {
|
||||
base_provider::notify_directory_added(api_path, api_parent);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
api_error skynet_provider::notify_file_added(const std::string &, const std::string &,
|
||||
const std::uint64_t &) {
|
||||
return api_error::not_implemented;
|
||||
}
|
||||
|
||||
void skynet_provider::populate_api_file(api_file &file) const {
|
||||
api_meta_map meta{};
|
||||
this->get_item_meta(file.api_path, meta);
|
||||
|
||||
file.api_parent = utils::path::get_parent_api_path(file.api_path);
|
||||
file.accessed_date = utils::string::to_uint64(meta[META_ACCESSED]);
|
||||
file.changed_date = utils::string::to_uint64(meta[META_MODIFIED]);
|
||||
file.created_date = utils::string::to_uint64(meta[META_CREATION]);
|
||||
file.encryption_token = meta[META_ENCRYPTION_TOKEN].empty() ? "" : meta[META_ENCRYPTION_TOKEN];
|
||||
file.file_size = utils::string::to_uint64(meta[META_SIZE]);
|
||||
file.modified_date = utils::string::to_uint64(meta[META_MODIFIED]);
|
||||
file.recoverable = not is_processing(file.api_path);
|
||||
file.redundancy = 3.0;
|
||||
file.source_path = meta[META_SOURCE];
|
||||
}
|
||||
|
||||
void skynet_provider::process_export(json &result, const std::string &api_path) const {
|
||||
try {
|
||||
std::string id;
|
||||
std::string token;
|
||||
if (is_file(api_path) && (get_item_meta(api_path, META_ID, id) == api_error::success) &&
|
||||
(get_item_meta(api_path, META_ENCRYPTION_TOKEN, token) == api_error::success)) {
|
||||
auto directory = utils::path::get_parent_api_path(api_path);
|
||||
|
||||
const auto skylink = json::parse(id)["skylink"].get<std::string>();
|
||||
if (utils::string::contains(skylink, "/")) {
|
||||
const auto pos = skylink.find('/');
|
||||
const auto path =
|
||||
utils::path::create_api_path(utils::path::remove_file_name(skylink.substr(pos)));
|
||||
if (path != "/") {
|
||||
directory =
|
||||
utils::path::create_api_path(directory.substr(0, directory.length() - path.length()));
|
||||
}
|
||||
}
|
||||
|
||||
result["success"].emplace_back(
|
||||
json({{"skylink", skylink},
|
||||
{"token", token},
|
||||
{"directory", directory},
|
||||
{"filename", utils::path::strip_to_file_name(api_path)}}));
|
||||
} else {
|
||||
result["failed"].emplace_back(api_path);
|
||||
}
|
||||
} catch (const std::exception &e) {
|
||||
result["failed"].emplace_back(api_path);
|
||||
event_system::instance().raise<repertory_exception>(
|
||||
__FUNCTION__, e.what() ? e.what() : "export failed: " + api_path);
|
||||
}
|
||||
}
|
||||
|
||||
api_error skynet_provider::read_file_bytes(const std::string &api_path, const std::size_t &size,
|
||||
const std::uint64_t &offset, std::vector<char> &data,
|
||||
const bool &stop_requested) {
|
||||
if (size == 0u) {
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
std::string id;
|
||||
auto ret = get_item_meta(api_path, META_ID, id);
|
||||
if (ret == api_error::success) {
|
||||
ret = api_error::download_failed;
|
||||
|
||||
const auto skynet_data = json::parse(id);
|
||||
const auto path = utils::path::create_api_path(skynet_data["skylink"].get<std::string>());
|
||||
const auto ranges = http_ranges({{offset, offset + size - 1}});
|
||||
|
||||
std::string encryption_token;
|
||||
get_item_meta(api_path, META_ENCRYPTION_TOKEN, encryption_token);
|
||||
|
||||
std::uint64_t file_size{};
|
||||
{
|
||||
std::string temp;
|
||||
get_item_meta(api_path, META_SIZE, temp);
|
||||
file_size = utils::string::to_uint64(temp);
|
||||
}
|
||||
|
||||
const auto retry_count = get_retry_count();
|
||||
for (std::size_t i = 0u; not stop_requested && (ret != api_error::success) && (i < retry_count);
|
||||
i++) {
|
||||
json error;
|
||||
ret = (comm_.get_range(get_host_config(false), path, file_size, {{"format", "concat"}},
|
||||
encryption_token, data, ranges, error,
|
||||
stop_requested) == api_error::success)
|
||||
? api_error::success
|
||||
: api_error::download_failed;
|
||||
if (ret != api_error::success) {
|
||||
event_system::instance().raise<file_read_bytes_failed>(api_path, error.dump(2), i + 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error skynet_provider::remove_directory(const std::string &api_path) {
|
||||
const auto ret = directory_db_.remove_directory(api_path);
|
||||
if (ret == api_error::success) {
|
||||
remove_item_meta(api_path);
|
||||
event_system::instance().raise<directory_removed>(api_path);
|
||||
} else {
|
||||
event_system::instance().raise<directory_remove_failed>(api_path,
|
||||
std::to_string(static_cast<int>(ret)));
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error skynet_provider::remove_file(const std::string &api_path) {
|
||||
upload_manager_.remove_upload(api_path);
|
||||
const auto ret =
|
||||
directory_db_.remove_file(api_path) ? api_error::success : api_error::item_not_found;
|
||||
if (ret == api_error::success) {
|
||||
remove_item_meta(api_path);
|
||||
event_system::instance().raise<file_removed>(api_path);
|
||||
} else {
|
||||
event_system::instance().raise<file_remove_failed>(api_path,
|
||||
std::to_string(static_cast<int>(ret)));
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error skynet_provider::rename_file(const std::string &from_api_path,
|
||||
const std::string &to_api_path) {
|
||||
std::string id;
|
||||
auto ret = get_item_meta(from_api_path, META_ID, id);
|
||||
if (ret == api_error::success) {
|
||||
ret = api_error::access_denied;
|
||||
|
||||
const auto skynet_data = json::parse(id.empty() ? R"({"skylink":""})" : id);
|
||||
const auto skylink = skynet_data["skylink"].get<std::string>();
|
||||
if (utils::string::contains(skylink, "/")) {
|
||||
const auto pos = skylink.find('/');
|
||||
const auto len = skylink.size() - pos;
|
||||
if (to_api_path.size() >= len) {
|
||||
const auto comp1 = to_api_path.substr(to_api_path.size() - len);
|
||||
const auto comp2 = skylink.substr(pos);
|
||||
ret = (comp1 == comp2) ? api_error::success : ret;
|
||||
}
|
||||
} else {
|
||||
ret = (skylink.empty() || (utils::path::strip_to_file_name(from_api_path) ==
|
||||
utils::path::strip_to_file_name(to_api_path)))
|
||||
? api_error::success
|
||||
: ret;
|
||||
}
|
||||
|
||||
if (ret == api_error::success) {
|
||||
std::string current_source;
|
||||
if ((ret = get_item_meta(from_api_path, META_SOURCE, current_source)) == api_error::success) {
|
||||
if (not upload_manager_.execute_if_not_processing({from_api_path, to_api_path}, [&]() {
|
||||
if ((ret = directory_db_.rename_file(from_api_path, to_api_path)) ==
|
||||
api_error::success) {
|
||||
meta_db_.rename_item_meta(current_source, from_api_path, to_api_path);
|
||||
}
|
||||
})) {
|
||||
ret = api_error::file_in_use;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool skynet_provider::start(api_item_added_callback api_item_added, i_open_file_table *oft) {
|
||||
const auto ret = base_provider::start(api_item_added, oft);
|
||||
if (not ret) {
|
||||
api_file_list list;
|
||||
if (get_file_list(list) != api_error::success) {
|
||||
throw startup_exception("failed to determine used space");
|
||||
}
|
||||
|
||||
const auto total_size =
|
||||
std::accumulate(list.begin(), list.end(), std::uint64_t(0),
|
||||
[](std::uint64_t t, const api_file &file) { return t + file.file_size; });
|
||||
global_data::instance().initialize_used_drive_space(total_size);
|
||||
upload_manager_.start();
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
void skynet_provider::stop() {
|
||||
stop_requested_ = true;
|
||||
upload_manager_.stop();
|
||||
}
|
||||
|
||||
bool skynet_provider::update_portal_list() {
|
||||
auto portal_list = get_config().get_skynet_config().portal_list;
|
||||
auto download_portal_list = std::make_shared<std::vector<host_config>>();
|
||||
auto upload_portal_list = std::make_shared<std::vector<host_config>>();
|
||||
|
||||
std::copy_if(portal_list.begin(), portal_list.end(), std::back_inserter(*upload_portal_list),
|
||||
[](const auto &portal) -> bool {
|
||||
return not portal.auth_url.empty() && not portal.auth_user.empty();
|
||||
});
|
||||
for (const auto &portal : portal_list) {
|
||||
if (upload_portal_list->empty()) {
|
||||
upload_portal_list->emplace_back(portal);
|
||||
}
|
||||
download_portal_list->emplace_back(portal);
|
||||
}
|
||||
|
||||
unique_mutex_lock portal_lock(portal_mutex_);
|
||||
download_list_ = std::move(download_portal_list);
|
||||
upload_list_ = std::move(upload_portal_list);
|
||||
portal_lock.unlock();
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void skynet_provider::upload_completed(const std::string &api_path, const std::string &,
|
||||
const json &data) {
|
||||
set_item_meta(api_path, META_ID, data.dump());
|
||||
}
|
||||
|
||||
api_error skynet_provider::upload_file(const std::string &api_path, const std::string &source_path,
|
||||
const std::string &encryption_token) {
|
||||
std::uint64_t file_size = 0u;
|
||||
utils::file::get_file_size(source_path, file_size);
|
||||
auto ret = set_source_path(api_path, source_path);
|
||||
if (ret == api_error::success) {
|
||||
if (((ret = set_item_meta(api_path, META_SIZE, std::to_string(file_size))) ==
|
||||
api_error::success) &&
|
||||
((ret = set_item_meta(api_path, META_ENCRYPTION_TOKEN, encryption_token)) ==
|
||||
api_error::success) &&
|
||||
(file_size != 0u)) {
|
||||
ret = upload_manager_.queue_upload(api_path, source_path, encryption_token);
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error skynet_provider::upload_handler(const upload_manager::upload &upload, json &data,
|
||||
json &error) {
|
||||
const auto file_name = utils::path::strip_to_file_name(upload.api_path);
|
||||
|
||||
auto ret = api_error::upload_failed;
|
||||
if (not upload.cancel) {
|
||||
event_system::instance().raise<file_upload_begin>(upload.api_path, upload.source_path);
|
||||
ret = (comm_.post_multipart_file(get_host_config(true), "", file_name, upload.source_path,
|
||||
upload.encryption_token, data, error,
|
||||
upload.cancel) == api_error::success)
|
||||
? api_error::success
|
||||
: api_error::upload_failed;
|
||||
}
|
||||
|
||||
event_system::instance().raise<file_upload_end>(upload.api_path, upload.source_path, ret);
|
||||
return ret;
|
||||
}
|
||||
} // namespace repertory
|
||||
|
||||
#endif // defined(REPERTORY_ENABLE_SKYNET)
|
||||
240
src/rpc/client/client.cpp
Normal file
240
src/rpc/client/client.cpp
Normal file
@@ -0,0 +1,240 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "rpc/client/client.hpp"
|
||||
#include "comm/curl/curl_resolver.hpp"
|
||||
#include "types/repertory.hpp"
|
||||
#include "utils/Base64.hpp"
|
||||
#include "utils/utils.hpp"
|
||||
|
||||
namespace repertory {
|
||||
client::client(rpc_host_info host_info) : host_info_(std::move(host_info)) { request_id_ = 0u; }
|
||||
|
||||
rpc_response client::export_list(const std::vector<std::string> &paths) {
|
||||
auto ret = make_request(rpc_method::export_links, {paths});
|
||||
if (ret.response_type == rpc_response_type::success) {
|
||||
ret.data = ret.data["result"];
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
rpc_response client::export_all() {
|
||||
auto ret = make_request(rpc_method::export_links, {}, 60000u);
|
||||
if (ret.response_type == rpc_response_type::success) {
|
||||
ret.data = ret.data["result"];
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
rpc_response client::get_drive_information() {
|
||||
auto ret = make_request(rpc_method::get_drive_information, {});
|
||||
if (ret.response_type == rpc_response_type::success) {
|
||||
ret.data = ret.data["result"];
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
rpc_response client::get_config() {
|
||||
auto ret = make_request(rpc_method::get_config, {});
|
||||
if (ret.response_type == rpc_response_type::success) {
|
||||
ret.data = ret.data["result"];
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
rpc_response client::get_config_value_by_name(const std::string &name) {
|
||||
auto ret = make_request(rpc_method::get_config_value_by_name, {name});
|
||||
if (ret.response_type == rpc_response_type::success) {
|
||||
if (ret.data["result"]["value"].get<std::string>().empty()) {
|
||||
ret.response_type = rpc_response_type::config_value_not_found;
|
||||
} else {
|
||||
ret.data = ret.data["result"];
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
rpc_response client::get_directory_items(const std::string &api_path) {
|
||||
auto ret = make_request(rpc_method::get_directory_items, {api_path});
|
||||
if (ret.response_type == rpc_response_type::success) {
|
||||
ret.data = ret.data["result"];
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
rpc_response client::get_open_files() {
|
||||
auto ret = make_request(rpc_method::get_open_files, {});
|
||||
if (ret.response_type == rpc_response_type::success) {
|
||||
ret.data = ret.data["result"];
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
rpc_response client::get_pinned_files() {
|
||||
auto ret = make_request(rpc_method::get_pinned_files, {});
|
||||
if (ret.response_type == rpc_response_type::success) {
|
||||
ret.data = ret.data["result"];
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
rpc_response client::import_skylink(const skylink_import_list &list) {
|
||||
std::vector<json> json_list;
|
||||
for (const auto &skynet_import : list) {
|
||||
json_list.emplace_back(skynet_import.to_json());
|
||||
}
|
||||
|
||||
auto ret = make_request(rpc_method::import, {json(json_list)}, 60000u);
|
||||
if (ret.response_type == rpc_response_type::success) {
|
||||
ret.data = ret.data["result"];
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
rpc_response client::make_request(const std::string &command, const std::vector<json> &args,
|
||||
std::uint32_t timeout_ms) {
|
||||
auto error = rpc_response_type::success;
|
||||
|
||||
auto *curl_handle = utils::create_curl();
|
||||
|
||||
const auto port = static_cast<std::uint16_t>(host_info_.port);
|
||||
const auto url = "http://" + host_info_.host + ":" + std::to_string(port) + "/api";
|
||||
const auto request = json({{"jsonrpc", "2.0"},
|
||||
{"id", std::to_string(++request_id_)},
|
||||
{"method", command},
|
||||
{"params", args}})
|
||||
.dump();
|
||||
|
||||
struct curl_slist *hs = nullptr;
|
||||
hs = curl_slist_append(hs, "Content-Type: application/json;");
|
||||
if (not(host_info_.password.empty() && host_info_.user.empty())) {
|
||||
curl_easy_setopt(curl_handle, CURLOPT_USERNAME, &host_info_.user[0]);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_PASSWORD, &host_info_.password[0]);
|
||||
}
|
||||
#ifndef __APPLE__
|
||||
curl_resolver resolver(curl_handle,
|
||||
{"localhost:" + std::to_string(host_info_.port) + ":127.0.0.1"}, true);
|
||||
#endif
|
||||
if (timeout_ms > 0) {
|
||||
curl_easy_setopt(curl_handle, CURLOPT_TIMEOUT_MS, timeout_ms);
|
||||
}
|
||||
curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, hs);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_URL, url.c_str());
|
||||
curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, request.c_str());
|
||||
curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDSIZE, request.size());
|
||||
curl_easy_setopt(curl_handle, CURLOPT_CONNECTTIMEOUT, 5L);
|
||||
curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION,
|
||||
static_cast<size_t (*)(char *, size_t, size_t, void *)>(
|
||||
[](char *buffer, size_t size, size_t nitems, void *outstream) -> size_t {
|
||||
(*reinterpret_cast<std::string *>(outstream)) +=
|
||||
std::string(buffer, size * nitems);
|
||||
return size * nitems;
|
||||
}));
|
||||
|
||||
std::string response;
|
||||
curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, &response);
|
||||
|
||||
json json_data;
|
||||
const auto res = curl_easy_perform(curl_handle);
|
||||
if (res == CURLE_OK) {
|
||||
long httpErrorCode;
|
||||
curl_easy_getinfo(curl_handle, CURLINFO_RESPONSE_CODE, &httpErrorCode);
|
||||
if (httpErrorCode == 200) {
|
||||
json_data = json::parse(response.begin(), response.end());
|
||||
} else {
|
||||
json parsed;
|
||||
try {
|
||||
parsed = json::parse(response.begin(), response.end());
|
||||
} catch (...) {
|
||||
}
|
||||
error = rpc_response_type::http_error;
|
||||
json_data = {{"error", {{"code", std::to_string(httpErrorCode)}}}};
|
||||
if (parsed.empty()) {
|
||||
json_data["error"]["response"] = response;
|
||||
} else {
|
||||
json_data["error"]["response"] = parsed;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
error = rpc_response_type::curl_error;
|
||||
json_data = {{"error", {{"message", curl_easy_strerror(res)}}}};
|
||||
}
|
||||
curl_easy_cleanup(curl_handle);
|
||||
curl_slist_free_all(hs);
|
||||
|
||||
return rpc_response({error, json_data});
|
||||
}
|
||||
|
||||
rpc_response client::pin_file(const std::string &api_path) {
|
||||
auto ret = make_request(rpc_method::pin_file, {api_path});
|
||||
if (ret.response_type == rpc_response_type::success) {
|
||||
ret.data = ret.data["result"];
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
rpc_response client::pinned_status(const std::string &api_path) {
|
||||
auto ret = make_request(rpc_method::pinned_status, {api_path});
|
||||
if (ret.response_type == rpc_response_type::success) {
|
||||
ret.data = ret.data["result"];
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
rpc_response client::set_config_value_by_name(const std::string &name, const std::string &value) {
|
||||
auto ret = make_request(rpc_method::set_config_value_by_name, {name, value});
|
||||
if (ret.response_type == rpc_response_type::success) {
|
||||
if (ret.data["result"]["value"].get<std::string>().empty()) {
|
||||
ret.response_type = rpc_response_type::config_value_not_found;
|
||||
} else {
|
||||
ret.data = ret.data["result"];
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
rpc_response client::unmount() {
|
||||
auto ret = make_request(rpc_method::unmount, {});
|
||||
if (ret.response_type == rpc_response_type::success) {
|
||||
ret.data = ret.data["result"];
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
rpc_response client::unpin_file(const std::string &api_path) {
|
||||
auto ret = make_request(rpc_method::unpin_file, {api_path});
|
||||
if (ret.response_type == rpc_response_type::success) {
|
||||
ret.data = ret.data["result"];
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
} // namespace repertory
|
||||
165
src/rpc/server/full_server.cpp
Normal file
165
src/rpc/server/full_server.cpp
Normal file
@@ -0,0 +1,165 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "rpc/server/full_server.hpp"
|
||||
#include "app_config.hpp"
|
||||
#include "drives/directory_iterator.hpp"
|
||||
#include "drives/i_open_file_table.hpp"
|
||||
#include "providers/i_provider.hpp"
|
||||
#include "providers/skynet/skynet_provider.hpp"
|
||||
#include "types/repertory.hpp"
|
||||
#include "types/rpc.hpp"
|
||||
#include "types/skynet.hpp"
|
||||
#include "utils/global_data.hpp"
|
||||
#include "utils/path_utils.hpp"
|
||||
|
||||
namespace repertory {
|
||||
full_server::full_server(app_config &config, i_provider &provider, i_open_file_table &oft)
|
||||
: server(config), provider_(provider), oft_(oft) {}
|
||||
|
||||
bool full_server::handle_request(jsonrpcpp::request_ptr &request,
|
||||
std::unique_ptr<jsonrpcpp::Response> &response) {
|
||||
auto handled = true;
|
||||
if (request->method == rpc_method::get_drive_information) {
|
||||
if (request->params.param_array.empty()) {
|
||||
response = std::make_unique<jsonrpcpp::Response>(
|
||||
*request, json({{"cache_space_used", global_data::instance().get_used_cache_space()},
|
||||
{"drive_space_total", provider_.get_total_drive_space()},
|
||||
{"drive_space_used", global_data::instance().get_used_drive_space()},
|
||||
{"item_count", provider_.get_total_item_count()}}));
|
||||
} else {
|
||||
throw jsonrpcpp::InvalidParamsException(*request);
|
||||
}
|
||||
} else if (request->method == rpc_method::get_pinned_files) {
|
||||
if (request->params.param_array.empty()) {
|
||||
response = std::make_unique<jsonrpcpp::Response>(*request, provider_.get_pinned_files());
|
||||
} else {
|
||||
throw jsonrpcpp::InvalidParamsException(*request);
|
||||
}
|
||||
} else if (request->method == rpc_method::get_directory_items) {
|
||||
if (request->params.param_array.size() == 1u) {
|
||||
const auto api_path = utils::path::create_api_path(request->params.param_array[0]);
|
||||
auto directoryItems = oft_.get_directory_items(api_path);
|
||||
std::vector<json> items;
|
||||
for (const auto &item : directoryItems) {
|
||||
items.emplace_back(item.to_json());
|
||||
}
|
||||
|
||||
response = std::make_unique<jsonrpcpp::Response>(*request, json({{"items", items}}));
|
||||
} else {
|
||||
throw jsonrpcpp::InvalidParamsException(*request);
|
||||
}
|
||||
} else if (request->method == rpc_method::pin_file) {
|
||||
if (request->params.param_array.size() == 1u) {
|
||||
const auto api_path = utils::path::create_api_path(request->params.param_array[0]);
|
||||
auto success = provider_.is_file(api_path);
|
||||
if (success) {
|
||||
success = api_error::success ==
|
||||
provider_.set_item_meta(api_path, META_PINNED, utils::string::from_bool(true));
|
||||
}
|
||||
|
||||
if (success) {
|
||||
event_system::instance().raise<file_pinned>(api_path);
|
||||
}
|
||||
|
||||
response = std::make_unique<jsonrpcpp::Response>(*request, json({{"success", success}}));
|
||||
} else {
|
||||
throw jsonrpcpp::InvalidParamsException(*request);
|
||||
}
|
||||
} else if (request->method == rpc_method::pinned_status) {
|
||||
if (request->params.param_array.size() == 1u) {
|
||||
const auto api_path = utils::path::create_api_path(request->params.param_array[0]);
|
||||
std::string pinned;
|
||||
const auto success =
|
||||
api_error::success == provider_.get_item_meta(api_path, META_PINNED, pinned);
|
||||
|
||||
response = std::make_unique<jsonrpcpp::Response>(
|
||||
*request, json({{"success", success},
|
||||
{"pinned", pinned.empty() ? false : utils::string::to_bool(pinned)}}));
|
||||
} else {
|
||||
throw jsonrpcpp::InvalidParamsException(*request);
|
||||
}
|
||||
} else if (request->method == rpc_method::unpin_file) {
|
||||
if (request->params.param_array.size() == 1u) {
|
||||
const auto api_path = utils::path::create_api_path(request->params.param_array[0]);
|
||||
auto success = provider_.is_file(api_path);
|
||||
if (success) {
|
||||
success = api_error::success ==
|
||||
provider_.set_item_meta(api_path, META_PINNED, utils::string::from_bool(false));
|
||||
}
|
||||
|
||||
if (success) {
|
||||
event_system::instance().raise<file_unpinned>(api_path);
|
||||
}
|
||||
|
||||
response = std::make_unique<jsonrpcpp::Response>(*request, json({{"success", success}}));
|
||||
} else {
|
||||
throw jsonrpcpp::InvalidParamsException(*request);
|
||||
}
|
||||
} else if (request->method == rpc_method::get_open_files) {
|
||||
if (request->params.param_array.empty()) {
|
||||
const auto list = oft_.get_open_files();
|
||||
json open_files = {{"file_list", std::vector<json>()}};
|
||||
for (const auto &kv : list) {
|
||||
open_files["file_list"].emplace_back(json({{"path", kv.first}, {"count", kv.second}}));
|
||||
}
|
||||
response = std::make_unique<jsonrpcpp::Response>(*request, open_files);
|
||||
} else {
|
||||
throw jsonrpcpp::InvalidParamsException(*request);
|
||||
}
|
||||
#if defined(REPERTORY_ENABLE_SKYNET)
|
||||
} else if (get_config().get_provider_type() == provider_type::skynet) {
|
||||
if (request->method == rpc_method::import) {
|
||||
if (request->params.param_array.size() == 1) {
|
||||
json results = {{"success", std::vector<json>()}, {"failed", std::vector<std::string>()}};
|
||||
for (const auto &link : request->params.param_array[0]) {
|
||||
const auto si = skylink_import::from_json(link);
|
||||
auto &provider = dynamic_cast<skynet_provider &>(provider_);
|
||||
const auto res = provider.import_skylink(si);
|
||||
if (res == api_error::success) {
|
||||
results["success"].emplace_back(link);
|
||||
} else {
|
||||
results["failed"].emplace_back(link["skylink"].get<std::string>());
|
||||
}
|
||||
}
|
||||
response = std::make_unique<jsonrpcpp::Response>(*request, results);
|
||||
} else {
|
||||
throw jsonrpcpp::InvalidParamsException(*request);
|
||||
}
|
||||
} else if (request->method == rpc_method::export_links) {
|
||||
if (request->params.param_array.empty()) {
|
||||
auto &provider = dynamic_cast<skynet_provider &>(provider_);
|
||||
response = std::make_unique<jsonrpcpp::Response>(*request, provider.export_all());
|
||||
} else if (request->params.param_array.size() == 1) {
|
||||
auto &provider = dynamic_cast<skynet_provider &>(provider_);
|
||||
response = std::make_unique<jsonrpcpp::Response>(
|
||||
*request,
|
||||
provider.export_list(request->params.param_array[0].get<std::vector<std::string>>()));
|
||||
} else {
|
||||
throw jsonrpcpp::InvalidParamsException(*request);
|
||||
}
|
||||
} else {
|
||||
handled = server::handle_request(request, response);
|
||||
}
|
||||
#endif
|
||||
} else {
|
||||
handled = server::handle_request(request, response);
|
||||
}
|
||||
return handled;
|
||||
}
|
||||
} // namespace repertory
|
||||
163
src/rpc/server/server.cpp
Normal file
163
src/rpc/server/server.cpp
Normal file
@@ -0,0 +1,163 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "rpc/server/server.hpp"
|
||||
#include "app_config.hpp"
|
||||
#include "utils/Base64.hpp"
|
||||
|
||||
namespace repertory {
|
||||
server::rpc_resource::rpc_resource(server &owner) : httpserver::http_resource(), owner_(owner) {
|
||||
disallow_all();
|
||||
set_allowing("POST", true);
|
||||
}
|
||||
|
||||
const std::shared_ptr<httpserver::http_response>
|
||||
server::rpc_resource::render(const httpserver::http_request &request) {
|
||||
std::shared_ptr<json_response> ret;
|
||||
|
||||
try {
|
||||
if (owner_.check_authorization(request)) {
|
||||
std::unique_ptr<jsonrpcpp::Response> response;
|
||||
auto entity = jsonrpcpp::Parser::do_parse(utils::string::to_utf8(request.get_content()));
|
||||
if (entity->is_request()) {
|
||||
auto request = std::dynamic_pointer_cast<jsonrpcpp::Request>(entity);
|
||||
if (not owner_.handle_request(request, response)) {
|
||||
throw jsonrpcpp::MethodNotFoundException(*request);
|
||||
}
|
||||
ret = std::make_shared<json_response>(response->to_json());
|
||||
}
|
||||
} else {
|
||||
ret = std::make_shared<json_response>(json({{"error", "unauthorized"}}),
|
||||
httpserver::http::http_utils::http_unauthorized);
|
||||
}
|
||||
} catch (const jsonrpcpp::RequestException &e) {
|
||||
ret = std::make_shared<json_response>(e.to_json(),
|
||||
httpserver::http::http_utils::http_bad_request);
|
||||
event_system::instance().raise<rpc_server_exception>(e.to_json().dump());
|
||||
} catch (const std::exception &e2) {
|
||||
ret = std::make_shared<json_response>(json({{"exception", e2.what()}}),
|
||||
httpserver::http::http_utils::http_internal_server_error);
|
||||
event_system::instance().raise<rpc_server_exception>(e2.what());
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
server::server(app_config &config) : config_(config), resource_(*this) {}
|
||||
|
||||
bool server::check_authorization(const httpserver::http_request &request) {
|
||||
auto ret = (config_.get_api_auth().empty() && config_.get_api_user().empty());
|
||||
if (not ret) {
|
||||
const auto authorization = request.get_header("Authorization");
|
||||
if (not authorization.empty()) {
|
||||
const auto auth_parts = utils::string::split(authorization, ' ');
|
||||
if (not auth_parts.empty()) {
|
||||
const auto auth_type = auth_parts[0];
|
||||
if (auth_type == "Basic") {
|
||||
const auto data = macaron::Base64::Decode(authorization.substr(6));
|
||||
const auto auth = utils::string::split(std::string(data.begin(), data.end()), ':');
|
||||
if (auth.size() == 2) {
|
||||
const auto &user = auth[0];
|
||||
const auto &pwd = auth[1];
|
||||
ret = (user == config_.get_api_user()) && (pwd == config_.get_api_auth());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool server::handle_request(jsonrpcpp::request_ptr &request,
|
||||
std::unique_ptr<jsonrpcpp::Response> &response) {
|
||||
auto handled = true;
|
||||
if (request->method == rpc_method::get_config) {
|
||||
if (request->params.param_array.empty()) {
|
||||
response = std::make_unique<jsonrpcpp::Response>(*request, config_.get_json());
|
||||
} else {
|
||||
throw jsonrpcpp::InvalidParamsException(*request);
|
||||
}
|
||||
} else if (request->method == rpc_method::get_config_value_by_name) {
|
||||
if (request->params.param_array.size() == 1) {
|
||||
response = std::make_unique<jsonrpcpp::Response>(
|
||||
*request, json({{"value", config_.get_value_by_name(request->params.param_array[0])}}));
|
||||
} else {
|
||||
throw jsonrpcpp::InvalidParamsException(*request);
|
||||
}
|
||||
} else if (request->method == rpc_method::set_config_value_by_name) {
|
||||
if (request->params.param_array.size() == 2) {
|
||||
response = std::make_unique<jsonrpcpp::Response>(
|
||||
*request, json({{"value", config_.set_value_by_name(request->params.param_array[0],
|
||||
request->params.param_array[1])}}));
|
||||
} else {
|
||||
throw jsonrpcpp::InvalidParamsException(*request);
|
||||
}
|
||||
} else if (request->method == rpc_method::unmount) {
|
||||
if (request->params.param_array.empty()) {
|
||||
event_system::instance().raise<unmount_requested>();
|
||||
response = std::make_unique<jsonrpcpp::Response>(*request, json({{"success", true}}));
|
||||
} else {
|
||||
throw jsonrpcpp::InvalidParamsException(*request);
|
||||
}
|
||||
} else {
|
||||
handled = false;
|
||||
}
|
||||
return handled;
|
||||
}
|
||||
|
||||
/*void server::Start() {
|
||||
mutex_lock l(startStopMutex_);
|
||||
if (not started_) {
|
||||
struct sockaddr_in localHost{};
|
||||
inet_pton(AF_INET, "127.0.0.1", &localHost.sin_addr);
|
||||
|
||||
ws_ = std::make_unique<httpserver::webserver>(
|
||||
httpserver::create_webserver(config_.GetAPIPort())
|
||||
.bind_address((sockaddr*)&localHost)
|
||||
.default_policy(httpserver::http::http_utils::REJECT));
|
||||
ws_->allow_ip("127.0.0.1");
|
||||
ws_->register_resource("/api", &rpcResource_);
|
||||
ws_->start(false);
|
||||
started_ = true;
|
||||
}
|
||||
}*/
|
||||
|
||||
void server::start() {
|
||||
mutex_lock l(start_stop_mutex_);
|
||||
if (not started_) {
|
||||
ws_ = std::make_unique<httpserver::webserver>(
|
||||
httpserver::create_webserver(config_.get_api_port())
|
||||
.default_policy(httpserver::http::http_utils::REJECT));
|
||||
ws_->allow_ip("127.0.0.1");
|
||||
ws_->register_resource("/api", &resource_);
|
||||
ws_->start(false);
|
||||
started_ = true;
|
||||
}
|
||||
}
|
||||
|
||||
void server::stop() {
|
||||
mutex_lock l(start_stop_mutex_);
|
||||
if (started_) {
|
||||
event_system::instance().raise<service_shutdown>("server");
|
||||
ws_->stop();
|
||||
ws_.reset();
|
||||
started_ = false;
|
||||
}
|
||||
}
|
||||
} // namespace repertory
|
||||
183
src/types/remote.cpp
Normal file
183
src/types/remote.cpp
Normal file
@@ -0,0 +1,183 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "types/remote.hpp"
|
||||
|
||||
namespace repertory::remote {
|
||||
#ifndef _WIN32
|
||||
open_flags create_open_flags(const std::uint32_t &flags) {
|
||||
open_flags ret{};
|
||||
{
|
||||
const auto f = (flags & 3u);
|
||||
ret |= (f == 1u) ? open_flags::write_only
|
||||
: (f == 2u) ? open_flags::read_write
|
||||
: open_flags::read_only;
|
||||
}
|
||||
if (flags & static_cast<std::uint32_t>(O_CREAT)) {
|
||||
ret |= open_flags::create;
|
||||
}
|
||||
|
||||
if (flags & static_cast<std::uint32_t>(O_EXCL)) {
|
||||
ret |= open_flags::excl;
|
||||
}
|
||||
|
||||
if (flags & static_cast<std::uint32_t>(O_NOCTTY)) {
|
||||
ret |= open_flags::no_ctty;
|
||||
}
|
||||
|
||||
if (flags & static_cast<std::uint32_t>(O_TRUNC)) {
|
||||
ret |= open_flags::truncate;
|
||||
}
|
||||
|
||||
if (flags & static_cast<std::uint32_t>(O_APPEND)) {
|
||||
ret |= open_flags::append;
|
||||
}
|
||||
|
||||
if (flags & static_cast<std::uint32_t>(O_NONBLOCK)) {
|
||||
ret |= open_flags::non_blocking;
|
||||
}
|
||||
|
||||
if (flags & static_cast<std::uint32_t>(O_SYNC)) {
|
||||
ret |= open_flags::sync;
|
||||
}
|
||||
|
||||
if (flags & static_cast<std::uint32_t>(O_ASYNC)) {
|
||||
ret |= open_flags::async;
|
||||
}
|
||||
|
||||
if (flags & static_cast<std::uint32_t>(O_DIRECTORY)) {
|
||||
ret |= open_flags::directory;
|
||||
}
|
||||
|
||||
if (flags & static_cast<std::uint32_t>(O_NOFOLLOW)) {
|
||||
ret |= open_flags::no_follow;
|
||||
}
|
||||
|
||||
if (flags & static_cast<std::uint32_t>(O_CLOEXEC)) {
|
||||
ret |= open_flags::clo_exec;
|
||||
}
|
||||
#ifdef O_DIRECT
|
||||
if (flags & static_cast<std::uint32_t>(O_DIRECT)) {
|
||||
ret |= open_flags::direct;
|
||||
}
|
||||
#endif
|
||||
#ifdef O_NOATIME
|
||||
if (flags & static_cast<std::uint32_t>(O_NOATIME)) {
|
||||
ret |= open_flags::no_atime;
|
||||
}
|
||||
#endif
|
||||
#ifdef O_PATH
|
||||
if (flags & static_cast<std::uint32_t>(O_PATH)) {
|
||||
ret |= open_flags::path;
|
||||
}
|
||||
#endif
|
||||
#ifdef O_TMPFILE
|
||||
if (flags & static_cast<std::uint32_t>(O_TMPFILE)) {
|
||||
ret |= open_flags::temp_file;
|
||||
}
|
||||
#endif
|
||||
#ifdef O_DSYNC
|
||||
if (flags & static_cast<std::uint32_t>(O_DSYNC)) {
|
||||
ret |= open_flags::dsync;
|
||||
}
|
||||
#endif
|
||||
return ret;
|
||||
}
|
||||
|
||||
std::uint32_t create_os_open_flags(const open_flags &flags) {
|
||||
std::uint32_t ret = 0u;
|
||||
if ((flags & open_flags::read_write) == open_flags::read_write) {
|
||||
ret |= static_cast<std::uint32_t>(O_RDWR);
|
||||
} else if ((flags & open_flags::write_only) == open_flags::write_only) {
|
||||
ret |= static_cast<std::uint32_t>(O_WRONLY);
|
||||
} else {
|
||||
ret |= static_cast<std::uint32_t>(O_RDONLY);
|
||||
}
|
||||
|
||||
if ((flags & open_flags::create) == open_flags::create) {
|
||||
ret |= static_cast<std::uint32_t>(O_CREAT);
|
||||
}
|
||||
|
||||
if ((flags & open_flags::excl) == open_flags::excl) {
|
||||
ret |= static_cast<std::uint32_t>(O_EXCL);
|
||||
}
|
||||
|
||||
if ((flags & open_flags::no_ctty) == open_flags::no_ctty) {
|
||||
ret |= static_cast<std::uint32_t>(O_NOCTTY);
|
||||
}
|
||||
|
||||
if ((flags & open_flags::truncate) == open_flags::truncate) {
|
||||
ret |= static_cast<std::uint32_t>(O_TRUNC);
|
||||
}
|
||||
|
||||
if ((flags & open_flags::append) == open_flags::append) {
|
||||
ret |= static_cast<std::uint32_t>(O_APPEND);
|
||||
}
|
||||
|
||||
if ((flags & open_flags::non_blocking) == open_flags::non_blocking) {
|
||||
ret |= static_cast<std::uint32_t>(O_NONBLOCK);
|
||||
}
|
||||
|
||||
if ((flags & open_flags::sync) == open_flags::sync) {
|
||||
ret |= static_cast<std::uint32_t>(O_SYNC);
|
||||
}
|
||||
|
||||
if ((flags & open_flags::async) == open_flags::async) {
|
||||
ret |= static_cast<std::uint32_t>(O_ASYNC);
|
||||
}
|
||||
|
||||
if ((flags & open_flags::directory) == open_flags::directory) {
|
||||
ret |= static_cast<std::uint32_t>(O_DIRECTORY);
|
||||
}
|
||||
|
||||
if ((flags & open_flags::no_follow) == open_flags::no_follow) {
|
||||
ret |= static_cast<std::uint32_t>(O_NOFOLLOW);
|
||||
}
|
||||
|
||||
if ((flags & open_flags::clo_exec) == open_flags::clo_exec) {
|
||||
ret |= static_cast<std::uint32_t>(O_CLOEXEC);
|
||||
}
|
||||
#ifdef O_DIRECT
|
||||
if ((flags & open_flags::direct) == open_flags::direct) {
|
||||
ret |= static_cast<std::uint32_t>(O_DIRECT);
|
||||
}
|
||||
#endif
|
||||
#ifdef O_NOATIME
|
||||
if ((flags & open_flags::no_atime) == open_flags::no_atime) {
|
||||
ret |= static_cast<std::uint32_t>(O_NOATIME);
|
||||
}
|
||||
#endif
|
||||
#ifdef O_PATH
|
||||
if ((flags & open_flags::path) == open_flags::path) {
|
||||
ret |= static_cast<std::uint32_t>(O_PATH);
|
||||
}
|
||||
#endif
|
||||
#ifdef O_TMPFILE
|
||||
if ((flags & open_flags::temp_file) == open_flags::temp_file) {
|
||||
ret |= static_cast<std::uint32_t>(O_TMPFILE);
|
||||
}
|
||||
#endif
|
||||
#ifdef O_DSYNC
|
||||
if ((flags & open_flags::dsync) == open_flags::dsync) {
|
||||
ret |= static_cast<std::uint32_t>(O_DSYNC);
|
||||
}
|
||||
#endif
|
||||
return ret;
|
||||
}
|
||||
#endif
|
||||
} // namespace repertory::remote
|
||||
73
src/types/repertory.cpp
Normal file
73
src/types/repertory.cpp
Normal file
@@ -0,0 +1,73 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "types/repertory.hpp"
|
||||
#include "types/startup_exception.hpp"
|
||||
|
||||
namespace repertory {
|
||||
const std::string &api_error_to_string(const api_error &error) {
|
||||
static const std::unordered_map<api_error, std::string> LOOKUP = {
|
||||
{api_error::success, "success"},
|
||||
{api_error::access_denied, "access_denied"},
|
||||
{api_error::bad_address, "bad_address"},
|
||||
{api_error::buffer_overflow, "buffer_overflow"},
|
||||
{api_error::buffer_too_small, "buffer_too_small"},
|
||||
{api_error::comm_error, "comm_error"},
|
||||
{api_error::decryption_error, "decryption_error"},
|
||||
{api_error::directory_end_of_files, "directory_end_of_files"},
|
||||
{api_error::directory_exists, "directory_exists"},
|
||||
{api_error::directory_not_empty, "directory_not_empty"},
|
||||
{api_error::directory_not_found, "directory_not_found"},
|
||||
{api_error::download_failed, "download_failed"},
|
||||
{api_error::download_incomplete, "download_incomplete"},
|
||||
{api_error::download_stopped, "download_stopped"},
|
||||
{api_error::download_timeout, "download_timeout"},
|
||||
{api_error::empty_ring_buffer_chunk_size, "empty_ring_buffer_chunk_size"},
|
||||
{api_error::empty_ring_buffer_size, "empty_ring_buffer_size"},
|
||||
{api_error::error, "error"},
|
||||
{api_error::file_exists, "file_exists"},
|
||||
{api_error::file_in_use, "file_in_use"},
|
||||
{api_error::incompatible_version, "incompatible_version"},
|
||||
{api_error::invalid_handle, "invalid_handle"},
|
||||
{api_error::invalid_operation, "invalid_operation"},
|
||||
{api_error::invalid_ring_buffer_multiple, "invalid_ring_buffer_multiple"},
|
||||
{api_error::invalid_ring_buffer_size, "invalid_ring_buffer_size"},
|
||||
{api_error::invalid_version, "invalid_version"},
|
||||
{api_error::item_is_file, "item_is_file"},
|
||||
{api_error::item_not_found, "item_not_found"},
|
||||
{api_error::not_implemented, "not_implemented"},
|
||||
{api_error::not_supported, "not_supported"},
|
||||
{api_error::os_error, "os_error"},
|
||||
{api_error::permission_denied, "permission_denied"},
|
||||
{api_error::upload_failed, "upload_failed"},
|
||||
{api_error::upload_stopped, "upload_stopped"},
|
||||
{api_error::xattr_buffer_small, "xattr_buffer_small"},
|
||||
{api_error::xattr_exists, "xattr_exists"},
|
||||
{api_error::xattr_invalid_namespace, "xattr_invalid_namespace"},
|
||||
{api_error::xattr_not_found, "xattr_not_found"},
|
||||
{api_error::xattr_osx_invalid, "xattr_osx_invalid"},
|
||||
{api_error::xattr_too_big, "xattr_too_big"},
|
||||
};
|
||||
|
||||
if (LOOKUP.size() != static_cast<std::size_t>(api_error::ERROR_COUNT)) {
|
||||
throw startup_exception("undefined api_error strings");
|
||||
}
|
||||
|
||||
return LOOKUP.at(error);
|
||||
}
|
||||
} // namespace repertory
|
||||
111
src/types/skynet.cpp
Normal file
111
src/types/skynet.cpp
Normal file
@@ -0,0 +1,111 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "types/skynet.hpp"
|
||||
#include "utils/string_utils.hpp"
|
||||
|
||||
namespace repertory {
|
||||
std::vector<host_config> skynet_config::from_string(const std::string &list) {
|
||||
const auto obj = json::parse(list);
|
||||
std::vector<host_config> ret;
|
||||
std::transform(obj.begin(), obj.end(), std::back_inserter(ret),
|
||||
[](const json &portal) -> host_config {
|
||||
host_config hc{};
|
||||
hc.agent_string = portal["AgentString"].get<std::string>();
|
||||
hc.api_password = portal["ApiPassword"].get<std::string>();
|
||||
hc.api_port = portal["ApiPort"].get<std::uint16_t>();
|
||||
hc.host_name_or_ip = portal["HostNameOrIp"].get<std::string>();
|
||||
hc.path = portal["Path"].get<std::string>();
|
||||
hc.protocol = portal["Protocol"].get<std::string>();
|
||||
hc.timeout_ms = portal["TimeoutMs"].get<std::uint32_t>();
|
||||
hc.auth_url = portal["AuthURL"].get<std::string>();
|
||||
hc.auth_user = portal["AuthUser"].get<std::string>();
|
||||
hc.auth_password = portal["AuthPassword"].get<std::string>();
|
||||
return hc;
|
||||
});
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
std::string skynet_config::to_string(const std::vector<host_config> &list) {
|
||||
json j;
|
||||
std::transform(list.begin(), list.end(), std::back_inserter(j), [](const auto &hc) -> json {
|
||||
return {
|
||||
{"AgentString", hc.agent_string},
|
||||
{"ApiPassword", hc.api_password},
|
||||
{"ApiPort", hc.api_port},
|
||||
{"HostNameOrIp", hc.host_name_or_ip},
|
||||
{"Path", hc.path},
|
||||
{"Protocol", hc.protocol},
|
||||
{"TimeoutMs", hc.timeout_ms},
|
||||
{"AuthURL", hc.auth_url},
|
||||
{"AuthUser", hc.auth_user},
|
||||
{"AuthPassword", hc.auth_password},
|
||||
};
|
||||
});
|
||||
|
||||
return j.dump();
|
||||
}
|
||||
|
||||
skylink_import skylink_import::from_json(const json &j) {
|
||||
return skylink_import{
|
||||
j.contains("directory") ? j["directory"].get<std::string>() : "/",
|
||||
j.contains("filename") ? j["filename"].get<std::string>() : "",
|
||||
j["skylink"].get<std::string>(),
|
||||
j.contains("token") ? j["token"].get<std::string>() : "",
|
||||
};
|
||||
}
|
||||
|
||||
skylink_import skylink_import::from_string(const std::string &str) {
|
||||
auto parts = utils::string::split(str, ',');
|
||||
if (parts.empty() || (parts.size() > 4)) {
|
||||
throw std::runtime_error("unable to parse skylink from string: " + str);
|
||||
}
|
||||
|
||||
std::unordered_map<std::string, std::string> linkData;
|
||||
for (auto &part : parts) {
|
||||
utils::string::replace(part, "@comma@", ",");
|
||||
auto parts2 = utils::string::split(part, '=', false);
|
||||
if (parts2.size() != 2) {
|
||||
throw std::runtime_error("unable to parse skylink from string: " + str);
|
||||
}
|
||||
|
||||
for (auto &part2 : parts2) {
|
||||
utils::string::replace(part2, "@equal@", "=");
|
||||
}
|
||||
|
||||
linkData[parts2[0]] = parts2[1];
|
||||
}
|
||||
|
||||
return skylink_import{
|
||||
linkData["directory"],
|
||||
linkData["filename"],
|
||||
linkData["skylink"],
|
||||
linkData["token"],
|
||||
};
|
||||
}
|
||||
|
||||
json skylink_import::to_json() const {
|
||||
return {
|
||||
{"directory", directory},
|
||||
{"filename", file_name},
|
||||
{"skylink", skylink},
|
||||
{"token", token},
|
||||
};
|
||||
}
|
||||
} // namespace repertory
|
||||
259
src/upload/upload_manager.cpp
Normal file
259
src/upload/upload_manager.cpp
Normal file
@@ -0,0 +1,259 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "upload/upload_manager.hpp"
|
||||
#include "comm/i_comm.hpp"
|
||||
#include "app_config.hpp"
|
||||
#include "events/events.hpp"
|
||||
#include "events/event_system.hpp"
|
||||
#include "utils/file_utils.hpp"
|
||||
#include "utils/rocksdb_utils.hpp"
|
||||
|
||||
namespace repertory {
|
||||
upload_manager::upload_manager(const app_config &config, api_file_exists_callback api_file_exists,
|
||||
upload_handler_callback upload_handler,
|
||||
upload_completed_callback upload_completed)
|
||||
: config_(config),
|
||||
api_file_exists_(std::move(api_file_exists)),
|
||||
upload_handler_(std::move(upload_handler)),
|
||||
upload_completed_(std::move(upload_completed)) {
|
||||
utils::db::create_rocksdb(config, ROCKS_DB_NAME, upload_db_);
|
||||
auto iterator =
|
||||
std::unique_ptr<rocksdb::Iterator>(upload_db_->NewIterator(rocksdb::ReadOptions()));
|
||||
for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
|
||||
auto data = json::parse(iterator->value().ToString());
|
||||
auto u = std::make_shared<upload>(upload{
|
||||
data["path"].get<std::string>(),
|
||||
false,
|
||||
data["token"].get<std::string>(),
|
||||
data["source"].get<std::string>(),
|
||||
});
|
||||
|
||||
upload_lookup_[iterator->key().ToString()] = u;
|
||||
upload_queue_.emplace_back(u.get());
|
||||
}
|
||||
}
|
||||
|
||||
upload_manager::~upload_manager() {
|
||||
stop();
|
||||
upload_db_.reset();
|
||||
}
|
||||
|
||||
bool upload_manager::execute_if_not_processing(const std::vector<std::string> &api_paths,
|
||||
const std::function<void()> &action) const {
|
||||
mutex_lock upload_lock(upload_mutex_);
|
||||
const auto ret =
|
||||
(std::find_if(api_paths.begin(), api_paths.end(), [this](const auto &path) -> bool {
|
||||
return upload_lookup_.find(path) != upload_lookup_.end();
|
||||
}) == api_paths.end());
|
||||
if (ret) {
|
||||
action();
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool upload_manager::is_processing(const std::string &api_path) const {
|
||||
mutex_lock upload_lock(upload_mutex_);
|
||||
return upload_lookup_.find(api_path) != upload_lookup_.end();
|
||||
}
|
||||
|
||||
api_error upload_manager::queue_upload(const std::string &api_path, const std::string &source_path,
|
||||
const std::string &encryption_token) {
|
||||
auto ret = remove_upload(api_path);
|
||||
if ((ret == api_error::success) || (ret == api_error::item_not_found)) {
|
||||
ret = api_error::success;
|
||||
|
||||
auto u = std::make_shared<upload>(upload{api_path, false, encryption_token, source_path});
|
||||
|
||||
unique_mutex_lock upload_lock(upload_mutex_);
|
||||
upload_db_->Put(rocksdb::WriteOptions(), api_path,
|
||||
json({
|
||||
{"path", api_path},
|
||||
{"source", source_path},
|
||||
{"token", encryption_token},
|
||||
})
|
||||
.dump());
|
||||
if (not stop_requested_) {
|
||||
upload_lookup_[api_path] = u;
|
||||
upload_queue_.emplace_back(u.get());
|
||||
}
|
||||
upload_notify_.notify_all();
|
||||
event_system::instance().raise<file_upload_queued>(api_path, source_path);
|
||||
upload_lock.unlock();
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error upload_manager::remove_upload(const std::string &api_path) {
|
||||
auto ret = api_error::item_not_found;
|
||||
|
||||
unique_mutex_lock upload_lock(upload_mutex_);
|
||||
if (upload_lookup_.find(api_path) != upload_lookup_.end()) {
|
||||
auto upload = upload_lookup_[api_path];
|
||||
utils::remove_element_from(upload_queue_, upload.get());
|
||||
upload_lookup_.erase(api_path);
|
||||
upload_db_->Delete(rocksdb::WriteOptions(), api_path);
|
||||
upload->cancel = true;
|
||||
upload_lock.unlock();
|
||||
|
||||
upload->wait();
|
||||
|
||||
event_system::instance().raise<file_upload_removed>(api_path, upload->source_path);
|
||||
ret = api_error::success;
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
void upload_manager::start() {
|
||||
mutex_lock upload_lock(upload_mutex_);
|
||||
if (not upload_thread_) {
|
||||
stop_requested_ = false;
|
||||
upload_thread_ = std::make_unique<std::thread>([this]() { this->upload_thread(); });
|
||||
}
|
||||
}
|
||||
|
||||
void upload_manager::stop() {
|
||||
unique_mutex_lock upload_lock(upload_mutex_);
|
||||
if (upload_thread_) {
|
||||
stop_requested_ = true;
|
||||
upload_notify_.notify_all();
|
||||
upload_lock.unlock();
|
||||
|
||||
upload_thread_->join();
|
||||
upload_thread_.reset();
|
||||
stop_requested_ = false;
|
||||
|
||||
event_system::instance().raise<service_shutdown>("upload_manager");
|
||||
}
|
||||
}
|
||||
|
||||
void upload_manager::upload_thread() {
|
||||
unique_mutex_lock upload_lock(upload_mutex_);
|
||||
upload_lock.unlock();
|
||||
|
||||
std::deque<std::shared_ptr<upload>> completed;
|
||||
const auto process_completed = [&]() {
|
||||
while (not completed.empty()) {
|
||||
upload_lock.lock();
|
||||
auto u = completed.front();
|
||||
completed.pop_front();
|
||||
upload_lock.unlock();
|
||||
|
||||
u->thread->join();
|
||||
u->thread.reset();
|
||||
u->completed = true;
|
||||
}
|
||||
};
|
||||
|
||||
const auto process_next = [this, &completed]() {
|
||||
if (not upload_queue_.empty()) {
|
||||
auto *next_upload = upload_queue_.front();
|
||||
if (not next_upload->thread) {
|
||||
upload_queue_.pop_front();
|
||||
next_upload->completed = false;
|
||||
active_uploads_.emplace_back(next_upload);
|
||||
auto u = upload_lookup_[next_upload->api_path];
|
||||
auto upload_handler = [this, &completed, u]() {
|
||||
json data, err;
|
||||
const auto ret = this->upload_handler_(*u, data, err);
|
||||
|
||||
unique_mutex_lock upload_lock(upload_mutex_);
|
||||
if (ret == api_error::success) {
|
||||
upload_lookup_.erase(u->api_path);
|
||||
upload_db_->Delete(rocksdb::WriteOptions(), u->api_path);
|
||||
upload_completed_(u->api_path, u->source_path, data);
|
||||
event_system::instance().raise<file_upload_completed>(u->api_path, u->source_path);
|
||||
} else {
|
||||
event_system::instance().raise<file_upload_failed>(u->api_path, u->source_path,
|
||||
err.dump(2));
|
||||
if (u->cancel) {
|
||||
upload_lookup_.erase(u->api_path);
|
||||
} else if (not api_file_exists_(u->api_path) ||
|
||||
not utils::file::is_file(u->source_path)) {
|
||||
upload_lookup_.erase(u->api_path);
|
||||
upload_db_->Delete(rocksdb::WriteOptions(), u->api_path);
|
||||
event_system::instance().raise<file_upload_not_found>(u->api_path, u->source_path);
|
||||
} else {
|
||||
upload_queue_.emplace_back(u.get());
|
||||
u->retry = true;
|
||||
}
|
||||
}
|
||||
utils::remove_element_from(active_uploads_, u.get());
|
||||
|
||||
completed.emplace_back(u);
|
||||
upload_notify_.notify_all();
|
||||
upload_lock.unlock();
|
||||
};
|
||||
|
||||
u->thread = std::make_unique<std::thread>(std::move(upload_handler));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const auto has_only_retries = [this]() -> bool {
|
||||
return std::find_if(upload_queue_.begin(), upload_queue_.end(), [](const auto &upload) -> bool {
|
||||
return not upload->retry;
|
||||
}) == upload_queue_.end();
|
||||
};
|
||||
|
||||
const auto cancel_all_active = [this, &process_completed, &upload_lock]() {
|
||||
upload_lock.lock();
|
||||
for (auto &upload : upload_lookup_) {
|
||||
upload.second->cancel = true;
|
||||
}
|
||||
upload_lock.unlock();
|
||||
|
||||
process_completed();
|
||||
};
|
||||
|
||||
const auto drain_queue = [this, &process_completed, &process_next, &upload_lock]() {
|
||||
while (not upload_lookup_.empty()) {
|
||||
upload_lock.lock();
|
||||
process_next();
|
||||
upload_lock.unlock();
|
||||
|
||||
process_completed();
|
||||
}
|
||||
};
|
||||
|
||||
const auto run_queue_loop = [this, &has_only_retries, &process_completed, &process_next,
|
||||
&upload_lock]() {
|
||||
while (not stop_requested_) {
|
||||
upload_lock.lock();
|
||||
if ((active_uploads_.size() >= config_.get_max_upload_count()) || upload_queue_.empty()) {
|
||||
upload_notify_.wait_for(upload_lock, 5s);
|
||||
if (not stop_requested_ && has_only_retries()) {
|
||||
upload_notify_.wait_for(upload_lock, 5s);
|
||||
}
|
||||
} else {
|
||||
process_next();
|
||||
}
|
||||
upload_lock.unlock();
|
||||
|
||||
process_completed();
|
||||
}
|
||||
};
|
||||
|
||||
run_queue_loop();
|
||||
cancel_all_active();
|
||||
drain_queue();
|
||||
}
|
||||
} // namespace repertory
|
||||
236
src/utils/cli_utils.cpp
Normal file
236
src/utils/cli_utils.cpp
Normal file
@@ -0,0 +1,236 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "utils/cli_utils.hpp"
|
||||
#include "app_config.hpp"
|
||||
#include "utils/file_utils.hpp"
|
||||
#include "utils/path_utils.hpp"
|
||||
#include "utils/string_utils.hpp"
|
||||
#include "utils/utils.hpp"
|
||||
|
||||
namespace repertory::utils::cli {
|
||||
void get_api_authentication_data(std::string &user, std::string &password, std::uint16_t &port,
|
||||
const provider_type &pt, const std::string &data_directory) {
|
||||
const auto cfg_file_path = utils::path::combine(
|
||||
data_directory.empty() ? app_config::default_data_directory(pt) : data_directory,
|
||||
{"config.json"});
|
||||
|
||||
json data;
|
||||
auto success = false;
|
||||
for (auto i = 0; not(success = utils::file::read_json_file(cfg_file_path, data)) && (i < 20);
|
||||
i++) {
|
||||
std::this_thread::sleep_for(100ms);
|
||||
}
|
||||
|
||||
if (success) {
|
||||
if (user.empty() && password.empty()) {
|
||||
password = data["ApiAuth"].get<std::string>();
|
||||
user = data["ApiUser"].get<std::string>();
|
||||
}
|
||||
port = data["ApiPort"].get<std::uint16_t>();
|
||||
}
|
||||
}
|
||||
|
||||
provider_type get_provider_type_from_args(const int &argc, char *argv[]) {
|
||||
auto pt = provider_type::unknown;
|
||||
#if defined(REPERTORY_ENABLE_S3)
|
||||
if ((pt == provider_type::unknown) && (has_option(argc, argv, options::s3_option))) {
|
||||
pt = provider_type::s3;
|
||||
}
|
||||
#endif // defined(REPERTORY_ENABLE_S3)
|
||||
#if defined(REPERTORY_ENABLE_SKYNET)
|
||||
if ((pt == provider_type::unknown) && (has_option(argc, argv, options::skynet_option))) {
|
||||
pt = provider_type::skynet;
|
||||
}
|
||||
#endif // defined(REPERTORY_ENABLE_SKYNET)
|
||||
if ((pt == provider_type::unknown) && (has_option(argc, argv, options::remote_mount_option))) {
|
||||
pt = provider_type::remote;
|
||||
}
|
||||
|
||||
if (pt == provider_type::unknown) {
|
||||
pt = provider_type::sia;
|
||||
}
|
||||
|
||||
return pt;
|
||||
}
|
||||
bool has_option(const int &argc, char *argv[], const std::string &option_name) {
|
||||
auto ret = false;
|
||||
for (int i = 0; not ret && (i < argc); i++) {
|
||||
ret = (option_name == argv[i]);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool has_option(const int &argc, char *argv[], const option &opt) {
|
||||
return has_option(argc, argv, opt[0u]) || has_option(argc, argv, opt[1u]);
|
||||
}
|
||||
|
||||
std::vector<std::string> parse_option(const int &argc, char *argv[], const std::string &option_name,
|
||||
std::uint8_t count) {
|
||||
std::vector<std::string> ret;
|
||||
auto found = false;
|
||||
for (auto i = 0; not found && (i < argc); i++) {
|
||||
if ((found = (option_name == argv[i]))) {
|
||||
if ((++i + count) <= argc) {
|
||||
while (count--) {
|
||||
ret.emplace_back(argv[i++]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
exit_code parse_string_option(const int &argc, char **argv, const option &opt, std::string &value) {
|
||||
auto ret = exit_code::success;
|
||||
if (has_option(argc, argv, opt[0u]) || has_option(argc, argv, opt[1u])) {
|
||||
auto data = parse_option(argc, argv, opt[0u], 1u);
|
||||
if (data.empty()) {
|
||||
data = parse_option(argc, argv, opt[1u], 1u);
|
||||
if (data.empty()) {
|
||||
std::cerr << "Invalid syntax for '" << opt[0u] << "," << opt[1u] << '\'' << std::endl;
|
||||
ret = exit_code::invalid_syntax;
|
||||
}
|
||||
}
|
||||
|
||||
if (not data.empty()) {
|
||||
value = data[0u];
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
std::vector<std::string> parse_drive_options(const int &argc, char **argv, provider_type &pt,
|
||||
std::string &data_directory) {
|
||||
// Strip out options from command line
|
||||
const auto &option_list = options::option_list;
|
||||
std::vector<std::string> drive_args;
|
||||
for (int i = 0; i < argc; i++) {
|
||||
const auto &a = argv[i];
|
||||
if (std::find_if(option_list.begin(), option_list.end(), [&a](const auto &pair) -> bool {
|
||||
return ((pair[0] == a) || (pair[1] == a));
|
||||
}) == option_list.end()) {
|
||||
drive_args.emplace_back(argv[i]);
|
||||
} else if ((std::string(argv[i]) == options::remote_mount_option[0]) ||
|
||||
(std::string(argv[i]) == options::remote_mount_option[1])) {
|
||||
i++;
|
||||
}
|
||||
#ifdef REPERTORY_ENABLE_S3
|
||||
else if ((std::string(argv[i]) == options::name_option[0]) ||
|
||||
(std::string(argv[i]) == options::name_option[1])) {
|
||||
i++;
|
||||
}
|
||||
#endif // REPERTORY_ENABLE_S3
|
||||
}
|
||||
|
||||
#ifndef _WIN32
|
||||
std::vector<std::string> fuse_flags_list;
|
||||
for (std::size_t i = 1; i < drive_args.size(); i++) {
|
||||
if (drive_args[i].find("-o") == 0) {
|
||||
std::string options = "";
|
||||
if (drive_args[i].size() == 2) {
|
||||
if ((i + 1) < drive_args.size()) {
|
||||
options = drive_args[++i];
|
||||
}
|
||||
} else {
|
||||
options = drive_args[i].substr(2);
|
||||
}
|
||||
|
||||
const auto fuse_option_list = utils::string::split(options, ',');
|
||||
for (const auto &fuse_option : fuse_option_list) {
|
||||
#if defined(REPERTORY_ENABLE_S3)
|
||||
if (fuse_option.find("s3") == 0) {
|
||||
pt = provider_type::s3;
|
||||
}
|
||||
#endif // defined(REPERTORY_ENABLE_S3)
|
||||
#if defined(REPERTORY_ENABLE_SKYNET)
|
||||
if ((fuse_option.find("sk") == 0) || (fuse_option.find("skynet") == 0)) {
|
||||
pt = provider_type::skynet;
|
||||
}
|
||||
#endif // defined(REPERTORY_ENABLE_SKYNET)
|
||||
if ((fuse_option.find("dd") == 0) || (fuse_option.find("data_directory") == 0)) {
|
||||
const auto data = utils::string::split(fuse_option, '=');
|
||||
if (data.size() == 2) {
|
||||
data_directory = data[1];
|
||||
} else {
|
||||
std::cerr << "Invalid syntax for '-dd,--data_directory'" << std::endl;
|
||||
exit(3);
|
||||
}
|
||||
} else {
|
||||
fuse_flags_list.push_back(fuse_option);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
std::vector<std::string> new_drive_args(drive_args);
|
||||
for (std::size_t i = 1; i < new_drive_args.size(); i++) {
|
||||
if (new_drive_args[i].find("-o") == 0) {
|
||||
if (new_drive_args[i].size() == 2) {
|
||||
if ((i + 1) < drive_args.size()) {
|
||||
utils::remove_element_from(new_drive_args, new_drive_args[i]);
|
||||
utils::remove_element_from(new_drive_args, new_drive_args[i]);
|
||||
i--;
|
||||
}
|
||||
} else {
|
||||
utils::remove_element_from(new_drive_args, new_drive_args[i]);
|
||||
i--;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef __APPLE__
|
||||
auto it = std::remove_if(fuse_flags_list.begin(), fuse_flags_list.end(),
|
||||
[](const auto &opt) -> bool { return opt.find("volname") == 0; });
|
||||
if (it != fuse_flags_list.end()) {
|
||||
fuse_flags_list.erase(it, fuse_flags_list.end());
|
||||
}
|
||||
fuse_flags_list.emplace_back("volname=Repertory" + app_config::get_provider_display_name(pt));
|
||||
|
||||
it = std::remove_if(fuse_flags_list.begin(), fuse_flags_list.end(),
|
||||
[](const auto &opt) -> bool { return opt.find("daemon_timeout") == 0; });
|
||||
if (it != fuse_flags_list.end()) {
|
||||
fuse_flags_list.erase(it, fuse_flags_list.end());
|
||||
}
|
||||
fuse_flags_list.emplace_back("daemon_timeout=300");
|
||||
|
||||
it = std::remove_if(fuse_flags_list.begin(), fuse_flags_list.end(),
|
||||
[](const auto &opt) -> bool { return opt.find("noappledouble") == 0; });
|
||||
if (it == fuse_flags_list.end()) {
|
||||
fuse_flags_list.emplace_back("noappledouble");
|
||||
}
|
||||
#endif // __APPLE__
|
||||
|
||||
if (not fuse_flags_list.empty()) {
|
||||
std::string fuse_options;
|
||||
for (const auto &flag : fuse_flags_list) {
|
||||
fuse_options += (fuse_options.empty()) ? flag : (',' + flag);
|
||||
}
|
||||
new_drive_args.emplace_back("-o");
|
||||
new_drive_args.emplace_back(fuse_options);
|
||||
}
|
||||
drive_args = std::move(new_drive_args);
|
||||
}
|
||||
#endif // _WIN32
|
||||
|
||||
return drive_args;
|
||||
}
|
||||
} // namespace repertory::utils::cli
|
||||
244
src/utils/encrypting_reader.cpp
Normal file
244
src/utils/encrypting_reader.cpp
Normal file
@@ -0,0 +1,244 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "utils/encrypting_reader.hpp"
|
||||
#include "events/events.hpp"
|
||||
#include "events/event_system.hpp"
|
||||
#include "utils/encryption.hpp"
|
||||
#include "utils/utils.hpp"
|
||||
|
||||
namespace repertory::utils::encryption {
|
||||
class encrypting_streambuf final : public encrypting_reader::streambuf {
|
||||
public:
|
||||
explicit encrypting_streambuf(const encrypting_reader &reader) : reader_(reader) {
|
||||
setg(reinterpret_cast<char *>(0), reinterpret_cast<char *>(0),
|
||||
reinterpret_cast<char *>(reader_.get_total_size()));
|
||||
}
|
||||
|
||||
~encrypting_streambuf() override = default;
|
||||
|
||||
private:
|
||||
encrypting_reader reader_;
|
||||
|
||||
protected:
|
||||
pos_type seekoff(off_type off, std::ios_base::seekdir dir,
|
||||
std::ios_base::openmode which = std::ios_base::out |
|
||||
std::ios_base::in) override {
|
||||
if ((which & std::ios_base::in) != std::ios_base::in) {
|
||||
throw std::runtime_error("output is not supported");
|
||||
}
|
||||
|
||||
const auto set_position = [this](char *next) -> pos_type {
|
||||
if ((next <= egptr()) && (next >= eback())) {
|
||||
setg(eback(), next, reinterpret_cast<char *>(reader_.get_total_size()));
|
||||
return pos_type(reinterpret_cast<std::uintptr_t>(gptr()));
|
||||
}
|
||||
return pos_type(traits_type::eof());
|
||||
};
|
||||
|
||||
switch (dir) {
|
||||
case std::ios_base::beg:
|
||||
return set_position(eback() + off);
|
||||
|
||||
case std::ios_base::cur:
|
||||
return set_position(gptr() + off);
|
||||
|
||||
case std::ios_base::end:
|
||||
return set_position(egptr() + off);
|
||||
}
|
||||
|
||||
return encrypting_reader::streambuf::seekoff(off, dir, which);
|
||||
}
|
||||
|
||||
pos_type seekpos(pos_type pos, std::ios_base::openmode which = std::ios_base::out |
|
||||
std::ios_base::in) override {
|
||||
return seekoff(pos, std::ios_base::beg, which);
|
||||
}
|
||||
|
||||
int_type uflow() override {
|
||||
auto ret = underflow();
|
||||
if (ret == traits_type::eof()) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
gbump(1);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
int_type underflow() override {
|
||||
if (gptr() == egptr()) {
|
||||
return traits_type::eof();
|
||||
}
|
||||
|
||||
reader_.set_read_position(static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(gptr())));
|
||||
|
||||
char c{};
|
||||
const auto res = reader_.reader_function(&c, 1u, 1u, &reader_);
|
||||
if (res != 1) {
|
||||
return traits_type::eof();
|
||||
}
|
||||
|
||||
return c;
|
||||
}
|
||||
|
||||
std::streamsize xsgetn(char *ptr, std::streamsize count) override {
|
||||
if (gptr() == egptr()) {
|
||||
return traits_type::eof();
|
||||
}
|
||||
|
||||
reader_.set_read_position(static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(gptr())));
|
||||
|
||||
const auto res = reader_.reader_function(ptr, 1u, count, &reader_);
|
||||
if ((res == reader_.get_error_return()) ||
|
||||
(reader_.get_stop_requested() && (res == CURL_READFUNC_ABORT))) {
|
||||
return traits_type::eof();
|
||||
}
|
||||
|
||||
setg(eback(), gptr() + res, reinterpret_cast<char *>(reader_.get_total_size()));
|
||||
return res;
|
||||
}
|
||||
};
|
||||
|
||||
class encrypting_reader_iostream final : public encrypting_reader::iostream {
|
||||
public:
|
||||
explicit encrypting_reader_iostream(std::unique_ptr<encrypting_streambuf> buffer)
|
||||
: encrypting_reader::iostream(buffer.get()), buffer_(std::move(buffer)) {}
|
||||
|
||||
~encrypting_reader_iostream() override = default;
|
||||
|
||||
private:
|
||||
std::unique_ptr<encrypting_streambuf> buffer_;
|
||||
};
|
||||
|
||||
const std::size_t encrypting_reader::header_size_ = ([]() {
|
||||
CryptoPP::XChaCha20Poly1305::Encryption enc;
|
||||
return enc.IVSize() + enc.DigestSize();
|
||||
})();
|
||||
const std::size_t encrypting_reader::data_chunk_size_ = (8u * 1024u * 1024u);
|
||||
const std::size_t encrypting_reader::encrypted_chunk_size_ = data_chunk_size_ + header_size_;
|
||||
|
||||
encrypting_reader::encrypting_reader(const std::string &file_name, const std::string &source_path,
|
||||
const bool &stop_requested, const std::string &token,
|
||||
const size_t error_return)
|
||||
: key_(utils::encryption::generate_key(token)),
|
||||
stop_requested_(stop_requested),
|
||||
error_return_(error_return) {
|
||||
const auto res = native_file::open(source_path, source_file_);
|
||||
if (res != api_error::success) {
|
||||
throw std::runtime_error("file open failed: " + source_path + ':' + api_error_to_string(res));
|
||||
}
|
||||
|
||||
std::vector<char> result;
|
||||
utils::encryption::encrypt_data(key_, file_name.c_str(),
|
||||
strnlen(file_name.c_str(), file_name.size()), result);
|
||||
encrypted_file_name_ = utils::to_hex_string(result);
|
||||
|
||||
std::uint64_t file_size{};
|
||||
if (not utils::file::get_file_size(source_path, file_size)) {
|
||||
throw std::runtime_error("get file size failed: " + source_path + ':' +
|
||||
std::to_string(utils::get_last_error_code()));
|
||||
}
|
||||
|
||||
const auto total_chunks = static_cast<std::size_t>(
|
||||
utils::divide_with_ceiling(file_size, static_cast<std::uint64_t>(data_chunk_size_)));
|
||||
total_size_ = file_size + (total_chunks * encrypting_reader::get_header_size());
|
||||
last_data_chunk_ = total_chunks - 1u;
|
||||
last_data_chunk_size_ =
|
||||
static_cast<std::size_t>((file_size <= data_chunk_size_) ? file_size
|
||||
: (file_size % data_chunk_size_) ? file_size % data_chunk_size_
|
||||
: data_chunk_size_);
|
||||
}
|
||||
|
||||
encrypting_reader::encrypting_reader(const encrypting_reader &r)
|
||||
: key_(r.key_),
|
||||
stop_requested_(r.stop_requested_),
|
||||
error_return_(r.error_return_),
|
||||
chunk_buffers_(r.chunk_buffers_),
|
||||
encrypted_file_name_(r.encrypted_file_name_),
|
||||
last_data_chunk_(r.last_data_chunk_),
|
||||
last_data_chunk_size_(r.last_data_chunk_size_),
|
||||
read_offset_(r.read_offset_),
|
||||
source_file_(native_file::clone(r.source_file_)),
|
||||
total_size_(r.total_size_) {}
|
||||
|
||||
encrypting_reader::~encrypting_reader() {
|
||||
if (source_file_) {
|
||||
source_file_->close();
|
||||
}
|
||||
}
|
||||
|
||||
std::uint64_t encrypting_reader::calculate_decrypted_size(const std::uint64_t &total_size) {
|
||||
return total_size - (utils::divide_with_ceiling(
|
||||
total_size, static_cast<std::uint64_t>(get_encrypted_chunk_size())) *
|
||||
get_header_size());
|
||||
}
|
||||
|
||||
std::shared_ptr<encrypting_reader::iostream> encrypting_reader::create_iostream() const {
|
||||
return std::make_shared<encrypting_reader_iostream>(
|
||||
std::make_unique<encrypting_streambuf>(*this));
|
||||
}
|
||||
|
||||
size_t encrypting_reader::reader_function(char *buffer, size_t size, size_t nitems) {
|
||||
const auto read_size = static_cast<std::size_t>(
|
||||
std::min(static_cast<std::uint64_t>(size * nitems), total_size_ - read_offset_));
|
||||
|
||||
auto chunk = static_cast<std::size_t>(read_offset_ / encrypted_chunk_size_);
|
||||
auto chunk_offset = static_cast<std::size_t>(read_offset_ % encrypted_chunk_size_);
|
||||
std::size_t total_read = 0u;
|
||||
|
||||
auto ret = false;
|
||||
if (read_offset_ < total_size_) {
|
||||
try {
|
||||
ret = true;
|
||||
auto remain = read_size;
|
||||
while (not stop_requested_ && ret && remain) {
|
||||
if (chunk_buffers_.find(chunk) == chunk_buffers_.end()) {
|
||||
auto &chunk_buffer = chunk_buffers_[chunk];
|
||||
std::vector<char> file_data(chunk == last_data_chunk_ ? last_data_chunk_size_
|
||||
: data_chunk_size_);
|
||||
chunk_buffer.resize(file_data.size() + get_header_size());
|
||||
|
||||
std::size_t bytes_read{};
|
||||
if ((ret = source_file_->read_bytes(&file_data[0u], file_data.size(),
|
||||
chunk * data_chunk_size_, bytes_read))) {
|
||||
utils::encryption::encrypt_data(key_, file_data, chunk_buffer);
|
||||
}
|
||||
} else if (chunk) {
|
||||
chunk_buffers_.erase(chunk - 1u);
|
||||
}
|
||||
|
||||
auto &chunk_buffer = chunk_buffers_[chunk];
|
||||
const auto to_read = std::min(chunk_buffer.size() - chunk_offset, remain);
|
||||
std::memcpy(buffer + total_read, &chunk_buffer[chunk_offset], to_read);
|
||||
total_read += to_read;
|
||||
remain -= to_read;
|
||||
chunk_offset = 0u;
|
||||
chunk++;
|
||||
read_offset_ += to_read;
|
||||
}
|
||||
} catch (const std::exception &e) {
|
||||
event_system::instance().raise<repertory_exception>(__FUNCTION__,
|
||||
e.what() ? e.what() : "unkown error");
|
||||
ret = false;
|
||||
}
|
||||
}
|
||||
|
||||
return stop_requested_ ? CURL_READFUNC_ABORT : ret ? total_read : error_return_;
|
||||
}
|
||||
} // namespace repertory::utils::encryption
|
||||
95
src/utils/encryption.cpp
Normal file
95
src/utils/encryption.cpp
Normal file
@@ -0,0 +1,95 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "utils/encryption.hpp"
|
||||
#include "events/events.hpp"
|
||||
#include "events/event_system.hpp"
|
||||
#include "utils/encrypting_reader.hpp"
|
||||
#include "utils/utils.hpp"
|
||||
|
||||
namespace repertory::utils::encryption {
|
||||
api_error decrypt_file_name(const std::string &encryption_token, std::string &file_name) {
|
||||
std::vector<char> buffer;
|
||||
if (not utils::from_hex_string(file_name, buffer)) {
|
||||
return api_error::error;
|
||||
}
|
||||
|
||||
file_name.clear();
|
||||
if (not utils::encryption::decrypt_data(encryption_token, buffer, file_name)) {
|
||||
return api_error::decryption_error;
|
||||
}
|
||||
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
CryptoPP::SecByteBlock generate_key(const std::string &encryption_token) {
|
||||
CryptoPP::SecByteBlock key(CryptoPP::SHA256::DIGESTSIZE);
|
||||
CryptoPP::SHA256().CalculateDigest(
|
||||
reinterpret_cast<CryptoPP::byte *>(&key[0u]),
|
||||
reinterpret_cast<const CryptoPP::byte *>(encryption_token.c_str()),
|
||||
strnlen(encryption_token.c_str(), encryption_token.size()));
|
||||
|
||||
return key;
|
||||
}
|
||||
|
||||
api_error read_encrypted_range(
|
||||
const http_range &range, const CryptoPP::SecByteBlock &key,
|
||||
const std::function<api_error(std::vector<char> &ct, const std::uint64_t &start_offset,
|
||||
const std::uint64_t &end_offset)> &reader,
|
||||
const std::uint64_t &total_size, std::vector<char> &data) {
|
||||
static constexpr const auto &encrypted_chunk_size =
|
||||
utils::encryption::encrypting_reader::get_encrypted_chunk_size();
|
||||
static constexpr const auto &data_chunk_size =
|
||||
utils::encryption::encrypting_reader::get_data_chunk_size();
|
||||
static constexpr const auto &header_size =
|
||||
utils::encryption::encrypting_reader::get_header_size();
|
||||
|
||||
const auto start_chunk = static_cast<std::size_t>(range.begin / data_chunk_size);
|
||||
const auto end_chunk = static_cast<std::size_t>(range.end / data_chunk_size);
|
||||
auto remain = range.end - range.begin + 1u;
|
||||
auto source_offset = static_cast<std::size_t>(range.begin % data_chunk_size);
|
||||
|
||||
for (std::size_t chunk = start_chunk; chunk <= end_chunk; chunk++) {
|
||||
std::vector<char> ct;
|
||||
const auto start_offset = chunk * encrypted_chunk_size;
|
||||
const auto end_offset =
|
||||
std::min(start_offset + (total_size - (chunk * data_chunk_size)) + header_size - 1u,
|
||||
static_cast<std::uint64_t>(start_offset + encrypted_chunk_size - 1u));
|
||||
|
||||
const auto result = reader(ct, start_offset, end_offset);
|
||||
if (result != api_error::success) {
|
||||
return result;
|
||||
}
|
||||
|
||||
std::vector<char> source_buffer;
|
||||
if (not utils::encryption::decrypt_data(key, ct, source_buffer)) {
|
||||
return api_error::decryption_error;
|
||||
}
|
||||
ct.clear();
|
||||
|
||||
const auto data_size = static_cast<std::size_t>(
|
||||
std::min(remain, static_cast<std::uint64_t>(data_chunk_size - source_offset)));
|
||||
std::copy(source_buffer.begin() + source_offset,
|
||||
source_buffer.begin() + source_offset + data_size, std::back_inserter(data));
|
||||
remain -= data_size;
|
||||
source_offset = 0u;
|
||||
}
|
||||
|
||||
return api_error::success;
|
||||
}
|
||||
} // namespace repertory::utils::encryption
|
||||
612
src/utils/file_utils.cpp
Normal file
612
src/utils/file_utils.cpp
Normal file
@@ -0,0 +1,612 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "utils/file_utils.hpp"
|
||||
#include "utils/path_utils.hpp"
|
||||
#include "utils/string_utils.hpp"
|
||||
#include "utils/utils.hpp"
|
||||
|
||||
namespace repertory::utils::file {
|
||||
api_error assign_and_get_native_file(filesystem_item &fi, native_file_ptr &nf) {
|
||||
recur_mutex_lock l(*fi.lock);
|
||||
if (fi.handle == REPERTORY_INVALID_HANDLE) {
|
||||
const auto res = native_file::create_or_open(fi.source_path, nf);
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
|
||||
fi.handle = nf->get_handle();
|
||||
} else {
|
||||
nf = native_file::attach(fi.handle);
|
||||
}
|
||||
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
std::uint64_t calculate_used_space(std::string path, const bool &recursive) {
|
||||
path = utils::path::absolute(path);
|
||||
std::uint64_t ret = 0u;
|
||||
#ifdef _WIN32
|
||||
WIN32_FIND_DATA fd = {0};
|
||||
const auto search = utils::path::combine(path, {"*.*"});
|
||||
auto find = ::FindFirstFile(search.c_str(), &fd);
|
||||
if (find != INVALID_HANDLE_VALUE) {
|
||||
do {
|
||||
const auto file_name = std::string(fd.cFileName);
|
||||
if (fd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) {
|
||||
if (recursive && (file_name != ".") && (file_name != "..")) {
|
||||
ret += calculate_used_space(utils::path::combine(path, {file_name}), recursive);
|
||||
}
|
||||
} else {
|
||||
std::uint64_t file_size{};
|
||||
if (get_file_size(utils::path::combine(path, {file_name}), file_size)) {
|
||||
ret += file_size;
|
||||
}
|
||||
}
|
||||
} while (::FindNextFile(find, &fd) != 0);
|
||||
::FindClose(find);
|
||||
}
|
||||
#else
|
||||
auto *root = opendir(path.c_str());
|
||||
if (root) {
|
||||
struct dirent *de{};
|
||||
while ((de = readdir(root)) != nullptr) {
|
||||
if (de->d_type == DT_DIR) {
|
||||
if (recursive && (strcmp(de->d_name, ".") != 0) && (strcmp(de->d_name, "..") != 0)) {
|
||||
ret += calculate_used_space(utils::path::combine(path, {de->d_name}), recursive);
|
||||
}
|
||||
} else {
|
||||
std::uint64_t file_size{};
|
||||
if (get_file_size(utils::path::combine(path, {de->d_name}), file_size)) {
|
||||
ret += file_size;
|
||||
}
|
||||
}
|
||||
}
|
||||
closedir(root);
|
||||
}
|
||||
#endif
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
void change_to_process_directory() {
|
||||
#ifdef _WIN32
|
||||
std::string file_name;
|
||||
file_name.resize(MAX_PATH);
|
||||
::GetModuleFileNameA(nullptr, &file_name[0u], static_cast<DWORD>(file_name.size()));
|
||||
|
||||
std::string path = file_name.c_str();
|
||||
::PathRemoveFileSpecA(&path[0u]);
|
||||
::SetCurrentDirectoryA(&path[0u]);
|
||||
#else
|
||||
std::string path;
|
||||
path.resize(PATH_MAX + 1);
|
||||
#ifdef __APPLE__
|
||||
proc_pidpath(getpid(), &path[0u], path.size());
|
||||
#else
|
||||
readlink("/proc/self/exe", &path[0u], path.size());
|
||||
#endif
|
||||
path = utils::path::get_parent_directory(path);
|
||||
chdir(path.c_str());
|
||||
#endif
|
||||
}
|
||||
|
||||
bool copy_file(std::string from_path, std::string to_path) {
|
||||
from_path = utils::path::absolute(from_path);
|
||||
to_path = utils::path::absolute(to_path);
|
||||
if (is_file(from_path) && not is_directory(to_path)) {
|
||||
boost::system::error_code ec{};
|
||||
boost::filesystem::copy_file(from_path, to_path, ec);
|
||||
return not ec.failed();
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
bool copy_directory_recursively(std::string from_path, std::string to_path) {
|
||||
from_path = utils::path::absolute(from_path);
|
||||
to_path = utils::path::absolute(to_path);
|
||||
auto ret = create_full_directory_path(to_path);
|
||||
if (ret) {
|
||||
#ifdef _WIN32
|
||||
WIN32_FIND_DATA fd = {0};
|
||||
const auto search = utils::path::combine(from_path, {"*.*"});
|
||||
auto find = ::FindFirstFile(search.c_str(), &fd);
|
||||
if (find != INVALID_HANDLE_VALUE) {
|
||||
ret = true;
|
||||
do {
|
||||
if (fd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) {
|
||||
if ((std::string(fd.cFileName) != ".") && (std::string(fd.cFileName) != "..")) {
|
||||
ret = copy_directory_recursively(utils::path::combine(from_path, {fd.cFileName}),
|
||||
utils::path::combine(to_path, {fd.cFileName}));
|
||||
}
|
||||
} else {
|
||||
ret = copy_file(utils::path::combine(from_path, {fd.cFileName}),
|
||||
utils::path::combine(to_path, {fd.cFileName}));
|
||||
}
|
||||
} while (ret && (::FindNextFile(find, &fd) != 0));
|
||||
|
||||
::FindClose(find);
|
||||
}
|
||||
#else
|
||||
auto *root = opendir(from_path.c_str());
|
||||
if (root) {
|
||||
struct dirent *de{};
|
||||
while (ret && (de = readdir(root))) {
|
||||
if (de->d_type == DT_DIR) {
|
||||
if ((strcmp(de->d_name, ".") != 0) && (strcmp(de->d_name, "..") != 0)) {
|
||||
ret = copy_directory_recursively(utils::path::combine(from_path, {de->d_name}),
|
||||
utils::path::combine(to_path, {de->d_name}));
|
||||
}
|
||||
} else {
|
||||
ret = copy_file(utils::path::combine(from_path, {de->d_name}),
|
||||
utils::path::combine(to_path, {de->d_name}));
|
||||
}
|
||||
}
|
||||
|
||||
closedir(root);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool create_full_directory_path(std::string path) {
|
||||
#ifdef _WIN32
|
||||
const auto unicode_path = utils::string::from_utf8(utils::path::absolute(path));
|
||||
return is_directory(path) ||
|
||||
(::SHCreateDirectory(nullptr, unicode_path.c_str()) == ERROR_SUCCESS);
|
||||
#else
|
||||
auto ret = true;
|
||||
const auto paths =
|
||||
utils::string::split(utils::path::absolute(path), utils::path::directory_seperator[0u]);
|
||||
std::string current_path;
|
||||
for (std::size_t i = 0u; ret && (i < paths.size()); i++) {
|
||||
if (paths[i].empty()) { // Skip root
|
||||
current_path = utils::path::directory_seperator;
|
||||
} else {
|
||||
current_path = utils::path::combine(current_path, {paths[i]});
|
||||
const auto status = mkdir(current_path.c_str(), S_IRWXU);
|
||||
ret = ((status == 0) || (errno == EEXIST));
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
#endif
|
||||
}
|
||||
|
||||
bool delete_directory(std::string path, const bool &recursive) {
|
||||
if (recursive) {
|
||||
return delete_directory_recursively(path);
|
||||
}
|
||||
|
||||
path = utils::path::absolute(path);
|
||||
#ifdef _WIN32
|
||||
return (not is_directory(path) ||
|
||||
utils::retryable_action([&]() -> bool { return !!::RemoveDirectoryA(path.c_str()); }));
|
||||
#else
|
||||
return not is_directory(path) || (rmdir(path.c_str()) == 0);
|
||||
#endif
|
||||
}
|
||||
|
||||
bool delete_directory_recursively(std::string path) {
|
||||
path = utils::path::absolute(path);
|
||||
#ifdef _WIN32
|
||||
|
||||
WIN32_FIND_DATA fd = {0};
|
||||
const auto search = utils::path::combine(path, {"*.*"});
|
||||
auto find = ::FindFirstFile(search.c_str(), &fd);
|
||||
if (find != INVALID_HANDLE_VALUE) {
|
||||
auto res = false;
|
||||
do {
|
||||
if (fd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) {
|
||||
if ((std::string(fd.cFileName) != ".") && (std::string(fd.cFileName) != "..")) {
|
||||
res = delete_directory_recursively(utils::path::combine(path, {fd.cFileName}));
|
||||
}
|
||||
} else {
|
||||
res = delete_file(utils::path::combine(path, {fd.cFileName}));
|
||||
}
|
||||
} while (res && (::FindNextFile(find, &fd) != 0));
|
||||
|
||||
::FindClose(find);
|
||||
}
|
||||
#else
|
||||
auto *root = opendir(path.c_str());
|
||||
if (root) {
|
||||
auto res = true;
|
||||
struct dirent *de{};
|
||||
while (res && (de = readdir(root))) {
|
||||
if (de->d_type == DT_DIR) {
|
||||
if ((strcmp(de->d_name, ".") != 0) && (strcmp(de->d_name, "..") != 0)) {
|
||||
res = delete_directory_recursively(utils::path::combine(path, {de->d_name}));
|
||||
}
|
||||
} else {
|
||||
res = delete_file(utils::path::combine(path, {de->d_name}));
|
||||
}
|
||||
}
|
||||
|
||||
closedir(root);
|
||||
}
|
||||
#endif
|
||||
|
||||
return delete_directory(path, false);
|
||||
}
|
||||
|
||||
bool delete_file(std::string path) {
|
||||
path = utils::path::absolute(path);
|
||||
#ifdef _WIN32
|
||||
return (not is_file(path) || utils::retryable_action([&]() -> bool {
|
||||
const auto ret = !!::DeleteFileA(path.c_str());
|
||||
if (not ret) {
|
||||
std::cout << "delete failed:" << path << std::endl;
|
||||
}
|
||||
|
||||
return ret;
|
||||
}));
|
||||
#else
|
||||
return (not is_file(path) || (unlink(path.c_str()) == 0));
|
||||
#endif
|
||||
}
|
||||
|
||||
std::string generate_sha256(const std::string &file_path) {
|
||||
std::string value;
|
||||
CryptoPP::SHA256 hash;
|
||||
CryptoPP::FileSource(
|
||||
file_path.c_str(), true,
|
||||
new CryptoPP::HashFilter(hash, new CryptoPP::HexEncoder(new CryptoPP::StringSink(value))));
|
||||
return value;
|
||||
}
|
||||
|
||||
std::uint64_t get_available_drive_space(const std::string &path) {
|
||||
#ifdef _WIN32
|
||||
ULARGE_INTEGER li = {0};
|
||||
::GetDiskFreeSpaceEx(path.c_str(), &li, nullptr, nullptr);
|
||||
return li.QuadPart;
|
||||
#endif
|
||||
#ifdef __linux__
|
||||
std::uint64_t ret = 0;
|
||||
struct statfs64 st {};
|
||||
if (statfs64(path.c_str(), &st) == 0) {
|
||||
ret = st.f_bfree * st.f_bsize;
|
||||
}
|
||||
return ret;
|
||||
#endif
|
||||
#if __APPLE__
|
||||
struct statvfs st {};
|
||||
statvfs(path.c_str(), &st);
|
||||
return st.f_bavail * st.f_frsize;
|
||||
#endif
|
||||
}
|
||||
|
||||
std::deque<std::string> get_directory_files(std::string path, const bool &oldest_first,
|
||||
const bool &recursive) {
|
||||
path = utils::path::absolute(path);
|
||||
std::deque<std::string> ret;
|
||||
std::unordered_map<std::string, std::uint64_t> lookup;
|
||||
#ifdef _WIN32
|
||||
WIN32_FIND_DATA fd = {0};
|
||||
const auto search = utils::path::combine(path, {"*.*"});
|
||||
auto find = ::FindFirstFile(search.c_str(), &fd);
|
||||
if (find != INVALID_HANDLE_VALUE) {
|
||||
do {
|
||||
if ((fd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != FILE_ATTRIBUTE_DIRECTORY) {
|
||||
ULARGE_INTEGER li{};
|
||||
li.HighPart = fd.ftLastWriteTime.dwHighDateTime;
|
||||
li.LowPart = fd.ftLastWriteTime.dwLowDateTime;
|
||||
const auto full_path = utils::path::combine(path, {fd.cFileName});
|
||||
lookup[full_path] = li.QuadPart;
|
||||
ret.emplace_back(full_path);
|
||||
}
|
||||
} while (::FindNextFile(find, &fd) != 0);
|
||||
::FindClose(find);
|
||||
|
||||
std::sort(ret.begin(), ret.end(), [&](const auto &p1, const auto &p2) -> bool {
|
||||
return (oldest_first != 0) == (lookup[p1] < lookup[p2]);
|
||||
});
|
||||
}
|
||||
#else
|
||||
auto *root = opendir(path.c_str());
|
||||
if (root) {
|
||||
struct dirent *de{};
|
||||
while ((de = readdir(root)) != nullptr) {
|
||||
if (de->d_type == DT_DIR) {
|
||||
if (recursive) {
|
||||
const auto subFiles = get_directory_files(utils::path::combine(path, {de->d_name}),
|
||||
oldest_first, recursive);
|
||||
ret.insert(ret.end(), subFiles.begin(), subFiles.end());
|
||||
}
|
||||
} else {
|
||||
ret.emplace_back(utils::path::combine(path, {de->d_name}));
|
||||
}
|
||||
}
|
||||
closedir(root);
|
||||
|
||||
const auto add_to_lookup = [&](const std::string &lookup_path) {
|
||||
if (lookup.find(lookup_path) == lookup.end()) {
|
||||
struct stat st {};
|
||||
stat(lookup_path.c_str(), &st);
|
||||
#ifdef __APPLE__
|
||||
lookup[lookup_path] = static_cast<std::uint64_t>(
|
||||
(st.st_mtimespec.tv_sec * NANOS_PER_SECOND) + st.st_mtimespec.tv_nsec);
|
||||
#else
|
||||
lookup[lookup_path] =
|
||||
static_cast<std::uint64_t>((st.st_mtim.tv_sec * NANOS_PER_SECOND) + st.st_mtim.tv_nsec);
|
||||
#endif
|
||||
}
|
||||
};
|
||||
|
||||
std::sort(ret.begin(), ret.end(), [&](const auto &p1, const auto &p2) -> bool {
|
||||
add_to_lookup(p1);
|
||||
add_to_lookup(p2);
|
||||
return (oldest_first != 0) == (lookup[p1] < lookup[p2]);
|
||||
});
|
||||
}
|
||||
#endif
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool get_accessed_time(const std::string &path, std::uint64_t &accessed) {
|
||||
auto ret = false;
|
||||
accessed = 0;
|
||||
#ifdef _WIN32
|
||||
struct _stat64 st {};
|
||||
if (_stat64(path.c_str(), &st) != -1) {
|
||||
accessed = static_cast<uint64_t>(st.st_atime);
|
||||
#else
|
||||
struct stat st {};
|
||||
if (stat(path.c_str(), &st) != -1) {
|
||||
#ifdef __APPLE__
|
||||
accessed = static_cast<uint64_t>(st.st_atimespec.tv_nsec +
|
||||
(st.st_atimespec.tv_sec * NANOS_PER_SECOND));
|
||||
#else
|
||||
accessed = static_cast<uint64_t>(st.st_atim.tv_nsec + (st.st_atim.tv_sec * NANOS_PER_SECOND));
|
||||
#endif
|
||||
#endif
|
||||
ret = true;
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool get_modified_time(const std::string &path, std::uint64_t &modified) {
|
||||
auto ret = false;
|
||||
modified = 0u;
|
||||
#ifdef _WIN32
|
||||
struct _stat64 st {};
|
||||
if (_stat64(path.c_str(), &st) != -1) {
|
||||
modified = static_cast<uint64_t>(st.st_mtime);
|
||||
#else
|
||||
struct stat st {};
|
||||
if (stat(path.c_str(), &st) != -1) {
|
||||
#ifdef __APPLE__
|
||||
modified = static_cast<uint64_t>(st.st_mtimespec.tv_nsec +
|
||||
(st.st_mtimespec.tv_sec * NANOS_PER_SECOND));
|
||||
#else
|
||||
modified = static_cast<uint64_t>(st.st_mtim.tv_nsec + (st.st_mtim.tv_sec * NANOS_PER_SECOND));
|
||||
#endif
|
||||
#endif
|
||||
ret = true;
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool get_file_size(std::string path, std::uint64_t &file_size) {
|
||||
file_size = 0u;
|
||||
path = utils::path::finalize(path);
|
||||
|
||||
#ifdef _WIN32
|
||||
struct _stat64 st {};
|
||||
if (_stat64(path.c_str(), &st) != 0) {
|
||||
#else
|
||||
#if __APPLE__
|
||||
struct stat st {};
|
||||
if (stat(path.c_str(), &st) != 0) {
|
||||
#else
|
||||
struct stat64 st {};
|
||||
if (stat64(path.c_str(), &st) != 0) {
|
||||
#endif
|
||||
#endif
|
||||
return false;
|
||||
}
|
||||
|
||||
if (st.st_size >= 0) {
|
||||
file_size = static_cast<std::uint64_t>(st.st_size);
|
||||
}
|
||||
|
||||
return (st.st_size >= 0);
|
||||
}
|
||||
|
||||
bool is_directory(const std::string &path) {
|
||||
#ifdef _WIN32
|
||||
return ::PathIsDirectory(path.c_str()) != 0;
|
||||
#else
|
||||
struct stat st {};
|
||||
return (not stat(path.c_str(), &st) && S_ISDIR(st.st_mode));
|
||||
#endif
|
||||
}
|
||||
|
||||
bool is_file(const std::string &path) {
|
||||
#ifdef _WIN32
|
||||
return (::PathFileExists(path.c_str()) && not ::PathIsDirectory(path.c_str()));
|
||||
#else
|
||||
struct stat st {};
|
||||
return (not stat(path.c_str(), &st) && not S_ISDIR(st.st_mode));
|
||||
#endif
|
||||
}
|
||||
|
||||
bool is_modified_date_older_than(const std::string &path, const std::chrono::hours &hours) {
|
||||
auto ret = false;
|
||||
std::uint64_t modified = 0;
|
||||
if (get_modified_time(path, modified)) {
|
||||
const auto seconds = std::chrono::duration_cast<std::chrono::seconds>(hours);
|
||||
#ifdef _WIN32
|
||||
return (std::chrono::system_clock::from_time_t(modified) + seconds) <
|
||||
std::chrono::system_clock::now();
|
||||
#else
|
||||
return (modified + (seconds.count() * NANOS_PER_SECOND)) < utils::get_time_now();
|
||||
#endif
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool move_file(std::string from, std::string to) {
|
||||
from = utils::path::finalize(from);
|
||||
to = utils::path::finalize(to);
|
||||
|
||||
const auto directory = utils::path::remove_file_name(to);
|
||||
create_full_directory_path(directory);
|
||||
#ifdef _WIN32
|
||||
const bool ret = ::MoveFile(from.c_str(), to.c_str()) != 0;
|
||||
#else
|
||||
const bool ret = (rename(from.c_str(), to.c_str()) == 0);
|
||||
#endif
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error read_from_source(filesystem_item &fi, const std::size_t &read_size,
|
||||
const std::uint64_t &read_offset, std::vector<char> &data) {
|
||||
native_file_ptr nf;
|
||||
const auto res = assign_and_get_native_file(fi, nf);
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
|
||||
data.resize(read_size);
|
||||
std::size_t bytes_read = 0u;
|
||||
if (not nf->read_bytes(&data[0u], data.size(), read_offset, bytes_read)) {
|
||||
return api_error::os_error;
|
||||
}
|
||||
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
std::vector<std::string> read_file_lines(const std::string &path) {
|
||||
std::vector<std::string> ret;
|
||||
if (is_file(path)) {
|
||||
std::ifstream fs(path);
|
||||
std::string current_line;
|
||||
while (not fs.eof() && std::getline(fs, current_line)) {
|
||||
ret.emplace_back(utils::string::right_trim(current_line, '\r'));
|
||||
}
|
||||
fs.close();
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool read_json_file(const std::string &path, json &data) {
|
||||
auto ret = false;
|
||||
|
||||
try {
|
||||
std::ifstream file_stream(path.c_str());
|
||||
if (file_stream.is_open()) {
|
||||
std::stringstream ss;
|
||||
ss << file_stream.rdbuf();
|
||||
std::string json_text = ss.str();
|
||||
if (json_text.empty()) {
|
||||
file_stream.close();
|
||||
} else {
|
||||
data = json::parse(json_text.c_str());
|
||||
ret = true;
|
||||
file_stream.close();
|
||||
}
|
||||
}
|
||||
} catch (const std::exception &) {
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool reset_modified_time(const std::string &path) {
|
||||
auto ret = false;
|
||||
#ifdef _WIN32
|
||||
SYSTEMTIME st{};
|
||||
::GetSystemTime(&st);
|
||||
|
||||
FILETIME ft{};
|
||||
if ((ret = !!::SystemTimeToFileTime(&st, &ft))) {
|
||||
auto handle = ::CreateFileA(path.c_str(), FILE_READ_ATTRIBUTES | FILE_WRITE_ATTRIBUTES,
|
||||
FILE_SHARE_DELETE | FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr,
|
||||
OPEN_EXISTING, 0, nullptr);
|
||||
if ((ret = (handle != INVALID_HANDLE_VALUE))) {
|
||||
ret = !!::SetFileTime(handle, nullptr, &ft, &ft);
|
||||
::CloseHandle(handle);
|
||||
}
|
||||
}
|
||||
#else
|
||||
auto fd = open(path.c_str(), O_RDWR);
|
||||
if ((ret = (fd != -1))) {
|
||||
ret = not futimens(fd, nullptr);
|
||||
close(fd);
|
||||
}
|
||||
#endif
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error truncate_source(filesystem_item &fi, const std::uint64_t &size) {
|
||||
native_file_ptr nf;
|
||||
auto res = assign_and_get_native_file(fi, nf);
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
|
||||
if (not nf->truncate(size)) {
|
||||
return api_error::os_error;
|
||||
}
|
||||
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
bool write_json_file(const std::string &path, const json &j) {
|
||||
std::string data;
|
||||
{
|
||||
std::stringstream ss;
|
||||
ss << std::setw(2) << j;
|
||||
data = ss.str();
|
||||
}
|
||||
native_file_ptr nf;
|
||||
auto ret = (native_file::create_or_open(path, nf) == api_error::success);
|
||||
if (ret) {
|
||||
std::size_t bytes_written = 0u;
|
||||
ret = nf->truncate(0) && nf->write_bytes(&data[0u], data.size(), 0u, bytes_written);
|
||||
nf->close();
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
api_error write_to_source(filesystem_item &fi, const std::uint64_t &write_offset,
|
||||
const std::vector<char> &data, std::size_t &bytes_written) {
|
||||
bytes_written = 0u;
|
||||
|
||||
native_file_ptr nf;
|
||||
auto res = assign_and_get_native_file(fi, nf);
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
|
||||
if (not nf->write_bytes(&data[0u], data.size(), write_offset, bytes_written)) {
|
||||
return api_error::os_error;
|
||||
}
|
||||
|
||||
return api_error::success;
|
||||
}
|
||||
} // namespace repertory::utils::file
|
||||
40
src/utils/global_data.cpp
Normal file
40
src/utils/global_data.cpp
Normal file
@@ -0,0 +1,40 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "utils/global_data.hpp"
|
||||
|
||||
namespace repertory {
|
||||
global_data global_data::instance_;
|
||||
|
||||
void global_data::update_used_space(const std::uint64_t &file_size,
|
||||
const std::uint64_t &new_file_size, const bool &cache_only) {
|
||||
if (file_size > new_file_size) {
|
||||
const auto diff = (file_size - new_file_size);
|
||||
used_cache_space_ -= diff;
|
||||
if (not cache_only) {
|
||||
used_drive_space_ -= diff;
|
||||
}
|
||||
} else if (file_size < new_file_size) {
|
||||
const auto diff = (new_file_size - file_size);
|
||||
used_cache_space_ += diff;
|
||||
if (not cache_only) {
|
||||
used_drive_space_ += diff;
|
||||
}
|
||||
}
|
||||
}
|
||||
} // namespace repertory
|
||||
255
src/utils/native_file.cpp
Normal file
255
src/utils/native_file.cpp
Normal file
@@ -0,0 +1,255 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "utils/native_file.hpp"
|
||||
#include "utils/string_utils.hpp"
|
||||
|
||||
namespace repertory {
|
||||
native_file_ptr native_file::clone(const native_file_ptr &nf) {
|
||||
std::string source_path;
|
||||
|
||||
#ifdef _WIN32
|
||||
source_path.resize(MAX_PATH + 1);
|
||||
::GetFinalPathNameByHandleA(nf->get_handle(), &source_path[0u], MAX_PATH + 1,
|
||||
FILE_NAME_NORMALIZED | VOLUME_NAME_DOS);
|
||||
#else
|
||||
source_path.resize(PATH_MAX + 1);
|
||||
#ifdef __APPLE__
|
||||
fcntl(nf->get_handle(), F_GETPATH, &source_path[0u]);
|
||||
a
|
||||
#else
|
||||
readlink(("/proc/self/fd/" + utils::string::from_int64(nf->get_handle())).c_str(),
|
||||
&source_path[0u], source_path.size());
|
||||
#endif
|
||||
#endif
|
||||
source_path = source_path.c_str();
|
||||
|
||||
native_file_ptr clone;
|
||||
if (native_file::open(source_path, clone) != api_error::success) {
|
||||
throw std::runtime_error("unable to clone file " + source_path);
|
||||
}
|
||||
|
||||
return clone;
|
||||
}
|
||||
|
||||
api_error native_file::create_or_open(const std::string &source_path, native_file_ptr &nf) {
|
||||
#ifdef _WIN32
|
||||
auto handle = ::CreateFileA(source_path.c_str(), GENERIC_READ | GENERIC_WRITE,
|
||||
FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, OPEN_ALWAYS,
|
||||
FILE_FLAG_RANDOM_ACCESS, nullptr);
|
||||
#else
|
||||
auto handle = ::open(source_path.c_str(), O_CREAT | O_RDWR | O_CLOEXEC, 0600u);
|
||||
chmod(source_path.c_str(), 0600u);
|
||||
#endif
|
||||
nf = native_file::attach(handle);
|
||||
return ((handle == REPERTORY_INVALID_HANDLE) ? api_error::os_error : api_error::success);
|
||||
}
|
||||
|
||||
api_error native_file::open(const std::string &source_path, native_file_ptr &nf) {
|
||||
#ifdef _WIN32
|
||||
auto handle = ::CreateFileA(source_path.c_str(), GENERIC_READ | GENERIC_WRITE,
|
||||
FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, OPEN_EXISTING,
|
||||
FILE_FLAG_RANDOM_ACCESS, nullptr);
|
||||
#else
|
||||
auto handle = ::open(source_path.c_str(), O_RDWR | O_CLOEXEC, 0600u);
|
||||
chmod(source_path.c_str(), 0600u);
|
||||
#endif
|
||||
nf = native_file::attach(handle);
|
||||
return ((handle == REPERTORY_INVALID_HANDLE) ? api_error::os_error : api_error::success);
|
||||
}
|
||||
|
||||
bool native_file::allocate(const std::uint64_t &file_size) {
|
||||
#ifdef _WIN32
|
||||
LARGE_INTEGER li{};
|
||||
li.QuadPart = file_size;
|
||||
return (::SetFilePointerEx(handle_, li, nullptr, FILE_BEGIN) && ::SetEndOfFile(handle_));
|
||||
#endif
|
||||
#ifdef __linux__
|
||||
return (fallocate(handle_, 0, 0, file_size) >= 0);
|
||||
#endif
|
||||
#ifdef __APPLE__
|
||||
return (ftruncate(handle_, file_size) >= 0);
|
||||
#endif
|
||||
}
|
||||
|
||||
void native_file::close() {
|
||||
if (handle_ != REPERTORY_INVALID_HANDLE) {
|
||||
#ifdef WIN32
|
||||
::CloseHandle(handle_);
|
||||
#else
|
||||
::close(handle_);
|
||||
#endif
|
||||
handle_ = REPERTORY_INVALID_HANDLE;
|
||||
}
|
||||
}
|
||||
|
||||
bool native_file::copy_from(const native_file_ptr &nf) {
|
||||
auto ret = false;
|
||||
std::uint64_t file_size = 0u;
|
||||
if ((ret = nf->get_file_size(file_size))) {
|
||||
std::vector<char> buffer;
|
||||
buffer.resize(65536u * 2u);
|
||||
std::uint64_t offset = 0u;
|
||||
while (ret && (file_size > 0u)) {
|
||||
std::size_t bytes_read{};
|
||||
if ((ret = nf->read_bytes(&buffer[0u], buffer.size(), offset, bytes_read))) {
|
||||
std::size_t bytes_written = 0u;
|
||||
ret = write_bytes(&buffer[0u], bytes_read, offset, bytes_written);
|
||||
file_size -= bytes_read;
|
||||
offset += bytes_read;
|
||||
}
|
||||
}
|
||||
flush();
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool native_file::copy_from(const std::string &path) {
|
||||
auto ret = false;
|
||||
native_file_ptr nf;
|
||||
if (native_file::create_or_open(path, nf) == api_error ::success) {
|
||||
ret = copy_from(nf);
|
||||
nf->close();
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
void native_file::flush() {
|
||||
#ifdef _WIN32
|
||||
recur_mutex_lock l(read_write_mutex_);
|
||||
::FlushFileBuffers(handle_);
|
||||
#else
|
||||
fsync(handle_);
|
||||
#endif
|
||||
}
|
||||
|
||||
bool native_file::get_file_size(std::uint64_t &file_size) {
|
||||
auto ret = false;
|
||||
#ifdef _WIN32
|
||||
LARGE_INTEGER li{};
|
||||
if ((ret = ::GetFileSizeEx(handle_, &li) && (li.QuadPart >= 0))) {
|
||||
file_size = static_cast<std::uint64_t>(li.QuadPart);
|
||||
}
|
||||
#else
|
||||
#if __APPLE__
|
||||
struct stat st {};
|
||||
if (fstat(handle_, &st) >= 0) {
|
||||
#else
|
||||
struct stat64 st {};
|
||||
if (fstat64(handle_, &st) >= 0) {
|
||||
#endif
|
||||
if ((ret = (st.st_size >= 0))) {
|
||||
file_size = static_cast<uint64_t>(st.st_size);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
return ret;
|
||||
}
|
||||
|
||||
#ifdef _WIN32
|
||||
bool native_file::read_bytes(char *buffer, const std::size_t &read_size,
|
||||
const std::uint64_t &read_offset, std::size_t &bytes_read) {
|
||||
recur_mutex_lock l(read_write_mutex_);
|
||||
|
||||
auto ret = false;
|
||||
bytes_read = 0u;
|
||||
LARGE_INTEGER li{};
|
||||
li.QuadPart = read_offset;
|
||||
if ((ret = !!::SetFilePointerEx(handle_, li, nullptr, FILE_BEGIN))) {
|
||||
DWORD current_read = 0u;
|
||||
do {
|
||||
current_read = 0u;
|
||||
ret = !!::ReadFile(handle_, &buffer[bytes_read], static_cast<DWORD>(read_size - bytes_read),
|
||||
¤t_read, nullptr);
|
||||
bytes_read += current_read;
|
||||
} while (ret && (bytes_read < read_size) && (current_read != 0));
|
||||
}
|
||||
|
||||
if (ret && (read_size != bytes_read)) {
|
||||
::SetLastError(ERROR_HANDLE_EOF);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
#else
|
||||
bool native_file::read_bytes(char *buffer, const std::size_t &read_size,
|
||||
const std::uint64_t &read_offset, std::size_t &bytes_read) {
|
||||
bytes_read = 0u;
|
||||
ssize_t result = 0;
|
||||
do {
|
||||
result =
|
||||
pread64(handle_, &buffer[bytes_read], read_size - bytes_read, read_offset + bytes_read);
|
||||
if (result > 0) {
|
||||
bytes_read += static_cast<size_t>(result);
|
||||
}
|
||||
} while ((result > 0) && (bytes_read < read_size));
|
||||
|
||||
return (result >= 0);
|
||||
}
|
||||
#endif
|
||||
bool native_file::truncate(const std::uint64_t &file_size) {
|
||||
#ifdef _WIN32
|
||||
recur_mutex_lock l(read_write_mutex_);
|
||||
LARGE_INTEGER li{};
|
||||
li.QuadPart = file_size;
|
||||
return (::SetFilePointerEx(handle_, li, nullptr, FILE_BEGIN) && ::SetEndOfFile(handle_));
|
||||
#else
|
||||
return (ftruncate(handle_, file_size) >= 0);
|
||||
#endif
|
||||
}
|
||||
|
||||
#ifdef _WIN32
|
||||
bool native_file::write_bytes(const char *buffer, const std::size_t &write_size,
|
||||
const std::uint64_t &write_offset, std::size_t &bytes_written) {
|
||||
recur_mutex_lock l(read_write_mutex_);
|
||||
|
||||
bytes_written = 0u;
|
||||
auto ret = true;
|
||||
|
||||
LARGE_INTEGER li{};
|
||||
li.QuadPart = write_offset;
|
||||
if ((ret = !!::SetFilePointerEx(handle_, li, nullptr, FILE_BEGIN))) {
|
||||
do {
|
||||
DWORD current_write = 0u;
|
||||
ret = !!::WriteFile(handle_, &buffer[bytes_written],
|
||||
static_cast<DWORD>(write_size - bytes_written), ¤t_write, nullptr);
|
||||
bytes_written += current_write;
|
||||
} while (ret && (bytes_written < write_size));
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
#else
|
||||
bool native_file::write_bytes(const char *buffer, const std::size_t &write_size,
|
||||
const std::uint64_t &write_offset, std::size_t &bytes_written) {
|
||||
bytes_written = 0;
|
||||
ssize_t result = 0;
|
||||
do {
|
||||
result = pwrite64(handle_, &buffer[bytes_written], write_size - bytes_written,
|
||||
write_offset + bytes_written);
|
||||
if (result > 0) {
|
||||
bytes_written += static_cast<size_t>(result);
|
||||
}
|
||||
} while ((result >= 0) && (bytes_written < write_size));
|
||||
|
||||
return (bytes_written == write_size);
|
||||
}
|
||||
#endif
|
||||
} // namespace repertory
|
||||
181
src/utils/path_utils.cpp
Normal file
181
src/utils/path_utils.cpp
Normal file
@@ -0,0 +1,181 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "utils/path_utils.hpp"
|
||||
#include "types/repertory.hpp"
|
||||
#include "utils/string_utils.hpp"
|
||||
#include "utils/utils.hpp"
|
||||
|
||||
namespace repertory::utils::path {
|
||||
static std::string &format_path(std::string &path, const std::string &sep,
|
||||
const std::string ¬_sep);
|
||||
|
||||
std::string absolute(std::string path) {
|
||||
#ifdef _WIN32
|
||||
if (not path.empty() && ::PathIsRelative(&path[0u])) {
|
||||
std::string temp;
|
||||
temp.resize(MAX_PATH + 1);
|
||||
path = _fullpath(&temp[0u], &path[0u], MAX_PATH);
|
||||
}
|
||||
#else
|
||||
if (not path.empty() && (path[0u] != '/')) {
|
||||
auto found = false;
|
||||
auto tmp = path;
|
||||
do {
|
||||
auto *res = realpath(&tmp[0u], nullptr);
|
||||
if (res) {
|
||||
path = combine(res, {path.substr(tmp.size())});
|
||||
free(res);
|
||||
found = true;
|
||||
} else if (tmp == ".") {
|
||||
found = true;
|
||||
} else {
|
||||
tmp = dirname(&tmp[0u]);
|
||||
}
|
||||
} while (not found);
|
||||
}
|
||||
#endif
|
||||
|
||||
return path;
|
||||
}
|
||||
|
||||
std::string combine(std::string path, const std::vector<std::string> &paths) {
|
||||
return finalize(
|
||||
std::accumulate(paths.begin(), paths.end(), path, [](std::string path, const auto &pathPart) {
|
||||
path += (directory_seperator + pathPart);
|
||||
return path;
|
||||
}));
|
||||
}
|
||||
|
||||
std::string create_api_path(std::string path) {
|
||||
if (path.empty() || (path == ".")) {
|
||||
path = "/";
|
||||
} else {
|
||||
format_path(path, "/", "\\");
|
||||
if (path.find("./") == 0) {
|
||||
path = path.substr(1);
|
||||
}
|
||||
if (path[0u] != '/') {
|
||||
path = "/" + path;
|
||||
}
|
||||
}
|
||||
|
||||
return path;
|
||||
}
|
||||
|
||||
static std::string &format_path(std::string &path, const std::string &sep,
|
||||
const std::string ¬_sep) {
|
||||
std::replace(path.begin(), path.end(), not_sep[0u], sep[0u]);
|
||||
|
||||
while (utils::string::contains(path, sep + sep)) {
|
||||
utils::string::replace(path, sep + sep, sep);
|
||||
}
|
||||
|
||||
return path;
|
||||
}
|
||||
|
||||
std::string finalize(std::string path) {
|
||||
format_path(path, directory_seperator, not_directory_seperator);
|
||||
if ((path.size() > 1) && (path[path.size() - 1] == directory_seperator[0u])) {
|
||||
path = path.substr(0, path.size() - 1);
|
||||
}
|
||||
|
||||
return path;
|
||||
}
|
||||
|
||||
std::string get_parent_api_path(const std::string &path) {
|
||||
std::string ret;
|
||||
if (path != "/") {
|
||||
ret = path.substr(0, path.rfind('/') + 1);
|
||||
if (ret != "/") {
|
||||
ret = utils::string::right_trim(ret, '/');
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
#ifndef _WIN32
|
||||
std::string get_parent_directory(std::string path) {
|
||||
auto ret = std::string(dirname(&path[0u]));
|
||||
if (ret == ".") {
|
||||
ret = "/";
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
#endif
|
||||
|
||||
bool is_ads_file_path(const std::string &path) {
|
||||
#ifdef _WIN32
|
||||
return utils::string::contains(path, ":");
|
||||
#else
|
||||
return false;
|
||||
#endif
|
||||
}
|
||||
|
||||
bool is_trash_directory(std::string path) {
|
||||
path = create_api_path(utils::string::to_lower(path));
|
||||
if (utils::string::begins_with(path, "/.trash-") ||
|
||||
utils::string::begins_with(path, "/.trashes") ||
|
||||
utils::string::begins_with(path, "/$recycle.bin")) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
std::string remove_file_name(std::string path) {
|
||||
path = finalize(path);
|
||||
|
||||
#ifdef _WIN32
|
||||
::PathRemoveFileSpec(&path[0u]);
|
||||
path = path.c_str();
|
||||
#else
|
||||
if (path != "/") {
|
||||
auto i = path.size() - 1;
|
||||
while ((i != 0) && (path[i] != '/')) {
|
||||
i--;
|
||||
}
|
||||
|
||||
path = (i > 0) ? finalize(path.substr(0, i)) : "/";
|
||||
}
|
||||
#endif
|
||||
|
||||
return path;
|
||||
}
|
||||
|
||||
#ifndef _WIN32
|
||||
std::string resolve(std::string path) {
|
||||
struct passwd *pw = getpwuid(getuid());
|
||||
std::string home = (pw->pw_dir ? pw->pw_dir : "");
|
||||
if (home.empty() || ((home == "/") && (getuid() != 0))) {
|
||||
home = combine("/home", {pw->pw_name});
|
||||
}
|
||||
|
||||
return finalize(utils::string::replace(path, "~", home));
|
||||
}
|
||||
#endif
|
||||
|
||||
std::string strip_to_file_name(std::string path) {
|
||||
#ifdef _WIN32
|
||||
return ::PathFindFileName(&path[0u]);
|
||||
#else
|
||||
return utils::string::contains(path, "/") ? basename(&path[0u]) : path;
|
||||
#endif
|
||||
}
|
||||
} // namespace repertory::utils::path
|
||||
91
src/utils/polling.cpp
Normal file
91
src/utils/polling.cpp
Normal file
@@ -0,0 +1,91 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "utils/polling.hpp"
|
||||
#include "app_config.hpp"
|
||||
|
||||
namespace repertory {
|
||||
polling polling::instance_;
|
||||
|
||||
void polling::frequency_thread(std::function<std::uint32_t()> get_frequency_seconds,
|
||||
bool low_frequency) {
|
||||
while (not stop_requested_) {
|
||||
std::deque<std::future<void>> futures;
|
||||
unique_mutex_lock l(mutex_);
|
||||
if (not stop_requested_ && notify_.wait_for(l, std::chrono::seconds(get_frequency_seconds())) ==
|
||||
std::cv_status::timeout) {
|
||||
for (auto &kv : items_) {
|
||||
if (kv.second.low_frequency == low_frequency) {
|
||||
futures.emplace_back(std::async(std::launch::async, [kv]() -> void {
|
||||
event_system::instance().raise<polling_item_begin>(kv.first);
|
||||
kv.second.action();
|
||||
event_system::instance().raise<polling_item_end>(kv.first);
|
||||
}));
|
||||
}
|
||||
}
|
||||
l.unlock();
|
||||
while (not futures.empty()) {
|
||||
futures.front().wait();
|
||||
futures.pop_front();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void polling::remove_callback(const std::string &name) {
|
||||
mutex_lock l(mutex_);
|
||||
items_.erase(name);
|
||||
}
|
||||
|
||||
void polling::set_callback(const polling_item &pollingItem) {
|
||||
mutex_lock l(mutex_);
|
||||
items_[pollingItem.name] = pollingItem;
|
||||
}
|
||||
|
||||
void polling::start(app_config *config) {
|
||||
mutex_lock l(start_stop_mutex_);
|
||||
if (not high_frequency_thread_) {
|
||||
config_ = config;
|
||||
stop_requested_ = false;
|
||||
high_frequency_thread_ = std::make_unique<std::thread>([this]() -> void {
|
||||
this->frequency_thread(
|
||||
[this]() -> std::uint32_t { return config_->get_high_frequency_interval_secs(); }, false);
|
||||
});
|
||||
low_frequency_thread_ = std::make_unique<std::thread>([this]() -> void {
|
||||
this->frequency_thread(
|
||||
[this]() -> std::uint32_t { return config_->get_low_frequency_interval_secs(); }, true);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
void polling::stop() {
|
||||
mutex_lock l(start_stop_mutex_);
|
||||
if (high_frequency_thread_) {
|
||||
event_system::instance().raise<service_shutdown>("polling");
|
||||
{
|
||||
mutex_lock l2(mutex_);
|
||||
stop_requested_ = true;
|
||||
notify_.notify_all();
|
||||
}
|
||||
high_frequency_thread_->join();
|
||||
low_frequency_thread_->join();
|
||||
high_frequency_thread_.reset();
|
||||
low_frequency_thread_.reset();
|
||||
}
|
||||
}
|
||||
} // namespace repertory
|
||||
65
src/utils/rocksdb_utils.cpp
Normal file
65
src/utils/rocksdb_utils.cpp
Normal file
@@ -0,0 +1,65 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "utils/rocksdb_utils.hpp"
|
||||
#include "app_config.hpp"
|
||||
#include "events/event_system.hpp"
|
||||
#include "types/startup_exception.hpp"
|
||||
#include "utils/path_utils.hpp"
|
||||
|
||||
namespace repertory::utils::db {
|
||||
void create_rocksdb(const app_config &config, const std::string &name,
|
||||
std::unique_ptr<rocksdb::DB> &db) {
|
||||
rocksdb::Options options{};
|
||||
options.create_if_missing = true;
|
||||
options.db_log_dir = config.get_log_directory();
|
||||
options.keep_log_file_num = 10;
|
||||
|
||||
rocksdb::DB *db_ptr = nullptr;
|
||||
const auto status = rocksdb::DB::Open(
|
||||
options, utils::path::combine(config.get_data_directory(), {name}), &db_ptr);
|
||||
if (status.ok()) {
|
||||
db.reset(db_ptr);
|
||||
} else {
|
||||
event_system::instance().raise<repertory_exception>(__FUNCTION__, status.ToString());
|
||||
throw startup_exception(status.ToString());
|
||||
}
|
||||
}
|
||||
|
||||
void create_rocksdb(const app_config &config, const std::string &name,
|
||||
const std::vector<rocksdb::ColumnFamilyDescriptor> &families,
|
||||
std::vector<rocksdb::ColumnFamilyHandle *> &handles,
|
||||
std::unique_ptr<rocksdb::DB> &db) {
|
||||
rocksdb::Options options{};
|
||||
options.create_if_missing = true;
|
||||
options.create_missing_column_families = true;
|
||||
options.db_log_dir = config.get_log_directory();
|
||||
options.keep_log_file_num = 10;
|
||||
|
||||
rocksdb::DB *db_ptr = nullptr;
|
||||
const auto status =
|
||||
rocksdb::DB::Open(options, utils::path::combine(config.get_data_directory(), {name}),
|
||||
families, &handles, &db_ptr);
|
||||
if (status.ok()) {
|
||||
db.reset(db_ptr);
|
||||
} else {
|
||||
event_system::instance().raise<repertory_exception>(__FUNCTION__, status.ToString());
|
||||
throw startup_exception(status.ToString());
|
||||
}
|
||||
}
|
||||
} // namespace repertory::utils::db
|
||||
195
src/utils/string_utils.cpp
Normal file
195
src/utils/string_utils.cpp
Normal file
@@ -0,0 +1,195 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "utils/string_utils.hpp"
|
||||
#include "common.hpp"
|
||||
|
||||
namespace repertory::utils::string {
|
||||
bool begins_with(const std::string &str, const std::string &val) { return (str.find(val) == 0); }
|
||||
|
||||
bool contains(const std::string &str, const std::string &search) {
|
||||
return (str.find(search) != std::string::npos);
|
||||
}
|
||||
|
||||
bool ends_with(const std::string &str, const std::string &val) {
|
||||
if (val.size() > str.size()) {
|
||||
return false;
|
||||
}
|
||||
return std::equal(val.rbegin(), val.rend(), str.rbegin());
|
||||
}
|
||||
|
||||
std::string from_bool(const bool &val) { return std::to_string(val); }
|
||||
|
||||
std::string from_double(const double &value) { return std::to_string(value); }
|
||||
|
||||
std::string from_dynamic_bitset(const boost::dynamic_bitset<> &bitset) {
|
||||
std::stringstream ss;
|
||||
boost::archive::text_oarchive archive(ss);
|
||||
archive << bitset;
|
||||
return ss.str();
|
||||
}
|
||||
|
||||
std::string from_int32(const std::int32_t &val) { return std::to_string(val); }
|
||||
|
||||
std::string from_int64(const std::int64_t &val) { return std::to_string(val); }
|
||||
|
||||
std::string from_uint8(const std::uint8_t &val) { return std::to_string(val); }
|
||||
|
||||
std::string from_uint16(const std::uint16_t &val) { return std::to_string(val); }
|
||||
|
||||
std::string from_uint32(const std::uint32_t &val) { return std::to_string(val); }
|
||||
|
||||
std::string from_uint64(const std::uint64_t &val) { return std::to_string(val); }
|
||||
|
||||
std::wstring from_utf8(const std::string &str) {
|
||||
return str.empty() ? L""
|
||||
: std::wstring_convert<std::codecvt_utf8<wchar_t>, wchar_t>().from_bytes(str);
|
||||
}
|
||||
|
||||
bool is_numeric(const std::string &s) {
|
||||
static const auto r = std::regex(R"(^(\+|\-)?(([0-9]*)|(([0-9]*)\.([0-9]*)))$)");
|
||||
return std::regex_match(s, r);
|
||||
}
|
||||
|
||||
std::string join(const std::vector<std::string> &arr, const char &delim) {
|
||||
if (arr.empty()) {
|
||||
return "";
|
||||
}
|
||||
|
||||
return std::accumulate(std::next(arr.begin()), arr.end(), arr[0u],
|
||||
[&delim](auto s, const auto &v) { return s + delim + v; });
|
||||
}
|
||||
|
||||
std::string &left_trim(std::string &s) { return left_trim(s, ' '); }
|
||||
|
||||
std::string &left_trim(std::string &s, const char &c) {
|
||||
s.erase(0, s.find_first_not_of(c));
|
||||
return s;
|
||||
}
|
||||
|
||||
std::string &replace(std::string &src, const char &character, const char &with) {
|
||||
std::replace(src.begin(), src.end(), character, with);
|
||||
return src;
|
||||
}
|
||||
|
||||
std::string &replace(std::string &src, const std::string &find, const std::string &with,
|
||||
size_t startPos) {
|
||||
if (!src.empty() && (startPos < src.size())) {
|
||||
while ((startPos = src.find(find, startPos)) != std::string::npos) {
|
||||
src.replace(startPos, find.size(), with);
|
||||
startPos += with.size();
|
||||
}
|
||||
}
|
||||
return src;
|
||||
}
|
||||
|
||||
std::string replace_copy(std::string src, const char &character, const char &with) {
|
||||
std::replace(src.begin(), src.end(), character, with);
|
||||
return src;
|
||||
}
|
||||
|
||||
std::string replace_copy(std::string src, const std::string &find, const std::string &with,
|
||||
size_t startPos) {
|
||||
return replace(src, find, with, startPos);
|
||||
}
|
||||
|
||||
std::string &right_trim(std::string &s) { return right_trim(s, ' '); }
|
||||
|
||||
std::string &right_trim(std::string &s, const char &c) {
|
||||
s.erase(s.find_last_not_of(c) + 1);
|
||||
return s;
|
||||
}
|
||||
|
||||
std::vector<std::string> split(const std::string &str, const char &delim, const bool &should_trim) {
|
||||
std::vector<std::string> ret;
|
||||
std::stringstream ss(str);
|
||||
std::string item;
|
||||
while (std::getline(ss, item, delim)) {
|
||||
ret.push_back(should_trim ? trim(item) : item);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool to_bool(std::string val) {
|
||||
auto b = false;
|
||||
|
||||
trim(val);
|
||||
if (is_numeric(val)) {
|
||||
if (contains(val, ".")) {
|
||||
b = (to_double(val) != 0.0);
|
||||
} else {
|
||||
std::istringstream(val) >> b;
|
||||
}
|
||||
} else {
|
||||
std::istringstream(to_lower(val)) >> std::boolalpha >> b;
|
||||
}
|
||||
|
||||
return b;
|
||||
}
|
||||
|
||||
double to_double(const std::string &str) { return std::stod(str); }
|
||||
|
||||
boost::dynamic_bitset<> to_dynamic_bitset(const std::string &val) {
|
||||
std::stringstream ss(val);
|
||||
boost::dynamic_bitset<> bitset;
|
||||
boost::archive::text_iarchive archive(ss);
|
||||
archive >> bitset;
|
||||
return bitset;
|
||||
}
|
||||
|
||||
std::int32_t to_int32(const std::string &val) { return std::stoi(val); }
|
||||
|
||||
std::int64_t to_int64(const std::string &val) { return std::stoll(val); }
|
||||
|
||||
std::string to_lower(std::string str) {
|
||||
std::transform(str.begin(), str.end(), str.begin(), ::tolower);
|
||||
return str;
|
||||
}
|
||||
|
||||
std::uint8_t to_uint8(const std::string &val) { return static_cast<std::uint8_t>(std::stoul(val)); }
|
||||
|
||||
std::uint16_t to_uint16(const std::string &val) {
|
||||
return static_cast<std::uint16_t>(std::stoul(val));
|
||||
}
|
||||
|
||||
std::uint32_t to_uint32(const std::string &val) {
|
||||
return static_cast<std::uint32_t>(std::stoul(val));
|
||||
}
|
||||
|
||||
std::uint64_t to_uint64(const std::string &val) { return std::stoull(val); }
|
||||
|
||||
std::string to_upper(std::string str) {
|
||||
std::transform(str.begin(), str.end(), str.begin(), ::toupper);
|
||||
return str;
|
||||
}
|
||||
|
||||
const std::string &to_utf8(const std::string &str) { return str; }
|
||||
|
||||
std::string to_utf8(const std::wstring &str) {
|
||||
return str.empty() ? ""
|
||||
: std::wstring_convert<std::codecvt_utf8<wchar_t>, wchar_t>().to_bytes(str);
|
||||
}
|
||||
|
||||
std::string &trim(std::string &str) { return right_trim(left_trim(str)); }
|
||||
|
||||
std::string &trim(std::string &str, const char &c) { return right_trim(left_trim(str, c), c); }
|
||||
|
||||
std::string trim_copy(std::string str) { return right_trim(left_trim(str)); }
|
||||
|
||||
std::string trim_copy(std::string str, const char &c) { return right_trim(left_trim(str, c), c); }
|
||||
} // namespace repertory::utils::string
|
||||
60
src/utils/throttle.cpp
Normal file
60
src/utils/throttle.cpp
Normal file
@@ -0,0 +1,60 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "utils/throttle.hpp"
|
||||
#include "types/repertory.hpp"
|
||||
|
||||
namespace repertory {
|
||||
void throttle::decrement() {
|
||||
mutex_lock l(mutex_);
|
||||
if (not shutdown_) {
|
||||
if (count_ > 0) {
|
||||
count_--;
|
||||
}
|
||||
notify_.notify_one();
|
||||
}
|
||||
}
|
||||
|
||||
void throttle::increment_or_wait() {
|
||||
unique_mutex_lock l(mutex_);
|
||||
if (not shutdown_) {
|
||||
if (count_ >= max_size_) {
|
||||
notify_.wait(l);
|
||||
}
|
||||
if (not shutdown_) {
|
||||
count_++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void throttle::reset() {
|
||||
unique_mutex_lock l(mutex_);
|
||||
if (shutdown_) {
|
||||
count_ = 0;
|
||||
shutdown_ = false;
|
||||
}
|
||||
}
|
||||
|
||||
void throttle::shutdown() {
|
||||
if (not shutdown_) {
|
||||
unique_mutex_lock l(mutex_);
|
||||
shutdown_ = true;
|
||||
notify_.notify_all();
|
||||
}
|
||||
}
|
||||
} // namespace repertory
|
||||
49
src/utils/timeout.cpp
Normal file
49
src/utils/timeout.cpp
Normal file
@@ -0,0 +1,49 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "utils/timeout.hpp"
|
||||
#include "types/repertory.hpp"
|
||||
|
||||
namespace repertory {
|
||||
timeout::timeout(std::function<void()> timeout_callback,
|
||||
const std::chrono::system_clock::duration &duration)
|
||||
: timeout_killed_(duration == 0s) {
|
||||
if (not timeout_killed_) {
|
||||
timeout_thread_ = std::make_unique<std::thread>([this, duration, timeout_callback]() {
|
||||
unique_mutex_lock lock(timeout_mutex_);
|
||||
if (not timeout_killed_) {
|
||||
timeout_notify_.wait_for(lock, duration);
|
||||
if (not timeout_killed_) {
|
||||
timeout_callback();
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
void timeout::disable() {
|
||||
if (not timeout_killed_) {
|
||||
timeout_killed_ = true;
|
||||
unique_mutex_lock lock(timeout_mutex_);
|
||||
timeout_notify_.notify_all();
|
||||
lock.unlock();
|
||||
timeout_thread_->join();
|
||||
timeout_thread_.reset();
|
||||
}
|
||||
}
|
||||
} // namespace repertory
|
||||
191
src/utils/unix/unix_utils.cpp
Normal file
191
src/utils/unix/unix_utils.cpp
Normal file
@@ -0,0 +1,191 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#ifndef _WIN32
|
||||
|
||||
#include "utils/unix/unix_utils.hpp"
|
||||
#include "utils/utils.hpp"
|
||||
|
||||
namespace repertory::utils {
|
||||
#ifndef __APPLE__
|
||||
std::uint64_t convert_to_uint64(const pthread_t &t) { return static_cast<std::uint64_t>(t); }
|
||||
#endif
|
||||
|
||||
int get_last_error_code() { return errno; }
|
||||
|
||||
std::uint64_t get_thread_id() { return convert_to_uint64(pthread_self()); }
|
||||
|
||||
bool is_uid_member_of_group(const uid_t &uid, const gid_t &gid) {
|
||||
auto *pw = getpwuid(uid);
|
||||
std::vector<gid_t> groups;
|
||||
|
||||
int group_count = 0;
|
||||
if (getgrouplist(pw->pw_name, pw->pw_gid, nullptr, &group_count) < 0) {
|
||||
groups.resize(static_cast<unsigned long>(group_count));
|
||||
#ifdef __APPLE__
|
||||
getgrouplist(pw->pw_name, pw->pw_gid, reinterpret_cast<int *>(&groups[0]), &group_count);
|
||||
#else
|
||||
getgrouplist(pw->pw_name, pw->pw_gid, &groups[0], &group_count);
|
||||
#endif
|
||||
}
|
||||
|
||||
return collection_includes(groups, gid);
|
||||
}
|
||||
|
||||
int translate_api_error(const api_error &e) {
|
||||
switch (e) {
|
||||
case api_error::access_denied:
|
||||
return -EACCES;
|
||||
case api_error::bad_address:
|
||||
return -EFAULT;
|
||||
case api_error::directory_end_of_files:
|
||||
return -EOF;
|
||||
case api_error::directory_exists:
|
||||
return -EISDIR;
|
||||
case api_error::directory_not_empty:
|
||||
return -ENOTEMPTY;
|
||||
case api_error::directory_not_found:
|
||||
return -ENOTDIR;
|
||||
case api_error::download_failed:
|
||||
#if __APPLE__
|
||||
return -EBADMSG;
|
||||
#else
|
||||
return -EREMOTEIO;
|
||||
#endif
|
||||
case api_error::error:
|
||||
return -EIO;
|
||||
case api_error::file_exists:
|
||||
return -EEXIST;
|
||||
case api_error::file_in_use:
|
||||
return -EBUSY;
|
||||
case api_error::invalid_operation:
|
||||
return -EINVAL;
|
||||
case api_error::item_not_found:
|
||||
return -ENOENT;
|
||||
case api_error::item_is_file:
|
||||
return -ENOTDIR;
|
||||
case api_error::os_error:
|
||||
return -errno;
|
||||
case api_error::permission_denied:
|
||||
return -EPERM;
|
||||
case api_error::success:
|
||||
return 0;
|
||||
case api_error::not_supported:
|
||||
return -ENOTSUP;
|
||||
case api_error::not_implemented:
|
||||
return -ENOSYS;
|
||||
case api_error::upload_failed:
|
||||
#if __APPLE__
|
||||
return -EBADMSG;
|
||||
#else
|
||||
return -EREMOTEIO;
|
||||
#endif
|
||||
case api_error::xattr_buffer_small:
|
||||
return -ERANGE;
|
||||
case api_error::xattr_exists:
|
||||
return -EEXIST;
|
||||
case api_error::xattr_invalid_namespace:
|
||||
return -ENOTSUP;
|
||||
case api_error::xattr_not_found:
|
||||
#ifdef __APPLE__
|
||||
return -ENOATTR;
|
||||
#else
|
||||
return -ENODATA;
|
||||
#endif
|
||||
case api_error::xattr_too_big:
|
||||
#ifdef __APPLE__
|
||||
return -ENAMETOOLONG;
|
||||
#else
|
||||
return -E2BIG;
|
||||
#endif
|
||||
#ifdef __APPLE__
|
||||
case api_error::XAttrOSXInvalid:
|
||||
return -EINVAL;
|
||||
#endif
|
||||
default:
|
||||
return -EIO;
|
||||
}
|
||||
}
|
||||
|
||||
void set_last_error_code(int error_code) { errno = error_code; }
|
||||
|
||||
std::int32_t unix_error_to_windows(const int &e) {
|
||||
switch (e) {
|
||||
case 0:
|
||||
return STATUS_SUCCESS;
|
||||
case EACCES:
|
||||
case EPERM:
|
||||
return STATUS_ACCESS_DENIED;
|
||||
case EBADF:
|
||||
return STATUS_INVALID_HANDLE;
|
||||
case EBUSY:
|
||||
return STATUS_DEVICE_BUSY;
|
||||
case EEXIST:
|
||||
return STATUS_OBJECT_NAME_EXISTS;
|
||||
case EFAULT:
|
||||
return STATUS_INVALID_ADDRESS;
|
||||
case EFBIG:
|
||||
return STATUS_FILE_TOO_LARGE;
|
||||
case EINVAL:
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
case EIO:
|
||||
return STATUS_UNEXPECTED_IO_ERROR;
|
||||
case EISDIR:
|
||||
return STATUS_FILE_IS_A_DIRECTORY;
|
||||
case EMFILE:
|
||||
return STATUS_INSUFFICIENT_RESOURCES;
|
||||
case ENOENT:
|
||||
return STATUS_OBJECT_NAME_NOT_FOUND;
|
||||
case ENOEXEC:
|
||||
return STATUS_INVALID_IMAGE_FORMAT;
|
||||
case ENOMEM:
|
||||
return STATUS_NO_MEMORY;
|
||||
case ENOSPC:
|
||||
return STATUS_DEVICE_INSUFFICIENT_RESOURCES;
|
||||
case ENOTDIR:
|
||||
return STATUS_OBJECT_PATH_INVALID;
|
||||
default:
|
||||
return STATUS_INTERNAL_ERROR;
|
||||
}
|
||||
}
|
||||
|
||||
UINT64 unix_time_to_windows_time(const remote::file_time &ts) {
|
||||
return (ts / 100ull) + 116444736000000000ull;
|
||||
}
|
||||
|
||||
void windows_create_to_unix(const UINT32 &create_options, const UINT32 &granted_access,
|
||||
std::uint32_t &flags, remote::file_mode &mode) {
|
||||
mode = S_IRUSR | S_IWUSR;
|
||||
flags = O_CREAT | O_RDWR;
|
||||
if (create_options & FILE_DIRECTORY_FILE) {
|
||||
mode |= S_IXUSR;
|
||||
flags = O_DIRECTORY;
|
||||
}
|
||||
|
||||
if ((granted_access & GENERIC_EXECUTE) || (granted_access & FILE_GENERIC_EXECUTE) ||
|
||||
(granted_access & FILE_EXECUTE)) {
|
||||
mode |= (S_IXUSR);
|
||||
}
|
||||
}
|
||||
|
||||
remote::file_time windows_time_to_unix_time(const std::uint64_t &t) {
|
||||
return (t - 116444736000000000ull) * 100ull;
|
||||
}
|
||||
} // namespace repertory::utils
|
||||
|
||||
#endif // !_WIN32
|
||||
371
src/utils/utils.cpp
Normal file
371
src/utils/utils.cpp
Normal file
@@ -0,0 +1,371 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "utils/utils.hpp"
|
||||
#include "app_config.hpp"
|
||||
#include "events/events.hpp"
|
||||
#include "events/event_system.hpp"
|
||||
#include "providers/i_provider.hpp"
|
||||
#include "types/skynet.hpp"
|
||||
#include "types/startup_exception.hpp"
|
||||
#include "utils/com_init_wrapper.hpp"
|
||||
#include "utils/native_file.hpp"
|
||||
#include "utils/path_utils.hpp"
|
||||
#include "utils/string_utils.hpp"
|
||||
|
||||
namespace repertory::utils {
|
||||
hastings api_currency_to_hastings(const api_currency ¤cy) {
|
||||
ttmath::Parser<api_currency> parser;
|
||||
parser.Parse(currency.ToString() + " * (10 ^ 24)");
|
||||
ttmath::Conv conv;
|
||||
conv.scient_from = 256;
|
||||
conv.base = 10u;
|
||||
conv.round = 0;
|
||||
return parser.stack[0u].value.ToString(conv);
|
||||
}
|
||||
|
||||
void calculate_allocation_size(const bool &directory, const std::uint64_t &file_size,
|
||||
UINT64 allocation_size, std::string &allocation_meta_size) {
|
||||
if (directory) {
|
||||
allocation_meta_size = "0";
|
||||
return;
|
||||
}
|
||||
|
||||
if (file_size > allocation_size) {
|
||||
allocation_size = file_size;
|
||||
}
|
||||
allocation_size = ((allocation_size == 0u) ? WINFSP_ALLOCATION_UNIT : allocation_size);
|
||||
allocation_size =
|
||||
utils::divide_with_ceiling(allocation_size, WINFSP_ALLOCATION_UNIT) * WINFSP_ALLOCATION_UNIT;
|
||||
allocation_meta_size = std::to_string(allocation_size);
|
||||
}
|
||||
|
||||
std::size_t calculate_read_size(const uint64_t &total_size, const std::size_t &read_size,
|
||||
const uint64_t &offset) {
|
||||
return static_cast<std::size_t>(((offset + read_size) > total_size)
|
||||
? ((offset < total_size) ? total_size - offset : 0u)
|
||||
: read_size);
|
||||
}
|
||||
|
||||
int compare_version_strings(std::string version1, std::string version2) {
|
||||
if (utils::string::contains(version1, "-")) {
|
||||
version1 = utils::string::split(version1, '-')[0u];
|
||||
}
|
||||
|
||||
if (utils::string::contains(version2, "-")) {
|
||||
version2 = utils::string::split(version2, '-')[0u];
|
||||
}
|
||||
|
||||
auto nums1 = utils::string::split(version1, '.');
|
||||
auto nums2 = utils::string::split(version2, '.');
|
||||
|
||||
while (nums1.size() > nums2.size()) {
|
||||
nums2.emplace_back("0");
|
||||
}
|
||||
|
||||
while (nums2.size() > nums1.size()) {
|
||||
nums1.emplace_back("0");
|
||||
}
|
||||
|
||||
for (std::size_t i = 0u; i < nums1.size(); i++) {
|
||||
const auto int1 = utils::string::to_uint32(nums1[i]);
|
||||
const auto int2 = utils::string::to_uint32(nums2[i]);
|
||||
const auto res = std::memcmp(&int1, &int2, sizeof(int1));
|
||||
if (res) {
|
||||
return res;
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
std::uint64_t convert_api_date(const std::string &date) {
|
||||
//"2019-02-21T02:24:37.653091916-06:00"
|
||||
const auto parts = utils::string::split(date, '.');
|
||||
const auto dt = parts[0];
|
||||
const auto nanos = utils::string::to_uint64(utils::string::split(parts[1u], '-')[0u]);
|
||||
|
||||
struct tm tm1 {};
|
||||
#ifdef _WIN32
|
||||
auto convert_time = [](time_t t) -> std::uint64_t {
|
||||
const auto ll = Int32x32To64(t, 10000000) + 116444736000000000ull;
|
||||
ULARGE_INTEGER ft{};
|
||||
ft.LowPart = static_cast<DWORD>(ll);
|
||||
ft.HighPart = static_cast<DWORD>(ll >> 32);
|
||||
return ft.QuadPart;
|
||||
};
|
||||
|
||||
const auto parts2 = utils::string::split(dt, 'T');
|
||||
const auto date_parts = utils::string::split(parts2[0u], '-');
|
||||
const auto time_parts = utils::string::split(parts2[1u], ':');
|
||||
tm1.tm_year = utils::string::to_int32(date_parts[0u]) - 1900;
|
||||
tm1.tm_mon = utils::string::to_int32(date_parts[1u]) - 1;
|
||||
tm1.tm_mday = utils::string::to_int32(date_parts[2u]);
|
||||
tm1.tm_hour = utils::string::to_uint32(time_parts[0u]);
|
||||
tm1.tm_min = utils::string::to_uint32(time_parts[1u]);
|
||||
tm1.tm_sec = utils::string::to_uint32(time_parts[2u]);
|
||||
tm1.tm_wday = -1;
|
||||
tm1.tm_yday = -1;
|
||||
tm1.tm_isdst = -1;
|
||||
return (nanos / 100) + convert_time(mktime(&tm1));
|
||||
#else
|
||||
strptime(&dt[0], "%Y-%m-%dT%T", &tm1);
|
||||
return nanos + (mktime(&tm1) * NANOS_PER_SECOND);
|
||||
#endif
|
||||
}
|
||||
|
||||
CURL *create_curl() { return reset_curl(curl_easy_init()); }
|
||||
|
||||
std::string create_uuid_string() {
|
||||
#ifdef _WIN32
|
||||
UUID guid{};
|
||||
UuidCreate(&guid);
|
||||
|
||||
unsigned char *s;
|
||||
UuidToStringA(&guid, &s);
|
||||
|
||||
std::string ret(reinterpret_cast<char *>(s));
|
||||
RpcStringFreeA(&s);
|
||||
|
||||
return ret;
|
||||
#else
|
||||
#if __linux__
|
||||
uuid id;
|
||||
id.make(UUID_MAKE_V4);
|
||||
return id.string();
|
||||
#else
|
||||
uuid_t guid;
|
||||
uuid_generate_random(guid);
|
||||
|
||||
std::string ret;
|
||||
ret.resize(37);
|
||||
uuid_unparse(guid, &ret[0]);
|
||||
|
||||
return ret.c_str();
|
||||
#endif
|
||||
#endif
|
||||
}
|
||||
|
||||
std::string create_volume_label(const provider_type &pt) {
|
||||
return "repertory_" + app_config::get_provider_name(pt);
|
||||
}
|
||||
|
||||
download_type download_type_from_string(std::string type, const download_type &default_type) {
|
||||
type = utils::string::to_lower(utils::string::trim(type));
|
||||
if (type == "direct") {
|
||||
return download_type::direct;
|
||||
} else if (type == "fallback") {
|
||||
return download_type::fallback;
|
||||
} else if (type == "ring_buffer") {
|
||||
return download_type::ring_buffer;
|
||||
}
|
||||
|
||||
return default_type;
|
||||
}
|
||||
|
||||
std::string download_type_to_string(const download_type &type) {
|
||||
switch (type) {
|
||||
case download_type::direct:
|
||||
return "direct";
|
||||
case download_type::fallback:
|
||||
return "fallback";
|
||||
case download_type::ring_buffer:
|
||||
return "ring_buffer";
|
||||
default:
|
||||
return "fallback";
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef _WIN32
|
||||
// https://www.frenk.com/2009/12/convert-filetime-to-unix-timestamp/
|
||||
remote::file_time filetime_to_unix_time(const FILETIME &ft) {
|
||||
LARGE_INTEGER date{};
|
||||
date.HighPart = ft.dwHighDateTime;
|
||||
date.LowPart = ft.dwLowDateTime;
|
||||
date.QuadPart -= 116444736000000000ull;
|
||||
|
||||
return date.QuadPart * 100ull;
|
||||
}
|
||||
|
||||
void unix_time_to_filetime(const remote::file_time &ts, FILETIME &ft) {
|
||||
const auto winTime = (ts / 100ull) + 116444736000000000ull;
|
||||
ft.dwHighDateTime = winTime >> 32u;
|
||||
ft.dwLowDateTime = winTime & 0xFFFFFFFF;
|
||||
}
|
||||
#endif
|
||||
|
||||
std::string generate_random_string(const std::uint16_t &length) {
|
||||
srand(static_cast<unsigned int>(get_time_now()));
|
||||
|
||||
std::string ret;
|
||||
ret.resize(length);
|
||||
for (std::uint16_t i = 0u; i < length; i++) {
|
||||
do {
|
||||
ret[i] = static_cast<char>(rand() % 74 + 48);
|
||||
} while (((ret[i] >= 91) && (ret[i] <= 96)) || ((ret[i] >= 58) && (ret[i] <= 64)));
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
std::string get_environment_variable(const std::string &variable) {
|
||||
#ifdef _WIN32
|
||||
std::string value;
|
||||
auto sz = ::GetEnvironmentVariable(&variable[0], nullptr, 0);
|
||||
if (sz > 0) {
|
||||
value.resize(sz);
|
||||
::GetEnvironmentVariable(&variable[0], &value[0], sz);
|
||||
}
|
||||
|
||||
return value.c_str();
|
||||
#else
|
||||
const auto *v = getenv(variable.c_str());
|
||||
return std::string(v ? v : "");
|
||||
#endif
|
||||
}
|
||||
|
||||
std::uint64_t get_file_time_now() {
|
||||
#ifdef _WIN32
|
||||
SYSTEMTIME st{};
|
||||
::GetSystemTime(&st);
|
||||
FILETIME ft{};
|
||||
::SystemTimeToFileTime(&st, &ft);
|
||||
return static_cast<std::uint64_t>(((LARGE_INTEGER *)&ft)->QuadPart);
|
||||
#else
|
||||
return get_time_now();
|
||||
#endif
|
||||
}
|
||||
|
||||
void get_local_time_now(struct tm &local_time) {
|
||||
memset(&local_time, 0, sizeof(local_time));
|
||||
|
||||
const auto now = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now());
|
||||
#ifdef _WIN32
|
||||
localtime_s(&local_time, &now);
|
||||
#else
|
||||
const auto *tmp = std::localtime(&now);
|
||||
if (tmp) {
|
||||
memcpy(&local_time, tmp, sizeof(local_time));
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
#ifdef _WIN32
|
||||
#endif
|
||||
|
||||
bool get_next_available_port(std::uint16_t first_port, std::uint16_t &available_port) {
|
||||
using namespace boost::asio;
|
||||
using ip::tcp;
|
||||
boost::system::error_code ec;
|
||||
do {
|
||||
io_service svc;
|
||||
tcp::acceptor a(svc);
|
||||
a.open(tcp::v4(), ec) || a.bind({tcp::v4(), first_port}, ec);
|
||||
} while (ec && (first_port++ < 65535u));
|
||||
|
||||
if (not ec) {
|
||||
available_port = first_port;
|
||||
}
|
||||
|
||||
return not ec;
|
||||
}
|
||||
|
||||
std::uint64_t get_time_now() {
|
||||
#ifdef _WIN32
|
||||
return static_cast<std::uint64_t>(
|
||||
std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()));
|
||||
#else
|
||||
#if __APPLE__
|
||||
return std::chrono::nanoseconds(std::chrono::system_clock::now().time_since_epoch()).count();
|
||||
#else
|
||||
return static_cast<std::uint64_t>(
|
||||
std::chrono::nanoseconds(std::chrono::high_resolution_clock::now().time_since_epoch())
|
||||
.count());
|
||||
#endif
|
||||
#endif
|
||||
}
|
||||
|
||||
api_currency hastings_string_to_api_currency(const std::string &amount) {
|
||||
ttmath::Parser<api_currency> parser;
|
||||
parser.Parse(amount + " / (10 ^ 24)");
|
||||
return parser.stack[0].value;
|
||||
}
|
||||
|
||||
CURL *reset_curl(CURL *curl_handle) {
|
||||
curl_easy_reset(curl_handle);
|
||||
#if __APPLE__
|
||||
curl_easy_setopt(curl_handle, CURLOPT_NOSIGNAL, 1);
|
||||
#endif
|
||||
return curl_handle;
|
||||
}
|
||||
|
||||
bool retryable_action(const std::function<bool()> &action) {
|
||||
auto succeeded = false;
|
||||
for (auto i = 0; not(succeeded = action()) && (i < 10); i++) {
|
||||
std::this_thread::sleep_for(100ms);
|
||||
}
|
||||
return succeeded;
|
||||
}
|
||||
|
||||
/* bool parse_url(const std::string &url, HostConfig &hc) { */
|
||||
/* static const auto pathRegex = std::regex( */
|
||||
/* R"(^((\w+):)?(\/\/((\w+)?(:(\w+))?@)?([^\/\?:]+)(:(\d+))?)?(\/?([^\/\?#][^\?#]*)?)?(\?([^#]+))?(#(\w*))?$)");
|
||||
*/
|
||||
/* */
|
||||
/* std::smatch results; */
|
||||
/* if (std::regex_search(url, results, pathRegex)) { */
|
||||
/* hc.HostNameOrIp = results[8u].str(); */
|
||||
/* hc.Path = utils::path::create_api_path(results[11u].str()); */
|
||||
/* hc.Protocol = utils::string::to_lower(results[2u].str()); */
|
||||
/* hc.ApiPort = results[10u].str().empty() ? ((hc.Protocol == "https") ? 443u : 80u) */
|
||||
/* : utils::string::to_uint16(results[10u].str()); */
|
||||
/* return not hc.HostNameOrIp.empty() && not hc.Protocol.empty() && */
|
||||
/* ((hc.Protocol == "http") || (hc.Protocol == "https")); */
|
||||
/* } */
|
||||
/* */
|
||||
/* return false; */
|
||||
/* } */
|
||||
|
||||
void spin_wait_for_mutex(std::function<bool()> complete, std::condition_variable &cv,
|
||||
std::mutex &mtx, const std::string &text) {
|
||||
while (not complete()) {
|
||||
unique_mutex_lock l(mtx);
|
||||
if (not complete()) {
|
||||
if (not text.empty()) {
|
||||
/* event_system::instance().raise<DebugLog>(__FUNCTION__, "spin_wait_for_mutex", text); */
|
||||
}
|
||||
cv.wait_for(l, 5s);
|
||||
}
|
||||
l.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
void spin_wait_for_mutex(bool &complete, std::condition_variable &cv, std::mutex &mtx,
|
||||
const std::string &text) {
|
||||
while (not complete) {
|
||||
unique_mutex_lock l(mtx);
|
||||
if (not complete) {
|
||||
if (not text.empty()) {
|
||||
/* event_system::instance().raise<DebugLog>(__FUNCTION__, "spin_wait_for_mutex", text); */
|
||||
}
|
||||
cv.wait_for(l, 5s);
|
||||
}
|
||||
l.unlock();
|
||||
}
|
||||
}
|
||||
} // namespace repertory::utils
|
||||
196
src/utils/windows/windows_utils.cpp
Normal file
196
src/utils/windows/windows_utils.cpp
Normal file
@@ -0,0 +1,196 @@
|
||||
/*
|
||||
Copyright <2018-2022> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#ifdef _WIN32
|
||||
|
||||
#include "utils/windows/windows_utils.hpp"
|
||||
#include "types/startup_exception.hpp"
|
||||
#include "utils/com_init_wrapper.hpp"
|
||||
#include "utils/string_utils.hpp"
|
||||
|
||||
namespace repertory::utils {
|
||||
DWORD get_last_error_code() { return ::GetLastError(); }
|
||||
|
||||
const std::string &get_local_app_data_directory() {
|
||||
static std::string app_data = ([]() -> std::string {
|
||||
com_init_wrapper cw;
|
||||
PWSTR local_app_data{};
|
||||
if (SUCCEEDED(::SHGetKnownFolderPath(FOLDERID_LocalAppData, 0, nullptr, &local_app_data))) {
|
||||
auto app_data = utils::string::to_utf8(local_app_data);
|
||||
::CoTaskMemFree(local_app_data);
|
||||
return app_data;
|
||||
}
|
||||
|
||||
throw startup_exception("unable to detect local application data folder");
|
||||
})();
|
||||
|
||||
return app_data;
|
||||
}
|
||||
|
||||
std::uint64_t get_accessed_time_from_meta(const api_meta_map &meta) {
|
||||
return utils::string::to_uint64(meta.at(META_ACCESSED));
|
||||
}
|
||||
|
||||
DWORD get_attributes_from_meta(const api_meta_map &meta) {
|
||||
return static_cast<DWORD>(utils::string::to_uint32(meta.at(META_ATTRIBUTES)));
|
||||
}
|
||||
|
||||
std::uint64_t get_changed_time_from_meta(const api_meta_map &meta) {
|
||||
return utils::string::to_uint64(meta.at(META_MODIFIED));
|
||||
}
|
||||
|
||||
std::uint64_t get_creation_time_from_meta(const api_meta_map &meta) {
|
||||
return utils::string::to_uint64(meta.at(META_CREATION));
|
||||
}
|
||||
|
||||
std::uint64_t get_written_time_from_meta(const api_meta_map &meta) {
|
||||
return utils::string::to_uint64(meta.at(META_WRITTEN));
|
||||
}
|
||||
|
||||
std::uint64_t get_thread_id() { return static_cast<std::uint64_t>(::GetCurrentThreadId()); }
|
||||
|
||||
NTSTATUS translate_api_error(const api_error &e) {
|
||||
switch (e) {
|
||||
case api_error::access_denied:
|
||||
return FspNtStatusFromWin32(ERROR_ACCESS_DENIED);
|
||||
case api_error::buffer_too_small:
|
||||
return STATUS_BUFFER_TOO_SMALL;
|
||||
case api_error::buffer_overflow:
|
||||
return STATUS_BUFFER_OVERFLOW;
|
||||
case api_error::directory_end_of_files:
|
||||
return STATUS_NO_MORE_FILES;
|
||||
case api_error::directory_exists:
|
||||
return STATUS_OBJECT_NAME_EXISTS;
|
||||
case api_error::directory_not_empty:
|
||||
return STATUS_DIRECTORY_NOT_EMPTY;
|
||||
case api_error::directory_not_found:
|
||||
return STATUS_OBJECT_NAME_NOT_FOUND;
|
||||
case api_error::download_failed:
|
||||
return FspNtStatusFromWin32(ERROR_INTERNAL_ERROR);
|
||||
case api_error::error:
|
||||
return FspNtStatusFromWin32(ERROR_INTERNAL_ERROR);
|
||||
case api_error::file_exists:
|
||||
return FspNtStatusFromWin32(ERROR_FILE_EXISTS);
|
||||
case api_error::file_in_use:
|
||||
return FspNtStatusFromWin32(ERROR_BUSY);
|
||||
case api_error::incompatible_version:
|
||||
return STATUS_CLIENT_SERVER_PARAMETERS_INVALID;
|
||||
case api_error::invalid_operation:
|
||||
return FspNtStatusFromWin32(ERROR_INTERNAL_ERROR);
|
||||
case api_error::item_not_found:
|
||||
return STATUS_OBJECT_NAME_NOT_FOUND;
|
||||
case api_error::item_is_file:
|
||||
return STATUS_ACCESS_DENIED;
|
||||
case api_error::os_error:
|
||||
return FspNtStatusFromWin32(::GetLastError());
|
||||
case api_error::permission_denied:
|
||||
return FspNtStatusFromWin32(ERROR_ACCESS_DENIED);
|
||||
case api_error::success:
|
||||
return 0;
|
||||
case api_error::not_supported:
|
||||
return FspNtStatusFromWin32(ERROR_INTERNAL_ERROR);
|
||||
case api_error::not_implemented:
|
||||
return STATUS_NOT_IMPLEMENTED;
|
||||
case api_error::upload_failed:
|
||||
return FspNtStatusFromWin32(ERROR_INTERNAL_ERROR);
|
||||
default:
|
||||
return FspNtStatusFromWin32(ERROR_INTERNAL_ERROR);
|
||||
}
|
||||
}
|
||||
bool is_process_elevated() {
|
||||
auto ret = false;
|
||||
HANDLE token = INVALID_HANDLE_VALUE;
|
||||
if (::OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY, &token)) {
|
||||
TOKEN_ELEVATION te{};
|
||||
DWORD sz = sizeof(te);
|
||||
if (::GetTokenInformation(token, TokenElevation, &te, sizeof(te), &sz)) {
|
||||
ret = (te.TokenIsElevated != 0);
|
||||
}
|
||||
}
|
||||
|
||||
if (token != INVALID_HANDLE_VALUE) {
|
||||
::CloseHandle(token);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
int run_process_elevated(int argc, char *argv[]) {
|
||||
std::cout << "Elevating Process" << std::endl;
|
||||
std::string parameters = "-hidden";
|
||||
for (int i = 1; i < argc; i++) {
|
||||
parameters += (parameters.empty() ? argv[i] : " " + std::string(argv[i]));
|
||||
}
|
||||
|
||||
char full_path[MAX_PATH] = {0};
|
||||
if (::GetModuleFileName(nullptr, full_path, MAX_PATH)) {
|
||||
SHELLEXECUTEINFO sei{};
|
||||
sei.fMask = SEE_MASK_NOCLOSEPROCESS;
|
||||
sei.cbSize = sizeof(sei);
|
||||
sei.lpVerb = "runas";
|
||||
sei.lpFile = &full_path[0];
|
||||
sei.lpParameters = ¶meters[0];
|
||||
sei.hwnd = nullptr;
|
||||
sei.nShow = SW_NORMAL;
|
||||
if (::ShellExecuteEx(&sei)) {
|
||||
::WaitForSingleObject(sei.hProcess, INFINITE);
|
||||
DWORD exit_code = 0u;
|
||||
::GetExitCodeProcess(sei.hProcess, &exit_code);
|
||||
::CloseHandle(sei.hProcess);
|
||||
return exit_code;
|
||||
}
|
||||
}
|
||||
|
||||
return ::GetLastError();
|
||||
}
|
||||
|
||||
void set_last_error_code(DWORD error_code) { ::SetLastError(error_code); }
|
||||
|
||||
int unix_access_mask_to_windows(std::int32_t mask) {
|
||||
if (mask & 1) {
|
||||
mask -= 1;
|
||||
if (not mask) {
|
||||
mask = 4;
|
||||
}
|
||||
}
|
||||
|
||||
return mask & 6;
|
||||
}
|
||||
|
||||
int unix_open_flags_to_flags_and_perms(const remote::file_mode &mode,
|
||||
const remote::open_flags &flags, std::int32_t &perms) {
|
||||
auto ret = _O_BINARY | _O_RANDOM;
|
||||
ret |= (((flags & remote::open_flags::create) == remote::open_flags::create) ? _O_CREAT : 0);
|
||||
ret |= (((flags & remote::open_flags::excl) == remote::open_flags::excl) ? _O_EXCL : 0);
|
||||
ret |= (((flags & remote::open_flags::truncate) == remote::open_flags::truncate) ? _O_TRUNC : 0);
|
||||
ret |= (((flags & remote::open_flags::append) == remote::open_flags::append) ? _O_APPEND : 0);
|
||||
ret |= (((flags & remote::open_flags::temp_file) == remote::open_flags::temp_file) ? _O_TEMPORARY
|
||||
: 0);
|
||||
ret |= ((flags & remote::open_flags::write_only) == remote::open_flags::write_only) ? _O_WRONLY
|
||||
: ((flags & remote::open_flags::read_write) == remote::open_flags::read_write) ? _O_RDWR
|
||||
: _O_RDONLY;
|
||||
|
||||
perms = _S_IREAD | _S_IWRITE;
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
remote::file_time time64_to_unix_time(const __time64_t &t) { return t * NANOS_PER_SECOND; }
|
||||
} // namespace repertory::utils
|
||||
|
||||
#endif // _WIN32
|
||||
Reference in New Issue
Block a user