extract common behavior
This commit is contained in:
529
src/providers/base_provider.cpp
Normal file
529
src/providers/base_provider.cpp
Normal file
@ -0,0 +1,529 @@
|
||||
/*
|
||||
Copyright <2018-2023> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
*/
|
||||
#include "providers/base_provider.hpp"
|
||||
|
||||
#include "app_config.hpp"
|
||||
#include "events/event_system.hpp"
|
||||
#include "events/events.hpp"
|
||||
#include "file_manager/i_file_manager.hpp"
|
||||
#include "utils/file_utils.hpp"
|
||||
#include "utils/path_utils.hpp"
|
||||
#include "utils/polling.hpp"
|
||||
#include "utils/rocksdb_utils.hpp"
|
||||
|
||||
namespace repertory {
|
||||
auto base_provider::create_api_file(std::string path, std::string key,
|
||||
std::uint64_t size) -> api_file {
|
||||
api_file file{};
|
||||
file.api_path = utils::path::create_api_path(path);
|
||||
file.api_parent = utils::path::get_parent_api_path(file.api_path);
|
||||
file.accessed_date = utils::get_file_time_now();
|
||||
file.changed_date = utils::get_file_time_now();
|
||||
file.creation_date = utils::get_file_time_now();
|
||||
file.modified_date = utils::get_file_time_now();
|
||||
file.key = key;
|
||||
file.file_size = size;
|
||||
return file;
|
||||
}
|
||||
|
||||
auto base_provider::create_api_file(std::string path, std::uint64_t size,
|
||||
api_meta_map &meta) -> api_file {
|
||||
auto current_size = utils::string::to_uint64(meta[META_SIZE]);
|
||||
if (current_size == 0U) {
|
||||
current_size = size;
|
||||
}
|
||||
|
||||
api_file file{};
|
||||
file.api_path = utils::path::create_api_path(path);
|
||||
file.api_parent = utils::path::get_parent_api_path(file.api_path);
|
||||
file.accessed_date = utils::string::to_uint64(meta[META_ACCESSED]);
|
||||
file.changed_date = utils::string::to_uint64(meta[META_CHANGED]);
|
||||
file.creation_date = utils::string::to_uint64(meta[META_CREATION]);
|
||||
file.file_size = current_size;
|
||||
file.modified_date = utils::string::to_uint64(meta[META_MODIFIED]);
|
||||
return file;
|
||||
}
|
||||
|
||||
auto base_provider::create_directory_clone_source_meta(
|
||||
const std::string &source_api_path, const std::string &api_path)
|
||||
-> api_error {
|
||||
bool exists{};
|
||||
auto res = is_file(source_api_path, exists);
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
if (exists) {
|
||||
utils::error::raise_api_path_error(__FUNCTION__, api_path,
|
||||
api_error::item_exists,
|
||||
"failed to create directory");
|
||||
return api_error::item_exists;
|
||||
}
|
||||
|
||||
res = is_directory(api_path, exists);
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
if (exists) {
|
||||
utils::error::raise_api_path_error(__FUNCTION__, api_path,
|
||||
api_error::directory_exists,
|
||||
"failed to create directory");
|
||||
return api_error::directory_exists;
|
||||
}
|
||||
|
||||
res = is_file(api_path, exists);
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
if (exists) {
|
||||
utils::error::raise_api_path_error(__FUNCTION__, api_path,
|
||||
api_error::item_exists,
|
||||
"failed to create directory");
|
||||
return api_error::item_exists;
|
||||
}
|
||||
|
||||
api_meta_map meta{};
|
||||
res = get_item_meta(source_api_path, meta);
|
||||
if (res != api_error::success) {
|
||||
if (res == api_error::item_not_found) {
|
||||
res = api_error::directory_not_found;
|
||||
}
|
||||
utils::error::raise_api_path_error(__FUNCTION__, api_path, res,
|
||||
"failed to create directory");
|
||||
return res;
|
||||
}
|
||||
|
||||
return create_directory(api_path, meta);
|
||||
}
|
||||
|
||||
auto base_provider::get_api_path_from_source(const std::string &source_path,
|
||||
std::string &api_path) const
|
||||
-> api_error {
|
||||
if (source_path.empty()) {
|
||||
utils::error::raise_api_path_error(__FUNCTION__, api_path,
|
||||
api_error::item_not_found,
|
||||
"failed to source path from api path");
|
||||
return api_error::item_not_found;
|
||||
}
|
||||
|
||||
auto iterator = std::unique_ptr<rocksdb::Iterator>(
|
||||
db_->NewIterator(rocksdb::ReadOptions()));
|
||||
for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
|
||||
std::string current_source_path{};
|
||||
if (get_item_meta(iterator->key().ToString(), META_SOURCE,
|
||||
current_source_path) != api_error::success) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (current_source_path.empty()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (current_source_path == source_path) {
|
||||
api_path = iterator->key().ToString();
|
||||
return api_error::success;
|
||||
}
|
||||
}
|
||||
|
||||
return api_error::item_not_found;
|
||||
}
|
||||
|
||||
auto base_provider::get_filesystem_item(const std::string &api_path,
|
||||
bool directory,
|
||||
filesystem_item &fsi) const
|
||||
-> api_error {
|
||||
bool exists{};
|
||||
auto res = is_directory(api_path, exists);
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
if (directory && not exists) {
|
||||
return api_error::directory_not_found;
|
||||
}
|
||||
|
||||
res = is_file(api_path, exists);
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
if (not directory && not exists) {
|
||||
return api_error::item_not_found;
|
||||
}
|
||||
|
||||
api_meta_map meta{};
|
||||
res = get_item_meta(api_path, meta);
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
|
||||
fsi.api_parent = utils::path::get_parent_api_path(api_path);
|
||||
fsi.api_path = api_path;
|
||||
fsi.directory = directory;
|
||||
fsi.size = fsi.directory ? 0U : utils::string::to_uint64(meta[META_SIZE]);
|
||||
fsi.source_path = meta[META_SOURCE];
|
||||
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
auto base_provider::get_filesystem_item_and_file(const std::string &api_path,
|
||||
api_file &file,
|
||||
filesystem_item &fsi) const
|
||||
-> api_error {
|
||||
auto res = get_file(api_path, file);
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
|
||||
api_meta_map meta{};
|
||||
res = get_item_meta(api_path, meta);
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
|
||||
fsi.api_parent = utils::path::get_parent_api_path(api_path);
|
||||
fsi.api_path = api_path;
|
||||
fsi.directory = false;
|
||||
fsi.size = utils::string::to_uint64(meta[META_SIZE]);
|
||||
fsi.source_path = meta[META_SOURCE];
|
||||
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
auto base_provider::get_filesystem_item_from_source_path(
|
||||
const std::string &source_path, filesystem_item &fsi) const -> api_error {
|
||||
std::string api_path{};
|
||||
auto res = get_api_path_from_source(source_path, api_path);
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
|
||||
bool exists{};
|
||||
res = is_directory(api_path, exists);
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
if (exists) {
|
||||
return api_error::directory_exists;
|
||||
}
|
||||
|
||||
return get_filesystem_item(api_path, false, fsi);
|
||||
}
|
||||
|
||||
auto base_provider::get_item_meta(const std::string &api_path,
|
||||
api_meta_map &meta) const -> api_error {
|
||||
std::string meta_value{};
|
||||
db_->Get(rocksdb::ReadOptions(), api_path, &meta_value);
|
||||
if (meta_value.empty()) {
|
||||
return api_error::item_not_found;
|
||||
}
|
||||
|
||||
try {
|
||||
meta = json::parse(meta_value).get<api_meta_map>();
|
||||
|
||||
return api_error::success;
|
||||
} catch (const std::exception &e) {
|
||||
utils::error::raise_api_path_error(__FUNCTION__, api_path, e,
|
||||
"failed to get item meta");
|
||||
}
|
||||
|
||||
return api_error::error;
|
||||
}
|
||||
|
||||
auto base_provider::get_item_meta(const std::string &api_path,
|
||||
const std::string &key,
|
||||
std::string &value) const -> api_error {
|
||||
std::string meta_value{};
|
||||
db_->Get(rocksdb::ReadOptions(), api_path, &meta_value);
|
||||
if (meta_value.empty()) {
|
||||
return api_error::item_not_found;
|
||||
}
|
||||
|
||||
try {
|
||||
value = json::parse(meta_value)[key];
|
||||
return api_error::success;
|
||||
} catch (const std::exception &e) {
|
||||
utils::error::raise_api_path_error(__FUNCTION__, api_path, e,
|
||||
"failed to get item meta");
|
||||
}
|
||||
|
||||
return api_error::error;
|
||||
}
|
||||
|
||||
auto base_provider::get_pinned_files() const -> std::vector<std::string> {
|
||||
std::vector<std::string> ret{};
|
||||
|
||||
auto iterator = std::unique_ptr<rocksdb::Iterator>(
|
||||
db_->NewIterator(rocksdb::ReadOptions()));
|
||||
for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
|
||||
std::string pinned{};
|
||||
if (get_item_meta(iterator->key().ToString(), META_PINNED, pinned) !=
|
||||
api_error::success) {
|
||||
continue;
|
||||
}
|
||||
if (pinned.empty() || not utils::string::to_bool(pinned)) {
|
||||
continue;
|
||||
}
|
||||
ret.emplace_back(iterator->key().ToString());
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
auto base_provider::get_total_item_count() const -> std::uint64_t {
|
||||
std::uint64_t ret{};
|
||||
auto iterator = std::unique_ptr<rocksdb::Iterator>(
|
||||
db_->NewIterator(rocksdb::ReadOptions()));
|
||||
for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
|
||||
ret++;
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
auto base_provider::is_file_writeable(const std::string &api_path) const
|
||||
-> bool {
|
||||
bool exists{};
|
||||
auto res = is_directory(api_path, exists);
|
||||
if (res != api_error::success) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return not exists;
|
||||
}
|
||||
|
||||
void base_provider::remove_deleted_files() {
|
||||
struct removed_item {
|
||||
std::string api_path{};
|
||||
bool directory{};
|
||||
std::string source_path{};
|
||||
};
|
||||
|
||||
api_file_list list{};
|
||||
auto res = get_file_list(list);
|
||||
if (res != api_error::success) {
|
||||
utils::error::raise_error(__FUNCTION__, res, "failed to get file list");
|
||||
return;
|
||||
}
|
||||
|
||||
std::vector<removed_item> removed_list{};
|
||||
auto iterator = std::unique_ptr<rocksdb::Iterator>(
|
||||
db_->NewIterator(rocksdb::ReadOptions()));
|
||||
for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
|
||||
api_meta_map meta{};
|
||||
if (get_item_meta(iterator->key().ToString(), meta) == api_error::success) {
|
||||
if (utils::string::to_bool(meta[META_DIRECTORY])) {
|
||||
bool exists{};
|
||||
if (is_directory(iterator->key().ToString(), exists) !=
|
||||
api_error::success) {
|
||||
continue;
|
||||
}
|
||||
if (not exists) {
|
||||
removed_list.emplace_back(
|
||||
removed_item{iterator->key().ToString(), true, ""});
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
bool exists{};
|
||||
if (is_file(iterator->key().ToString(), exists) != api_error::success) {
|
||||
continue;
|
||||
}
|
||||
if (not exists) {
|
||||
removed_list.emplace_back(
|
||||
removed_item{iterator->key().ToString(), false, meta[META_SOURCE]});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const auto &item : removed_list) {
|
||||
if (not item.directory) {
|
||||
if (utils::file::is_file(item.source_path)) {
|
||||
const auto orphaned_directory =
|
||||
utils::path::combine(config_.get_data_directory(), {"orphaned"});
|
||||
if (utils::file::create_full_directory_path(orphaned_directory)) {
|
||||
const auto parts = utils::string::split(item.api_path, '/', false);
|
||||
const auto orphaned_file = utils::path::combine(
|
||||
orphaned_directory,
|
||||
{utils::path::strip_to_file_name(item.source_path) + '_' +
|
||||
parts[parts.size() - 1U]});
|
||||
|
||||
event_system::instance().raise<orphaned_file_detected>(
|
||||
item.source_path);
|
||||
if (utils::file::reset_modified_time(item.source_path) &&
|
||||
utils::file::copy_file(item.source_path, orphaned_file)) {
|
||||
event_system::instance().raise<orphaned_file_processed>(
|
||||
item.source_path, orphaned_file);
|
||||
} else {
|
||||
event_system::instance().raise<orphaned_file_processing_failed>(
|
||||
item.source_path, orphaned_file,
|
||||
std::to_string(utils::get_last_error_code()));
|
||||
}
|
||||
} else {
|
||||
utils::error::raise_error(
|
||||
__FUNCTION__, std::to_string(utils::get_last_error_code()),
|
||||
"failed to create orphaned director|sp|" + orphaned_directory);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if (fm_->evict_file(item.api_path)) {
|
||||
db_->Delete(rocksdb::WriteOptions(), item.api_path);
|
||||
event_system::instance().raise<file_removed_externally>(
|
||||
item.api_path, item.source_path);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const auto &item : removed_list) {
|
||||
if (item.directory) {
|
||||
db_->Delete(rocksdb::WriteOptions(), item.api_path);
|
||||
event_system::instance().raise<directory_removed_externally>(
|
||||
item.api_path, item.source_path);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
auto base_provider::remove_item_meta(const std::string &api_path,
|
||||
const std::string &key) -> api_error {
|
||||
api_meta_map meta{};
|
||||
auto res = get_item_meta(api_path, meta);
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
|
||||
meta.erase(key);
|
||||
|
||||
auto res2 = db_->Put(rocksdb::WriteOptions(), api_path, json(meta).dump());
|
||||
if (not res2.ok()) {
|
||||
utils::error::raise_api_path_error(__FUNCTION__, api_path, res2.code(),
|
||||
"failed to remove item meta");
|
||||
return api_error::error;
|
||||
}
|
||||
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
auto base_provider::set_item_meta(const std::string &api_path,
|
||||
const std::string &key,
|
||||
const std::string &value) -> api_error {
|
||||
json meta_json{};
|
||||
std::string meta_value{};
|
||||
db_->Get(rocksdb::ReadOptions(), api_path, &meta_value);
|
||||
if (not meta_value.empty()) {
|
||||
try {
|
||||
meta_json = json::parse(meta_value);
|
||||
} catch (const std::exception &e) {
|
||||
utils::error::raise_api_path_error(__FUNCTION__, api_path, e,
|
||||
"failed to set item meta");
|
||||
return api_error::error;
|
||||
}
|
||||
}
|
||||
|
||||
meta_json[key] = value;
|
||||
|
||||
const auto res =
|
||||
db_->Put(rocksdb::WriteOptions(), api_path, meta_json.dump());
|
||||
if (not res.ok()) {
|
||||
utils::error::raise_api_path_error(__FUNCTION__, api_path, res.code(),
|
||||
"failed to set item meta");
|
||||
return api_error::error;
|
||||
}
|
||||
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
auto base_provider::set_item_meta(const std::string &api_path,
|
||||
const api_meta_map &meta) -> api_error {
|
||||
json meta_json{};
|
||||
std::string meta_value{};
|
||||
db_->Get(rocksdb::ReadOptions(), api_path, &meta_value);
|
||||
if (not meta_value.empty()) {
|
||||
try {
|
||||
meta_json = json::parse(meta_value);
|
||||
} catch (const std::exception &e) {
|
||||
utils::error::raise_api_path_error(__FUNCTION__, api_path, e,
|
||||
"failed to set item meta");
|
||||
return api_error::error;
|
||||
}
|
||||
}
|
||||
|
||||
for (const auto &kv : meta) {
|
||||
meta_json[kv.first] = kv.second;
|
||||
}
|
||||
|
||||
const auto res =
|
||||
db_->Put(rocksdb::WriteOptions(), api_path, meta_json.dump());
|
||||
if (not res.ok()) {
|
||||
utils::error::raise_api_path_error(__FUNCTION__, api_path, res.code(),
|
||||
"failed to set item meta");
|
||||
return api_error::error;
|
||||
}
|
||||
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
auto base_provider::start(api_item_added_callback api_item_added,
|
||||
i_file_manager *mgr) -> bool {
|
||||
event_system::instance().raise<service_started>("sia_provider");
|
||||
utils::db::create_rocksdb(config_, DB_NAME, db_);
|
||||
|
||||
api_item_added_ = api_item_added;
|
||||
fm_ = mgr;
|
||||
|
||||
api_meta_map meta{};
|
||||
if (get_item_meta("/", meta) == api_error::item_not_found) {
|
||||
auto dir = create_api_file("/", "", 0U);
|
||||
api_item_added_(true, dir);
|
||||
}
|
||||
|
||||
auto online = false;
|
||||
auto unmount_requested = false;
|
||||
{
|
||||
repertory::event_consumer consumer(
|
||||
"unmount_requested",
|
||||
[&unmount_requested](const event &) { unmount_requested = true; });
|
||||
for (std::uint16_t i = 0U; not online && not unmount_requested &&
|
||||
(i < config_.get_online_check_retry_secs());
|
||||
i++) {
|
||||
online = is_online();
|
||||
if (not online) {
|
||||
event_system::instance().raise<provider_offline>(
|
||||
config_.get_host_config().host_name_or_ip,
|
||||
config_.get_host_config().api_port);
|
||||
std::this_thread::sleep_for(1s);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (online && not unmount_requested) {
|
||||
polling::instance().set_callback({"check_deleted", polling::frequency::low,
|
||||
[this]() { remove_deleted_files(); }});
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
void base_provider::stop() {
|
||||
event_system::instance().raise<service_shutdown_begin>("sia_provider");
|
||||
polling::instance().remove_callback("check_deleted");
|
||||
db_.reset();
|
||||
event_system::instance().raise<service_shutdown_end>("sia_provider");
|
||||
}
|
||||
} // namespace repertory
|
||||
Reference in New Issue
Block a user