BlockStorage/repertory
1
0
Fork 0
Code Issues 11 Pull Requests Releases 7 Wiki Activity

revert
All checks were successful
BlockStorage/repertory_linux_builds/pipeline/head This commit looks good
Details
BlockStorage/repertory_osx_builds/pipeline/head This commit looks good
Details

Browse Source
This commit is contained in:
Scott E. Graves
2024-01-29 11:36:26 -06:00
parent d175a38ad1
commit 99533a9687
213 changed files with 43429 additions and 41103 deletions
Download Patch File Download Diff File Expand all files Collapse all files

266
src/providers/base_provider.cpp
View File

@@ -1,5 +1,5 @@
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Copyright <2018-2024> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
@@ -22,13 +22,15 @@
#include "providers/base_provider.hpp"
#include "app_config.hpp"
#include "database/db_common.hpp"
#include "database/db_insert.hpp"
#include "database/db_select.hpp"
#include "events/event_system.hpp"
#include "events/events.hpp"
#include "file_manager/i_file_manager.hpp"
#include "utils/file_utils.hpp"
#include "utils/path_utils.hpp"
#include "utils/polling.hpp"
#include "utils/rocksdb_utils.hpp"
namespace repertory {
auto base_provider::create_api_file(std::string path, std::string key,
@@ -66,13 +68,15 @@ auto base_provider::create_api_file(std::string path, std::uint64_t size,
auto base_provider::create_directory_clone_source_meta(
const std::string &source_api_path, const std::string &api_path)
-> api_error {
constexpr const auto *function_name = static_cast<const char *>(__FUNCTION__);
bool exists{};
auto res = is_file(source_api_path, exists);
if (res != api_error::success) {
return res;
}
if (exists) {
utils::error::raise_api_path_error(__FUNCTION__, api_path,
utils::error::raise_api_path_error(function_name, api_path,
api_error::item_exists,
"failed to create directory");
return api_error::item_exists;
@@ -83,7 +87,7 @@ auto base_provider::create_directory_clone_source_meta(
return res;
}
if (exists) {
utils::error::raise_api_path_error(__FUNCTION__, api_path,
utils::error::raise_api_path_error(function_name, api_path,
api_error::directory_exists,
"failed to create directory");
return api_error::directory_exists;
@@ -94,7 +98,7 @@ auto base_provider::create_directory_clone_source_meta(
return res;
}
if (exists) {
utils::error::raise_api_path_error(__FUNCTION__, api_path,
utils::error::raise_api_path_error(function_name, api_path,
api_error::item_exists,
"failed to create directory");
return api_error::item_exists;
@@ -106,7 +110,7 @@ auto base_provider::create_directory_clone_source_meta(
if (res == api_error::item_not_found) {
res = api_error::directory_not_found;
}
utils::error::raise_api_path_error(__FUNCTION__, api_path, res,
utils::error::raise_api_path_error(function_name, api_path, res,
"failed to create directory");
return res;
}
@@ -116,13 +120,15 @@ auto base_provider::create_directory_clone_source_meta(
auto base_provider::create_directory(const std::string &api_path,
api_meta_map &meta) -> api_error {
constexpr const auto *function_name = static_cast<const char *>(__FUNCTION__);
bool exists{};
auto res = is_directory(api_path, exists);
if (res != api_error::success) {
return res;
}
if (exists) {
utils::error::raise_api_path_error(__FUNCTION__, api_path,
utils::error::raise_api_path_error(function_name, api_path,
api_error::directory_exists,
"failed to create directory");
return api_error::directory_exists;
@@ -130,12 +136,12 @@ auto base_provider::create_directory(const std::string &api_path,
res = is_file(api_path, exists);
if (res != api_error::success) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, res,
utils::error::raise_api_path_error(function_name, api_path, res,
"failed to create directory");
return res;
}
if (exists) {
utils::error::raise_api_path_error(__FUNCTION__, api_path,
utils::error::raise_api_path_error(function_name, api_path,
api_error::item_exists,
"failed to create directory");
return api_error::item_exists;
@@ -144,12 +150,12 @@ auto base_provider::create_directory(const std::string &api_path,
try {
res = create_directory_impl(api_path, meta);
if (res != api_error::success) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, res,
utils::error::raise_api_path_error(function_name, api_path, res,
"failed to create directory");
return res;
}
} catch (const std::exception &e) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, e,
utils::error::raise_api_path_error(function_name, api_path, e,
"failed to create directory");
return api_error::error;
}
@@ -160,13 +166,15 @@ auto base_provider::create_directory(const std::string &api_path,
auto base_provider::create_file(const std::string &api_path, api_meta_map &meta)
-> api_error {
constexpr const auto *function_name = static_cast<const char *>(__FUNCTION__);
bool exists{};
auto res = is_directory(api_path, exists);
if (res != api_error::success && res != api_error::item_not_found) {
return res;
}
if (exists) {
utils::error::raise_api_path_error(__FUNCTION__, api_path,
utils::error::raise_api_path_error(function_name, api_path,
api_error::directory_exists,
"failed to create file");
return api_error::directory_exists;
@@ -177,7 +185,7 @@ auto base_provider::create_file(const std::string &api_path, api_meta_map &meta)
return res;
}
if (exists) {
utils::error::raise_api_path_error(__FUNCTION__, api_path,
utils::error::raise_api_path_error(function_name, api_path,
api_error::item_exists,
"failed to create file");
return api_error::item_exists;
@@ -186,7 +194,7 @@ auto base_provider::create_file(const std::string &api_path, api_meta_map &meta)
try {
res = create_file_extra(api_path, meta);
if (res != api_error::success) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, res,
utils::error::raise_api_path_error(function_name, api_path, res,
"failed to create file");
return res;
}
@@ -194,7 +202,7 @@ auto base_provider::create_file(const std::string &api_path, api_meta_map &meta)
meta[META_DIRECTORY] = utils::string::from_bool(false);
res = set_item_meta(api_path, meta);
if (res != api_error::success) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, res,
utils::error::raise_api_path_error(function_name, api_path, res,
"failed to create file");
return res;
}
@@ -202,12 +210,12 @@ auto base_provider::create_file(const std::string &api_path, api_meta_map &meta)
stop_type stop_requested{false};
res = upload_file(api_path, meta[META_SOURCE], stop_requested);
if (res != api_error::success) {
get_db()->Delete(rocksdb::WriteOptions(), api_path);
db3_->remove_api_path(api_path);
}
return res;
} catch (const std::exception &e) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, e,
utils::error::raise_api_path_error(function_name, api_path, e,
"failed to create file");
}
@@ -217,38 +225,23 @@ auto base_provider::create_file(const std::string &api_path, api_meta_map &meta)
auto base_provider::get_api_path_from_source(const std::string &source_path,
std::string &api_path) const
-> api_error {
constexpr const auto *function_name = static_cast<const char *>(__FUNCTION__);
if (source_path.empty()) {
utils::error::raise_api_path_error(__FUNCTION__, api_path,
utils::error::raise_api_path_error(function_name, api_path,
api_error::item_not_found,
"failed to source path from api path");
return api_error::item_not_found;
}
auto iterator = std::unique_ptr<rocksdb::Iterator>(
db_->NewIterator(rocksdb::ReadOptions()));
for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
std::string current_source_path{};
if (get_item_meta(iterator->key().ToString(), META_SOURCE,
current_source_path) != api_error::success) {
continue;
}
if (current_source_path.empty()) {
continue;
}
if (current_source_path == source_path) {
api_path = iterator->key().ToString();
return api_error::success;
}
}
return api_error::item_not_found;
return db3_->get_api_path(source_path, api_path);
}
auto base_provider::get_directory_items(const std::string &api_path,
directory_item_list &list) const
-> api_error {
constexpr const auto *function_name = static_cast<const char *>(__FUNCTION__);
bool exists{};
auto res = is_directory(api_path, exists);
if (res != api_error::success) {
@@ -264,7 +257,7 @@ auto base_provider::get_directory_items(const std::string &api_path,
return res;
}
} catch (const std::exception &e) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, e,
utils::error::raise_api_path_error(function_name, api_path, e,
"failed to get directory items");
return api_error::error;
}
@@ -392,81 +385,32 @@ auto base_provider::get_filesystem_item_from_source_path(
auto base_provider::get_item_meta(const std::string &api_path,
api_meta_map &meta) const -> api_error {
std::string meta_value{};
db_->Get(rocksdb::ReadOptions(), api_path, &meta_value);
if (meta_value.empty()) {
return api_error::item_not_found;
}
try {
meta = json::parse(meta_value).get<api_meta_map>();
return api_error::success;
} catch (const std::exception &e) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, e,
"failed to get item meta");
}
return api_error::error;
return db3_->get_item_meta(api_path, meta);
}
auto base_provider::get_item_meta(const std::string &api_path,
const std::string &key,
std::string &value) const -> api_error {
std::string meta_value{};
db_->Get(rocksdb::ReadOptions(), api_path, &meta_value);
if (meta_value.empty()) {
return api_error::item_not_found;
}
try {
value = json::parse(meta_value)[key];
return api_error::success;
} catch (const std::exception &e) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, e,
"failed to get item meta");
}
return api_error::error;
return db3_->get_item_meta(api_path, key, value);
}
auto base_provider::get_pinned_files() const -> std::vector<std::string> {
std::vector<std::string> ret{};
auto iterator = std::unique_ptr<rocksdb::Iterator>(
db_->NewIterator(rocksdb::ReadOptions()));
for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
std::string pinned{};
if (get_item_meta(iterator->key().ToString(), META_PINNED, pinned) !=
api_error::success) {
continue;
}
if (pinned.empty() || not utils::string::to_bool(pinned)) {
continue;
}
ret.emplace_back(iterator->key().ToString());
}
return ret;
return db3_->get_pinned_files();
}
auto base_provider::get_total_item_count() const -> std::uint64_t {
std::uint64_t ret{};
auto iterator = std::unique_ptr<rocksdb::Iterator>(
db_->NewIterator(rocksdb::ReadOptions()));
for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
ret++;
}
return ret;
return db3_->get_total_item_count();
}
auto base_provider::get_used_drive_space() const -> std::uint64_t {
constexpr const auto *function_name = static_cast<const char *>(__FUNCTION__);
try {
auto used_space = get_used_drive_space_impl();
get_file_mgr()->update_used_space(used_space);
return used_space;
} catch (const std::exception &ex) {
utils::error::raise_error(__FUNCTION__, ex,
utils::error::raise_error(function_name, ex,
"failed to get used drive space");
}
@@ -485,6 +429,8 @@ auto base_provider::is_file_writeable(const std::string &api_path) const
}
void base_provider::remove_deleted_files() {
constexpr const auto *function_name = static_cast<const char *>(__FUNCTION__);
struct removed_item {
std::string api_path{};
bool directory{};
@@ -494,36 +440,33 @@ void base_provider::remove_deleted_files() {
api_file_list list{};
auto res = get_file_list(list);
if (res != api_error::success) {
utils::error::raise_error(__FUNCTION__, res, "failed to get file list");
utils::error::raise_error(function_name, res, "failed to get file list");
return;
}
std::vector<removed_item> removed_list{};
auto iterator = std::unique_ptr<rocksdb::Iterator>(
db_->NewIterator(rocksdb::ReadOptions()));
for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
for (const auto &api_path : db3_->get_api_path_list()) {
api_meta_map meta{};
if (get_item_meta(iterator->key().ToString(), meta) == api_error::success) {
if (get_item_meta(api_path, meta) == api_error::success) {
if (utils::string::to_bool(meta[META_DIRECTORY])) {
bool exists{};
if (is_directory(iterator->key().ToString(), exists) !=
api_error::success) {
if (is_directory(api_path, exists) != api_error::success) {
continue;
}
if (not exists) {
removed_list.emplace_back(
removed_item{iterator->key().ToString(), true, ""});
removed_list.emplace_back(removed_item{api_path, true, ""});
}
continue;
}
bool exists{};
if (is_file(iterator->key().ToString(), exists) != api_error::success) {
if (is_file(api_path, exists) != api_error::success) {
continue;
}
if (not exists) {
removed_list.emplace_back(
removed_item{iterator->key().ToString(), false, meta[META_SOURCE]});
removed_item{api_path, false, meta[META_SOURCE]});
}
}
}
@@ -553,14 +496,14 @@ void base_provider::remove_deleted_files() {
}
} else {
utils::error::raise_error(
__FUNCTION__, std::to_string(utils::get_last_error_code()),
function_name, std::to_string(utils::get_last_error_code()),
"failed to create orphaned director|sp|" + orphaned_directory);
continue;
}
}
if (fm_->evict_file(item.api_path)) {
db_->Delete(rocksdb::WriteOptions(), item.api_path);
db3_->remove_api_path(item.api_path);
event_system::instance().raise<file_removed_externally>(
item.api_path, item.source_path);
}
@@ -569,7 +512,7 @@ void base_provider::remove_deleted_files() {
for (const auto &item : removed_list) {
if (item.directory) {
db_->Delete(rocksdb::WriteOptions(), item.api_path);
db3_->remove_api_path(item.api_path);
event_system::instance().raise<directory_removed_externally>(
item.api_path, item.source_path);
}
@@ -588,20 +531,10 @@ auto base_provider::remove_file(const std::string &api_path) -> api_error {
return error;
};
const auto *const function_name = __FUNCTION__;
const auto remove_file_meta = [this, &api_path, &function_name,
&notify_end]() -> api_error {
const auto remove_file_meta = [this, &api_path, &notify_end]() -> api_error {
api_meta_map meta{};
auto res = get_item_meta(api_path, meta);
auto res2 = get_db()->Delete(rocksdb::WriteOptions(), api_path);
if (not res2.ok()) {
utils::error::raise_api_path_error(function_name, api_path, res2.code(),
"failed to remove file");
return notify_end(api_error::error);
}
db3_->remove_api_path(api_path);
return notify_end(res);
};
@@ -658,109 +591,42 @@ auto base_provider::remove_directory(const std::string &api_path) -> api_error {
return notify_end(res);
}
auto status = get_db()->Delete(rocksdb::WriteOptions(), api_path);
if (not status.ok()) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, status.code(),
"failed to remove directory");
return notify_end(api_error::error);
}
db3_->remove_api_path(api_path);
return notify_end(api_error::success);
}
auto base_provider::remove_item_meta(const std::string &api_path,
const std::string &key) -> api_error {
api_meta_map meta{};
auto res = get_item_meta(api_path, meta);
if (res != api_error::success) {
return res;
}
meta.erase(key);
auto res2 = db_->Put(rocksdb::WriteOptions(), api_path, json(meta).dump());
if (not res2.ok()) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, res2.code(),
"failed to remove item meta");
return api_error::error;
}
return api_error::success;
return db3_->remove_item_meta(api_path, key);
}
auto base_provider::set_item_meta(const std::string &api_path,
const std::string &key,
const std::string &value) -> api_error {
json meta_json{};
std::string meta_value{};
db_->Get(rocksdb::ReadOptions(), api_path, &meta_value);
if (not meta_value.empty()) {
try {
meta_json = json::parse(meta_value);
} catch (const std::exception &e) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, e,
"failed to set item meta");
return api_error::error;
}
}
meta_json[key] = value;
const auto res =
db_->Put(rocksdb::WriteOptions(), api_path, meta_json.dump());
if (not res.ok()) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, res.code(),
"failed to set item meta");
return api_error::error;
}
return api_error::success;
return db3_->set_item_meta(api_path, key, value);
}
auto base_provider::set_item_meta(const std::string &api_path,
const api_meta_map &meta) -> api_error {
json meta_json{};
std::string meta_value{};
db_->Get(rocksdb::ReadOptions(), api_path, &meta_value);
if (not meta_value.empty()) {
try {
meta_json = json::parse(meta_value);
} catch (const std::exception &e) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, e,
"failed to set item meta");
return api_error::error;
}
}
for (const auto &kv : meta) {
meta_json[kv.first] = kv.second;
}
const auto res =
db_->Put(rocksdb::WriteOptions(), api_path, meta_json.dump());
if (not res.ok()) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, res.code(),
"failed to set item meta");
return api_error::error;
}
return api_error::success;
return db3_->set_item_meta(api_path, meta);
}
auto base_provider::start(api_item_added_callback api_item_added,
i_file_manager *mgr) -> bool {
utils::db::create_rocksdb(config_, DB_NAME, db_);
api_item_added_ = api_item_added;
fm_ = mgr;
db3_ = std::make_unique<meta_db>(config_);
api_meta_map meta{};
if (get_item_meta("/", meta) == api_error::item_not_found) {
auto dir = create_api_file("/", "", 0U);
api_item_added_(true, dir);
}
auto online = false;
auto unmount_requested = false;
auto online{false};
auto unmount_requested{false};
{
repertory::event_consumer consumer(
"unmount_requested",
@@ -789,12 +655,14 @@ auto base_provider::start(api_item_added_callback api_item_added,
void base_provider::stop() {
polling::instance().remove_callback("check_deleted");
db_.reset();
db3_.reset();
}
auto base_provider::upload_file(const std::string &api_path,
const std::string &source_path,
stop_type &stop_requested) -> api_error {
constexpr const auto *function_name = static_cast<const char *>(__FUNCTION__);
event_system::instance().raise<provider_upload_begin>(api_path, source_path);
const auto notify_end = [&api_path,
@@ -806,7 +674,7 @@ auto base_provider::upload_file(const std::string &api_path,
try {
return notify_end(upload_file_impl(api_path, source_path, stop_requested));
} catch (const std::exception &e) {
utils::error::raise_error(__FUNCTION__, e, "exception occurred");
utils::error::raise_error(function_name, e, "exception occurred");
}
return notify_end(api_error::error);