refactor encrypt provider db

This commit is contained in:
Scott E. Graves 2024-12-16 13:52:15 -06:00
parent 9af77225f2
commit c420ff1c81

View File

@ -36,42 +36,28 @@
#include "utils/polling.hpp"
namespace {
const std::string directory_table = "directory";
const std::string file_table = "file";
const std::string source_table = "source";
const std::string file_table = "provider_meta";
const std::map<std::string, std::string> sql_create_tables = {
{
{directory_table},
{"CREATE TABLE IF NOT EXISTS " + directory_table +
"("
"source_path TEXT PRIMARY KEY ASC, "
"api_path TEXT"
");"},
},
{
{file_table},
{"CREATE TABLE IF NOT EXISTS " + file_table +
"("
"source_path TEXT PRIMARY KEY ASC, "
"data TEXT"
"api_path TEXT, "
"data TEXT, "
"directory INTEGER, "
");"},
},
{
{source_table},
{"CREATE TABLE IF NOT EXISTS " + source_table +
"("
"api_path TEXT PRIMARY KEY ASC, "
"source_path TEXT"
");"},
}};
};
} // namespace
namespace repertory {
encrypt_provider::encrypt_provider(app_config &config) : config_(config) {}
auto encrypt_provider::create_api_file(
const std::string &api_path, bool directory,
const std::string &source_path) -> api_file {
auto encrypt_provider::create_api_file(const std::string &api_path,
bool directory,
const std::string &source_path)
-> api_file {
auto times = utils::file::get_times(source_path);
if (not times.has_value()) {
throw std::runtime_error("failed to get file times");
@ -97,10 +83,10 @@ auto encrypt_provider::create_api_file(
void encrypt_provider::create_item_meta(api_meta_map &meta, bool directory,
const api_file &file) {
#if defined(_WIN32)
struct _stat64 buf {};
struct _stat64 buf{};
_stat64(file.source_path.c_str(), &buf);
#else // !defined(_WIN32)
struct stat buf {};
struct stat buf{};
stat(file.source_path.c_str(), &buf);
#endif // defined(_WIN32)
@ -159,11 +145,7 @@ auto encrypt_provider::do_fs_operation(
: api_error::item_not_found;
}
auto exists =
utils::file::file{
source_path,
}
.exists();
auto exists = utils::file::file{source_path}.exists();
if (exists && directory) {
return api_error::item_exists;
}
@ -171,11 +153,7 @@ auto encrypt_provider::do_fs_operation(
return api_error::item_not_found;
}
exists =
utils::file::directory{
source_path,
}
.exists();
exists = utils::file::directory{source_path}.exists();
if (exists && not directory) {
return api_error::item_exists;
}
@ -187,38 +165,24 @@ auto encrypt_provider::do_fs_operation(
return callback(cfg, source_path);
}
auto encrypt_provider::get_api_path_from_source(
const std::string &source_path, std::string &api_path) const -> api_error {
auto encrypt_provider::get_api_path_from_source(const std::string &source_path,
std::string &api_path) const
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
try {
auto result = utils::db::sqlite::db_select{*db_, file_table}
.column("data")
.column("api_path")
.where("source_path")
.equals(source_path)
.go();
std::optional<utils::db::sqlite::db_result::row> row;
if (result.get_row(row) && row.has_value()) {
api_path = row->get_column("data")
.get_value_as_json()
.at("api_path")
.get<std::string>();
return api_error::success;
}
result = utils::db::sqlite::db_select{*db_, directory_table}
.column("api_path")
.where("source_path")
.equals(source_path)
.go();
row.reset();
if (result.get_row(row) && row.has_value()) {
api_path = row->get_column("api_path").get_value<std::string>();
return api_error::success;
}
return api_error::item_not_found;
} catch (const std::exception &ex) {
utils::error::raise_error(function_name, ex, source_path,
"failed to get api path from source path");
@ -253,8 +217,9 @@ auto encrypt_provider::get_directory_item_count(
return count;
}
auto encrypt_provider::get_directory_items(
const std::string &api_path, directory_item_list &list) const -> api_error {
auto encrypt_provider::get_directory_items(const std::string &api_path,
directory_item_list &list) const
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
return do_fs_operation(
@ -267,12 +232,14 @@ auto encrypt_provider::get_directory_items(
try {
std::string current_api_path{};
if (dir_entry->is_directory_item()) {
auto result =
utils::db::sqlite::db_select{*db_, directory_table}
.column("api_path")
.where("source_path")
.equals(dir_entry->get_path())
.go();
auto result = utils::db::sqlite::db_select{*db_, file_table}
.column("api_path")
.where("source_path")
.equals(dir_entry->get_path())
.and_()
.where("directory")
.equals(1)
.go();
std::optional<utils::db::sqlite::db_result::row> row;
if (result.get_row(row) && row.has_value()) {
current_api_path =
@ -282,10 +249,13 @@ auto encrypt_provider::get_directory_items(
process_directory_entry(*dir_entry.get(), cfg,
current_api_path);
result = utils::db::sqlite::db_select{*db_, directory_table}
result = utils::db::sqlite::db_select{*db_, file_table}
.column("api_path")
.where("source_path")
.equals(dir_entry->get_path())
.and_()
.where("directory")
.equals(1)
.go();
row.reset();
if (not(result.get_row(row) && row.has_value())) {
@ -298,25 +268,26 @@ auto encrypt_provider::get_directory_items(
} else {
std::string api_path_data{};
auto result = utils::db::sqlite::db_select{*db_, file_table}
.column("api_path")
.column("data")
.where("source_path")
.equals(dir_entry->get_path())
.and_()
.where("directory")
.equals(0)
.go();
std::optional<utils::db::sqlite::db_result::row> row;
if (result.get_row(row) && row.has_value()) {
api_path_data =
row->get_column("data").get_value<std::string>();
current_api_path =
row->get_column("api_path").get_value<std::string>();
}
if (api_path_data.empty()) {
if (not process_directory_entry(*dir_entry.get(), cfg,
current_api_path)) {
continue;
}
} else {
current_api_path = json::parse(api_path_data)
.at("api_path")
.get<std::string>();
if (api_path_data.empty() &&
not process_directory_entry(*dir_entry.get(), cfg,
current_api_path)) {
continue;
}
}
@ -356,11 +327,15 @@ auto encrypt_provider::get_directory_items(
"..",
"",
true,
0U,
{},
});
list.insert(list.begin(), directory_item{
".",
"",
true,
0U,
{},
});
return api_error::success;
});
@ -380,10 +355,13 @@ auto encrypt_provider::get_file(const std::string &api_path,
return api_error::directory_exists;
}
auto result = utils::db::sqlite::db_select{*db_, source_table}
auto result = utils::db::sqlite::db_select{*db_, file_table}
.column("source_path")
.where("api_path")
.equals(api_path)
.and_()
.where("directory")
.equals(0)
.go();
std::optional<utils::db::sqlite::db_result::row> row;
if (not(result.get_row(row) && row.has_value())) {
@ -402,8 +380,9 @@ auto encrypt_provider::get_file(const std::string &api_path,
return api_error::error;
}
auto encrypt_provider::get_file_list(
api_file_list &list, std::string & /* marker */) const -> api_error {
auto encrypt_provider::get_file_list(api_file_list &list,
std::string & /* marker */) const
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
const auto cfg = config_.get_encrypt_config();
@ -426,15 +405,19 @@ auto encrypt_provider::get_file_list(
return api_error::error;
}
auto encrypt_provider::get_file_size(
const std::string &api_path, std::uint64_t &file_size) const -> api_error {
auto encrypt_provider::get_file_size(const std::string &api_path,
std::uint64_t &file_size) const
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
try {
auto result = utils::db::sqlite::db_select{*db_, source_table}
auto result = utils::db::sqlite::db_select{*db_, file_table}
.column("source_path")
.where("api_path")
.equals(api_path)
.and_()
.where("directory")
.equals(0)
.go();
std::optional<utils::db::sqlite::db_result::row> row;
if (not(result.get_row(row) && row.has_value())) {
@ -453,26 +436,34 @@ auto encrypt_provider::get_file_size(
return api_error::error;
}
auto encrypt_provider::get_filesystem_item(
const std::string &api_path, bool directory,
filesystem_item &fsi) const -> api_error {
auto result = utils::db::sqlite::db_select{*db_, source_table}
auto encrypt_provider::get_filesystem_item(const std::string &api_path,
bool directory,
filesystem_item &fsi) const
-> api_error {
auto result = utils::db::sqlite::db_select{*db_, file_table}
.column("source_path")
.where("api_path")
.equals(api_path)
.and_()
.where("directory")
.equals(directory ? 1 : 0)
.go();
std::optional<utils::db::sqlite::db_result::row> row;
if (not(result.get_row(row) && row.has_value())) {
return api_error::item_not_found;
return directory ? api_error::directory_not_found
: api_error::item_not_found;
}
auto source_path = row->get_column("source_path").get_value<std::string>();
if (directory) {
result = utils::db::sqlite::db_select{*db_, directory_table}
result = utils::db::sqlite::db_select{*db_, file_table}
.column("api_path")
.where("source_path")
.equals(source_path)
.and_()
.where("directory")
.equals(1)
.go();
row.reset();
if (not(result.get_row(row) && row.has_value())) {
@ -489,19 +480,20 @@ auto encrypt_provider::get_filesystem_item(
}
result = utils::db::sqlite::db_select{*db_, file_table}
.column("api_path")
.column("data")
.where("source_path")
.equals(source_path)
.and_()
.where("directory")
.equals(0)
.go();
row.reset();
if (not(result.get_row(row) && row.has_value())) {
return api_error::item_not_found;
}
fsi.api_path = row->get_column("data")
.get_value_as_json()
.at("api_path")
.get<std::string>();
fsi.api_path = row->get_column("api_path").get_value<std::string>();
fsi.api_parent = utils::path::get_parent_api_path(fsi.api_path);
fsi.directory = false;
fsi.size = utils::encryption::encrypting_reader::calculate_encrypted_size(
@ -531,9 +523,10 @@ auto encrypt_provider::get_filesystem_item_from_source_path(
return get_filesystem_item(api_path, false, fsi);
}
auto encrypt_provider::get_filesystem_item_and_file(
const std::string &api_path, api_file &file,
filesystem_item &fsi) const -> api_error {
auto encrypt_provider::get_filesystem_item_and_file(const std::string &api_path,
api_file &file,
filesystem_item &fsi) const
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
try {
@ -570,7 +563,7 @@ auto encrypt_provider::get_item_meta(const std::string &api_path,
REPERTORY_USES_FUNCTION_NAME();
try {
auto result = utils::db::sqlite::db_select{*db_, source_table}
auto result = utils::db::sqlite::db_select{*db_, file_table}
.column("source_path")
.where("api_path")
.equals(api_path)
@ -619,7 +612,7 @@ auto encrypt_provider::get_total_drive_space() const -> std::uint64_t {
}
auto encrypt_provider::get_total_item_count() const -> std::uint64_t {
auto result = utils::db::sqlite::db_select{*db_, source_table}
auto result = utils::db::sqlite::db_select{*db_, file_table}
.count("api_path", "count")
.go();
@ -641,10 +634,13 @@ auto encrypt_provider::get_used_drive_space() const -> std::uint64_t {
auto encrypt_provider::is_directory(const std::string &api_path,
bool &exists) const -> api_error {
auto result = utils::db::sqlite::db_select{*db_, source_table}
auto result = utils::db::sqlite::db_select{*db_, file_table}
.column("source_path")
.where("api_path")
.equals(api_path)
.and_()
.where("directory")
.equals(1)
.go();
std::optional<utils::db::sqlite::db_result::row> row;
if (not(result.get_row(row) && row.has_value())) {
@ -660,12 +656,15 @@ auto encrypt_provider::is_directory(const std::string &api_path,
return api_error::success;
}
auto encrypt_provider::is_file(const std::string &api_path,
bool &exists) const -> api_error {
auto result = utils::db::sqlite::db_select{*db_, source_table}
auto encrypt_provider::is_file(const std::string &api_path, bool &exists) const
-> api_error {
auto result = utils::db::sqlite::db_select{*db_, file_table}
.column("source_path")
.where("api_path")
.equals(api_path)
.and_()
.where("directory")
.equals(0)
.go();
std::optional<utils::db::sqlite::db_result::row> row;
if (not(result.get_row(row) && row.has_value())) {
@ -722,10 +721,13 @@ auto encrypt_provider::process_directory_entry(
current_source_path = utils::path::combine(current_source_path, {part});
std::string current_api_path{};
auto result = utils::db::sqlite::db_select{*db_, directory_table}
auto result = utils::db::sqlite::db_select{*db_, file_table}
.column("api_path")
.where("source_path")
.equals(current_source_path)
.and_()
.where("directory")
.equals(1)
.go();
std::optional<utils::db::sqlite::db_result::row> row;
if (result.get_row(row) && row.has_value()) {
@ -736,16 +738,13 @@ auto encrypt_provider::process_directory_entry(
current_api_path = utils::path::create_api_path(
current_encrypted_path + '/' + encrypted_parts.at(current_idx));
auto ins_res = utils::db::sqlite::db_insert{*db_, directory_table}
.column_value("source_path", current_source_path)
// TODO handle error
auto ins_res = utils::db::sqlite::db_insert{*db_, file_table}
.column_value("api_path", current_api_path)
.column_value("data", "")
.column_value("directory", 1)
.column_value("source_path", current_source_path)
.go();
// TODO handle error
ins_res = utils::db::sqlite::db_insert{*db_, source_table}
.column_value("api_path", current_api_path)
.column_value("source_path", current_source_path)
.go();
// TODO handle error
event_system::instance().raise<filesystem_item_added>(
current_api_path,
utils::path::get_parent_api_path(current_api_path), true);
@ -773,20 +772,28 @@ auto encrypt_provider::process_directory_entry(
std::string api_path_data{};
auto result = utils::db::sqlite::db_select{*db_, file_table}
.column("api_path")
.column("data")
.where("source_path")
.equals(dir_entry.get_path())
.and_()
.where("directory")
.equals(0)
.go();
std::optional<utils::db::sqlite::db_result::row> row;
if (result.get_row(row) && row.has_value()) {
api_path_data = row->get_column("data").get_value<std::string>();
api_path = row->get_column("api_path").get_value<std::string>();
}
std::string api_parent{};
result = utils::db::sqlite::db_select{*db_, directory_table}
result = utils::db::sqlite::db_select{*db_, file_table}
.column("api_path")
.where("source_path")
.equals(utils::path::get_parent_path(dir_entry.get_path()))
.and_()
.where("directory")
.equals(1)
.go();
row.reset();
if (result.get_row(row) && row.has_value()) {
@ -797,7 +804,7 @@ auto encrypt_provider::process_directory_entry(
api_parent = add_directory(utils::path::get_parent_path(relative_path));
}
if (api_path_data.empty()) {
if (api_path.empty()) {
stop_type stop_requested = false;
utils::encryption::encrypting_reader reader(
utils::path::strip_to_file_name(relative_path), dir_entry.get_path(),
@ -808,7 +815,6 @@ auto encrypt_provider::process_directory_entry(
auto iv_list = reader.get_iv_list();
json data = {
{"api_path", api_path},
{"iv_list", iv_list},
{
"original_file_size",
@ -817,21 +823,17 @@ auto encrypt_provider::process_directory_entry(
.value_or(0U)),
},
};
auto ins_res = utils::db::sqlite::db_insert{*db_, file_table}
.column_value("source_path", dir_entry.get_path())
.column_value("data", data.dump())
.go();
// TODO handle error
ins_res = utils::db::sqlite::db_insert{*db_, source_table}
.column_value("api_path", api_path)
.column_value("source_path", dir_entry.get_path())
.go();
// TODO handle error
auto ins_res = utils::db::sqlite::db_insert{*db_, file_table}
.column_value("api_path", api_path)
.column_value("data", data.dump())
.column_value("directory", 0)
.column_value("source_path", dir_entry.get_path())
.go();
event_system::instance().raise<filesystem_item_added>(api_path,
api_parent, false);
} else {
api_path = json::parse(api_path_data)["api_path"].get<std::string>();
}
return true;
@ -846,10 +848,14 @@ auto encrypt_provider::read_file_bytes(const std::string &api_path,
stop_type &stop_requested) -> api_error {
REPERTORY_USES_FUNCTION_NAME();
auto result = utils::db::sqlite::db_select{*db_, source_table}
auto result = utils::db::sqlite::db_select{*db_, file_table}
.column("data")
.column("source_path")
.where("api_path")
.equals(api_path)
.and_()
.where("directory")
.equals(0)
.go();
std::optional<utils::db::sqlite::db_result::row> row;
if (not(result.get_row(row) && row.has_value())) {
@ -861,16 +867,6 @@ auto encrypt_provider::read_file_bytes(const std::string &api_path,
return api_error::item_not_found;
}
result = utils::db::sqlite::db_select{*db_, file_table}
.column("data")
.where("source_path")
.equals(source_path)
.go();
row.reset();
if (not(result.get_row(row) && row.has_value())) {
return api_error::item_not_found;
}
auto file_data = row->get_column("data").get_value_as_json();
auto opt_size = utils::file::file{source_path}.size();
@ -902,8 +898,8 @@ auto encrypt_provider::read_file_bytes(const std::string &api_path,
file_data["iv_list"] = iv_list;
auto ins_res = utils::db::sqlite::db_insert{*db_, file_table}
.or_replace()
.column_value("source_path", source_path)
.column_value("data", file_data.dump())
.column_value("source_path", source_path)
.go();
if (not ins_res.ok()) {
utils::error::raise_error(function_name, ins_res.get_error(), source_path,
@ -957,7 +953,7 @@ void encrypt_provider::remove_deleted_files(const stop_type &stop_requested) {
std::vector<removed_item> removed_list{};
std::vector<utils::db::sqlite::db_result::row> row_list{};
auto result = utils::db::sqlite::db_select{*db_, source_table}.go();
auto result = utils::db::sqlite::db_select{*db_, file_table}.go();
while (result.has_row()) {
if (stop_requested) {
return;
@ -982,8 +978,11 @@ void encrypt_provider::remove_deleted_files(const stop_type &stop_requested) {
.where("source_path")
.equals(source_path)
.go();
removed_list.emplace_back(
removed_item{api_path, not result.has_row(), source_path});
removed_list.emplace_back(removed_item{
api_path,
row.get_column("directory").get_value<std::int64_t>() == 1,
source_path,
});
}
}
@ -996,16 +995,14 @@ void encrypt_provider::remove_deleted_files(const stop_type &stop_requested) {
continue;
}
auto del_res = utils::db::sqlite::db_delete{*db_, source_table}
.where("api_path")
.equals(item.api_path)
// TODO handle error
auto del_res = utils::db::sqlite::db_delete{*db_, file_table}
.where("source_path")
.equals(item.source_path)
.and_()
.where("directory")
.equals(0)
.go();
// TODO handle error
del_res = utils::db::sqlite::db_delete{*db_, file_table}
.where("source_path")
.equals(item.source_path)
.go();
// TODO handle error
event_system::instance().raise<file_removed_externally>(item.api_path,
item.source_path);
}
@ -1018,17 +1015,15 @@ void encrypt_provider::remove_deleted_files(const stop_type &stop_requested) {
if (not item.directory) {
continue;
}
auto del_res = utils::db::sqlite::db_delete{*db_, source_table}
.where("api_path")
.equals(item.api_path)
// TODO handle error
auto del_res = utils::db::sqlite::db_delete{*db_, file_table}
.where("source_path")
.equals(item.source_path)
.and_()
.where("directory")
.equals(1)
.go();
// TODO handle error
del_res = utils::db::sqlite::db_delete{*db_, directory_table}
.where("source_path")
.equals(item.source_path)
.go();
// TODO handle error
event_system::instance().raise<directory_removed_externally>(
item.api_path, item.source_path);
}
@ -1049,34 +1044,44 @@ auto encrypt_provider::start(api_item_added_callback /*api_item_added*/,
const auto cfg = config_.get_encrypt_config();
std::string source_path;
auto result = utils::db::sqlite::db_select{*db_, source_table}
auto result = utils::db::sqlite::db_select{*db_, file_table}
.column("source_path")
.where("api_path")
.equals("/")
.and_()
.where("directory")
.equals(1)
.go();
std::optional<utils::db::sqlite::db_result::row> row;
if (result.get_row(row) && row.has_value()) {
source_path = row->get_column("source_path").get_value<std::string>();
} else {
auto ins_res = utils::db::sqlite::db_insert{*db_, source_table}
// TODO error handling
auto ins_res = utils::db::sqlite::db_insert{*db_, file_table}
.column_value("api_path", "/")
.column_value("data", "")
.column_value("directory", 1)
.column_value("source_path", cfg.path)
.go();
// TODO error handling
source_path = cfg.path;
}
result = utils::db::sqlite::db_select{*db_, directory_table}
result = utils::db::sqlite::db_select{*db_, file_table}
.column("api_path")
.where("source_path")
.equals(source_path)
.and_()
.where("directory")
.equals(1)
.go();
if (not result.has_row()) {
auto ins_res = utils::db::sqlite::db_insert{*db_, directory_table}
.column_value("source_path", source_path)
.column_value("api_path", "/")
.go();
// TODO error handling
auto ins_res = utils::db::sqlite::db_insert{*db_, file_table}
.column_value("api_path", "/")
.column_value("data", "")
.column_value("directory", 1)
.column_value("source_path", source_path)
.go();
}
polling::instance().set_callback({