8 Commits

Author SHA1 Message Date
f44972b8b3 refactor
All checks were successful
BlockStorage/repertory_osx_builds/pipeline/head This commit looks good
BlockStorage/repertory_linux_builds/pipeline/head This commit looks good
2023-12-13 15:32:30 -06:00
94675a3011 refactor 2023-12-13 15:22:19 -06:00
7112fbee7e refactor 2023-12-13 15:21:21 -06:00
ff13633962 refactor 2023-12-13 14:57:08 -06:00
cb93e34de0 refactor 2023-12-13 14:53:39 -06:00
883968f53e refactor 2023-12-13 14:50:19 -06:00
d6d4b579c9 fix encryption provider 2023-12-13 13:34:21 -06:00
0e83d84360 fix encription provider 2023-12-13 10:02:05 -06:00
4 changed files with 348 additions and 295 deletions

View File

@ -65,10 +65,17 @@ private:
static void create_item_meta(api_meta_map &meta, bool directory,
const api_file &file);
[[nodiscard]] auto
auto do_directory_operation(
const std::string &api_path, bool directory,
std::function<api_error(const encrypt_config &cfg,
const std::string &source_path)>
callback) const -> api_error;
auto
process_directory_entry(const std::filesystem::directory_entry &dir_entry,
const encrypt_config &cfg,
std::string &api_path) const -> bool;
void remove_deleted_files();
public:

View File

@ -51,7 +51,7 @@ pushd ..
ln -sf ${BUILD_FOLDER}/compile_commands.json .
ln -sf ${BUILD_FOLDER}/repertory${EXE_EXT} .
ln -sf ${BUILD_FOLDER}/unittests${EXE_EXT} .
if [ "${IS_MINGW}" == "1" ]; then
if [ "${IS_MINGW}" == "1" ] || [ "${IS_WIN32}" == "1" ]; then
ln -sf ${BUILD_FOLDER}/winfsp-x64.dll .
fi
popd
@ -67,7 +67,7 @@ pushd ../${BUILD_ROOT}
ln -sf ${BUILD_FOLDER}/compile_commands.json .
ln -sf ${BUILD_FOLDER}/repertory${EXE_EXT} .
ln -sf ${BUILD_FOLDER}/unittests${EXE_EXT} .
if [ "${IS_MINGW}" == "1" ]; then
if [ "${IS_MINGW}" == "1" ] || [ "${IS_WIN32}" == "1" ]; then
ln -sf ${BUILD_FOLDER}/winfsp-x64.dll .
fi
popd

View File

@ -26,9 +26,9 @@
#include "database/db_select.hpp"
#include "events/event_system.hpp"
#include "events/events.hpp"
#include "platform/platform.hpp"
#include "types/repertory.hpp"
#include "utils/encrypting_reader.hpp"
#include "utils/encryption.hpp"
#include "utils/path_utils.hpp"
#include "utils/polling.hpp"
@ -169,6 +169,49 @@ auto encrypt_provider::create_directory(const std::string &api_path,
return api_error::not_implemented;
}
auto encrypt_provider::do_directory_operation(
const std::string &api_path, bool directory,
std::function<api_error(const encrypt_config &cfg,
const std::string &source_path)>
callback) const -> api_error {
auto cfg = config_.get_encrypt_config();
std::string source_path{api_path};
if (api_path != "/") {
auto res =
utils::encryption::decrypt_file_path(cfg.encryption_token, source_path);
if (res != api_error::success) {
return res;
}
}
source_path =
utils::path::absolute(utils::path::combine(cfg.path, {source_path}));
if (source_path != cfg.path &&
not source_path.starts_with(cfg.path +
utils::path::directory_seperator)) {
return api_error::directory_not_found;
}
auto exists = utils::file::is_file(source_path);
if (exists && directory) {
return api_error::item_exists;
}
if (not exists && not directory) {
return api_error::item_not_found;
}
exists = utils::file::is_directory(source_path);
if (exists && not directory) {
return api_error::item_exists;
}
if (not exists && directory) {
return api_error::directory_not_found;
}
return callback(cfg, source_path);
}
auto encrypt_provider::get_api_path_from_source(const std::string &source_path,
std::string &api_path) const
-> api_error {
@ -210,38 +253,28 @@ auto encrypt_provider::get_api_path_from_source(const std::string &source_path,
auto encrypt_provider::get_directory_item_count(
const std::string &api_path) const -> std::uint64_t {
auto result = db::db_select{*db_, source_table}
.column("source_path")
.where("api_path")
.equals(api_path)
.go();
std::optional<db::db_select::row> row;
if (not(result.get_row(row) && row.has_value())) {
return 0U;
}
auto source_path = row->get_column("source_path").get_value<std::string>();
result = db::db_select{*db_, directory_table}
.column("api_path")
.where("source_path")
.equals(source_path)
.go();
if (not result.has_row()) {
return 0U;
}
const auto cfg = config_.get_encrypt_config();
static const auto *function_name = __FUNCTION__;
std::uint64_t count{};
try {
for ([[maybe_unused]] const auto &dir_entry :
std::filesystem::directory_iterator(source_path)) {
count++;
}
} catch (const std::exception &ex) {
utils::error::raise_error(__FUNCTION__, ex, cfg.path,
"failed to get directory item count");
return 0U;
auto res = do_directory_operation(
api_path, true,
[&api_path, &count](const encrypt_config & /* cfg */,
const std::string &source_path) -> api_error {
try {
for ([[maybe_unused]] const auto &dir_entry :
std::filesystem::directory_iterator(source_path)) {
count++;
}
} catch (const std::exception &ex) {
utils::error::raise_api_path_error(
function_name, api_path, source_path, ex,
"failed to get directory item count");
}
return api_error::success;
});
if (res != api_error::success) {
utils::error::raise_api_path_error(function_name, api_path, res,
"failed to get directory item count");
}
return count;
@ -250,131 +283,113 @@ auto encrypt_provider::get_directory_item_count(
auto encrypt_provider::get_directory_items(const std::string &api_path,
directory_item_list &list) const
-> api_error {
bool exists{};
auto res = is_file(api_path, exists);
if (res != api_error::success) {
return res;
}
if (exists) {
return api_error::item_exists;
}
static const auto *function_name = __FUNCTION__;
auto result = db::db_select{*db_, source_table}
.column("source_path")
.where("api_path")
.equals(api_path)
.go();
std::optional<db::db_select::row> row;
if (not(result.get_row(row) && row.has_value())) {
return api_error::directory_not_found;
}
auto source_path = row->get_column("source_path").get_value<std::string>();
return do_directory_operation(
api_path, true,
[this, &list](const encrypt_config &cfg,
const std::string &source_path) -> api_error {
try {
for (const auto &dir_entry :
std::filesystem::directory_iterator(source_path)) {
try {
std::string current_api_path{};
if (dir_entry.is_directory()) {
auto result = db::db_select{*db_, directory_table}
.column("api_path")
.where("source_path")
.equals(dir_entry.path().string())
.go();
std::optional<db::db_select::row> row;
if (result.get_row(row) && row.has_value()) {
current_api_path =
row->get_column("api_path").get_value<std::string>();
}
if (current_api_path.empty()) {
process_directory_entry(dir_entry, cfg, current_api_path);
result = db::db_select{*db_, directory_table}
.column("api_path")
.where("source_path")
.equals(source_path)
.go();
if (not result.has_row()) {
return api_error::directory_not_found;
}
result = db::db_select{*db_, directory_table}
.column("api_path")
.where("source_path")
.equals(dir_entry.path().string())
.go();
row.reset();
if (not(result.get_row(row) && row.has_value())) {
continue;
}
try {
for (const auto &dir_entry :
std::filesystem::directory_iterator(source_path)) {
try {
std::string current_api_path{};
if (dir_entry.is_directory()) {
result = db::db_select{*db_, directory_table}
.column("api_path")
.where("source_path")
.equals(dir_entry.path().string())
.go();
row.reset();
if (result.get_row(row) && row.has_value()) {
current_api_path =
row->get_column("api_path").get_value<std::string>();
}
if (current_api_path.empty()) {
const auto cfg = config_.get_encrypt_config();
for (const auto &child_dir_entry :
std::filesystem::directory_iterator(dir_entry.path())) {
if (process_directory_entry(child_dir_entry, cfg,
current_api_path)) {
current_api_path =
utils::path::get_parent_api_path(current_api_path);
break;
current_api_path =
row->get_column("api_path").get_value<std::string>();
}
} else {
std::string api_path_data{};
auto result = db::db_select{*db_, file_table}
.column("data")
.where("source_path")
.equals(dir_entry.path().string())
.go();
std::optional<db::db_select::row> row;
if (result.get_row(row) && row.has_value()) {
api_path_data =
row->get_column("data").get_value<std::string>();
}
if (api_path_data.empty()) {
if (not process_directory_entry(dir_entry, cfg,
current_api_path)) {
continue;
}
} else {
current_api_path = json::parse(api_path_data)
.at("api_path")
.get<std::string>();
}
}
}
if (current_api_path.empty()) {
continue;
}
}
} else {
std::string api_path_data{};
result = db::db_select{*db_, file_table}
.column("data")
.where("source_path")
.equals(dir_entry.path().string())
.go();
row.reset();
if (result.get_row(row) && row.has_value()) {
api_path_data = row->get_column("data").get_value<std::string>();
}
auto file =
create_api_file(current_api_path, dir_entry.is_directory(),
dir_entry.path().string());
if (api_path_data.empty()) {
const auto cfg = config_.get_encrypt_config();
if (not process_directory_entry(dir_entry, cfg, current_api_path)) {
continue;
directory_item dir_item{};
dir_item.api_parent = file.api_parent;
dir_item.api_path = file.api_path;
dir_item.directory = dir_entry.is_directory();
dir_item.resolved = true;
dir_item.size = file.file_size;
create_item_meta(dir_item.meta, dir_item.directory, file);
list.emplace_back(std::move(dir_item));
} catch (const std::exception &ex) {
utils::error::raise_error(function_name, ex,
dir_entry.path().string(),
"failed to process directory item");
}
} else {
current_api_path =
json::parse(api_path_data).at("api_path").get<std::string>();
}
} catch (const std::exception &ex) {
utils::error::raise_error(function_name, ex, source_path,
"failed to get directory items");
return api_error::error;
}
auto file = create_api_file(current_api_path, dir_entry.is_directory(),
dir_entry.path().string());
std::sort(list.begin(), list.end(),
[](const auto &item1, const auto &item2) -> bool {
return (item1.directory && not item2.directory) ||
(not(item2.directory && not item1.directory) &&
(item1.api_path.compare(item2.api_path) < 0));
});
directory_item dir_item{};
dir_item.api_parent = file.api_parent;
dir_item.api_path = file.api_path;
dir_item.directory = dir_entry.is_directory();
dir_item.resolved = true;
dir_item.size = file.file_size;
create_item_meta(dir_item.meta, dir_item.directory, file);
list.emplace_back(std::move(dir_item));
} catch (const std::exception &ex) {
utils::error::raise_error(__FUNCTION__, ex, dir_entry.path().string(),
"failed to process directory item");
}
}
} catch (const std::exception &ex) {
utils::error::raise_error(__FUNCTION__, ex, source_path,
"failed to get directory items");
return api_error::error;
}
std::sort(list.begin(), list.end(), [](const auto &a, const auto &b) -> bool {
return (a.directory && not b.directory) ||
(not(b.directory && not a.directory) &&
(a.api_path.compare(b.api_path) < 0));
});
list.insert(list.begin(), directory_item{
"..",
"",
true,
});
list.insert(list.begin(), directory_item{
".",
"",
true,
});
return api_error::success;
list.insert(list.begin(), directory_item{
"..",
"",
true,
});
list.insert(list.begin(), directory_item{
".",
"",
true,
});
return api_error::success;
});
}
auto encrypt_provider::get_file(const std::string &api_path,
@ -403,135 +418,6 @@ auto encrypt_provider::get_file(const std::string &api_path,
return api_error::success;
}
auto encrypt_provider::process_directory_entry(
const std::filesystem::directory_entry &dir_entry,
const encrypt_config &cfg, std::string &api_path) const -> bool {
if (dir_entry.is_regular_file() && not dir_entry.is_symlink() &&
not dir_entry.is_directory()) {
const auto relative_path = dir_entry.path().lexically_relative(cfg.path);
std::string api_path_data{};
auto result = db::db_select{*db_, file_table}
.column("data")
.where("source_path")
.equals(dir_entry.path().string())
.go();
std::optional<db::db_select::row> row;
if (result.get_row(row) && row.has_value()) {
api_path_data = row->get_column("data").get_value<std::string>();
}
std::string api_parent{};
result = db::db_select{*db_, directory_table}
.column("api_path")
.where("source_path")
.equals(dir_entry.path().parent_path().string())
.go();
row.reset();
if (result.get_row(row) && row.has_value()) {
api_parent = row->get_column("api_path").get_value<std::string>();
}
if (api_path_data.empty() || api_parent.empty()) {
stop_type stop_requested = false;
utils::encryption::encrypting_reader reader(
relative_path.filename().string(), dir_entry.path().string(),
stop_requested, cfg.encryption_token,
relative_path.parent_path().string());
if (api_parent.empty()) {
auto encrypted_parts =
utils::string::split(reader.get_encrypted_file_path(), '/', false);
std::size_t idx{1U};
std::string current_source_path{cfg.path};
std::string current_encrypted_path{};
for (const auto &part : relative_path.parent_path()) {
if (part.string() == "/") {
continue;
}
current_source_path =
utils::path::combine(current_source_path, {part.string()});
std::string parent_api_path{};
result = db::db_select{*db_, directory_table}
.column("api_path")
.where("source_path")
.equals(current_source_path)
.go();
row.reset();
if (result.get_row(row) && row.has_value()) {
parent_api_path =
row->get_column("api_path").get_value<std::string>();
}
if (parent_api_path.empty()) {
parent_api_path = utils::path::create_api_path(
current_encrypted_path + '/' + encrypted_parts[idx]);
auto ins_res = db::db_insert{*db_, directory_table}
.column_value("source_path", current_source_path)
.column_value("api_path", parent_api_path)
.go();
// TODO handle error
ins_res = db::db_insert{*db_, source_table}
.column_value("api_path", parent_api_path)
.column_value("source_path", current_source_path)
.go();
// TODO handle error
event_system::instance().raise<filesystem_item_added>(
parent_api_path,
utils::path::get_parent_api_path(parent_api_path), true);
} else {
encrypted_parts[idx] =
utils::string::split(parent_api_path, '/', false)[idx];
}
current_encrypted_path = utils::path::create_api_path(
current_encrypted_path + '/' + encrypted_parts[idx++]);
}
api_parent = current_encrypted_path;
}
if (api_path_data.empty()) {
api_path = utils::path::create_api_path(
api_parent + "/" + reader.get_encrypted_file_name());
auto iv_list = reader.get_iv_list();
json data = {
{"api_path", api_path},
{"iv_list", iv_list},
{"original_file_size", dir_entry.file_size()},
};
auto ins_res =
db::db_insert{*db_, file_table}
.column_value("source_path", dir_entry.path().string())
.column_value("data", data.dump())
.go();
// TODO handle error
ins_res = db::db_insert{*db_, source_table}
.column_value("api_path", api_path)
.column_value("source_path", dir_entry.path().string())
.go();
// TODO handle error
event_system::instance().raise<filesystem_item_added>(
api_path, api_parent, false);
} else {
api_path = json::parse(api_path_data)["api_path"].get<std::string>();
}
} else {
api_path = json::parse(api_path_data)["api_path"].get<std::string>();
}
return true;
}
return false;
}
auto encrypt_provider::get_file_list(api_file_list &list) const -> api_error {
const auto cfg = config_.get_encrypt_config();
@ -793,6 +679,149 @@ auto encrypt_provider::is_online() const -> bool {
auto encrypt_provider::is_rename_supported() const -> bool { return false; }
auto encrypt_provider::process_directory_entry(
const std::filesystem::directory_entry &dir_entry,
const encrypt_config &cfg, std::string &api_path) const -> bool {
const auto add_directory = [this, &cfg](auto dir_path) -> std::string {
auto encrypted_parts = utils::string::split(
utils::path::create_api_path(dir_path.string()), '/', false);
for (std::size_t part_idx = 1U; part_idx < encrypted_parts.size();
part_idx++) {
data_buffer encrypted_data;
utils::encryption::encrypt_data(
cfg.encryption_token, encrypted_parts.at(part_idx).c_str(),
strnlen(encrypted_parts.at(part_idx).c_str(),
encrypted_parts.at(part_idx).size()),
encrypted_data);
encrypted_parts[part_idx] = utils::to_hex_string(encrypted_data);
}
std::size_t current_idx{1U};
std::string current_encrypted_path{};
std::string current_source_path{cfg.path};
for (const auto &part : dir_path) {
if (part.string() == "/") {
continue;
}
current_source_path =
utils::path::combine(current_source_path, {part.string()});
std::string current_api_path{};
auto result = db::db_select{*db_, directory_table}
.column("api_path")
.where("source_path")
.equals(current_source_path)
.go();
std::optional<db::db_select::row> row;
if (result.get_row(row) && row.has_value()) {
current_api_path = row->get_column("api_path").get_value<std::string>();
}
if (current_api_path.empty()) {
current_api_path = utils::path::create_api_path(
current_encrypted_path + '/' + encrypted_parts.at(current_idx));
auto ins_res = db::db_insert{*db_, directory_table}
.column_value("source_path", current_source_path)
.column_value("api_path", current_api_path)
.go();
// TODO handle error
ins_res = db::db_insert{*db_, source_table}
.column_value("api_path", current_api_path)
.column_value("source_path", current_source_path)
.go();
// TODO handle error
event_system::instance().raise<filesystem_item_added>(
current_api_path,
utils::path::get_parent_api_path(current_api_path), true);
} else {
encrypted_parts[current_idx] =
utils::string::split(current_api_path, '/', false)[current_idx];
}
current_encrypted_path = utils::path::create_api_path(
current_encrypted_path + '/' + encrypted_parts.at(current_idx++));
std::cout << current_source_path << ':' << current_encrypted_path
<< std::endl;
}
return current_encrypted_path;
};
if (dir_entry.is_directory()) {
api_path = add_directory(dir_entry.path().lexically_relative(cfg.path));
return false;
}
if (dir_entry.is_regular_file() && not dir_entry.is_symlink()) {
const auto relative_path = dir_entry.path().lexically_relative(cfg.path);
std::string api_path_data{};
auto result = db::db_select{*db_, file_table}
.column("data")
.where("source_path")
.equals(dir_entry.path().string())
.go();
std::optional<db::db_select::row> row;
if (result.get_row(row) && row.has_value()) {
api_path_data = row->get_column("data").get_value<std::string>();
}
std::string api_parent{};
result = db::db_select{*db_, directory_table}
.column("api_path")
.where("source_path")
.equals(dir_entry.path().parent_path().string())
.go();
row.reset();
if (result.get_row(row) && row.has_value()) {
api_parent = row->get_column("api_path").get_value<std::string>();
}
if (api_parent.empty()) {
api_parent = add_directory(relative_path.parent_path());
}
if (api_path_data.empty()) {
stop_type stop_requested = false;
utils::encryption::encrypting_reader reader(
relative_path.filename().string(), dir_entry.path().string(),
stop_requested, cfg.encryption_token,
relative_path.parent_path().string());
api_path = utils::path::create_api_path(api_parent + "/" +
reader.get_encrypted_file_name());
auto iv_list = reader.get_iv_list();
json data = {
{"api_path", api_path},
{"iv_list", iv_list},
{"original_file_size", dir_entry.file_size()},
};
auto ins_res = db::db_insert{*db_, file_table}
.column_value("source_path", dir_entry.path().string())
.column_value("data", data.dump())
.go();
// TODO handle error
ins_res = db::db_insert{*db_, source_table}
.column_value("api_path", api_path)
.column_value("source_path", dir_entry.path().string())
.go();
// TODO handle error
event_system::instance().raise<filesystem_item_added>(api_path,
api_parent, false);
} else {
api_path = json::parse(api_path_data)["api_path"].get<std::string>();
}
return true;
}
return false;
}
auto encrypt_provider::read_file_bytes(const std::string &api_path,
std::size_t size, std::uint64_t offset,
data_buffer &data,
@ -841,12 +870,12 @@ auto encrypt_provider::read_file_bytes(const std::string &api_path,
const auto relative_path =
std::filesystem::path(source_path).lexically_relative(cfg.path);
auto ri = std::make_shared<reader_info>();
ri->reader = std::make_unique<utils::encryption::encrypting_reader>(
auto info = std::make_shared<reader_info>();
info->reader = std::make_unique<utils::encryption::encrypting_reader>(
relative_path.filename().string(), source_path, stop_requested,
cfg.encryption_token, relative_path.parent_path().string());
reader_lookup_[source_path] = ri;
iv_list = ri->reader->get_iv_list();
reader_lookup_[source_path] = info;
iv_list = info->reader->get_iv_list();
file_data["original_file_size"] = file_size;
file_data["iv_list"] = iv_list;
@ -866,11 +895,11 @@ auto encrypt_provider::read_file_bytes(const std::string &api_path,
std::array<unsigned char,
crypto_aead_xchacha20poly1305_IETF_NPUBBYTES>>>();
if (reader_lookup_.find(source_path) == reader_lookup_.end()) {
auto ri = std::make_shared<reader_info>();
ri->reader = std::make_unique<utils::encryption::encrypting_reader>(
auto info = std::make_shared<reader_info>();
info->reader = std::make_unique<utils::encryption::encrypting_reader>(
api_path, source_path, stop_requested, cfg.encryption_token,
std::move(iv_list));
reader_lookup_[source_path] = ri;
reader_lookup_[source_path] = info;
}
}
@ -878,16 +907,16 @@ auto encrypt_provider::read_file_bytes(const std::string &api_path,
return api_error::success;
}
auto ri = reader_lookup_.at(source_path);
ri->last_access_time = std::chrono::system_clock::now();
auto info = reader_lookup_.at(source_path);
info->last_access_time = std::chrono::system_clock::now();
reader_lookup_lock.unlock();
mutex_lock reader_lock(ri->reader_mtx);
ri->reader->set_read_position(offset);
mutex_lock reader_lock(info->reader_mtx);
info->reader->set_read_position(offset);
data.resize(size);
const auto res = ri->reader->reader_function(data.data(), 1u, data.size(),
ri->reader.get());
const auto res = info->reader->reader_function(data.data(), 1U, data.size(),
info->reader.get());
if (res == 0) {
return api_error::os_error;
}

View File

@ -22,25 +22,42 @@
#include "test_common.hpp"
#include "app_config.hpp"
#include "file_manager/file_manager.hpp"
#include "events/consumers/console_consumer.hpp"
#include "providers/encrypt/encrypt_provider.hpp"
#include "utils/path_utils.hpp"
namespace repertory {
/* TEST(encrypt_provider, can_construct_encrypt_provider) {
/*
TEST(encrypt_provider, can_construct_encrypt_provider) {
ASSERT_TRUE(
utils::file::delete_directory_recursively("./encrypt_provider_test"));
// console_consumer consumer{};
// event_system::instance().start();
{
console_consumer cc{};
event_system::instance().start();
app_config cfg(provider_type::encrypt, "./encrypt_provider_test");
EXPECT_FALSE(
cfg.set_value_by_name("EncryptConfig.Path", "c:\\src").empty());
encrypt_provider provider(cfg);
file_manager mgr(cfg, provider);
mgr.start();
event_system::instance().stop();
EXPECT_TRUE(provider.start(
[&provider](bool directory, api_file &file) -> api_error {
return provider_meta_handler(provider, directory, file);
},
&mgr));
directory_item_list list{};
EXPECT_EQ(api_error::success, provider.get_directory_items("/", list));
provider.stop();
mgr.stop();
}
// event_system::instance().stop();
ASSERT_TRUE(
utils::file::delete_directory_recursively("./encrypt_provider_test"));
}