BlockStorage/repertory
1
0
Fork 0
Code Issues 3 Pull Requests Releases 9 Wiki Activity

Implement secure key via KDF for transparent data encryption/decryption #60
Some checks reported errors
BlockStorage/repertory/pipeline/head Something is wrong with the build of this commit
Details

Browse Source
This commit is contained in:
sgraves76
2025-08-31 12:08:04 -05:00
parent 5a4e1302f1
commit e542aa64ad
16 changed files with 808 additions and 552 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
repertory/librepertory/include/db/i_file_db.hpp
View File

@@ -29,6 +29,13 @@ class i_file_db {
INTERFACE_SETUP(i_file_db);
public:
struct directory_data final {
std::string api_path;
std::pair<utils::encryption::kdf_config, utils::encryption::kdf_config>
kdf_configs;
std::string source_path;
};
struct file_info final {
std::string api_path;
bool directory{};
@@ -40,13 +47,14 @@ public:
std::uint64_t file_size{};
std::vector<
std::array<unsigned char, crypto_aead_xchacha20poly1305_IETF_NPUBBYTES>>
iv_list{};
iv_list;
std::pair<utils::encryption::kdf_config, utils::encryption::kdf_config>
kdf_configs;
std::string source_path;
};
public:
[[nodiscard]] virtual auto add_directory(const std::string &api_path,
const std::string &source_path)
[[nodiscard]] virtual auto add_or_update_directory(const directory_data &data)
-> api_error = 0;
[[nodiscard]] virtual auto add_or_update_file(const file_data &data)
@@ -68,6 +76,10 @@ public:
get_directory_api_path(const std::string &source_path,
std::string &api_path) const -> api_error = 0;
[[nodiscard]] virtual auto get_directory_data(const std::string &api_path,
directory_data &data) const
-> api_error = 0;
[[nodiscard]] virtual auto
get_directory_source_path(const std::string &api_path,
std::string &source_path) const -> api_error = 0;

7
repertory/librepertory/include/db/impl/rdb_file_db.hpp
View File

@@ -67,8 +67,7 @@ private:
rocksdb::Transaction *txn) -> rocksdb::Status;
public:
[[nodiscard]] auto add_directory(const std::string &api_path,
const std::string &source_path)
[[nodiscard]] auto add_or_update_directory(const directory_data &data)
-> api_error override;
[[nodiscard]] auto add_or_update_file(const i_file_db::file_data &data)
@@ -90,6 +89,10 @@ public:
std::string &api_path) const
-> api_error override;
[[nodiscard]] auto get_directory_data(const std::string &api_path,
i_file_db::directory_data &data) const
-> api_error override;
[[nodiscard]] auto get_directory_source_path(const std::string &api_path,
std::string &source_path) const
-> api_error override;

8
repertory/librepertory/include/db/impl/sqlite_file_db.hpp
View File

@@ -42,8 +42,8 @@ private:
utils::db::sqlite::db3_t db_;
public:
[[nodiscard]] auto add_directory(const std::string &api_path,
const std::string &source_path)
[[nodiscard]] auto
add_or_update_directory(const i_file_db::directory_data &data)
-> api_error override;
[[nodiscard]] auto add_or_update_file(const i_file_db::file_data &data)
@@ -65,6 +65,10 @@ public:
std::string &api_path) const
-> api_error override;
[[nodiscard]] auto get_directory_data(const std::string &api_path,
i_file_db::directory_data &data) const
-> api_error override;
[[nodiscard]] auto get_directory_source_path(const std::string &api_path,
std::string &source_path) const
-> api_error override;

8
repertory/librepertory/include/providers/base_provider.hpp
View File

@@ -49,8 +49,8 @@ private:
private:
api_item_added_callback api_item_added_;
std::unique_ptr<i_meta_db> db3_;
i_file_manager *fm_{};
i_file_manager *fm_{nullptr};
std::unique_ptr<i_meta_db> meta_db_;
private:
void add_all_items(stop_type &stop_requested);
@@ -106,9 +106,9 @@ protected:
return config_;
}
[[nodiscard]] auto get_db() -> i_meta_db & { return *db3_; }
[[nodiscard]] auto get_db() -> i_meta_db & { return *meta_db_; }
[[nodiscard]] auto get_db() const -> const i_meta_db & { return *db3_; }
[[nodiscard]] auto get_db() const -> const i_meta_db & { return *meta_db_; }
[[nodiscard]] virtual auto
get_directory_items_impl(const std::string &api_path,

18
repertory/librepertory/include/providers/encrypt/encrypt_provider.hpp
View File

@@ -57,22 +57,26 @@ private:
encrypt_config encrypt_config_;
private:
std::unique_ptr<i_file_db> db_{nullptr};
std::unique_ptr<i_file_db> file_db_{nullptr};
i_file_manager *fm_{nullptr};
utils::hash::hash_256_t master_key_{};
std::unordered_map<std::string, std::shared_ptr<reader_info>> reader_lookup_;
std::recursive_mutex reader_lookup_mtx_;
private:
static auto create_api_file(const std::string &api_path, bool directory,
const std::string &source_path) -> api_file;
[[nodiscard]] static auto create_api_file(const std::string &api_path,
bool directory,
const std::string &source_path)
-> api_file;
static void create_item_meta(api_meta_map &meta, bool directory,
const api_file &file);
auto do_fs_operation(const std::string &api_path, bool directory,
std::function<api_error(const encrypt_config &cfg,
const std::string &source_path)>
callback) const -> api_error;
[[nodiscard]] auto do_fs_operation(
const std::string &api_path, bool directory,
std::function<api_error(const encrypt_config &cfg,
const std::string &source_path)> callback) const
-> api_error;
[[nodiscard]] auto get_encrypt_config() const -> const encrypt_config & {
return encrypt_config_;

2
repertory/librepertory/include/types/repertory.hpp
View File

@@ -490,7 +490,7 @@ template <> struct adl_serializer<repertory::encrypt_config> {
}
if (not repertory::utils::encryption::kdf_config::from_header(
buffer, value.kdf_cfg)) {
buffer, value.kdf_cfg, true)) {
throw repertory::utils::error::create_exception(
function_name, {"failed to parse kdf header"});
}

71
repertory/librepertory/src/db/impl/rdb_file_db.cpp
View File

@@ -57,12 +57,12 @@ void rdb_file_db::create_or_open(bool clear) {
source_family_ = handles.at(idx++);
}
auto rdb_file_db::add_directory(const std::string &api_path,
const std::string &source_path) -> api_error {
auto rdb_file_db::add_or_update_directory(const i_file_db::directory_data &data)
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
std::string existing_source_path;
auto result = get_directory_source_path(api_path, existing_source_path);
auto result = get_directory_source_path(data.api_path, existing_source_path);
if (result != api_error::success &&
result != api_error::directory_not_found) {
return result;
@@ -71,23 +71,28 @@ auto rdb_file_db::add_directory(const std::string &api_path,
return perform_action(
function_name, [&](rocksdb::Transaction *txn) -> rocksdb::Status {
if (not existing_source_path.empty()) {
auto res = remove_item(api_path, existing_source_path, txn);
auto res = remove_item(data.api_path, existing_source_path, txn);
if (not res.ok() && not res.IsNotFound()) {
return res;
}
}
auto res = txn->Put(directory_family_, api_path, source_path);
json json_data = {
{"kdf_configs", data.kdf_configs},
{"source_path", data.source_path},
};
auto res = txn->Put(directory_family_, data.api_path, json_data.dump());
if (not res.ok()) {
return res;
}
res = txn->Put(path_family_, api_path, source_path);
res = txn->Put(path_family_, data.api_path, data.source_path);
if (not res.ok()) {
return res;
}
return txn->Put(source_family_, source_path, api_path);
return txn->Put(source_family_, data.source_path, data.api_path);
});
}
@@ -113,6 +118,7 @@ auto rdb_file_db::add_or_update_file(const i_file_db::file_data &data)
json json_data = {
{"file_size", data.file_size},
{"iv", data.iv_list},
{"kdf_configs", data.kdf_configs},
{"source_path", data.source_path},
};
@@ -207,8 +213,9 @@ auto rdb_file_db::get_api_path(const std::string &source_path,
});
}
auto rdb_file_db::get_directory_api_path(
const std::string &source_path, std::string &api_path) const -> api_error {
auto rdb_file_db::get_directory_api_path(const std::string &source_path,
std::string &api_path) const
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
auto result = perform_action(function_name, [&]() -> rocksdb::Status {
@@ -231,13 +238,45 @@ auto rdb_file_db::get_directory_api_path(
: result;
}
auto rdb_file_db::get_directory_source_path(
const std::string &api_path, std::string &source_path) const -> api_error {
auto rdb_file_db::get_directory_data(const std::string &api_path,
i_file_db::directory_data &data) const
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
auto result = perform_action(function_name, [&]() -> rocksdb::Status {
return db_->Get(rocksdb::ReadOptions{}, directory_family_, api_path,
&source_path);
std::string value;
auto res =
db_->Get(rocksdb::ReadOptions{}, directory_family_, api_path, &value);
if (not res.ok()) {
return res;
}
auto json_data = json::parse(value);
data.api_path = api_path;
json_data.at("kdf_configs").get_to(data.kdf_configs);
data.source_path = json_data.at("source_path").get<std::string>();
return res;
});
return result;
}
auto rdb_file_db::get_directory_source_path(const std::string &api_path,
std::string &source_path) const
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
auto result = perform_action(function_name, [&]() -> rocksdb::Status {
std::string data;
auto ret =
db_->Get(rocksdb::ReadOptions{}, directory_family_, api_path, &data);
if (ret.ok()) {
source_path =
nlohmann::json::parse(data).at("source_path").get<std::string>();
}
return ret;
});
return result == api_error::item_not_found ? api_error::directory_not_found
@@ -285,6 +324,7 @@ auto rdb_file_db::get_file_data(const std::string &api_path,
.get<std::vector<
std::array<unsigned char,
crypto_aead_xchacha20poly1305_IETF_NPUBBYTES>>>();
json_data.at("kdf_configs").get_to(data.kdf_configs);
data.source_path = json_data.at("source_path").get<std::string>();
return res;
@@ -293,8 +333,9 @@ auto rdb_file_db::get_file_data(const std::string &api_path,
return result;
}
auto rdb_file_db::get_file_source_path(
const std::string &api_path, std::string &source_path) const -> api_error {
auto rdb_file_db::get_file_source_path(const std::string &api_path,
std::string &source_path) const
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
auto result = perform_action(function_name, [&]() -> rocksdb::Status {

83
repertory/librepertory/src/db/impl/sqlite_file_db.cpp
View File

@@ -43,8 +43,9 @@ const std::map<std::string, std::string> sql_create_tables = {
"("
"source_path TEXT PRIMARY KEY ASC, "
"api_path TEXT UNIQUE NOT NULL, "
"iv TEXT DEFAULT '' NOT NULL, "
"directory INTEGER NOT NULL, "
"iv TEXT DEFAULT '' NOT NULL, "
"kdf_configs TEXT NOT NULL, "
"size INTEGER DEFAULT 0 NOT NULL"
");"},
},
@@ -65,21 +66,23 @@ sqlite_file_db::sqlite_file_db(const app_config &cfg) {
sqlite_file_db::~sqlite_file_db() { db_.reset(); }
auto sqlite_file_db::add_directory(
const std::string &api_path, const std::string &source_path) -> api_error {
auto sqlite_file_db::add_or_update_directory(
const i_file_db::directory_data &data) -> api_error {
REPERTORY_USES_FUNCTION_NAME();
auto result = utils::db::sqlite::db_insert{*db_, file_table}
.column_value("api_path", api_path)
.column_value("directory", 1)
.column_value("source_path", source_path)
.go();
auto result =
utils::db::sqlite::db_insert{*db_, file_table}
.column_value("api_path", data.api_path)
.column_value("directory", 1)
.column_value("kdf_configs", nlohmann::json(data.kdf_configs).dump())
.column_value("source_path", data.source_path)
.go();
if (result.ok()) {
return api_error::success;
}
utils::error::raise_api_path_error(
function_name, api_path, api_error::error,
function_name, data.api_path, api_error::error,
fmt::format("failed to add directory|{}", result.get_error_str()));
return api_error::error;
}
@@ -94,6 +97,7 @@ auto sqlite_file_db::add_or_update_file(const i_file_db::file_data &data)
.column_value("api_path", data.api_path)
.column_value("directory", 0)
.column_value("iv", json(data.iv_list).dump())
.column_value("kdf_configs", json(data.kdf_configs).dump())
.column_value("size", static_cast<std::int64_t>(data.file_size))
.column_value("source_path", data.source_path)
.go();
@@ -182,8 +186,9 @@ auto sqlite_file_db::get_api_path(const std::string &source_path,
return api_error::item_not_found;
}
auto sqlite_file_db::get_directory_api_path(
const std::string &source_path, std::string &api_path) const -> api_error {
auto sqlite_file_db::get_directory_api_path(const std::string &source_path,
std::string &api_path) const
-> api_error {
auto result = utils::db::sqlite::db_select{*db_, file_table}
.column("api_path")
.where("source_path")
@@ -204,8 +209,39 @@ auto sqlite_file_db::get_directory_api_path(
return api_error::directory_not_found;
}
auto sqlite_file_db::get_directory_source_path(
const std::string &api_path, std::string &source_path) const -> api_error {
auto sqlite_file_db::get_directory_data(const std::string &api_path,
i_file_db::directory_data &data) const
-> api_error {
auto result = utils::db::sqlite::db_select{*db_, file_table}
.column("kdf_configs")
.column("source_path")
.where("api_path")
.equals(api_path)
.and_()
.where("directory")
.equals(0)
.op()
.limit(1)
.go();
std::optional<utils::db::sqlite::db_result::row> row;
if (result.get_row(row) && row.has_value()) {
data.api_path = api_path;
data.source_path = row->get_column("source_path").get_value<std::string>();
auto str_data = row->get_column("kdf_configs").get_value<std::string>();
if (not str_data.empty()) {
json::parse(str_data).get_to(data.kdf_configs);
}
return api_error::success;
}
return api_error::item_not_found;
}
auto sqlite_file_db::get_directory_source_path(const std::string &api_path,
std::string &source_path) const
-> api_error {
auto result = utils::db::sqlite::db_select{*db_, file_table}
.column("source_path")
.where("api_path")
@@ -226,8 +262,9 @@ auto sqlite_file_db::get_directory_source_path(
return api_error::directory_not_found;
}
auto sqlite_file_db::get_file_api_path(
const std::string &source_path, std::string &api_path) const -> api_error {
auto sqlite_file_db::get_file_api_path(const std::string &source_path,
std::string &api_path) const
-> api_error {
auto result = utils::db::sqlite::db_select{*db_, file_table}
.column("api_path")
.where("source_path")
@@ -253,6 +290,7 @@ auto sqlite_file_db::get_file_data(const std::string &api_path,
-> api_error {
auto result = utils::db::sqlite::db_select{*db_, file_table}
.column("iv")
.column("kdf_configs")
.column("size")
.column("source_path")
.where("api_path")
@@ -280,14 +318,20 @@ auto sqlite_file_db::get_file_data(const std::string &api_path,
crypto_aead_xchacha20poly1305_IETF_NPUBBYTES>>>();
}
str_data = row->get_column("kdf_configs").get_value<std::string>();
if (not str_data.empty()) {
json::parse(str_data).get_to(data.kdf_configs);
}
return api_error::success;
}
return api_error::item_not_found;
}
auto sqlite_file_db::get_file_source_path(
const std::string &api_path, std::string &source_path) const -> api_error {
auto sqlite_file_db::get_file_source_path(const std::string &api_path,
std::string &source_path) const
-> api_error {
auto result = utils::db::sqlite::db_select{*db_, file_table}
.column("source_path")
.where("api_path")
@@ -329,8 +373,9 @@ auto sqlite_file_db::get_item_list(stop_type_callback stop_requested_cb) const
return ret;
}
auto sqlite_file_db::get_source_path(
const std::string &api_path, std::string &source_path) const -> api_error {
auto sqlite_file_db::get_source_path(const std::string &api_path,
std::string &source_path) const
-> api_error {
auto result = utils::db::sqlite::db_select{*db_, file_table}
.column("source_path")
.where("api_path")

37
repertory/librepertory/src/providers/base_provider.cpp
View File

@@ -248,7 +248,7 @@ auto base_provider::create_file(const std::string &api_path, api_meta_map &meta)
stop_type stop_requested{false};
res = upload_file(api_path, meta[META_SOURCE], stop_requested);
if (res != api_error::success) {
db3_->remove_api_path(api_path);
meta_db_->remove_api_path(api_path);
}
return res;
@@ -272,7 +272,7 @@ auto base_provider::get_api_path_from_source(const std::string &source_path,
return api_error::item_not_found;
}
return db3_->get_api_path(source_path, api_path);
return meta_db_->get_api_path(source_path, api_path);
}
auto base_provider::get_directory_item(const std::string &api_path,
@@ -481,25 +481,25 @@ auto base_provider::get_filesystem_item_from_source_path(
auto base_provider::get_item_meta(const std::string &api_path,
api_meta_map &meta) const -> api_error {
return db3_->get_item_meta(api_path, meta);
return meta_db_->get_item_meta(api_path, meta);
}
auto base_provider::get_item_meta(const std::string &api_path,
const std::string &key,
std::string &value) const -> api_error {
return db3_->get_item_meta(api_path, key, value);
return meta_db_->get_item_meta(api_path, key, value);
}
auto base_provider::get_pinned_files() const -> std::vector<std::string> {
return db3_->get_pinned_files();
return meta_db_->get_pinned_files();
}
auto base_provider::get_total_item_count() const -> std::uint64_t {
return db3_->get_total_item_count();
return meta_db_->get_total_item_count();
}
auto base_provider::get_used_drive_space() const -> std::uint64_t {
return db3_->get_total_size();
return meta_db_->get_total_size();
}
auto base_provider::is_file_writeable(const std::string &api_path) const
@@ -530,7 +530,7 @@ void base_provider::process_removed_directories(
continue;
}
db3_->remove_api_path(item.api_path);
meta_db_->remove_api_path(item.api_path);
event_system::instance().raise<directory_removed_externally>(
item.api_path, function_name, item.source_path);
}
@@ -557,7 +557,7 @@ void base_provider::process_removed_files(std::deque<removed_item> removed_list,
}
if (not utils::file::file{item.source_path}.exists()) {
db3_->remove_api_path(item.api_path);
meta_db_->remove_api_path(item.api_path);
event_system::instance().raise<file_removed_externally>(
item.api_path, function_name, item.source_path);
continue;
@@ -601,7 +601,7 @@ void base_provider::process_removed_files(std::deque<removed_item> removed_list,
continue;
}
db3_->remove_api_path(item.api_path);
meta_db_->remove_api_path(item.api_path);
event_system::instance().raise<file_removed_externally>(
item.api_path, function_name, item.source_path);
}
@@ -612,7 +612,7 @@ void base_provider::process_removed_items(stop_type &stop_requested) {
return stop_requested || app_config::get_stop_requested();
};
db3_->enumerate_api_path_list(
meta_db_->enumerate_api_path_list(
[this, &get_stop_requested](auto &&list) {
[[maybe_unused]] auto res =
std::all_of(list.begin(), list.end(), [&](auto &&api_path) -> bool {
@@ -706,7 +706,7 @@ auto base_provider::remove_file(const std::string &api_path) -> api_error {
const auto remove_file_meta = [this, &api_path, &notify_end]() -> api_error {
api_meta_map meta{};
auto res = get_item_meta(api_path, meta);
db3_->remove_api_path(api_path);
meta_db_->remove_api_path(api_path);
return notify_end(res);
};
@@ -766,14 +766,14 @@ auto base_provider::remove_directory(const std::string &api_path) -> api_error {
return notify_end(res);
}
db3_->remove_api_path(api_path);
meta_db_->remove_api_path(api_path);
return notify_end(api_error::success);
}
auto base_provider::remove_item_meta(const std::string &api_path,
const std::string &key) -> api_error {
return db3_->remove_item_meta(api_path, key);
return meta_db_->remove_item_meta(api_path, key);
}
void base_provider::remove_unmatched_source_files(stop_type &stop_requested) {
@@ -831,12 +831,12 @@ void base_provider::remove_unmatched_source_files(stop_type &stop_requested) {
auto base_provider::set_item_meta(const std::string &api_path,
const std::string &key,
const std::string &value) -> api_error {
return db3_->set_item_meta(api_path, key, value);
return meta_db_->set_item_meta(api_path, key, value);
}
auto base_provider::set_item_meta(const std::string &api_path,
const api_meta_map &meta) -> api_error {
return db3_->set_item_meta(api_path, meta);
return meta_db_->set_item_meta(api_path, meta);
}
auto base_provider::start(api_item_added_callback api_item_added,
@@ -845,8 +845,7 @@ auto base_provider::start(api_item_added_callback api_item_added,
api_item_added_ = api_item_added;
fm_ = mgr;
db3_ = create_meta_db(config_);
meta_db_ = create_meta_db(config_);
api_meta_map meta{};
if (get_item_meta("/", meta) == api_error::item_not_found) {
@@ -906,7 +905,7 @@ auto base_provider::start(api_item_added_callback api_item_added,
void base_provider::stop() {
cache_size_mgr::instance().stop();
polling::instance().remove_callback("check_deleted");
db3_.reset();
meta_db_.reset();
}
auto base_provider::upload_file(const std::string &api_path,

289
repertory/librepertory/src/providers/encrypt/encrypt_provider.cpp
View File

@@ -22,6 +22,7 @@
#include "providers/encrypt/encrypt_provider.hpp"
#include "db/file_db.hpp"
#include "db/meta_db.hpp"
#include "events/event_system.hpp"
#include "events/types/directory_removed_externally.hpp"
#include "events/types/file_removed_externally.hpp"
@@ -32,7 +33,8 @@
#include "events/types/service_stop_end.hpp"
#include "types/repertory.hpp"
#include "types/startup_exception.hpp"
#include "utils/collection.hpp"
#include "utils/base64.hpp"
#include "utils/common.hpp"
#include "utils/config.hpp"
#include "utils/encrypting_reader.hpp"
#include "utils/encryption.hpp"
@@ -64,7 +66,7 @@ auto encrypt_provider::create_api_file(const std::string &api_path,
directory
? 0U
: utils::encryption::encrypting_reader::calculate_encrypted_size(
source_path, false);
source_path, true);
file.modified_date = times->get(utils::file::time_type::written);
file.source_path = source_path;
@@ -163,7 +165,7 @@ auto encrypt_provider::get_api_path_from_source(const std::string &source_path,
REPERTORY_USES_FUNCTION_NAME();
try {
return db_->get_api_path(source_path, api_path);
return file_db_->get_api_path(source_path, api_path);
} catch (const std::exception &ex) {
utils::error::raise_error(function_name, ex, source_path,
"failed to get api path from source path");
@@ -245,38 +247,38 @@ auto encrypt_provider::get_directory_items(const std::string &api_path,
try {
std::string current_api_path;
if (dir_entry->is_directory_item()) {
auto result{
db_->get_directory_api_path(dir_entry->get_path(),
current_api_path),
auto res{
file_db_->get_directory_api_path(dir_entry->get_path(),
current_api_path),
};
if (result != api_error::success &&
result != api_error::directory_not_found) {
if (res != api_error::success &&
res != api_error::directory_not_found) {
// TODO raise error
continue;
}
if (result == api_error::directory_not_found) {
if (res == api_error::directory_not_found) {
process_directory_entry(*dir_entry, cfg, current_api_path);
result = db_->get_directory_api_path(dir_entry->get_path(),
current_api_path);
if (result != api_error::success &&
result != api_error::directory_not_found) {
res = file_db_->get_directory_api_path(dir_entry->get_path(),
current_api_path);
if (res != api_error::success &&
res != api_error::directory_not_found) {
// TODO raise error
continue;
}
}
} else {
auto result{
db_->get_file_api_path(dir_entry->get_path(),
current_api_path),
auto res{
file_db_->get_file_api_path(dir_entry->get_path(),
current_api_path),
};
if (result != api_error::success &&
result != api_error::item_not_found) {
if (res != api_error::success &&
res != api_error::item_not_found) {
// TODO raise error
continue;
}
if (result == api_error::item_not_found &&
if (res == api_error::item_not_found &&
not process_directory_entry(*dir_entry, cfg,
current_api_path)) {
continue;
@@ -357,9 +359,9 @@ auto encrypt_provider::get_file(const std::string &api_path,
}
std::string source_path;
auto result{db_->get_file_source_path(api_path, source_path)};
if (result != api_error::success) {
return result;
res = file_db_->get_file_source_path(api_path, source_path);
if (res != api_error::success) {
return res;
}
file = create_api_file(api_path, false, source_path);
@@ -414,13 +416,13 @@ auto encrypt_provider::get_file_size(const std::string &api_path,
try {
std::string source_path;
auto result{db_->get_file_source_path(api_path, source_path)};
if (result != api_error::success) {
return result;
auto res{file_db_->get_file_source_path(api_path, source_path)};
if (res != api_error::success) {
return res;
}
file_size = utils::encryption::encrypting_reader::calculate_encrypted_size(
source_path, false);
source_path, true);
return api_error::success;
} catch (const std::exception &ex) {
utils::error::raise_error(function_name, ex, api_path,
@@ -436,9 +438,9 @@ auto encrypt_provider::get_filesystem_item(const std::string &api_path,
-> api_error {
std::string source_path;
if (directory) {
auto result{db_->get_directory_source_path(api_path, source_path)};
if (result != api_error::success) {
return result;
auto res{file_db_->get_directory_source_path(api_path, source_path)};
if (res != api_error::success) {
return res;
}
fsi.api_parent = utils::path::get_parent_api_path(api_path);
@@ -449,16 +451,16 @@ auto encrypt_provider::get_filesystem_item(const std::string &api_path,
return api_error::success;
}
auto result{db_->get_file_source_path(api_path, source_path)};
if (result != api_error::success) {
return result;
auto res{file_db_->get_file_source_path(api_path, source_path)};
if (res != api_error::success) {
return res;
}
fsi.api_path = api_path;
fsi.api_parent = utils::path::get_parent_api_path(fsi.api_path);
fsi.directory = false;
fsi.size = utils::encryption::encrypting_reader::calculate_encrypted_size(
source_path, false);
source_path, true);
fsi.source_path = source_path;
return api_error::success;
@@ -525,15 +527,15 @@ auto encrypt_provider::get_item_meta(const std::string &api_path,
try {
std::string source_path;
auto result{db_->get_source_path(api_path, source_path)};
if (result != api_error::success) {
return result;
auto res{file_db_->get_source_path(api_path, source_path)};
if (res != api_error::success) {
return res;
}
bool is_dir{};
result = is_directory(api_path, is_dir);
if (result != api_error::success) {
return result;
res = is_directory(api_path, is_dir);
if (res != api_error::success) {
return res;
}
auto file{create_api_file(api_path, is_dir, source_path)};
@@ -569,7 +571,7 @@ auto encrypt_provider::get_total_item_count() const -> std::uint64_t {
REPERTORY_USES_FUNCTION_NAME();
try {
return db_->count();
return file_db_->count();
} catch (const std::exception &ex) {
utils::error::raise_error(function_name, ex,
"failed to get total item count");
@@ -592,11 +594,11 @@ auto encrypt_provider::is_directory(const std::string &api_path,
try {
std::string source_path;
auto result{db_->get_directory_source_path(api_path, source_path)};
auto res{file_db_->get_directory_source_path(api_path, source_path)};
if (result != api_error::success) {
if (result != api_error::directory_not_found) {
return result;
if (res != api_error::success) {
if (res != api_error::directory_not_found) {
return res;
}
exists = false;
@@ -619,10 +621,10 @@ auto encrypt_provider::is_file(const std::string &api_path, bool &exists) const
try {
std::string source_path;
auto result{db_->get_file_source_path(api_path, source_path)};
if (result != api_error::success) {
if (result != api_error::item_not_found) {
return result;
auto res{file_db_->get_file_source_path(api_path, source_path)};
if (res != api_error::success) {
if (res != api_error::item_not_found) {
return res;
}
exists = false;
@@ -661,53 +663,100 @@ auto encrypt_provider::process_directory_entry(
false),
};
auto current_source_path{cfg.path};
std::string current_api_path;
for (std::size_t part_idx = 1U; part_idx < encrypted_parts.size();
++part_idx) {
current_source_path = utils::path::combine(
current_source_path, {encrypted_parts.at(part_idx)});
i_file_db::directory_data dir_data{};
std::string dir_api_path;
auto res =
file_db_->get_directory_api_path(current_source_path, dir_api_path);
if (res == api_error::success) {
res = file_db_->get_directory_data(dir_api_path, dir_data);
if (res != api_error::success) {
throw std::runtime_error(
fmt::format("failed to get directory file data|{}",
api_error_to_string(res)));
}
} else if (res == api_error::directory_not_found) {
dir_data.kdf_configs.first = dir_data.kdf_configs.second =
cfg.kdf_cfg;
dir_data.kdf_configs.first.unique_id =
utils::generate_secure_random<std::uint64_t>();
dir_data.kdf_configs.second.unique_id =
utils::generate_secure_random<std::uint64_t>();
dir_data.kdf_configs.first.seal();
dir_data.kdf_configs.second.seal();
} else {
throw std::runtime_error(fmt::format(
"failed to get directory api path|{}", api_error_to_string(res)));
}
auto path_key = dir_data.kdf_configs.second.recreate_subkey(
utils::encryption::kdf_context::path, master_key_);
data_buffer encrypted_data;
utils::encryption::encrypt_data(
cfg.encryption_token,
path_key,
reinterpret_cast<const unsigned char *>(
encrypted_parts.at(part_idx).c_str()),
strnlen(encrypted_parts.at(part_idx).c_str(),
encrypted_parts.at(part_idx).size()),
encrypted_data);
encrypted_parts[part_idx] =
utils::collection::to_hex_string(encrypted_data);
auto hdr = dir_data.kdf_configs.second.to_header();
encrypted_data.insert(encrypted_data.begin(), hdr.begin(), hdr.end());
encrypted_parts[part_idx] = macaron::Base64::EncodeUrlSafe(
encrypted_data.data(), encrypted_data.size());
if (dir_api_path.empty()) {
current_api_path = utils::path::create_api_path(
current_api_path + '/' + encrypted_parts.at(part_idx));
dir_data.api_path = current_api_path;
dir_data.source_path = current_source_path;
res = file_db_->add_or_update_directory(dir_data);
if (res != api_error::success) {
throw std::runtime_error(
fmt::format("failed to set directory file data|{}",
api_error_to_string(res)));
}
event_system::instance().raise<filesystem_item_added>(
utils::path::get_parent_api_path(dir_api_path), dir_api_path,
true, function_name);
}
}
std::size_t current_idx{1U};
std::string current_encrypted_path{};
auto current_source_path{cfg.path};
current_source_path = cfg.path;
for (const auto &part : utils::path::get_parts(dir_path)) {
current_source_path = utils::path::combine(current_source_path, {part});
std::string current_api_path{};
auto result{
db_->get_directory_api_path(current_source_path, current_api_path),
};
if (result == api_error::directory_not_found) {
current_api_path = utils::path::create_api_path(
current_encrypted_path + '/' + encrypted_parts.at(current_idx));
result = db_->add_directory(current_api_path, current_source_path);
if (result != api_error::success) {
std::runtime_error(
auto dir_api_path = utils::path::create_api_path(
current_encrypted_path + '/' + encrypted_parts.at(current_idx));
i_file_db::directory_data dir_data;
auto res = file_db_->get_directory_data(dir_api_path, dir_data);
if (res == api_error::directory_not_found) {
dir_data.source_path = current_api_path;
res = file_db_->add_or_update_directory(dir_data);
if (res != api_error::success) {
throw std::runtime_error(
fmt::format("failed to get directory api path|{}",
api_error_to_string(result)));
api_error_to_string(res)));
}
event_system::instance().raise<filesystem_item_added>(
utils::path::get_parent_api_path(current_api_path),
current_api_path, true, function_name);
} else {
if (result != api_error::success) {
std::runtime_error(
fmt::format("failed to get directory api path|{}",
api_error_to_string(result)));
}
utils::path::get_parent_api_path(dir_api_path), dir_api_path,
true, function_name);
} else if (res == api_error::success) {
encrypted_parts[current_idx] =
utils::string::split(current_api_path, '/', false)[current_idx];
utils::string::split(dir_api_path, '/', false)[current_idx];
} else {
throw std::runtime_error(fmt::format(
"failed to get directory api path|{}", api_error_to_string(res)));
}
current_encrypted_path = utils::path::create_api_path(
@@ -728,17 +777,16 @@ auto encrypt_provider::process_directory_entry(
utils::path::get_relative_path(dir_entry.get_path(), cfg.path),
};
i_file_db::file_data data;
auto file_res{db_->get_file_data(dir_entry.get_path(), data)};
if (file_res != api_error::success &&
file_res != api_error::item_not_found) {
i_file_db::file_data file_data{};
auto file_res{file_db_->get_file_data(api_path, file_data)};
if (file_res != api_error::success) {
// TODO raise error
return false;
}
std::string api_parent{};
auto parent_res{
db_->get_directory_api_path(
file_db_->get_directory_api_path(
utils::path::get_parent_path(dir_entry.get_path()), api_parent),
};
if (parent_res != api_error::success &&
@@ -757,16 +805,22 @@ auto encrypt_provider::process_directory_entry(
utils::path::strip_to_file_name(relative_path),
dir_entry.get_path(),
[]() -> bool { return app_config::get_stop_requested(); },
cfg.encryption_token, utils::path::get_parent_path(relative_path));
master_key_, file_data.kdf_configs,
utils::path::get_parent_path(relative_path));
api_path = utils::path::create_api_path(
api_parent + "/" + reader.get_encrypted_file_name());
file_res = db_->add_or_update_file(i_file_db::file_data{
file_res = file_db_->add_or_update_file(i_file_db::file_data{
.api_path = api_path,
.file_size = dynamic_cast<const utils::file::i_file *>(&dir_entry)
->size()
.value_or(0U),
.iv_list = reader.get_iv_list(),
.kdf_configs =
{
*reader.get_kdf_config_for_data(),
*reader.get_kdf_config_for_path(),
},
.source_path = dir_entry.get_path(),
});
if (file_res != api_error::success) {
@@ -795,9 +849,9 @@ auto encrypt_provider::read_file_bytes(const std::string &api_path,
REPERTORY_USES_FUNCTION_NAME();
i_file_db::file_data file_data{};
auto result{db_->get_file_data(api_path, file_data)};
if (result != api_error::success) {
return result;
auto res{file_db_->get_file_data(api_path, file_data)};
if (res != api_error::success) {
return res;
}
auto opt_size{utils::file::file{file_data.source_path}.size()};
@@ -822,18 +876,19 @@ auto encrypt_provider::read_file_bytes(const std::string &api_path,
[&stop_requested]() -> bool {
return stop_requested || app_config::get_stop_requested();
},
cfg.encryption_token, utils::path::get_parent_path(relative_path));
master_key_, file_data.kdf_configs,
utils::path::get_parent_path(relative_path));
reader_lookup_[file_data.source_path] = info;
file_data.file_size = file_size;
file_data.iv_list = info->reader->get_iv_list();
result = db_->add_or_update_file(file_data);
res = file_db_->add_or_update_file(file_data);
file_data.iv_list.clear();
if (result != api_error::success) {
utils::error::raise_error(function_name, result, file_data.source_path,
if (res != api_error::success) {
utils::error::raise_error(function_name, res, file_data.source_path,
"failed to update file");
return result;
return res;
}
} else if (not reader_lookup_.contains(file_data.source_path)) {
auto info{std::make_shared<reader_info>()};
@@ -841,7 +896,7 @@ auto encrypt_provider::read_file_bytes(const std::string &api_path,
[&stop_requested]() -> bool {
return stop_requested || app_config::get_stop_requested();
},
api_path, file_data.source_path, cfg.encryption_token,
api_path, file_data.source_path, master_key_, file_data.kdf_configs,
std::move(file_data.iv_list));
reader_lookup_[file_data.source_path] = info;
}
@@ -858,12 +913,12 @@ auto encrypt_provider::read_file_bytes(const std::string &api_path,
info->reader->set_read_position(offset);
data.resize(size);
auto res{
auto ret{
utils::encryption::encrypting_reader::reader_function(
reinterpret_cast<char *>(data.data()), 1U, data.size(),
info->reader.get()),
};
return res == 0 ? api_error::os_error : api_error::success;
return ret == 0 ? api_error::os_error : api_error::success;
}
void encrypt_provider::remove_deleted_files(stop_type &stop_requested) {
@@ -873,7 +928,7 @@ void encrypt_provider::remove_deleted_files(stop_type &stop_requested) {
return stop_requested || app_config::get_stop_requested();
};
db_->enumerate_item_list(
file_db_->enumerate_item_list(
[this, &get_stop_requested](auto &&list) {
std::vector<i_file_db::file_info> removed_list{};
for (const auto &item : list) {
@@ -893,7 +948,7 @@ void encrypt_provider::remove_deleted_files(stop_type &stop_requested) {
return;
}
auto res{db_->remove_item(item.api_path)};
auto res{file_db_->remove_item(item.api_path)};
if (res != api_error::success) {
utils::error::raise_api_path_error(
function_name, item.api_path, item.source_path, res,
@@ -943,18 +998,37 @@ auto encrypt_provider::start(api_item_added_callback /*api_item_added*/,
event_system::instance().raise<service_start_begin>(function_name,
"encrypt_provider");
db_ = create_file_db(config_);
if (encrypt_config_.kdf_cfg.checksum == 0U) {
encrypt_config_.kdf_cfg.seal();
config_.set_encrypt_config(encrypt_config_);
}
if (encrypt_config_.kdf_cfg.checksum !=
encrypt_config_.kdf_cfg.generate_checksum()) {
utils::error::raise_error(
function_name, "existing kdf configuration failed checksum validation");
return false;
}
if (not utils::encryption::recreate_key_argon2id(
get_encrypt_config().encryption_token, get_encrypt_config().kdf_cfg,
master_key_)) {
utils::error::raise_error(function_name,
"failed to recreate master key from kdf config");
return false;
}
file_db_ = create_file_db(config_);
std::string source_path;
auto result{db_->get_directory_source_path("/", source_path)};
if (result != api_error::success &&
result != api_error::directory_not_found) {
auto res{file_db_->get_directory_source_path("/", source_path)};
if (res != api_error::success && res != api_error::directory_not_found) {
throw startup_exception(
fmt::format("failed to get root|{}", api_error_to_string(result)));
fmt::format("failed to get root|{}", api_error_to_string(res)));
}
auto cfg_path{utils::path::absolute(get_encrypt_config().path)};
if (result == api_error::success) {
if (res == api_error::success) {
auto cur_path{utils::path::absolute(source_path)};
#if defined(_WIN32)
if (utils::string::to_lower(cur_path) !=
@@ -966,10 +1040,15 @@ auto encrypt_provider::start(api_item_added_callback /*api_item_added*/,
"source path has changed|cur|{}|cfg|{}", cur_path, cfg_path));
}
} else {
result = db_->add_directory("/", utils::path::absolute(cfg_path));
if (result != api_error::success) {
i_file_db::directory_data dir_data{
.api_path = "/",
.kdf_configs = {},
.source_path = cfg_path,
};
res = file_db_->add_or_update_directory(dir_data);
if (res != api_error::success) {
throw startup_exception(
fmt::format("failed to create root|{}", api_error_to_string(result)));
fmt::format("failed to create root|{}", api_error_to_string(res)));
}
}
@@ -984,7 +1063,7 @@ auto encrypt_provider::start(api_item_added_callback /*api_item_added*/,
polling::instance().set_callback({
.name = "remove_expired",
.freq = polling::frequency::high,
.freq = polling::frequency::second,
.action =
[this](auto && /* stop_requested */) { remove_expired_files(); },
});
@@ -1006,7 +1085,7 @@ void encrypt_provider::stop() {
reader_lookup_.clear();
reader_lookup_lock.unlock();
db_.reset();
file_db_.reset();
event_system::instance().raise<service_stop_end>(function_name,
"encrypt_provider");
}

8
repertory/librepertory/src/providers/s3/s3_provider.cpp
View File

@@ -1025,7 +1025,8 @@ auto s3_provider::search_keys_for_master_kdf(
std::string token{};
std::string response_data{};
long response_code{};
if (not get_object_list(response_data, response_code, "/", "", token)) {
if (not get_object_list(response_data, response_code, std::nullopt,
std::nullopt, token)) {
throw utils::error::create_exception(function_name,
{"failed to get object list"});
}
@@ -1060,9 +1061,10 @@ auto s3_provider::search_keys_for_master_kdf(
auto buffer = macaron::Base64::Decode(object_name);
if (not utils::encryption::kdf_config::from_header(buffer,
master_kdf_cfg_)) {
continue;
throw std::runtime_error("from_header failed");
}
} catch (...) {
} catch (const std::exception &e) {
utils::error::raise_error(function_name, e, "exception occurred");
continue;
}