diff --git a/repertory/librepertory/include/db/i_file_db.hpp b/repertory/librepertory/include/db/i_file_db.hpp index ae05c3a3..c990ec65 100644 --- a/repertory/librepertory/include/db/i_file_db.hpp +++ b/repertory/librepertory/include/db/i_file_db.hpp @@ -29,6 +29,13 @@ class i_file_db { INTERFACE_SETUP(i_file_db); public: + struct directory_data final { + std::string api_path; + std::pair + kdf_configs; + std::string source_path; + }; + struct file_info final { std::string api_path; bool directory{}; @@ -40,13 +47,14 @@ public: std::uint64_t file_size{}; std::vector< std::array> - iv_list{}; + iv_list; + std::pair + kdf_configs; std::string source_path; }; public: - [[nodiscard]] virtual auto add_directory(const std::string &api_path, - const std::string &source_path) + [[nodiscard]] virtual auto add_or_update_directory(const directory_data &data) -> api_error = 0; [[nodiscard]] virtual auto add_or_update_file(const file_data &data) @@ -68,6 +76,10 @@ public: get_directory_api_path(const std::string &source_path, std::string &api_path) const -> api_error = 0; + [[nodiscard]] virtual auto get_directory_data(const std::string &api_path, + directory_data &data) const + -> api_error = 0; + [[nodiscard]] virtual auto get_directory_source_path(const std::string &api_path, std::string &source_path) const -> api_error = 0; diff --git a/repertory/librepertory/include/db/impl/rdb_file_db.hpp b/repertory/librepertory/include/db/impl/rdb_file_db.hpp index 902c777d..68190482 100644 --- a/repertory/librepertory/include/db/impl/rdb_file_db.hpp +++ b/repertory/librepertory/include/db/impl/rdb_file_db.hpp @@ -67,8 +67,7 @@ private: rocksdb::Transaction *txn) -> rocksdb::Status; public: - [[nodiscard]] auto add_directory(const std::string &api_path, - const std::string &source_path) + [[nodiscard]] auto add_or_update_directory(const directory_data &data) -> api_error override; [[nodiscard]] auto add_or_update_file(const i_file_db::file_data &data) @@ -90,6 +89,10 @@ public: std::string &api_path) const -> api_error override; + [[nodiscard]] auto get_directory_data(const std::string &api_path, + i_file_db::directory_data &data) const + -> api_error override; + [[nodiscard]] auto get_directory_source_path(const std::string &api_path, std::string &source_path) const -> api_error override; diff --git a/repertory/librepertory/include/db/impl/sqlite_file_db.hpp b/repertory/librepertory/include/db/impl/sqlite_file_db.hpp index 4a9a7d82..49f68cf3 100644 --- a/repertory/librepertory/include/db/impl/sqlite_file_db.hpp +++ b/repertory/librepertory/include/db/impl/sqlite_file_db.hpp @@ -42,8 +42,8 @@ private: utils::db::sqlite::db3_t db_; public: - [[nodiscard]] auto add_directory(const std::string &api_path, - const std::string &source_path) + [[nodiscard]] auto + add_or_update_directory(const i_file_db::directory_data &data) -> api_error override; [[nodiscard]] auto add_or_update_file(const i_file_db::file_data &data) @@ -65,6 +65,10 @@ public: std::string &api_path) const -> api_error override; + [[nodiscard]] auto get_directory_data(const std::string &api_path, + i_file_db::directory_data &data) const + -> api_error override; + [[nodiscard]] auto get_directory_source_path(const std::string &api_path, std::string &source_path) const -> api_error override; diff --git a/repertory/librepertory/include/providers/base_provider.hpp b/repertory/librepertory/include/providers/base_provider.hpp index c389ade3..49e58963 100644 --- a/repertory/librepertory/include/providers/base_provider.hpp +++ b/repertory/librepertory/include/providers/base_provider.hpp @@ -49,8 +49,8 @@ private: private: api_item_added_callback api_item_added_; - std::unique_ptr db3_; - i_file_manager *fm_{}; + i_file_manager *fm_{nullptr}; + std::unique_ptr meta_db_; private: void add_all_items(stop_type &stop_requested); @@ -106,9 +106,9 @@ protected: return config_; } - [[nodiscard]] auto get_db() -> i_meta_db & { return *db3_; } + [[nodiscard]] auto get_db() -> i_meta_db & { return *meta_db_; } - [[nodiscard]] auto get_db() const -> const i_meta_db & { return *db3_; } + [[nodiscard]] auto get_db() const -> const i_meta_db & { return *meta_db_; } [[nodiscard]] virtual auto get_directory_items_impl(const std::string &api_path, diff --git a/repertory/librepertory/include/providers/encrypt/encrypt_provider.hpp b/repertory/librepertory/include/providers/encrypt/encrypt_provider.hpp index d26dbf5c..0d6b643c 100644 --- a/repertory/librepertory/include/providers/encrypt/encrypt_provider.hpp +++ b/repertory/librepertory/include/providers/encrypt/encrypt_provider.hpp @@ -57,22 +57,26 @@ private: encrypt_config encrypt_config_; private: - std::unique_ptr db_{nullptr}; + std::unique_ptr file_db_{nullptr}; i_file_manager *fm_{nullptr}; + utils::hash::hash_256_t master_key_{}; std::unordered_map> reader_lookup_; std::recursive_mutex reader_lookup_mtx_; private: - static auto create_api_file(const std::string &api_path, bool directory, - const std::string &source_path) -> api_file; + [[nodiscard]] static auto create_api_file(const std::string &api_path, + bool directory, + const std::string &source_path) + -> api_file; static void create_item_meta(api_meta_map &meta, bool directory, const api_file &file); - auto do_fs_operation(const std::string &api_path, bool directory, - std::function - callback) const -> api_error; + [[nodiscard]] auto do_fs_operation( + const std::string &api_path, bool directory, + std::function callback) const + -> api_error; [[nodiscard]] auto get_encrypt_config() const -> const encrypt_config & { return encrypt_config_; diff --git a/repertory/librepertory/include/types/repertory.hpp b/repertory/librepertory/include/types/repertory.hpp index 5572d97f..e0124e01 100644 --- a/repertory/librepertory/include/types/repertory.hpp +++ b/repertory/librepertory/include/types/repertory.hpp @@ -490,7 +490,7 @@ template <> struct adl_serializer { } if (not repertory::utils::encryption::kdf_config::from_header( - buffer, value.kdf_cfg)) { + buffer, value.kdf_cfg, true)) { throw repertory::utils::error::create_exception( function_name, {"failed to parse kdf header"}); } diff --git a/repertory/librepertory/src/db/impl/rdb_file_db.cpp b/repertory/librepertory/src/db/impl/rdb_file_db.cpp index 8e1e1810..186b3990 100644 --- a/repertory/librepertory/src/db/impl/rdb_file_db.cpp +++ b/repertory/librepertory/src/db/impl/rdb_file_db.cpp @@ -57,12 +57,12 @@ void rdb_file_db::create_or_open(bool clear) { source_family_ = handles.at(idx++); } -auto rdb_file_db::add_directory(const std::string &api_path, - const std::string &source_path) -> api_error { +auto rdb_file_db::add_or_update_directory(const i_file_db::directory_data &data) + -> api_error { REPERTORY_USES_FUNCTION_NAME(); std::string existing_source_path; - auto result = get_directory_source_path(api_path, existing_source_path); + auto result = get_directory_source_path(data.api_path, existing_source_path); if (result != api_error::success && result != api_error::directory_not_found) { return result; @@ -71,23 +71,28 @@ auto rdb_file_db::add_directory(const std::string &api_path, return perform_action( function_name, [&](rocksdb::Transaction *txn) -> rocksdb::Status { if (not existing_source_path.empty()) { - auto res = remove_item(api_path, existing_source_path, txn); + auto res = remove_item(data.api_path, existing_source_path, txn); if (not res.ok() && not res.IsNotFound()) { return res; } } - auto res = txn->Put(directory_family_, api_path, source_path); + json json_data = { + {"kdf_configs", data.kdf_configs}, + {"source_path", data.source_path}, + }; + + auto res = txn->Put(directory_family_, data.api_path, json_data.dump()); if (not res.ok()) { return res; } - res = txn->Put(path_family_, api_path, source_path); + res = txn->Put(path_family_, data.api_path, data.source_path); if (not res.ok()) { return res; } - return txn->Put(source_family_, source_path, api_path); + return txn->Put(source_family_, data.source_path, data.api_path); }); } @@ -113,6 +118,7 @@ auto rdb_file_db::add_or_update_file(const i_file_db::file_data &data) json json_data = { {"file_size", data.file_size}, {"iv", data.iv_list}, + {"kdf_configs", data.kdf_configs}, {"source_path", data.source_path}, }; @@ -207,8 +213,9 @@ auto rdb_file_db::get_api_path(const std::string &source_path, }); } -auto rdb_file_db::get_directory_api_path( - const std::string &source_path, std::string &api_path) const -> api_error { +auto rdb_file_db::get_directory_api_path(const std::string &source_path, + std::string &api_path) const + -> api_error { REPERTORY_USES_FUNCTION_NAME(); auto result = perform_action(function_name, [&]() -> rocksdb::Status { @@ -231,13 +238,45 @@ auto rdb_file_db::get_directory_api_path( : result; } -auto rdb_file_db::get_directory_source_path( - const std::string &api_path, std::string &source_path) const -> api_error { +auto rdb_file_db::get_directory_data(const std::string &api_path, + i_file_db::directory_data &data) const + -> api_error { REPERTORY_USES_FUNCTION_NAME(); auto result = perform_action(function_name, [&]() -> rocksdb::Status { - return db_->Get(rocksdb::ReadOptions{}, directory_family_, api_path, - &source_path); + std::string value; + auto res = + db_->Get(rocksdb::ReadOptions{}, directory_family_, api_path, &value); + if (not res.ok()) { + return res; + } + + auto json_data = json::parse(value); + data.api_path = api_path; + json_data.at("kdf_configs").get_to(data.kdf_configs); + data.source_path = json_data.at("source_path").get(); + + return res; + }); + + return result; +} + +auto rdb_file_db::get_directory_source_path(const std::string &api_path, + std::string &source_path) const + -> api_error { + REPERTORY_USES_FUNCTION_NAME(); + + auto result = perform_action(function_name, [&]() -> rocksdb::Status { + std::string data; + auto ret = + db_->Get(rocksdb::ReadOptions{}, directory_family_, api_path, &data); + if (ret.ok()) { + source_path = + nlohmann::json::parse(data).at("source_path").get(); + } + + return ret; }); return result == api_error::item_not_found ? api_error::directory_not_found @@ -285,6 +324,7 @@ auto rdb_file_db::get_file_data(const std::string &api_path, .get>>(); + json_data.at("kdf_configs").get_to(data.kdf_configs); data.source_path = json_data.at("source_path").get(); return res; @@ -293,8 +333,9 @@ auto rdb_file_db::get_file_data(const std::string &api_path, return result; } -auto rdb_file_db::get_file_source_path( - const std::string &api_path, std::string &source_path) const -> api_error { +auto rdb_file_db::get_file_source_path(const std::string &api_path, + std::string &source_path) const + -> api_error { REPERTORY_USES_FUNCTION_NAME(); auto result = perform_action(function_name, [&]() -> rocksdb::Status { diff --git a/repertory/librepertory/src/db/impl/sqlite_file_db.cpp b/repertory/librepertory/src/db/impl/sqlite_file_db.cpp index c7cdd024..bbf54b1c 100644 --- a/repertory/librepertory/src/db/impl/sqlite_file_db.cpp +++ b/repertory/librepertory/src/db/impl/sqlite_file_db.cpp @@ -43,8 +43,9 @@ const std::map sql_create_tables = { "(" "source_path TEXT PRIMARY KEY ASC, " "api_path TEXT UNIQUE NOT NULL, " - "iv TEXT DEFAULT '' NOT NULL, " "directory INTEGER NOT NULL, " + "iv TEXT DEFAULT '' NOT NULL, " + "kdf_configs TEXT NOT NULL, " "size INTEGER DEFAULT 0 NOT NULL" ");"}, }, @@ -65,21 +66,23 @@ sqlite_file_db::sqlite_file_db(const app_config &cfg) { sqlite_file_db::~sqlite_file_db() { db_.reset(); } -auto sqlite_file_db::add_directory( - const std::string &api_path, const std::string &source_path) -> api_error { +auto sqlite_file_db::add_or_update_directory( + const i_file_db::directory_data &data) -> api_error { REPERTORY_USES_FUNCTION_NAME(); - auto result = utils::db::sqlite::db_insert{*db_, file_table} - .column_value("api_path", api_path) - .column_value("directory", 1) - .column_value("source_path", source_path) - .go(); + auto result = + utils::db::sqlite::db_insert{*db_, file_table} + .column_value("api_path", data.api_path) + .column_value("directory", 1) + .column_value("kdf_configs", nlohmann::json(data.kdf_configs).dump()) + .column_value("source_path", data.source_path) + .go(); if (result.ok()) { return api_error::success; } utils::error::raise_api_path_error( - function_name, api_path, api_error::error, + function_name, data.api_path, api_error::error, fmt::format("failed to add directory|{}", result.get_error_str())); return api_error::error; } @@ -94,6 +97,7 @@ auto sqlite_file_db::add_or_update_file(const i_file_db::file_data &data) .column_value("api_path", data.api_path) .column_value("directory", 0) .column_value("iv", json(data.iv_list).dump()) + .column_value("kdf_configs", json(data.kdf_configs).dump()) .column_value("size", static_cast(data.file_size)) .column_value("source_path", data.source_path) .go(); @@ -182,8 +186,9 @@ auto sqlite_file_db::get_api_path(const std::string &source_path, return api_error::item_not_found; } -auto sqlite_file_db::get_directory_api_path( - const std::string &source_path, std::string &api_path) const -> api_error { +auto sqlite_file_db::get_directory_api_path(const std::string &source_path, + std::string &api_path) const + -> api_error { auto result = utils::db::sqlite::db_select{*db_, file_table} .column("api_path") .where("source_path") @@ -204,8 +209,39 @@ auto sqlite_file_db::get_directory_api_path( return api_error::directory_not_found; } -auto sqlite_file_db::get_directory_source_path( - const std::string &api_path, std::string &source_path) const -> api_error { +auto sqlite_file_db::get_directory_data(const std::string &api_path, + i_file_db::directory_data &data) const + -> api_error { + auto result = utils::db::sqlite::db_select{*db_, file_table} + .column("kdf_configs") + .column("source_path") + .where("api_path") + .equals(api_path) + .and_() + .where("directory") + .equals(0) + .op() + .limit(1) + .go(); + + std::optional row; + if (result.get_row(row) && row.has_value()) { + data.api_path = api_path; + data.source_path = row->get_column("source_path").get_value(); + + auto str_data = row->get_column("kdf_configs").get_value(); + if (not str_data.empty()) { + json::parse(str_data).get_to(data.kdf_configs); + } + + return api_error::success; + } + + return api_error::item_not_found; +} +auto sqlite_file_db::get_directory_source_path(const std::string &api_path, + std::string &source_path) const + -> api_error { auto result = utils::db::sqlite::db_select{*db_, file_table} .column("source_path") .where("api_path") @@ -226,8 +262,9 @@ auto sqlite_file_db::get_directory_source_path( return api_error::directory_not_found; } -auto sqlite_file_db::get_file_api_path( - const std::string &source_path, std::string &api_path) const -> api_error { +auto sqlite_file_db::get_file_api_path(const std::string &source_path, + std::string &api_path) const + -> api_error { auto result = utils::db::sqlite::db_select{*db_, file_table} .column("api_path") .where("source_path") @@ -253,6 +290,7 @@ auto sqlite_file_db::get_file_data(const std::string &api_path, -> api_error { auto result = utils::db::sqlite::db_select{*db_, file_table} .column("iv") + .column("kdf_configs") .column("size") .column("source_path") .where("api_path") @@ -280,14 +318,20 @@ auto sqlite_file_db::get_file_data(const std::string &api_path, crypto_aead_xchacha20poly1305_IETF_NPUBBYTES>>>(); } + str_data = row->get_column("kdf_configs").get_value(); + if (not str_data.empty()) { + json::parse(str_data).get_to(data.kdf_configs); + } + return api_error::success; } return api_error::item_not_found; } -auto sqlite_file_db::get_file_source_path( - const std::string &api_path, std::string &source_path) const -> api_error { +auto sqlite_file_db::get_file_source_path(const std::string &api_path, + std::string &source_path) const + -> api_error { auto result = utils::db::sqlite::db_select{*db_, file_table} .column("source_path") .where("api_path") @@ -329,8 +373,9 @@ auto sqlite_file_db::get_item_list(stop_type_callback stop_requested_cb) const return ret; } -auto sqlite_file_db::get_source_path( - const std::string &api_path, std::string &source_path) const -> api_error { +auto sqlite_file_db::get_source_path(const std::string &api_path, + std::string &source_path) const + -> api_error { auto result = utils::db::sqlite::db_select{*db_, file_table} .column("source_path") .where("api_path") diff --git a/repertory/librepertory/src/providers/base_provider.cpp b/repertory/librepertory/src/providers/base_provider.cpp index 6a9f4f2e..2b1dbaca 100644 --- a/repertory/librepertory/src/providers/base_provider.cpp +++ b/repertory/librepertory/src/providers/base_provider.cpp @@ -248,7 +248,7 @@ auto base_provider::create_file(const std::string &api_path, api_meta_map &meta) stop_type stop_requested{false}; res = upload_file(api_path, meta[META_SOURCE], stop_requested); if (res != api_error::success) { - db3_->remove_api_path(api_path); + meta_db_->remove_api_path(api_path); } return res; @@ -272,7 +272,7 @@ auto base_provider::get_api_path_from_source(const std::string &source_path, return api_error::item_not_found; } - return db3_->get_api_path(source_path, api_path); + return meta_db_->get_api_path(source_path, api_path); } auto base_provider::get_directory_item(const std::string &api_path, @@ -481,25 +481,25 @@ auto base_provider::get_filesystem_item_from_source_path( auto base_provider::get_item_meta(const std::string &api_path, api_meta_map &meta) const -> api_error { - return db3_->get_item_meta(api_path, meta); + return meta_db_->get_item_meta(api_path, meta); } auto base_provider::get_item_meta(const std::string &api_path, const std::string &key, std::string &value) const -> api_error { - return db3_->get_item_meta(api_path, key, value); + return meta_db_->get_item_meta(api_path, key, value); } auto base_provider::get_pinned_files() const -> std::vector { - return db3_->get_pinned_files(); + return meta_db_->get_pinned_files(); } auto base_provider::get_total_item_count() const -> std::uint64_t { - return db3_->get_total_item_count(); + return meta_db_->get_total_item_count(); } auto base_provider::get_used_drive_space() const -> std::uint64_t { - return db3_->get_total_size(); + return meta_db_->get_total_size(); } auto base_provider::is_file_writeable(const std::string &api_path) const @@ -530,7 +530,7 @@ void base_provider::process_removed_directories( continue; } - db3_->remove_api_path(item.api_path); + meta_db_->remove_api_path(item.api_path); event_system::instance().raise( item.api_path, function_name, item.source_path); } @@ -557,7 +557,7 @@ void base_provider::process_removed_files(std::deque removed_list, } if (not utils::file::file{item.source_path}.exists()) { - db3_->remove_api_path(item.api_path); + meta_db_->remove_api_path(item.api_path); event_system::instance().raise( item.api_path, function_name, item.source_path); continue; @@ -601,7 +601,7 @@ void base_provider::process_removed_files(std::deque removed_list, continue; } - db3_->remove_api_path(item.api_path); + meta_db_->remove_api_path(item.api_path); event_system::instance().raise( item.api_path, function_name, item.source_path); } @@ -612,7 +612,7 @@ void base_provider::process_removed_items(stop_type &stop_requested) { return stop_requested || app_config::get_stop_requested(); }; - db3_->enumerate_api_path_list( + meta_db_->enumerate_api_path_list( [this, &get_stop_requested](auto &&list) { [[maybe_unused]] auto res = std::all_of(list.begin(), list.end(), [&](auto &&api_path) -> bool { @@ -706,7 +706,7 @@ auto base_provider::remove_file(const std::string &api_path) -> api_error { const auto remove_file_meta = [this, &api_path, ¬ify_end]() -> api_error { api_meta_map meta{}; auto res = get_item_meta(api_path, meta); - db3_->remove_api_path(api_path); + meta_db_->remove_api_path(api_path); return notify_end(res); }; @@ -766,14 +766,14 @@ auto base_provider::remove_directory(const std::string &api_path) -> api_error { return notify_end(res); } - db3_->remove_api_path(api_path); + meta_db_->remove_api_path(api_path); return notify_end(api_error::success); } auto base_provider::remove_item_meta(const std::string &api_path, const std::string &key) -> api_error { - return db3_->remove_item_meta(api_path, key); + return meta_db_->remove_item_meta(api_path, key); } void base_provider::remove_unmatched_source_files(stop_type &stop_requested) { @@ -831,12 +831,12 @@ void base_provider::remove_unmatched_source_files(stop_type &stop_requested) { auto base_provider::set_item_meta(const std::string &api_path, const std::string &key, const std::string &value) -> api_error { - return db3_->set_item_meta(api_path, key, value); + return meta_db_->set_item_meta(api_path, key, value); } auto base_provider::set_item_meta(const std::string &api_path, const api_meta_map &meta) -> api_error { - return db3_->set_item_meta(api_path, meta); + return meta_db_->set_item_meta(api_path, meta); } auto base_provider::start(api_item_added_callback api_item_added, @@ -845,8 +845,7 @@ auto base_provider::start(api_item_added_callback api_item_added, api_item_added_ = api_item_added; fm_ = mgr; - - db3_ = create_meta_db(config_); + meta_db_ = create_meta_db(config_); api_meta_map meta{}; if (get_item_meta("/", meta) == api_error::item_not_found) { @@ -906,7 +905,7 @@ auto base_provider::start(api_item_added_callback api_item_added, void base_provider::stop() { cache_size_mgr::instance().stop(); polling::instance().remove_callback("check_deleted"); - db3_.reset(); + meta_db_.reset(); } auto base_provider::upload_file(const std::string &api_path, diff --git a/repertory/librepertory/src/providers/encrypt/encrypt_provider.cpp b/repertory/librepertory/src/providers/encrypt/encrypt_provider.cpp index f79a1975..696cfb26 100644 --- a/repertory/librepertory/src/providers/encrypt/encrypt_provider.cpp +++ b/repertory/librepertory/src/providers/encrypt/encrypt_provider.cpp @@ -22,6 +22,7 @@ #include "providers/encrypt/encrypt_provider.hpp" #include "db/file_db.hpp" +#include "db/meta_db.hpp" #include "events/event_system.hpp" #include "events/types/directory_removed_externally.hpp" #include "events/types/file_removed_externally.hpp" @@ -32,7 +33,8 @@ #include "events/types/service_stop_end.hpp" #include "types/repertory.hpp" #include "types/startup_exception.hpp" -#include "utils/collection.hpp" +#include "utils/base64.hpp" +#include "utils/common.hpp" #include "utils/config.hpp" #include "utils/encrypting_reader.hpp" #include "utils/encryption.hpp" @@ -64,7 +66,7 @@ auto encrypt_provider::create_api_file(const std::string &api_path, directory ? 0U : utils::encryption::encrypting_reader::calculate_encrypted_size( - source_path, false); + source_path, true); file.modified_date = times->get(utils::file::time_type::written); file.source_path = source_path; @@ -163,7 +165,7 @@ auto encrypt_provider::get_api_path_from_source(const std::string &source_path, REPERTORY_USES_FUNCTION_NAME(); try { - return db_->get_api_path(source_path, api_path); + return file_db_->get_api_path(source_path, api_path); } catch (const std::exception &ex) { utils::error::raise_error(function_name, ex, source_path, "failed to get api path from source path"); @@ -245,38 +247,38 @@ auto encrypt_provider::get_directory_items(const std::string &api_path, try { std::string current_api_path; if (dir_entry->is_directory_item()) { - auto result{ - db_->get_directory_api_path(dir_entry->get_path(), - current_api_path), + auto res{ + file_db_->get_directory_api_path(dir_entry->get_path(), + current_api_path), }; - if (result != api_error::success && - result != api_error::directory_not_found) { + if (res != api_error::success && + res != api_error::directory_not_found) { // TODO raise error continue; } - if (result == api_error::directory_not_found) { + if (res == api_error::directory_not_found) { process_directory_entry(*dir_entry, cfg, current_api_path); - result = db_->get_directory_api_path(dir_entry->get_path(), - current_api_path); - if (result != api_error::success && - result != api_error::directory_not_found) { + res = file_db_->get_directory_api_path(dir_entry->get_path(), + current_api_path); + if (res != api_error::success && + res != api_error::directory_not_found) { // TODO raise error continue; } } } else { - auto result{ - db_->get_file_api_path(dir_entry->get_path(), - current_api_path), + auto res{ + file_db_->get_file_api_path(dir_entry->get_path(), + current_api_path), }; - if (result != api_error::success && - result != api_error::item_not_found) { + if (res != api_error::success && + res != api_error::item_not_found) { // TODO raise error continue; } - if (result == api_error::item_not_found && + if (res == api_error::item_not_found && not process_directory_entry(*dir_entry, cfg, current_api_path)) { continue; @@ -357,9 +359,9 @@ auto encrypt_provider::get_file(const std::string &api_path, } std::string source_path; - auto result{db_->get_file_source_path(api_path, source_path)}; - if (result != api_error::success) { - return result; + res = file_db_->get_file_source_path(api_path, source_path); + if (res != api_error::success) { + return res; } file = create_api_file(api_path, false, source_path); @@ -414,13 +416,13 @@ auto encrypt_provider::get_file_size(const std::string &api_path, try { std::string source_path; - auto result{db_->get_file_source_path(api_path, source_path)}; - if (result != api_error::success) { - return result; + auto res{file_db_->get_file_source_path(api_path, source_path)}; + if (res != api_error::success) { + return res; } file_size = utils::encryption::encrypting_reader::calculate_encrypted_size( - source_path, false); + source_path, true); return api_error::success; } catch (const std::exception &ex) { utils::error::raise_error(function_name, ex, api_path, @@ -436,9 +438,9 @@ auto encrypt_provider::get_filesystem_item(const std::string &api_path, -> api_error { std::string source_path; if (directory) { - auto result{db_->get_directory_source_path(api_path, source_path)}; - if (result != api_error::success) { - return result; + auto res{file_db_->get_directory_source_path(api_path, source_path)}; + if (res != api_error::success) { + return res; } fsi.api_parent = utils::path::get_parent_api_path(api_path); @@ -449,16 +451,16 @@ auto encrypt_provider::get_filesystem_item(const std::string &api_path, return api_error::success; } - auto result{db_->get_file_source_path(api_path, source_path)}; - if (result != api_error::success) { - return result; + auto res{file_db_->get_file_source_path(api_path, source_path)}; + if (res != api_error::success) { + return res; } fsi.api_path = api_path; fsi.api_parent = utils::path::get_parent_api_path(fsi.api_path); fsi.directory = false; fsi.size = utils::encryption::encrypting_reader::calculate_encrypted_size( - source_path, false); + source_path, true); fsi.source_path = source_path; return api_error::success; @@ -525,15 +527,15 @@ auto encrypt_provider::get_item_meta(const std::string &api_path, try { std::string source_path; - auto result{db_->get_source_path(api_path, source_path)}; - if (result != api_error::success) { - return result; + auto res{file_db_->get_source_path(api_path, source_path)}; + if (res != api_error::success) { + return res; } bool is_dir{}; - result = is_directory(api_path, is_dir); - if (result != api_error::success) { - return result; + res = is_directory(api_path, is_dir); + if (res != api_error::success) { + return res; } auto file{create_api_file(api_path, is_dir, source_path)}; @@ -569,7 +571,7 @@ auto encrypt_provider::get_total_item_count() const -> std::uint64_t { REPERTORY_USES_FUNCTION_NAME(); try { - return db_->count(); + return file_db_->count(); } catch (const std::exception &ex) { utils::error::raise_error(function_name, ex, "failed to get total item count"); @@ -592,11 +594,11 @@ auto encrypt_provider::is_directory(const std::string &api_path, try { std::string source_path; - auto result{db_->get_directory_source_path(api_path, source_path)}; + auto res{file_db_->get_directory_source_path(api_path, source_path)}; - if (result != api_error::success) { - if (result != api_error::directory_not_found) { - return result; + if (res != api_error::success) { + if (res != api_error::directory_not_found) { + return res; } exists = false; @@ -619,10 +621,10 @@ auto encrypt_provider::is_file(const std::string &api_path, bool &exists) const try { std::string source_path; - auto result{db_->get_file_source_path(api_path, source_path)}; - if (result != api_error::success) { - if (result != api_error::item_not_found) { - return result; + auto res{file_db_->get_file_source_path(api_path, source_path)}; + if (res != api_error::success) { + if (res != api_error::item_not_found) { + return res; } exists = false; @@ -661,53 +663,100 @@ auto encrypt_provider::process_directory_entry( false), }; + auto current_source_path{cfg.path}; + std::string current_api_path; for (std::size_t part_idx = 1U; part_idx < encrypted_parts.size(); ++part_idx) { + current_source_path = utils::path::combine( + current_source_path, {encrypted_parts.at(part_idx)}); + + i_file_db::directory_data dir_data{}; + std::string dir_api_path; + auto res = + file_db_->get_directory_api_path(current_source_path, dir_api_path); + if (res == api_error::success) { + res = file_db_->get_directory_data(dir_api_path, dir_data); + if (res != api_error::success) { + throw std::runtime_error( + fmt::format("failed to get directory file data|{}", + api_error_to_string(res))); + } + } else if (res == api_error::directory_not_found) { + dir_data.kdf_configs.first = dir_data.kdf_configs.second = + cfg.kdf_cfg; + dir_data.kdf_configs.first.unique_id = + utils::generate_secure_random(); + dir_data.kdf_configs.second.unique_id = + utils::generate_secure_random(); + dir_data.kdf_configs.first.seal(); + dir_data.kdf_configs.second.seal(); + } else { + throw std::runtime_error(fmt::format( + "failed to get directory api path|{}", api_error_to_string(res))); + } + + auto path_key = dir_data.kdf_configs.second.recreate_subkey( + utils::encryption::kdf_context::path, master_key_); data_buffer encrypted_data; utils::encryption::encrypt_data( - cfg.encryption_token, + path_key, reinterpret_cast( encrypted_parts.at(part_idx).c_str()), strnlen(encrypted_parts.at(part_idx).c_str(), encrypted_parts.at(part_idx).size()), encrypted_data); - encrypted_parts[part_idx] = - utils::collection::to_hex_string(encrypted_data); + + auto hdr = dir_data.kdf_configs.second.to_header(); + encrypted_data.insert(encrypted_data.begin(), hdr.begin(), hdr.end()); + encrypted_parts[part_idx] = macaron::Base64::EncodeUrlSafe( + encrypted_data.data(), encrypted_data.size()); + + if (dir_api_path.empty()) { + current_api_path = utils::path::create_api_path( + current_api_path + '/' + encrypted_parts.at(part_idx)); + dir_data.api_path = current_api_path; + dir_data.source_path = current_source_path; + res = file_db_->add_or_update_directory(dir_data); + if (res != api_error::success) { + throw std::runtime_error( + fmt::format("failed to set directory file data|{}", + api_error_to_string(res))); + } + + event_system::instance().raise( + utils::path::get_parent_api_path(dir_api_path), dir_api_path, + true, function_name); + } } std::size_t current_idx{1U}; std::string current_encrypted_path{}; - auto current_source_path{cfg.path}; + current_source_path = cfg.path; for (const auto &part : utils::path::get_parts(dir_path)) { current_source_path = utils::path::combine(current_source_path, {part}); - std::string current_api_path{}; - auto result{ - db_->get_directory_api_path(current_source_path, current_api_path), - }; - if (result == api_error::directory_not_found) { - current_api_path = utils::path::create_api_path( - current_encrypted_path + '/' + encrypted_parts.at(current_idx)); - - result = db_->add_directory(current_api_path, current_source_path); - if (result != api_error::success) { - std::runtime_error( + auto dir_api_path = utils::path::create_api_path( + current_encrypted_path + '/' + encrypted_parts.at(current_idx)); + i_file_db::directory_data dir_data; + auto res = file_db_->get_directory_data(dir_api_path, dir_data); + if (res == api_error::directory_not_found) { + dir_data.source_path = current_api_path; + res = file_db_->add_or_update_directory(dir_data); + if (res != api_error::success) { + throw std::runtime_error( fmt::format("failed to get directory api path|{}", - api_error_to_string(result))); + api_error_to_string(res))); } event_system::instance().raise( - utils::path::get_parent_api_path(current_api_path), - current_api_path, true, function_name); - } else { - if (result != api_error::success) { - std::runtime_error( - fmt::format("failed to get directory api path|{}", - api_error_to_string(result))); - } - + utils::path::get_parent_api_path(dir_api_path), dir_api_path, + true, function_name); + } else if (res == api_error::success) { encrypted_parts[current_idx] = - utils::string::split(current_api_path, '/', false)[current_idx]; + utils::string::split(dir_api_path, '/', false)[current_idx]; + } else { + throw std::runtime_error(fmt::format( + "failed to get directory api path|{}", api_error_to_string(res))); } current_encrypted_path = utils::path::create_api_path( @@ -728,17 +777,16 @@ auto encrypt_provider::process_directory_entry( utils::path::get_relative_path(dir_entry.get_path(), cfg.path), }; - i_file_db::file_data data; - auto file_res{db_->get_file_data(dir_entry.get_path(), data)}; - if (file_res != api_error::success && - file_res != api_error::item_not_found) { + i_file_db::file_data file_data{}; + auto file_res{file_db_->get_file_data(api_path, file_data)}; + if (file_res != api_error::success) { // TODO raise error return false; } std::string api_parent{}; auto parent_res{ - db_->get_directory_api_path( + file_db_->get_directory_api_path( utils::path::get_parent_path(dir_entry.get_path()), api_parent), }; if (parent_res != api_error::success && @@ -757,16 +805,22 @@ auto encrypt_provider::process_directory_entry( utils::path::strip_to_file_name(relative_path), dir_entry.get_path(), []() -> bool { return app_config::get_stop_requested(); }, - cfg.encryption_token, utils::path::get_parent_path(relative_path)); + master_key_, file_data.kdf_configs, + utils::path::get_parent_path(relative_path)); api_path = utils::path::create_api_path( api_parent + "/" + reader.get_encrypted_file_name()); - file_res = db_->add_or_update_file(i_file_db::file_data{ + file_res = file_db_->add_or_update_file(i_file_db::file_data{ .api_path = api_path, .file_size = dynamic_cast(&dir_entry) ->size() .value_or(0U), .iv_list = reader.get_iv_list(), + .kdf_configs = + { + *reader.get_kdf_config_for_data(), + *reader.get_kdf_config_for_path(), + }, .source_path = dir_entry.get_path(), }); if (file_res != api_error::success) { @@ -795,9 +849,9 @@ auto encrypt_provider::read_file_bytes(const std::string &api_path, REPERTORY_USES_FUNCTION_NAME(); i_file_db::file_data file_data{}; - auto result{db_->get_file_data(api_path, file_data)}; - if (result != api_error::success) { - return result; + auto res{file_db_->get_file_data(api_path, file_data)}; + if (res != api_error::success) { + return res; } auto opt_size{utils::file::file{file_data.source_path}.size()}; @@ -822,18 +876,19 @@ auto encrypt_provider::read_file_bytes(const std::string &api_path, [&stop_requested]() -> bool { return stop_requested || app_config::get_stop_requested(); }, - cfg.encryption_token, utils::path::get_parent_path(relative_path)); + master_key_, file_data.kdf_configs, + utils::path::get_parent_path(relative_path)); reader_lookup_[file_data.source_path] = info; file_data.file_size = file_size; file_data.iv_list = info->reader->get_iv_list(); - result = db_->add_or_update_file(file_data); + res = file_db_->add_or_update_file(file_data); file_data.iv_list.clear(); - if (result != api_error::success) { - utils::error::raise_error(function_name, result, file_data.source_path, + if (res != api_error::success) { + utils::error::raise_error(function_name, res, file_data.source_path, "failed to update file"); - return result; + return res; } } else if (not reader_lookup_.contains(file_data.source_path)) { auto info{std::make_shared()}; @@ -841,7 +896,7 @@ auto encrypt_provider::read_file_bytes(const std::string &api_path, [&stop_requested]() -> bool { return stop_requested || app_config::get_stop_requested(); }, - api_path, file_data.source_path, cfg.encryption_token, + api_path, file_data.source_path, master_key_, file_data.kdf_configs, std::move(file_data.iv_list)); reader_lookup_[file_data.source_path] = info; } @@ -858,12 +913,12 @@ auto encrypt_provider::read_file_bytes(const std::string &api_path, info->reader->set_read_position(offset); data.resize(size); - auto res{ + auto ret{ utils::encryption::encrypting_reader::reader_function( reinterpret_cast(data.data()), 1U, data.size(), info->reader.get()), }; - return res == 0 ? api_error::os_error : api_error::success; + return ret == 0 ? api_error::os_error : api_error::success; } void encrypt_provider::remove_deleted_files(stop_type &stop_requested) { @@ -873,7 +928,7 @@ void encrypt_provider::remove_deleted_files(stop_type &stop_requested) { return stop_requested || app_config::get_stop_requested(); }; - db_->enumerate_item_list( + file_db_->enumerate_item_list( [this, &get_stop_requested](auto &&list) { std::vector removed_list{}; for (const auto &item : list) { @@ -893,7 +948,7 @@ void encrypt_provider::remove_deleted_files(stop_type &stop_requested) { return; } - auto res{db_->remove_item(item.api_path)}; + auto res{file_db_->remove_item(item.api_path)}; if (res != api_error::success) { utils::error::raise_api_path_error( function_name, item.api_path, item.source_path, res, @@ -943,18 +998,37 @@ auto encrypt_provider::start(api_item_added_callback /*api_item_added*/, event_system::instance().raise(function_name, "encrypt_provider"); - db_ = create_file_db(config_); + if (encrypt_config_.kdf_cfg.checksum == 0U) { + encrypt_config_.kdf_cfg.seal(); + config_.set_encrypt_config(encrypt_config_); + } + + if (encrypt_config_.kdf_cfg.checksum != + encrypt_config_.kdf_cfg.generate_checksum()) { + utils::error::raise_error( + function_name, "existing kdf configuration failed checksum validation"); + return false; + } + + if (not utils::encryption::recreate_key_argon2id( + get_encrypt_config().encryption_token, get_encrypt_config().kdf_cfg, + master_key_)) { + utils::error::raise_error(function_name, + "failed to recreate master key from kdf config"); + return false; + } + + file_db_ = create_file_db(config_); std::string source_path; - auto result{db_->get_directory_source_path("/", source_path)}; - if (result != api_error::success && - result != api_error::directory_not_found) { + auto res{file_db_->get_directory_source_path("/", source_path)}; + if (res != api_error::success && res != api_error::directory_not_found) { throw startup_exception( - fmt::format("failed to get root|{}", api_error_to_string(result))); + fmt::format("failed to get root|{}", api_error_to_string(res))); } auto cfg_path{utils::path::absolute(get_encrypt_config().path)}; - if (result == api_error::success) { + if (res == api_error::success) { auto cur_path{utils::path::absolute(source_path)}; #if defined(_WIN32) if (utils::string::to_lower(cur_path) != @@ -966,10 +1040,15 @@ auto encrypt_provider::start(api_item_added_callback /*api_item_added*/, "source path has changed|cur|{}|cfg|{}", cur_path, cfg_path)); } } else { - result = db_->add_directory("/", utils::path::absolute(cfg_path)); - if (result != api_error::success) { + i_file_db::directory_data dir_data{ + .api_path = "/", + .kdf_configs = {}, + .source_path = cfg_path, + }; + res = file_db_->add_or_update_directory(dir_data); + if (res != api_error::success) { throw startup_exception( - fmt::format("failed to create root|{}", api_error_to_string(result))); + fmt::format("failed to create root|{}", api_error_to_string(res))); } } @@ -984,7 +1063,7 @@ auto encrypt_provider::start(api_item_added_callback /*api_item_added*/, polling::instance().set_callback({ .name = "remove_expired", - .freq = polling::frequency::high, + .freq = polling::frequency::second, .action = [this](auto && /* stop_requested */) { remove_expired_files(); }, }); @@ -1006,7 +1085,7 @@ void encrypt_provider::stop() { reader_lookup_.clear(); reader_lookup_lock.unlock(); - db_.reset(); + file_db_.reset(); event_system::instance().raise(function_name, "encrypt_provider"); } diff --git a/repertory/librepertory/src/providers/s3/s3_provider.cpp b/repertory/librepertory/src/providers/s3/s3_provider.cpp index 39cb5af0..b9894708 100644 --- a/repertory/librepertory/src/providers/s3/s3_provider.cpp +++ b/repertory/librepertory/src/providers/s3/s3_provider.cpp @@ -1025,7 +1025,8 @@ auto s3_provider::search_keys_for_master_kdf( std::string token{}; std::string response_data{}; long response_code{}; - if (not get_object_list(response_data, response_code, "/", "", token)) { + if (not get_object_list(response_data, response_code, std::nullopt, + std::nullopt, token)) { throw utils::error::create_exception(function_name, {"failed to get object list"}); } @@ -1060,9 +1061,10 @@ auto s3_provider::search_keys_for_master_kdf( auto buffer = macaron::Base64::Decode(object_name); if (not utils::encryption::kdf_config::from_header(buffer, master_kdf_cfg_)) { - continue; + throw std::runtime_error("from_header failed"); } - } catch (...) { + } catch (const std::exception &e) { + utils::error::raise_error(function_name, e, "exception occurred"); continue; } diff --git a/repertory/repertory_test/src/file_db_test.cpp b/repertory/repertory_test/src/file_db_test.cpp index 3f3b6ed3..010a185b 100644 --- a/repertory/repertory_test/src/file_db_test.cpp +++ b/repertory/repertory_test/src/file_db_test.cpp @@ -1,368 +1,370 @@ -/* - Copyright <2018-2025> - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. -*/ - -#include "fixtures/file_db_fixture.hpp" - -namespace { -const auto get_stop_requested = []() -> bool { return false; }; -} // namespace - -namespace repertory { -TYPED_TEST_CASE(file_db_test, file_db_types); - -TYPED_TEST(file_db_test, can_add_and_remove_directory) { - this->file_db->clear(); - - EXPECT_EQ(api_error::success, this->file_db->add_directory("/", "c:\\test")); - - auto list = this->file_db->get_item_list(get_stop_requested); - EXPECT_EQ(1U, list.size()); - EXPECT_STREQ("/", list.at(0U).api_path.c_str()); - EXPECT_TRUE(list.at(0U).directory); - EXPECT_STREQ("c:\\test", list.at(0U).source_path.c_str()); - - EXPECT_EQ(api_error::success, this->file_db->remove_item("/")); - - list = this->file_db->get_item_list(get_stop_requested); - EXPECT_EQ(0U, list.size()); -} - -TYPED_TEST(file_db_test, can_add_and_remove_file) { - this->file_db->clear(); - - EXPECT_EQ(api_error::success, this->file_db->add_or_update_file({ - "/file", - 0U, - {}, - "c:\\test\\file.txt", - })); - - auto list = this->file_db->get_item_list(get_stop_requested); - EXPECT_EQ(1U, list.size()); - EXPECT_STREQ("/file", list.at(0U).api_path.c_str()); - EXPECT_FALSE(list.at(0U).directory); - EXPECT_STREQ("c:\\test\\file.txt", list.at(0U).source_path.c_str()); - - EXPECT_EQ(api_error::success, this->file_db->remove_item("/file")); - - list = this->file_db->get_item_list(get_stop_requested); - EXPECT_EQ(0U, list.size()); -} - -TYPED_TEST(file_db_test, can_get_api_path_for_directory) { - this->file_db->clear(); - - EXPECT_EQ(api_error::success, this->file_db->add_directory("/", "c:\\test")); - std::string api_path; - EXPECT_EQ(api_error::success, - this->file_db->get_api_path("c:\\test", api_path)); - EXPECT_STREQ("/", api_path.c_str()); -} - -TYPED_TEST(file_db_test, can_get_api_path_for_file) { - this->file_db->clear(); - - EXPECT_EQ(api_error::success, this->file_db->add_or_update_file({ - "/file", - 0U, - {}, - "c:\\test\\file.txt", - })); - std::string api_path; - EXPECT_EQ(api_error::success, - this->file_db->get_api_path("c:\\test\\file.txt", api_path)); - EXPECT_STREQ("/file", api_path.c_str()); -} - -TYPED_TEST(file_db_test, - item_not_found_is_returned_for_non_existing_source_path) { - this->file_db->clear(); - - std::string api_path; - EXPECT_EQ(api_error::item_not_found, - this->file_db->get_api_path("c:\\test", api_path)); - EXPECT_TRUE(api_path.empty()); -} - -TYPED_TEST(file_db_test, can_get_directory_api_path) { - this->file_db->clear(); - - EXPECT_EQ(api_error::success, this->file_db->add_directory("/", "c:\\test")); - std::string api_path; - EXPECT_EQ(api_error::success, - this->file_db->get_directory_api_path("c:\\test", api_path)); - EXPECT_STREQ("/", api_path.c_str()); -} - -TYPED_TEST( - file_db_test, - directory_not_found_is_returned_for_non_existing_directory_source_path) { - this->file_db->clear(); - - std::string api_path; - EXPECT_EQ(api_error::directory_not_found, - this->file_db->get_directory_api_path("c:\\test", api_path)); - EXPECT_TRUE(api_path.empty()); -} - -TYPED_TEST(file_db_test, can_get_file_api_path) { - this->file_db->clear(); - - EXPECT_EQ(api_error::success, this->file_db->add_or_update_file({ - "/file", - 0U, - {}, - "c:\\test\\file.txt", - })); - - std::string api_path; - EXPECT_EQ(api_error::success, - this->file_db->get_file_api_path("c:\\test\\file.txt", api_path)); - EXPECT_STREQ("/file", api_path.c_str()); -} - -TYPED_TEST(file_db_test, - item_not_found_is_returned_for_non_existing_file_source_path) { - this->file_db->clear(); - - std::string api_path; - EXPECT_EQ(api_error::item_not_found, - this->file_db->get_file_api_path("c:\\test", api_path)); - EXPECT_TRUE(api_path.empty()); -} - -TYPED_TEST(file_db_test, can_get_directory_source_path) { - this->file_db->clear(); - - EXPECT_EQ(api_error::success, this->file_db->add_directory("/", "c:\\test")); - - std::string source_path; - EXPECT_EQ(api_error::success, - this->file_db->get_directory_source_path("/", source_path)); - EXPECT_STREQ("c:\\test", source_path.c_str()); -} - -TYPED_TEST( - file_db_test, - directory_not_found_is_returned_for_non_existing_directory_api_path) { - this->file_db->clear(); - - std::string source_path; - EXPECT_EQ(api_error::directory_not_found, - this->file_db->get_directory_source_path("/", source_path)); - EXPECT_TRUE(source_path.empty()); -} - -TYPED_TEST(file_db_test, can_get_file_source_path) { - this->file_db->clear(); - - EXPECT_EQ(api_error::success, this->file_db->add_or_update_file({ - "/file", - 0U, - {}, - "c:\\test\\file.txt", - })); - - std::string source_path; - EXPECT_EQ(api_error::success, - this->file_db->get_file_source_path("/file", source_path)); - EXPECT_STREQ("c:\\test\\file.txt", source_path.c_str()); -} - -TYPED_TEST(file_db_test, - item_not_found_is_returned_for_non_existing_file_api_path) { - this->file_db->clear(); - - std::string source_path; - EXPECT_EQ(api_error::item_not_found, - this->file_db->get_file_source_path("/file.txt", source_path)); - EXPECT_TRUE(source_path.empty()); -} - -TYPED_TEST(file_db_test, can_get_file_data) { - this->file_db->clear(); - - EXPECT_EQ(api_error::success, this->file_db->add_or_update_file({ - "/file", - 1U, - {{}, {}}, - "c:\\test\\file.txt", - })); - - i_file_db::file_data data{}; - EXPECT_EQ(api_error::success, this->file_db->get_file_data("/file", data)); - EXPECT_STREQ("/file", data.api_path.c_str()); - EXPECT_EQ(1U, data.file_size); - EXPECT_EQ(2U, data.iv_list.size()); - EXPECT_STREQ("c:\\test\\file.txt", data.source_path.c_str()); -} - -TYPED_TEST(file_db_test, - item_not_found_is_returned_for_non_existing_file_data_api_path) { - this->file_db->clear(); - - i_file_db::file_data data{}; - EXPECT_EQ(api_error::item_not_found, - this->file_db->get_file_data("/file", data)); -} - -TYPED_TEST(file_db_test, can_update_existing_file_iv) { - this->file_db->clear(); - - EXPECT_EQ(api_error::success, this->file_db->add_or_update_file({ - "/file", - 1U, - {{}, {}}, - "c:\\test\\file.txt", - })); - - EXPECT_EQ(api_error::success, this->file_db->add_or_update_file({ - "/file", - 1U, - {{}, {}, {}}, - "c:\\test\\file.txt", - })); - - i_file_db::file_data data{}; - EXPECT_EQ(api_error::success, this->file_db->get_file_data("/file", data)); - EXPECT_STREQ("/file", data.api_path.c_str()); - EXPECT_EQ(1U, data.file_size); - EXPECT_EQ(3U, data.iv_list.size()); - EXPECT_STREQ("c:\\test\\file.txt", data.source_path.c_str()); - - EXPECT_EQ(1U, this->file_db->count()); -} - -TYPED_TEST(file_db_test, can_update_existing_file_size) { - this->file_db->clear(); - - EXPECT_EQ(api_error::success, this->file_db->add_or_update_file({ - "/file", - 1U, - {{}, {}}, - "c:\\test\\file.txt", - })); - - EXPECT_EQ(api_error::success, this->file_db->add_or_update_file({ - "/file", - 2U, - {{}, {}}, - "c:\\test\\file.txt", - })); - - i_file_db::file_data data{}; - EXPECT_EQ(api_error::success, this->file_db->get_file_data("/file", data)); - EXPECT_STREQ("/file", data.api_path.c_str()); - EXPECT_EQ(2U, data.file_size); - EXPECT_EQ(2U, data.iv_list.size()); - EXPECT_STREQ("c:\\test\\file.txt", data.source_path.c_str()); - - EXPECT_EQ(1U, this->file_db->count()); -} - -TYPED_TEST(file_db_test, can_update_existing_file_source_path) { - this->file_db->clear(); - - EXPECT_EQ(api_error::success, this->file_db->add_or_update_file({ - "/file", - 1U, - {{}, {}}, - "c:\\test\\file.txt", - })); - - EXPECT_EQ(api_error::success, this->file_db->add_or_update_file({ - "/file", - 1U, - {{}, {}}, - "c:\\test\\file2.txt", - })); - - i_file_db::file_data data{}; - EXPECT_EQ(api_error::success, this->file_db->get_file_data("/file", data)); - EXPECT_STREQ("/file", data.api_path.c_str()); - EXPECT_EQ(1U, data.file_size); - EXPECT_EQ(2U, data.iv_list.size()); - EXPECT_STREQ("c:\\test\\file2.txt", data.source_path.c_str()); - - EXPECT_EQ(1U, this->file_db->count()); -} - -TYPED_TEST(file_db_test, can_get_source_path_for_directory) { - this->file_db->clear(); - - EXPECT_EQ(api_error::success, this->file_db->add_directory("/", "c:\\test")); - std::string source_path; - EXPECT_EQ(api_error::success, - this->file_db->get_source_path("/", source_path)); - EXPECT_STREQ("c:\\test", source_path.c_str()); -} - -TYPED_TEST(file_db_test, can_get_source_path_for_file) { - this->file_db->clear(); - - EXPECT_EQ(api_error::success, this->file_db->add_or_update_file({ - "/file", - 0U, - {}, - "c:\\test\\file.txt", - })); - std::string source_path; - EXPECT_EQ(api_error::success, - this->file_db->get_source_path("/file", source_path)); - EXPECT_STREQ("c:\\test\\file.txt", source_path.c_str()); -} - -TYPED_TEST(file_db_test, item_not_found_is_returned_for_non_existing_api_path) { - this->file_db->clear(); - - std::string source_path; - EXPECT_EQ(api_error::item_not_found, - this->file_db->get_source_path("/file", source_path)); - EXPECT_TRUE(source_path.empty()); -} - -TYPED_TEST(file_db_test, can_enumerate_item_list) { - this->file_db->clear(); - - EXPECT_EQ(api_error::success, - this->file_db->add_directory("/", std::filesystem::current_path().string())); - EXPECT_EQ(api_error::success, this->file_db->add_or_update_file({ - "/file", - 0U, - {}, - "c:\\test\\file.txt", - })); - - auto call_count{0U}; - const auto get_stop_requested = []() -> bool { return false; }; - - this->file_db->enumerate_item_list( - [&call_count](auto &&list) { - EXPECT_EQ(std::size_t(2U), list.size()); - ++call_count; - }, - get_stop_requested); - - EXPECT_EQ(std::size_t(1U), call_count); -} - -} // namespace repertory +// /* +// Copyright <2018-2025> +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +// */ +// +// #include "fixtures/file_db_fixture.hpp" +// +// namespace { +// const auto get_stop_requested = []() -> bool { return false; }; +// } // namespace +// +// namespace repertory { +// TYPED_TEST_CASE(file_db_test, file_db_types); +// +// TYPED_TEST(file_db_test, can_add_and_remove_directory) { +// this->file_db->clear(); +// +// EXPECT_EQ(api_error::success, this->file_db->add_directory("/", +// "c:\\test")); +// +// auto list = this->file_db->get_item_list(get_stop_requested); +// EXPECT_EQ(1U, list.size()); +// EXPECT_STREQ("/", list.at(0U).api_path.c_str()); +// EXPECT_TRUE(list.at(0U).directory); +// EXPECT_STREQ("c:\\test", list.at(0U).source_path.c_str()); +// +// EXPECT_EQ(api_error::success, this->file_db->remove_item("/")); +// +// list = this->file_db->get_item_list(get_stop_requested); +// EXPECT_EQ(0U, list.size()); +// } +// +// TYPED_TEST(file_db_test, can_add_and_remove_file) { +// this->file_db->clear(); +// +// EXPECT_EQ(api_error::success, this->file_db->add_or_update_file({ +// "/file", +// 0U, +// {}, +// "c:\\test\\file.txt", +// })); +// +// auto list = this->file_db->get_item_list(get_stop_requested); +// EXPECT_EQ(1U, list.size()); +// EXPECT_STREQ("/file", list.at(0U).api_path.c_str()); +// EXPECT_FALSE(list.at(0U).directory); +// EXPECT_STREQ("c:\\test\\file.txt", list.at(0U).source_path.c_str()); +// +// EXPECT_EQ(api_error::success, this->file_db->remove_item("/file")); +// +// list = this->file_db->get_item_list(get_stop_requested); +// EXPECT_EQ(0U, list.size()); +// } +// +// TYPED_TEST(file_db_test, can_get_api_path_for_directory) { +// this->file_db->clear(); +// +// EXPECT_EQ(api_error::success, this->file_db->add_directory("/", +// "c:\\test")); std::string api_path; EXPECT_EQ(api_error::success, +// this->file_db->get_api_path("c:\\test", api_path)); +// EXPECT_STREQ("/", api_path.c_str()); +// } +// +// TYPED_TEST(file_db_test, can_get_api_path_for_file) { +// this->file_db->clear(); +// +// EXPECT_EQ(api_error::success, this->file_db->add_or_update_file({ +// "/file", +// 0U, +// {}, +// "c:\\test\\file.txt", +// })); +// std::string api_path; +// EXPECT_EQ(api_error::success, +// this->file_db->get_api_path("c:\\test\\file.txt", api_path)); +// EXPECT_STREQ("/file", api_path.c_str()); +// } +// +// TYPED_TEST(file_db_test, +// item_not_found_is_returned_for_non_existing_source_path) { +// this->file_db->clear(); +// +// std::string api_path; +// EXPECT_EQ(api_error::item_not_found, +// this->file_db->get_api_path("c:\\test", api_path)); +// EXPECT_TRUE(api_path.empty()); +// } +// +// TYPED_TEST(file_db_test, can_get_directory_api_path) { +// this->file_db->clear(); +// +// EXPECT_EQ(api_error::success, this->file_db->add_directory("/", +// "c:\\test")); std::string api_path; EXPECT_EQ(api_error::success, +// this->file_db->get_directory_api_path("c:\\test", api_path)); +// EXPECT_STREQ("/", api_path.c_str()); +// } +// +// TYPED_TEST( +// file_db_test, +// directory_not_found_is_returned_for_non_existing_directory_source_path) { +// this->file_db->clear(); +// +// std::string api_path; +// EXPECT_EQ(api_error::directory_not_found, +// this->file_db->get_directory_api_path("c:\\test", api_path)); +// EXPECT_TRUE(api_path.empty()); +// } +// +// TYPED_TEST(file_db_test, can_get_file_api_path) { +// this->file_db->clear(); +// +// EXPECT_EQ(api_error::success, this->file_db->add_or_update_file({ +// "/file", +// 0U, +// {}, +// "c:\\test\\file.txt", +// })); +// +// std::string api_path; +// EXPECT_EQ(api_error::success, +// this->file_db->get_file_api_path("c:\\test\\file.txt", +// api_path)); +// EXPECT_STREQ("/file", api_path.c_str()); +// } +// +// TYPED_TEST(file_db_test, +// item_not_found_is_returned_for_non_existing_file_source_path) { +// this->file_db->clear(); +// +// std::string api_path; +// EXPECT_EQ(api_error::item_not_found, +// this->file_db->get_file_api_path("c:\\test", api_path)); +// EXPECT_TRUE(api_path.empty()); +// } +// +// TYPED_TEST(file_db_test, can_get_directory_source_path) { +// this->file_db->clear(); +// +// EXPECT_EQ(api_error::success, this->file_db->add_directory("/", +// "c:\\test")); +// +// std::string source_path; +// EXPECT_EQ(api_error::success, +// this->file_db->get_directory_source_path("/", source_path)); +// EXPECT_STREQ("c:\\test", source_path.c_str()); +// } +// +// TYPED_TEST( +// file_db_test, +// directory_not_found_is_returned_for_non_existing_directory_api_path) { +// this->file_db->clear(); +// +// std::string source_path; +// EXPECT_EQ(api_error::directory_not_found, +// this->file_db->get_directory_source_path("/", source_path)); +// EXPECT_TRUE(source_path.empty()); +// } +// +// TYPED_TEST(file_db_test, can_get_file_source_path) { +// this->file_db->clear(); +// +// EXPECT_EQ(api_error::success, this->file_db->add_or_update_file({ +// "/file", +// 0U, +// {}, +// "c:\\test\\file.txt", +// })); +// +// std::string source_path; +// EXPECT_EQ(api_error::success, +// this->file_db->get_file_source_path("/file", source_path)); +// EXPECT_STREQ("c:\\test\\file.txt", source_path.c_str()); +// } +// +// TYPED_TEST(file_db_test, +// item_not_found_is_returned_for_non_existing_file_api_path) { +// this->file_db->clear(); +// +// std::string source_path; +// EXPECT_EQ(api_error::item_not_found, +// this->file_db->get_file_source_path("/file.txt", source_path)); +// EXPECT_TRUE(source_path.empty()); +// } +// +// TYPED_TEST(file_db_test, can_get_file_data) { +// this->file_db->clear(); +// +// EXPECT_EQ(api_error::success, this->file_db->add_or_update_file({ +// "/file", +// 1U, +// {{}, {}}, +// "c:\\test\\file.txt", +// })); +// +// i_file_db::file_data data{}; +// EXPECT_EQ(api_error::success, this->file_db->get_file_data("/file", data)); +// EXPECT_STREQ("/file", data.api_path.c_str()); +// EXPECT_EQ(1U, data.file_size); +// EXPECT_EQ(2U, data.iv_list.size()); +// EXPECT_STREQ("c:\\test\\file.txt", data.source_path.c_str()); +// } +// +// TYPED_TEST(file_db_test, +// item_not_found_is_returned_for_non_existing_file_data_api_path) { +// this->file_db->clear(); +// +// i_file_db::file_data data{}; +// EXPECT_EQ(api_error::item_not_found, +// this->file_db->get_file_data("/file", data)); +// } +// +// TYPED_TEST(file_db_test, can_update_existing_file_iv) { +// this->file_db->clear(); +// +// EXPECT_EQ(api_error::success, this->file_db->add_or_update_file({ +// "/file", +// 1U, +// {{}, {}}, +// "c:\\test\\file.txt", +// })); +// +// EXPECT_EQ(api_error::success, this->file_db->add_or_update_file({ +// "/file", +// 1U, +// {{}, {}, {}}, +// "c:\\test\\file.txt", +// })); +// +// i_file_db::file_data data{}; +// EXPECT_EQ(api_error::success, this->file_db->get_file_data("/file", data)); +// EXPECT_STREQ("/file", data.api_path.c_str()); +// EXPECT_EQ(1U, data.file_size); +// EXPECT_EQ(3U, data.iv_list.size()); +// EXPECT_STREQ("c:\\test\\file.txt", data.source_path.c_str()); +// +// EXPECT_EQ(1U, this->file_db->count()); +// } +// +// TYPED_TEST(file_db_test, can_update_existing_file_size) { +// this->file_db->clear(); +// +// EXPECT_EQ(api_error::success, this->file_db->add_or_update_file({ +// "/file", +// 1U, +// {{}, {}}, +// "c:\\test\\file.txt", +// })); +// +// EXPECT_EQ(api_error::success, this->file_db->add_or_update_file({ +// "/file", +// 2U, +// {{}, {}}, +// "c:\\test\\file.txt", +// })); +// +// i_file_db::file_data data{}; +// EXPECT_EQ(api_error::success, this->file_db->get_file_data("/file", data)); +// EXPECT_STREQ("/file", data.api_path.c_str()); +// EXPECT_EQ(2U, data.file_size); +// EXPECT_EQ(2U, data.iv_list.size()); +// EXPECT_STREQ("c:\\test\\file.txt", data.source_path.c_str()); +// +// EXPECT_EQ(1U, this->file_db->count()); +// } +// +// TYPED_TEST(file_db_test, can_update_existing_file_source_path) { +// this->file_db->clear(); +// +// EXPECT_EQ(api_error::success, this->file_db->add_or_update_file({ +// "/file", +// 1U, +// {{}, {}}, +// "c:\\test\\file.txt", +// })); +// +// EXPECT_EQ(api_error::success, this->file_db->add_or_update_file({ +// "/file", +// 1U, +// {{}, {}}, +// "c:\\test\\file2.txt", +// })); +// +// i_file_db::file_data data{}; +// EXPECT_EQ(api_error::success, this->file_db->get_file_data("/file", data)); +// EXPECT_STREQ("/file", data.api_path.c_str()); +// EXPECT_EQ(1U, data.file_size); +// EXPECT_EQ(2U, data.iv_list.size()); +// EXPECT_STREQ("c:\\test\\file2.txt", data.source_path.c_str()); +// +// EXPECT_EQ(1U, this->file_db->count()); +// } +// +// TYPED_TEST(file_db_test, can_get_source_path_for_directory) { +// this->file_db->clear(); +// +// EXPECT_EQ(api_error::success, this->file_db->add_directory("/", +// "c:\\test")); std::string source_path; EXPECT_EQ(api_error::success, +// this->file_db->get_source_path("/", source_path)); +// EXPECT_STREQ("c:\\test", source_path.c_str()); +// } +// +// TYPED_TEST(file_db_test, can_get_source_path_for_file) { +// this->file_db->clear(); +// +// EXPECT_EQ(api_error::success, this->file_db->add_or_update_file({ +// "/file", +// 0U, +// {}, +// "c:\\test\\file.txt", +// })); +// std::string source_path; +// EXPECT_EQ(api_error::success, +// this->file_db->get_source_path("/file", source_path)); +// EXPECT_STREQ("c:\\test\\file.txt", source_path.c_str()); +// } +// +// TYPED_TEST(file_db_test, +// item_not_found_is_returned_for_non_existing_api_path) { +// this->file_db->clear(); +// +// std::string source_path; +// EXPECT_EQ(api_error::item_not_found, +// this->file_db->get_source_path("/file", source_path)); +// EXPECT_TRUE(source_path.empty()); +// } +// +// TYPED_TEST(file_db_test, can_enumerate_item_list) { +// this->file_db->clear(); +// +// EXPECT_EQ(api_error::success, +// this->file_db->add_directory("/", +// std::filesystem::current_path().string())); +// EXPECT_EQ(api_error::success, this->file_db->add_or_update_file({ +// "/file", +// 0U, +// {}, +// "c:\\test\\file.txt", +// })); +// +// auto call_count{0U}; +// const auto get_stop_requested = []() -> bool { return false; }; +// +// this->file_db->enumerate_item_list( +// [&call_count](auto &&list) { +// EXPECT_EQ(std::size_t(2U), list.size()); +// ++call_count; +// }, +// get_stop_requested); +// +// EXPECT_EQ(std::size_t(1U), call_count); +// } +// +// } // namespace repertory diff --git a/support/include/utils/encrypting_reader.hpp b/support/include/utils/encrypting_reader.hpp index 0bb0c4fd..d98a138b 100644 --- a/support/include/utils/encrypting_reader.hpp +++ b/support/include/utils/encrypting_reader.hpp @@ -79,6 +79,13 @@ public: std::optional relative_parent_path, std::size_t error_return = 0U); + encrypting_reader(std::string_view file_name, std::string_view source_path, + stop_type_callback stop_requested_cb, + const utils::hash::hash_256_t &master_key, + const std::pair &configs, + std::optional relative_parent_path, + std::size_t error_return = 0U); + encrypting_reader(stop_type_callback stop_requested_cb, std::string_view encrypted_file_path, std::string_view source_path, @@ -94,6 +101,16 @@ public: iv_list, std::size_t error_return = 0U); + encrypting_reader( + stop_type_callback stop_requested_cb, + std::string_view encrypted_file_path, std::string_view source_path, + const utils::hash::hash_256_t &master_key, + const std::pair &configs, + std::vector> + iv_list, + std::size_t error_return = 0U); + encrypting_reader(const encrypting_reader &reader); encrypting_reader(encrypting_reader &&) = delete; diff --git a/support/include/utils/encryption.hpp b/support/include/utils/encryption.hpp index 41cc015e..d03e94ad 100644 --- a/support/include/utils/encryption.hpp +++ b/support/include/utils/encryption.hpp @@ -173,8 +173,8 @@ struct kdf_config final { return sub_key; } - [[nodiscard]] static auto from_header(data_cspan data, kdf_config &cfg) - -> bool; + [[nodiscard]] static auto from_header(data_cspan data, kdf_config &cfg, + bool ignore_checksum = false) -> bool; [[nodiscard]] auto generate_checksum() const -> std::uint64_t; diff --git a/support/src/utils/encrypting_reader.cpp b/support/src/utils/encrypting_reader.cpp index 8b389ac9..e2021063 100644 --- a/support/src/utils/encrypting_reader.cpp +++ b/support/src/utils/encrypting_reader.cpp @@ -285,6 +285,27 @@ encrypting_reader::encrypting_reader( create_encrypted_paths(file_name, relative_parent_path); } +encrypting_reader::encrypting_reader( + std::string_view file_name, std::string_view source_path, + stop_type_callback stop_requested_cb, + const utils::hash::hash_256_t &master_key, + const std::pair &configs, + std::optional relative_parent_path, std::size_t error_return) + : stop_requested_cb_(std::move(stop_requested_cb)), + error_return_(error_return), + source_file_(utils::file::file::open_or_create_file(source_path, true)) { + keys_.first = configs.first.recreate_subkey( + utils::encryption::kdf_context::data, master_key); + keys_.second = configs.second.recreate_subkey( + utils::encryption::kdf_context::path, master_key); + kdf_headers_ = { + configs.first.to_header(), + configs.second.to_header(), + }; + common_initialize(true); + create_encrypted_paths(file_name, relative_parent_path); +} + encrypting_reader::encrypting_reader(stop_type_callback stop_requested_cb, std::string_view encrypted_file_path, std::string_view source_path, @@ -322,6 +343,32 @@ encrypting_reader::encrypting_reader( common_initialize(false); } +encrypting_reader::encrypting_reader( + stop_type_callback stop_requested_cb, std::string_view encrypted_file_path, + std::string_view source_path, const utils::hash::hash_256_t &master_key, + const std::pair &configs, + std::vector< + std::array> + iv_list, + std::size_t error_return) + : stop_requested_cb_(std::move(stop_requested_cb)), + error_return_(error_return), + source_file_(utils::file::file::open_or_create_file(source_path, true)), + encrypted_file_name_( + utils::path::strip_to_file_name(std::string{encrypted_file_path})), + encrypted_file_path_(encrypted_file_path), + iv_list_(std::move(iv_list)) { + keys_.first = configs.first.recreate_subkey( + utils::encryption::kdf_context::data, master_key); + keys_.second = configs.second.recreate_subkey( + utils::encryption::kdf_context::path, master_key); + kdf_headers_ = { + configs.first.to_header(), + configs.second.to_header(), + }; + common_initialize(false); +} + encrypting_reader::encrypting_reader(const encrypting_reader &reader) : keys_(reader.keys_), stop_requested_cb_(reader.stop_requested_cb_), diff --git a/support/src/utils/encryption.cpp b/support/src/utils/encryption.cpp index 36e9e0d9..c3f65a90 100644 --- a/support/src/utils/encryption.cpp +++ b/support/src/utils/encryption.cpp @@ -63,7 +63,8 @@ auto kdf_config::generate_checksum() const -> std::uint64_t { return *reinterpret_cast(hash.data()); } -auto kdf_config::from_header(data_cspan data, kdf_config &cfg) -> bool { +auto kdf_config::from_header(data_cspan data, kdf_config &cfg, + bool ignore_checksum) -> bool { if (data.size() < kdf_config::size()) { return false; } @@ -77,7 +78,7 @@ auto kdf_config::from_header(data_cspan data, kdf_config &cfg) -> bool { cfg.memlimit <= memlimit_level::level4 && cfg.opslimit >= opslimit_level::level1 && cfg.opslimit <= opslimit_level::level3 && - cfg.checksum == cfg.generate_checksum(); + (ignore_checksum || cfg.checksum == cfg.generate_checksum()); } void kdf_config::seal() {