From 73d1d993d7bd8626cfa9159fe7690483a9ed5872 Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Wed, 18 Dec 2024 08:23:22 -0600 Subject: [PATCH] file db unit tests and fixes --- .../include/db/impl/rdb_file_db.hpp | 10 +- .../librepertory/src/db/impl/rdb_file_db.cpp | 256 ++++++++++++++++-- .../include/fixtures/file_db_fixture.hpp | 3 +- 3 files changed, 237 insertions(+), 32 deletions(-) diff --git a/repertory/librepertory/include/db/impl/rdb_file_db.hpp b/repertory/librepertory/include/db/impl/rdb_file_db.hpp index 37e639f2..efaecaf2 100644 --- a/repertory/librepertory/include/db/impl/rdb_file_db.hpp +++ b/repertory/librepertory/include/db/impl/rdb_file_db.hpp @@ -42,20 +42,24 @@ private: private: std::unique_ptr db_{nullptr}; + rocksdb::ColumnFamilyHandle *default_family_{}; + rocksdb::ColumnFamilyHandle *directory_family_{}; + rocksdb::ColumnFamilyHandle *file_family_{}; private: void create_or_open(bool clear); - [[nodiscard]] auto create_iterator() const + [[nodiscard]] auto create_iterator(rocksdb::ColumnFamilyHandle *family) const -> std::shared_ptr; [[nodiscard]] static auto perform_action(std::string_view function_name, - std::function action) -> bool; + std::function action) -> api_error; [[nodiscard]] auto perform_action( std::string_view function_name, - std::function action) -> bool; + std::function action) + -> api_error; public: [[nodiscard]] auto add_directory(const std::string &api_path, diff --git a/repertory/librepertory/src/db/impl/rdb_file_db.cpp b/repertory/librepertory/src/db/impl/rdb_file_db.cpp index 7ac44ca9..c4db7714 100644 --- a/repertory/librepertory/src/db/impl/rdb_file_db.cpp +++ b/repertory/librepertory/src/db/impl/rdb_file_db.cpp @@ -75,75 +75,254 @@ void rdb_file_db::create_or_open(bool clear) { auto families = std::vector(); families.emplace_back(rocksdb::kDefaultColumnFamilyName, rocksdb::ColumnFamilyOptions()); + families.emplace_back("directory", rocksdb::ColumnFamilyOptions()); + families.emplace_back("file", rocksdb::ColumnFamilyOptions()); auto handles = std::vector(); db_ = create_rocksdb(cfg_, "file", families, handles, clear); + + std::size_t idx{}; + default_family_ = handles[idx++]; + file_family_ = handles[idx++]; + directory_family_ = handles[idx++]; } auto rdb_file_db::add_directory(const std::string &api_path, - const std::string &source_path) -> api_error {} + const std::string &source_path) -> api_error { + REPERTORY_USES_FUNCTION_NAME(); + + return perform_action( + function_name, [&](rocksdb::Transaction *txn) -> rocksdb::Status { + auto res = txn->Put(directory_family_, api_path, source_path); + if (not res.ok()) { + return res; + } + + return txn->Put(default_family_, source_path, api_path); + }); +} auto rdb_file_db::add_or_update_file(const i_file_db::file_data &data) - -> api_error {} + -> api_error { + REPERTORY_USES_FUNCTION_NAME(); + + return perform_action( + function_name, [&](rocksdb::Transaction *txn) -> rocksdb::Status { + json json_data = { + {"file_size", data.file_size}, + {"iv", data.iv_list}, + {"source_path", data.source_path}, + }; + + auto res = txn->Put(file_family_, data.api_path, json_data.dump()); + if (not res.ok()) { + return res; + } + + return txn->Put(default_family_, data.source_path, data.api_path); + }); +} void rdb_file_db::clear() { create_or_open(true); } -auto rdb_file_db::create_iterator() const +auto rdb_file_db::create_iterator(rocksdb::ColumnFamilyHandle *family) const -> std::shared_ptr { return std::shared_ptr( - db_->NewIterator(rocksdb::ReadOptions())); + db_->NewIterator(rocksdb::ReadOptions{}, family)); } -auto rdb_file_db::count() const -> std::uint64_t {} +auto rdb_file_db::count() const -> std::uint64_t { + std::uint64_t ret{}; + + auto iter = create_iterator(default_family_); + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ++ret; + } + + return ret; +} auto rdb_file_db::get_api_path(const std::string &source_path, - std::string &api_path) const -> api_error {} + std::string &api_path) const -> api_error { + REPERTORY_USES_FUNCTION_NAME(); + + return perform_action(function_name, [&]() -> rocksdb::Status { + return db_->Get(rocksdb::ReadOptions{}, default_family_, source_path, + &api_path); + }); +} auto rdb_file_db::get_directory_api_path(const std::string &source_path, std::string &api_path) const - -> api_error {} + -> api_error { + REPERTORY_USES_FUNCTION_NAME(); + + auto result = perform_action(function_name, [&]() -> rocksdb::Status { + std::string api_path; + auto res = db_->Get(rocksdb::ReadOptions{}, default_family_, source_path, + &api_path); + if (not res.ok()) { + return res; + } + + std::string value; + return db_->Get(rocksdb::ReadOptions{}, directory_family_, api_path, + &value); + }); + + if (result != api_error::success) { + api_path.clear(); + } + + return result == api_error::item_not_found ? api_error::directory_not_found + : result; +} auto rdb_file_db::get_directory_source_path(const std::string &api_path, std::string &source_path) const - -> api_error {} + -> api_error { + REPERTORY_USES_FUNCTION_NAME(); + + auto result = perform_action(function_name, [&]() -> rocksdb::Status { + return db_->Get(rocksdb::ReadOptions{}, directory_family_, api_path, + &source_path); + }); + + return result == api_error::item_not_found ? api_error::directory_not_found + : result; +} auto rdb_file_db::get_file_api_path(const std::string &source_path, - std::string &api_path) const -> api_error {} + std::string &api_path) const -> api_error { + REPERTORY_USES_FUNCTION_NAME(); + + auto result = perform_action(function_name, [&]() -> rocksdb::Status { + auto res = db_->Get(rocksdb::ReadOptions{}, default_family_, source_path, + &api_path); + if (not res.ok()) { + return res; + } + + std::string value; + return db_->Get(rocksdb::ReadOptions{}, file_family_, api_path, &value); + }); + + if (result != api_error::success) { + api_path.clear(); + } + + return result; +} auto rdb_file_db::get_file_data(const std::string &api_path, i_file_db::file_data &data) const -> api_error { + REPERTORY_USES_FUNCTION_NAME(); + + auto result = perform_action(function_name, [&]() -> rocksdb::Status { + std::string value; + auto res = db_->Get(rocksdb::ReadOptions{}, file_family_, api_path, &value); + if (not res.ok()) { + return res; + } + + auto json_data = json::parse(value); + data.api_path = api_path; + data.file_size = json_data.at("file_size").get(); + data.iv_list = + json_data.at("iv") + .get>>(); + data.source_path = json_data.at("source_path").get(); + + return res; + }); + + return result; } auto rdb_file_db::get_file_source_path(const std::string &api_path, std::string &source_path) const - -> api_error {} + -> api_error { + REPERTORY_USES_FUNCTION_NAME(); -auto rdb_file_db::get_item_list() const -> std::vector {} + auto result = perform_action(function_name, [&]() -> rocksdb::Status { + std::string value; + auto res = db_->Get(rocksdb::ReadOptions{}, file_family_, api_path, &value); + if (not res.ok()) { + return res; + } + + auto json_data = json::parse(value); + source_path = json_data.at("source_path").get(); + return res; + }); + + return result; +} + +auto rdb_file_db::get_item_list() const -> std::vector { + std::vector ret{}; + { + auto iter = create_iterator(directory_family_); + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ret.emplace_back(i_file_db::file_info{ + iter->key().ToString(), + true, + iter->value().ToString(), + }); + } + } + + { + auto iter = create_iterator(file_family_); + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + auto json_data = json::parse(iter->value().ToString()); + ret.emplace_back(i_file_db::file_info{ + iter->key().ToString(), + true, + json_data.at("source_path").get(), + }); + } + } + + return ret; +} auto rdb_file_db::get_source_path(const std::string &api_path, std::string &source_path) const -> api_error { + auto iter = create_iterator(default_family_); + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + if (iter->value().ToString() != api_path) { + continue; + } + + source_path = iter->key().ToString(); + return api_error::success; + } + + return api_error::item_not_found; } auto rdb_file_db::perform_action(std::string_view function_name, std::function action) - -> bool { - try { - auto res = action(); - if (not res.ok()) { - utils::error::raise_error(function_name, res.ToString()); - } - - return res.ok(); - } catch (const std::exception &ex) { - utils::error::raise_error(function_name, ex); + -> api_error { + auto res = action(); + if (res.ok()) { + return api_error::success; } - return false; + if (not res.IsNotFound()) { + utils::error::raise_error(function_name, res.ToString()); + } + + return res.IsNotFound() ? api_error::item_not_found : api_error::error; } auto rdb_file_db::perform_action( std::string_view function_name, - std::function action) -> bool { + std::function action) + -> api_error { std::unique_ptr txn{ db_->BeginTransaction(rocksdb::WriteOptions{}, rocksdb::TransactionOptions{}), @@ -154,12 +333,12 @@ auto rdb_file_db::perform_action( if (res.ok()) { auto commit_res = txn->Commit(); if (commit_res.ok()) { - return true; + return api_error::success; } utils::error::raise_error(function_name, "rocksdb commit failed|" + res.ToString()); - return false; + return api_error::error; } utils::error::raise_error(function_name, @@ -172,8 +351,31 @@ auto rdb_file_db::perform_action( auto rollback_res = txn->Rollback(); utils::error::raise_error(function_name, "rocksdb rollback failed|" + rollback_res.ToString()); - return false; + return api_error::error; } -auto rdb_file_db::remove_item(const std::string &api_path) -> api_error {} +auto rdb_file_db::remove_item(const std::string &api_path) -> api_error { + REPERTORY_USES_FUNCTION_NAME(); + + std::string source_path; + auto res = get_source_path(api_path, source_path); + if (res != api_error::success) { + return res; + } + + return perform_action(function_name, + [&](rocksdb::Transaction *txn) -> rocksdb::Status { + auto res = txn->Delete(default_family_, source_path); + if (not res.ok()) { + return res; + } + + res = txn->Delete(directory_family_, api_path); + if (not res.ok()) { + return res; + } + + return txn->Delete(file_family_, api_path); + }); +} } // namespace repertory diff --git a/repertory/repertory_test/include/fixtures/file_db_fixture.hpp b/repertory/repertory_test/include/fixtures/file_db_fixture.hpp index 3846c758..db54a473 100644 --- a/repertory/repertory_test/include/fixtures/file_db_fixture.hpp +++ b/repertory/repertory_test/include/fixtures/file_db_fixture.hpp @@ -58,8 +58,7 @@ protected: } }; -// using file_db_types = ::testing::Types; -using file_db_types = ::testing::Types; +using file_db_types = ::testing::Types; template std::unique_ptr file_db_test::config;