BlockStorage/repertory
1
0
Fork 0
Code Issues 7 Pull Requests Releases 6 Wiki Activity

file db unit tests and fixes

Browse Source
This commit is contained in:
Scott E. Graves 2024-12-18 08:23:22 -06:00
parent efa5e07549
commit 73d1d993d7
3 changed files with 237 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
repertory/librepertory/include/db/impl/rdb_file_db.hpp
View File

@ -42,20 +42,24 @@ private:
private:
std::unique_ptr<rocksdb::TransactionDB> db_{nullptr};
rocksdb::ColumnFamilyHandle *default_family_{};
rocksdb::ColumnFamilyHandle *directory_family_{};
rocksdb::ColumnFamilyHandle *file_family_{};
private:
void create_or_open(bool clear);
[[nodiscard]] auto create_iterator() const
[[nodiscard]] auto create_iterator(rocksdb::ColumnFamilyHandle *family) const
-> std::shared_ptr<rocksdb::Iterator>;
[[nodiscard]] static auto
perform_action(std::string_view function_name,
std::function<rocksdb::Status()> action) -> bool;
std::function<rocksdb::Status()> action) -> api_error;
[[nodiscard]] auto perform_action(
std::string_view function_name,
std::function<rocksdb::Status(rocksdb::Transaction *txn)> action) -> bool;
std::function<rocksdb::Status(rocksdb::Transaction *txn)> action)
-> api_error;
public:
[[nodiscard]] auto add_directory(const std::string &api_path,

256
repertory/librepertory/src/db/impl/rdb_file_db.cpp
View File

@ -75,75 +75,254 @@ void rdb_file_db::create_or_open(bool clear) {
auto families = std::vector<rocksdb::ColumnFamilyDescriptor>();
families.emplace_back(rocksdb::kDefaultColumnFamilyName,
rocksdb::ColumnFamilyOptions());
families.emplace_back("directory", rocksdb::ColumnFamilyOptions());
families.emplace_back("file", rocksdb::ColumnFamilyOptions());
auto handles = std::vector<rocksdb::ColumnFamilyHandle *>();
db_ = create_rocksdb(cfg_, "file", families, handles, clear);
std::size_t idx{};
default_family_ = handles[idx++];
file_family_ = handles[idx++];
directory_family_ = handles[idx++];
}
auto rdb_file_db::add_directory(const std::string &api_path,
const std::string &source_path) -> api_error {}
const std::string &source_path) -> api_error {
REPERTORY_USES_FUNCTION_NAME();
return perform_action(
function_name, [&](rocksdb::Transaction *txn) -> rocksdb::Status {
auto res = txn->Put(directory_family_, api_path, source_path);
if (not res.ok()) {
return res;
}
return txn->Put(default_family_, source_path, api_path);
});
}
auto rdb_file_db::add_or_update_file(const i_file_db::file_data &data)
-> api_error {}
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
return perform_action(
function_name, [&](rocksdb::Transaction *txn) -> rocksdb::Status {
json json_data = {
{"file_size", data.file_size},
{"iv", data.iv_list},
{"source_path", data.source_path},
};
auto res = txn->Put(file_family_, data.api_path, json_data.dump());
if (not res.ok()) {
return res;
}
return txn->Put(default_family_, data.source_path, data.api_path);
});
}
void rdb_file_db::clear() { create_or_open(true); }
auto rdb_file_db::create_iterator() const
auto rdb_file_db::create_iterator(rocksdb::ColumnFamilyHandle *family) const
-> std::shared_ptr<rocksdb::Iterator> {
return std::shared_ptr<rocksdb::Iterator>(
db_->NewIterator(rocksdb::ReadOptions()));
db_->NewIterator(rocksdb::ReadOptions{}, family));
}
auto rdb_file_db::count() const -> std::uint64_t {}
auto rdb_file_db::count() const -> std::uint64_t {
std::uint64_t ret{};
auto iter = create_iterator(default_family_);
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
++ret;
}
return ret;
}
auto rdb_file_db::get_api_path(const std::string &source_path,
std::string &api_path) const -> api_error {}
std::string &api_path) const -> api_error {
REPERTORY_USES_FUNCTION_NAME();
return perform_action(function_name, [&]() -> rocksdb::Status {
return db_->Get(rocksdb::ReadOptions{}, default_family_, source_path,
&api_path);
});
}
auto rdb_file_db::get_directory_api_path(const std::string &source_path,
std::string &api_path) const
-> api_error {}
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
auto result = perform_action(function_name, [&]() -> rocksdb::Status {
std::string api_path;
auto res = db_->Get(rocksdb::ReadOptions{}, default_family_, source_path,
&api_path);
if (not res.ok()) {
return res;
}
std::string value;
return db_->Get(rocksdb::ReadOptions{}, directory_family_, api_path,
&value);
});
if (result != api_error::success) {
api_path.clear();
}
return result == api_error::item_not_found ? api_error::directory_not_found
: result;
}
auto rdb_file_db::get_directory_source_path(const std::string &api_path,
std::string &source_path) const
-> api_error {}
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
auto result = perform_action(function_name, [&]() -> rocksdb::Status {
return db_->Get(rocksdb::ReadOptions{}, directory_family_, api_path,
&source_path);
});
return result == api_error::item_not_found ? api_error::directory_not_found
: result;
}
auto rdb_file_db::get_file_api_path(const std::string &source_path,
std::string &api_path) const -> api_error {}
std::string &api_path) const -> api_error {
REPERTORY_USES_FUNCTION_NAME();
auto result = perform_action(function_name, [&]() -> rocksdb::Status {
auto res = db_->Get(rocksdb::ReadOptions{}, default_family_, source_path,
&api_path);
if (not res.ok()) {
return res;
}
std::string value;
return db_->Get(rocksdb::ReadOptions{}, file_family_, api_path, &value);
});
if (result != api_error::success) {
api_path.clear();
}
return result;
}
auto rdb_file_db::get_file_data(const std::string &api_path,
i_file_db::file_data &data) const -> api_error {
REPERTORY_USES_FUNCTION_NAME();
auto result = perform_action(function_name, [&]() -> rocksdb::Status {
std::string value;
auto res = db_->Get(rocksdb::ReadOptions{}, file_family_, api_path, &value);
if (not res.ok()) {
return res;
}
auto json_data = json::parse(value);
data.api_path = api_path;
data.file_size = json_data.at("file_size").get<std::uint64_t>();
data.iv_list =
json_data.at("iv")
.get<std::vector<
std::array<unsigned char,
crypto_aead_xchacha20poly1305_IETF_NPUBBYTES>>>();
data.source_path = json_data.at("source_path").get<std::string>();
return res;
});
return result;
}
auto rdb_file_db::get_file_source_path(const std::string &api_path,
std::string &source_path) const
-> api_error {}
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
auto rdb_file_db::get_item_list() const -> std::vector<i_file_db::file_info> {}
auto result = perform_action(function_name, [&]() -> rocksdb::Status {
std::string value;
auto res = db_->Get(rocksdb::ReadOptions{}, file_family_, api_path, &value);
if (not res.ok()) {
return res;
}
auto json_data = json::parse(value);
source_path = json_data.at("source_path").get<std::string>();
return res;
});
return result;
}
auto rdb_file_db::get_item_list() const -> std::vector<i_file_db::file_info> {
std::vector<i_file_db::file_info> ret{};
{
auto iter = create_iterator(directory_family_);
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
ret.emplace_back(i_file_db::file_info{
iter->key().ToString(),
true,
iter->value().ToString(),
});
}
}
{
auto iter = create_iterator(file_family_);
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
auto json_data = json::parse(iter->value().ToString());
ret.emplace_back(i_file_db::file_info{
iter->key().ToString(),
true,
json_data.at("source_path").get<std::string>(),
});
}
}
return ret;
}
auto rdb_file_db::get_source_path(const std::string &api_path,
std::string &source_path) const -> api_error {
auto iter = create_iterator(default_family_);
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
if (iter->value().ToString() != api_path) {
continue;
}
source_path = iter->key().ToString();
return api_error::success;
}
return api_error::item_not_found;
}
auto rdb_file_db::perform_action(std::string_view function_name,
std::function<rocksdb::Status()> action)
-> bool {
try {
auto res = action();
if (not res.ok()) {
utils::error::raise_error(function_name, res.ToString());
}
return res.ok();
} catch (const std::exception &ex) {
utils::error::raise_error(function_name, ex);
-> api_error {
auto res = action();
if (res.ok()) {
return api_error::success;
}
return false;
if (not res.IsNotFound()) {
utils::error::raise_error(function_name, res.ToString());
}
return res.IsNotFound() ? api_error::item_not_found : api_error::error;
}
auto rdb_file_db::perform_action(
std::string_view function_name,
std::function<rocksdb::Status(rocksdb::Transaction *txn)> action) -> bool {
std::function<rocksdb::Status(rocksdb::Transaction *txn)> action)
-> api_error {
std::unique_ptr<rocksdb::Transaction> txn{
db_->BeginTransaction(rocksdb::WriteOptions{},
rocksdb::TransactionOptions{}),
@ -154,12 +333,12 @@ auto rdb_file_db::perform_action(
if (res.ok()) {
auto commit_res = txn->Commit();
if (commit_res.ok()) {
return true;
return api_error::success;
}
utils::error::raise_error(function_name,
"rocksdb commit failed|" + res.ToString());
return false;
return api_error::error;
}
utils::error::raise_error(function_name,
@ -172,8 +351,31 @@ auto rdb_file_db::perform_action(
auto rollback_res = txn->Rollback();
utils::error::raise_error(function_name, "rocksdb rollback failed|" +
rollback_res.ToString());
return false;
return api_error::error;
}
auto rdb_file_db::remove_item(const std::string &api_path) -> api_error {}
auto rdb_file_db::remove_item(const std::string &api_path) -> api_error {
REPERTORY_USES_FUNCTION_NAME();
std::string source_path;
auto res = get_source_path(api_path, source_path);
if (res != api_error::success) {
return res;
}
return perform_action(function_name,
[&](rocksdb::Transaction *txn) -> rocksdb::Status {
auto res = txn->Delete(default_family_, source_path);
if (not res.ok()) {
return res;
}
res = txn->Delete(directory_family_, api_path);
if (not res.ok()) {
return res;
}
return txn->Delete(file_family_, api_path);
});
}
} // namespace repertory

3
repertory/repertory_test/include/fixtures/file_db_fixture.hpp
View File

@ -58,8 +58,7 @@ protected:
}
};
// using file_db_types = ::testing::Types<sqlite_file_db, rdb_file_db>;
using file_db_types = ::testing::Types<sqlite_file_db>;
using file_db_types = ::testing::Types<sqlite_file_db, rdb_file_db>;
template <typename db_t> std::unique_ptr<app_config> file_db_test<db_t>::config;