BlockStorage/repertory
1
0
Fork 0
Code Issues 7 Pull Requests Releases 6 Wiki Activity

meta db unit tests and fixes

Browse Source
This commit is contained in:
Scott E. Graves 2024-12-05 08:34:35 -06:00
parent 08379ea622
commit 7100708dfd
6 changed files with 132 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
repertory/librepertory/include/db/i_meta_db.hpp
View File

@ -29,6 +29,8 @@ class i_meta_db {
INTERFACE_SETUP(i_meta_db);
public:
virtual void clear() = 0;
[[nodiscard]] virtual auto get_api_path(const std::string &source_path,
std::string &api_path) const
-> api_error = 0;

5
repertory/librepertory/include/db/rdb_meta_db.hpp
View File

@ -39,6 +39,7 @@ public:
auto operator=(rdb_meta_db &&) -> rdb_meta_db & = delete;
private:
const app_config &cfg_;
std::unique_ptr<rocksdb::DB> db_{nullptr};
rocksdb::ColumnFamilyHandle *default_family_{};
rocksdb::ColumnFamilyHandle *pinned_family_{};
@ -49,6 +50,8 @@ private:
[[nodiscard]] auto create_iterator(rocksdb::ColumnFamilyHandle *family) const
-> std::shared_ptr<rocksdb::Iterator>;
void create_or_open(bool clear);
[[nodiscard]] auto get_item_meta_json(const std::string &api_path,
json &json_data) const -> api_error;
@ -60,6 +63,8 @@ private:
json json_data) -> api_error;
public:
void clear() override;
[[nodiscard]] auto get_api_path(const std::string &source_path,
std::string &api_path) const
-> api_error override;

2
repertory/librepertory/include/db/sqlite_meta_db.hpp
View File

@ -48,6 +48,8 @@ private:
api_meta_map meta) -> api_error;
public:
void clear() override;
[[nodiscard]] auto get_api_path(const std::string &source_path,
std::string &api_path) const
-> api_error override;

63
repertory/librepertory/src/db/rdb_meta_db.cpp
View File

@ -24,6 +24,7 @@
#include "app_config.hpp"
#include "types/startup_exception.hpp"
#include "utils/error_utils.hpp"
#include "utils/file.hpp"
#include "utils/path.hpp"
#include "utils/string.hpp"
@ -31,10 +32,17 @@ namespace {
[[nodiscard]] auto
create_rocksdb(const repertory::app_config &cfg, const std::string &name,
const std::vector<rocksdb::ColumnFamilyDescriptor> &families,
std::vector<rocksdb::ColumnFamilyHandle *> &handles)
std::vector<rocksdb::ColumnFamilyHandle *> &handles, bool clear)
-> std::unique_ptr<rocksdb::DB> {
REPERTORY_USES_FUNCTION_NAME();
auto path = repertory::utils::path::combine(cfg.get_data_directory(), {name});
if (clear &&
not repertory::utils::file::directory{path}.remove_recursively()) {
repertory::utils::error::raise_error(function_name,
"failed to remove meta db|" + path);
}
rocksdb::Options options{};
options.create_if_missing = true;
options.create_missing_column_families = true;
@ -42,10 +50,7 @@ create_rocksdb(const repertory::app_config &cfg, const std::string &name,
options.keep_log_file_num = 10;
rocksdb::DB *ptr{};
auto status = rocksdb::DB::Open(
options,
repertory::utils::path::combine(cfg.get_data_directory(), {name}),
families, &handles, &ptr);
auto status = rocksdb::DB::Open(options, path, families, &handles, &ptr);
if (not status.ok()) {
repertory::utils::error::raise_error(function_name, status.ToString());
throw repertory::startup_exception(status.ToString());
@ -56,30 +61,34 @@ create_rocksdb(const repertory::app_config &cfg, const std::string &name,
} // namespace
namespace repertory {
rdb_meta_db::rdb_meta_db(const app_config &cfg) {
const auto create_resources = [this, &cfg](const std::string &name) {
auto families = std::vector<rocksdb::ColumnFamilyDescriptor>();
families.emplace_back(rocksdb::kDefaultColumnFamilyName,
rocksdb::ColumnFamilyOptions());
families.emplace_back("pinned", rocksdb::ColumnFamilyOptions());
families.emplace_back("size", rocksdb::ColumnFamilyOptions());
families.emplace_back("source", rocksdb::ColumnFamilyOptions());
auto handles = std::vector<rocksdb::ColumnFamilyHandle *>();
db_ = create_rocksdb(cfg, name, families, handles);
std::size_t idx{};
default_family_ = handles[idx++];
pinned_family_ = handles[idx++];
size_family_ = handles[idx++];
source_family_ = handles[idx++];
};
create_resources("provider_meta");
rdb_meta_db::rdb_meta_db(const app_config &cfg) : cfg_(cfg) {
create_or_open(false);
}
rdb_meta_db::~rdb_meta_db() { db_.reset(); }
void rdb_meta_db::create_or_open(bool clear) {
db_.reset();
auto families = std::vector<rocksdb::ColumnFamilyDescriptor>();
families.emplace_back(rocksdb::kDefaultColumnFamilyName,
rocksdb::ColumnFamilyOptions());
families.emplace_back("pinned", rocksdb::ColumnFamilyOptions());
families.emplace_back("size", rocksdb::ColumnFamilyOptions());
families.emplace_back("source", rocksdb::ColumnFamilyOptions());
auto handles = std::vector<rocksdb::ColumnFamilyHandle *>();
db_ = create_rocksdb(cfg_, "provider_meta", families, handles, clear);
std::size_t idx{};
default_family_ = handles[idx++];
pinned_family_ = handles[idx++];
size_family_ = handles[idx++];
source_family_ = handles[idx++];
}
void rdb_meta_db::clear() { create_or_open(true); }
auto rdb_meta_db::create_iterator(rocksdb::ColumnFamilyHandle *family) const
-> std::shared_ptr<rocksdb::Iterator> {
return std::shared_ptr<rocksdb::Iterator>(
@ -353,8 +362,8 @@ auto rdb_meta_db::set_item_meta(const std::string &api_path,
return update_item_meta(api_path, json_data);
}
auto rdb_meta_db::update_item_meta(const std::string &api_path,
json json_data) -> api_error {
auto rdb_meta_db::update_item_meta(const std::string &api_path, json json_data)
-> api_error {
REPERTORY_USES_FUNCTION_NAME();
try {

12
repertory/librepertory/src/db/sqlite_meta_db.cpp
View File

@ -55,6 +55,18 @@ sqlite_meta_db::sqlite_meta_db(const app_config &cfg) {
sqlite_meta_db::~sqlite_meta_db() { db_.reset(); }
void sqlite_meta_db::clear() {
REPERTORY_USES_FUNCTION_NAME();
auto result = utils::db::sqlite::db_delete{*db_, table_name}.go();
if (result.ok()) {
return;
}
utils::error::raise_error(function_name,
"failed to clear meta db|" + result.get_error());
}
auto sqlite_meta_db::get_api_path(const std::string &source_path,
std::string &api_path) const -> api_error {
auto result = utils::db::sqlite::db_select{*db_, table_name}

75
repertory/repertory_test/src/meta_db_test.cpp
View File

@ -280,4 +280,79 @@ TYPED_TEST(meta_db_test, can_get_pinned_files) {
EXPECT_TRUE(utils::collection::excludes(pinned, api_path));
}
}
TYPED_TEST(meta_db_test, can_get_total_item_count) {
this->meta_db->clear();
EXPECT_EQ(0U, this->meta_db->get_total_item_count());
auto test_file = create_test_file();
auto test_source = create_test_file();
EXPECT_EQ(
api_error::success,
this->meta_db->set_item_meta(
test_file, {
{META_DIRECTORY, utils::string::from_bool(false)},
{META_SOURCE, test_source},
}));
auto test_dir = create_test_file();
EXPECT_EQ(api_error::success,
this->meta_db->set_item_meta(
test_dir, {
{META_DIRECTORY, utils::string::from_bool(true)},
}));
EXPECT_EQ(2U, this->meta_db->get_total_item_count());
}
TYPED_TEST(meta_db_test,
get_total_item_count_decreases_after_directory_is_removed) {
this->meta_db->clear();
EXPECT_EQ(0U, this->meta_db->get_total_item_count());
auto test_file = create_test_file();
auto test_source = create_test_file();
EXPECT_EQ(
api_error::success,
this->meta_db->set_item_meta(
test_file, {
{META_DIRECTORY, utils::string::from_bool(false)},
{META_SOURCE, test_source},
}));
auto test_dir = create_test_file();
EXPECT_EQ(api_error::success,
this->meta_db->set_item_meta(
test_dir, {
{META_DIRECTORY, utils::string::from_bool(true)},
}));
this->meta_db->remove_api_path(test_dir);
EXPECT_EQ(1U, this->meta_db->get_total_item_count());
}
TYPED_TEST(meta_db_test, get_total_item_count_decreases_after_file_is_removed) {
this->meta_db->clear();
EXPECT_EQ(0U, this->meta_db->get_total_item_count());
auto test_file = create_test_file();
auto test_source = create_test_file();
EXPECT_EQ(
api_error::success,
this->meta_db->set_item_meta(
test_file, {
{META_DIRECTORY, utils::string::from_bool(false)},
{META_SOURCE, test_source},
}));
auto test_dir = create_test_file();
EXPECT_EQ(api_error::success,
this->meta_db->set_item_meta(
test_dir, {
{META_DIRECTORY, utils::string::from_bool(true)},
}));
this->meta_db->remove_api_path(test_file);
EXPECT_EQ(1U, this->meta_db->get_total_item_count());
}
} // namespace repertory