BlockStorage/repertory
1
0
Fork 0
Code Issues 8 Pull Requests Releases 6 Wiki Activity

meta db unit tests and fixes

Browse Source
This commit is contained in:
Scott E. Graves 2024-12-04 13:51:41 -06:00
parent c6b895ced2
commit b4621f6a4e
3 changed files with 71 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
repertory/librepertory/include/db/rdb_meta_db.hpp
View File

@ -41,8 +41,6 @@ public:
private: private:
std::unique_ptr<rocksdb::DB> db_{nullptr}; std::unique_ptr<rocksdb::DB> db_{nullptr};
rocksdb::ColumnFamilyHandle *default_family_{}; rocksdb::ColumnFamilyHandle *default_family_{};
rocksdb::ColumnFamilyHandle *directory_family_{};
rocksdb::ColumnFamilyHandle *keys_family_{};
rocksdb::ColumnFamilyHandle *pinned_family_{}; rocksdb::ColumnFamilyHandle *pinned_family_{};
rocksdb::ColumnFamilyHandle *size_family_{}; rocksdb::ColumnFamilyHandle *size_family_{};
rocksdb::ColumnFamilyHandle *source_family_{}; rocksdb::ColumnFamilyHandle *source_family_{};

134
repertory/librepertory/src/db/rdb_meta_db.cpp
View File

@ -61,8 +61,6 @@ rdb_meta_db::rdb_meta_db(const app_config &cfg) {
auto families = std::vector<rocksdb::ColumnFamilyDescriptor>(); auto families = std::vector<rocksdb::ColumnFamilyDescriptor>();
families.emplace_back(rocksdb::kDefaultColumnFamilyName, families.emplace_back(rocksdb::kDefaultColumnFamilyName,
rocksdb::ColumnFamilyOptions()); rocksdb::ColumnFamilyOptions());
families.emplace_back("directory", rocksdb::ColumnFamilyOptions());
families.emplace_back("keys", rocksdb::ColumnFamilyOptions());
families.emplace_back("pinned", rocksdb::ColumnFamilyOptions()); families.emplace_back("pinned", rocksdb::ColumnFamilyOptions());
families.emplace_back("size", rocksdb::ColumnFamilyOptions()); families.emplace_back("size", rocksdb::ColumnFamilyOptions());
families.emplace_back("source", rocksdb::ColumnFamilyOptions()); families.emplace_back("source", rocksdb::ColumnFamilyOptions());
@ -72,8 +70,6 @@ rdb_meta_db::rdb_meta_db(const app_config &cfg) {
std::size_t idx{}; std::size_t idx{};
default_family_ = handles[idx++]; default_family_ = handles[idx++];
directory_family_ = handles[idx++];
keys_family_ = handles[idx++];
pinned_family_ = handles[idx++]; pinned_family_ = handles[idx++];
size_family_ = handles[idx++]; size_family_ = handles[idx++];
source_family_ = handles[idx++]; source_family_ = handles[idx++];
@ -123,7 +119,7 @@ auto rdb_meta_db::get_item_meta_json(const std::string &api_path,
{ {
std::string value; std::string value;
auto res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status { auto res = perform_action(function_name, [&]() -> rocksdb::Status {
return db_->Get(rocksdb::ReadOptions(), default_family_, api_path, return db_->Get(rocksdb::ReadOptions(), default_family_, api_path,
&value); &value);
}); });
@ -139,32 +135,7 @@ auto rdb_meta_db::get_item_meta_json(const std::string &api_path,
{ {
std::string value; std::string value;
auto res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status { auto res = perform_action(function_name, [&]() -> rocksdb::Status {
return db_->Get(rocksdb::ReadOptions(), directory_family_, api_path,
&value);
});
if (res != api_error::success) {
return res;
}
json_data[META_DIRECTORY] = value;
found = found || not value.empty();
}
{
std::string value;
auto res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
return db_->Get(rocksdb::ReadOptions(), keys_family_, api_path, &value);
});
if (res != api_error::success) {
return res;
}
json_data[META_KEY] = value;
found = found || not value.empty();
}
{
std::string value;
auto res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
return db_->Get(rocksdb::ReadOptions(), pinned_family_, api_path, return db_->Get(rocksdb::ReadOptions(), pinned_family_, api_path,
&value); &value);
}); });
@ -177,7 +148,7 @@ auto rdb_meta_db::get_item_meta_json(const std::string &api_path,
{ {
std::string value; std::string value;
auto res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status { auto res = perform_action(function_name, [&]() -> rocksdb::Status {
return db_->Get(rocksdb::ReadOptions(), size_family_, api_path, &value); return db_->Get(rocksdb::ReadOptions(), size_family_, api_path, &value);
}); });
if (res != api_error::success) { if (res != api_error::success) {
@ -187,19 +158,6 @@ auto rdb_meta_db::get_item_meta_json(const std::string &api_path,
found = found || not value.empty(); found = found || not value.empty();
} }
{
std::string value;
auto res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
return db_->Get(rocksdb::ReadOptions(), source_family_, api_path,
&value);
});
if (res != api_error::success) {
return res;
}
json_data[META_SOURCE] = value;
found = found || not value.empty();
}
return found ? api_error::success : api_error::item_not_found; return found ? api_error::success : api_error::item_not_found;
} catch (const std::exception &e) { } catch (const std::exception &e) {
utils::error::raise_api_path_error(function_name, api_path, e, utils::error::raise_api_path_error(function_name, api_path, e,
@ -227,6 +185,20 @@ auto rdb_meta_db::get_item_meta(const std::string &api_path,
auto rdb_meta_db::get_item_meta(const std::string &api_path, auto rdb_meta_db::get_item_meta(const std::string &api_path,
const std::string &key, const std::string &key,
std::string &value) const -> api_error { std::string &value) const -> api_error {
REPERTORY_USES_FUNCTION_NAME();
if (key == META_PINNED) {
return perform_action(function_name, [&]() -> rocksdb::Status {
return db_->Get(rocksdb::ReadOptions(), pinned_family_, api_path, &value);
});
}
if (key == META_SIZE) {
return perform_action(function_name, [&]() -> rocksdb::Status {
return db_->Get(rocksdb::ReadOptions(), size_family_, api_path, &value);
});
}
json json_data; json json_data;
auto ret = get_item_meta_json(api_path, json_data); auto ret = get_item_meta_json(api_path, json_data);
if (ret != api_error::success) { if (ret != api_error::success) {
@ -290,12 +262,18 @@ auto rdb_meta_db::perform_action(std::string_view function_name,
} }
void rdb_meta_db::remove_api_path(const std::string &api_path) { void rdb_meta_db::remove_api_path(const std::string &api_path) {
[[maybe_unused]] auto res = REPERTORY_USES_FUNCTION_NAME();
perform_action(__FUNCTION__, [this, &api_path]() -> rocksdb::Status {
db_->Delete(rocksdb::WriteOptions(), directory_family_, api_path); std::string source_path;
[[maybe_unused]] auto res = get_item_meta(api_path, META_SOURCE, source_path);
res = perform_action(
function_name, [this, &api_path, &source_path]() -> rocksdb::Status {
db_->Delete(rocksdb::WriteOptions(), pinned_family_, api_path); db_->Delete(rocksdb::WriteOptions(), pinned_family_, api_path);
db_->Delete(rocksdb::WriteOptions(), size_family_, api_path); db_->Delete(rocksdb::WriteOptions(), size_family_, api_path);
db_->Delete(rocksdb::WriteOptions(), source_family_, api_path); if (not source_path.empty()) {
db_->Delete(rocksdb::WriteOptions(), source_family_, source_path);
}
return db_->Delete(rocksdb::WriteOptions(), default_family_, api_path); return db_->Delete(rocksdb::WriteOptions(), default_family_, api_path);
}); });
} }
@ -334,6 +312,20 @@ auto rdb_meta_db::rename_item_meta(const std::string &from_api_path,
auto rdb_meta_db::set_item_meta(const std::string &api_path, auto rdb_meta_db::set_item_meta(const std::string &api_path,
const std::string &key, const std::string &key,
const std::string &value) -> api_error { const std::string &value) -> api_error {
REPERTORY_USES_FUNCTION_NAME();
if (key == META_PINNED) {
return perform_action(function_name, [&]() -> rocksdb::Status {
return db_->Put(rocksdb::WriteOptions(), pinned_family_, api_path, value);
});
}
if (key == META_SIZE) {
return perform_action(function_name, [&]() -> rocksdb::Status {
return db_->Put(rocksdb::WriteOptions(), size_family_, api_path, value);
});
}
json json_data; json json_data;
auto res = get_item_meta_json(api_path, json_data); auto res = get_item_meta_json(api_path, json_data);
if (res != api_error::success && res != api_error::item_not_found) { if (res != api_error::success && res != api_error::item_not_found) {
@ -365,8 +357,11 @@ auto rdb_meta_db::update_item_meta(const std::string &api_path, json json_data)
REPERTORY_USES_FUNCTION_NAME(); REPERTORY_USES_FUNCTION_NAME();
try { try {
auto directory = std::string orig_source_path;
utils::string::to_bool(json_data.at(META_DIRECTORY).get<std::string>()); auto ret = get_item_meta(api_path, META_SOURCE, orig_source_path);
if (ret != api_error::success && ret != api_error::item_not_found) {
return ret;
}
if (not json_data.contains(META_PINNED)) { if (not json_data.contains(META_PINNED)) {
json_data[META_PINNED] = utils::string::from_bool(false); json_data[META_PINNED] = utils::string::from_bool(false);
@ -378,30 +373,31 @@ auto rdb_meta_db::update_item_meta(const std::string &api_path, json json_data)
json_data[META_SOURCE] = ""; json_data[META_SOURCE] = "";
} }
auto directory =
utils::string::to_bool(json_data.at(META_DIRECTORY).get<std::string>());
auto pinned = auto pinned =
utils::string::to_bool(json_data.at(META_PINNED).get<std::string>()); utils::string::to_bool(json_data.at(META_PINNED).get<std::string>());
auto size = directory ? std::uint64_t(0U) auto size = directory ? std::uint64_t(0U)
: utils::string::to_uint64( : utils::string::to_uint64(
json_data.at(META_SIZE).get<std::string>()); json_data.at(META_SIZE).get<std::string>());
auto source_path = json_data.at(META_SOURCE).get<std::string>(); auto source_path = json_data.at(META_SOURCE).get<std::string>();
if (orig_source_path != source_path && not orig_source_path.empty()) {
ret = perform_action(function_name, [&]() -> rocksdb::Status {
return db_->Delete(rocksdb::WriteOptions(), source_family_,
orig_source_path);
});
if (ret != api_error::success && ret != api_error::item_not_found) {
return ret;
}
}
json_data.erase(META_DIRECTORY);
json_data.erase(META_PINNED); json_data.erase(META_PINNED);
json_data.erase(META_SIZE); json_data.erase(META_SIZE);
json_data.erase(META_SOURCE);
return perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
auto res = db_->Put(rocksdb::WriteOptions(), directory_family_, api_path,
utils::string::from_bool(directory));
if (not res.ok()) {
return res;
}
return perform_action(function_name, [&]() -> rocksdb::Status {
if (not directory) { if (not directory) {
db_->Put(rocksdb::WriteOptions(), pinned_family_, api_path, auto res = db_->Put(rocksdb::WriteOptions(), pinned_family_, api_path,
utils::string::from_bool(pinned)); utils::string::from_bool(pinned));
if (not res.ok()) { if (not res.ok()) {
return res; return res;
} }
@ -413,10 +409,12 @@ auto rdb_meta_db::update_item_meta(const std::string &api_path, json json_data)
} }
} }
res = db_->Put(rocksdb::WriteOptions(), source_family_, api_path, if (not source_path.empty()) {
source_path); auto res = db_->Put(rocksdb::WriteOptions(), source_family_,
if (not res.ok()) { source_path, api_path);
return res; if (not res.ok()) {
return res;
}
} }
return db_->Put(rocksdb::WriteOptions(), default_family_, api_path, return db_->Put(rocksdb::WriteOptions(), default_family_, api_path,

8
repertory/librepertory/src/db/sqlite_meta_db.cpp
View File

@ -307,19 +307,21 @@ auto sqlite_meta_db::update_item_meta(const std::string &api_path,
REPERTORY_USES_FUNCTION_NAME(); REPERTORY_USES_FUNCTION_NAME();
try { try {
auto directory = utils::string::to_bool(meta.at(META_DIRECTORY));
if (meta[META_PINNED].empty()) { if (meta[META_PINNED].empty()) {
meta[META_PINNED] = utils::string::from_bool(false); meta[META_PINNED] = utils::string::from_bool(false);
} }
if (meta[META_SIZE].empty()) { if (meta[META_SIZE].empty()) {
meta[META_SIZE] = "0"; meta[META_SIZE] = "0";
} }
if (meta[META_SOURCE].empty()) {
meta[META_SOURCE] = "";
}
auto directory = utils::string::to_bool(meta.at(META_DIRECTORY));
auto pinned = utils::string::to_bool(meta.at(META_PINNED)); auto pinned = utils::string::to_bool(meta.at(META_PINNED));
auto size = directory ? std::uint64_t(0U) auto size = directory ? std::uint64_t(0U)
: utils::string::to_uint64(meta.at(META_SIZE)); : utils::string::to_uint64(meta.at(META_SIZE));
auto source_path = meta[META_SOURCE]; auto source_path = meta.at(META_SOURCE);
meta.erase(META_DIRECTORY); meta.erase(META_DIRECTORY);
meta.erase(META_PINNED); meta.erase(META_PINNED);