From b4621f6a4e48ccfb5de1982b42de535c3610748e Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Wed, 4 Dec 2024 13:51:41 -0600 Subject: [PATCH] meta db unit tests and fixes --- .../librepertory/include/db/rdb_meta_db.hpp | 2 - repertory/librepertory/src/db/rdb_meta_db.cpp | 134 +++++++++--------- .../librepertory/src/db/sqlite_meta_db.cpp | 8 +- 3 files changed, 71 insertions(+), 73 deletions(-) diff --git a/repertory/librepertory/include/db/rdb_meta_db.hpp b/repertory/librepertory/include/db/rdb_meta_db.hpp index 2d6479d2..1249f572 100644 --- a/repertory/librepertory/include/db/rdb_meta_db.hpp +++ b/repertory/librepertory/include/db/rdb_meta_db.hpp @@ -41,8 +41,6 @@ public: private: std::unique_ptr db_{nullptr}; rocksdb::ColumnFamilyHandle *default_family_{}; - rocksdb::ColumnFamilyHandle *directory_family_{}; - rocksdb::ColumnFamilyHandle *keys_family_{}; rocksdb::ColumnFamilyHandle *pinned_family_{}; rocksdb::ColumnFamilyHandle *size_family_{}; rocksdb::ColumnFamilyHandle *source_family_{}; diff --git a/repertory/librepertory/src/db/rdb_meta_db.cpp b/repertory/librepertory/src/db/rdb_meta_db.cpp index ed5c571c..526533e2 100644 --- a/repertory/librepertory/src/db/rdb_meta_db.cpp +++ b/repertory/librepertory/src/db/rdb_meta_db.cpp @@ -61,8 +61,6 @@ rdb_meta_db::rdb_meta_db(const app_config &cfg) { auto families = std::vector(); families.emplace_back(rocksdb::kDefaultColumnFamilyName, rocksdb::ColumnFamilyOptions()); - families.emplace_back("directory", rocksdb::ColumnFamilyOptions()); - families.emplace_back("keys", rocksdb::ColumnFamilyOptions()); families.emplace_back("pinned", rocksdb::ColumnFamilyOptions()); families.emplace_back("size", rocksdb::ColumnFamilyOptions()); families.emplace_back("source", rocksdb::ColumnFamilyOptions()); @@ -72,8 +70,6 @@ rdb_meta_db::rdb_meta_db(const app_config &cfg) { std::size_t idx{}; default_family_ = handles[idx++]; - directory_family_ = handles[idx++]; - keys_family_ = handles[idx++]; pinned_family_ = handles[idx++]; size_family_ = handles[idx++]; source_family_ = handles[idx++]; @@ -123,7 +119,7 @@ auto rdb_meta_db::get_item_meta_json(const std::string &api_path, { std::string value; - auto res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status { + auto res = perform_action(function_name, [&]() -> rocksdb::Status { return db_->Get(rocksdb::ReadOptions(), default_family_, api_path, &value); }); @@ -139,32 +135,7 @@ auto rdb_meta_db::get_item_meta_json(const std::string &api_path, { std::string value; - auto res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status { - return db_->Get(rocksdb::ReadOptions(), directory_family_, api_path, - &value); - }); - if (res != api_error::success) { - return res; - } - json_data[META_DIRECTORY] = value; - found = found || not value.empty(); - } - - { - std::string value; - auto res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status { - return db_->Get(rocksdb::ReadOptions(), keys_family_, api_path, &value); - }); - if (res != api_error::success) { - return res; - } - json_data[META_KEY] = value; - found = found || not value.empty(); - } - - { - std::string value; - auto res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status { + auto res = perform_action(function_name, [&]() -> rocksdb::Status { return db_->Get(rocksdb::ReadOptions(), pinned_family_, api_path, &value); }); @@ -177,7 +148,7 @@ auto rdb_meta_db::get_item_meta_json(const std::string &api_path, { std::string value; - auto res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status { + auto res = perform_action(function_name, [&]() -> rocksdb::Status { return db_->Get(rocksdb::ReadOptions(), size_family_, api_path, &value); }); if (res != api_error::success) { @@ -187,19 +158,6 @@ auto rdb_meta_db::get_item_meta_json(const std::string &api_path, found = found || not value.empty(); } - { - std::string value; - auto res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status { - return db_->Get(rocksdb::ReadOptions(), source_family_, api_path, - &value); - }); - if (res != api_error::success) { - return res; - } - json_data[META_SOURCE] = value; - found = found || not value.empty(); - } - return found ? api_error::success : api_error::item_not_found; } catch (const std::exception &e) { utils::error::raise_api_path_error(function_name, api_path, e, @@ -227,6 +185,20 @@ auto rdb_meta_db::get_item_meta(const std::string &api_path, auto rdb_meta_db::get_item_meta(const std::string &api_path, const std::string &key, std::string &value) const -> api_error { + REPERTORY_USES_FUNCTION_NAME(); + + if (key == META_PINNED) { + return perform_action(function_name, [&]() -> rocksdb::Status { + return db_->Get(rocksdb::ReadOptions(), pinned_family_, api_path, &value); + }); + } + + if (key == META_SIZE) { + return perform_action(function_name, [&]() -> rocksdb::Status { + return db_->Get(rocksdb::ReadOptions(), size_family_, api_path, &value); + }); + } + json json_data; auto ret = get_item_meta_json(api_path, json_data); if (ret != api_error::success) { @@ -290,12 +262,18 @@ auto rdb_meta_db::perform_action(std::string_view function_name, } void rdb_meta_db::remove_api_path(const std::string &api_path) { - [[maybe_unused]] auto res = - perform_action(__FUNCTION__, [this, &api_path]() -> rocksdb::Status { - db_->Delete(rocksdb::WriteOptions(), directory_family_, api_path); + REPERTORY_USES_FUNCTION_NAME(); + + std::string source_path; + [[maybe_unused]] auto res = get_item_meta(api_path, META_SOURCE, source_path); + + res = perform_action( + function_name, [this, &api_path, &source_path]() -> rocksdb::Status { db_->Delete(rocksdb::WriteOptions(), pinned_family_, api_path); db_->Delete(rocksdb::WriteOptions(), size_family_, api_path); - db_->Delete(rocksdb::WriteOptions(), source_family_, api_path); + if (not source_path.empty()) { + db_->Delete(rocksdb::WriteOptions(), source_family_, source_path); + } return db_->Delete(rocksdb::WriteOptions(), default_family_, api_path); }); } @@ -334,6 +312,20 @@ auto rdb_meta_db::rename_item_meta(const std::string &from_api_path, auto rdb_meta_db::set_item_meta(const std::string &api_path, const std::string &key, const std::string &value) -> api_error { + REPERTORY_USES_FUNCTION_NAME(); + + if (key == META_PINNED) { + return perform_action(function_name, [&]() -> rocksdb::Status { + return db_->Put(rocksdb::WriteOptions(), pinned_family_, api_path, value); + }); + } + + if (key == META_SIZE) { + return perform_action(function_name, [&]() -> rocksdb::Status { + return db_->Put(rocksdb::WriteOptions(), size_family_, api_path, value); + }); + } + json json_data; auto res = get_item_meta_json(api_path, json_data); if (res != api_error::success && res != api_error::item_not_found) { @@ -365,8 +357,11 @@ auto rdb_meta_db::update_item_meta(const std::string &api_path, json json_data) REPERTORY_USES_FUNCTION_NAME(); try { - auto directory = - utils::string::to_bool(json_data.at(META_DIRECTORY).get()); + std::string orig_source_path; + auto ret = get_item_meta(api_path, META_SOURCE, orig_source_path); + if (ret != api_error::success && ret != api_error::item_not_found) { + return ret; + } if (not json_data.contains(META_PINNED)) { json_data[META_PINNED] = utils::string::from_bool(false); @@ -378,30 +373,31 @@ auto rdb_meta_db::update_item_meta(const std::string &api_path, json json_data) json_data[META_SOURCE] = ""; } + auto directory = + utils::string::to_bool(json_data.at(META_DIRECTORY).get()); auto pinned = utils::string::to_bool(json_data.at(META_PINNED).get()); - auto size = directory ? std::uint64_t(0U) : utils::string::to_uint64( json_data.at(META_SIZE).get()); - auto source_path = json_data.at(META_SOURCE).get(); + if (orig_source_path != source_path && not orig_source_path.empty()) { + ret = perform_action(function_name, [&]() -> rocksdb::Status { + return db_->Delete(rocksdb::WriteOptions(), source_family_, + orig_source_path); + }); + if (ret != api_error::success && ret != api_error::item_not_found) { + return ret; + } + } - json_data.erase(META_DIRECTORY); json_data.erase(META_PINNED); json_data.erase(META_SIZE); - json_data.erase(META_SOURCE); - - return perform_action(__FUNCTION__, [&]() -> rocksdb::Status { - auto res = db_->Put(rocksdb::WriteOptions(), directory_family_, api_path, - utils::string::from_bool(directory)); - if (not res.ok()) { - return res; - } + return perform_action(function_name, [&]() -> rocksdb::Status { if (not directory) { - db_->Put(rocksdb::WriteOptions(), pinned_family_, api_path, - utils::string::from_bool(pinned)); + auto res = db_->Put(rocksdb::WriteOptions(), pinned_family_, api_path, + utils::string::from_bool(pinned)); if (not res.ok()) { return res; } @@ -413,10 +409,12 @@ auto rdb_meta_db::update_item_meta(const std::string &api_path, json json_data) } } - res = db_->Put(rocksdb::WriteOptions(), source_family_, api_path, - source_path); - if (not res.ok()) { - return res; + if (not source_path.empty()) { + auto res = db_->Put(rocksdb::WriteOptions(), source_family_, + source_path, api_path); + if (not res.ok()) { + return res; + } } return db_->Put(rocksdb::WriteOptions(), default_family_, api_path, diff --git a/repertory/librepertory/src/db/sqlite_meta_db.cpp b/repertory/librepertory/src/db/sqlite_meta_db.cpp index b4d940ac..bf37305f 100644 --- a/repertory/librepertory/src/db/sqlite_meta_db.cpp +++ b/repertory/librepertory/src/db/sqlite_meta_db.cpp @@ -307,19 +307,21 @@ auto sqlite_meta_db::update_item_meta(const std::string &api_path, REPERTORY_USES_FUNCTION_NAME(); try { - auto directory = utils::string::to_bool(meta.at(META_DIRECTORY)); - if (meta[META_PINNED].empty()) { meta[META_PINNED] = utils::string::from_bool(false); } if (meta[META_SIZE].empty()) { meta[META_SIZE] = "0"; } + if (meta[META_SOURCE].empty()) { + meta[META_SOURCE] = ""; + } + auto directory = utils::string::to_bool(meta.at(META_DIRECTORY)); auto pinned = utils::string::to_bool(meta.at(META_PINNED)); auto size = directory ? std::uint64_t(0U) : utils::string::to_uint64(meta.at(META_SIZE)); - auto source_path = meta[META_SOURCE]; + auto source_path = meta.at(META_SOURCE); meta.erase(META_DIRECTORY); meta.erase(META_PINNED);