From be96d7928193e77cdfb385924551036d5de9b5cf Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Wed, 4 Dec 2024 10:11:09 -0600 Subject: [PATCH] added rocksdb meta db --- .../librepertory/include/db/rdb_meta_db.hpp | 109 +++++ repertory/librepertory/src/db/rdb_meta_db.cpp | 393 ++++++++++++++++++ .../librepertory/src/db/sqlite_meta_db.cpp | 2 - 3 files changed, 502 insertions(+), 2 deletions(-) create mode 100644 repertory/librepertory/include/db/rdb_meta_db.hpp create mode 100644 repertory/librepertory/src/db/rdb_meta_db.cpp diff --git a/repertory/librepertory/include/db/rdb_meta_db.hpp b/repertory/librepertory/include/db/rdb_meta_db.hpp new file mode 100644 index 00000000..2d6479d2 --- /dev/null +++ b/repertory/librepertory/include/db/rdb_meta_db.hpp @@ -0,0 +1,109 @@ +/* + Copyright <2018-2024> + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ +#ifndef REPERTORY_INCLUDE_DB_RDB_META_DB_HPP_ +#define REPERTORY_INCLUDE_DB_RDB_META_DB_HPP_ + +#include "db/i_meta_db.hpp" +#include "types/repertory.hpp" + +namespace repertory { +class app_config; + +class rdb_meta_db final : public i_meta_db { +public: + rdb_meta_db(const app_config &cfg); + ~rdb_meta_db() override; + + rdb_meta_db(const rdb_meta_db &) = delete; + rdb_meta_db(rdb_meta_db &&) = delete; + auto operator=(const rdb_meta_db &) -> rdb_meta_db & = delete; + auto operator=(rdb_meta_db &&) -> rdb_meta_db & = delete; + +private: + std::unique_ptr db_{nullptr}; + rocksdb::ColumnFamilyHandle *default_family_{}; + rocksdb::ColumnFamilyHandle *directory_family_{}; + rocksdb::ColumnFamilyHandle *keys_family_{}; + rocksdb::ColumnFamilyHandle *pinned_family_{}; + rocksdb::ColumnFamilyHandle *size_family_{}; + rocksdb::ColumnFamilyHandle *source_family_{}; + +private: + [[nodiscard]] auto create_iterator(rocksdb::ColumnFamilyHandle *family) const + -> std::shared_ptr; + + [[nodiscard]] auto get_item_meta_json(const std::string &api_path, + json &json_data) const -> api_error; + + [[nodiscard]] static auto + perform_action(std::string_view function_name, + std::function action) -> api_error; + + [[nodiscard]] auto update_item_meta(const std::string &api_path, + json json_data) -> api_error; + +public: + [[nodiscard]] auto get_api_path(const std::string &source_path, + std::string &api_path) const + -> api_error override; + + [[nodiscard]] auto get_api_path_list() const + -> std::vector override; + + [[nodiscard]] auto get_item_meta(const std::string &api_path, + api_meta_map &meta) const + -> api_error override; + + [[nodiscard]] auto get_item_meta(const std::string &api_path, + const std::string &key, + std::string &value) const + -> api_error override; + + [[nodiscard]] auto get_pinned_files() const + -> std::vector override; + + [[nodiscard]] auto get_total_item_count() const -> std::uint64_t override; + + [[nodiscard]] auto get_total_size() const -> std::uint64_t override; + + void remove_api_path(const std::string &api_path) override; + + [[nodiscard]] auto remove_item_meta(const std::string &api_path, + const std::string &key) + -> api_error override; + + [[nodiscard]] auto rename_item_meta(const std::string &from_api_path, + const std::string &to_api_path) + -> api_error override; + + [[nodiscard]] auto set_item_meta(const std::string &api_path, + const std::string &key, + const std::string &value) + -> api_error override; + + [[nodiscard]] auto set_item_meta(const std::string &api_path, + const api_meta_map &meta) + -> api_error override; +}; +} // namespace repertory + +#endif // REPERTORY_INCLUDE_DB_RDB_META_DB_HPP_ diff --git a/repertory/librepertory/src/db/rdb_meta_db.cpp b/repertory/librepertory/src/db/rdb_meta_db.cpp new file mode 100644 index 00000000..6ae00d15 --- /dev/null +++ b/repertory/librepertory/src/db/rdb_meta_db.cpp @@ -0,0 +1,393 @@ +/* + Copyright <2018-2024> + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ +#include "db/rdb_meta_db.hpp" + +#include "app_config.hpp" +#include "types/startup_exception.hpp" +#include "utils/error_utils.hpp" +#include "utils/path.hpp" +#include "utils/string.hpp" + +namespace { +void create_rocksdb( + const repertory::app_config &cfg, const std::string &name, + const std::vector &families, + std::vector &handles, + std::unique_ptr &db) { + rocksdb::Options options{}; + options.create_if_missing = true; + options.create_missing_column_families = true; + options.db_log_dir = cfg.get_log_directory(); + options.keep_log_file_num = 10; + + rocksdb::DB *ptr{}; + auto status = rocksdb::DB::Open( + options, + repertory::utils::path::combine(cfg.get_data_directory(), {name}), + families, &handles, &ptr); + if (status.ok()) { + db.reset(ptr); + return; + } + + repertory::utils::error::raise_error(__FUNCTION__, status.ToString()); + throw repertory::startup_exception(status.ToString()); +} +} // namespace + +namespace repertory { +rdb_meta_db::rdb_meta_db(const app_config &cfg) { + const auto create_resources = [this, &cfg](const std::string &name) { + auto families = std::vector(); + families.emplace_back(rocksdb::kDefaultColumnFamilyName, + rocksdb::ColumnFamilyOptions()); + families.emplace_back("directory", rocksdb::ColumnFamilyOptions()); + families.emplace_back("keys", rocksdb::ColumnFamilyOptions()); + families.emplace_back("pinned", rocksdb::ColumnFamilyOptions()); + families.emplace_back("size", rocksdb::ColumnFamilyOptions()); + families.emplace_back("source", rocksdb::ColumnFamilyOptions()); + + auto handles = std::vector(); + create_rocksdb(cfg, name, families, handles, db_); + + std::size_t idx{}; + default_family_ = handles[idx++]; + directory_family_ = handles[idx++]; + keys_family_ = handles[idx++]; + pinned_family_ = handles[idx++]; + size_family_ = handles[idx++]; + source_family_ = handles[idx++]; + }; + + create_resources("provider_meta"); +} + +rdb_meta_db::~rdb_meta_db() { db_.reset(); } + +auto rdb_meta_db::create_iterator(rocksdb::ColumnFamilyHandle *family) const + -> std::shared_ptr { + return std::shared_ptr( + db_->NewIterator(rocksdb::ReadOptions(), family)); +} + +auto rdb_meta_db::get_api_path(const std::string &source_path, + std::string &api_path) const -> api_error { + REPERTORY_USES_FUNCTION_NAME(); + + if (source_path.empty()) { + return api_error::item_not_found; + } + + return perform_action(function_name, [&]() -> rocksdb::Status { + return db_->Get(rocksdb::ReadOptions(), source_family_, source_path, + &api_path); + }); +} + +auto rdb_meta_db::get_api_path_list() const -> std::vector { + std::vector ret; + auto iter = create_iterator(default_family_); + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ret.push_back(iter->key().ToString()); + } + + return ret; +} + +auto rdb_meta_db::get_item_meta_json(const std::string &api_path, + json &json_data) const -> api_error { + { + std::string value; + auto res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status { + return db_->Get(rocksdb::ReadOptions(), default_family_, api_path, + &value); + }); + if (res != api_error::success) { + return res; + } + + if (value.empty()) { + return api_error::item_not_found; + } + + json_data = json::parse(value); + } + + { + std::string value; + auto res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status { + return db_->Get(rocksdb::ReadOptions(), directory_family_, api_path, + &value); + }); + if (res != api_error::success) { + return res; + } + json_data[META_DIRECTORY] = value; + } + + { + std::string value; + auto res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status { + return db_->Get(rocksdb::ReadOptions(), keys_family_, api_path, &value); + }); + if (res != api_error::success) { + return res; + } + json_data[META_KEY] = value; + } + + { + std::string value; + auto res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status { + return db_->Get(rocksdb::ReadOptions(), pinned_family_, api_path, &value); + }); + if (res != api_error::success) { + return res; + } + json_data[META_PINNED] = value; + } + + { + std::string value; + auto res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status { + return db_->Get(rocksdb::ReadOptions(), size_family_, api_path, &value); + }); + if (res != api_error::success) { + return res; + } + json_data[META_SIZE] = value; + } + + { + std::string value; + auto res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status { + return db_->Get(rocksdb::ReadOptions(), source_family_, api_path, &value); + }); + if (res != api_error::success) { + return res; + } + json_data[META_SOURCE] = value; + } + + return api_error::success; +} + +auto rdb_meta_db::get_item_meta(const std::string &api_path, + api_meta_map &meta) const -> api_error { + json json_data; + auto ret = get_item_meta_json(api_path, json_data); + if (ret != api_error::success) { + return ret; + } + + for (auto it = json_data.begin(); it != json_data.end(); ++it) { + meta[it.key()] = it.value().get(); + } + + return api_error::success; +} + +auto rdb_meta_db::get_item_meta(const std::string &api_path, + const std::string &key, + std::string &value) const -> api_error { + json json_data; + auto ret = get_item_meta_json(api_path, json_data); + if (ret != api_error::success) { + return ret; + } + + if (json_data.find(key) != json_data.end()) { + value = json_data[key].get(); + } + + return api_error::success; +} + +auto rdb_meta_db::get_pinned_files() const -> std::vector { + std::vector ret; + auto iter = create_iterator(pinned_family_); + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + if (not utils::string::to_bool(iter->value().ToString())) { + continue; + } + + ret.push_back(iter->key().ToString()); + } + + return ret; +} + +auto rdb_meta_db::get_total_item_count() const -> std::uint64_t { + std::uint64_t ret{}; + auto iter = create_iterator(default_family_); + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ++ret; + } + + return ret; +} + +auto rdb_meta_db::get_total_size() const -> std::uint64_t { + std::uint64_t ret{}; + auto iter = create_iterator(size_family_); + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ret += utils::string::to_uint64(iter->value().ToString()); + } + + return ret; +} + +auto rdb_meta_db::perform_action(std::string_view function_name, + std::function action) + -> api_error { + auto res = action(); + if (res.ok()) { + return api_error::success; + } + + if (not res.IsNotFound()) { + utils::error::raise_error(function_name, res.ToString()); + } + + return res.IsNotFound() ? api_error::item_not_found : api_error::error; +} + +void rdb_meta_db::remove_api_path(const std::string &api_path) { + [[maybe_unused]] auto res = + perform_action(__FUNCTION__, [this, &api_path]() -> rocksdb::Status { + db_->Delete(rocksdb::WriteOptions(), directory_family_, api_path); + db_->Delete(rocksdb::WriteOptions(), pinned_family_, api_path); + db_->Delete(rocksdb::WriteOptions(), size_family_, api_path); + db_->Delete(rocksdb::WriteOptions(), source_family_, api_path); + return db_->Delete(rocksdb::WriteOptions(), default_family_, api_path); + }); +} + +auto rdb_meta_db::remove_item_meta(const std::string &api_path, + const std::string &key) -> api_error { + if (key == META_DIRECTORY || key == META_PINNED || key == META_SIZE || + key == META_SOURCE) { + // TODO log warning for unsupported attributes + return api_error::success; + } + + json json_data; + auto res = get_item_meta_json(api_path, json_data); + if (res != api_error::success) { + return res; + } + + json_data.erase(key); + return update_item_meta(api_path, json_data); +} + +auto rdb_meta_db::rename_item_meta(const std::string &from_api_path, + const std::string &to_api_path) + -> api_error { + json json_data; + auto res = get_item_meta_json(from_api_path, json_data); + if (res != api_error::success) { + return res; + } + + remove_api_path(from_api_path); + return update_item_meta(to_api_path, json_data); +} + +auto rdb_meta_db::set_item_meta(const std::string &api_path, + const std::string &key, + const std::string &value) -> api_error { + json json_data; + auto res = get_item_meta_json(api_path, json_data); + if (res != api_error::success && res != api_error::item_not_found) { + return res; + } + + json_data[key] = value; + + return update_item_meta(api_path, json_data); +} + +auto rdb_meta_db::set_item_meta(const std::string &api_path, + const api_meta_map &meta) -> api_error { + json json_data; + auto res = get_item_meta_json(api_path, json_data); + if (res != api_error::success && res != api_error::item_not_found) { + return res; + } + + for (const auto &data : meta) { + json_data[data.first] = data.second; + } + + return update_item_meta(api_path, json_data); +} + +auto rdb_meta_db::update_item_meta(const std::string &api_path, json json_data) + -> api_error { + auto directory = + utils::string::to_bool(json_data[META_DIRECTORY].get()); + auto pinned = + utils::string::to_bool(json_data[META_PINNED].get()); + auto size = + directory + ? std::uint64_t(0U) + : utils::string::to_uint64(json_data[META_SIZE].get()); + auto source_path = json_data[META_SOURCE].get(); + + json_data.erase(META_DIRECTORY); + json_data.erase(META_PINNED); + json_data.erase(META_SIZE); + json_data.erase(META_SOURCE); + + return perform_action(__FUNCTION__, [&]() -> rocksdb::Status { + auto res = db_->Put(rocksdb::WriteOptions(), directory_family_, api_path, + utils::string::from_bool(directory)); + if (not res.ok()) { + return res; + } + + if (not directory) { + db_->Put(rocksdb::WriteOptions(), pinned_family_, api_path, + utils::string::from_bool(pinned)); + if (not res.ok()) { + return res; + } + + res = db_->Put(rocksdb::WriteOptions(), size_family_, api_path, + std::to_string(size)); + if (not res.ok()) { + return res; + } + } + + res = db_->Put(rocksdb::WriteOptions(), source_family_, api_path, + source_path); + if (not res.ok()) { + return res; + } + + return db_->Put(rocksdb::WriteOptions(), default_family_, api_path, + json_data.dump()); + }); +} +} // namespace repertory diff --git a/repertory/librepertory/src/db/sqlite_meta_db.cpp b/repertory/librepertory/src/db/sqlite_meta_db.cpp index 8adeae60..c7975875 100644 --- a/repertory/librepertory/src/db/sqlite_meta_db.cpp +++ b/repertory/librepertory/src/db/sqlite_meta_db.cpp @@ -32,8 +32,6 @@ namespace repertory { sqlite_meta_db::sqlite_meta_db(const app_config &cfg) { - REPERTORY_USES_FUNCTION_NAME(); - const std::map sql_create_tables{ { {"meta"},