added rocksdb meta db
This commit is contained in:
109
repertory/librepertory/include/db/rdb_meta_db.hpp
Normal file
109
repertory/librepertory/include/db/rdb_meta_db.hpp
Normal file
@@ -0,0 +1,109 @@
|
||||
/*
|
||||
Copyright <2018-2024> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
*/
|
||||
#ifndef REPERTORY_INCLUDE_DB_RDB_META_DB_HPP_
|
||||
#define REPERTORY_INCLUDE_DB_RDB_META_DB_HPP_
|
||||
|
||||
#include "db/i_meta_db.hpp"
|
||||
#include "types/repertory.hpp"
|
||||
|
||||
namespace repertory {
|
||||
class app_config;
|
||||
|
||||
class rdb_meta_db final : public i_meta_db {
|
||||
public:
|
||||
rdb_meta_db(const app_config &cfg);
|
||||
~rdb_meta_db() override;
|
||||
|
||||
rdb_meta_db(const rdb_meta_db &) = delete;
|
||||
rdb_meta_db(rdb_meta_db &&) = delete;
|
||||
auto operator=(const rdb_meta_db &) -> rdb_meta_db & = delete;
|
||||
auto operator=(rdb_meta_db &&) -> rdb_meta_db & = delete;
|
||||
|
||||
private:
|
||||
std::unique_ptr<rocksdb::DB> db_{nullptr};
|
||||
rocksdb::ColumnFamilyHandle *default_family_{};
|
||||
rocksdb::ColumnFamilyHandle *directory_family_{};
|
||||
rocksdb::ColumnFamilyHandle *keys_family_{};
|
||||
rocksdb::ColumnFamilyHandle *pinned_family_{};
|
||||
rocksdb::ColumnFamilyHandle *size_family_{};
|
||||
rocksdb::ColumnFamilyHandle *source_family_{};
|
||||
|
||||
private:
|
||||
[[nodiscard]] auto create_iterator(rocksdb::ColumnFamilyHandle *family) const
|
||||
-> std::shared_ptr<rocksdb::Iterator>;
|
||||
|
||||
[[nodiscard]] auto get_item_meta_json(const std::string &api_path,
|
||||
json &json_data) const -> api_error;
|
||||
|
||||
[[nodiscard]] static auto
|
||||
perform_action(std::string_view function_name,
|
||||
std::function<rocksdb::Status()> action) -> api_error;
|
||||
|
||||
[[nodiscard]] auto update_item_meta(const std::string &api_path,
|
||||
json json_data) -> api_error;
|
||||
|
||||
public:
|
||||
[[nodiscard]] auto get_api_path(const std::string &source_path,
|
||||
std::string &api_path) const
|
||||
-> api_error override;
|
||||
|
||||
[[nodiscard]] auto get_api_path_list() const
|
||||
-> std::vector<std::string> override;
|
||||
|
||||
[[nodiscard]] auto get_item_meta(const std::string &api_path,
|
||||
api_meta_map &meta) const
|
||||
-> api_error override;
|
||||
|
||||
[[nodiscard]] auto get_item_meta(const std::string &api_path,
|
||||
const std::string &key,
|
||||
std::string &value) const
|
||||
-> api_error override;
|
||||
|
||||
[[nodiscard]] auto get_pinned_files() const
|
||||
-> std::vector<std::string> override;
|
||||
|
||||
[[nodiscard]] auto get_total_item_count() const -> std::uint64_t override;
|
||||
|
||||
[[nodiscard]] auto get_total_size() const -> std::uint64_t override;
|
||||
|
||||
void remove_api_path(const std::string &api_path) override;
|
||||
|
||||
[[nodiscard]] auto remove_item_meta(const std::string &api_path,
|
||||
const std::string &key)
|
||||
-> api_error override;
|
||||
|
||||
[[nodiscard]] auto rename_item_meta(const std::string &from_api_path,
|
||||
const std::string &to_api_path)
|
||||
-> api_error override;
|
||||
|
||||
[[nodiscard]] auto set_item_meta(const std::string &api_path,
|
||||
const std::string &key,
|
||||
const std::string &value)
|
||||
-> api_error override;
|
||||
|
||||
[[nodiscard]] auto set_item_meta(const std::string &api_path,
|
||||
const api_meta_map &meta)
|
||||
-> api_error override;
|
||||
};
|
||||
} // namespace repertory
|
||||
|
||||
#endif // REPERTORY_INCLUDE_DB_RDB_META_DB_HPP_
|
||||
393
repertory/librepertory/src/db/rdb_meta_db.cpp
Normal file
393
repertory/librepertory/src/db/rdb_meta_db.cpp
Normal file
@@ -0,0 +1,393 @@
|
||||
/*
|
||||
Copyright <2018-2024> <scott.e.graves@protonmail.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
*/
|
||||
#include "db/rdb_meta_db.hpp"
|
||||
|
||||
#include "app_config.hpp"
|
||||
#include "types/startup_exception.hpp"
|
||||
#include "utils/error_utils.hpp"
|
||||
#include "utils/path.hpp"
|
||||
#include "utils/string.hpp"
|
||||
|
||||
namespace {
|
||||
void create_rocksdb(
|
||||
const repertory::app_config &cfg, const std::string &name,
|
||||
const std::vector<rocksdb::ColumnFamilyDescriptor> &families,
|
||||
std::vector<rocksdb::ColumnFamilyHandle *> &handles,
|
||||
std::unique_ptr<rocksdb::DB> &db) {
|
||||
rocksdb::Options options{};
|
||||
options.create_if_missing = true;
|
||||
options.create_missing_column_families = true;
|
||||
options.db_log_dir = cfg.get_log_directory();
|
||||
options.keep_log_file_num = 10;
|
||||
|
||||
rocksdb::DB *ptr{};
|
||||
auto status = rocksdb::DB::Open(
|
||||
options,
|
||||
repertory::utils::path::combine(cfg.get_data_directory(), {name}),
|
||||
families, &handles, &ptr);
|
||||
if (status.ok()) {
|
||||
db.reset(ptr);
|
||||
return;
|
||||
}
|
||||
|
||||
repertory::utils::error::raise_error(__FUNCTION__, status.ToString());
|
||||
throw repertory::startup_exception(status.ToString());
|
||||
}
|
||||
} // namespace
|
||||
|
||||
namespace repertory {
|
||||
rdb_meta_db::rdb_meta_db(const app_config &cfg) {
|
||||
const auto create_resources = [this, &cfg](const std::string &name) {
|
||||
auto families = std::vector<rocksdb::ColumnFamilyDescriptor>();
|
||||
families.emplace_back(rocksdb::kDefaultColumnFamilyName,
|
||||
rocksdb::ColumnFamilyOptions());
|
||||
families.emplace_back("directory", rocksdb::ColumnFamilyOptions());
|
||||
families.emplace_back("keys", rocksdb::ColumnFamilyOptions());
|
||||
families.emplace_back("pinned", rocksdb::ColumnFamilyOptions());
|
||||
families.emplace_back("size", rocksdb::ColumnFamilyOptions());
|
||||
families.emplace_back("source", rocksdb::ColumnFamilyOptions());
|
||||
|
||||
auto handles = std::vector<rocksdb::ColumnFamilyHandle *>();
|
||||
create_rocksdb(cfg, name, families, handles, db_);
|
||||
|
||||
std::size_t idx{};
|
||||
default_family_ = handles[idx++];
|
||||
directory_family_ = handles[idx++];
|
||||
keys_family_ = handles[idx++];
|
||||
pinned_family_ = handles[idx++];
|
||||
size_family_ = handles[idx++];
|
||||
source_family_ = handles[idx++];
|
||||
};
|
||||
|
||||
create_resources("provider_meta");
|
||||
}
|
||||
|
||||
rdb_meta_db::~rdb_meta_db() { db_.reset(); }
|
||||
|
||||
auto rdb_meta_db::create_iterator(rocksdb::ColumnFamilyHandle *family) const
|
||||
-> std::shared_ptr<rocksdb::Iterator> {
|
||||
return std::shared_ptr<rocksdb::Iterator>(
|
||||
db_->NewIterator(rocksdb::ReadOptions(), family));
|
||||
}
|
||||
|
||||
auto rdb_meta_db::get_api_path(const std::string &source_path,
|
||||
std::string &api_path) const -> api_error {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
if (source_path.empty()) {
|
||||
return api_error::item_not_found;
|
||||
}
|
||||
|
||||
return perform_action(function_name, [&]() -> rocksdb::Status {
|
||||
return db_->Get(rocksdb::ReadOptions(), source_family_, source_path,
|
||||
&api_path);
|
||||
});
|
||||
}
|
||||
|
||||
auto rdb_meta_db::get_api_path_list() const -> std::vector<std::string> {
|
||||
std::vector<std::string> ret;
|
||||
auto iter = create_iterator(default_family_);
|
||||
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
|
||||
ret.push_back(iter->key().ToString());
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
auto rdb_meta_db::get_item_meta_json(const std::string &api_path,
|
||||
json &json_data) const -> api_error {
|
||||
{
|
||||
std::string value;
|
||||
auto res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
|
||||
return db_->Get(rocksdb::ReadOptions(), default_family_, api_path,
|
||||
&value);
|
||||
});
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
|
||||
if (value.empty()) {
|
||||
return api_error::item_not_found;
|
||||
}
|
||||
|
||||
json_data = json::parse(value);
|
||||
}
|
||||
|
||||
{
|
||||
std::string value;
|
||||
auto res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
|
||||
return db_->Get(rocksdb::ReadOptions(), directory_family_, api_path,
|
||||
&value);
|
||||
});
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
json_data[META_DIRECTORY] = value;
|
||||
}
|
||||
|
||||
{
|
||||
std::string value;
|
||||
auto res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
|
||||
return db_->Get(rocksdb::ReadOptions(), keys_family_, api_path, &value);
|
||||
});
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
json_data[META_KEY] = value;
|
||||
}
|
||||
|
||||
{
|
||||
std::string value;
|
||||
auto res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
|
||||
return db_->Get(rocksdb::ReadOptions(), pinned_family_, api_path, &value);
|
||||
});
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
json_data[META_PINNED] = value;
|
||||
}
|
||||
|
||||
{
|
||||
std::string value;
|
||||
auto res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
|
||||
return db_->Get(rocksdb::ReadOptions(), size_family_, api_path, &value);
|
||||
});
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
json_data[META_SIZE] = value;
|
||||
}
|
||||
|
||||
{
|
||||
std::string value;
|
||||
auto res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
|
||||
return db_->Get(rocksdb::ReadOptions(), source_family_, api_path, &value);
|
||||
});
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
json_data[META_SOURCE] = value;
|
||||
}
|
||||
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
auto rdb_meta_db::get_item_meta(const std::string &api_path,
|
||||
api_meta_map &meta) const -> api_error {
|
||||
json json_data;
|
||||
auto ret = get_item_meta_json(api_path, json_data);
|
||||
if (ret != api_error::success) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
for (auto it = json_data.begin(); it != json_data.end(); ++it) {
|
||||
meta[it.key()] = it.value().get<std::string>();
|
||||
}
|
||||
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
auto rdb_meta_db::get_item_meta(const std::string &api_path,
|
||||
const std::string &key,
|
||||
std::string &value) const -> api_error {
|
||||
json json_data;
|
||||
auto ret = get_item_meta_json(api_path, json_data);
|
||||
if (ret != api_error::success) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
if (json_data.find(key) != json_data.end()) {
|
||||
value = json_data[key].get<std::string>();
|
||||
}
|
||||
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
auto rdb_meta_db::get_pinned_files() const -> std::vector<std::string> {
|
||||
std::vector<std::string> ret;
|
||||
auto iter = create_iterator(pinned_family_);
|
||||
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
|
||||
if (not utils::string::to_bool(iter->value().ToString())) {
|
||||
continue;
|
||||
}
|
||||
|
||||
ret.push_back(iter->key().ToString());
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
auto rdb_meta_db::get_total_item_count() const -> std::uint64_t {
|
||||
std::uint64_t ret{};
|
||||
auto iter = create_iterator(default_family_);
|
||||
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
|
||||
++ret;
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
auto rdb_meta_db::get_total_size() const -> std::uint64_t {
|
||||
std::uint64_t ret{};
|
||||
auto iter = create_iterator(size_family_);
|
||||
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
|
||||
ret += utils::string::to_uint64(iter->value().ToString());
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
auto rdb_meta_db::perform_action(std::string_view function_name,
|
||||
std::function<rocksdb::Status()> action)
|
||||
-> api_error {
|
||||
auto res = action();
|
||||
if (res.ok()) {
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
if (not res.IsNotFound()) {
|
||||
utils::error::raise_error(function_name, res.ToString());
|
||||
}
|
||||
|
||||
return res.IsNotFound() ? api_error::item_not_found : api_error::error;
|
||||
}
|
||||
|
||||
void rdb_meta_db::remove_api_path(const std::string &api_path) {
|
||||
[[maybe_unused]] auto res =
|
||||
perform_action(__FUNCTION__, [this, &api_path]() -> rocksdb::Status {
|
||||
db_->Delete(rocksdb::WriteOptions(), directory_family_, api_path);
|
||||
db_->Delete(rocksdb::WriteOptions(), pinned_family_, api_path);
|
||||
db_->Delete(rocksdb::WriteOptions(), size_family_, api_path);
|
||||
db_->Delete(rocksdb::WriteOptions(), source_family_, api_path);
|
||||
return db_->Delete(rocksdb::WriteOptions(), default_family_, api_path);
|
||||
});
|
||||
}
|
||||
|
||||
auto rdb_meta_db::remove_item_meta(const std::string &api_path,
|
||||
const std::string &key) -> api_error {
|
||||
if (key == META_DIRECTORY || key == META_PINNED || key == META_SIZE ||
|
||||
key == META_SOURCE) {
|
||||
// TODO log warning for unsupported attributes
|
||||
return api_error::success;
|
||||
}
|
||||
|
||||
json json_data;
|
||||
auto res = get_item_meta_json(api_path, json_data);
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
|
||||
json_data.erase(key);
|
||||
return update_item_meta(api_path, json_data);
|
||||
}
|
||||
|
||||
auto rdb_meta_db::rename_item_meta(const std::string &from_api_path,
|
||||
const std::string &to_api_path)
|
||||
-> api_error {
|
||||
json json_data;
|
||||
auto res = get_item_meta_json(from_api_path, json_data);
|
||||
if (res != api_error::success) {
|
||||
return res;
|
||||
}
|
||||
|
||||
remove_api_path(from_api_path);
|
||||
return update_item_meta(to_api_path, json_data);
|
||||
}
|
||||
|
||||
auto rdb_meta_db::set_item_meta(const std::string &api_path,
|
||||
const std::string &key,
|
||||
const std::string &value) -> api_error {
|
||||
json json_data;
|
||||
auto res = get_item_meta_json(api_path, json_data);
|
||||
if (res != api_error::success && res != api_error::item_not_found) {
|
||||
return res;
|
||||
}
|
||||
|
||||
json_data[key] = value;
|
||||
|
||||
return update_item_meta(api_path, json_data);
|
||||
}
|
||||
|
||||
auto rdb_meta_db::set_item_meta(const std::string &api_path,
|
||||
const api_meta_map &meta) -> api_error {
|
||||
json json_data;
|
||||
auto res = get_item_meta_json(api_path, json_data);
|
||||
if (res != api_error::success && res != api_error::item_not_found) {
|
||||
return res;
|
||||
}
|
||||
|
||||
for (const auto &data : meta) {
|
||||
json_data[data.first] = data.second;
|
||||
}
|
||||
|
||||
return update_item_meta(api_path, json_data);
|
||||
}
|
||||
|
||||
auto rdb_meta_db::update_item_meta(const std::string &api_path, json json_data)
|
||||
-> api_error {
|
||||
auto directory =
|
||||
utils::string::to_bool(json_data[META_DIRECTORY].get<std::string>());
|
||||
auto pinned =
|
||||
utils::string::to_bool(json_data[META_PINNED].get<std::string>());
|
||||
auto size =
|
||||
directory
|
||||
? std::uint64_t(0U)
|
||||
: utils::string::to_uint64(json_data[META_SIZE].get<std::string>());
|
||||
auto source_path = json_data[META_SOURCE].get<std::string>();
|
||||
|
||||
json_data.erase(META_DIRECTORY);
|
||||
json_data.erase(META_PINNED);
|
||||
json_data.erase(META_SIZE);
|
||||
json_data.erase(META_SOURCE);
|
||||
|
||||
return perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
|
||||
auto res = db_->Put(rocksdb::WriteOptions(), directory_family_, api_path,
|
||||
utils::string::from_bool(directory));
|
||||
if (not res.ok()) {
|
||||
return res;
|
||||
}
|
||||
|
||||
if (not directory) {
|
||||
db_->Put(rocksdb::WriteOptions(), pinned_family_, api_path,
|
||||
utils::string::from_bool(pinned));
|
||||
if (not res.ok()) {
|
||||
return res;
|
||||
}
|
||||
|
||||
res = db_->Put(rocksdb::WriteOptions(), size_family_, api_path,
|
||||
std::to_string(size));
|
||||
if (not res.ok()) {
|
||||
return res;
|
||||
}
|
||||
}
|
||||
|
||||
res = db_->Put(rocksdb::WriteOptions(), source_family_, api_path,
|
||||
source_path);
|
||||
if (not res.ok()) {
|
||||
return res;
|
||||
}
|
||||
|
||||
return db_->Put(rocksdb::WriteOptions(), default_family_, api_path,
|
||||
json_data.dump());
|
||||
});
|
||||
}
|
||||
} // namespace repertory
|
||||
@@ -32,8 +32,6 @@
|
||||
|
||||
namespace repertory {
|
||||
sqlite_meta_db::sqlite_meta_db(const app_config &cfg) {
|
||||
REPERTORY_USES_FUNCTION_NAME();
|
||||
|
||||
const std::map<std::string, std::string> sql_create_tables{
|
||||
{
|
||||
{"meta"},
|
||||
|
||||
Reference in New Issue
Block a user