added rocksdb meta db

repertory/librepertory/include/db/rdb_meta_db.hpp

@@ -0,0 +1,109 @@
+/*
+  Copyright <2018-2024> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all
+  copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  SOFTWARE.
+*/
+#ifndef REPERTORY_INCLUDE_DB_RDB_META_DB_HPP_
+#define REPERTORY_INCLUDE_DB_RDB_META_DB_HPP_
+
+#include "db/i_meta_db.hpp"
+#include "types/repertory.hpp"
+
+namespace repertory {
+class app_config;
+
+class rdb_meta_db final : public i_meta_db {
+public:
+  rdb_meta_db(const app_config &cfg);
+  ~rdb_meta_db() override;
+
+  rdb_meta_db(const rdb_meta_db &) = delete;
+  rdb_meta_db(rdb_meta_db &&) = delete;
+  auto operator=(const rdb_meta_db &) -> rdb_meta_db & = delete;
+  auto operator=(rdb_meta_db &&) -> rdb_meta_db & = delete;
+
+private:
+  std::unique_ptr<rocksdb::DB> db_{nullptr};
+  rocksdb::ColumnFamilyHandle *default_family_{};
+  rocksdb::ColumnFamilyHandle *directory_family_{};
+  rocksdb::ColumnFamilyHandle *keys_family_{};
+  rocksdb::ColumnFamilyHandle *pinned_family_{};
+  rocksdb::ColumnFamilyHandle *size_family_{};
+  rocksdb::ColumnFamilyHandle *source_family_{};
+
+private:
+  [[nodiscard]] auto create_iterator(rocksdb::ColumnFamilyHandle *family) const
+      -> std::shared_ptr<rocksdb::Iterator>;
+
+  [[nodiscard]] auto get_item_meta_json(const std::string &api_path,
+                                        json &json_data) const -> api_error;
+
+  [[nodiscard]] static auto
+  perform_action(std::string_view function_name,
+                 std::function<rocksdb::Status()> action) -> api_error;
+
+  [[nodiscard]] auto update_item_meta(const std::string &api_path,
+                                      json json_data) -> api_error;
+
+public:
+  [[nodiscard]] auto get_api_path(const std::string &source_path,
+                                  std::string &api_path) const
+      -> api_error override;
+
+  [[nodiscard]] auto get_api_path_list() const
+      -> std::vector<std::string> override;
+
+  [[nodiscard]] auto get_item_meta(const std::string &api_path,
+                                   api_meta_map &meta) const
+      -> api_error override;
+
+  [[nodiscard]] auto get_item_meta(const std::string &api_path,
+                                   const std::string &key,
+                                   std::string &value) const
+      -> api_error override;
+
+  [[nodiscard]] auto get_pinned_files() const
+      -> std::vector<std::string> override;
+
+  [[nodiscard]] auto get_total_item_count() const -> std::uint64_t override;
+
+  [[nodiscard]] auto get_total_size() const -> std::uint64_t override;
+
+  void remove_api_path(const std::string &api_path) override;
+
+  [[nodiscard]] auto remove_item_meta(const std::string &api_path,
+                                      const std::string &key)
+      -> api_error override;
+
+  [[nodiscard]] auto rename_item_meta(const std::string &from_api_path,
+                                      const std::string &to_api_path)
+      -> api_error override;
+
+  [[nodiscard]] auto set_item_meta(const std::string &api_path,
+                                   const std::string &key,
+                                   const std::string &value)
+      -> api_error override;
+
+  [[nodiscard]] auto set_item_meta(const std::string &api_path,
+                                   const api_meta_map &meta)
+      -> api_error override;
+};
+} // namespace repertory
+
+#endif // REPERTORY_INCLUDE_DB_RDB_META_DB_HPP_

repertory/librepertory/src/db/rdb_meta_db.cpp

@@ -0,0 +1,393 @@
+/*
+  Copyright <2018-2024> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all
+  copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  SOFTWARE.
+*/
+#include "db/rdb_meta_db.hpp"
+
+#include "app_config.hpp"
+#include "types/startup_exception.hpp"
+#include "utils/error_utils.hpp"
+#include "utils/path.hpp"
+#include "utils/string.hpp"
+
+namespace {
+void create_rocksdb(
+    const repertory::app_config &cfg, const std::string &name,
+    const std::vector<rocksdb::ColumnFamilyDescriptor> &families,
+    std::vector<rocksdb::ColumnFamilyHandle *> &handles,
+    std::unique_ptr<rocksdb::DB> &db) {
+  rocksdb::Options options{};
+  options.create_if_missing = true;
+  options.create_missing_column_families = true;
+  options.db_log_dir = cfg.get_log_directory();
+  options.keep_log_file_num = 10;
+
+  rocksdb::DB *ptr{};
+  auto status = rocksdb::DB::Open(
+      options,
+      repertory::utils::path::combine(cfg.get_data_directory(), {name}),
+      families, &handles, &ptr);
+  if (status.ok()) {
+    db.reset(ptr);
+    return;
+  }
+
+  repertory::utils::error::raise_error(__FUNCTION__, status.ToString());
+  throw repertory::startup_exception(status.ToString());
+}
+} // namespace
+
+namespace repertory {
+rdb_meta_db::rdb_meta_db(const app_config &cfg) {
+  const auto create_resources = [this, &cfg](const std::string &name) {
+    auto families = std::vector<rocksdb::ColumnFamilyDescriptor>();
+    families.emplace_back(rocksdb::kDefaultColumnFamilyName,
+                          rocksdb::ColumnFamilyOptions());
+    families.emplace_back("directory", rocksdb::ColumnFamilyOptions());
+    families.emplace_back("keys", rocksdb::ColumnFamilyOptions());
+    families.emplace_back("pinned", rocksdb::ColumnFamilyOptions());
+    families.emplace_back("size", rocksdb::ColumnFamilyOptions());
+    families.emplace_back("source", rocksdb::ColumnFamilyOptions());
+
+    auto handles = std::vector<rocksdb::ColumnFamilyHandle *>();
+    create_rocksdb(cfg, name, families, handles, db_);
+
+    std::size_t idx{};
+    default_family_ = handles[idx++];
+    directory_family_ = handles[idx++];
+    keys_family_ = handles[idx++];
+    pinned_family_ = handles[idx++];
+    size_family_ = handles[idx++];
+    source_family_ = handles[idx++];
+  };
+
+  create_resources("provider_meta");
+}
+
+rdb_meta_db::~rdb_meta_db() { db_.reset(); }
+
+auto rdb_meta_db::create_iterator(rocksdb::ColumnFamilyHandle *family) const
+    -> std::shared_ptr<rocksdb::Iterator> {
+  return std::shared_ptr<rocksdb::Iterator>(
+      db_->NewIterator(rocksdb::ReadOptions(), family));
+}
+
+auto rdb_meta_db::get_api_path(const std::string &source_path,
+                               std::string &api_path) const -> api_error {
+  REPERTORY_USES_FUNCTION_NAME();
+
+  if (source_path.empty()) {
+    return api_error::item_not_found;
+  }
+
+  return perform_action(function_name, [&]() -> rocksdb::Status {
+    return db_->Get(rocksdb::ReadOptions(), source_family_, source_path,
+                    &api_path);
+  });
+}
+
+auto rdb_meta_db::get_api_path_list() const -> std::vector<std::string> {
+  std::vector<std::string> ret;
+  auto iter = create_iterator(default_family_);
+  for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
+    ret.push_back(iter->key().ToString());
+  }
+
+  return ret;
+}
+
+auto rdb_meta_db::get_item_meta_json(const std::string &api_path,
+                                     json &json_data) const -> api_error {
+  {
+    std::string value;
+    auto res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
+      return db_->Get(rocksdb::ReadOptions(), default_family_, api_path,
+                      &value);
+    });
+    if (res != api_error::success) {
+      return res;
+    }
+
+    if (value.empty()) {
+      return api_error::item_not_found;
+    }
+
+    json_data = json::parse(value);
+  }
+
+  {
+    std::string value;
+    auto res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
+      return db_->Get(rocksdb::ReadOptions(), directory_family_, api_path,
+                      &value);
+    });
+    if (res != api_error::success) {
+      return res;
+    }
+    json_data[META_DIRECTORY] = value;
+  }
+
+  {
+    std::string value;
+    auto res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
+      return db_->Get(rocksdb::ReadOptions(), keys_family_, api_path, &value);
+    });
+    if (res != api_error::success) {
+      return res;
+    }
+    json_data[META_KEY] = value;
+  }
+
+  {
+    std::string value;
+    auto res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
+      return db_->Get(rocksdb::ReadOptions(), pinned_family_, api_path, &value);
+    });
+    if (res != api_error::success) {
+      return res;
+    }
+    json_data[META_PINNED] = value;
+  }
+
+  {
+    std::string value;
+    auto res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
+      return db_->Get(rocksdb::ReadOptions(), size_family_, api_path, &value);
+    });
+    if (res != api_error::success) {
+      return res;
+    }
+    json_data[META_SIZE] = value;
+  }
+
+  {
+    std::string value;
+    auto res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
+      return db_->Get(rocksdb::ReadOptions(), source_family_, api_path, &value);
+    });
+    if (res != api_error::success) {
+      return res;
+    }
+    json_data[META_SOURCE] = value;
+  }
+
+  return api_error::success;
+}
+
+auto rdb_meta_db::get_item_meta(const std::string &api_path,
+                                api_meta_map &meta) const -> api_error {
+  json json_data;
+  auto ret = get_item_meta_json(api_path, json_data);
+  if (ret != api_error::success) {
+    return ret;
+  }
+
+  for (auto it = json_data.begin(); it != json_data.end(); ++it) {
+    meta[it.key()] = it.value().get<std::string>();
+  }
+
+  return api_error::success;
+}
+
+auto rdb_meta_db::get_item_meta(const std::string &api_path,
+                                const std::string &key,
+                                std::string &value) const -> api_error {
+  json json_data;
+  auto ret = get_item_meta_json(api_path, json_data);
+  if (ret != api_error::success) {
+    return ret;
+  }
+
+  if (json_data.find(key) != json_data.end()) {
+    value = json_data[key].get<std::string>();
+  }
+
+  return api_error::success;
+}
+
+auto rdb_meta_db::get_pinned_files() const -> std::vector<std::string> {
+  std::vector<std::string> ret;
+  auto iter = create_iterator(pinned_family_);
+  for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
+    if (not utils::string::to_bool(iter->value().ToString())) {
+      continue;
+    }
+
+    ret.push_back(iter->key().ToString());
+  }
+
+  return ret;
+}
+
+auto rdb_meta_db::get_total_item_count() const -> std::uint64_t {
+  std::uint64_t ret{};
+  auto iter = create_iterator(default_family_);
+  for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
+    ++ret;
+  }
+
+  return ret;
+}
+
+auto rdb_meta_db::get_total_size() const -> std::uint64_t {
+  std::uint64_t ret{};
+  auto iter = create_iterator(size_family_);
+  for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
+    ret += utils::string::to_uint64(iter->value().ToString());
+  }
+
+  return ret;
+}
+
+auto rdb_meta_db::perform_action(std::string_view function_name,
+                                 std::function<rocksdb::Status()> action)
+    -> api_error {
+  auto res = action();
+  if (res.ok()) {
+    return api_error::success;
+  }
+
+  if (not res.IsNotFound()) {
+    utils::error::raise_error(function_name, res.ToString());
+  }
+
+  return res.IsNotFound() ? api_error::item_not_found : api_error::error;
+}
+
+void rdb_meta_db::remove_api_path(const std::string &api_path) {
+  [[maybe_unused]] auto res =
+      perform_action(__FUNCTION__, [this, &api_path]() -> rocksdb::Status {
+        db_->Delete(rocksdb::WriteOptions(), directory_family_, api_path);
+        db_->Delete(rocksdb::WriteOptions(), pinned_family_, api_path);
+        db_->Delete(rocksdb::WriteOptions(), size_family_, api_path);
+        db_->Delete(rocksdb::WriteOptions(), source_family_, api_path);
+        return db_->Delete(rocksdb::WriteOptions(), default_family_, api_path);
+      });
+}
+
+auto rdb_meta_db::remove_item_meta(const std::string &api_path,
+                                   const std::string &key) -> api_error {
+  if (key == META_DIRECTORY || key == META_PINNED || key == META_SIZE ||
+      key == META_SOURCE) {
+    // TODO log warning for unsupported attributes
+    return api_error::success;
+  }
+
+  json json_data;
+  auto res = get_item_meta_json(api_path, json_data);
+  if (res != api_error::success) {
+    return res;
+  }
+
+  json_data.erase(key);
+  return update_item_meta(api_path, json_data);
+}
+
+auto rdb_meta_db::rename_item_meta(const std::string &from_api_path,
+                                   const std::string &to_api_path)
+    -> api_error {
+  json json_data;
+  auto res = get_item_meta_json(from_api_path, json_data);
+  if (res != api_error::success) {
+    return res;
+  }
+
+  remove_api_path(from_api_path);
+  return update_item_meta(to_api_path, json_data);
+}
+
+auto rdb_meta_db::set_item_meta(const std::string &api_path,
+                                const std::string &key,
+                                const std::string &value) -> api_error {
+  json json_data;
+  auto res = get_item_meta_json(api_path, json_data);
+  if (res != api_error::success && res != api_error::item_not_found) {
+    return res;
+  }
+
+  json_data[key] = value;
+
+  return update_item_meta(api_path, json_data);
+}
+
+auto rdb_meta_db::set_item_meta(const std::string &api_path,
+                                const api_meta_map &meta) -> api_error {
+  json json_data;
+  auto res = get_item_meta_json(api_path, json_data);
+  if (res != api_error::success && res != api_error::item_not_found) {
+    return res;
+  }
+
+  for (const auto &data : meta) {
+    json_data[data.first] = data.second;
+  }
+
+  return update_item_meta(api_path, json_data);
+}
+
+auto rdb_meta_db::update_item_meta(const std::string &api_path, json json_data)
+    -> api_error {
+  auto directory =
+      utils::string::to_bool(json_data[META_DIRECTORY].get<std::string>());
+  auto pinned =
+      utils::string::to_bool(json_data[META_PINNED].get<std::string>());
+  auto size =
+      directory
+          ? std::uint64_t(0U)
+          : utils::string::to_uint64(json_data[META_SIZE].get<std::string>());
+  auto source_path = json_data[META_SOURCE].get<std::string>();
+
+  json_data.erase(META_DIRECTORY);
+  json_data.erase(META_PINNED);
+  json_data.erase(META_SIZE);
+  json_data.erase(META_SOURCE);
+
+  return perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
+    auto res = db_->Put(rocksdb::WriteOptions(), directory_family_, api_path,
+                        utils::string::from_bool(directory));
+    if (not res.ok()) {
+      return res;
+    }
+
+    if (not directory) {
+      db_->Put(rocksdb::WriteOptions(), pinned_family_, api_path,
+               utils::string::from_bool(pinned));
+      if (not res.ok()) {
+        return res;
+      }
+
+      res = db_->Put(rocksdb::WriteOptions(), size_family_, api_path,
+                     std::to_string(size));
+      if (not res.ok()) {
+        return res;
+      }
+    }
+
+    res = db_->Put(rocksdb::WriteOptions(), source_family_, api_path,
+                   source_path);
+    if (not res.ok()) {
+      return res;
+    }
+
+    return db_->Put(rocksdb::WriteOptions(), default_family_, api_path,
+                    json_data.dump());
+  });
+}
+} // namespace repertory

repertory/librepertory/src/db/sqlite_meta_db.cpp

@@ -32,8 +32,6 @@
 
 namespace repertory {
 sqlite_meta_db::sqlite_meta_db(const app_config &cfg) {
-  REPERTORY_USES_FUNCTION_NAME();
-
   const std::map<std::string, std::string> sql_create_tables{
       {
           {"meta"},