updates

CMakeLists.txt

@@ -148,6 +148,7 @@ endif()
       -DPROJECT_ENABLE_LIBSODIUM=${PROJECT_ENABLE_LIBSODIUM}
       -DPROJECT_ENABLE_OPENSSL=${PROJECT_ENABLE_OPENSSL}
       -DPROJECT_ENABLE_PUGIXML=${PROJECT_ENABLE_PUGIXML}
+      -DPROJECT_ENABLE_ROCKSDB=${PROJECT_ENABLE_ROCKSDB}
       -DPROJECT_ENABLE_SPDLOG=${PROJECT_ENABLE_SPDLOG}
       -DPROJECT_ENABLE_SQLITE=${PROJECT_ENABLE_SQLITE}
       -DPROJECT_ENABLE_STDUUID=${PROJECT_ENABLE_STDUUID}

cmake/hashes.cmake

@@ -13,6 +13,7 @@ set(MINGW_HASH 3f66bce069ee8bed7439a1a13da7cb91a5e67ea6170f21317ac7f5794625ee10)
 set(OPENSSL_HASH e15dda82fe2fe8139dc2ac21a36d4ca01d5313c75f99f46c4e8a27709b7294bf)
 set(PKG_CONFIG_HASH 6fc69c01688c9458a57eb9a1664c9aba372ccda420a02bf4429fe610e7e7d591)
 set(PUGIXML_HASH 2f10e276870c64b1db6809050a75e11a897a8d7456c4be5c6b2e35a11168a015)
+set(ROCKSDB_HASH 9b810c81731835fda0d4bbdb51d3199d901fa4395733ab63752d297da84c5a47)
 set(SPDLOG_HASH 9962648c9b4f1a7bbc76fd8d9172555bad1871fdb14ff4f842ef87949682caa5)
 set(SQLITE_HASH 77823cb110929c2bcb0f5d48e4833b5c59a8a6e40cdea3936b99e199dbbe5784)
 set(STDUUID_HASH b1176597e789531c38481acbbed2a6894ad419aab0979c10410d59eb0ebf40d3)

cmake/libraries.cmake

@@ -17,6 +17,7 @@ include(cmake/libraries/fuse.cmake)
 include(cmake/libraries/json.cmake)
 include(cmake/libraries/libsodium.cmake)
 include(cmake/libraries/pugixml.cmake)
+include(cmake/libraries/rocksdb.cmake)
 include(cmake/libraries/spdlog.cmake)
 include(cmake/libraries/sqlite.cmake)
 include(cmake/libraries/stduuid.cmake)

cmake/libraries/rocksdb.cmake

@@ -0,0 +1,36 @@
+if(PROJECT_ENABLE_ROCKSDB)
+  if(PROJECT_BUILD)
+    add_definitions(-DPROJECT_ENABLE_ROCKSDB)
+
+    find_library(ROCKSDB_LIBRARY NAMES librocksdb.a REQUIRED)
+
+    link_libraries(${ROCKSDB_LIBRARY})
+  elseif(NOT PROJECT_IS_MINGW OR CMAKE_HOST_WIN32)
+    ExternalProject_Add(rocksdb_project
+      PREFIX external
+      URL ${PROJECT_3RD_PARTY_DIR}/rocksdb-${ROCKSDB_VERSION}.tar.gz
+      URL_HASH SHA256=${ROCKSDB_HASH}
+      LIST_SEPARATOR |
+      CMAKE_ARGS ${PROJECT_EXTERNAL_CMAKE_FLAGS}
+        -DBUILD_SHARED_LIBS=${PROJECT_BUILD_SHARED_LIBS}
+        -DBUILD_STATIC_LIBS=ON
+        -DFAIL_ON_WARNINGS=OFF
+        -DPORTABLE=1
+        -DROCKSDB_BUILD_SHARED=${PROJECT_BUILD_SHARED_LIBS}
+        -DROCKSDB_INSTALL_ON_WINDOWS=ON
+        -DWITH_BENCHMARK=OFF
+        -DWITH_BENCHMARK_TOOLS=OFF
+        -DWITH_CORE_TOOLS=OFF
+        -DWITH_EXAMPLES=OFF
+        -DWITH_GFLAGS=OFF
+        -DWITH_IOSTATS_CONTEXT=OFF
+        -DWITH_PERF_CONTEXT=OFF
+        -DWITH_TESTS=OFF
+        -DWITH_TOOLS=OFF
+        -DWITH_TRACE_TOOLS=OFF
+        -DWITH_ZLIB=ON
+    )
+
+    list(APPEND PROJECT_DEPENDENCIES rocksdb_project)
+  endif()
+endif()

cmake/options.cmake

@@ -6,6 +6,7 @@ option(PROJECT_ENABLE_JSON "Enable JSON for Modern C++ library" ON)
 option(PROJECT_ENABLE_LIBSODIUM "Enable libsodium library" ON)
 option(PROJECT_ENABLE_OPENSSL "Enable OpenSSL library" ON)
 option(PROJECT_ENABLE_PUGIXML "Enable PugiXML library" ON)
+option(PROJECT_ENABLE_ROCKSDB "Enable RocksDB library" ON)
 option(PROJECT_ENABLE_SPDLOG "Enable spdlog library" ON) 
 option(PROJECT_ENABLE_SQLITE "Enable SQLite" ON)
 option(PROJECT_ENABLE_STDUUID "Enable stduuid library" ON)

cmake/versions.cmake

@@ -1,15 +1,15 @@
 set(BINUTILS_VERSION 2.41)
-set(BOOST2_MAJOR_VERSION 1)
-set(BOOST2_MINOR_VERSION 76)
-set(BOOST2_PATCH_VERSION 0)
 set(BOOST_MAJOR_VERSION 1)
 set(BOOST_MINOR_VERSION 86)
 set(BOOST_PATCH_VERSION 0)
+set(BOOST2_MAJOR_VERSION 1)
+set(BOOST2_MINOR_VERSION 76)
+set(BOOST2_PATCH_VERSION 0)
 set(CPP_HTTPLIB_VERSION 0.18.1)
-set(CURL2_VERSION 8_11_0)
 set(CURL_VERSION 8.11.0)
-set(EXPAT2_VERSION 2_6_4)
+set(CURL2_VERSION 8_11_0)
 set(EXPAT_VERSION 2.6.4)
+set(EXPAT2_VERSION 2_6_4)
 set(GCC_VERSION 14.2.0)
 set(GTEST_VERSION 1.15.2)
 set(ICU_VERSION 75-1)
@@ -20,8 +20,9 @@ set(MINGW_VERSION 11.0.1)
 set(OPENSSL_VERSION 3.4.0)
 set(PKG_CONFIG_VERSION 0.29.2)
 set(PUGIXML_VERSION 1.14)
+set(ROCKSDB_VERSION 9.7.4)
 set(SPDLOG_VERSION 1.15.0)
-set(SQLITE2_VERSION 3.46.1)
 set(SQLITE_VERSION 3460100)
+set(SQLITE2_VERSION 3.46.1)
 set(STDUUID_VERSION 1.2.3)
 set(ZLIB_VERSION 1.3.1)

config.sh

@@ -30,6 +30,7 @@ PROJECT_ENABLE_JSON=ON
 PROJECT_ENABLE_LIBSODIUM=ON
 PROJECT_ENABLE_OPENSSL=ON
 PROJECT_ENABLE_PUGIXML=ON
+PROJECT_ENABLE_ROCKSDB=ON
 PROJECT_ENABLE_SPDLOG=ON
 PROJECT_ENABLE_SQLITE=ON
 PROJECT_ENABLE_STDUUID=ON

repertory/librepertory/include/db/rdb_meta_db.hpp

@@ -0,0 +1,107 @@
+/*
+  Copyright <2018-2024> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all
+  copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  SOFTWARE.
+*/
+#ifndef REPERTORY_INCLUDE_DB_RDB_META_DB_HPP_
+#define REPERTORY_INCLUDE_DB_RDB_META_DB_HPP_
+
+#include "db/i_meta_db.hpp"
+#include "types/repertory.hpp"
+
+namespace repertory {
+class app_config;
+
+class rdb_meta_db final : public i_meta_db {
+public:
+  rdb_meta_db(const app_config &cfg);
+  ~rdb_meta_db() override;
+
+  rdb_meta_db(const rdb_meta_db &) = delete;
+  rdb_meta_db(rdb_meta_db &&) = delete;
+  auto operator=(const rdb_meta_db &) -> rdb_meta_db & = delete;
+  auto operator=(rdb_meta_db &&) -> rdb_meta_db & = delete;
+
+private:
+  std::unique_ptr<rocksdb::DB> db_{nullptr};
+  rocksdb::ColumnFamilyHandle *default_family_{};
+  rocksdb::ColumnFamilyHandle *pinned_family_{};
+  rocksdb::ColumnFamilyHandle *size_family_{};
+  rocksdb::ColumnFamilyHandle *source_family_{};
+
+private:
+  [[nodiscard]] auto create_iterator(rocksdb::ColumnFamilyHandle *family) const
+      -> std::shared_ptr<rocksdb::Iterator>;
+
+  [[nodiscard]] auto get_item_meta_json(const std::string &api_path,
+                                        json &json_data) const -> api_error;
+
+  [[nodiscard]] static auto
+  perform_action(std::string_view function_name,
+                 std::function<rocksdb::Status()> action) -> api_error;
+
+  [[nodiscard]] auto update_item_meta(const std::string &api_path,
+                                      json json_data) -> api_error;
+
+public:
+  [[nodiscard]] auto get_api_path(const std::string &source_path,
+                                  std::string &api_path) const
+      -> api_error override;
+
+  [[nodiscard]] auto get_api_path_list() const
+      -> std::vector<std::string> override;
+
+  [[nodiscard]] auto get_item_meta(const std::string &api_path,
+                                   api_meta_map &meta) const
+      -> api_error override;
+
+  [[nodiscard]] auto get_item_meta(const std::string &api_path,
+                                   const std::string &key,
+                                   std::string &value) const
+      -> api_error override;
+
+  [[nodiscard]] auto get_pinned_files() const
+      -> std::vector<std::string> override;
+
+  [[nodiscard]] auto get_total_item_count() const -> std::uint64_t override;
+
+  [[nodiscard]] auto get_total_size() const -> std::uint64_t override;
+
+  void remove_api_path(const std::string &api_path) override;
+
+  [[nodiscard]] auto remove_item_meta(const std::string &api_path,
+                                      const std::string &key)
+      -> api_error override;
+
+  [[nodiscard]] auto rename_item_meta(const std::string &from_api_path,
+                                      const std::string &to_api_path)
+      -> api_error override;
+
+  [[nodiscard]] auto set_item_meta(const std::string &api_path,
+                                   const std::string &key,
+                                   const std::string &value)
+      -> api_error override;
+
+  [[nodiscard]] auto set_item_meta(const std::string &api_path,
+                                   const api_meta_map &meta)
+      -> api_error override;
+};
+} // namespace repertory
+
+#endif // REPERTORY_INCLUDE_DB_RDB_META_DB_HPP_

repertory/librepertory/include/providers/base_provider.hpp

@@ -55,16 +55,14 @@ private:
 private:
   void add_all_items(const stop_type &stop_requested);
 
-  void get_removed_items(std::deque<removed_item> &directories,
-                         std::deque<removed_item> &files,
-                         const stop_type &stop_requested) const;
-
-  void process_removed_directories(std::deque<removed_item> &removed_list,
+  void process_removed_directories(std::deque<removed_item> removed_list,
                                    const stop_type &stop_requested);
 
-  void process_removed_files(std::deque<removed_item> &removed_list,
+  void process_removed_files(std::deque<removed_item> removed_list,
                              const stop_type &stop_requested);
 
+  void process_removed_items(const stop_type &stop_requested);
+
   void remove_deleted_items(const stop_type &stop_requested);
 
   void remove_unmatched_source_files(const stop_type &stop_requested);

repertory/librepertory/include/utils/polling.hpp

@@ -35,7 +35,7 @@ public:
     second,
   };
 
-  struct polling_item {
+  struct polling_item final {
     std::string name;
     frequency freq;
     std::function<void(const stop_type &stop_requested)> action;

repertory/librepertory/include/utils/tasks.hpp

@@ -0,0 +1,74 @@
+/*
+  Copyright <2018-2024> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all
+  copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  SOFTWARE.
+*/
+#ifndef REPERTORY_INCLUDE_UTILS_TASKS_HPP_
+#define REPERTORY_INCLUDE_UTILS_TASKS_HPP_
+
+#include "types/repertory.hpp"
+
+namespace repertory {
+class app_config;
+
+class tasks final {
+public:
+  struct task_item final {
+    std::function<void(const stop_type &stop_requested)> action;
+  };
+
+public:
+  tasks(const tasks &) = delete;
+  tasks(tasks &&) = delete;
+  auto operator=(const tasks &) -> tasks & = delete;
+  auto operator=(tasks &&) -> tasks & = delete;
+
+private:
+  tasks() = default;
+
+  ~tasks() { stop(); }
+
+private:
+  static tasks instance_;
+
+public:
+  static auto instance() -> tasks & { return instance_; }
+
+private:
+  app_config *config_{nullptr};
+  std::mutex mutex_;
+  std::condition_variable notify_;
+  std::mutex start_stop_mutex_;
+  stop_type stop_requested_{false};
+  std::vector<std::unique_ptr<std::jthread>> task_threads_;
+  std::deque<task_item> tasks_;
+
+private:
+  void task_thread();
+
+public:
+  void schedule(task_item task);
+
+  void start(app_config *config);
+
+  void stop();
+};
+} // namespace repertory
+
+#endif // REPERTORY_INCLUDE_UTILS_TASKS_HPP_

repertory/librepertory/src/db/meta_db.cpp

@@ -22,10 +22,11 @@
 #include "db/meta_db.hpp"
 
 #include "app_config.hpp"
+#include "db/rdb_meta_db.hpp"
 #include "db/sqlite_meta_db.hpp"
 
 namespace repertory {
 auto create_meta_db(const app_config &cfg) -> std::unique_ptr<i_meta_db> {
-  return std::make_unique<sqlite_meta_db>(cfg);
+  return std::make_unique<rdb_meta_db>(cfg);
 }
 } // namespace repertory

repertory/librepertory/src/db/rdb_meta_db.cpp

@@ -0,0 +1,436 @@
+/*
+  Copyright <2018-2024> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all
+  copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  SOFTWARE.
+*/
+#include "db/rdb_meta_db.hpp"
+
+#include "app_config.hpp"
+#include "types/startup_exception.hpp"
+#include "utils/error_utils.hpp"
+#include "utils/path.hpp"
+#include "utils/string.hpp"
+
+namespace {
+[[nodiscard]] auto
+create_rocksdb(const repertory::app_config &cfg, const std::string &name,
+               const std::vector<rocksdb::ColumnFamilyDescriptor> &families,
+               std::vector<rocksdb::ColumnFamilyHandle *> &handles)
+    -> std::unique_ptr<rocksdb::DB> {
+  REPERTORY_USES_FUNCTION_NAME();
+
+  rocksdb::Options options{};
+  options.create_if_missing = true;
+  options.create_missing_column_families = true;
+  options.db_log_dir = cfg.get_log_directory();
+  options.keep_log_file_num = 10;
+
+  rocksdb::DB *ptr{};
+  auto status = rocksdb::DB::Open(
+      options,
+      repertory::utils::path::combine(cfg.get_data_directory(), {name}),
+      families, &handles, &ptr);
+  if (not status.ok()) {
+    repertory::utils::error::raise_error(function_name, status.ToString());
+    throw repertory::startup_exception(status.ToString());
+  }
+
+  return std::unique_ptr<rocksdb::DB>(ptr);
+}
+} // namespace
+
+namespace repertory {
+rdb_meta_db::rdb_meta_db(const app_config &cfg) {
+  const auto create_resources = [this, &cfg](const std::string &name) {
+    auto families = std::vector<rocksdb::ColumnFamilyDescriptor>();
+    families.emplace_back(rocksdb::kDefaultColumnFamilyName,
+                          rocksdb::ColumnFamilyOptions());
+    families.emplace_back("pinned", rocksdb::ColumnFamilyOptions());
+    families.emplace_back("size", rocksdb::ColumnFamilyOptions());
+    families.emplace_back("source", rocksdb::ColumnFamilyOptions());
+
+    auto handles = std::vector<rocksdb::ColumnFamilyHandle *>();
+    db_ = create_rocksdb(cfg, name, families, handles);
+
+    std::size_t idx{};
+    default_family_ = handles[idx++];
+    pinned_family_ = handles[idx++];
+    size_family_ = handles[idx++];
+    source_family_ = handles[idx++];
+  };
+
+  create_resources("provider_meta");
+}
+
+rdb_meta_db::~rdb_meta_db() { db_.reset(); }
+
+auto rdb_meta_db::create_iterator(rocksdb::ColumnFamilyHandle *family) const
+    -> std::shared_ptr<rocksdb::Iterator> {
+  return std::shared_ptr<rocksdb::Iterator>(
+      db_->NewIterator(rocksdb::ReadOptions(), family));
+}
+
+auto rdb_meta_db::get_api_path(const std::string &source_path,
+                               std::string &api_path) const -> api_error {
+  REPERTORY_USES_FUNCTION_NAME();
+
+  if (source_path.empty()) {
+    return api_error::item_not_found;
+  }
+
+  return perform_action(function_name, [&]() -> rocksdb::Status {
+    return db_->Get(rocksdb::ReadOptions(), source_family_, source_path,
+                    &api_path);
+  });
+}
+
+auto rdb_meta_db::get_api_path_list() const -> std::vector<std::string> {
+  std::vector<std::string> ret;
+  auto iter = create_iterator(default_family_);
+  for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
+    ret.push_back(iter->key().ToString());
+  }
+
+  return ret;
+}
+
+auto rdb_meta_db::get_item_meta_json(const std::string &api_path,
+                                     json &json_data) const -> api_error {
+  REPERTORY_USES_FUNCTION_NAME();
+
+  try {
+    auto found{false};
+
+    {
+      std::string value;
+      auto res = perform_action(function_name, [&]() -> rocksdb::Status {
+        return db_->Get(rocksdb::ReadOptions(), default_family_, api_path,
+                        &value);
+      });
+      if (res != api_error::success) {
+        return res;
+      }
+
+      found = not value.empty();
+      if (found) {
+        json_data = json::parse(value);
+      }
+    }
+
+    {
+      std::string value;
+      auto res = perform_action(function_name, [&]() -> rocksdb::Status {
+        return db_->Get(rocksdb::ReadOptions(), pinned_family_, api_path,
+                        &value);
+      });
+      if (res != api_error::success) {
+        return res;
+      }
+      json_data[META_PINNED] = value;
+      found = found || not value.empty();
+    }
+
+    {
+      std::string value;
+      auto res = perform_action(function_name, [&]() -> rocksdb::Status {
+        return db_->Get(rocksdb::ReadOptions(), size_family_, api_path, &value);
+      });
+      if (res != api_error::success) {
+        return res;
+      }
+      json_data[META_SIZE] = value;
+      found = found || not value.empty();
+    }
+
+    return found ? api_error::success : api_error::item_not_found;
+  } catch (const std::exception &e) {
+    utils::error::raise_api_path_error(function_name, api_path, e,
+                                       "failed to get item meta");
+  }
+
+  return api_error::error;
+}
+
+auto rdb_meta_db::get_item_meta(const std::string &api_path,
+                                api_meta_map &meta) const -> api_error {
+  json json_data;
+  auto ret = get_item_meta_json(api_path, json_data);
+  if (ret != api_error::success) {
+    return ret;
+  }
+
+  for (auto it = json_data.begin(); it != json_data.end(); ++it) {
+    meta[it.key()] = it.value().get<std::string>();
+  }
+
+  return api_error::success;
+}
+
+auto rdb_meta_db::get_item_meta(const std::string &api_path,
+                                const std::string &key,
+                                std::string &value) const -> api_error {
+  REPERTORY_USES_FUNCTION_NAME();
+
+  if (key == META_PINNED) {
+    return perform_action(function_name, [&]() -> rocksdb::Status {
+      return db_->Get(rocksdb::ReadOptions(), pinned_family_, api_path, &value);
+    });
+  }
+
+  if (key == META_SIZE) {
+    return perform_action(function_name, [&]() -> rocksdb::Status {
+      return db_->Get(rocksdb::ReadOptions(), size_family_, api_path, &value);
+    });
+  }
+
+  json json_data;
+  auto ret = get_item_meta_json(api_path, json_data);
+  if (ret != api_error::success) {
+    return ret;
+  }
+
+  if (json_data.find(key) != json_data.end()) {
+    value = json_data[key].get<std::string>();
+  }
+
+  return api_error::success;
+}
+
+auto rdb_meta_db::get_pinned_files() const -> std::vector<std::string> {
+  std::vector<std::string> ret;
+  auto iter = create_iterator(pinned_family_);
+  for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
+    if (not utils::string::to_bool(iter->value().ToString())) {
+      continue;
+    }
+
+    ret.push_back(iter->key().ToString());
+  }
+
+  return ret;
+}
+
+auto rdb_meta_db::get_total_item_count() const -> std::uint64_t {
+  std::uint64_t ret{};
+  auto iter = create_iterator(default_family_);
+  for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
+    ++ret;
+  }
+
+  return ret;
+}
+
+auto rdb_meta_db::get_total_size() const -> std::uint64_t {
+  std::uint64_t ret{};
+  auto iter = create_iterator(size_family_);
+  for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
+    ret += utils::string::to_uint64(iter->value().ToString());
+  }
+
+  return ret;
+}
+
+auto rdb_meta_db::perform_action(std::string_view function_name,
+                                 std::function<rocksdb::Status()> action)
+    -> api_error {
+  auto res = action();
+  if (res.ok()) {
+    return api_error::success;
+  }
+
+  if (not res.IsNotFound()) {
+    utils::error::raise_error(function_name, res.ToString());
+  }
+
+  return res.IsNotFound() ? api_error::item_not_found : api_error::error;
+}
+
+void rdb_meta_db::remove_api_path(const std::string &api_path) {
+  REPERTORY_USES_FUNCTION_NAME();
+
+  std::string source_path;
+  [[maybe_unused]] auto res = get_item_meta(api_path, META_SOURCE, source_path);
+
+  res = perform_action(
+      function_name, [this, &api_path, &source_path]() -> rocksdb::Status {
+        db_->Delete(rocksdb::WriteOptions(), pinned_family_, api_path);
+        db_->Delete(rocksdb::WriteOptions(), size_family_, api_path);
+        if (not source_path.empty()) {
+          db_->Delete(rocksdb::WriteOptions(), source_family_, source_path);
+        }
+        return db_->Delete(rocksdb::WriteOptions(), default_family_, api_path);
+      });
+}
+
+auto rdb_meta_db::remove_item_meta(const std::string &api_path,
+                                   const std::string &key) -> api_error {
+  if (key == META_DIRECTORY || key == META_PINNED || key == META_SIZE ||
+      key == META_SOURCE) {
+    // TODO log warning for unsupported attributes
+    return api_error::success;
+  }
+
+  json json_data;
+  auto res = get_item_meta_json(api_path, json_data);
+  if (res != api_error::success) {
+    return res;
+  }
+
+  json_data.erase(key);
+  return update_item_meta(api_path, json_data);
+}
+
+auto rdb_meta_db::rename_item_meta(const std::string &from_api_path,
+                                   const std::string &to_api_path)
+    -> api_error {
+  json json_data;
+  auto res = get_item_meta_json(from_api_path, json_data);
+  if (res != api_error::success) {
+    return res;
+  }
+
+  remove_api_path(from_api_path);
+  return update_item_meta(to_api_path, json_data);
+}
+
+auto rdb_meta_db::set_item_meta(const std::string &api_path,
+                                const std::string &key,
+                                const std::string &value) -> api_error {
+  REPERTORY_USES_FUNCTION_NAME();
+
+  if (key == META_PINNED) {
+    return perform_action(function_name, [&]() -> rocksdb::Status {
+      return db_->Put(rocksdb::WriteOptions(), pinned_family_, api_path, value);
+    });
+  }
+
+  if (key == META_SIZE) {
+    return perform_action(function_name, [&]() -> rocksdb::Status {
+      return db_->Put(rocksdb::WriteOptions(), size_family_, api_path, value);
+    });
+  }
+
+  json json_data;
+  auto res = get_item_meta_json(api_path, json_data);
+  if (res != api_error::success && res != api_error::item_not_found) {
+    return res;
+  }
+
+  json_data[key] = value;
+
+  return update_item_meta(api_path, json_data);
+}
+
+auto rdb_meta_db::set_item_meta(const std::string &api_path,
+                                const api_meta_map &meta) -> api_error {
+  json json_data;
+  auto res = get_item_meta_json(api_path, json_data);
+  if (res != api_error::success && res != api_error::item_not_found) {
+    return res;
+  }
+
+  for (const auto &data : meta) {
+    json_data[data.first] = data.second;
+  }
+
+  return update_item_meta(api_path, json_data);
+}
+
+auto rdb_meta_db::update_item_meta(const std::string &api_path, json json_data)
+    -> api_error {
+  REPERTORY_USES_FUNCTION_NAME();
+
+  try {
+    if (not json_data.contains(META_PINNED)) {
+      json_data[META_PINNED] = utils::string::from_bool(false);
+    }
+    if (not json_data.contains(META_SIZE)) {
+      json_data[META_SIZE] = "0";
+    }
+    if (not json_data.contains(META_SOURCE)) {
+      json_data[META_SOURCE] = "";
+    }
+
+    auto directory =
+        utils::string::to_bool(json_data.at(META_DIRECTORY).get<std::string>());
+
+    auto pinned = directory ? false
+                            : utils::string::to_bool(
+                                  json_data.at(META_PINNED).get<std::string>());
+    auto size = directory ? std::uint64_t(0U)
+                          : utils::string::to_uint64(
+                                json_data.at(META_SIZE).get<std::string>());
+    auto source_path = directory ? std::string("")
+                                 : json_data.at(META_SOURCE).get<std::string>();
+
+    if (not directory) {
+      std::string orig_source_path;
+      auto res = get_item_meta(api_path, META_SOURCE, orig_source_path);
+      if (res != api_error::success && res != api_error::item_not_found) {
+        return res;
+      }
+
+      if (not orig_source_path.empty() && orig_source_path != source_path) {
+        res = perform_action(function_name, [&]() -> rocksdb::Status {
+          return db_->Delete(rocksdb::WriteOptions(), source_family_,
+                             orig_source_path);
+        });
+        if (res != api_error::success && res != api_error::item_not_found) {
+          return res;
+        }
+      }
+    }
+
+    json_data.erase(META_PINNED);
+    json_data.erase(META_SIZE);
+
+    return perform_action(function_name, [&]() -> rocksdb::Status {
+      if (not directory) {
+        auto res = db_->Put(rocksdb::WriteOptions(), pinned_family_, api_path,
+                            utils::string::from_bool(pinned));
+        if (not res.ok()) {
+          return res;
+        }
+
+        res = db_->Put(rocksdb::WriteOptions(), size_family_, api_path,
+                       std::to_string(size));
+        if (not res.ok()) {
+          return res;
+        }
+
+        if (not source_path.empty()) {
+          res = db_->Put(rocksdb::WriteOptions(), source_family_, source_path,
+                         api_path);
+          if (not res.ok()) {
+            return res;
+          }
+        }
+      }
+
+      return db_->Put(rocksdb::WriteOptions(), default_family_, api_path,
+                      json_data.dump());
+    });
+  } catch (const std::exception &e) {
+    utils::error::raise_api_path_error(function_name, api_path, e,
+                                       "failed to update item meta");
+  }
+
+  return api_error::error;
+}
+} // namespace repertory

repertory/librepertory/src/db/sqlite_meta_db.cpp

@@ -32,8 +32,6 @@
 
 namespace repertory {
 sqlite_meta_db::sqlite_meta_db(const app_config &cfg) {
-  REPERTORY_USES_FUNCTION_NAME();
-
   const std::map<std::string, std::string> sql_create_tables{
       {
           {"meta"},
@@ -308,33 +306,51 @@ auto sqlite_meta_db::update_item_meta(const std::string &api_path,
                                       api_meta_map meta) -> api_error {
   REPERTORY_USES_FUNCTION_NAME();
 
-  auto directory = utils::string::to_bool(meta[META_DIRECTORY]);
-  auto pinned = utils::string::to_bool(meta[META_PINNED]);
-  auto size =
-      directory ? std::uint64_t(0U) : utils::string::to_uint64(meta[META_SIZE]);
-  auto source_path = meta[META_SOURCE];
+  try {
+    if (meta[META_PINNED].empty()) {
+      meta[META_PINNED] = utils::string::from_bool(false);
+    }
+    if (meta[META_SIZE].empty()) {
+      meta[META_SIZE] = "0";
+    }
+    if (meta[META_SOURCE].empty()) {
+      meta[META_SOURCE] = "";
+    }
 
-  meta.erase(META_DIRECTORY);
-  meta.erase(META_PINNED);
-  meta.erase(META_SIZE);
-  meta.erase(META_SOURCE);
+    auto directory = utils::string::to_bool(meta.at(META_DIRECTORY));
+    auto pinned =
+        directory ? false : utils::string::to_bool(meta.at(META_PINNED));
+    auto size = directory ? std::uint64_t(0U)
+                          : utils::string::to_uint64(meta.at(META_SIZE));
+    auto source_path = directory ? std::string("") : meta.at(META_SOURCE);
 
-  auto result = utils::db::sqlite::db_insert{*db_, table_name}
-                    .or_replace()
-                    .column_value("api_path", api_path)
-                    .column_value("data", nlohmann::json(meta).dump())
-                    .column_value("directory", directory ? 1 : 0)
-                    .column_value("pinned", pinned ? 1 : 0)
-                    .column_value("size", static_cast<std::int64_t>(size))
-                    .column_value("source_path", source_path)
-                    .go();
-  if (not result.ok()) {
-    utils::error::raise_api_path_error(function_name, api_path,
-                                       result.get_error(),
+    meta.erase(META_DIRECTORY);
+    meta.erase(META_PINNED);
+    meta.erase(META_SIZE);
+    meta.erase(META_SOURCE);
+
+    auto result = utils::db::sqlite::db_insert{*db_, table_name}
+                      .or_replace()
+                      .column_value("api_path", api_path)
+                      .column_value("data", nlohmann::json(meta).dump())
+                      .column_value("directory", directory ? 1 : 0)
+                      .column_value("pinned", pinned ? 1 : 0)
+                      .column_value("size", static_cast<std::int64_t>(size))
+                      .column_value("source_path", source_path)
+                      .go();
+    if (not result.ok()) {
+      utils::error::raise_api_path_error(function_name, api_path,
+                                         result.get_error(),
+                                         "failed to update item meta");
+      return api_error::error;
+    }
+
+    return api_error::success;
+  } catch (const std::exception &e) {
+    utils::error::raise_api_path_error(function_name, api_path, e,
                                        "failed to update item meta");
-    return api_error::error;
   }
 
-  return api_error::success;
+  return api_error::error;
 }
 } // namespace repertory

repertory/librepertory/src/drives/fuse/fuse_drive.cpp

@@ -42,6 +42,7 @@
 #include "utils/common.hpp"
 #include "utils/error_utils.hpp"
 #include "utils/polling.hpp"
+#include "utils/tasks.hpp"
 #include "utils/time.hpp"
 #include "utils/utils.hpp"
 
@@ -258,6 +259,7 @@ void fuse_drive::destroy_impl(void *ptr) {
   }
 
   polling::instance().stop();
+  tasks::instance().stop();
 
   if (eviction_) {
     eviction_->stop();
@@ -622,6 +624,7 @@ void *fuse_drive::init_impl(struct fuse_conn_info *conn) {
     }
 
     polling::instance().start(&config_);
+    tasks::instance().start(&config_);
 
     event_system::instance().raise<drive_mounted>(get_mount_location());
   } catch (const std::exception &e) {

repertory/librepertory/src/drives/winfsp/winfsp_drive.cpp

@@ -38,6 +38,7 @@
 #include "utils/file_utils.hpp"
 #include "utils/polling.hpp"
 #include "utils/string.hpp"
+#include "utils/tasks.hpp"
 #include "utils/time.hpp"
 #include "utils/utils.hpp"
 
@@ -652,6 +653,8 @@ auto winfsp_drive::Mounted(PVOID host) -> NTSTATUS {
     }
 
     polling::instance().start(&config_);
+    tasks::instance().start(&config_);
+
     event_system::instance().raise<drive_mounted>(mount_location);
   } catch (const std::exception &e) {
     utils::error::raise_error(function_name, e, "exception occurred");
@@ -660,6 +663,7 @@ auto winfsp_drive::Mounted(PVOID host) -> NTSTATUS {
     }
     server_->stop();
     polling::instance().stop();
+    tasks::instance().stop();
     if (eviction_) {
       eviction_->stop();
     }
@@ -1172,6 +1176,7 @@ VOID winfsp_drive::Unmounted(PVOID host) {
   }
   server_->stop();
   polling::instance().stop();
+  tasks::instance().stop();
   if (eviction_) {
     eviction_->stop();
   }

repertory/librepertory/src/providers/base_provider.cpp

@@ -19,8 +19,6 @@
   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
   SOFTWARE.
 */
-#include <execution>
-
 #include "providers/base_provider.hpp"
 
 #include "app_config.hpp"
@@ -32,6 +30,7 @@
 #include "utils/file_utils.hpp"
 #include "utils/path.hpp"
 #include "utils/polling.hpp"
+#include "utils/tasks.hpp"
 #include "utils/time.hpp"
 
 namespace repertory {
@@ -50,8 +49,8 @@ void base_provider::add_all_items(const stop_type &stop_requested) {
 }
 
 auto base_provider::create_api_file(std::string path, std::string key,
-                                    std::uint64_t size,
-                                    std::uint64_t file_time) -> api_file {
+                                    std::uint64_t size, std::uint64_t file_time)
+    -> api_file {
   api_file file{};
   file.api_path = utils::path::create_api_path(path);
   file.api_parent = utils::path::get_parent_api_path(file.api_path);
@@ -83,8 +82,8 @@ auto base_provider::create_api_file(std::string path, std::uint64_t size,
 }
 
 auto base_provider::create_directory_clone_source_meta(
-    const std::string &source_api_path,
-    const std::string &api_path) -> api_error {
+    const std::string &source_api_path, const std::string &api_path)
+    -> api_error {
   REPERTORY_USES_FUNCTION_NAME();
 
   bool exists{};
@@ -181,8 +180,8 @@ auto base_provider::create_directory(const std::string &api_path,
   return set_item_meta(api_path, meta);
 }
 
-auto base_provider::create_file(const std::string &api_path,
-                                api_meta_map &meta) -> api_error {
+auto base_provider::create_file(const std::string &api_path, api_meta_map &meta)
+    -> api_error {
   REPERTORY_USES_FUNCTION_NAME();
 
   bool exists{};
@@ -239,8 +238,9 @@ auto base_provider::create_file(const std::string &api_path,
   return api_error::error;
 }
 
-auto base_provider::get_api_path_from_source(
-    const std::string &source_path, std::string &api_path) const -> api_error {
+auto base_provider::get_api_path_from_source(const std::string &source_path,
+                                             std::string &api_path) const
+    -> api_error {
   REPERTORY_USES_FUNCTION_NAME();
 
   if (source_path.empty()) {
@@ -253,8 +253,9 @@ auto base_provider::get_api_path_from_source(
   return db3_->get_api_path(source_path, api_path);
 }
 
-auto base_provider::get_directory_items(
-    const std::string &api_path, directory_item_list &list) const -> api_error {
+auto base_provider::get_directory_items(const std::string &api_path,
+                                        directory_item_list &list) const
+    -> api_error {
   REPERTORY_USES_FUNCTION_NAME();
 
   bool exists{};
@@ -318,9 +319,10 @@ auto base_provider::get_file_size(const std::string &api_path,
   return api_error::success;
 }
 
-auto base_provider::get_filesystem_item(
-    const std::string &api_path, bool directory,
-    filesystem_item &fsi) const -> api_error {
+auto base_provider::get_filesystem_item(const std::string &api_path,
+                                        bool directory,
+                                        filesystem_item &fsi) const
+    -> api_error {
   bool exists{};
   auto res = is_directory(api_path, exists);
   if (res != api_error::success) {
@@ -353,9 +355,10 @@ auto base_provider::get_filesystem_item(
   return api_error::success;
 }
 
-auto base_provider::get_filesystem_item_and_file(
-    const std::string &api_path, api_file &file,
-    filesystem_item &fsi) const -> api_error {
+auto base_provider::get_filesystem_item_and_file(const std::string &api_path,
+                                                 api_file &file,
+                                                 filesystem_item &fsi) const
+    -> api_error {
   auto res = get_file(api_path, file);
   if (res != api_error::success) {
     return res;
@@ -411,49 +414,6 @@ auto base_provider::get_pinned_files() const -> std::vector<std::string> {
   return db3_->get_pinned_files();
 }
 
-void base_provider::get_removed_items(std::deque<removed_item> &directories,
-                                      std::deque<removed_item> &files,
-                                      const stop_type &stop_requested) const {
-  auto list = db3_->get_api_path_list();
-  std::all_of(std::execution::par, list.begin(), list.end(),
-              [&](auto &&api_path) -> bool {
-                if (stop_requested) {
-                  return false;
-                }
-
-                api_meta_map meta{};
-                if (get_item_meta(api_path, meta) != api_error::success) {
-                  return true;
-                }
-
-                if (utils::string::to_bool(meta[META_DIRECTORY])) {
-                  bool exists{};
-                  if (is_directory(api_path, exists) != api_error::success) {
-                    return true;
-                  }
-
-                  if (not exists) {
-                    directories.emplace_back(removed_item{api_path, true, ""});
-                  }
-
-                  return true;
-                }
-
-                bool exists{};
-                if (is_file(api_path, exists) != api_error::success) {
-                  return true;
-                }
-
-                if (exists) {
-                  return true;
-                }
-
-                files.emplace_back(
-                    removed_item{api_path, false, meta[META_SOURCE]});
-                return true;
-              });
-}
-
 auto base_provider::get_total_item_count() const -> std::uint64_t {
   return db3_->get_total_item_count();
 }
@@ -474,7 +434,7 @@ auto base_provider::is_file_writeable(const std::string &api_path) const
 }
 
 void base_provider::process_removed_directories(
-    std::deque<removed_item> &removed_list, const stop_type &stop_requested) {
+    std::deque<removed_item> removed_list, const stop_type &stop_requested) {
   for (auto &&item : removed_list) {
     if (stop_requested) {
       return;
@@ -490,8 +450,8 @@ void base_provider::process_removed_directories(
   }
 }
 
-void base_provider::process_removed_files(
-    std::deque<removed_item> &removed_list, const stop_type &stop_requested) {
+void base_provider::process_removed_files(std::deque<removed_item> removed_list,
+                                          const stop_type &stop_requested) {
   REPERTORY_USES_FUNCTION_NAME();
 
   auto orphaned_directory =
@@ -550,6 +510,59 @@ void base_provider::process_removed_files(
   }
 }
 
+void base_provider::process_removed_items(const stop_type &stop_requested) {
+  auto list = db3_->get_api_path_list();
+  [[maybe_unused]] auto res =
+      std::all_of(list.begin(), list.end(), [&](auto &&api_path) -> bool {
+        if (stop_requested) {
+          return false;
+        }
+
+        tasks::instance().schedule({
+            [this, api_path](auto &&stop_requested2) {
+              api_meta_map meta{};
+              if (get_item_meta(api_path, meta) != api_error::success) {
+                return;
+              }
+
+              if (utils::string::to_bool(meta[META_DIRECTORY])) {
+                bool exists{};
+                if (is_directory(api_path, exists) != api_error::success) {
+                  return;
+                }
+
+                if (exists) {
+                  return;
+                }
+
+                // process_removed_directories({
+                //     removed_item{api_path, true, ""},
+                // }, stop_requested2);
+
+                return;
+              }
+
+              bool exists{};
+              if (is_file(api_path, exists) != api_error::success) {
+                return;
+              }
+
+              if (exists) {
+                return;
+              }
+
+              process_removed_files(
+                  {
+                      removed_item{api_path, false, meta[META_SOURCE]},
+                  },
+                  stop_requested2);
+            },
+        });
+
+        return true;
+      });
+}
+
 void base_provider::remove_deleted_items(const stop_type &stop_requested) {
   add_all_items(stop_requested);
   if (stop_requested) {
@@ -561,19 +574,7 @@ void base_provider::remove_deleted_items(const stop_type &stop_requested) {
     return;
   }
 
-  std::deque<removed_item> directories;
-  std::deque<removed_item> files;
-  get_removed_items(directories, files, stop_requested);
-  if (stop_requested) {
-    return;
-  }
-
-  process_removed_files(files, stop_requested);
-  if (stop_requested) {
-    return;
-  }
-
-  process_removed_directories(directories, stop_requested);
+  process_removed_items(stop_requested);
 }
 
 auto base_provider::remove_file(const std::string &api_path) -> api_error {

repertory/librepertory/src/utils/tasks.cpp

@@ -0,0 +1,118 @@
+/*
+  Copyright <2018-2024> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all
+  copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  SOFTWARE.
+*/
+#include "utils/tasks.hpp"
+
+#include "app_config.hpp"
+
+namespace repertory {
+tasks tasks::instance_;
+
+void tasks::schedule(task_item task) {
+  unique_mutex_lock lock(mutex_);
+  while (not stop_requested_ && tasks_.size() > (task_threads_.size() * 4U)) {
+    notify_.wait(lock);
+  }
+
+  if (stop_requested_) {
+    notify_.notify_all();
+    return;
+  }
+
+  tasks_.push_back(std::move(task));
+  notify_.notify_all();
+}
+
+void tasks::start(app_config *config) {
+  mutex_lock start_stop_lock(start_stop_mutex_);
+  if (not task_threads_.empty()) {
+    return;
+  }
+
+  config_ = config;
+  stop_requested_ = false;
+  for (std::uint32_t idx = 0U; idx < std::thread::hardware_concurrency();
+       ++idx) {
+    task_threads_.emplace_back(
+        std::make_unique<std::jthread>([this]() { task_thread(); }));
+  }
+}
+
+void tasks::stop() {
+  mutex_lock start_stop_lock(start_stop_mutex_);
+  if (task_threads_.empty()) {
+    return;
+  }
+
+  stop_requested_ = true;
+
+  unique_mutex_lock lock(mutex_);
+  notify_.notify_all();
+  lock.unlock();
+
+  task_threads_.clear();
+}
+
+void tasks::task_thread() {
+  REPERTORY_USES_FUNCTION_NAME();
+
+  unique_mutex_lock lock(mutex_);
+
+  const auto release = [&]() {
+    notify_.notify_all();
+    lock.unlock();
+  };
+
+  release();
+
+  while (not stop_requested_) {
+    lock.lock();
+
+    if (stop_requested_) {
+      release();
+      return;
+    }
+
+    if (tasks_.empty()) {
+      notify_.wait(lock);
+      if (stop_requested_) {
+        release();
+        return;
+      }
+    }
+
+    if (tasks_.empty()) {
+      release();
+      continue;
+    }
+
+    auto task = tasks_.front();
+    tasks_.pop_front();
+    release();
+
+    try {
+      task.action(stop_requested_);
+    } catch (const std::exception &e) {
+      utils::error::raise_error(function_name, e, "failed to execute task");
+    }
+  }
+}
+} // namespace repertory

repertory/repertory_test/include/fixtures/meta_db_fixture.hpp

@@ -0,0 +1,70 @@
+/*
+  Copyright <2018-2024> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all
+  copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  SOFTWARE.
+*/
+#ifndef REPERTORY_TEST_INCLUDE_FIXTURES_META_DB_FIXTURE_HPP
+#define REPERTORY_TEST_INCLUDE_FIXTURES_META_DB_FIXTURE_HPP
+
+#include "test_common.hpp"
+
+#include "app_config.hpp"
+#include "db/rdb_meta_db.hpp"
+#include "db/sqlite_meta_db.hpp"
+#include "events/consumers/console_consumer.hpp"
+#include "events/event_system.hpp"
+
+namespace repertory {
+template <typename db_t> class meta_db_test : public ::testing::Test {
+protected:
+  static std::unique_ptr<app_config> config;
+  static console_consumer console_;
+  static std::unique_ptr<db_t> meta_db;
+
+protected:
+  static void SetUpTestCase() {
+    static std::uint64_t idx{};
+
+    event_system::instance().start();
+    auto cfg_directory = utils::path::combine(test::get_test_output_dir(),
+                                              {
+                                                  "meta_db_test",
+                                                  std::to_string(++idx),
+                                              });
+    config = std::make_unique<app_config>(provider_type::s3, cfg_directory);
+    meta_db = std::make_unique<db_t>(*config);
+  }
+
+  static void TearDownTestCase() {
+    meta_db.reset();
+    config.reset();
+    event_system::instance().stop();
+  }
+};
+
+using meta_db_types = ::testing::Types<rdb_meta_db, sqlite_meta_db>;
+
+template <typename db_t> std::unique_ptr<app_config> meta_db_test<db_t>::config;
+
+template <typename db_t> console_consumer meta_db_test<db_t>::console_;
+
+template <typename db_t> std::unique_ptr<db_t> meta_db_test<db_t>::meta_db;
+} // namespace repertory
+
+#endif // REPERTORY_TEST_INCLUDE_FIXTURES_META_DB_FIXTURE_HPP

repertory/repertory_test/src/file_manager_test.cpp

@@ -37,6 +37,7 @@
 #include "utils/path.hpp"
 #include "utils/polling.hpp"
 #include "utils/string.hpp"
+#include "utils/tasks.hpp"
 #include "utils/time.hpp"
 #include "utils/utils.hpp"
 
@@ -108,6 +109,7 @@ TEST_F(file_manager_test, can_create_and_close_file) {
   EXPECT_CALL(mp, is_read_only()).WillRepeatedly(Return(false));
 
   polling::instance().start(cfg.get());
+  tasks::instance().start(cfg.get());
 
   file_manager mgr(*cfg, mp);
   mgr.start();
@@ -209,6 +211,7 @@ TEST_F(file_manager_test, can_create_and_close_file) {
   mgr.stop();
 
   polling::instance().stop();
+  tasks::instance().stop();
 }
 
 TEST_F(file_manager_test, can_open_and_close_file) {
@@ -217,6 +220,8 @@ TEST_F(file_manager_test, can_open_and_close_file) {
   EXPECT_CALL(mp, is_read_only()).WillRepeatedly(Return(false));
 
   polling::instance().start(cfg.get());
+  tasks::instance().start(cfg.get());
+
   file_manager mgr(*cfg, mp);
   mgr.start();
 
@@ -316,12 +321,15 @@ TEST_F(file_manager_test, can_open_and_close_file) {
   mgr.stop();
 
   polling::instance().stop();
+  tasks::instance().stop();
 }
 
 TEST_F(file_manager_test, can_open_and_close_multiple_handles_for_same_file) {
   EXPECT_CALL(mp, is_read_only()).WillRepeatedly(Return(false));
 
   polling::instance().start(cfg.get());
+  tasks::instance().start(cfg.get());
+
   file_manager mgr(*cfg, mp);
   mgr.start();
 
@@ -377,6 +385,7 @@ TEST_F(file_manager_test, can_open_and_close_multiple_handles_for_same_file) {
   EXPECT_EQ(std::size_t(0U), mgr.get_open_handle_count());
 
   polling::instance().stop();
+  tasks::instance().stop();
 }
 
 TEST_F(file_manager_test,
@@ -535,6 +544,8 @@ TEST_F(file_manager_test, upload_occurs_after_write_if_fully_downloaded) {
   EXPECT_CALL(mp, is_read_only()).WillRepeatedly(Return(false));
 
   polling::instance().start(cfg.get());
+  tasks::instance().start(cfg.get());
+
   file_manager mgr(*cfg, mp);
   mgr.start();
 
@@ -638,6 +649,7 @@ TEST_F(file_manager_test, upload_occurs_after_write_if_fully_downloaded) {
   file.close();
 
   polling::instance().stop();
+  tasks::instance().stop();
 }
 
 TEST_F(file_manager_test, can_evict_file) {
@@ -1419,6 +1431,7 @@ TEST_F(file_manager_test, file_is_closed_after_download_timeout) {
   cfg->set_chunk_downloader_timeout_secs(3U);
 
   polling::instance().start(cfg.get());
+  tasks::instance().start(cfg.get());
 
   EXPECT_CALL(mp, is_read_only()).WillRepeatedly(Return(false));
 
@@ -1501,6 +1514,7 @@ TEST_F(file_manager_test, file_is_closed_after_download_timeout) {
   mgr.stop();
 
   polling::instance().stop();
+  tasks::instance().stop();
 }
 
 TEST_F(file_manager_test, remove_file_fails_if_file_does_not_exist) {
@@ -1548,6 +1562,7 @@ TEST_F(file_manager_test,
   EXPECT_CALL(mp, is_read_only()).WillRepeatedly(Return(false));
 
   polling::instance().start(cfg.get());
+  tasks::instance().start(cfg.get());
 
   file_manager mgr(*cfg, mp);
   mgr.start();
@@ -1620,5 +1635,6 @@ TEST_F(file_manager_test,
   mgr.stop();
 
   polling::instance().stop();
+  tasks::instance().stop();
 }
 } // namespace repertory

repertory/repertory_test/src/meta_db_test.cpp

@@ -0,0 +1,172 @@
+/*
+  Copyright <2018-2024> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all
+  copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  SOFTWARE.
+*/
+
+#include "fixtures/meta_db_fixture.hpp"
+
+namespace {
+[[nodiscard]] auto create_test_file() -> std::string {
+  static std::atomic<std::uint64_t> idx{};
+  return "/test" + std::to_string(++idx);
+}
+} // namespace
+
+namespace repertory {
+TYPED_TEST_CASE(meta_db_test, meta_db_types);
+
+TYPED_TEST(meta_db_test, can_get_api_path_from_source_path) {
+  auto test_file = create_test_file();
+  auto test_source = create_test_file();
+  EXPECT_EQ(
+      api_error::success,
+      this->meta_db->set_item_meta(
+          test_file, {
+                         {META_DIRECTORY, utils::string::from_bool(false)},
+                         {META_SOURCE, test_source},
+                     }));
+
+  std::string api_path;
+  EXPECT_EQ(api_error::success,
+            this->meta_db->get_api_path(test_source, api_path));
+  EXPECT_STREQ(test_file.c_str(), api_path.c_str());
+}
+
+TYPED_TEST(meta_db_test, can_change_source_path) {
+  auto test_file = create_test_file();
+  auto test_source = create_test_file();
+  auto test_source2 = create_test_file();
+  EXPECT_EQ(
+      api_error::success,
+      this->meta_db->set_item_meta(
+          test_file, {
+                         {META_DIRECTORY, utils::string::from_bool(false)},
+                         {META_SOURCE, test_source},
+                     }));
+
+  EXPECT_EQ(api_error::success,
+            this->meta_db->set_item_meta(test_file, META_SOURCE, test_source2));
+
+  std::string api_path;
+  EXPECT_EQ(api_error::success,
+            this->meta_db->get_api_path(test_source2, api_path));
+  EXPECT_STREQ(test_file.c_str(), api_path.c_str());
+
+  std::string api_path2;
+  EXPECT_EQ(api_error::item_not_found,
+            this->meta_db->get_api_path(test_source, api_path2));
+  EXPECT_TRUE(api_path2.empty());
+}
+
+TYPED_TEST(meta_db_test,
+           get_api_path_returns_item_not_found_if_source_does_not_exist) {
+  std::string api_path;
+  EXPECT_EQ(api_error::item_not_found,
+            this->meta_db->get_api_path(create_test_file(), api_path));
+  EXPECT_TRUE(api_path.empty());
+}
+
+TYPED_TEST(meta_db_test, set_item_meta_fails_with_missing_directory_meta) {
+  auto test_file = create_test_file();
+  auto test_source = create_test_file();
+  EXPECT_EQ(api_error::error, this->meta_db->set_item_meta(
+                                  test_file, {
+                                                 {META_SOURCE, test_source},
+                                             }));
+  EXPECT_EQ(api_error::error,
+            this->meta_db->set_item_meta(test_file, META_SOURCE, test_source));
+}
+
+TYPED_TEST(meta_db_test, can_get_api_file_list) {
+  std::vector<std::string> directories{};
+  for (auto idx = 0U; idx < 5U; ++idx) {
+    auto test_dir = create_test_file();
+    directories.push_back(test_dir);
+    EXPECT_EQ(
+        api_error::success,
+        this->meta_db->set_item_meta(
+            test_dir, {
+                          {META_DIRECTORY, utils::string::from_bool(true)},
+                      }));
+  }
+
+  std::vector<std::string> files{};
+  for (auto idx = 0U; idx < 5U; ++idx) {
+    auto test_file = create_test_file();
+    files.push_back(test_file);
+    EXPECT_EQ(
+        api_error::success,
+        this->meta_db->set_item_meta(
+            test_file, {
+                           {META_DIRECTORY, utils::string::from_bool(false)},
+                       }));
+  }
+
+  auto file_list = this->meta_db->get_api_path_list();
+  for (const auto &api_path : directories) {
+    EXPECT_TRUE(utils::collection::includes(file_list, api_path));
+  }
+
+  for (const auto &api_path : files) {
+    EXPECT_TRUE(utils::collection::includes(file_list, api_path));
+  }
+}
+
+TYPED_TEST(meta_db_test, can_get_full_item_meta_for_directory) {
+  auto api_path = create_test_file();
+  auto source_path = create_test_file();
+  EXPECT_EQ(api_error::success,
+            this->meta_db->set_item_meta(
+                api_path, {
+                              {META_DIRECTORY, utils::string::from_bool(true)},
+                              {META_PINNED, utils::string::from_bool(true)},
+                              {META_SIZE, std::to_string(2ULL)},
+                              {META_SOURCE, source_path},
+                          }));
+  api_meta_map meta;
+  EXPECT_EQ(api_error::success, this->meta_db->get_item_meta(api_path, meta));
+
+  EXPECT_TRUE(utils::string::to_bool(meta[META_DIRECTORY]));
+  EXPECT_FALSE(utils::string::to_bool(meta[META_PINNED]));
+  EXPECT_EQ(0U, utils::string::to_uint64(meta[META_SIZE]));
+  EXPECT_TRUE(meta[META_SOURCE].empty());
+}
+
+TYPED_TEST(meta_db_test, can_get_full_item_meta_for_file) {
+  auto api_path = create_test_file();
+  auto source_path = create_test_file();
+  EXPECT_EQ(api_error::success,
+            this->meta_db->set_item_meta(
+                api_path, {
+                              {META_DIRECTORY, utils::string::from_bool(false)},
+                              {META_PINNED, utils::string::from_bool(true)},
+                              {META_SIZE, std::to_string(2ULL)},
+                              {META_SOURCE, source_path},
+                          }));
+
+  api_meta_map meta;
+  EXPECT_EQ(api_error::success, this->meta_db->get_item_meta(api_path, meta));
+
+  EXPECT_FALSE(utils::string::to_bool(meta[META_DIRECTORY]));
+  EXPECT_TRUE(utils::string::to_bool(meta[META_PINNED]));
+  EXPECT_EQ(2ULL, utils::string::to_uint64(meta[META_SIZE]));
+  EXPECT_STREQ(source_path.c_str(), meta[META_SOURCE].c_str());
+}
+} // namespace repertory

scripts/options.sh

@@ -9,6 +9,7 @@ PROJECT_LIBRARIES=(
   LIBSODIUM
   OPENSSL
   PUGIXML
+  ROCKSDB
   SPDLOG
   SQLITE
   STDUUID
@@ -25,6 +26,7 @@ PROJECT_CLEANUP[LIBBITCOIN_SYSTEM_ON]="3rd_party/boost_${PROJECT_VERSIONS[BOOST_
 PROJECT_CLEANUP[LIBSODIUM]="3rd_party/libsodium-*:3rd_party/libsodium*"
 PROJECT_CLEANUP[OPENSSL]="3rd_party/openssl-*"
 PROJECT_CLEANUP[PUGIXML]="3rd_party/pugixml-*"
+PROJECT_CLEANUP[ROCKSDB]="3rd_party/rocksdb-*"
 PROJECT_CLEANUP[SPDLOG]="3rd_party/spdlog-*"
 PROJECT_CLEANUP[SQLITE]="3rd_party/sqlite*"
 PROJECT_CLEANUP[STDUUID]="3rd_party/stduuid-*"

scripts/versions.sh

@@ -23,6 +23,7 @@ PROJECT_VERSIONS[MINGW]="11.0.1"
 PROJECT_VERSIONS[OPENSSL]="3.4.0"
 PROJECT_VERSIONS[PKG_CONFIG]="0.29.2"
 PROJECT_VERSIONS[PUGIXML]="1.14"
+PROJECT_VERSIONS[ROCKSDB]="9.7.4"
 PROJECT_VERSIONS[SPDLOG]="1.15.0"
 PROJECT_VERSIONS[SQLITE]="3460100"
 PROJECT_VERSIONS[SQLITE2]="3.46.1"
@@ -46,6 +47,7 @@ PROJECT_DOWNLOADS[MINGW]="https://phoenixnap.dl.sourceforge.net/project/mingw-w6
 PROJECT_DOWNLOADS[OPENSSL]="https://github.com/openssl/openssl/releases/download/openssl-${PROJECT_VERSIONS[OPENSSL]}/openssl-${PROJECT_VERSIONS[OPENSSL]}.tar.gz;openssl-${PROJECT_VERSIONS[OPENSSL]}.tar.gz;3rd_party"
 PROJECT_DOWNLOADS[PKG_CONFIG]="https://pkgconfig.freedesktop.org/releases/pkg-config-${PROJECT_VERSIONS[PKG_CONFIG]}.tar.gz;pkg-config-${PROJECT_VERSIONS[PKG_CONFIG]}.tar.gz;3rd_party/mingw64"
 PROJECT_DOWNLOADS[PUGIXML]="https://github.com/zeux/pugixml/releases/download/v${PROJECT_VERSIONS[PUGIXML]}/pugixml-${PROJECT_VERSIONS[PUGIXML]}.tar.gz;pugixml-${PROJECT_VERSIONS[PUGIXML]}.tar.gz;3rd_party"
+PROJECT_DOWNLOADS[ROCKSDB]="https://github.com/facebook/rocksdb/archive/refs/tags/v${PROJECT_VERSIONS[ROCKSDB]}.tar.gz;rocksdb-${PROJECT_VERSIONS[ROCKSDB]}.tar.gz;3rd_party"
 PROJECT_DOWNLOADS[SPDLOG]="https://github.com/gabime/spdlog/archive/refs/tags/v${PROJECT_VERSIONS[SPDLOG]}.tar.gz;spdlog-${PROJECT_VERSIONS[SPDLOG]}.tar.gz;3rd_party"
 PROJECT_DOWNLOADS[SQLITE]="https://www.sqlite.org/2024/sqlite-amalgamation-${PROJECT_VERSIONS[SQLITE]}.zip;sqlite-amalgamation-${PROJECT_VERSIONS[SQLITE]}.zip;3rd_party"
 PROJECT_DOWNLOADS[STDUUID]="https://github.com/mariusbancila/stduuid/archive/refs/tags/v${PROJECT_VERSIONS[STDUUID]}.tar.gz;stduuid-${PROJECT_VERSIONS[STDUUID]}.tar.gz;3rd_party"

support/3rd_party/rocksdb-9.7.4.tar.gz.sha256

@@ -0,0 +1 @@
+9b810c81731835fda0d4bbdb51d3199d901fa4395733ab63752d297da84c5a47 *rocksdb-9.7.4.tar.gz