From 12a373324e5a4a8aaf4735ff118c16115f672bde Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Wed, 11 Dec 2024 14:02:23 -0600 Subject: [PATCH] RocksDB implementations should be transactional #24 --- .../include/utils/db/rocksdb/utils.hpp | 33 ----------- .../src/db/impl/rdb_file_mgr_db.cpp | 36 +++++++++++- .../librepertory/src/db/impl/rdb_meta_db.cpp | 36 +++++++++++- .../src/utils/db/rocksdb/utils.cpp | 55 ------------------- 4 files changed, 68 insertions(+), 92 deletions(-) delete mode 100644 repertory/librepertory/include/utils/db/rocksdb/utils.hpp delete mode 100644 repertory/librepertory/src/utils/db/rocksdb/utils.cpp diff --git a/repertory/librepertory/include/utils/db/rocksdb/utils.hpp b/repertory/librepertory/include/utils/db/rocksdb/utils.hpp deleted file mode 100644 index 5a5ef6f0..00000000 --- a/repertory/librepertory/include/utils/db/rocksdb/utils.hpp +++ /dev/null @@ -1,33 +0,0 @@ -/* - Copyright <2018-2024> - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. -*/ -#ifndef REPERTORY_INCLUDE_UTILS_DB_ROCKSDB_UTILS_HPP_ -#define REPERTORY_INCLUDE_UTILS_DB_ROCKSDB_UTILS_HPP_ - -namespace repertory::utils::db::rocksdb { -[[nodiscard]] auto -create_rocksdb(const repertory::app_config &cfg, const std::string &name, - const std::vector &families, - std::vector &handles, bool clear) - -> std::unique_ptr; -} // namespace repertory::utils::db::rocksdb - -#endif // REPERTORY_INCLUDE_UTILS_DB_ROCKSDB_UTILS_HPP_ diff --git a/repertory/librepertory/src/db/impl/rdb_file_mgr_db.cpp b/repertory/librepertory/src/db/impl/rdb_file_mgr_db.cpp index 971f012e..76fdad6a 100644 --- a/repertory/librepertory/src/db/impl/rdb_file_mgr_db.cpp +++ b/repertory/librepertory/src/db/impl/rdb_file_mgr_db.cpp @@ -30,6 +30,39 @@ #include "utils/path.hpp" #include "utils/string.hpp" +namespace { +[[nodiscard]] auto +create_rocksdb(const repertory::app_config &cfg, const std::string &name, + const std::vector &families, + std::vector &handles, bool clear) + -> std::unique_ptr { + REPERTORY_USES_FUNCTION_NAME(); + + auto path = repertory::utils::path::combine(cfg.get_data_directory(), {name}); + if (clear && + not repertory::utils::file::directory{path}.remove_recursively()) { + repertory::utils::error::raise_error( + function_name, "failed to remove " + name + " db|" + path); + } + + rocksdb::Options options{}; + options.create_if_missing = true; + options.create_missing_column_families = true; + options.db_log_dir = cfg.get_log_directory(); + options.keep_log_file_num = 10; + + rocksdb::TransactionDB *ptr{}; + auto status = rocksdb::TransactionDB::Open( + options, rocksdb::TransactionDBOptions{}, path, families, &handles, &ptr); + if (not status.ok()) { + repertory::utils::error::raise_error(function_name, status.ToString()); + throw repertory::startup_exception(status.ToString()); + } + + return std::unique_ptr(ptr); +} +} // namespace + namespace repertory { rdb_file_mgr_db::rdb_file_mgr_db(const app_config &cfg) : cfg_(cfg) { create_or_open(false); @@ -47,8 +80,7 @@ void rdb_file_mgr_db::create_or_open(bool clear) { families.emplace_back("upload", rocksdb::ColumnFamilyOptions()); auto handles = std::vector(); - db_ = utils::db::rocksdb::create_rocksdb(cfg_, "file_mgr", families, handles, - clear); + db_ = create_rocksdb(cfg_, "file_mgr", families, handles, clear); std::size_t idx{}; resume_family_ = handles[idx++]; diff --git a/repertory/librepertory/src/db/impl/rdb_meta_db.cpp b/repertory/librepertory/src/db/impl/rdb_meta_db.cpp index d1097295..81352662 100644 --- a/repertory/librepertory/src/db/impl/rdb_meta_db.cpp +++ b/repertory/librepertory/src/db/impl/rdb_meta_db.cpp @@ -29,6 +29,39 @@ #include "utils/path.hpp" #include "utils/string.hpp" +namespace { +[[nodiscard]] auto +create_rocksdb(const repertory::app_config &cfg, const std::string &name, + const std::vector &families, + std::vector &handles, bool clear) + -> std::unique_ptr { + REPERTORY_USES_FUNCTION_NAME(); + + auto path = repertory::utils::path::combine(cfg.get_data_directory(), {name}); + if (clear && + not repertory::utils::file::directory{path}.remove_recursively()) { + repertory::utils::error::raise_error( + function_name, "failed to remove " + name + " db|" + path); + } + + rocksdb::Options options{}; + options.create_if_missing = true; + options.create_missing_column_families = true; + options.db_log_dir = cfg.get_log_directory(); + options.keep_log_file_num = 10; + + rocksdb::TransactionDB *ptr{}; + auto status = rocksdb::TransactionDB::Open( + options, rocksdb::TransactionDBOptions{}, path, families, &handles, &ptr); + if (not status.ok()) { + repertory::utils::error::raise_error(function_name, status.ToString()); + throw repertory::startup_exception(status.ToString()); + } + + return std::unique_ptr(ptr); +} +} // namespace + namespace repertory { rdb_meta_db::rdb_meta_db(const app_config &cfg) : cfg_(cfg) { create_or_open(false); @@ -47,8 +80,7 @@ void rdb_meta_db::create_or_open(bool clear) { families.emplace_back("source", rocksdb::ColumnFamilyOptions()); auto handles = std::vector(); - db_ = utils::db::rocksdb::create_rocksdb(cfg_, "provider_meta", families, - handles, clear); + db_ = create_rocksdb(cfg_, "provider_meta", families, handles, clear); std::size_t idx{}; default_family_ = handles[idx++]; diff --git a/repertory/librepertory/src/utils/db/rocksdb/utils.cpp b/repertory/librepertory/src/utils/db/rocksdb/utils.cpp deleted file mode 100644 index e41e02f0..00000000 --- a/repertory/librepertory/src/utils/db/rocksdb/utils.cpp +++ /dev/null @@ -1,55 +0,0 @@ -/* - Copyright <2018-2024> - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. -*/ -#include "utils/db/rocksdb/utils.hpp" - -namespace repertory::utils::db::rocksdb { -auto create_rocksdb( - const repertory::app_config &cfg, const std::string &name, - const std::vector &families, - std::vector &handles, bool clear) - -> std::unique_ptr { - REPERTORY_USES_FUNCTION_NAME(); - - auto path = repertory::utils::path::combine(cfg.get_data_directory(), {name}); - if (clear && - not repertory::utils::file::directory{path}.remove_recursively()) { - repertory::utils::error::raise_error( - function_name, "failed to remove " + name + " db|" + path); - } - - rocksdb::Options options{}; - options.create_if_missing = true; - options.create_missing_column_families = true; - options.db_log_dir = cfg.get_log_directory(); - options.keep_log_file_num = 10; - - rocksdb::TransactionDB *ptr{}; - auto status = rocksdb::TransactionDB::Open( - options, rocksdb::TransactionDBOptions{}, path, families, &handles, &ptr); - if (not status.ok()) { - repertory::utils::error::raise_error(function_name, status.ToString()); - throw repertory::startup_exception(status.ToString()); - } - - return std::unique_ptr(ptr); -} -} // namespace repertory::utils::db::rocksdb