BlockStorage/repertory
1
0
Fork 0
Code Issues 8 Pull Requests Releases 6 Wiki Activity

refactor create rocksdb

Browse Source
This commit is contained in:
Scott E. Graves 2024-12-18 12:32:47 -06:00
parent f1ab604fe9
commit 95b9464c9f
5 changed files with 58 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
repertory/librepertory/include/utils/utils.hpp
View File

@ -24,15 +24,25 @@
#include "types/repertory.hpp"
namespace repertory::utils {
namespace repertory {
class app_config;
namespace utils {
void calculate_allocation_size(bool directory, std::uint64_t file_size,
UINT64 allocation_size,
std::string &allocation_meta_size);
[[nodiscard]] auto
create_volume_label(const provider_type &prov) -> std::string;
create_rocksdb(const app_config &cfg, const std::string &name,
const std::vector<rocksdb::ColumnFamilyDescriptor> &families,
std::vector<rocksdb::ColumnFamilyHandle *> &handles, bool clear)
-> std::unique_ptr<rocksdb::TransactionDB>;
[[nodiscard]] auto create_volume_label(const provider_type &prov)
-> std::string;
[[nodiscard]] auto get_attributes_from_meta(const api_meta_map &meta) -> DWORD;
} // namespace repertory::utils
} // namespace utils
} // namespace repertory
#endif // REPERTORY_INCLUDE_UTILS_UTILS_HPP_

43
repertory/librepertory/src/db/impl/rdb_file_db.cpp
View File

@ -28,46 +28,7 @@
#include "utils/file.hpp"
#include "utils/path.hpp"
#include "utils/string.hpp"
namespace {
[[nodiscard]] auto
create_rocksdb(const repertory::app_config &cfg, const std::string &name,
const std::vector<rocksdb::ColumnFamilyDescriptor> &families,
std::vector<rocksdb::ColumnFamilyHandle *> &handles, bool clear)
-> std::unique_ptr<rocksdb::TransactionDB> {
REPERTORY_USES_FUNCTION_NAME();
auto db_dir =
repertory::utils::path::combine(cfg.get_data_directory(), {"db"});
if (not repertory::utils::file::directory{db_dir}.create_directory()) {
throw repertory::startup_exception(
fmt::format("failed to create db directory|", db_dir));
}
auto path = repertory::utils::path::combine(db_dir, {name});
if (clear &&
not repertory::utils::file::directory{path}.remove_recursively()) {
repertory::utils::error::raise_error(
function_name, "failed to remove " + name + " db|" + path);
}
rocksdb::Options options{};
options.create_if_missing = true;
options.create_missing_column_families = true;
options.db_log_dir = cfg.get_log_directory();
options.keep_log_file_num = 10;
rocksdb::TransactionDB *ptr{};
auto status = rocksdb::TransactionDB::Open(
options, rocksdb::TransactionDBOptions{}, path, families, &handles, &ptr);
if (not status.ok()) {
repertory::utils::error::raise_error(function_name, status.ToString());
throw repertory::startup_exception(status.ToString());
}
return std::unique_ptr<rocksdb::TransactionDB>(ptr);
}
} // namespace
#include "utils/utils.hpp"
namespace repertory {
rdb_file_db::rdb_file_db(const app_config &cfg) : cfg_(cfg) {
@ -86,7 +47,7 @@ void rdb_file_db::create_or_open(bool clear) {
families.emplace_back("file", rocksdb::ColumnFamilyOptions());
auto handles = std::vector<rocksdb::ColumnFamilyHandle *>();
db_ = create_rocksdb(cfg_, "file", families, handles, clear);
db_ = utils::create_rocksdb(cfg_, "file", families, handles, clear);
std::size_t idx{};
default_family_ = handles[idx++];

43
repertory/librepertory/src/db/impl/rdb_file_mgr_db.cpp
View File

@ -28,46 +28,7 @@
#include "utils/file.hpp"
#include "utils/path.hpp"
#include "utils/string.hpp"
namespace {
[[nodiscard]] auto
create_rocksdb(const repertory::app_config &cfg, const std::string &name,
const std::vector<rocksdb::ColumnFamilyDescriptor> &families,
std::vector<rocksdb::ColumnFamilyHandle *> &handles, bool clear)
-> std::unique_ptr<rocksdb::TransactionDB> {
REPERTORY_USES_FUNCTION_NAME();
auto db_dir =
repertory::utils::path::combine(cfg.get_data_directory(), {"db"});
if (not repertory::utils::file::directory{db_dir}.create_directory()) {
throw repertory::startup_exception(
fmt::format("failed to create db directory|", db_dir));
}
auto path = repertory::utils::path::combine(db_dir, {name});
if (clear &&
not repertory::utils::file::directory{path}.remove_recursively()) {
repertory::utils::error::raise_error(
function_name, "failed to remove " + name + " db|" + path);
}
rocksdb::Options options{};
options.create_if_missing = true;
options.create_missing_column_families = true;
options.db_log_dir = cfg.get_log_directory();
options.keep_log_file_num = 10;
rocksdb::TransactionDB *ptr{};
auto status = rocksdb::TransactionDB::Open(
options, rocksdb::TransactionDBOptions{}, path, families, &handles, &ptr);
if (not status.ok()) {
repertory::utils::error::raise_error(function_name, status.ToString());
throw repertory::startup_exception(status.ToString());
}
return std::unique_ptr<rocksdb::TransactionDB>(ptr);
}
} // namespace
#include "utils/utils.hpp"
namespace repertory {
rdb_file_mgr_db::rdb_file_mgr_db(const app_config &cfg) : cfg_(cfg) {
@ -86,7 +47,7 @@ void rdb_file_mgr_db::create_or_open(bool clear) {
families.emplace_back("upload", rocksdb::ColumnFamilyOptions());
auto handles = std::vector<rocksdb::ColumnFamilyHandle *>();
db_ = create_rocksdb(cfg_, "file_mgr", families, handles, clear);
db_ = utils::create_rocksdb(cfg_, "file_mgr", families, handles, clear);
std::size_t idx{};
resume_family_ = handles[idx++];

43
repertory/librepertory/src/db/impl/rdb_meta_db.cpp
View File

@ -27,46 +27,7 @@
#include "utils/file.hpp"
#include "utils/path.hpp"
#include "utils/string.hpp"
namespace {
[[nodiscard]] auto
create_rocksdb(const repertory::app_config &cfg, const std::string &name,
const std::vector<rocksdb::ColumnFamilyDescriptor> &families,
std::vector<rocksdb::ColumnFamilyHandle *> &handles, bool clear)
-> std::unique_ptr<rocksdb::TransactionDB> {
REPERTORY_USES_FUNCTION_NAME();
auto db_dir =
repertory::utils::path::combine(cfg.get_data_directory(), {"db"});
if (not repertory::utils::file::directory{db_dir}.create_directory()) {
throw repertory::startup_exception(
fmt::format("failed to create db directory|", db_dir));
}
auto path = repertory::utils::path::combine(db_dir, {name});
if (clear &&
not repertory::utils::file::directory{path}.remove_recursively()) {
repertory::utils::error::raise_error(
function_name, "failed to remove " + name + " db|" + path);
}
rocksdb::Options options{};
options.create_if_missing = true;
options.create_missing_column_families = true;
options.db_log_dir = cfg.get_log_directory();
options.keep_log_file_num = 10;
rocksdb::TransactionDB *ptr{};
auto status = rocksdb::TransactionDB::Open(
options, rocksdb::TransactionDBOptions{}, path, families, &handles, &ptr);
if (not status.ok()) {
repertory::utils::error::raise_error(function_name, status.ToString());
throw repertory::startup_exception(status.ToString());
}
return std::unique_ptr<rocksdb::TransactionDB>(ptr);
}
} // namespace
#include "utils/utils.hpp"
namespace repertory {
rdb_meta_db::rdb_meta_db(const app_config &cfg) : cfg_(cfg) {
@ -86,7 +47,7 @@ void rdb_meta_db::create_or_open(bool clear) {
families.emplace_back("source", rocksdb::ColumnFamilyOptions());
auto handles = std::vector<rocksdb::ColumnFamilyHandle *>();
db_ = create_rocksdb(cfg_, "provider_meta", families, handles, clear);
db_ = utils::create_rocksdb(cfg_, "provider_meta", families, handles, clear);
std::size_t idx{};
default_family_ = handles[idx++];

39
repertory/librepertory/src/utils/utils.cpp
View File

@ -22,7 +22,10 @@
#include "utils/utils.hpp"
#include "app_config.hpp"
#include "types/startup_exception.hpp"
#include "utils/common.hpp"
#include "utils/file.hpp"
#include "utils/path.hpp"
#include "utils/string.hpp"
namespace repertory::utils {
@ -44,6 +47,42 @@ void calculate_allocation_size(bool directory, std::uint64_t file_size,
allocation_meta_size = std::to_string(allocation_size);
}
auto create_rocksdb(
const app_config &cfg, const std::string &name,
const std::vector<rocksdb::ColumnFamilyDescriptor> &families,
std::vector<rocksdb::ColumnFamilyHandle *> &handles, bool clear)
-> std::unique_ptr<rocksdb::TransactionDB> {
REPERTORY_USES_FUNCTION_NAME();
auto db_dir = utils::path::combine(cfg.get_data_directory(), {"db"});
if (not utils::file::directory{db_dir}.create_directory()) {
throw startup_exception(
fmt::format("failed to create db directory|", db_dir));
}
auto path = utils::path::combine(db_dir, {name});
if (clear && not utils::file::directory{path}.remove_recursively()) {
utils::error::raise_error(function_name,
"failed to remove " + name + " db|" + path);
}
rocksdb::Options options{};
options.create_if_missing = true;
options.create_missing_column_families = true;
options.db_log_dir = cfg.get_log_directory();
options.keep_log_file_num = 10;
rocksdb::TransactionDB *ptr{};
auto status = rocksdb::TransactionDB::Open(
options, rocksdb::TransactionDBOptions{}, path, families, &handles, &ptr);
if (not status.ok()) {
throw startup_exception(fmt::format("failed to open rocksdb|path{}|error{}",
path, status.ToString()));
}
return std::unique_ptr<rocksdb::TransactionDB>(ptr);
}
auto create_volume_label(const provider_type &prov) -> std::string {
return "repertory_" + app_config::get_provider_name(prov);
}