refactor create rocksdb
This commit is contained in:
parent
f1ab604fe9
commit
95b9464c9f
@ -24,15 +24,25 @@
|
|||||||
|
|
||||||
#include "types/repertory.hpp"
|
#include "types/repertory.hpp"
|
||||||
|
|
||||||
namespace repertory::utils {
|
namespace repertory {
|
||||||
|
class app_config;
|
||||||
|
|
||||||
|
namespace utils {
|
||||||
void calculate_allocation_size(bool directory, std::uint64_t file_size,
|
void calculate_allocation_size(bool directory, std::uint64_t file_size,
|
||||||
UINT64 allocation_size,
|
UINT64 allocation_size,
|
||||||
std::string &allocation_meta_size);
|
std::string &allocation_meta_size);
|
||||||
|
|
||||||
[[nodiscard]] auto
|
[[nodiscard]] auto
|
||||||
create_volume_label(const provider_type &prov) -> std::string;
|
create_rocksdb(const app_config &cfg, const std::string &name,
|
||||||
|
const std::vector<rocksdb::ColumnFamilyDescriptor> &families,
|
||||||
|
std::vector<rocksdb::ColumnFamilyHandle *> &handles, bool clear)
|
||||||
|
-> std::unique_ptr<rocksdb::TransactionDB>;
|
||||||
|
|
||||||
|
[[nodiscard]] auto create_volume_label(const provider_type &prov)
|
||||||
|
-> std::string;
|
||||||
|
|
||||||
[[nodiscard]] auto get_attributes_from_meta(const api_meta_map &meta) -> DWORD;
|
[[nodiscard]] auto get_attributes_from_meta(const api_meta_map &meta) -> DWORD;
|
||||||
} // namespace repertory::utils
|
} // namespace utils
|
||||||
|
} // namespace repertory
|
||||||
|
|
||||||
#endif // REPERTORY_INCLUDE_UTILS_UTILS_HPP_
|
#endif // REPERTORY_INCLUDE_UTILS_UTILS_HPP_
|
||||||
|
@ -28,46 +28,7 @@
|
|||||||
#include "utils/file.hpp"
|
#include "utils/file.hpp"
|
||||||
#include "utils/path.hpp"
|
#include "utils/path.hpp"
|
||||||
#include "utils/string.hpp"
|
#include "utils/string.hpp"
|
||||||
|
#include "utils/utils.hpp"
|
||||||
namespace {
|
|
||||||
[[nodiscard]] auto
|
|
||||||
create_rocksdb(const repertory::app_config &cfg, const std::string &name,
|
|
||||||
const std::vector<rocksdb::ColumnFamilyDescriptor> &families,
|
|
||||||
std::vector<rocksdb::ColumnFamilyHandle *> &handles, bool clear)
|
|
||||||
-> std::unique_ptr<rocksdb::TransactionDB> {
|
|
||||||
REPERTORY_USES_FUNCTION_NAME();
|
|
||||||
|
|
||||||
auto db_dir =
|
|
||||||
repertory::utils::path::combine(cfg.get_data_directory(), {"db"});
|
|
||||||
if (not repertory::utils::file::directory{db_dir}.create_directory()) {
|
|
||||||
throw repertory::startup_exception(
|
|
||||||
fmt::format("failed to create db directory|", db_dir));
|
|
||||||
}
|
|
||||||
|
|
||||||
auto path = repertory::utils::path::combine(db_dir, {name});
|
|
||||||
if (clear &&
|
|
||||||
not repertory::utils::file::directory{path}.remove_recursively()) {
|
|
||||||
repertory::utils::error::raise_error(
|
|
||||||
function_name, "failed to remove " + name + " db|" + path);
|
|
||||||
}
|
|
||||||
|
|
||||||
rocksdb::Options options{};
|
|
||||||
options.create_if_missing = true;
|
|
||||||
options.create_missing_column_families = true;
|
|
||||||
options.db_log_dir = cfg.get_log_directory();
|
|
||||||
options.keep_log_file_num = 10;
|
|
||||||
|
|
||||||
rocksdb::TransactionDB *ptr{};
|
|
||||||
auto status = rocksdb::TransactionDB::Open(
|
|
||||||
options, rocksdb::TransactionDBOptions{}, path, families, &handles, &ptr);
|
|
||||||
if (not status.ok()) {
|
|
||||||
repertory::utils::error::raise_error(function_name, status.ToString());
|
|
||||||
throw repertory::startup_exception(status.ToString());
|
|
||||||
}
|
|
||||||
|
|
||||||
return std::unique_ptr<rocksdb::TransactionDB>(ptr);
|
|
||||||
}
|
|
||||||
} // namespace
|
|
||||||
|
|
||||||
namespace repertory {
|
namespace repertory {
|
||||||
rdb_file_db::rdb_file_db(const app_config &cfg) : cfg_(cfg) {
|
rdb_file_db::rdb_file_db(const app_config &cfg) : cfg_(cfg) {
|
||||||
@ -86,7 +47,7 @@ void rdb_file_db::create_or_open(bool clear) {
|
|||||||
families.emplace_back("file", rocksdb::ColumnFamilyOptions());
|
families.emplace_back("file", rocksdb::ColumnFamilyOptions());
|
||||||
|
|
||||||
auto handles = std::vector<rocksdb::ColumnFamilyHandle *>();
|
auto handles = std::vector<rocksdb::ColumnFamilyHandle *>();
|
||||||
db_ = create_rocksdb(cfg_, "file", families, handles, clear);
|
db_ = utils::create_rocksdb(cfg_, "file", families, handles, clear);
|
||||||
|
|
||||||
std::size_t idx{};
|
std::size_t idx{};
|
||||||
default_family_ = handles[idx++];
|
default_family_ = handles[idx++];
|
||||||
|
@ -28,46 +28,7 @@
|
|||||||
#include "utils/file.hpp"
|
#include "utils/file.hpp"
|
||||||
#include "utils/path.hpp"
|
#include "utils/path.hpp"
|
||||||
#include "utils/string.hpp"
|
#include "utils/string.hpp"
|
||||||
|
#include "utils/utils.hpp"
|
||||||
namespace {
|
|
||||||
[[nodiscard]] auto
|
|
||||||
create_rocksdb(const repertory::app_config &cfg, const std::string &name,
|
|
||||||
const std::vector<rocksdb::ColumnFamilyDescriptor> &families,
|
|
||||||
std::vector<rocksdb::ColumnFamilyHandle *> &handles, bool clear)
|
|
||||||
-> std::unique_ptr<rocksdb::TransactionDB> {
|
|
||||||
REPERTORY_USES_FUNCTION_NAME();
|
|
||||||
|
|
||||||
auto db_dir =
|
|
||||||
repertory::utils::path::combine(cfg.get_data_directory(), {"db"});
|
|
||||||
if (not repertory::utils::file::directory{db_dir}.create_directory()) {
|
|
||||||
throw repertory::startup_exception(
|
|
||||||
fmt::format("failed to create db directory|", db_dir));
|
|
||||||
}
|
|
||||||
|
|
||||||
auto path = repertory::utils::path::combine(db_dir, {name});
|
|
||||||
if (clear &&
|
|
||||||
not repertory::utils::file::directory{path}.remove_recursively()) {
|
|
||||||
repertory::utils::error::raise_error(
|
|
||||||
function_name, "failed to remove " + name + " db|" + path);
|
|
||||||
}
|
|
||||||
|
|
||||||
rocksdb::Options options{};
|
|
||||||
options.create_if_missing = true;
|
|
||||||
options.create_missing_column_families = true;
|
|
||||||
options.db_log_dir = cfg.get_log_directory();
|
|
||||||
options.keep_log_file_num = 10;
|
|
||||||
|
|
||||||
rocksdb::TransactionDB *ptr{};
|
|
||||||
auto status = rocksdb::TransactionDB::Open(
|
|
||||||
options, rocksdb::TransactionDBOptions{}, path, families, &handles, &ptr);
|
|
||||||
if (not status.ok()) {
|
|
||||||
repertory::utils::error::raise_error(function_name, status.ToString());
|
|
||||||
throw repertory::startup_exception(status.ToString());
|
|
||||||
}
|
|
||||||
|
|
||||||
return std::unique_ptr<rocksdb::TransactionDB>(ptr);
|
|
||||||
}
|
|
||||||
} // namespace
|
|
||||||
|
|
||||||
namespace repertory {
|
namespace repertory {
|
||||||
rdb_file_mgr_db::rdb_file_mgr_db(const app_config &cfg) : cfg_(cfg) {
|
rdb_file_mgr_db::rdb_file_mgr_db(const app_config &cfg) : cfg_(cfg) {
|
||||||
@ -86,7 +47,7 @@ void rdb_file_mgr_db::create_or_open(bool clear) {
|
|||||||
families.emplace_back("upload", rocksdb::ColumnFamilyOptions());
|
families.emplace_back("upload", rocksdb::ColumnFamilyOptions());
|
||||||
|
|
||||||
auto handles = std::vector<rocksdb::ColumnFamilyHandle *>();
|
auto handles = std::vector<rocksdb::ColumnFamilyHandle *>();
|
||||||
db_ = create_rocksdb(cfg_, "file_mgr", families, handles, clear);
|
db_ = utils::create_rocksdb(cfg_, "file_mgr", families, handles, clear);
|
||||||
|
|
||||||
std::size_t idx{};
|
std::size_t idx{};
|
||||||
resume_family_ = handles[idx++];
|
resume_family_ = handles[idx++];
|
||||||
|
@ -27,46 +27,7 @@
|
|||||||
#include "utils/file.hpp"
|
#include "utils/file.hpp"
|
||||||
#include "utils/path.hpp"
|
#include "utils/path.hpp"
|
||||||
#include "utils/string.hpp"
|
#include "utils/string.hpp"
|
||||||
|
#include "utils/utils.hpp"
|
||||||
namespace {
|
|
||||||
[[nodiscard]] auto
|
|
||||||
create_rocksdb(const repertory::app_config &cfg, const std::string &name,
|
|
||||||
const std::vector<rocksdb::ColumnFamilyDescriptor> &families,
|
|
||||||
std::vector<rocksdb::ColumnFamilyHandle *> &handles, bool clear)
|
|
||||||
-> std::unique_ptr<rocksdb::TransactionDB> {
|
|
||||||
REPERTORY_USES_FUNCTION_NAME();
|
|
||||||
|
|
||||||
auto db_dir =
|
|
||||||
repertory::utils::path::combine(cfg.get_data_directory(), {"db"});
|
|
||||||
if (not repertory::utils::file::directory{db_dir}.create_directory()) {
|
|
||||||
throw repertory::startup_exception(
|
|
||||||
fmt::format("failed to create db directory|", db_dir));
|
|
||||||
}
|
|
||||||
|
|
||||||
auto path = repertory::utils::path::combine(db_dir, {name});
|
|
||||||
if (clear &&
|
|
||||||
not repertory::utils::file::directory{path}.remove_recursively()) {
|
|
||||||
repertory::utils::error::raise_error(
|
|
||||||
function_name, "failed to remove " + name + " db|" + path);
|
|
||||||
}
|
|
||||||
|
|
||||||
rocksdb::Options options{};
|
|
||||||
options.create_if_missing = true;
|
|
||||||
options.create_missing_column_families = true;
|
|
||||||
options.db_log_dir = cfg.get_log_directory();
|
|
||||||
options.keep_log_file_num = 10;
|
|
||||||
|
|
||||||
rocksdb::TransactionDB *ptr{};
|
|
||||||
auto status = rocksdb::TransactionDB::Open(
|
|
||||||
options, rocksdb::TransactionDBOptions{}, path, families, &handles, &ptr);
|
|
||||||
if (not status.ok()) {
|
|
||||||
repertory::utils::error::raise_error(function_name, status.ToString());
|
|
||||||
throw repertory::startup_exception(status.ToString());
|
|
||||||
}
|
|
||||||
|
|
||||||
return std::unique_ptr<rocksdb::TransactionDB>(ptr);
|
|
||||||
}
|
|
||||||
} // namespace
|
|
||||||
|
|
||||||
namespace repertory {
|
namespace repertory {
|
||||||
rdb_meta_db::rdb_meta_db(const app_config &cfg) : cfg_(cfg) {
|
rdb_meta_db::rdb_meta_db(const app_config &cfg) : cfg_(cfg) {
|
||||||
@ -86,7 +47,7 @@ void rdb_meta_db::create_or_open(bool clear) {
|
|||||||
families.emplace_back("source", rocksdb::ColumnFamilyOptions());
|
families.emplace_back("source", rocksdb::ColumnFamilyOptions());
|
||||||
|
|
||||||
auto handles = std::vector<rocksdb::ColumnFamilyHandle *>();
|
auto handles = std::vector<rocksdb::ColumnFamilyHandle *>();
|
||||||
db_ = create_rocksdb(cfg_, "provider_meta", families, handles, clear);
|
db_ = utils::create_rocksdb(cfg_, "provider_meta", families, handles, clear);
|
||||||
|
|
||||||
std::size_t idx{};
|
std::size_t idx{};
|
||||||
default_family_ = handles[idx++];
|
default_family_ = handles[idx++];
|
||||||
|
@ -22,7 +22,10 @@
|
|||||||
#include "utils/utils.hpp"
|
#include "utils/utils.hpp"
|
||||||
|
|
||||||
#include "app_config.hpp"
|
#include "app_config.hpp"
|
||||||
|
#include "types/startup_exception.hpp"
|
||||||
#include "utils/common.hpp"
|
#include "utils/common.hpp"
|
||||||
|
#include "utils/file.hpp"
|
||||||
|
#include "utils/path.hpp"
|
||||||
#include "utils/string.hpp"
|
#include "utils/string.hpp"
|
||||||
|
|
||||||
namespace repertory::utils {
|
namespace repertory::utils {
|
||||||
@ -44,6 +47,42 @@ void calculate_allocation_size(bool directory, std::uint64_t file_size,
|
|||||||
allocation_meta_size = std::to_string(allocation_size);
|
allocation_meta_size = std::to_string(allocation_size);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
auto create_rocksdb(
|
||||||
|
const app_config &cfg, const std::string &name,
|
||||||
|
const std::vector<rocksdb::ColumnFamilyDescriptor> &families,
|
||||||
|
std::vector<rocksdb::ColumnFamilyHandle *> &handles, bool clear)
|
||||||
|
-> std::unique_ptr<rocksdb::TransactionDB> {
|
||||||
|
REPERTORY_USES_FUNCTION_NAME();
|
||||||
|
|
||||||
|
auto db_dir = utils::path::combine(cfg.get_data_directory(), {"db"});
|
||||||
|
if (not utils::file::directory{db_dir}.create_directory()) {
|
||||||
|
throw startup_exception(
|
||||||
|
fmt::format("failed to create db directory|", db_dir));
|
||||||
|
}
|
||||||
|
|
||||||
|
auto path = utils::path::combine(db_dir, {name});
|
||||||
|
if (clear && not utils::file::directory{path}.remove_recursively()) {
|
||||||
|
utils::error::raise_error(function_name,
|
||||||
|
"failed to remove " + name + " db|" + path);
|
||||||
|
}
|
||||||
|
|
||||||
|
rocksdb::Options options{};
|
||||||
|
options.create_if_missing = true;
|
||||||
|
options.create_missing_column_families = true;
|
||||||
|
options.db_log_dir = cfg.get_log_directory();
|
||||||
|
options.keep_log_file_num = 10;
|
||||||
|
|
||||||
|
rocksdb::TransactionDB *ptr{};
|
||||||
|
auto status = rocksdb::TransactionDB::Open(
|
||||||
|
options, rocksdb::TransactionDBOptions{}, path, families, &handles, &ptr);
|
||||||
|
if (not status.ok()) {
|
||||||
|
throw startup_exception(fmt::format("failed to open rocksdb|path{}|error{}",
|
||||||
|
path, status.ToString()));
|
||||||
|
}
|
||||||
|
|
||||||
|
return std::unique_ptr<rocksdb::TransactionDB>(ptr);
|
||||||
|
}
|
||||||
|
|
||||||
auto create_volume_label(const provider_type &prov) -> std::string {
|
auto create_volume_label(const provider_type &prov) -> std::string {
|
||||||
return "repertory_" + app_config::get_provider_name(prov);
|
return "repertory_" + app_config::get_provider_name(prov);
|
||||||
}
|
}
|
||||||
|
Loading…
x
Reference in New Issue
Block a user