BlockStorage/repertory
1
0
Fork 0
Code Issues 11 Pull Requests Releases 7 Wiki Activity

Merge branch 'development' of ssh://git.fifthgrid.com:3022/blockstorage/repertory into development

Browse Source
This commit is contained in:
Scott E. Graves
2024-01-29 11:18:24 -06:00
parent 5ec91e2d20 9b453327a8
commit 8c548bad92
15 changed files with 267797 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

88
src/comm/curl/requests/http_post.cpp Normal file
View File

@@ -0,0 +1,88 @@
<<<<<<< HEAD:include/utils/rocksdb_utils.hpp
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#ifndef INCLUDE_UTILS_ROCKSDB_UTILS_HPP_
#define INCLUDE_UTILS_ROCKSDB_UTILS_HPP_
namespace repertory {
class app_config;
namespace utils::db {
void create_rocksdb(const app_config &config, const std::string &name,
std::unique_ptr<rocksdb::DB> &db);
void create_rocksdb(
const app_config &config, const std::string &name,
const std::vector<rocksdb::ColumnFamilyDescriptor> &families,
std::vector<rocksdb::ColumnFamilyHandle *> &handles,
std::unique_ptr<rocksdb::DB> &db);
} // namespace utils::db
} // namespace repertory
#endif // INCLUDE_UTILS_ROCKSDB_UTILS_HPP_
=======
/*
Copyright <2018-2024> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "comm/curl/requests/http_post.hpp"
namespace repertory::curl::requests {
http_post::~http_post() {
if (headers != nullptr) {
curl_slist_free_all(headers);
}
}
auto http_post::set_method(CURL *curl, stop_type & /*stop_requested*/) const
-> bool {
curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "POST");
if (json.has_value()) {
headers = curl_slist_append(headers, "content-type: application/json");
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
json_str = json->dump();
curl_easy_setopt(curl, CURLOPT_POSTFIELDS, json_str->c_str());
curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, -1L);
}
return true;
}
} // namespace repertory::curl::requests
>>>>>>> 9b453327a81e182a9ec87ef1aae13169516436e0:src/comm/curl/requests/http_post.cpp

98
src/database/db_insert.cpp Normal file
View File

@@ -0,0 +1,98 @@
/*
Copyright <2018-2024> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "database/db_insert.hpp"
namespace repertory::db {
auto db_insert::column_value(std::string column_name, db_types_t value)
-> db_insert & {
context_->values[column_name] = value;
return *this;
}
auto db_insert::dump() const -> std::string {
std::stringstream query;
query << "INSERT ";
if (context_->or_replace) {
query << "OR REPLACE ";
}
query << "INTO \"" << context_->table_name << "\" (";
for (std::int32_t idx = 0;
idx < static_cast<std::int32_t>(context_->values.size()); idx++) {
if (idx > 0) {
query << ", ";
}
query << '"' << std::next(context_->values.begin(), idx)->first << '"';
}
query << ") VALUES (";
for (std::int32_t idx = 0;
idx < static_cast<std::int32_t>(context_->values.size()); idx++) {
if (idx > 0) {
query << ", ";
}
query << "?" << (idx + 1);
}
query << ");";
return query.str();
}
auto db_insert::go() const -> db_result<context> {
constexpr const auto *function_name = static_cast<const char *>(__FUNCTION__);
sqlite3_stmt *stmt_ptr{nullptr};
auto query_str = dump();
auto res = sqlite3_prepare_v2(&context_->db3, query_str.c_str(), -1,
&stmt_ptr, nullptr);
if (res != SQLITE_OK) {
utils::error::raise_error(function_name, "failed to prepare|" +
std::to_string(res) + '|' +
sqlite3_errstr(res));
return {context_, res};
}
context_->stmt.reset(stmt_ptr);
for (std::int32_t idx = 0;
idx < static_cast<std::int32_t>(context_->values.size()); idx++) {
res = std::visit(
overloaded{
[this, &idx](std::int64_t data) -> std::int32_t {
return sqlite3_bind_int64(context_->stmt.get(), idx + 1, data);
},
[this, &idx](const std::string &data) -> std::int32_t {
return sqlite3_bind_text(context_->stmt.get(), idx + 1,
data.c_str(), -1, nullptr);
},
},
std::next(context_->values.begin(), idx)->second);
if (res != SQLITE_OK) {
utils::error::raise_error(function_name, "failed to bind|" +
std::to_string(res) + '|' +
sqlite3_errstr(res));
return {context_, res};
}
}
return {context_, res};
}
} // namespace repertory::db

175
src/database/db_select.cpp Normal file
View File

@@ -0,0 +1,175 @@
/*
Copyright <2018-2024> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "database/db_select.hpp"
namespace repertory::db {
auto db_select::column(std::string column_name) -> db_select & {
if (context_->delete_query) {
throw std::runtime_error("columns may not be specified for delete");
}
context_->columns.push_back(column_name);
return *this;
}
auto db_select::count(std::string column_name, std::string as_column_name)
-> db_select & {
context_->count_columns[column_name] = as_column_name;
return *this;
}
auto db_select::delete_query() -> db_select & {
if (not context_->columns.empty()) {
throw std::runtime_error("columns must be empty for delete");
}
context_->delete_query = true;
return *this;
}
auto db_select::dump() const -> std::string {
std::stringstream query;
query << (context_->delete_query ? "DELETE " : "SELECT ");
if (not context_->delete_query) {
bool has_column{false};
if (context_->columns.empty()) {
if (context_->count_columns.empty()) {
query << "*";
has_column = true;
}
} else {
has_column = not context_->columns.empty();
for (std::size_t idx = 0U; idx < context_->columns.size(); idx++) {
if (idx > 0U) {
query << ", ";
}
query << context_->columns.at(idx);
}
}
for (std::int32_t idx = 0U;
idx < static_cast<std::int32_t>(context_->count_columns.size());
idx++) {
if (has_column || idx > 0) {
query << ", ";
}
query << "COUNT(\"";
auto &count_column = *std::next(context_->count_columns.begin(), idx);
query << count_column.first << "\") AS \"" << count_column.second << '"';
}
}
query << " FROM \"" << context_->table_name << "\"";
if (not context_->ands.empty()) {
query << " WHERE (";
for (std::int32_t idx = 0;
idx < static_cast<std::int32_t>(context_->ands.size()); idx++) {
if (idx > 0) {
query << " AND ";
}
auto &item = context_->ands.at(static_cast<std::size_t>(idx));
query << '"' << item.column_name << '"' << item.op_type << "?"
<< (idx + 1);
}
query << ")";
}
if (not context_->delete_query) {
if (context_->order_by.has_value()) {
query << " ORDER BY \"" << context_->order_by.value().first << "\" ";
query << (context_->order_by.value().second ? "ASC" : "DESC");
}
if (context_->limit.has_value()) {
query << " LIMIT " << context_->limit.value();
}
}
query << ';';
return query.str();
}
auto db_select::go() const -> db_result<context> {
constexpr const auto *function_name = static_cast<const char *>(__FUNCTION__);
sqlite3_stmt *stmt_ptr{nullptr};
auto query_str = dump();
auto res = sqlite3_prepare_v2(&context_->db3, query_str.c_str(), -1,
&stmt_ptr, nullptr);
if (res != SQLITE_OK) {
utils::error::raise_error(function_name,
"failed to prepare|" + std::to_string(res) + '|' +
sqlite3_errstr(res) + '|' + query_str);
return {context_, res};
}
context_->stmt.reset(stmt_ptr);
for (std::int32_t idx = 0;
idx < static_cast<std::int32_t>(context_->ands.size()); idx++) {
res = std::visit(
overloaded{
[this, &idx](std::int64_t data) -> std::int32_t {
return sqlite3_bind_int64(context_->stmt.get(), idx + 1, data);
},
[this, &idx](const std::string &data) -> std::int32_t {
return sqlite3_bind_text(context_->stmt.get(), idx + 1,
data.c_str(), -1, nullptr);
},
},
context_->ands.at(static_cast<std::size_t>(idx)).value);
if (res != SQLITE_OK) {
utils::error::raise_error(function_name,
"failed to bind|" + std::to_string(res) + '|' +
sqlite3_errstr(res) + '|' + query_str);
return {context_, res};
}
}
return {context_, res};
}
auto db_select::limit(std::int32_t value) -> db_select & {
if (context_->delete_query) {
throw std::runtime_error("limit may not be specified for delete");
}
context_->limit = value;
return *this;
}
auto db_select::order_by(std::string column_name, bool ascending)
-> db_select & {
if (context_->delete_query) {
throw std::runtime_error("order_by may not be specified for delete");
}
context_->order_by = {column_name, ascending};
return *this;
}
auto db_select::where(std::string column_name) const -> db_where {
return db_where{context_, column_name};
}
} // namespace repertory::db

315
src/providers/meta_db.cpp Normal file
View File

@@ -0,0 +1,315 @@
/*
Copyright <2018-2024> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "providers/meta_db.hpp"
#include "app_config.hpp"
#include "database/db_common.hpp"
#include "database/db_insert.hpp"
#include "database/db_select.hpp"
#include "utils/error_utils.hpp"
#include "utils/path_utils.hpp"
#include "utils/string_utils.hpp"
namespace repertory {
meta_db::meta_db(const app_config &cfg) {
constexpr const auto *function_name = static_cast<const char *>(__FUNCTION__);
auto db_path = utils::path::combine(cfg.get_data_directory(), {"meta.db3"});
sqlite3 *db3{nullptr};
auto res =
sqlite3_open_v2(db_path.c_str(), &db3,
SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, nullptr);
if (res != SQLITE_OK) {
utils::error::raise_error(function_name, "failed to open db|" + db_path +
'|' + std::to_string(res) +
'|' + sqlite3_errstr(res));
return;
}
db_.reset(db3);
const auto *create = "CREATE TABLE IF NOT EXISTS "
"meta "
"("
"api_path TEXT PRIMARY KEY ASC, "
"data TEXT, "
"directory INTEGER, "
"pinned INTEGER, "
"source_path TEXT"
");";
std::string err;
if (not db::execute_sql(*db_, create, err)) {
utils::error::raise_error(function_name,
"failed to create db|" + db_path + '|' + err);
db_.reset();
return;
}
db::set_journal_mode(*db_);
}
meta_db::~meta_db() { db_.reset(); }
auto meta_db::get_api_path(const std::string &source_path,
std::string &api_path) -> api_error {
auto result = db::db_select{*db_, table_name}
.column("api_path")
.where("source_path")
.equals(source_path)
.limit(1)
.go();
std::optional<db::db_select::row> row;
if (result.get_row(row) && row.has_value()) {
api_path = row->get_column("api_path").get_value<std::string>();
return api_error::success;
}
return api_error::item_not_found;
}
auto meta_db::get_api_path_list() -> std::vector<std::string> {
std::vector<std::string> ret{};
auto result = db::db_select{*db_, table_name}.column("api_path").go();
while (result.has_row()) {
std::optional<db::db_select::row> row;
if (result.get_row(row) && row.has_value()) {
ret.push_back(row->get_column("api_path").get_value<std::string>());
}
}
return ret;
}
auto meta_db::get_item_meta(const std::string &api_path, api_meta_map &meta)
-> api_error {
constexpr const auto *function_name = static_cast<const char *>(__FUNCTION__);
auto result = db::db_select{*db_, table_name}
.column("*")
.where("api_path")
.equals(api_path)
.limit(1)
.go();
if (not result.has_row()) {
return api_error::item_not_found;
}
try {
std::optional<db::db_select::row> row;
if (result.get_row(row) && row.has_value()) {
meta = json::parse(row->get_column("data").get_value<std::string>())
.get<api_meta_map>();
meta[META_DIRECTORY] = utils::string::from_bool(
row->get_column("directory").get_value<std::int64_t>() == 1);
meta[META_PINNED] = utils::string::from_bool(
row->get_column("pinned").get_value<std::int64_t>() == 1);
meta[META_SOURCE] =
row->get_column("source_path").get_value<std::string>();
return api_error::success;
}
return api_error::item_not_found;
} catch (const std::exception &e) {
utils::error::raise_api_path_error(function_name, api_path, e,
"failed to get item meta");
}
return api_error::error;
}
auto meta_db::get_item_meta(const std::string &api_path, const std::string &key,
std::string &value) const -> api_error {
constexpr const auto *function_name = static_cast<const char *>(__FUNCTION__);
auto result = db::db_select{*db_, table_name}
.column("*")
.where("api_path")
.equals(api_path)
.limit(1)
.go();
if (not result.has_row()) {
return api_error::item_not_found;
}
try {
std::optional<db::db_select::row> row;
if (result.get_row(row) && row.has_value()) {
value =
key == META_SOURCE
? row->get_column("source_path").get_value<std::string>()
: key == META_PINNED
? utils::string::from_bool(
row->get_column("pinned").get_value<std::int64_t>() == 1)
: key == META_DIRECTORY
? utils::string::from_bool(
row->get_column("directory").get_value<std::int64_t>() == 1)
: json::parse(
row->get_column("data").get_value<std::string>())[key]
.get<std::string>();
return api_error::success;
}
return api_error::item_not_found;
} catch (const std::exception &e) {
utils::error::raise_api_path_error(function_name, api_path, e,
"failed to get item meta");
}
return api_error::error;
}
auto meta_db::get_pinned_files() const -> std::vector<std::string> {
constexpr const auto *function_name = static_cast<const char *>(__FUNCTION__);
std::vector<std::string> ret{};
try {
auto result = db::db_select{*db_, table_name}
.column("api_path")
.where("pinned")
.equals(1)
.go();
while (result.has_row()) {
std::optional<db::db_select::row> row;
if (result.get_row(row) && row.has_value()) {
ret.emplace_back(row->get_column("api_path").get_value<std::string>());
}
}
} catch (const std::exception &e) {
utils::error::raise_error(function_name, e, "failed to get pinned files");
}
return ret;
}
auto meta_db::get_total_item_count() const -> std::uint64_t {
constexpr const auto *function_name = static_cast<const char *>(__FUNCTION__);
std::uint64_t ret{};
try {
auto result =
db::db_select{*db_, table_name}.count("api_path", "count").go();
std::optional<db::db_select::row> row;
if (result.get_row(row) && row.has_value()) {
ret = static_cast<std::uint64_t>(
row->get_column("count").get_value<std::int64_t>());
}
} catch (const std::exception &e) {
utils::error::raise_error(function_name, e,
"failed to get total item count");
}
return ret;
}
void meta_db::remove_api_path(const std::string &api_path) {
constexpr const auto *function_name = static_cast<const char *>(__FUNCTION__);
auto result = db::db_select{*db_, table_name}
.delete_query()
.where("api_path")
.equals(api_path)
.go();
if (not result.ok()) {
utils::error::raise_api_path_error(
function_name, api_path, result.get_error(), "failed to remove meta");
}
}
auto meta_db::remove_item_meta(const std::string &api_path,
const std::string &key) -> api_error {
api_meta_map meta{};
auto res = get_item_meta(api_path, meta);
if (res != api_error::success) {
return res;
}
meta.erase(key);
return update_item_meta(api_path, meta);
}
auto meta_db::rename_item_meta(const std::string &from_api_path,
const std::string &to_api_path) -> api_error {
api_meta_map meta{};
auto res = get_item_meta(from_api_path, meta);
if (res != api_error::success) {
return res;
}
remove_api_path(from_api_path);
return update_item_meta(to_api_path, meta);
}
auto meta_db::set_item_meta(const std::string &api_path, const std::string &key,
const std::string &value) -> api_error {
return set_item_meta(api_path, {{key, value}});
}
auto meta_db::set_item_meta(const std::string &api_path,
const api_meta_map &meta) -> api_error {
api_meta_map existing_meta{};
if (get_item_meta(api_path, existing_meta) != api_error::success) {
// TODO handle error
}
for (const auto &item : meta) {
existing_meta[item.first] = item.second;
}
return update_item_meta(api_path, existing_meta);
}
auto meta_db::update_item_meta(const std::string &api_path, api_meta_map meta)
-> api_error {
constexpr const auto *function_name = static_cast<const char *>(__FUNCTION__);
auto directory = utils::string::to_bool(meta[META_DIRECTORY]);
auto pinned = utils::string::to_bool(meta[META_PINNED]);
auto source_path = meta[META_SOURCE];
meta.erase(META_DIRECTORY);
meta.erase(META_PINNED);
meta.erase(META_SOURCE);
auto result = db::db_insert{*db_, table_name}
.or_replace()
.column_value("api_path", api_path)
.column_value("data", nlohmann::json(meta).dump())
.column_value("directory", directory ? 1 : 0)
.column_value("pinned", pinned ? 1 : 0)
.column_value("source_path", source_path)
.go();
if (not result.ok()) {
utils::error::raise_api_path_error(function_name, api_path,
result.get_error(),
"failed to update item meta");
return api_error::error;
}
return api_error::success;
}
} // namespace repertory