From 3f5756f01cc311c6da4fc51a585601d4b3ed4725 Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Fri, 21 Feb 2025 08:44:21 -0600 Subject: [PATCH] cleanup database --- monitarr/libmonitarr/include/actions.hpp | 5 ++- monitarr/libmonitarr/include/data_db.hpp | 2 ++ monitarr/libmonitarr/src/actions.cpp | 30 +++++++++++++++-- monitarr/libmonitarr/src/data_db.cpp | 28 ++++++++++++--- monitarr/monitarr/src/run_cmd.cpp | 43 +++++++++++++----------- 5 files changed, 81 insertions(+), 27 deletions(-) diff --git a/monitarr/libmonitarr/include/actions.hpp b/monitarr/libmonitarr/include/actions.hpp index 136985b..2a5b859 100644 --- a/monitarr/libmonitarr/include/actions.hpp +++ b/monitarr/libmonitarr/include/actions.hpp @@ -32,13 +32,16 @@ class data_db; [[nodiscard]] auto create_client(const server_cfg &server) -> httplib::Client; [[nodiscard]] auto get_download(std::uint64_t record_id, - const server_cfg &server) + const server_cfg &server, + bool *is_not_found = nullptr) -> std::optional; [[nodiscard]] auto list_queue(const server_cfg &server) -> int; [[nodiscard]] auto load_config(std::string &cfg_file) -> app_config; +[[nodiscard]] auto load_db(std::string_view log_dir) -> data_db; + void remove_stalled(std::string_view download_id, std::string_view title, std::uint64_t episode_id, std::uint64_t movie_id, const server_cfg &server, data_db *state_db = nullptr); diff --git a/monitarr/libmonitarr/include/data_db.hpp b/monitarr/libmonitarr/include/data_db.hpp index 5355ddf..a6cec6a 100644 --- a/monitarr/libmonitarr/include/data_db.hpp +++ b/monitarr/libmonitarr/include/data_db.hpp @@ -56,6 +56,8 @@ private: std::function action) -> bool; public: + [[nodiscard]] auto all(std::string_view server_id) -> std::vector; + void close(); [[nodiscard]] auto get(std::string_view download_id) const diff --git a/monitarr/libmonitarr/src/actions.cpp b/monitarr/libmonitarr/src/actions.cpp index 403b199..ffce0c5 100644 --- a/monitarr/libmonitarr/src/actions.cpp +++ b/monitarr/libmonitarr/src/actions.cpp @@ -30,7 +30,6 @@ #include "utils/string.hpp" #include "utils/unix.hpp" #include "utils/windows.hpp" -#include using namespace std::literals::chrono_literals; @@ -48,8 +47,8 @@ auto create_client(const server_cfg &server) -> httplib::Client { return cli; } -auto get_download(std::uint64_t record_id, const server_cfg &server) - -> std::optional { +auto get_download(std::uint64_t record_id, const server_cfg &server, + bool *is_not_found) -> std::optional { MONITARR_USES_FUNCTION_NAME(); auto cli = create_client(server); @@ -74,6 +73,9 @@ auto get_download(std::uint64_t record_id, const server_cfg &server) utils::error::handle_error( function_name, fmt::format("get download request failed|{}|{}|{}", server.id, server.url, response->status)); + if (is_not_found != nullptr) { + *is_not_found = response->status == httplib::StatusCode::NotFound_404; + } return std::nullopt; } @@ -228,6 +230,28 @@ auto load_config(std::string &cfg_file) -> app_config { return cfg; } +[[nodiscard]] auto load_db(std::string_view log_dir) -> data_db { + MONITARR_USES_FUNCTION_NAME(); + + auto data_dir = utils::get_environment_variable("MONITARR_DATA_DIR"); + if (data_dir.empty()) { + data_dir = utils::path::combine(".", {"data"}); + } + + if (not utils::file::directory{data_dir}.create_directory()) { + throw utils::error::create_exception( + function_name, { + "failed to create data dir", + data_dir, + std::to_string(utils::get_last_error_code()), + }); + } + + data_db state_db{}; + state_db.open(data_dir, log_dir); + return state_db; +} + void remove_stalled(std::string_view download_id, std::string_view title, std::uint64_t episode_id, std::uint64_t movie_id, const server_cfg &server, data_db *state_db) { diff --git a/monitarr/libmonitarr/src/data_db.cpp b/monitarr/libmonitarr/src/data_db.cpp index 408b1ca..aec3d47 100644 --- a/monitarr/libmonitarr/src/data_db.cpp +++ b/monitarr/libmonitarr/src/data_db.cpp @@ -27,11 +27,31 @@ namespace monitarr { data_db::~data_db() noexcept { close(); } -void data_db::close() { - if (db_) { - db_->Close(); - db_.reset(); +auto data_db::all(std::string_view server_id) -> std::vector { + std::unique_ptr iter{ + db_->NewIterator(rocksdb::ReadOptions{}), + }; + + auto prefix = fmt::format("{}/", server_id); + + std::vector list; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + if (utils::string::begins_with(iter->key().ToString(), prefix)) { + list.emplace_back( + nlohmann::json::parse(iter->value().ToString()).get()); + } } + + return list; +} + +void data_db::close() { + if (not db_) { + return; + } + + db_->Close(); + db_.reset(); } auto data_db::get(std::string_view download_id) const diff --git a/monitarr/monitarr/src/run_cmd.cpp b/monitarr/monitarr/src/run_cmd.cpp index 7ab3930..95c6c9b 100644 --- a/monitarr/monitarr/src/run_cmd.cpp +++ b/monitarr/monitarr/src/run_cmd.cpp @@ -42,7 +42,7 @@ static void check_server(const server_cfg &server, data_db &state_db) { auto cli = create_client(server); std::uint16_t page{0U}; - while (++page != 0U) { + while (++page != 0U && not stop_requested) { httplib::Params params; params.emplace("page", std::to_string(page)); params.emplace("pageSize", "100"); @@ -72,6 +72,10 @@ static void check_server(const server_cfg &server, data_db &state_db) { auto now = utils::time::get_time_now(); for (const auto &record : json_data.at("records")) { + if (stop_requested) { + return; + } + auto download_id = fmt::format("{}/{}", server.id, record.at("id").get()); auto episode_id = record.contains("episodeId") @@ -86,8 +90,8 @@ static void check_server(const server_cfg &server, data_db &state_db) { record.at("status").get(), "downloading"); auto data = state_db.get(download_id); - const auto update_entry = [&download_id, &now, &size_left, &state_db, - &title, + const auto update_entry = [&download_id, &now, server_id = server.id, + &size_left, &state_db, &title, url = server.url](auto last_size_left) { if (size_left == 0U) { state_db.remove(download_id); @@ -129,26 +133,26 @@ static void check_server(const server_cfg &server, data_db &state_db) { } } -[[nodiscard]] static auto load_db(std::string_view log_dir) -> data_db { +static void cleanup_entries(const server_cfg &server, data_db &state_db) { MONITARR_USES_FUNCTION_NAME(); - auto data_dir = utils::get_environment_variable("MONITARR_DATA_DIR"); - if (data_dir.empty()) { - data_dir = utils::path::combine(".", {"data"}); - } + for (const auto &entry : state_db.all(server.id)) { + if (stop_requested) { + return; + } - if (not utils::file::directory{data_dir}.create_directory()) { - throw utils::error::create_exception( - function_name, { - "failed to create data dir", - data_dir, - std::to_string(utils::get_last_error_code()), - }); - } + auto record_id = utils::string::to_uint64( + utils::string::split(entry.download_id, '/', false).at(1U)); - data_db state_db{}; - state_db.open(data_dir, log_dir); - return state_db; + bool is_not_found{}; + auto download = get_download(record_id, server, &is_not_found); + if (not download.has_value() && is_not_found) { + utils::error::handle_warn( + function_name, fmt::format("download not found|{}|{}|{}", server.id, + server.url, entry.download_id)); + state_db.remove(entry.download_id); + } + } } auto run_cmd(const app_config &cfg, std::string_view log_dir) -> int { @@ -179,6 +183,7 @@ auto run_cmd(const app_config &cfg, std::string_view log_dir) -> int { try { check_server(server, state_db); + cleanup_entries(server, state_db); } catch (const std::exception &ex) { utils::error::handle_exception(function_name, ex); } catch (...) {