cleanup database

monitarr/libmonitarr/include/actions.hpp

@@ -32,13 +32,16 @@ class data_db;
 [[nodiscard]] auto create_client(const server_cfg &server) -> httplib::Client;
 
 [[nodiscard]] auto get_download(std::uint64_t record_id,
-                                const server_cfg &server)
+                                const server_cfg &server,
+                                bool *is_not_found = nullptr)
     -> std::optional<nlohmann::json>;
 
 [[nodiscard]] auto list_queue(const server_cfg &server) -> int;
 
 [[nodiscard]] auto load_config(std::string &cfg_file) -> app_config;
 
+[[nodiscard]] auto load_db(std::string_view log_dir) -> data_db;
+
 void remove_stalled(std::string_view download_id, std::string_view title,
                     std::uint64_t episode_id, std::uint64_t movie_id,
                     const server_cfg &server, data_db *state_db = nullptr);

monitarr/libmonitarr/include/data_db.hpp

@@ -56,6 +56,8 @@ private:
       std::function<rocksdb::Status(rocksdb::Transaction *txn)> action) -> bool;
 
 public:
+  [[nodiscard]] auto all(std::string_view server_id) -> std::vector<data_entry>;
+
   void close();
 
   [[nodiscard]] auto get(std::string_view download_id) const

monitarr/libmonitarr/src/actions.cpp

@@ -30,7 +30,6 @@
 #include "utils/string.hpp"
 #include "utils/unix.hpp"
 #include "utils/windows.hpp"
-#include <optional>
 
 using namespace std::literals::chrono_literals;
 
@@ -48,8 +47,8 @@ auto create_client(const server_cfg &server) -> httplib::Client {
   return cli;
 }
 
-auto get_download(std::uint64_t record_id, const server_cfg &server)
+auto get_download(std::uint64_t record_id, const server_cfg &server,
-    -> std::optional<nlohmann::json> {
+                  bool *is_not_found) -> std::optional<nlohmann::json> {
   MONITARR_USES_FUNCTION_NAME();
 
   auto cli = create_client(server);
@@ -74,6 +73,9 @@ auto get_download(std::uint64_t record_id, const server_cfg &server)
       utils::error::handle_error(
           function_name, fmt::format("get download request failed|{}|{}|{}",
                                      server.id, server.url, response->status));
+      if (is_not_found != nullptr) {
+        *is_not_found = response->status == httplib::StatusCode::NotFound_404;
+      }
       return std::nullopt;
     }
 
@@ -228,6 +230,28 @@ auto load_config(std::string &cfg_file) -> app_config {
   return cfg;
 }
 
+[[nodiscard]] auto load_db(std::string_view log_dir) -> data_db {
+  MONITARR_USES_FUNCTION_NAME();
+
+  auto data_dir = utils::get_environment_variable("MONITARR_DATA_DIR");
+  if (data_dir.empty()) {
+    data_dir = utils::path::combine(".", {"data"});
+  }
+
+  if (not utils::file::directory{data_dir}.create_directory()) {
+    throw utils::error::create_exception(
+        function_name, {
+                           "failed to create data dir",
+                           data_dir,
+                           std::to_string(utils::get_last_error_code()),
+                       });
+  }
+
+  data_db state_db{};
+  state_db.open(data_dir, log_dir);
+  return state_db;
+}
+
 void remove_stalled(std::string_view download_id, std::string_view title,
                     std::uint64_t episode_id, std::uint64_t movie_id,
                     const server_cfg &server, data_db *state_db) {

monitarr/libmonitarr/src/data_db.cpp

@@ -27,12 +27,32 @@
 namespace monitarr {
 data_db::~data_db() noexcept { close(); }
 
+auto data_db::all(std::string_view server_id) -> std::vector<data_entry> {
+  std::unique_ptr<rocksdb::Iterator> iter{
+      db_->NewIterator(rocksdb::ReadOptions{}),
+  };
+
+  auto prefix = fmt::format("{}/", server_id);
+
+  std::vector<data_entry> list;
+  for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
+    if (utils::string::begins_with(iter->key().ToString(), prefix)) {
+      list.emplace_back(
+          nlohmann::json::parse(iter->value().ToString()).get<data_entry>());
+    }
+  }
+
+  return list;
+}
+
 void data_db::close() {
-  if (db_) {
+  if (not db_) {
+    return;
+  }
+
   db_->Close();
   db_.reset();
 }
-}
 
 auto data_db::get(std::string_view download_id) const
     -> std::optional<data_entry> {

monitarr/monitarr/src/run_cmd.cpp

@@ -42,7 +42,7 @@ static void check_server(const server_cfg &server, data_db &state_db) {
   auto cli = create_client(server);
 
   std::uint16_t page{0U};
-  while (++page != 0U) {
+  while (++page != 0U && not stop_requested) {
     httplib::Params params;
     params.emplace("page", std::to_string(page));
     params.emplace("pageSize", "100");
@@ -72,6 +72,10 @@ static void check_server(const server_cfg &server, data_db &state_db) {
     auto now = utils::time::get_time_now();
 
     for (const auto &record : json_data.at("records")) {
+      if (stop_requested) {
+        return;
+      }
+
       auto download_id =
           fmt::format("{}/{}", server.id, record.at("id").get<std::uint64_t>());
       auto episode_id = record.contains("episodeId")
@@ -86,8 +90,8 @@ static void check_server(const server_cfg &server, data_db &state_db) {
           record.at("status").get<std::string>(), "downloading");
 
       auto data = state_db.get(download_id);
-      const auto update_entry = [&download_id, &now, &size_left, &state_db,
+      const auto update_entry = [&download_id, &now, server_id = server.id,
-                                 &title,
+                                 &size_left, &state_db, &title,
                                  url = server.url](auto last_size_left) {
         if (size_left == 0U) {
           state_db.remove(download_id);
@@ -129,26 +133,26 @@ static void check_server(const server_cfg &server, data_db &state_db) {
   }
 }
 
-[[nodiscard]] static auto load_db(std::string_view log_dir) -> data_db {
+static void cleanup_entries(const server_cfg &server, data_db &state_db) {
   MONITARR_USES_FUNCTION_NAME();
 
-  auto data_dir = utils::get_environment_variable("MONITARR_DATA_DIR");
+  for (const auto &entry : state_db.all(server.id)) {
-  if (data_dir.empty()) {
+    if (stop_requested) {
-    data_dir = utils::path::combine(".", {"data"});
+      return;
     }
 
-  if (not utils::file::directory{data_dir}.create_directory()) {
+    auto record_id = utils::string::to_uint64(
-    throw utils::error::create_exception(
+        utils::string::split(entry.download_id, '/', false).at(1U));
-        function_name, {
-                           "failed to create data dir",
-                           data_dir,
-                           std::to_string(utils::get_last_error_code()),
-                       });
-  }
 
-  data_db state_db{};
+    bool is_not_found{};
-  state_db.open(data_dir, log_dir);
+    auto download = get_download(record_id, server, &is_not_found);
-  return state_db;
+    if (not download.has_value() && is_not_found) {
+      utils::error::handle_warn(
+          function_name, fmt::format("download not found|{}|{}|{}", server.id,
+                                     server.url, entry.download_id));
+      state_db.remove(entry.download_id);
+    }
+  }
 }
 
 auto run_cmd(const app_config &cfg, std::string_view log_dir) -> int {
@@ -179,6 +183,7 @@ auto run_cmd(const app_config &cfg, std::string_view log_dir) -> int {
 
                         try {
                           check_server(server, state_db);
+                          cleanup_entries(server, state_db);
                         } catch (const std::exception &ex) {
                           utils::error::handle_exception(function_name, ex);
                         } catch (...) {