4 Commits

Author SHA1 Message Date
49f0425e56 fixes
All checks were successful
BlockStorage/repertory/pipeline/head This commit looks good
2024-09-23 20:34:27 -05:00
7973e523c3 refactor 2024-09-23 20:24:49 -05:00
c4326520cd refactor 2024-09-23 20:22:43 -05:00
dfb9d78448 clean cache directory 2024-09-23 20:08:35 -05:00
2 changed files with 119 additions and 63 deletions

View File

@ -223,17 +223,20 @@ E_SIMPLE1(orphaned_file_detected, warn, true,
std::string, source, src, E_STRING
);
E_SIMPLE2(orphaned_file_processed, warn, true,
std::string, source, src, E_STRING,
std::string, dest, dest, E_STRING
);
E_SIMPLE3(orphaned_file_processing_failed, error, true,
std::string, source, src, E_STRING,
std::string, dest, dest, E_STRING,
std::string, result, res, E_STRING
);
E_SIMPLE1(orphaned_source_file_detected, info, true,
std::string, source, src, E_STRING
);
E_SIMPLE1(orphaned_source_file_removed, info, true,
std::string, source, src, E_STRING
);
E_SIMPLE1(polling_item_begin, debug, true,
std::string, item_name, item, E_STRING
);

View File

@ -440,6 +440,39 @@ void base_provider::remove_deleted_files() {
static_cast<const char *>(__FUNCTION__),
};
auto source_list =
utils::file::directory{config_.get_cache_directory()}.get_files();
for (auto &&source_file : source_list) {
filesystem_item fsi{};
auto sp_res =
get_filesystem_item_from_source_path(source_file->get_path(), fsi);
if (sp_res != api_error::item_not_found) {
continue;
}
auto reference_time =
source_file->get_time(config_.get_eviction_uses_accessed_time()
? utils::file::time_type::accessed
: utils::file::time_type::modified);
if (not reference_time.has_value()) {
continue;
}
auto delay = (config_.get_eviction_delay_mins() * 60UL) *
utils::time::NANOS_PER_SECOND;
if ((reference_time.value() + static_cast<std::uint64_t>(delay)) >=
utils::time::get_time_now()) {
continue;
}
event_system::instance().raise<orphaned_source_file_detected>(
source_file->get_path());
if (source_file->remove()) {
event_system::instance().raise<orphaned_source_file_removed>(
source_file->get_path());
}
}
struct removed_item {
std::string api_path{};
bool directory{};
@ -457,15 +490,20 @@ void base_provider::remove_deleted_files() {
for (auto &&api_path : db3_->get_api_path_list()) {
api_meta_map meta{};
if (get_item_meta(api_path, meta) == api_error::success) {
if (get_item_meta(api_path, meta) != api_error::success) {
continue;
}
if (utils::string::to_bool(meta[META_DIRECTORY])) {
bool exists{};
if (is_directory(api_path, exists) != api_error::success) {
continue;
}
if (not exists) {
removed_list.emplace_back(removed_item{api_path, true, ""});
}
continue;
}
@ -473,61 +511,74 @@ void base_provider::remove_deleted_files() {
if (is_file(api_path, exists) != api_error::success) {
continue;
}
if (not exists) {
removed_list.emplace_back(
removed_item{api_path, false, meta[META_SOURCE]});
}
}
}
for (auto &&item : removed_list) {
if (not item.directory) {
if (utils::file::file(item.source_path).exists()) {
if (item.directory) {
continue;
}
if (not utils::file::file(item.source_path).exists()) {
continue;
}
const auto orphaned_directory =
utils::path::combine(config_.get_data_directory(), {"orphaned"});
if (utils::file::directory(orphaned_directory).create_directory()) {
const auto parts = utils::string::split(item.api_path, '/', false);
const auto orphaned_file = utils::path::combine(
orphaned_directory,
{utils::path::strip_to_file_name(item.source_path) + '_' +
parts[parts.size() - 1U]});
event_system::instance().raise<orphaned_file_detected>(
item.source_path);
if (utils::file::reset_modified_time(item.source_path) &&
utils::file::file(item.source_path)
.copy_to(orphaned_file, true)) {
event_system::instance().raise<orphaned_file_processed>(
item.source_path, orphaned_file);
} else {
event_system::instance().raise<orphaned_file_processing_failed>(
item.source_path, orphaned_file,
std::to_string(utils::get_last_error_code()));
}
} else {
if (not utils::file::directory(orphaned_directory).create_directory()) {
utils::error::raise_error(
function_name, std::to_string(utils::get_last_error_code()),
"failed to create orphaned directory|sp|" + orphaned_directory);
continue;
}
const auto parts = utils::string::split(item.api_path, '/', false);
const auto orphaned_file = utils::path::combine(
orphaned_directory, {utils::path::strip_to_file_name(item.source_path) +
'_' + parts[parts.size() - 1U]});
event_system::instance().raise<orphaned_file_detected>(item.source_path);
if (not utils::file::reset_modified_time(item.source_path)) {
event_system::instance().raise<orphaned_file_processing_failed>(
item.source_path, orphaned_file,
"reset modified failed|" +
std::to_string(utils::get_last_error_code()));
continue;
}
if (not utils::file::file(item.source_path).copy_to(orphaned_file, true)) {
[[maybe_unused]] auto removed = utils::file::file{orphaned_file}.remove();
event_system::instance().raise<orphaned_file_processing_failed>(
item.source_path, orphaned_file,
"copy failed|" + std::to_string(utils::get_last_error_code()));
continue;
}
if (not fm_->evict_file(item.api_path)) {
event_system::instance().raise<orphaned_file_processing_failed>(
item.source_path, orphaned_file, "eviction failed");
continue;
}
if (fm_->evict_file(item.api_path)) {
db3_->remove_api_path(item.api_path);
event_system::instance().raise<file_removed_externally>(
item.api_path, item.source_path);
}
}
event_system::instance().raise<file_removed_externally>(item.api_path,
item.source_path);
}
for (auto &&item : removed_list) {
if (item.directory) {
if (not item.directory) {
continue;
}
db3_->remove_api_path(item.api_path);
event_system::instance().raise<directory_removed_externally>(
item.api_path, item.source_path);
}
}
}
auto base_provider::remove_file(const std::string &api_path) -> api_error {
const auto notify_end = [&api_path](api_error error) -> api_error {
@ -655,6 +706,8 @@ auto base_provider::start(api_item_added_callback api_item_added,
}
if (online && not unmount_requested) {
remove_deleted_files();
polling::instance().set_callback({"check_deleted", polling::frequency::low,
[this]() { remove_deleted_files(); }});
return true;