From f55b3ccba9d1a02296ebd50f03ce500ab14bca9e Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Tue, 28 Feb 2017 22:38:29 -0600 Subject: [PATCH] Upload manager changes --- SiaDrive.Api/UploadManager.cpp | 39 +++++++++++++++++++--------- SiaDrive.Api/UploadManager.h | 34 +++++++++++++++++++++++- SiaDrive.Dokan.Api/SiaDokanDrive.cpp | 1 + 3 files changed, 61 insertions(+), 13 deletions(-) diff --git a/SiaDrive.Api/UploadManager.cpp b/SiaDrive.Api/UploadManager.cpp index ca605e0..e99ade5 100644 --- a/SiaDrive.Api/UploadManager.cpp +++ b/SiaDrive.Api/UploadManager.cpp @@ -97,6 +97,7 @@ CUploadManager::CUploadManager(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriv { CreateTableIfNotFound(&_uploadDatabase, UPLOAD_TABLE, UPLOAD_TABLE_COLUMNS); + // Clean-up cache folder if (!RecurDeleteFilesByExtentsion(CA2W(siaDriveConfig->GetCacheFolder().c_str()).m_psz, L".siadrive")) { throw StartupException(L"Failed to remove '.siadrive' files"); @@ -107,20 +108,25 @@ CUploadManager::CUploadManager(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriv throw StartupException(L"Failed to remove '.siadrive.temp' files"); } - UpdateQueueOnStartup(); + // Re-add files to file action queue + UpdateFileQueueOnStartup(); + + // Detect files that have been removed since last startup DeleteFilesRemovedFromSia(siaCurl, siaDriveConfig, true); + // Begin normal processing StartAutoThread(); _fileThread.StartAutoThread(); } CUploadManager::~CUploadManager() { + // Stop all processing _fileThread.StopAutoThread(); StopAutoThread(); } -void CUploadManager::UpdateQueueOnStartup() +void CUploadManager::UpdateFileQueueOnStartup() { try { @@ -423,11 +429,20 @@ void CUploadManager::AutoThreadCallback(const CSiaCurl& siaCurl, CSiaDriveConfig { return ptr->GetSiaPath() == siaPath; }); - + + // Removed by another client if (it == fileList.end()) { - // error condition - should always exist. delete from db and log warning, but continue processing + SET_STATUS(UploadStatus::Remove, ExternallyRemovedFileDetected, ModifyUploadStatusFailed) + if (statusUpdated) + { + std::lock_guard l2(_fileQueueMutex); + _fileQueue.push_back([=]() { this->FileAction(CSiaCurl(GetHostConfig()), siaPath, filePath, nullptr, siaDriveFilePath, true); }); + + CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(FileRemoveAdded(siaPath))); + } } + // Changed file detected else if (uploadStatus == UploadStatus::Modified) { json response; @@ -442,6 +457,7 @@ void CUploadManager::AutoThreadCallback(const CSiaCurl& siaCurl, CSiaDriveConfig CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(FailedToDeleteFromSia(siaPath, filePath, cerror))); } } + // Upload is complete else if ((*it)->GetUploadProgress() >= 100) { SET_STATUS(UploadStatus::Complete, UploadComplete, ModifyUploadStatusFailed) @@ -450,9 +466,9 @@ void CUploadManager::AutoThreadCallback(const CSiaCurl& siaCurl, CSiaDriveConfig CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DeleteSiaDriveFileFailed(siaPath, filePath, siaDriveFilePath))); } } + // Upload still active, don't process another file else { - // upload still active processNext = false; } } @@ -594,18 +610,18 @@ UploadError CUploadManager::AddOrUpdate(const String& siaPath, String filePath) insert.bind("@file_path", CW2A(filePath.c_str()).m_psz); insert.bind("@sd_file_path", CW2A(siaDriveFileName.c_str()).m_psz); insert.bind("@status", static_cast(UploadStatus::Copying)); - if (insert.exec() != 1) - { - CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DatabaseInsertFailed(siaPath, filePath, CA2W(insert.getErrorMsg()).m_psz))); - ret = UploadError::DatabaseError; - } - else + if (insert.exec() == 1) { // Queue file upload operation std::lock_guard l2(_fileQueueMutex); _fileQueue.push_back([=]() { this->FileAction(CSiaCurl(GetHostConfig()), siaPath, filePath, tempSourcePath, siaDriveFilePath, false); }); CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(NewFileAdded(siaPath, filePath, siaDriveFilePath))); } + else + { + CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DatabaseInsertFailed(siaPath, filePath, CA2W(insert.getErrorMsg()).m_psz))); + ret = UploadError::DatabaseError; + } } catch (SQLite::Exception e) { @@ -681,7 +697,6 @@ UploadError CUploadManager::Remove(const String& siaPath) { CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DatabaseExceptionOccurred(e))); ret = UploadError::DatabaseError; - } return ret; diff --git a/SiaDrive.Api/UploadManager.h b/SiaDrive.Api/UploadManager.h index 17c22f5..c71e985 100644 --- a/SiaDrive.Api/UploadManager.h +++ b/SiaDrive.Api/UploadManager.h @@ -60,7 +60,7 @@ private: void FileThreadCallback(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig); void HandleFileRemove(const CSiaCurl& siaCurl, const String& siaPath, const String& siaDriveFilePath); void HandleAddFile(const String& siaPath, const String& filePath, const String& tempSourcePath, const String& siaDriveFilePath); - void UpdateQueueOnStartup(); + void UpdateFileQueueOnStartup(); void DeleteFilesRemovedFromSia(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig, const bool& isStartup = false); protected: @@ -145,6 +145,38 @@ public: } }; +class ExternallyRemovedFileDetected : + public CEvent +{ +public: + ExternallyRemovedFileDetected(const String& siaPath, const String& filePath) : + _siaPath(siaPath), + _filePath(filePath) + { + + } + +public: + virtual ~ExternallyRemovedFileDetected() + { + } + +private: + const String _siaPath; + const String _filePath; + +public: + virtual String GetSingleLineMessage() const override + { + return L"ExternallyRemovedFileDetected|SP|" + _siaPath + L"|FP|" + _filePath; + } + + virtual std::shared_ptr Clone() const override + { + return std::shared_ptr(new ExternallyRemovedFileDetected(_siaPath, _filePath)); + } +}; + class ModifiedUploadQueued : public CEvent { diff --git a/SiaDrive.Dokan.Api/SiaDokanDrive.cpp b/SiaDrive.Dokan.Api/SiaDokanDrive.cpp index b3b49d4..9ac0b89 100644 --- a/SiaDrive.Dokan.Api/SiaDokanDrive.cpp +++ b/SiaDrive.Dokan.Api/SiaDokanDrive.cpp @@ -397,6 +397,7 @@ public: { _siaApi = siaApi; _siaDriveConfig = siaDriveConfig; + // May spend a little wait time here while files are cleaned-up and re-added to queue _uploadManager.reset(new CUploadManager(CSiaCurl(siaApi->GetHostConfig()), siaDriveConfig)); _dokanOps.Cleanup = nullptr; _dokanOps.CloseFile = Sia_CloseFile;