From e0ec9427816a800d656440a4cb285a5e8b3f8fec Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Thu, 23 Feb 2017 00:55:18 -0600 Subject: [PATCH] Upload manager changes --- SiaDrive.Api/AutoThread.cpp | 16 +++- SiaDrive.Api/AutoThread.h | 6 +- SiaDrive.Api/UploadManager.cpp | 109 +++++++++++++++++++---- SiaDrive.Api/UploadManager.h | 14 ++- SiaDrive.sln | 6 +- UnitTests/TestCacheFolder/test1/test.rtf | 1 + UnitTests/UnitTests.vcxproj | 3 + UnitTests/UnitTests.vcxproj.filters | 11 +++ UnitTests/UploadManagerTest.cpp | 3 +- 9 files changed, 144 insertions(+), 25 deletions(-) create mode 100644 UnitTests/TestCacheFolder/test1/test.rtf diff --git a/SiaDrive.Api/AutoThread.cpp b/SiaDrive.Api/AutoThread.cpp index a6ca62f..c10b002 100644 --- a/SiaDrive.Api/AutoThread.cpp +++ b/SiaDrive.Api/AutoThread.cpp @@ -4,9 +4,15 @@ using namespace Sia::Api; CAutoThread::CAutoThread(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig) : + CAutoThread(siaCurl, siaDriveConfig, nullptr) +{ +} + +CAutoThread::CAutoThread(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig, std::function autoThreadCallback) : _siaCurl(siaCurl.GetHostConfig()), _siaDriveConfig(siaDriveConfig), - _stopEvent(::CreateEvent(nullptr, FALSE, FALSE, nullptr)) + _stopEvent(::CreateEvent(nullptr, FALSE, FALSE, nullptr)), + _AutoThreadCallback(autoThreadCallback) { } @@ -16,6 +22,14 @@ CAutoThread::~CAutoThread() ::CloseHandle(_stopEvent); } +void CAutoThread::AutoThreadCallback(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig) +{ + if (_AutoThreadCallback) + { + _AutoThreadCallback(siaCurl, siaDriveConfig); + } +} + void CAutoThread::StartAutoThread() { std::lock_guard l(_startStopMutex); diff --git a/SiaDrive.Api/AutoThread.h b/SiaDrive.Api/AutoThread.h index 12809a1..98e03c7 100644 --- a/SiaDrive.Api/AutoThread.h +++ b/SiaDrive.Api/AutoThread.h @@ -10,6 +10,7 @@ class AFX_EXT_CLASS CAutoThread { public: CAutoThread(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig); + CAutoThread(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig, std::function autoThreadCallback); public: virtual ~CAutoThread(); @@ -20,11 +21,14 @@ private: HANDLE _stopEvent; std::unique_ptr _thread; std::mutex _startStopMutex; + std::function _AutoThreadCallback; protected: + virtual void AutoThreadCallback(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig); + +public: void StartAutoThread(); void StopAutoThread(); - virtual void AutoThreadCallback(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig) = 0; }; NS_END(2) \ No newline at end of file diff --git a/SiaDrive.Api/UploadManager.cpp b/SiaDrive.Api/UploadManager.cpp index 1295c10..e7f2590 100644 --- a/SiaDrive.Api/UploadManager.cpp +++ b/SiaDrive.Api/UploadManager.cpp @@ -34,17 +34,38 @@ static void CreateTableIfNotFound(SQLite::Database* database, const String& tabl CUploadManager::CUploadManager(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig) : CAutoThread(siaCurl, siaDriveConfig), _uploadDatabase(siaDriveConfig->GetRenter_UploadDbFilePath(), SQLite::OPEN_CREATE | SQLite::OPEN_READWRITE), - _siaDriveConfig(siaDriveConfig) + _siaDriveConfig(siaDriveConfig), + _fileThread(siaCurl, siaDriveConfig, [this](const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig) { this->FileThreadCallback(siaCurl, siaDriveConfig); }) { CreateTableIfNotFound(&_uploadDatabase, UPLOAD_TABLE, UPLOAD_TABLE_COLUMNS); StartAutoThread(); + _fileThread.StartAutoThread(); } CUploadManager::~CUploadManager() { + _fileThread.StopAutoThread(); StopAutoThread(); } +void CUploadManager::FileThreadCallback(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig) +{ + std::function nextFile = nullptr; + { + std::lock_guard l(_fileQueueMutex); + if (_fileQueue.size()) + { + nextFile = _fileQueue.front(); + _fileQueue.pop_front(); + } + } + + if (nextFile) + { + nextFile(); + } +} + void CUploadManager::AutoThreadCallback(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig) { bool processNext = true; @@ -152,8 +173,31 @@ UploadStatus CUploadManager::GetUploadStatus(const String& siaPath) return uploadStatus; } -void CUploadManager::AddOrUpdate(const String& siaPath, const String& filePath) +// The real source file is copied to a hidden file. The hidden filename is constructed by generating an SHA25 hash of the +// source path (all lowercase). '.siadrive.temp' will be used as the extension. After copy is successful, the extension +// is renamed to '.siadrive'. '.siadrive' files will be hidden/system and used by the Dokan API for file i/o until Sia upload +// is complete. If a change occurs, the file will be deleted from Sia (cancelling the in-progress upload) and renamed to the +// real source path. The process will then start over again as if the file was new. +// If the file has been fully uploaded, the hidden file should not exist, so rename will not occur; however, the file +// will be deleted from Sia and treated as new. +// Uploads will always use the real file. User i/o will occur against the hidden file only temporarily. Since upload will take +// longer to handle than a normal file copy, this seems to be the best compromise for performance. +UploadError CUploadManager::AddOrUpdate(const String& siaPath, String filePath) { + UploadError ret = UploadError::Success; + + // Relative to absolute and grab parent folder of source + String rootPath; + { + String temp; + temp.resize(MAX_PATH + 1); + filePath = _wfullpath(&temp[0], filePath.c_str(), MAX_PATH); + + temp = filePath; + ::PathRemoveFileSpec(&temp[0]); + rootPath = temp; + } + // Lock here - if file is modified again before a prior upload is complete, delete it and // start again later std::lock_guard l(_uploadMutex); @@ -177,7 +221,7 @@ void CUploadManager::AddOrUpdate(const String& siaPath, const String& filePath) addOrUpdate.bind("@sia_path", CW2A(siaPath.c_str()).m_psz); addOrUpdate.bind("@file_path", CW2A(filePath.c_str()).m_psz); addOrUpdate.bind("@temp_path", CW2A(tempPath.c_str()).m_psz); - addOrUpdate.bind("@status", static_cast(UploadStatus::Queued));*/ + addOrUpdate.bind("@status", static_cast(UploadStatus::Copying));*/ // While copy is active, point to normal file. // After copy, use temp file as real file until upload is complete @@ -185,28 +229,56 @@ void CUploadManager::AddOrUpdate(const String& siaPath, const String& filePath) // Error Scenarios: // Crash before db update to status copying - file will be re-uploaded automatically, if complete; otherwise, deleted // Need to keep track of files as being copied and then there status - // Crash Scenarios: // Crash before copy begins - on startup, check for copying status with no .siadrive // Crash during copy - on startup, check for copying status and delete .siadrive // Crash after copy but before db update - on startup, check for copying status and delete .siadrive - String tempPath; - tempPath.resize(MAX_PATH + 1); - PathCombine(&tempPath[0], CA2W(_siaDriveConfig->GetTempFolder().c_str()), (GenerateSha256(filePath) + L".siadrive").c_str()); + + String siaDriveFileName = GenerateSha256(&filePath[3]) + L".siadrive"; - // Queue this - if (::CopyFile(filePath.c_str(), tempPath.c_str(), FALSE)) + String siaDriveFilePath; + siaDriveFilePath.resize(MAX_PATH + 1); + PathCombine(&siaDriveFilePath[0], rootPath.c_str(), siaDriveFileName.c_str()); + + String tempSourcePath; + tempSourcePath.resize(MAX_PATH + 1); + PathCombine(&tempSourcePath[0], rootPath.c_str(), (siaDriveFileName + L".temp").c_str()); + + // Queue file operations + std::lock_guard l2(_fileQueueMutex); + _fileQueue.push_back([this,filePath, tempSourcePath, siaDriveFilePath]() { - tempPath = L""; - } - else - { - if (!::DeleteFile(tempPath.c_str())) + // TODO Retry a few times + if (::CopyFile(filePath.c_str(), tempSourcePath.c_str(), FALSE)) { + // Rename to real temp + // TODO Retry a few times + if (::PathFileExists(siaDriveFilePath.c_str())) + { + // TODO Retry a few times + ::DeleteFile(siaDriveFilePath.c_str()); + } + if (::MoveFile(tempSourcePath.c_str(), siaDriveFilePath.c_str())) + { + // Change status to queued + } } - // error condition - } + else + { + // If temp copy fails, try to delete + // If partial copy and file is unable to be deleted, log warning + if (::PathFileExists(tempSourcePath.c_str())) + { + // TODO Retry a few times + ::DeleteFile(tempSourcePath.c_str()); + } + + // Requeue + } + }); } + + return ret; } void CUploadManager::PurgeCompleteStatus() @@ -219,7 +291,8 @@ void CUploadManager::PurgeErrorStatus() } -void CUploadManager::Remove(const String& siaPath) +UploadError CUploadManager::Remove(const String& siaPath) { - + UploadError ret = UploadError::Success; + return ret; } \ No newline at end of file diff --git a/SiaDrive.Api/UploadManager.h b/SiaDrive.Api/UploadManager.h index bda7f75..1ea271e 100644 --- a/SiaDrive.Api/UploadManager.h +++ b/SiaDrive.Api/UploadManager.h @@ -21,6 +21,11 @@ public: Error }; + enum class _UploadError + { + Success + }; + private: typedef struct { @@ -41,18 +46,23 @@ private: SQLite::Database _uploadDatabase; std::mutex _uploadMutex; CSiaDriveConfig* _siaDriveConfig; + CAutoThread _fileThread; + std::mutex _fileQueueMutex; + std::deque> _fileQueue; protected: virtual void AutoThreadCallback(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig) override; + void FileThreadCallback(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig); public: _UploadStatus GetUploadStatus(const String& siaPath); - void AddOrUpdate(const String& siaPath, const String& filePath); - void Remove(const String& siaPath); + _UploadError AddOrUpdate(const String& siaPath, String filePath); + _UploadError Remove(const String& siaPath); void PurgeCompleteStatus(); void PurgeErrorStatus(); }; typedef Sia::Api::CUploadManager::_UploadStatus UploadStatus; +typedef Sia::Api::CUploadManager::_UploadError UploadError; NS_END(2) \ No newline at end of file diff --git a/SiaDrive.sln b/SiaDrive.sln index b63fc19..fa987b3 100644 --- a/SiaDrive.sln +++ b/SiaDrive.sln @@ -734,7 +734,8 @@ Global {B3DF927F-A1CE-4F50-A621-A4C3A06E4F8A}.RelWithDebInfo|x64.Build.0 = Release|x64 {B3DF927F-A1CE-4F50-A621-A4C3A06E4F8A}.RelWithDebInfo|x86.ActiveCfg = Release|Win32 {B3DF927F-A1CE-4F50-A621-A4C3A06E4F8A}.RelWithDebInfo|x86.Build.0 = Release|Win32 - {BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.Debug|x64.ActiveCfg = Debug|Win32 + {BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.Debug|x64.ActiveCfg = Debug|x64 + {BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.Debug|x64.Build.0 = Debug|x64 {BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.Debug|x86.ActiveCfg = Debug|Win32 {BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.Debug|x86.Build.0 = Debug|Win32 {BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.DLL Debug - DLL OpenSSL - DLL LibSSH2|x64.ActiveCfg = Release|Win32 @@ -858,7 +859,8 @@ Global {BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.RelWithDebInfo|x64.ActiveCfg = RelWithDebInfo|Win32 {BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Win32 {BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Win32 - {92EF9CAE-3F0C-31D5-9556-62586CC5072D}.Debug|x64.ActiveCfg = Debug|Win32 + {92EF9CAE-3F0C-31D5-9556-62586CC5072D}.Debug|x64.ActiveCfg = Debug|x64 + {92EF9CAE-3F0C-31D5-9556-62586CC5072D}.Debug|x64.Build.0 = Debug|x64 {92EF9CAE-3F0C-31D5-9556-62586CC5072D}.Debug|x86.ActiveCfg = Debug|Win32 {92EF9CAE-3F0C-31D5-9556-62586CC5072D}.Debug|x86.Build.0 = Debug|Win32 {92EF9CAE-3F0C-31D5-9556-62586CC5072D}.DLL Debug - DLL OpenSSL - DLL LibSSH2|x64.ActiveCfg = Release|Win32 diff --git a/UnitTests/TestCacheFolder/test1/test.rtf b/UnitTests/TestCacheFolder/test1/test.rtf new file mode 100644 index 0000000..08778d7 --- /dev/null +++ b/UnitTests/TestCacheFolder/test1/test.rtf @@ -0,0 +1 @@ +{\rtf1} \ No newline at end of file diff --git a/UnitTests/UnitTests.vcxproj b/UnitTests/UnitTests.vcxproj index 177cb46..71026a1 100644 --- a/UnitTests/UnitTests.vcxproj +++ b/UnitTests/UnitTests.vcxproj @@ -198,6 +198,9 @@ {b3df927f-a1ce-4f50-a621-a4c3a06e4f8a} + + + diff --git a/UnitTests/UnitTests.vcxproj.filters b/UnitTests/UnitTests.vcxproj.filters index e7d0543..0edefe7 100644 --- a/UnitTests/UnitTests.vcxproj.filters +++ b/UnitTests/UnitTests.vcxproj.filters @@ -13,6 +13,12 @@ {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + {fe235f4d-95ea-4c37-9bb0-dc9e89f75676} + + + {18c83c4f-d98e-4993-b622-6d7c1fa75162} + @@ -57,4 +63,9 @@ Source Files + + + TestCacheFolder\test1 + + \ No newline at end of file diff --git a/UnitTests/UploadManagerTest.cpp b/UnitTests/UploadManagerTest.cpp index 2808b68..1a6cbdf 100644 --- a/UnitTests/UploadManagerTest.cpp +++ b/UnitTests/UploadManagerTest.cpp @@ -48,7 +48,8 @@ namespace UnitTests Assert::AreEqual(version.c_str(), TEST_SERVER_VERSION); // Connectivity test CUploadManager uploadManager(siaCurl, &driveConfig); - uploadManager.AddOrUpdate(L"/test1/test.txt", L"./test1/test.txt"); + uploadManager.AddOrUpdate(L"/test1/test.rtf", L"./TestCacheFolder/test1/test.rtf"); + Sleep(-1); } catch (SQLite::Exception e) {