Upload manager changes

SiaDrive.Api/AutoThread.cpp

@@ -4,9 +4,15 @@
 using namespace Sia::Api;
 
 CAutoThread::CAutoThread(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig) :
+	CAutoThread(siaCurl, siaDriveConfig, nullptr)
+{
+}
+
+CAutoThread::CAutoThread(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig, std::function<void(const CSiaCurl&, CSiaDriveConfig*)> autoThreadCallback) :
 	_siaCurl(siaCurl.GetHostConfig()),
 	_siaDriveConfig(siaDriveConfig),
-	_stopEvent(::CreateEvent(nullptr, FALSE, FALSE, nullptr))
+	_stopEvent(::CreateEvent(nullptr, FALSE, FALSE, nullptr)),
+	_AutoThreadCallback(autoThreadCallback)
 {
 }
 
@@ -16,6 +22,14 @@ CAutoThread::~CAutoThread()
 	::CloseHandle(_stopEvent);
 }
 
+void CAutoThread::AutoThreadCallback(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig)
+{
+	if (_AutoThreadCallback)
+	{
+		_AutoThreadCallback(siaCurl, siaDriveConfig);
+	}
+}
+
 void CAutoThread::StartAutoThread()
 {
 	std::lock_guard<std::mutex> l(_startStopMutex);

SiaDrive.Api/AutoThread.h

@@ -10,6 +10,7 @@ class AFX_EXT_CLASS CAutoThread
 {
 public:
 	CAutoThread(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig);
+	CAutoThread(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig, std::function<void(const CSiaCurl&, CSiaDriveConfig*)> autoThreadCallback);
 
 public:
 	virtual ~CAutoThread();
@@ -20,11 +21,14 @@ private:
 	HANDLE _stopEvent;
 	std::unique_ptr<std::thread>  _thread;
 	std::mutex _startStopMutex;
+	std::function<void(const CSiaCurl&, CSiaDriveConfig*)> _AutoThreadCallback;
 
 protected:
+	virtual void AutoThreadCallback(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig);
+
+public:
 	void StartAutoThread();
 	void StopAutoThread();
-	virtual void AutoThreadCallback(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig) = 0;
 };
 
 NS_END(2)

SiaDrive.Api/UploadManager.cpp

@@ -34,17 +34,38 @@ static void CreateTableIfNotFound(SQLite::Database* database, const String& tabl
 CUploadManager::CUploadManager(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig) :
 	CAutoThread(siaCurl, siaDriveConfig),
 	_uploadDatabase(siaDriveConfig->GetRenter_UploadDbFilePath(), SQLite::OPEN_CREATE | SQLite::OPEN_READWRITE),
-	_siaDriveConfig(siaDriveConfig)
+	_siaDriveConfig(siaDriveConfig),
+	_fileThread(siaCurl, siaDriveConfig, [this](const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig) { this->FileThreadCallback(siaCurl, siaDriveConfig); })
 {
 	CreateTableIfNotFound(&_uploadDatabase, UPLOAD_TABLE, UPLOAD_TABLE_COLUMNS);
 	StartAutoThread();
+	_fileThread.StartAutoThread();
 }
 
 CUploadManager::~CUploadManager()
 {
+	_fileThread.StopAutoThread();
 	StopAutoThread();
 }
 
+void CUploadManager::FileThreadCallback(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig)
+{
+	std::function<void()> nextFile = nullptr;
+	{
+		std::lock_guard<std::mutex> l(_fileQueueMutex);
+		if (_fileQueue.size())
+		{
+			nextFile = _fileQueue.front();
+			_fileQueue.pop_front();
+		}
+	}
+
+	if (nextFile)
+	{
+		nextFile();
+	}
+}
+
 void CUploadManager::AutoThreadCallback(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig)
 {
 	bool processNext = true;
@@ -152,8 +173,31 @@ UploadStatus CUploadManager::GetUploadStatus(const String& siaPath)
 	return uploadStatus;
 }
 
-void CUploadManager::AddOrUpdate(const String& siaPath, const String& filePath)
+// The real source file is copied to a hidden file. The hidden filename is constructed by generating an SHA25 hash of the
+//	source path (all lowercase). '.siadrive.temp' will be used as the extension. After copy is successful, the extension
+//	is renamed to '.siadrive'. '.siadrive' files will be hidden/system and used by the Dokan API for file i/o until Sia upload 
+//	is complete. If a change occurs, the file will be deleted from Sia (cancelling the in-progress upload) and renamed to the
+//	real source path. The process will then start over again as if the file was new.
+//	If the file has been fully uploaded, the hidden file should not exist, so rename will not occur; however, the file
+//	will be deleted from Sia and treated as new.
+//	Uploads will always use the real file. User i/o will occur against the hidden file only temporarily. Since upload will take
+//	longer to handle than a normal file copy, this seems to be the best compromise for performance.
+UploadError CUploadManager::AddOrUpdate(const String& siaPath, String filePath)
 {
+	UploadError ret = UploadError::Success;
+
+	// Relative to absolute and grab parent folder of source
+	String rootPath;
+	{
+		String temp;
+		temp.resize(MAX_PATH + 1);
+		filePath = _wfullpath(&temp[0], filePath.c_str(), MAX_PATH);
+
+		temp = filePath;
+		::PathRemoveFileSpec(&temp[0]);
+		rootPath = temp;
+	}
+
 	// Lock here - if file is modified again before a prior upload is complete, delete it and 
 	//	start again later
 	std::lock_guard<std::mutex> l(_uploadMutex);
@@ -177,7 +221,7 @@ void CUploadManager::AddOrUpdate(const String& siaPath, const String& filePath)
 		addOrUpdate.bind("@sia_path", CW2A(siaPath.c_str()).m_psz);
 		addOrUpdate.bind("@file_path", CW2A(filePath.c_str()).m_psz);
 		addOrUpdate.bind("@temp_path", CW2A(tempPath.c_str()).m_psz);
-		addOrUpdate.bind("@status", static_cast<unsigned>(UploadStatus::Queued));*/
+		addOrUpdate.bind("@status", static_cast<unsigned>(UploadStatus::Copying));*/
 
 		// While copy is active, point to normal file.
 		//	After copy, use temp file as real file until upload is complete
@@ -185,28 +229,56 @@ void CUploadManager::AddOrUpdate(const String& siaPath, const String& filePath)
 		//	Error Scenarios:
 		//		Crash before db update to status copying - file will be re-uploaded automatically, if complete; otherwise, deleted
 		//			Need to keep track of files as being copied and then there status
-		//	Crash Scenarios:
 		//		Crash before copy begins - on startup, check for copying status with no .siadrive
 		//		Crash during copy - on startup, check for copying status and delete .siadrive
 		//		Crash after copy but before db update - on startup, check for copying status and delete .siadrive
-		String tempPath;
-		tempPath.resize(MAX_PATH + 1);
-		PathCombine(&tempPath[0], CA2W(_siaDriveConfig->GetTempFolder().c_str()), (GenerateSha256(filePath) + L".siadrive").c_str());
 		
-		// Queue this
+		String siaDriveFileName = GenerateSha256(&filePath[3]) + L".siadrive";
-		if (::CopyFile(filePath.c_str(), tempPath.c_str(), FALSE))
+
+		String siaDriveFilePath;
+		siaDriveFilePath.resize(MAX_PATH + 1);
+		PathCombine(&siaDriveFilePath[0], rootPath.c_str(), siaDriveFileName.c_str());
+
+		String tempSourcePath;
+		tempSourcePath.resize(MAX_PATH + 1);
+		PathCombine(&tempSourcePath[0], rootPath.c_str(), (siaDriveFileName + L".temp").c_str());
+		
+		// Queue file operations
+		std::lock_guard<std::mutex> l2(_fileQueueMutex);
+		_fileQueue.push_back([this,filePath, tempSourcePath, siaDriveFilePath]()
 		{
-			tempPath = L"";
+			// TODO Retry a few times
-		}
+			if (::CopyFile(filePath.c_str(), tempSourcePath.c_str(), FALSE))
-		else
-		{
-			if (!::DeleteFile(tempPath.c_str()))
 			{
+				// Rename to real temp
+				// TODO Retry a few times
+				if (::PathFileExists(siaDriveFilePath.c_str()))
+				{
+					// TODO Retry a few times
+					::DeleteFile(siaDriveFilePath.c_str());
+				}
 
+				if (::MoveFile(tempSourcePath.c_str(), siaDriveFilePath.c_str()))
+				{
+					// Change status to queued
+				}
 			}
-			// error condition
+			else
-		}
+			{
+				// If temp copy fails, try to delete
+				//	If partial copy and file is unable to be deleted, log warning
+				if (::PathFileExists(tempSourcePath.c_str()))
+				{
+					// TODO Retry a few times
+					::DeleteFile(tempSourcePath.c_str());
+				}
+
+				// Requeue
+			}
+		});
 	}
+
+	return ret;
 }
 
 void CUploadManager::PurgeCompleteStatus()
@@ -219,7 +291,8 @@ void CUploadManager::PurgeErrorStatus()
 	
 }
 
-void CUploadManager::Remove(const String& siaPath)
+UploadError CUploadManager::Remove(const String& siaPath)
 {
-	
+	UploadError ret = UploadError::Success;
+	return ret;
 }

SiaDrive.Api/UploadManager.h

@@ -21,6 +21,11 @@ public:
 		Error
 	};
 
+	enum class _UploadError
+	{
+		Success
+	};
+
 private:
 	typedef struct
 	{
@@ -41,18 +46,23 @@ private:
 	SQLite::Database _uploadDatabase;
 	std::mutex _uploadMutex;
 	CSiaDriveConfig* _siaDriveConfig;
+	CAutoThread _fileThread;
+	std::mutex _fileQueueMutex;
+	std::deque<std::function<void()>> _fileQueue;
 
 protected:
 	virtual void AutoThreadCallback(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig) override;
+	void FileThreadCallback(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig);
 
 public:
 	_UploadStatus GetUploadStatus(const String& siaPath);
-	void AddOrUpdate(const String& siaPath, const String& filePath);
+	_UploadError AddOrUpdate(const String& siaPath, String filePath);
-	void Remove(const String& siaPath);
+	_UploadError Remove(const String& siaPath);
 	void PurgeCompleteStatus();
 	void PurgeErrorStatus();
 };
 
 typedef Sia::Api::CUploadManager::_UploadStatus UploadStatus;
+typedef Sia::Api::CUploadManager::_UploadError UploadError;
 
 NS_END(2)

SiaDrive.sln

@@ -734,7 +734,8 @@ Global
 		{B3DF927F-A1CE-4F50-A621-A4C3A06E4F8A}.RelWithDebInfo|x64.Build.0 = Release|x64
 		{B3DF927F-A1CE-4F50-A621-A4C3A06E4F8A}.RelWithDebInfo|x86.ActiveCfg = Release|Win32
 		{B3DF927F-A1CE-4F50-A621-A4C3A06E4F8A}.RelWithDebInfo|x86.Build.0 = Release|Win32
-		{BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.Debug|x64.ActiveCfg = Debug|Win32
+		{BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.Debug|x64.ActiveCfg = Debug|x64
+		{BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.Debug|x64.Build.0 = Debug|x64
 		{BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.Debug|x86.ActiveCfg = Debug|Win32
 		{BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.Debug|x86.Build.0 = Debug|Win32
 		{BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.DLL Debug - DLL OpenSSL - DLL LibSSH2|x64.ActiveCfg = Release|Win32
@@ -858,7 +859,8 @@ Global
 		{BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.RelWithDebInfo|x64.ActiveCfg = RelWithDebInfo|Win32
 		{BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Win32
 		{BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Win32
-		{92EF9CAE-3F0C-31D5-9556-62586CC5072D}.Debug|x64.ActiveCfg = Debug|Win32
+		{92EF9CAE-3F0C-31D5-9556-62586CC5072D}.Debug|x64.ActiveCfg = Debug|x64
+		{92EF9CAE-3F0C-31D5-9556-62586CC5072D}.Debug|x64.Build.0 = Debug|x64
 		{92EF9CAE-3F0C-31D5-9556-62586CC5072D}.Debug|x86.ActiveCfg = Debug|Win32
 		{92EF9CAE-3F0C-31D5-9556-62586CC5072D}.Debug|x86.Build.0 = Debug|Win32
 		{92EF9CAE-3F0C-31D5-9556-62586CC5072D}.DLL Debug - DLL OpenSSL - DLL LibSSH2|x64.ActiveCfg = Release|Win32

UnitTests/TestCacheFolder/test1/test.rtf

@@ -0,0 +1 @@
+{\rtf1}

UnitTests/UnitTests.vcxproj

@@ -198,6 +198,9 @@
       <Project>{b3df927f-a1ce-4f50-a621-a4c3a06e4f8a}</Project>
     </ProjectReference>
   </ItemGroup>
+  <ItemGroup>
+    <None Include="TestCacheFolder\test1\test.rtf" />
+  </ItemGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
   </ImportGroup>

UnitTests/UnitTests.vcxproj.filters

@@ -13,6 +13,12 @@
       <UniqueIdentifier>{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}</UniqueIdentifier>
       <Extensions>rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms</Extensions>
     </Filter>
+    <Filter Include="TestCacheFolder">
+      <UniqueIdentifier>{fe235f4d-95ea-4c37-9bb0-dc9e89f75676}</UniqueIdentifier>
+    </Filter>
+    <Filter Include="TestCacheFolder\test1">
+      <UniqueIdentifier>{18c83c4f-d98e-4993-b622-6d7c1fa75162}</UniqueIdentifier>
+    </Filter>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="stdafx.h">
@@ -57,4 +63,9 @@
       <Filter>Source Files</Filter>
     </ClCompile>
   </ItemGroup>
+  <ItemGroup>
+    <None Include="TestCacheFolder\test1\test.rtf">
+      <Filter>TestCacheFolder\test1</Filter>
+    </None>
+  </ItemGroup>
 </Project>

UnitTests/UploadManagerTest.cpp

@@ -48,7 +48,8 @@ namespace UnitTests
 				Assert::AreEqual(version.c_str(), TEST_SERVER_VERSION); // Connectivity test
 
 				CUploadManager uploadManager(siaCurl, &driveConfig);
-				uploadManager.AddOrUpdate(L"/test1/test.txt", L"./test1/test.txt");
+				uploadManager.AddOrUpdate(L"/test1/test.rtf", L"./TestCacheFolder/test1/test.rtf");
+				Sleep(-1);
 			}
 			catch (SQLite::Exception e)
 			{