BlockStorage/siadrive
Archived
1
0
Fork 0
Code Activity

Upload manager changes

Browse Source
This commit is contained in:
Scott E. Graves
2017-02-23 00:55:18 -06:00
parent 543854ca27
commit e0ec942781
9 changed files with 144 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
SiaDrive.Api/AutoThread.cpp
View File

@@ -4,9 +4,15 @@
using namespace Sia::Api;
CAutoThread::CAutoThread(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig) :
CAutoThread(siaCurl, siaDriveConfig, nullptr)
{
}
CAutoThread::CAutoThread(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig, std::function<void(const CSiaCurl&, CSiaDriveConfig*)> autoThreadCallback) :
_siaCurl(siaCurl.GetHostConfig()),
_siaDriveConfig(siaDriveConfig),
_stopEvent(::CreateEvent(nullptr, FALSE, FALSE, nullptr))
_stopEvent(::CreateEvent(nullptr, FALSE, FALSE, nullptr)),
_AutoThreadCallback(autoThreadCallback)
{
}
@@ -16,6 +22,14 @@ CAutoThread::~CAutoThread()
::CloseHandle(_stopEvent);
}
void CAutoThread::AutoThreadCallback(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig)
{
if (_AutoThreadCallback)
{
_AutoThreadCallback(siaCurl, siaDriveConfig);
}
}
void CAutoThread::StartAutoThread()
{
std::lock_guard<std::mutex> l(_startStopMutex);

6
SiaDrive.Api/AutoThread.h
View File

@@ -10,6 +10,7 @@ class AFX_EXT_CLASS CAutoThread
{
public:
CAutoThread(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig);
CAutoThread(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig, std::function<void(const CSiaCurl&, CSiaDriveConfig*)> autoThreadCallback);
public:
virtual ~CAutoThread();
@@ -20,11 +21,14 @@ private:
HANDLE _stopEvent;
std::unique_ptr<std::thread> _thread;
std::mutex _startStopMutex;
std::function<void(const CSiaCurl&, CSiaDriveConfig*)> _AutoThreadCallback;
protected:
virtual void AutoThreadCallback(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig);
public:
void StartAutoThread();
void StopAutoThread();
virtual void AutoThreadCallback(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig) = 0;
};
NS_END(2)

109
SiaDrive.Api/UploadManager.cpp
View File

@@ -34,17 +34,38 @@ static void CreateTableIfNotFound(SQLite::Database* database, const String& tabl
CUploadManager::CUploadManager(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig) :
CAutoThread(siaCurl, siaDriveConfig),
_uploadDatabase(siaDriveConfig->GetRenter_UploadDbFilePath(), SQLite::OPEN_CREATE | SQLite::OPEN_READWRITE),
_siaDriveConfig(siaDriveConfig)
_siaDriveConfig(siaDriveConfig),
_fileThread(siaCurl, siaDriveConfig, [this](const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig) { this->FileThreadCallback(siaCurl, siaDriveConfig); })
{
CreateTableIfNotFound(&_uploadDatabase, UPLOAD_TABLE, UPLOAD_TABLE_COLUMNS);
StartAutoThread();
_fileThread.StartAutoThread();
}
CUploadManager::~CUploadManager()
{
_fileThread.StopAutoThread();
StopAutoThread();
}
void CUploadManager::FileThreadCallback(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig)
{
std::function<void()> nextFile = nullptr;
{
std::lock_guard<std::mutex> l(_fileQueueMutex);
if (_fileQueue.size())
{
nextFile = _fileQueue.front();
_fileQueue.pop_front();
}
}
if (nextFile)
{
nextFile();
}
}
void CUploadManager::AutoThreadCallback(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig)
{
bool processNext = true;
@@ -152,8 +173,31 @@ UploadStatus CUploadManager::GetUploadStatus(const String& siaPath)
return uploadStatus;
}
void CUploadManager::AddOrUpdate(const String& siaPath, const String& filePath)
// The real source file is copied to a hidden file. The hidden filename is constructed by generating an SHA25 hash of the
// source path (all lowercase). '.siadrive.temp' will be used as the extension. After copy is successful, the extension
// is renamed to '.siadrive'. '.siadrive' files will be hidden/system and used by the Dokan API for file i/o until Sia upload
// is complete. If a change occurs, the file will be deleted from Sia (cancelling the in-progress upload) and renamed to the
// real source path. The process will then start over again as if the file was new.
// If the file has been fully uploaded, the hidden file should not exist, so rename will not occur; however, the file
// will be deleted from Sia and treated as new.
// Uploads will always use the real file. User i/o will occur against the hidden file only temporarily. Since upload will take
// longer to handle than a normal file copy, this seems to be the best compromise for performance.
UploadError CUploadManager::AddOrUpdate(const String& siaPath, String filePath)
{
UploadError ret = UploadError::Success;
// Relative to absolute and grab parent folder of source
String rootPath;
{
String temp;
temp.resize(MAX_PATH + 1);
filePath = _wfullpath(&temp[0], filePath.c_str(), MAX_PATH);
temp = filePath;
::PathRemoveFileSpec(&temp[0]);
rootPath = temp;
}
// Lock here - if file is modified again before a prior upload is complete, delete it and
// start again later
std::lock_guard<std::mutex> l(_uploadMutex);
@@ -177,7 +221,7 @@ void CUploadManager::AddOrUpdate(const String& siaPath, const String& filePath)
addOrUpdate.bind("@sia_path", CW2A(siaPath.c_str()).m_psz);
addOrUpdate.bind("@file_path", CW2A(filePath.c_str()).m_psz);
addOrUpdate.bind("@temp_path", CW2A(tempPath.c_str()).m_psz);
addOrUpdate.bind("@status", static_cast<unsigned>(UploadStatus::Queued));*/
addOrUpdate.bind("@status", static_cast<unsigned>(UploadStatus::Copying));*/
// While copy is active, point to normal file.
// After copy, use temp file as real file until upload is complete
@@ -185,28 +229,56 @@ void CUploadManager::AddOrUpdate(const String& siaPath, const String& filePath)
// Error Scenarios:
// Crash before db update to status copying - file will be re-uploaded automatically, if complete; otherwise, deleted
// Need to keep track of files as being copied and then there status
// Crash Scenarios:
// Crash before copy begins - on startup, check for copying status with no .siadrive
// Crash during copy - on startup, check for copying status and delete .siadrive
// Crash after copy but before db update - on startup, check for copying status and delete .siadrive
String tempPath;
tempPath.resize(MAX_PATH + 1);
PathCombine(&tempPath[0], CA2W(_siaDriveConfig->GetTempFolder().c_str()), (GenerateSha256(filePath) + L".siadrive").c_str());
String siaDriveFileName = GenerateSha256(&filePath[3]) + L".siadrive";
// Queue this
if (::CopyFile(filePath.c_str(), tempPath.c_str(), FALSE))
String siaDriveFilePath;
siaDriveFilePath.resize(MAX_PATH + 1);
PathCombine(&siaDriveFilePath[0], rootPath.c_str(), siaDriveFileName.c_str());
String tempSourcePath;
tempSourcePath.resize(MAX_PATH + 1);
PathCombine(&tempSourcePath[0], rootPath.c_str(), (siaDriveFileName + L".temp").c_str());
// Queue file operations
std::lock_guard<std::mutex> l2(_fileQueueMutex);
_fileQueue.push_back([this,filePath, tempSourcePath, siaDriveFilePath]()
{
tempPath = L"";
}
else
{
if (!::DeleteFile(tempPath.c_str()))
// TODO Retry a few times
if (::CopyFile(filePath.c_str(), tempSourcePath.c_str(), FALSE))
{
// Rename to real temp
// TODO Retry a few times
if (::PathFileExists(siaDriveFilePath.c_str()))
{
// TODO Retry a few times
::DeleteFile(siaDriveFilePath.c_str());
}
if (::MoveFile(tempSourcePath.c_str(), siaDriveFilePath.c_str()))
{
// Change status to queued
}
}
// error condition
}
else
{
// If temp copy fails, try to delete
// If partial copy and file is unable to be deleted, log warning
if (::PathFileExists(tempSourcePath.c_str()))
{
// TODO Retry a few times
::DeleteFile(tempSourcePath.c_str());
}
// Requeue
}
});
}
return ret;
}
void CUploadManager::PurgeCompleteStatus()
@@ -219,7 +291,8 @@ void CUploadManager::PurgeErrorStatus()
}
void CUploadManager::Remove(const String& siaPath)
UploadError CUploadManager::Remove(const String& siaPath)
{
UploadError ret = UploadError::Success;
return ret;
}

14
SiaDrive.Api/UploadManager.h
View File

@@ -21,6 +21,11 @@ public:
Error
};
enum class _UploadError
{
Success
};
private:
typedef struct
{
@@ -41,18 +46,23 @@ private:
SQLite::Database _uploadDatabase;
std::mutex _uploadMutex;
CSiaDriveConfig* _siaDriveConfig;
CAutoThread _fileThread;
std::mutex _fileQueueMutex;
std::deque<std::function<void()>> _fileQueue;
protected:
virtual void AutoThreadCallback(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig) override;
void FileThreadCallback(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig);
public:
_UploadStatus GetUploadStatus(const String& siaPath);
void AddOrUpdate(const String& siaPath, const String& filePath);
void Remove(const String& siaPath);
_UploadError AddOrUpdate(const String& siaPath, String filePath);
_UploadError Remove(const String& siaPath);
void PurgeCompleteStatus();
void PurgeErrorStatus();
};
typedef Sia::Api::CUploadManager::_UploadStatus UploadStatus;
typedef Sia::Api::CUploadManager::_UploadError UploadError;
NS_END(2)

6
SiaDrive.sln
View File

@@ -734,7 +734,8 @@ Global
{B3DF927F-A1CE-4F50-A621-A4C3A06E4F8A}.RelWithDebInfo|x64.Build.0 = Release|x64
{B3DF927F-A1CE-4F50-A621-A4C3A06E4F8A}.RelWithDebInfo|x86.ActiveCfg = Release|Win32
{B3DF927F-A1CE-4F50-A621-A4C3A06E4F8A}.RelWithDebInfo|x86.Build.0 = Release|Win32
{BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.Debug|x64.ActiveCfg = Debug|Win32
{BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.Debug|x64.ActiveCfg = Debug|x64
{BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.Debug|x64.Build.0 = Debug|x64
{BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.Debug|x86.ActiveCfg = Debug|Win32
{BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.Debug|x86.Build.0 = Debug|Win32
{BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.DLL Debug - DLL OpenSSL - DLL LibSSH2|x64.ActiveCfg = Release|Win32
@@ -858,7 +859,8 @@ Global
{BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.RelWithDebInfo|x64.ActiveCfg = RelWithDebInfo|Win32
{BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Win32
{BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Win32
{92EF9CAE-3F0C-31D5-9556-62586CC5072D}.Debug|x64.ActiveCfg = Debug|Win32
{92EF9CAE-3F0C-31D5-9556-62586CC5072D}.Debug|x64.ActiveCfg = Debug|x64
{92EF9CAE-3F0C-31D5-9556-62586CC5072D}.Debug|x64.Build.0 = Debug|x64
{92EF9CAE-3F0C-31D5-9556-62586CC5072D}.Debug|x86.ActiveCfg = Debug|Win32
{92EF9CAE-3F0C-31D5-9556-62586CC5072D}.Debug|x86.Build.0 = Debug|Win32
{92EF9CAE-3F0C-31D5-9556-62586CC5072D}.DLL Debug - DLL OpenSSL - DLL LibSSH2|x64.ActiveCfg = Release|Win32

1
UnitTests/TestCacheFolder/test1/test.rtf Normal file
View File

@@ -0,0 +1 @@
{\rtf1}

3
UnitTests/UnitTests.vcxproj
View File

@@ -198,6 +198,9 @@
<Project>{b3df927f-a1ce-4f50-a621-a4c3a06e4f8a}</Project>
</ProjectReference>
</ItemGroup>
<ItemGroup>
<None Include="TestCacheFolder\test1\test.rtf" />
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>

11
UnitTests/UnitTests.vcxproj.filters
View File

@@ -13,6 +13,12 @@
<UniqueIdentifier>{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}</UniqueIdentifier>
<Extensions>rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms</Extensions>
</Filter>
<Filter Include="TestCacheFolder">
<UniqueIdentifier>{fe235f4d-95ea-4c37-9bb0-dc9e89f75676}</UniqueIdentifier>
</Filter>
<Filter Include="TestCacheFolder\test1">
<UniqueIdentifier>{18c83c4f-d98e-4993-b622-6d7c1fa75162}</UniqueIdentifier>
</Filter>
</ItemGroup>
<ItemGroup>
<ClInclude Include="stdafx.h">
@@ -57,4 +63,9 @@
<Filter>Source Files</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<None Include="TestCacheFolder\test1\test.rtf">
<Filter>TestCacheFolder\test1</Filter>
</None>
</ItemGroup>
</Project>

3
UnitTests/UploadManagerTest.cpp
View File

@@ -48,7 +48,8 @@ namespace UnitTests
Assert::AreEqual(version.c_str(), TEST_SERVER_VERSION); // Connectivity test
CUploadManager uploadManager(siaCurl, &driveConfig);
uploadManager.AddOrUpdate(L"/test1/test.txt", L"./test1/test.txt");
uploadManager.AddOrUpdate(L"/test1/test.rtf", L"./TestCacheFolder/test1/test.rtf");
Sleep(-1);
}
catch (SQLite::Exception e)
{