BlockStorage/siadrive
Archived
1
0
Fork 0
Code Activity

Create and delete

Browse Source
This commit is contained in:
Scott E. Graves
2017-03-28 23:15:14 -05:00
parent 610df85f2e
commit 81c0aed78c
6 changed files with 143 additions and 186 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
include/siadrive_api/filepath.h
View File

@@ -42,6 +42,7 @@ public:
FilePath& MakeAbsolute();
FilePath& SkipRoot();
FilePath& StripToFileName();
bool CreateEmptyFile();
public:
FilePath& operator=(const FilePath& filePath);

34
include/siadrive_api/uploadmanager.h
View File

@@ -7,6 +7,7 @@
#include <deque>
#include <siacurl.h>
#include <eventsystem.h>
#include <filepath.h>
NS_BEGIN(Sia)
NS_BEGIN(Api)
@@ -21,7 +22,6 @@ public:
Queued,
Modified,
Uploading,
Remove,
Complete,
Error
};
@@ -57,8 +57,8 @@ private:
private:
void HandleFileRemove(const CSiaCurl& siaCurl, const SString& siaPath, const SString& siaDriveFilePath);
bool CreateSiaDriveFile(const SString& siaPath, const SString& filePath, const SString& tempSourcePath, const SString& siaDriveFilePath);
void UpdateFileQueueOnStartup();
void DeleteFilesRemovedFromSia(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig, const bool& isStartup = false);
void RemoveFileFromSia(const CSiaCurl& siaCurl, const SString& siaPath, FilePath removeFilePath);
protected:
virtual void AutoThreadCallback(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig) override;
@@ -746,36 +746,6 @@ public:
}
};
class FileRemoveAdded :
public CEvent
{
public:
FileRemoveAdded(const SString& siaPath) :
_siaPath(siaPath)
{
}
public:
virtual ~FileRemoveAdded()
{
}
private:
const SString _siaPath;
public:
virtual SString GetSingleLineMessage() const override
{
return L"FileRemoveAdded|SP|" + _siaPath;
}
virtual std::shared_ptr<CEvent> Clone() const override
{
return std::shared_ptr<CEvent>(new FileRemoveAdded(_siaPath));
}
};
class SourceFileNotFound :
public CEvent
{

11
src/siadrive_api/filepath.cpp
View File

@@ -253,3 +253,14 @@ FilePath& FilePath::StripToFileName()
return *this;
}
bool FilePath::CreateEmptyFile()
{
FILE* fp;
if (_wfopen_s(&fp, &_path[0], L"w+") == 0)
{
fclose(fp);
}
return false;
}

9
src/siadrive_api/siarenter.cpp
View File

@@ -46,15 +46,6 @@ using namespace Sia::Api;
}
}*/
/*
typedef struct
{
std::uint8_t DurationInMonths;
SiaCurrency Funds;
std::uint64_t Hosts;
std::uint64_t RenewWindowInBlocks;
} _SiaRenterAllowance;*/
CSiaApi::_CSiaRenter::_CSiaRenter(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig) :
CSiaBase(siaCurl, siaDriveConfig),
_Funds(0),

162
src/siadrive_api/uploadmanager.cpp
View File

@@ -77,9 +77,6 @@ SString CUploadManager::UploadStatusToString(const UploadStatus& uploadStatus)
case UploadStatus::Queued:
return L"Queued";
case UploadStatus::Remove:
return L"Remove";
case UploadStatus::Uploading:
return L"Uploading";
@@ -105,9 +102,6 @@ CUploadManager::CUploadManager(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriv
throw StartupException(L"Failed to remove '.siadrive.temp' files");
}
// Re-add files to file action queue
UpdateFileQueueOnStartup();
// Detect files that have been removed since last startup
DeleteFilesRemovedFromSia(siaCurl, siaDriveConfig, true);
@@ -121,42 +115,6 @@ CUploadManager::~CUploadManager()
StopAutoThread();
}
void CUploadManager::UpdateFileQueueOnStartup()
{
try
{
// Re-add Copying and Remove
SQLite::Statement query(_uploadDatabase, QUERY_UPLOADS_BY_STATUS);
query.bind("@status", static_cast<unsigned>(UploadStatus::Remove));
while (query.executeStep())
{
SString siaPath = static_cast<const char*>(query.getColumn(query.getColumnIndex("sia_path")));
SString filePath = static_cast<const char*>(query.getColumn(query.getColumnIndex("file_path")));
SString siaDriveFilePath = static_cast<const char*>(query.getColumn(query.getColumnIndex("sd_file_path")));
UploadStatus uploadStatus = static_cast<UploadStatus>(query.getColumn(query.getColumnIndex("status")).getUInt());
FilePath rootPath = filePath;
rootPath.RemoveFileName();
// Strip drive specification (i.e. C:\)
// TODO If mount to folder is ever enabled, this will need to change
SString siaDriveFileName = GenerateSha256(&filePath[3]) + L".siadrive";
FilePath tempSourcePath(rootPath, (siaDriveFileName + L".temp"));
if (uploadStatus == UploadStatus::Remove)
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(FileRemoveAdded(siaPath)));
this->HandleFileRemove(CSiaCurl(GetHostConfig()), siaPath, siaDriveFilePath);
}
}
}
catch (SQLite::Exception e)
{
SString msg = e.getErrorStr();
throw StartupException(msg);
}
}
void CUploadManager::DeleteFilesRemovedFromSia(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig, const bool& isStartup)
{
CSiaFileTreePtr fileTree(new CSiaFileTree(siaCurl, siaDriveConfig));
@@ -177,6 +135,30 @@ void CUploadManager::DeleteFilesRemovedFromSia(const CSiaCurl& siaCurl, CSiaDriv
}
}
void CUploadManager::RemoveFileFromSia(const CSiaCurl& siaCurl, const SString& siaPath, FilePath removeFilePath = SString("*"))
{
json response;
SiaCurlError cerror = siaCurl.Post(SString(L"/renter/delete/") + siaPath, {}, response);
if (ApiSuccess(cerror))
{
SQLite::Statement del(_uploadDatabase, DELETE_UPLOAD);
del.bind("@sia_path", SString::ToUtf8(siaPath).c_str());
auto ret = del.exec();
if (ret >= 0)
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(FileRemoved(siaPath, removeFilePath)));
}
else
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DatabaseDeleteFailed(siaPath, removeFilePath, del.getErrorMsg())));
}
}
else
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(FailedToDeleteFromSia(siaPath, removeFilePath, cerror)));
}
}
void CUploadManager::HandleFileRemove(const CSiaCurl& siaCurl, const SString& siaPath, const SString& siaDriveFilePath)
{
try
@@ -189,48 +171,27 @@ void CUploadManager::HandleFileRemove(const CSiaCurl& siaCurl, const SString& si
FilePath removeFilePath = SString(static_cast<const char*>(query.getColumn(query.getColumnIndex("file_path"))));
UploadStatus uploadStatus = static_cast<UploadStatus>(static_cast<unsigned>(query.getColumn(query.getColumnIndex("status"))));
// Make sure status is still remove
if (uploadStatus == UploadStatus::Remove)
bool deleteFromSia = true;
if (removeFilePath.IsFile())
{
bool deleteFromDb = true;
if (removeFilePath.IsFile())
if (RetryDeleteFileIfExists(removeFilePath))
{
if (RetryDeleteFileIfExists(removeFilePath))
if (!RetryDeleteFileIfExists(siaDriveFilePath))
{
if (!RetryDeleteFileIfExists(siaDriveFilePath))
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DeleteSiaDriveFileFailed(siaPath, removeFilePath, siaDriveFilePath)));
}
}
else
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(RemoveFileFailed(siaPath, removeFilePath)));
deleteFromDb = false;
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DeleteSiaDriveFileFailed(siaPath, removeFilePath, siaDriveFilePath)));
}
}
else
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(RemoveFileFailed(siaPath, removeFilePath)));
deleteFromSia = false;
}
}
if (deleteFromDb)
{
json response;
SiaCurlError cerror = siaCurl.Post(SString(L"/renter/delete/") + siaPath, {}, response);
if (ApiSuccess(cerror))
{
SQLite::Statement del(_uploadDatabase, DELETE_UPLOAD);
del.bind("@sia_path", SString::ToUtf8(siaPath).c_str());
if (del.exec() == 1)
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(FileRemoved(siaPath, removeFilePath)));
}
else
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DatabaseDeleteFailed(siaPath, removeFilePath, del.getErrorMsg())));
}
}
else
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(FailedToDeleteFromSia(siaPath, removeFilePath, cerror)));
}
}
if (deleteFromSia)
{
RemoveFileFromSia(siaCurl, siaPath, removeFilePath);
}
}
}
@@ -326,15 +287,11 @@ void CUploadManager::AutoThreadCallback(const CSiaCurl& siaCurl, CSiaDriveConfig
});
// Removed by another client
if (it == fileList.end())
{
SET_STATUS(UploadStatus::Remove, ExternallyRemovedFileDetected, ModifyUploadStatusFailed)
if (statusUpdated)
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(FileRemoveAdded(siaPath)));
this->HandleFileRemove(siaCurl, siaPath, siaDriveFilePath);
}
}
if (it == fileList.end())
{
// TODO SET_STATUS(UploadStatus::Remove, ExternallyRemovedFileDetected, ModifyUploadStatusFailed)
HandleFileRemove(siaCurl, siaPath, siaDriveFilePath);
}
// Changed file detected
else if (uploadStatus == UploadStatus::Modified)
{
@@ -473,7 +430,7 @@ UploadError CUploadManager::AddOrUpdate(const SString& siaPath, SString filePath
SQLite::Statement insert(_uploadDatabase, INSERT_UPLOAD);
insert.bind("@sia_path", SString::ToUtf8(siaPath).c_str());
insert.bind("@file_path", SString::ToUtf8(filePath).c_str());
insert.bind("@sd_file_path", SString::ToUtf8(siaDriveFileName).c_str());
insert.bind("@sd_file_path", SString::ToUtf8(static_cast<SString>(siaDriveFilePath)).c_str());
insert.bind("@status", static_cast<unsigned>(UploadStatus::Queued));
if (insert.exec() == 1)
{
@@ -510,17 +467,18 @@ UploadError CUploadManager::AddOrUpdate(const SString& siaPath, SString filePath
UploadError CUploadManager::Remove(const SString& siaPath)
{
UploadError ret = UploadError::Success;
std::lock_guard<std::mutex> l(_uploadMutex);
bool remove = false;
SString siaDriveFilePath;
try
{
bool remove;
std::lock_guard<std::mutex> l(_uploadMutex);
SQLite::Statement query(_uploadDatabase, QUERY_STATUS);
query.bind("@sia_path", SString::ToUtf8(siaPath).c_str());
if (query.executeStep())
{
SString filePath = static_cast<const char*>(query.getColumn(query.getColumnIndex("file_path")));
SString siaDriveFilePath = static_cast<const char*>(query.getColumn(query.getColumnIndex("sd_file_path")));
siaDriveFilePath = static_cast<const char*>(query.getColumn(query.getColumnIndex("sd_file_path")));
UploadStatus uploadStatus = static_cast<UploadStatus>(static_cast<unsigned>(query.getColumn(query.getColumnIndex("status"))));
switch (uploadStatus)
{
@@ -529,12 +487,7 @@ UploadError CUploadManager::Remove(const SString& siaPath)
case UploadStatus::Modified:
case UploadStatus::Uploading:
{
SET_STATUS(UploadStatus::Remove, UploadStatusSetToRemoved, ModifyUploadStatusFailed)
remove = statusUpdated;
if (!statusUpdated)
{
ret = UploadError::DatabaseError;
}
remove = true;
}
break;
@@ -544,13 +497,11 @@ UploadError CUploadManager::Remove(const SString& siaPath)
}
break;
}
if (remove)
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(FileRemoveAdded(siaPath)));
this->HandleFileRemove(CSiaCurl(GetHostConfig()), siaPath, siaDriveFilePath);
}
}
else
{
RemoveFileFromSia(CSiaCurl(GetHostConfig()), siaPath);
}
}
catch (SQLite::Exception e)
{
@@ -558,5 +509,10 @@ UploadError CUploadManager::Remove(const SString& siaPath)
ret = UploadError::DatabaseError;
}
if (remove)
{
HandleFileRemove(CSiaCurl(GetHostConfig()), siaPath, siaDriveFilePath);
}
return ret;
}

112
src/siadrive_dokan_api/siadokandrive.cpp
View File

@@ -20,6 +20,7 @@ private:
{
FilePath SiaPath;
FilePath CacheFilePath;
bool Dummy;
bool Changed;
} OpenFileInfo;
@@ -53,36 +54,49 @@ private:
return ret;
}
static bool AddFileToCache(const SString& siaPath, const SString& cacheLocation)
{
static bool AddFileToCache(const SString& siaPath)
{
FilePath tempFilePath = FilePath::GetTempDirectory();
tempFilePath.Append(GenerateSha256(siaPath) + ".siatmp");
// TODO Check cache size is large enough to hold new file
bool ret = ApiSuccess(_siaApi->GetRenter()->DownloadFile(siaPath, tempFilePath));
if (ret)
{
FilePath src(tempFilePath);
FilePath dest(GetCacheLocation(), siaPath);
// TODO Check cache size is large enough to hold new file
bool ret = ApiSuccess(_siaApi->GetRenter()->DownloadFile(siaPath, tempFilePath));
if (ret)
{
FilePath src(tempFilePath);
FilePath dest(GetCacheLocation(), siaPath);
ret = src.MoveFile(dest);
if (!ret)
{
src.DeleteFile();
}
}
}
return ret;
}
return ret;
}
static void HandleFileClose(const FilePath& filePath, const ULONG64 id, const std::uint64_t& fileSize)
inline static bool AddDummyFileToCache(const SString& siaPath)
{
FilePath dest(GetCacheLocation(), siaPath);
return dest.CreateEmptyFile();
}
static void HandleSiaFileClose(const FilePath& filePath, const ULONG64 id, const std::uint64_t& fileSize, const bool& deleteOnClose)
{
std::lock_guard<std::mutex> l(_dokanMutex);
auto ofi = _openFileMap.find(id);
if (ofi != _openFileMap.end())
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DokanCloseFile(ofi->second.CacheFilePath)));
QueueUploadIfChanged(id, fileSize);
if (deleteOnClose)
{
// TODO Handle failure
_uploadManager->Remove(ofi->second.SiaPath);
}
else
{
QueueUploadIfChanged(id, fileSize);
}
_openFileMap.erase(id);
}
}
@@ -330,16 +344,25 @@ private:
}
}
bool isDummy = false;
if (ret == STATUS_SUCCESS)
{
// If file must exist, then check for it in cache location. If not found,
// it must be downloaded first and placed in cache
if (!isCreateOp && !cacheFilePath.IsFile())
{
if (!AddFileToCache(siaPath, cacheFilePath))
{
ret = STATUS_INVALID_SERVER_STATE;
}
if (exists)
{
isDummy = AddDummyFileToCache(siaPath);
if (!isDummy)
{
ret = STATUS_ACCESS_DENIED;
}
}
else
{
ret = STATUS_NOT_FOUND;
}
}
if (ret == STATUS_SUCCESS)
@@ -373,6 +396,7 @@ private:
ofi.SiaPath = siaPath;
ofi.CacheFilePath = cacheFilePath;
// TODO Detect if file is read-only
ofi.Dummy = isDummy;
ofi.Changed = false;
std::lock_guard<std::mutex> l(_dokanMutex);
_openFileMap.insert({ dokanFileInfo->Context, ofi });
@@ -517,7 +541,7 @@ private:
{
LARGE_INTEGER li = { 0 };
::GetFileSizeEx(handle, &li);
HandleFileClose(filePath, id, li.QuadPart);
HandleSiaFileClose(filePath, id, li.QuadPart, dokanFileInfo->DeleteOnClose ? true : false);
}
::CloseHandle(reinterpret_cast<HANDLE>(dokanFileInfo->Context));
dokanFileInfo->Context = 0;
@@ -530,11 +554,7 @@ private:
BOOL opened = FALSE;
NTSTATUS ret = STATUS_SUCCESS;
FilePath cachePath = GetCacheLocation();
if (FilePath::DirSep != fileName)
{
cachePath.Append(fileName);
}
FilePath cachePath(GetCacheLocation(), fileName);
SString siaPath = CSiaApi::FormatToSiaPath(FilePath(fileName).SkipRoot());
auto siaFileTree = GetFileTree();
@@ -556,7 +576,23 @@ private:
if (ret == STATUS_SUCCESS)
{
if (!::GetFileInformationByHandle(handle, handleFileInfo))
bool isDummy = false;
if (siaFile)
{
std::lock_guard<std::mutex> l(_fileTreeMutex);
isDummy = _openFileMap.find(dokanFileInfo->Context) != _openFileMap.end();
}
if (isDummy)
{
LARGE_INTEGER li = { 0 };
li.QuadPart = siaFile->GetFileSize();
handleFileInfo->dwFileAttributes = FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_ARCHIVE;
handleFileInfo->nFileSizeHigh = li.HighPart;
handleFileInfo->nFileSizeLow = li.LowPart;
ret = STATUS_SUCCESS;
}
else if (!::GetFileInformationByHandle(handle, handleFileInfo))
{
// fileName is a root directory
// in this case, FindFirstFile can't get directory information
@@ -571,19 +607,7 @@ private:
if (findHandle == INVALID_HANDLE_VALUE)
{
DWORD error = ::GetLastError();
if (siaFile)
{
LARGE_INTEGER li = { 0 };
li.QuadPart = siaFile->GetFileSize();
handleFileInfo->dwFileAttributes = FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_ARCHIVE;
handleFileInfo->nFileSizeHigh = li.HighPart;
handleFileInfo->nFileSizeLow = li.LowPart;
ret = STATUS_SUCCESS;
}
else
{
ret = DokanNtStatusFromWin32(error);
}
ret = DokanNtStatusFromWin32(error);
}
else
{
@@ -669,6 +693,7 @@ private:
LONGLONG offset,
PDOKAN_FILE_INFO dokanFileInfo)
{
// TODO Check dummy and add to cache if not found
FilePath filePath(GetCacheLocation(), fileName);
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DokanReadFile(filePath)));
@@ -716,6 +741,7 @@ private:
LONGLONG offset,
PDOKAN_FILE_INFO dokanFileInfo)
{
// TODO Check dummy and add to cache if not found
FilePath filePath(GetCacheLocation(), fileName);
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DokanWriteFile(filePath)));
@@ -823,6 +849,7 @@ private:
static NTSTATUS DOKAN_CALLBACK Sia_SetEndOfFile(LPCWSTR fileName, LONGLONG byteOffset, PDOKAN_FILE_INFO dokanFileInfo)
{
NTSTATUS ret = STATUS_SUCCESS;
// TODO Check dummy and add to cache if not found
FilePath filePath(GetCacheLocation(), fileName);
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DokanSetEndOfFile(filePath)));
@@ -863,6 +890,7 @@ private:
static void DOKAN_CALLBACK Sia_Cleanup(LPCWSTR fileName, PDOKAN_FILE_INFO dokanFileInfo)
{
FilePath filePath(GetCacheLocation(), fileName);
ULONG64 id = dokanFileInfo->DeleteOnClose;
if (dokanFileInfo->Context)
{
HANDLE handle = reinterpret_cast<HANDLE>(dokanFileInfo->Context);
@@ -870,7 +898,7 @@ private:
{
LARGE_INTEGER li = { 0 };
::GetFileSizeEx(handle, &li);
HandleFileClose(filePath, dokanFileInfo->Context, li.QuadPart);
HandleSiaFileClose(filePath, dokanFileInfo->Context, li.QuadPart, dokanFileInfo->DeleteOnClose ? true : false);
}
::CloseHandle(handle);
@@ -894,7 +922,6 @@ private:
{
if (filePath.DeleteFile())
{
}
else
{
@@ -968,7 +995,7 @@ private:
static NTSTATUS DOKAN_CALLBACK Sia_DeleteFileW(LPCWSTR fileName, PDOKAN_FILE_INFO dokanFileInfo)
{
// TODO Handle files that aren't cached
// TODO Check dummy and add to cache if not found
NTSTATUS ret = STATUS_SUCCESS;
HANDLE handle = reinterpret_cast<HANDLE>(dokanFileInfo->Context);
@@ -995,7 +1022,7 @@ private:
static NTSTATUS DOKAN_CALLBACK Sia_MoveFileW(LPCWSTR fileName, LPCWSTR NewFileName, BOOL ReplaceIfExisting, PDOKAN_FILE_INFO dokanFileInfo)
{
// TODO Handle files that aren't cached
// TODO Check dummy and add to cache if not found
NTSTATUS ret = STATUS_SUCCESS;
FilePath filePath(GetCacheLocation(), fileName);
@@ -1157,6 +1184,7 @@ private:
static NTSTATUS DOKAN_CALLBACK Sia_SetAllocationSize(LPCWSTR fileName, LONGLONG allocSize, PDOKAN_FILE_INFO dokanFileInfo)
{
// TODO Check dummy and add to cache if not found
NTSTATUS ret = STATUS_SUCCESS;
FilePath filePath(GetCacheLocation(), fileName);
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DokanSetAllocationSize(filePath)));