BlockStorage/siadrive
Archived
1
0
Fork 0
Code Activity

Begin refactoring upload manager

Browse Source
This commit is contained in:
Scott E. Graves
2017-03-30 19:10:15 -05:00
parent 6e250d8ab3
commit 4f226eadbb
2 changed files with 153 additions and 180 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
include/siadrive_api/uploadmanager.h
View File

@@ -20,7 +20,6 @@ public:
{
NotFound,
Queued,
Modified,
Uploading,
Complete,
Error
@@ -50,19 +49,19 @@ public:
virtual ~CUploadManager();
private:
CSiaDriveConfig* _siaDriveConfig;
SQLite::Database _uploadDatabase;
std::mutex _uploadMutex;
SString _activeSiaPath;
private:
void HandleFileRemove(const CSiaCurl& siaCurl, const SString& siaPath, const SString& siaDriveFilePath);
CSiaDriveConfig* GetSiaDriveConfig() const { return _siaDriveConfig; }
bool HandleFileRemove(const CSiaCurl& siaCurl, const SString& siaPath, const SString& siaDriveFilePath = "");
bool CreateSiaDriveFile(const SString& siaPath, const SString& filePath, const SString& tempSourcePath, const SString& siaDriveFilePath);
void DeleteFilesRemovedFromSia(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig, const bool& isStartup = false);
void RemoveFileFromSia(const CSiaCurl& siaCurl, const SString& siaPath, FilePath removeFilePath);
protected:
virtual void AutoThreadCallback(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig) override;
public:
static SString UploadStatusToString(const _UploadStatus& uploadStatus);
@@ -507,6 +506,40 @@ public:
}
};
class RenameModifiedFileFailed :
public CEvent
{
public:
RenameModifiedFileFailed(const SString& siaPath, const SString& filePath, const SString& tempSourcePath) :
_siaPath(siaPath),
_filePath(filePath),
_tempSourcePath(tempSourcePath)
{
}
public:
virtual ~RenameModifiedFileFailed()
{
}
private:
const SString _siaPath;
const SString _filePath;
const SString _tempSourcePath;
public:
virtual SString GetSingleLineMessage() const override
{
return L"RenameModifiedFileFailed|SP|" + _siaPath + L"|FP|" + _filePath + L"|TSP|" + _tempSourcePath;
}
virtual std::shared_ptr<CEvent> Clone() const override
{
return std::shared_ptr<CEvent>(new RenameModifiedFileFailed(_siaPath, _filePath, _tempSourcePath));
}
};
class DeleteTemporarySiaDriveFileFailed :
public CEvent
{

290
src/siadrive_api/uploadmanager.cpp
View File

@@ -13,7 +13,7 @@ using namespace Sia::Api;
#define QUERY_UPLOADS_BY_STATUS "select * from upload_table where status=@status order by id desc limit 1;"
#define QUERY_UPLOADS_BY_2_STATUS "select * from upload_table where (status=@status1 or status=@status2) order by id desc limit 1;"
#define QUERY_UPLOADS_BY_SIA_PATH "select * from upload_table where sia_path=@sia_path order by id desc limit 1;"
#define QUERY_UPLOADS_BY_SIA_PATH_AND_2_STATUS "select * from upload_table where sia_path=@sia_path and (status=@status1 or status=@status2) order by id desc limit 1;"
#define QUERY_UPLOADS_BY_SIA_PATH_AND_STATUS "select * from upload_table where sia_path=@sia_path and status=@status order by id desc limit 1;"
#define UPDATE_STATUS "update upload_table set status=@status where sia_path=@sia_path;"
#define INSERT_UPLOAD "insert into upload_table (sia_path, status, file_path, sd_file_path) values (@sia_path, @status, @file_path, @sd_file_path);"
#define DELETE_UPLOAD "delete from upload_table where sia_path=@sia_path;"
@@ -66,10 +66,7 @@ SString CUploadManager::UploadStatusToString(const UploadStatus& uploadStatus)
return L"Complete";
case UploadStatus::Error:
return L"Error";
case UploadStatus::Modified:
return L"Modified";
return L"Error";
case UploadStatus::NotFound:
return L"Not Found";
@@ -87,6 +84,7 @@ SString CUploadManager::UploadStatusToString(const UploadStatus& uploadStatus)
CUploadManager::CUploadManager(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig) :
CAutoThread(siaCurl, siaDriveConfig),
_siaDriveConfig(siaDriveConfig),
_uploadDatabase(siaDriveConfig->GetRenter_UploadDbFilePath(), SQLite::OPEN_CREATE | SQLite::OPEN_READWRITE)
{
CreateTableIfNotFound(&_uploadDatabase, UPLOAD_TABLE, UPLOAD_TABLE_COLUMNS);
@@ -135,123 +133,103 @@ void CUploadManager::DeleteFilesRemovedFromSia(const CSiaCurl& siaCurl, CSiaDriv
}
}
void CUploadManager::RemoveFileFromSia(const CSiaCurl& siaCurl, const SString& siaPath, FilePath removeFilePath = SString("*"))
{
json response;
SiaCurlError cerror = siaCurl.Post(SString(L"/renter/delete/") + siaPath, {}, response);
if (ApiSuccess(cerror))
{
SQLite::Statement del(_uploadDatabase, DELETE_UPLOAD);
del.bind("@sia_path", SString::ToUtf8(siaPath).c_str());
auto ret = del.exec();
if (ret >= 0)
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(FileRemoved(siaPath, removeFilePath)));
}
else
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DatabaseDeleteFailed(siaPath, removeFilePath, del.getErrorMsg())));
}
}
else
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(FailedToDeleteFromSia(siaPath, removeFilePath, cerror)));
}
}
void CUploadManager::HandleFileRemove(const CSiaCurl& siaCurl, const SString& siaPath, const SString& siaDriveFilePath)
{
try
{
std::lock_guard<std::mutex> l(_uploadMutex);
SQLite::Statement query(_uploadDatabase, QUERY_STATUS);
query.bind("@sia_path", SString::ToUtf8(siaPath).c_str());
if (query.executeStep())
{
FilePath removeFilePath = SString(static_cast<const char*>(query.getColumn(query.getColumnIndex("file_path"))));
UploadStatus uploadStatus = static_cast<UploadStatus>(static_cast<unsigned>(query.getColumn(query.getColumnIndex("status"))));
// Make sure status is still remove
bool deleteFromSia = true;
if (removeFilePath.IsFile())
{
if (RetryDeleteFileIfExists(removeFilePath))
{
if (!RetryDeleteFileIfExists(siaDriveFilePath))
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DeleteSiaDriveFileFailed(siaPath, removeFilePath, siaDriveFilePath)));
}
}
else
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(RemoveFileFailed(siaPath, removeFilePath)));
deleteFromSia = false;
}
}
if (deleteFromSia)
{
RemoveFileFromSia(siaCurl, siaPath, removeFilePath);
}
}
}
catch (SQLite::Exception e)
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DatabaseExceptionOccurred("HandleFileRemove", e)));
}
}
bool CUploadManager::CreateSiaDriveFile(const SString& siaPath, const SString& filePath, const SString& tempSourcePath, const SString& siaDriveFilePath)
bool CUploadManager::HandleFileRemove(const CSiaCurl& siaCurl, const SString& siaPath, const SString& siaDriveFilePath)
{
bool ret = false;
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(CreatingTemporarySiaDriveFile(siaPath, filePath, tempSourcePath)));
if (RetryableAction(::CopyFile(filePath.str().c_str(), tempSourcePath.str().c_str(), FALSE), DEFAULT_RETRY_COUNT, DEFAULT_RETRY_DELAY_MS))
FilePath removeFilePath(GetSiaDriveConfig()->GetCacheFolder(), siaPath);
bool deleteFromSia = true;
if (removeFilePath.IsFile())
{
// Delete existing '.siadrive' file, if found
// !!Should never come here. If so, there was a problem with startup clean-up
if (!RetryDeleteFileIfExists(siaDriveFilePath))
if (RetryDeleteFileIfExists(removeFilePath))
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DeleteSiaDriveFileFailed(siaPath, filePath, siaDriveFilePath)));
}
// Rename '.siadrive.temp' to '.siadrive'
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(RenamingTemporarySiaDriveFile(siaPath, filePath, tempSourcePath, siaDriveFilePath)));
if (RetryableAction(::MoveFile(tempSourcePath.str().c_str(), siaDriveFilePath.str().c_str()), DEFAULT_RETRY_COUNT, DEFAULT_RETRY_DELAY_MS))
{
if (RetryDeleteFileIfExists(tempSourcePath))
if (!siaDriveFilePath.IsNullOrEmpty() && !RetryDeleteFileIfExists(siaDriveFilePath))
{
ret = true;
}
else
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DeleteTemporarySiaDriveFileFailed(siaPath, filePath, tempSourcePath)));
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DeleteSiaDriveFileFailed(siaPath, removeFilePath, siaDriveFilePath)));
}
}
else
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(RenamingTemporarySiaDriveFileFailed(siaPath, filePath, tempSourcePath, siaDriveFilePath)));
if (!RetryDeleteFileIfExists(tempSourcePath))
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DeleteTemporarySiaDriveFileFailed(siaPath, filePath, tempSourcePath)));
}
if (!RetryDeleteFileIfExists(siaDriveFilePath))
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DeleteSiaDriveFileFailed(siaPath, filePath, siaDriveFilePath)));
}
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(RemoveFileFailed(siaPath, removeFilePath)));
deleteFromSia = false;
}
}
else
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(CreatingTemporarySiaDriveFileFailed(siaPath, filePath, tempSourcePath)));
// If temp copy fails, try to delete
// If partial copy and file is unable to be deleted, log warning
if (!RetryDeleteFileIfExists(tempSourcePath))
if (deleteFromSia)
{
json response;
SiaCurlError cerror = siaCurl.Post(SString(L"/renter/delete/") + siaPath, {}, response);
if (ApiSuccess(cerror))
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DeleteTemporarySiaDriveFileFailed(siaPath, filePath, tempSourcePath)));
SQLite::Statement del(_uploadDatabase, DELETE_UPLOAD);
del.bind("@sia_path", SString::ToUtf8(siaPath).c_str());
if (del.exec() >= 0)
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(FileRemoved(siaPath, removeFilePath)));
ret = true;
}
else
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DatabaseDeleteFailed(siaPath, removeFilePath, del.getErrorMsg())));
}
}
else
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(FailedToDeleteFromSia(siaPath, removeFilePath, cerror)));
}
}
return ret;
}
bool CUploadManager::CreateSiaDriveFile(const SString& siaPath, const SString& filePath, const SString& tempSourcePath, const SString& siaDriveFilePath)
{
bool ret = FilePath(siaDriveFilePath).IsFile();
if (!ret)
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(CreatingTemporarySiaDriveFile(siaPath, filePath, tempSourcePath)));
if (RetryableAction(::CopyFile(filePath.str().c_str(), tempSourcePath.str().c_str(), FALSE), DEFAULT_RETRY_COUNT, DEFAULT_RETRY_DELAY_MS))
{
// Rename '.siadrive.temp' to '.siadrive'
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(RenamingTemporarySiaDriveFile(siaPath, filePath, tempSourcePath, siaDriveFilePath)));
if (RetryableAction(::MoveFile(tempSourcePath.str().c_str(), siaDriveFilePath.str().c_str()), DEFAULT_RETRY_COUNT, DEFAULT_RETRY_DELAY_MS))
{
if (RetryDeleteFileIfExists(tempSourcePath))
{
ret = true;
}
else
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DeleteTemporarySiaDriveFileFailed(siaPath, filePath, tempSourcePath)));
}
}
else
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(RenamingTemporarySiaDriveFileFailed(siaPath, filePath, tempSourcePath, siaDriveFilePath)));
if (!RetryDeleteFileIfExists(tempSourcePath))
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DeleteTemporarySiaDriveFileFailed(siaPath, filePath, tempSourcePath)));
}
if (!RetryDeleteFileIfExists(siaDriveFilePath))
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DeleteSiaDriveFileFailed(siaPath, filePath, siaDriveFilePath)));
}
}
}
else
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(CreatingTemporarySiaDriveFileFailed(siaPath, filePath, tempSourcePath)));
// If temp copy fails, try to delete
// If partial copy and file is unable to be deleted, log warning
if (!RetryDeleteFileIfExists(tempSourcePath))
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DeleteTemporarySiaDriveFileFailed(siaPath, filePath, tempSourcePath)));
}
}
}
return ret;
}
@@ -270,7 +248,6 @@ void CUploadManager::AutoThreadCallback(const CSiaCurl& siaCurl, CSiaDriveConfig
std::lock_guard<std::mutex> l(_uploadMutex);
SQLite::Statement query(_uploadDatabase, QUERY_UPLOADS_BY_2_STATUS);
query.bind("@status1", static_cast<unsigned>(UploadStatus::Uploading));
query.bind("@status2", static_cast<unsigned>(UploadStatus::Modified));
fileTree->BuildTree(result);
if (query.executeStep())
@@ -289,24 +266,8 @@ void CUploadManager::AutoThreadCallback(const CSiaCurl& siaCurl, CSiaDriveConfig
// Removed by another client
if (it == fileList.end())
{
// TODO SET_STATUS(UploadStatus::Remove, ExternallyRemovedFileDetected, ModifyUploadStatusFailed)
HandleFileRemove(siaCurl, siaPath, siaDriveFilePath);
}
// Changed file detected
else if (uploadStatus == UploadStatus::Modified)
{
json response;
SiaCurlError cerror = siaCurl.Post(SString(L"/renter/delete/") + siaPath, {}, response);
if (ApiSuccess(cerror))
{
// TODO validate response
SET_STATUS(UploadStatus::Queued, ModifiedUploadQueued, ModifyUploadStatusFailed)
}
else
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(FailedToDeleteFromSia(siaPath, filePath, cerror)));
}
}
// Upload is complete
else if ((*it)->GetAvailable())
{
@@ -319,7 +280,6 @@ void CUploadManager::AutoThreadCallback(const CSiaCurl& siaCurl, CSiaDriveConfig
// Upload still active, don't process another file
else
{
// TODO Check upload count
processNext = false;
}
}
@@ -401,29 +361,41 @@ UploadError CUploadManager::AddOrUpdate(const SString& siaPath, SString filePath
try
{
SQLite::Statement query(_uploadDatabase, QUERY_UPLOADS_BY_SIA_PATH_AND_2_STATUS);
SQLite::Statement query(_uploadDatabase, QUERY_UPLOADS_BY_SIA_PATH_AND_STATUS);
query.bind("@sia_path", SString::ToUtf8(siaPath).c_str());
query.bind("@status1", static_cast<unsigned>(UploadStatus::Uploading));
query.bind("@status2", static_cast<unsigned>(UploadStatus::Modified));
query.bind("@status", static_cast<unsigned>(UploadStatus::Uploading));
// Check copying
// Strip drive specification (i.e. C:\)
// TODO If mount to folder is ever enabled, this will need to change
// TODO Case sensative file names? Going to be a bit of an issue.
SString siaDriveFileName = GenerateSha256(&filePath[3]) + L".siadrive";
FilePath siaDriveFilePath(rootPath, siaDriveFileName);
// Check uploading
bool addToDatabase = true;
if (query.executeStep())
{
UploadStatus uploadStatus = static_cast<UploadStatus>(static_cast<unsigned>(query.getColumn(query.getColumnIndex("status"))));
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(ExistingUploadFound(siaPath, filePath, uploadStatus)));
if (uploadStatus == UploadStatus::Uploading)
{
SET_STATUS(UploadStatus::Modified, UploadStatusSetToModified, ModifyUploadStatusFailed)
addToDatabase = HandleFileRemove(CSiaCurl(GetHostConfig()), siaPath);
if (addToDatabase)
{
// Move modified file to real file
if (!siaDriveFilePath.MoveFile(filePath))
{
// TODO Critical failure
}
}
}
else if (uploadStatus == UploadStatus::Queued)
{
addToDatabase = false;
}
}
else
if (addToDatabase)
{
// Strip drive specification (i.e. C:\)
// TODO If mount to folder is ever enabled, this will need to change
// TODO Case sensative file names? Going to be a bit of an issue.
SString siaDriveFileName = GenerateSha256(&filePath[3]) + L".siadrive";
FilePath siaDriveFilePath(rootPath, siaDriveFileName);
// Add to db
try
{
@@ -467,52 +439,20 @@ UploadError CUploadManager::AddOrUpdate(const SString& siaPath, SString filePath
UploadError CUploadManager::Remove(const SString& siaPath)
{
UploadError ret;
bool remove = false;
SString siaDriveFilePath;
try
{
std::lock_guard<std::mutex> l(_uploadMutex);
SQLite::Statement query(_uploadDatabase, QUERY_STATUS);
query.bind("@sia_path", SString::ToUtf8(siaPath).c_str());
if (query.executeStep())
FilePath siaDriveFilePath(GetSiaDriveConfig()->GetCacheFolder(), siaPath + ".siadrive");
if (!HandleFileRemove(CSiaCurl(GetHostConfig()), siaPath, siaDriveFilePath))
{
SString filePath = static_cast<const char*>(query.getColumn(query.getColumnIndex("file_path")));
siaDriveFilePath = static_cast<const char*>(query.getColumn(query.getColumnIndex("sd_file_path")));
UploadStatus uploadStatus = static_cast<UploadStatus>(static_cast<unsigned>(query.getColumn(query.getColumnIndex("status"))));
switch (uploadStatus)
{
case UploadStatus::Complete:
case UploadStatus::Queued:
case UploadStatus::Modified:
case UploadStatus::Uploading:
{
remove = true;
}
break;
default:
{
remove = false;
}
break;
}
ret = UploadErrorCode::SourceFileNotFound;
}
else
{
RemoveFileFromSia(CSiaCurl(GetHostConfig()), siaPath);
}
}
catch (SQLite::Exception e)
catch (SQLite::Exception e)
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DatabaseExceptionOccurred("Remove", e)));
ret = { UploadErrorCode::DatabaseError, e.getErrorStr() };
}
if (remove)
{
HandleFileRemove(CSiaCurl(GetHostConfig()), siaPath, siaDriveFilePath);
}
return ret;
}