BlockStorage/siadrive
Archived
1
0
Fork 0
Code Activity

Begin refactoring upload manager

Browse Source
This commit is contained in:
Scott E. Graves
2017-03-30 19:10:15 -05:00
parent 6e250d8ab3
commit 4f226eadbb
2 changed files with 153 additions and 180 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
include/siadrive_api/uploadmanager.h
View File

@@ -20,7 +20,6 @@ public:
{ {
NotFound, NotFound,
Queued, Queued,
Modified,
Uploading, Uploading,
Complete, Complete,
Error Error
@@ -50,19 +49,19 @@ public:
virtual ~CUploadManager(); virtual ~CUploadManager();
private: private:
CSiaDriveConfig* _siaDriveConfig;
SQLite::Database _uploadDatabase; SQLite::Database _uploadDatabase;
std::mutex _uploadMutex; std::mutex _uploadMutex;
SString _activeSiaPath;
private: private:
void HandleFileRemove(const CSiaCurl& siaCurl, const SString& siaPath, const SString& siaDriveFilePath); CSiaDriveConfig* GetSiaDriveConfig() const { return _siaDriveConfig; }
bool HandleFileRemove(const CSiaCurl& siaCurl, const SString& siaPath, const SString& siaDriveFilePath = "");
bool CreateSiaDriveFile(const SString& siaPath, const SString& filePath, const SString& tempSourcePath, const SString& siaDriveFilePath); bool CreateSiaDriveFile(const SString& siaPath, const SString& filePath, const SString& tempSourcePath, const SString& siaDriveFilePath);
void DeleteFilesRemovedFromSia(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig, const bool& isStartup = false); void DeleteFilesRemovedFromSia(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig, const bool& isStartup = false);
void RemoveFileFromSia(const CSiaCurl& siaCurl, const SString& siaPath, FilePath removeFilePath);
protected: protected:
virtual void AutoThreadCallback(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig) override; virtual void AutoThreadCallback(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig) override;
public: public:
static SString UploadStatusToString(const _UploadStatus& uploadStatus); static SString UploadStatusToString(const _UploadStatus& uploadStatus);
@@ -507,6 +506,40 @@ public:
} }
}; };
class RenameModifiedFileFailed :
public CEvent
{
public:
RenameModifiedFileFailed(const SString& siaPath, const SString& filePath, const SString& tempSourcePath) :
_siaPath(siaPath),
_filePath(filePath),
_tempSourcePath(tempSourcePath)
{
}
public:
virtual ~RenameModifiedFileFailed()
{
}
private:
const SString _siaPath;
const SString _filePath;
const SString _tempSourcePath;
public:
virtual SString GetSingleLineMessage() const override
{
return L"RenameModifiedFileFailed|SP|" + _siaPath + L"|FP|" + _filePath + L"|TSP|" + _tempSourcePath;
}
virtual std::shared_ptr<CEvent> Clone() const override
{
return std::shared_ptr<CEvent>(new RenameModifiedFileFailed(_siaPath, _filePath, _tempSourcePath));
}
};
class DeleteTemporarySiaDriveFileFailed : class DeleteTemporarySiaDriveFileFailed :
public CEvent public CEvent
{ {

290
src/siadrive_api/uploadmanager.cpp
View File

@@ -13,7 +13,7 @@ using namespace Sia::Api;
#define QUERY_UPLOADS_BY_STATUS "select * from upload_table where status=@status order by id desc limit 1;" #define QUERY_UPLOADS_BY_STATUS "select * from upload_table where status=@status order by id desc limit 1;"
#define QUERY_UPLOADS_BY_2_STATUS "select * from upload_table where (status=@status1 or status=@status2) order by id desc limit 1;" #define QUERY_UPLOADS_BY_2_STATUS "select * from upload_table where (status=@status1 or status=@status2) order by id desc limit 1;"
#define QUERY_UPLOADS_BY_SIA_PATH "select * from upload_table where sia_path=@sia_path order by id desc limit 1;" #define QUERY_UPLOADS_BY_SIA_PATH "select * from upload_table where sia_path=@sia_path order by id desc limit 1;"
#define QUERY_UPLOADS_BY_SIA_PATH_AND_2_STATUS "select * from upload_table where sia_path=@sia_path and (status=@status1 or status=@status2) order by id desc limit 1;" #define QUERY_UPLOADS_BY_SIA_PATH_AND_STATUS "select * from upload_table where sia_path=@sia_path and status=@status order by id desc limit 1;"
#define UPDATE_STATUS "update upload_table set status=@status where sia_path=@sia_path;" #define UPDATE_STATUS "update upload_table set status=@status where sia_path=@sia_path;"
#define INSERT_UPLOAD "insert into upload_table (sia_path, status, file_path, sd_file_path) values (@sia_path, @status, @file_path, @sd_file_path);" #define INSERT_UPLOAD "insert into upload_table (sia_path, status, file_path, sd_file_path) values (@sia_path, @status, @file_path, @sd_file_path);"
#define DELETE_UPLOAD "delete from upload_table where sia_path=@sia_path;" #define DELETE_UPLOAD "delete from upload_table where sia_path=@sia_path;"
@@ -66,10 +66,7 @@ SString CUploadManager::UploadStatusToString(const UploadStatus& uploadStatus)
return L"Complete"; return L"Complete";
case UploadStatus::Error: case UploadStatus::Error:
return L"Error"; return L"Error";
case UploadStatus::Modified:
return L"Modified";
case UploadStatus::NotFound: case UploadStatus::NotFound:
return L"Not Found"; return L"Not Found";
@@ -87,6 +84,7 @@ SString CUploadManager::UploadStatusToString(const UploadStatus& uploadStatus)
CUploadManager::CUploadManager(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig) : CUploadManager::CUploadManager(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig) :
CAutoThread(siaCurl, siaDriveConfig), CAutoThread(siaCurl, siaDriveConfig),
_siaDriveConfig(siaDriveConfig),
_uploadDatabase(siaDriveConfig->GetRenter_UploadDbFilePath(), SQLite::OPEN_CREATE | SQLite::OPEN_READWRITE) _uploadDatabase(siaDriveConfig->GetRenter_UploadDbFilePath(), SQLite::OPEN_CREATE | SQLite::OPEN_READWRITE)
{ {
CreateTableIfNotFound(&_uploadDatabase, UPLOAD_TABLE, UPLOAD_TABLE_COLUMNS); CreateTableIfNotFound(&_uploadDatabase, UPLOAD_TABLE, UPLOAD_TABLE_COLUMNS);
@@ -135,123 +133,103 @@ void CUploadManager::DeleteFilesRemovedFromSia(const CSiaCurl& siaCurl, CSiaDriv
} }
} }
void CUploadManager::RemoveFileFromSia(const CSiaCurl& siaCurl, const SString& siaPath, FilePath removeFilePath = SString("*")) bool CUploadManager::HandleFileRemove(const CSiaCurl& siaCurl, const SString& siaPath, const SString& siaDriveFilePath)
{
json response;
SiaCurlError cerror = siaCurl.Post(SString(L"/renter/delete/") + siaPath, {}, response);
if (ApiSuccess(cerror))
{
SQLite::Statement del(_uploadDatabase, DELETE_UPLOAD);
del.bind("@sia_path", SString::ToUtf8(siaPath).c_str());
auto ret = del.exec();
if (ret >= 0)
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(FileRemoved(siaPath, removeFilePath)));
}
else
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DatabaseDeleteFailed(siaPath, removeFilePath, del.getErrorMsg())));
}
}
else
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(FailedToDeleteFromSia(siaPath, removeFilePath, cerror)));
}
}
void CUploadManager::HandleFileRemove(const CSiaCurl& siaCurl, const SString& siaPath, const SString& siaDriveFilePath)
{
try
{
std::lock_guard<std::mutex> l(_uploadMutex);
SQLite::Statement query(_uploadDatabase, QUERY_STATUS);
query.bind("@sia_path", SString::ToUtf8(siaPath).c_str());
if (query.executeStep())
{
FilePath removeFilePath = SString(static_cast<const char*>(query.getColumn(query.getColumnIndex("file_path"))));
UploadStatus uploadStatus = static_cast<UploadStatus>(static_cast<unsigned>(query.getColumn(query.getColumnIndex("status"))));
// Make sure status is still remove
bool deleteFromSia = true;
if (removeFilePath.IsFile())
{
if (RetryDeleteFileIfExists(removeFilePath))
{
if (!RetryDeleteFileIfExists(siaDriveFilePath))
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DeleteSiaDriveFileFailed(siaPath, removeFilePath, siaDriveFilePath)));
}
}
else
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(RemoveFileFailed(siaPath, removeFilePath)));
deleteFromSia = false;
}
}
if (deleteFromSia)
{
RemoveFileFromSia(siaCurl, siaPath, removeFilePath);
}
}
}
catch (SQLite::Exception e)
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DatabaseExceptionOccurred("HandleFileRemove", e)));
}
}
bool CUploadManager::CreateSiaDriveFile(const SString& siaPath, const SString& filePath, const SString& tempSourcePath, const SString& siaDriveFilePath)
{ {
bool ret = false; bool ret = false;
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(CreatingTemporarySiaDriveFile(siaPath, filePath, tempSourcePath))); FilePath removeFilePath(GetSiaDriveConfig()->GetCacheFolder(), siaPath);
if (RetryableAction(::CopyFile(filePath.str().c_str(), tempSourcePath.str().c_str(), FALSE), DEFAULT_RETRY_COUNT, DEFAULT_RETRY_DELAY_MS))
bool deleteFromSia = true;
if (removeFilePath.IsFile())
{ {
// Delete existing '.siadrive' file, if found if (RetryDeleteFileIfExists(removeFilePath))
// !!Should never come here. If so, there was a problem with startup clean-up
if (!RetryDeleteFileIfExists(siaDriveFilePath))
{ {
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DeleteSiaDriveFileFailed(siaPath, filePath, siaDriveFilePath))); if (!siaDriveFilePath.IsNullOrEmpty() && !RetryDeleteFileIfExists(siaDriveFilePath))
}
// Rename '.siadrive.temp' to '.siadrive'
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(RenamingTemporarySiaDriveFile(siaPath, filePath, tempSourcePath, siaDriveFilePath)));
if (RetryableAction(::MoveFile(tempSourcePath.str().c_str(), siaDriveFilePath.str().c_str()), DEFAULT_RETRY_COUNT, DEFAULT_RETRY_DELAY_MS))
{
if (RetryDeleteFileIfExists(tempSourcePath))
{ {
ret = true; CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DeleteSiaDriveFileFailed(siaPath, removeFilePath, siaDriveFilePath)));
}
else
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DeleteTemporarySiaDriveFileFailed(siaPath, filePath, tempSourcePath)));
} }
} }
else else
{ {
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(RenamingTemporarySiaDriveFileFailed(siaPath, filePath, tempSourcePath, siaDriveFilePath))); CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(RemoveFileFailed(siaPath, removeFilePath)));
if (!RetryDeleteFileIfExists(tempSourcePath)) deleteFromSia = false;
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DeleteTemporarySiaDriveFileFailed(siaPath, filePath, tempSourcePath)));
}
if (!RetryDeleteFileIfExists(siaDriveFilePath))
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DeleteSiaDriveFileFailed(siaPath, filePath, siaDriveFilePath)));
}
} }
} }
else
{ if (deleteFromSia)
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(CreatingTemporarySiaDriveFileFailed(siaPath, filePath, tempSourcePath))); {
json response;
// If temp copy fails, try to delete SiaCurlError cerror = siaCurl.Post(SString(L"/renter/delete/") + siaPath, {}, response);
// If partial copy and file is unable to be deleted, log warning if (ApiSuccess(cerror))
if (!RetryDeleteFileIfExists(tempSourcePath))
{ {
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DeleteTemporarySiaDriveFileFailed(siaPath, filePath, tempSourcePath))); SQLite::Statement del(_uploadDatabase, DELETE_UPLOAD);
del.bind("@sia_path", SString::ToUtf8(siaPath).c_str());
if (del.exec() >= 0)
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(FileRemoved(siaPath, removeFilePath)));
ret = true;
}
else
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DatabaseDeleteFailed(siaPath, removeFilePath, del.getErrorMsg())));
}
}
else
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(FailedToDeleteFromSia(siaPath, removeFilePath, cerror)));
}
}
return ret;
}
bool CUploadManager::CreateSiaDriveFile(const SString& siaPath, const SString& filePath, const SString& tempSourcePath, const SString& siaDriveFilePath)
{
bool ret = FilePath(siaDriveFilePath).IsFile();
if (!ret)
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(CreatingTemporarySiaDriveFile(siaPath, filePath, tempSourcePath)));
if (RetryableAction(::CopyFile(filePath.str().c_str(), tempSourcePath.str().c_str(), FALSE), DEFAULT_RETRY_COUNT, DEFAULT_RETRY_DELAY_MS))
{
// Rename '.siadrive.temp' to '.siadrive'
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(RenamingTemporarySiaDriveFile(siaPath, filePath, tempSourcePath, siaDriveFilePath)));
if (RetryableAction(::MoveFile(tempSourcePath.str().c_str(), siaDriveFilePath.str().c_str()), DEFAULT_RETRY_COUNT, DEFAULT_RETRY_DELAY_MS))
{
if (RetryDeleteFileIfExists(tempSourcePath))
{
ret = true;
}
else
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DeleteTemporarySiaDriveFileFailed(siaPath, filePath, tempSourcePath)));
}
}
else
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(RenamingTemporarySiaDriveFileFailed(siaPath, filePath, tempSourcePath, siaDriveFilePath)));
if (!RetryDeleteFileIfExists(tempSourcePath))
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DeleteTemporarySiaDriveFileFailed(siaPath, filePath, tempSourcePath)));
}
if (!RetryDeleteFileIfExists(siaDriveFilePath))
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DeleteSiaDriveFileFailed(siaPath, filePath, siaDriveFilePath)));
}
}
}
else
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(CreatingTemporarySiaDriveFileFailed(siaPath, filePath, tempSourcePath)));
// If temp copy fails, try to delete
// If partial copy and file is unable to be deleted, log warning
if (!RetryDeleteFileIfExists(tempSourcePath))
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DeleteTemporarySiaDriveFileFailed(siaPath, filePath, tempSourcePath)));
}
} }
} }
return ret; return ret;
} }
@@ -270,7 +248,6 @@ void CUploadManager::AutoThreadCallback(const CSiaCurl& siaCurl, CSiaDriveConfig
std::lock_guard<std::mutex> l(_uploadMutex); std::lock_guard<std::mutex> l(_uploadMutex);
SQLite::Statement query(_uploadDatabase, QUERY_UPLOADS_BY_2_STATUS); SQLite::Statement query(_uploadDatabase, QUERY_UPLOADS_BY_2_STATUS);
query.bind("@status1", static_cast<unsigned>(UploadStatus::Uploading)); query.bind("@status1", static_cast<unsigned>(UploadStatus::Uploading));
query.bind("@status2", static_cast<unsigned>(UploadStatus::Modified));
fileTree->BuildTree(result); fileTree->BuildTree(result);
if (query.executeStep()) if (query.executeStep())
@@ -289,24 +266,8 @@ void CUploadManager::AutoThreadCallback(const CSiaCurl& siaCurl, CSiaDriveConfig
// Removed by another client // Removed by another client
if (it == fileList.end()) if (it == fileList.end())
{ {
// TODO SET_STATUS(UploadStatus::Remove, ExternallyRemovedFileDetected, ModifyUploadStatusFailed)
HandleFileRemove(siaCurl, siaPath, siaDriveFilePath); HandleFileRemove(siaCurl, siaPath, siaDriveFilePath);
} }
// Changed file detected
else if (uploadStatus == UploadStatus::Modified)
{
json response;
SiaCurlError cerror = siaCurl.Post(SString(L"/renter/delete/") + siaPath, {}, response);
if (ApiSuccess(cerror))
{
// TODO validate response
SET_STATUS(UploadStatus::Queued, ModifiedUploadQueued, ModifyUploadStatusFailed)
}
else
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(FailedToDeleteFromSia(siaPath, filePath, cerror)));
}
}
// Upload is complete // Upload is complete
else if ((*it)->GetAvailable()) else if ((*it)->GetAvailable())
{ {
@@ -319,7 +280,6 @@ void CUploadManager::AutoThreadCallback(const CSiaCurl& siaCurl, CSiaDriveConfig
// Upload still active, don't process another file // Upload still active, don't process another file
else else
{ {
// TODO Check upload count
processNext = false; processNext = false;
} }
} }
@@ -401,29 +361,41 @@ UploadError CUploadManager::AddOrUpdate(const SString& siaPath, SString filePath
try try
{ {
SQLite::Statement query(_uploadDatabase, QUERY_UPLOADS_BY_SIA_PATH_AND_2_STATUS); SQLite::Statement query(_uploadDatabase, QUERY_UPLOADS_BY_SIA_PATH_AND_STATUS);
query.bind("@sia_path", SString::ToUtf8(siaPath).c_str()); query.bind("@sia_path", SString::ToUtf8(siaPath).c_str());
query.bind("@status1", static_cast<unsigned>(UploadStatus::Uploading)); query.bind("@status", static_cast<unsigned>(UploadStatus::Uploading));
query.bind("@status2", static_cast<unsigned>(UploadStatus::Modified));
// Check copying // Strip drive specification (i.e. C:\)
// TODO If mount to folder is ever enabled, this will need to change
// TODO Case sensative file names? Going to be a bit of an issue.
SString siaDriveFileName = GenerateSha256(&filePath[3]) + L".siadrive";
FilePath siaDriveFilePath(rootPath, siaDriveFileName);
// Check uploading
bool addToDatabase = true;
if (query.executeStep()) if (query.executeStep())
{ {
UploadStatus uploadStatus = static_cast<UploadStatus>(static_cast<unsigned>(query.getColumn(query.getColumnIndex("status")))); UploadStatus uploadStatus = static_cast<UploadStatus>(static_cast<unsigned>(query.getColumn(query.getColumnIndex("status"))));
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(ExistingUploadFound(siaPath, filePath, uploadStatus)));
if (uploadStatus == UploadStatus::Uploading) if (uploadStatus == UploadStatus::Uploading)
{ {
SET_STATUS(UploadStatus::Modified, UploadStatusSetToModified, ModifyUploadStatusFailed) addToDatabase = HandleFileRemove(CSiaCurl(GetHostConfig()), siaPath);
if (addToDatabase)
{
// Move modified file to real file
if (!siaDriveFilePath.MoveFile(filePath))
{
// TODO Critical failure
}
}
} }
else if (uploadStatus == UploadStatus::Queued)
{
addToDatabase = false;
}
} }
else
if (addToDatabase)
{ {
// Strip drive specification (i.e. C:\)
// TODO If mount to folder is ever enabled, this will need to change
// TODO Case sensative file names? Going to be a bit of an issue.
SString siaDriveFileName = GenerateSha256(&filePath[3]) + L".siadrive";
FilePath siaDriveFilePath(rootPath, siaDriveFileName);
// Add to db // Add to db
try try
{ {
@@ -467,52 +439,20 @@ UploadError CUploadManager::AddOrUpdate(const SString& siaPath, SString filePath
UploadError CUploadManager::Remove(const SString& siaPath) UploadError CUploadManager::Remove(const SString& siaPath)
{ {
UploadError ret; UploadError ret;
bool remove = false;
SString siaDriveFilePath;
try try
{ {
std::lock_guard<std::mutex> l(_uploadMutex); std::lock_guard<std::mutex> l(_uploadMutex);
FilePath siaDriveFilePath(GetSiaDriveConfig()->GetCacheFolder(), siaPath + ".siadrive");
SQLite::Statement query(_uploadDatabase, QUERY_STATUS); if (!HandleFileRemove(CSiaCurl(GetHostConfig()), siaPath, siaDriveFilePath))
query.bind("@sia_path", SString::ToUtf8(siaPath).c_str());
if (query.executeStep())
{ {
SString filePath = static_cast<const char*>(query.getColumn(query.getColumnIndex("file_path"))); ret = UploadErrorCode::SourceFileNotFound;
siaDriveFilePath = static_cast<const char*>(query.getColumn(query.getColumnIndex("sd_file_path")));
UploadStatus uploadStatus = static_cast<UploadStatus>(static_cast<unsigned>(query.getColumn(query.getColumnIndex("status"))));
switch (uploadStatus)
{
case UploadStatus::Complete:
case UploadStatus::Queued:
case UploadStatus::Modified:
case UploadStatus::Uploading:
{
remove = true;
}
break;
default:
{
remove = false;
}
break;
}
} }
else
{
RemoveFileFromSia(CSiaCurl(GetHostConfig()), siaPath);
}
} }
catch (SQLite::Exception e) catch (SQLite::Exception e)
{ {
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DatabaseExceptionOccurred("Remove", e))); CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DatabaseExceptionOccurred("Remove", e)));
ret = { UploadErrorCode::DatabaseError, e.getErrorStr() }; ret = { UploadErrorCode::DatabaseError, e.getErrorStr() };
} }
if (remove)
{
HandleFileRemove(CSiaCurl(GetHostConfig()), siaPath, siaDriveFilePath);
}
return ret; return ret;
} }