BlockStorage/siadrive
Archived
1
0
Fork 0
Code Activity

More CMake changes - BROKEN BUILD

Browse Source
This commit is contained in:
Scott E. Graves
2017-03-15 17:57:38 -05:00
parent 14349a082a
commit 836c6838fe
11 changed files with 2231 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

83
src/siadrive_api/eventsystem.cpp Normal file
View File

@@ -0,0 +1,83 @@
#include <eventsystem.h>
using namespace Sia::Api;
CEventSystem CEventSystem::EventSystem;
CEventSystem::CEventSystem() :
_stopEvent(INVALID_HANDLE_VALUE)
{
}
CEventSystem::~CEventSystem()
{
Stop();
::CloseHandle(_stopEvent);
}
void CEventSystem::ProcessEvents()
{
while (::WaitForSingleObject(_stopEvent, 10) == WAIT_TIMEOUT)
{
CEventPtr eventData;
do
{
{
std::lock_guard<std::mutex> l(_eventMutex);
if (_eventQueue.size())
{
eventData = _eventQueue.front();
_eventQueue.pop_front();
}
else
{
eventData.reset();
}
}
if (eventData)
{
for (auto& v : _eventConsumers)
{
v(*eventData);
}
}
} while (eventData);
}
}
void CEventSystem::NotifyEvent(CEventPtr eventData)
{
std::lock_guard<std::mutex> l(_eventMutex);
if (_eventConsumers.size())
{
_eventQueue.push_back(eventData);
}
}
void CEventSystem::AddEventConsumer(std::function<void(const CEvent&)> consumer)
{
if (!_processThread)
{
_eventConsumers.push_back(consumer);
}
}
void CEventSystem::Start()
{
if (!_processThread)
{
_stopEvent = ::CreateEvent(nullptr, FALSE, FALSE, nullptr);
_processThread.reset(new std::thread([this]() {ProcessEvents(); }));
}
}
void CEventSystem::Stop()
{
if (_processThread)
{
::SetEvent(_stopEvent);
_processThread->join();
_processThread.reset();
}
}

35
src/siadrive_api/siaconsensus.cpp Normal file
View File

@@ -0,0 +1,35 @@
#include <siaapi.h>
using namespace Sia::Api;
CSiaApi::_CSiaConsensus::_CSiaConsensus(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig) :
CSiaBase(siaCurl, siaDriveConfig),
CAutoThread(siaCurl, siaDriveConfig),
_Height(0),
_Synced(false),
_CurrentBlock(L"")
{
StartAutoThread();
}
CSiaApi::_CSiaConsensus::~_CSiaConsensus()
{
StopAutoThread();
}
void CSiaApi::_CSiaConsensus::AutoThreadCallback(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig)
{
json result;
if (ApiSuccess(siaCurl.Get(L"/consensus", result)))
{
SetHeight(result["height"].get<std::uint64_t>());
SetSynced(result["synced"].get<bool>());
SetCurrentBlock(result["currentblock"].get<std::string>());
}
else
{
SetHeight(0);
SetSynced(false);
SetCurrentBlock(L"");
}
}

21
src/siadrive_api/siafile.cpp Normal file
View File

@@ -0,0 +1,21 @@
#include <siaapi.h>
using namespace Sia::Api;
CSiaApi::_CSiaFile::_CSiaFile(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig, const json& fileJson) :
CSiaBase(siaCurl, siaDriveConfig),
_SiaPath(fileJson["siapath"].get<std::string>()),
_FileSize(fileJson["filesize"].get<std::uint64_t>()),
_Available(fileJson["available"].get<bool>()),
_Renewing(fileJson["renewing"].get<bool>()),
_Redundancy(fileJson["redundancy"].get<std::uint32_t>()),
_UploadProgress(fileJson["uploadprogress"].get<std::uint32_t>()),
_Expiration(fileJson["expiration"].get<std::uint32_t>())
{
}
CSiaApi::_CSiaFile::~_CSiaFile()
{
}

109
src/siadrive_api/siafiletree.cpp Normal file
View File

@@ -0,0 +1,109 @@
#include <siaapi.h>
#include <regex>
using namespace Sia::Api;
CSiaApi::_CSiaFileTree::_CSiaFileTree(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig) :
CSiaBase(siaCurl, siaDriveConfig)
{
}
CSiaApi::_CSiaFileTree::~_CSiaFileTree()
{
}
void CSiaApi::_CSiaFileTree::BuildTree(const json& result)
{
_fileList.clear();
for (const auto& file : result["files"])
{
_fileList.push_back(CSiaFilePtr(new CSiaFile(GetSiaCurl(), &GetSiaDriveConfig(), file)));
}
}
bool CSiaApi::_CSiaFileTree::FileExists(const SString& siaPath) const
{
auto result = std::find_if(_fileList.begin(), _fileList.end(), [&](const CSiaFilePtr& item)->bool
{
return (item->GetSiaPath() == siaPath);
});
return (result != _fileList.end());
}
CSiaFileCollection CSiaApi::_CSiaFileTree::GetFileList() const
{
return _fileList;
}
CSiaFilePtr CSiaApi::_CSiaFileTree::GetFile(const SString& siaPath) const
{
auto result = std::find_if(_fileList.begin(), _fileList.end(), [&](const CSiaFilePtr& item)->bool
{
return (item->GetSiaPath() == siaPath);
});
return ((result != _fileList.end()) ? *result : nullptr);
}
CSiaFileCollection CSiaApi::_CSiaFileTree::Query(SString query) const
{
query = CSiaApi::FormatToSiaPath(query);
query.Replace(".", "\\.").Replace("*", "[^/]+").Replace("?", "[^/]?");
std::wregex r(query.str());
CSiaFileCollection ret;
std::copy_if(_fileList.begin(), _fileList.end(), std::back_inserter(ret), [&](const CSiaFilePtr& v) -> bool
{
return std::regex_match(v->GetSiaPath().str(), r);
});
return std::move(ret);
}
std::vector<SString> CSiaApi::_CSiaFileTree::QueryDirectories(SString rootFolder) const
{
CSiaFileCollection col;
rootFolder.Replace("/", "\\");
if (rootFolder[0] == '\\')
{
rootFolder = rootFolder.SubString(1);
}
std::vector<SString> ret;
std::for_each(_fileList.begin(), _fileList.end(), [&](const CSiaFilePtr& v)
{
SString dir = v->GetSiaPath();
dir.Replace("/", "\\");
::PathRemoveFileSpec(&dir[0]);
::PathRemoveBackslash(&dir[0]);
::PathRemoveBackslash(&rootFolder[0]);
dir = dir.str().c_str();
rootFolder = rootFolder.str().c_str();
if ((dir.Length() > rootFolder.Length()) && (dir.SubString(0, rootFolder.Length()) == rootFolder))
{
SString subFolder = dir.SubString(rootFolder.Length());
int idx = subFolder.str().find('\\');
if (idx == 0)
{
subFolder = subFolder.SubString(1);
idx = subFolder.str().find('\\');
}
if (idx > 0)
{
subFolder = subFolder.SubString(0, idx);
}
auto it = std::find(ret.begin(), ret.end(), subFolder);
if (it == ret.end())
{
ret.push_back(subFolder);
}
}
});
return std::move(ret);
}

155
src/siadrive_api/siarenter.cpp Normal file
View File

@@ -0,0 +1,155 @@
#include <siaapi.h>
#include <SQLiteCpp/Database.h>
using namespace Sia::Api;
/*{
// Settings that control the behavior of the renter.
"settings": {
// Allowance dictates how much the renter is allowed to spend in a given
// period. Note that funds are spent on both storage and bandwidth.
"allowance": {
// Amount of money allocated for contracts. Funds are spent on both
// storage and bandwidth.
"funds": "1234", // hastings
// Number of hosts that contracts will be formed with.
"hosts":24,
// Duration of contracts formed, in number of blocks.
"period": 6048, // blocks
// If the current blockheight + the renew window >= the height the
// contract is scheduled to end, the contract is renewed automatically.
// Is always nonzero.
"renewwindow": 3024 // blocks
}
},
// Metrics about how much the Renter has spent on storage, uploads, and
// downloads.
"financialmetrics": {
// How much money, in hastings, the Renter has spent on file contracts,
// including fees.
"contractspending": "1234", // hastings
// Amount of money spent on downloads.
"downloadspending": "5678", // hastings
// Amount of money spend on storage.
"storagespending": "1234", // hastings
// Amount of money spent on uploads.
"uploadspending": "5678", // hastings
// Amount of money in the allowance that has not been spent.
"unspent": "1234" // hastings
}
}*/
CSiaApi::_CSiaRenter::_CSiaRenter(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig) :
CSiaBase(siaCurl, siaDriveConfig),
CAutoThread(siaCurl, siaDriveConfig),
_Funds(0),
_Hosts(0),
_Unspent(0),
_TotalUsedBytes(0),
_TotalUploadProgress(100)
{
StartAutoThread();
}
CSiaApi::_CSiaRenter::~_CSiaRenter()
{
StopAutoThread();
}
void CSiaApi::_CSiaRenter::AutoThreadCallback(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig)
{
json result;
if (ApiSuccess(siaCurl.Get(L"/renter", result)))
{
SiaCurrency funds = HastingsStringToSiaCurrency(result["settings"]["allowance"]["funds"].get<std::string>());
SiaCurrency unspent = HastingsStringToSiaCurrency(result["financialmetrics"]["unspent"].get<std::string>());
std::uint64_t hosts = result["settings"]["allowance"]["hosts"].get<std::uint64_t>();
SetFunds(funds);
SetHosts(hosts);
SetUnspent(unspent);
CSiaFileTreePtr fileTree(new CSiaFileTree(siaCurl, siaDriveConfig));
if (ApiSuccess(siaCurl.Get(L"/renter/files", result)))
{
fileTree->BuildTree(result);
auto fileList = fileTree->GetFileList();
if (fileList.size())
{
std::uint64_t total = std::accumulate(std::next(fileList.begin()), fileList.end(), fileList[0]->GetFileSize(), [](const std::uint64_t& sz, const CSiaFilePtr& file)
{
return sz + file->GetFileSize();
});
std::uint32_t totalProgress = std::accumulate(std::next(fileList.begin()), fileList.end(), fileList[0]->GetUploadProgress(), [](const std::uint32_t& progress, const CSiaFilePtr& file)
{
return progress + min(100, file->GetUploadProgress());
}) / fileList.size();
SetTotalUsedBytes(total);
SetTotalUploadProgress(totalProgress);
}
else
{
SetTotalUsedBytes(0);
SetTotalUploadProgress(100);
}
}
else
{
SetTotalUsedBytes(0);
SetTotalUploadProgress(100);
}
}
else
{
SetFunds(0);
SetHosts(0);
SetUnspent(0);
SetTotalUsedBytes(0);
SetTotalUploadProgress(100);
}
}
SiaApiError CSiaApi::_CSiaRenter::FileExists(const SString& siaPath, bool& exists) const
{
CSiaFileTreePtr siaFileTree;
SiaApiError ret = GetFileTree(siaFileTree);
if (ApiSuccess(ret))
{
exists = siaFileTree->FileExists(siaPath);
}
return ret;
}
SiaApiError CSiaApi::_CSiaRenter::DownloadFile(const SString& siaPath, const SString& location) const
{
SiaApiError ret = SiaApiError::RequestError;
json result;
if (ApiSuccess(GetSiaCurl().Get(L"/renter/download/" + siaPath, { { L"destination", location } }, result)))
{
ret = SiaApiError::Success;
}
return ret;
}
SiaApiError CSiaApi::_CSiaRenter::GetFileTree(CSiaFileTreePtr& siaFileTree) const
{
SiaApiError ret = SiaApiError::RequestError;
siaFileTree.reset(new CSiaFileTree(GetSiaCurl(), &GetSiaDriveConfig()));
json result;
if (ApiSuccess(GetSiaCurl().Get(L"/renter/files", result)))
{
siaFileTree->BuildTree(result);
ret = SiaApiError::Success;
}
return ret;
}

178
src/siadrive_api/siawallet.cpp Normal file
View File

@@ -0,0 +1,178 @@
#include <siaapi.h>
using namespace Sia::Api;
static SString SeedLangToString(const SiaSeedLanguage& lang)
{
switch (lang)
{
case SiaSeedLanguage::English:
return L"english";
case SiaSeedLanguage::German:
return L"german";
case SiaSeedLanguage::Japanese:
return L"japanese";
default:
throw std::exception("Seed language not implemented");
}
}
CSiaApi::_CSiaWallet::_CSiaWallet(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig) :
CSiaBase(siaCurl, siaDriveConfig),
_Created(false),
_Locked(false)
{
Refresh();
}
CSiaApi::_CSiaWallet::~_CSiaWallet()
{
}
bool CSiaApi::_CSiaWallet::Refresh()
{
json result;
SiaCurlError error = GetSiaCurl().Get(L"/wallet", result);
if (ApiSuccess(error))
{
SetCreated(result["encrypted"].get<bool>());
SetLocked(!result["unlocked"].get<bool>());
return true;
}
return false;
}
SiaApiError CSiaApi::_CSiaWallet::Create(const SiaSeedLanguage& seedLanguage, SString& seed)
{
SiaApiError error = SiaApiError::RequestError;
if (Refresh())
{
error = SiaApiError::WalletExists;
if (!GetCreated())
{
error = SiaApiError::RequestError;
json result;
SiaCurlError cerror = GetSiaCurl().Post(L"/wallet/init", { {L"dictionary", SeedLangToString(seedLanguage)} }, result);
if (ApiSuccess(cerror))
{
error = SiaApiError::Success;
seed = result["primaryseed"].get<std::string>();
Refresh();
}
}
}
return error;
}
SiaApiError CSiaApi::_CSiaWallet::Restore(const SString& seed)
{
SiaApiError error = SiaApiError::NotImplemented;
// TODO Future enhancement
return error;
}
SiaApiError CSiaApi::_CSiaWallet::Lock()
{
SiaApiError error = GetCreated() ? (GetLocked() ? SiaApiError::WalletLocked : SiaApiError::Success) : SiaApiError::WalletNotCreated;
if (ApiSuccess(error))
{
error = SiaApiError::RequestError;
json result;
SiaCurlError cerror = GetSiaCurl().Post(L"/wallet/lock", {}, result);
if (ApiSuccess(cerror))
{
Refresh();
error = SiaApiError::Success;
}
}
return error;
}
SiaApiError CSiaApi::_CSiaWallet::Unlock(const SString& password)
{
SiaApiError error = GetCreated() ? (GetLocked() ? SiaApiError::Success : SiaApiError::WalletUnlocked) : SiaApiError::WalletNotCreated;
if (ApiSuccess(error))
{
error = SiaApiError::RequestError;
json result;
SiaCurlError cerror = GetSiaCurl().Post(L"/wallet/unlock", { {L"encryptionpassword", password} }, result);
if (ApiSuccess(cerror))
{
Refresh();
error = SiaApiError::Success;
}
}
return error;
}
/*{
"encrypted": true,
"unlocked": true,
"confirmedsiacoinbalance": "123456", // hastings, big int
"unconfirmedoutgoingsiacoins": "0", // hastings, big int
"unconfirmedincomingsiacoins": "789", // hastings, big int
"siafundbalance": "1", // siafunds, big int
"siacoinclaimbalance": "9001", // hastings, big int
}*/
SiaApiError CSiaApi::_CSiaWallet::GetConfirmedBalance(SiaCurrency& balance) const
{
SiaApiError ret = SiaApiError::RequestError;
balance = 0;
json result;
SiaCurlError cerror = GetSiaCurl().Get(L"/wallet", result);
if (ApiSuccess(cerror))
{
balance = HastingsStringToSiaCurrency(result["confirmedsiacoinbalance"].get<std::string>());
ret = SiaApiError::Success;
}
return ret;
}
SiaApiError CSiaApi::_CSiaWallet::GetUnonfirmedBalance(SiaCurrency& balance) const
{
SiaApiError ret = SiaApiError::RequestError;
balance = 0;
json result;
SiaCurlError cerror = GetSiaCurl().Get(L"/wallet", result);
if (ApiSuccess(cerror))
{
SiaCurrency sc1 = HastingsStringToSiaCurrency(result["unconfirmedincomingsiacoins"].get<std::string>());
SiaCurrency sc2 = HastingsStringToSiaCurrency(result["unconfirmedoutgoingsiacoins"].get<std::string>());
balance = sc1 - sc2;
ret = SiaApiError::Success;
}
return ret;
}
SiaApiError CSiaApi::_CSiaWallet::GetAddress(SString& address) const
{
SiaApiError ret = SiaApiError::RequestError;
address = L"";
json result;
SiaCurlError cerror = GetSiaCurl().Get(L"/wallet/address", result);
if (ApiSuccess(cerror))
{
address = result["address"].get<std::string>();
ret = SiaApiError::Success;
}
return ret;
}

703
src/siadrive_api/uploadmanager.cpp Normal file
View File

@@ -0,0 +1,703 @@
#include <uploadmanager.h>
#include <siaapi.h>
#include <eventsystem.h>
using namespace Sia::Api;
#define TABLE_CREATE L"create table if not exists %s (%s);"
#define UPLOAD_TABLE L"upload_table"
#define UPLOAD_TABLE_COLUMNS L"id integer primary key autoincrement, sia_path text unique not null, file_path text unique not null, sd_file_path text unique not null, status integer not null"
#define QUERY_STATUS "select id, sia_path, file_path, sd_file_path, status from upload_table where sia_path=@sia_path order by id desc limit 1;"
#define QUERY_UPLOADS_BY_STATUS "select id, sia_path, status from upload_table where status=@status order by id desc limit 1;"
#define QUERY_UPLOADS_BY_2_STATUS "select id, sia_path, status from upload_table where (status=@status1 or status=@status2) order by id desc limit 1;"
#define QUERY_UPLOADS_BY_SIA_PATH "select id, sia_path, status from upload_table where sia_path=@sia_path order by id desc limit 1;"
#define QUERY_UPLOADS_BY_SIA_PATH_AND_2_STATUS "select id, sia_path, file_path, sd_file_path, status from upload_table where sia_path=@sia_path and (status=@status1 or status=@status2) order by id desc limit 1;"
#define UPDATE_STATUS "update upload_table set status=@status where sia_path=@sia_path;"
#define INSERT_UPLOAD "insert into upload_table (sia_path, status, file_path, sd_file_path) values (@sia_path, @status, @file_path, @sd_file_path);"
#define DELETE_UPLOAD "delete from upload_table where sia_path=@sia_path;"
#define SET_STATUS(status, success_event, fail_event)\
bool statusUpdated = false;\
try\
{\
SQLite::Statement update(_uploadDatabase, UPDATE_STATUS);\
update.bind("@sia_path", CW2A(siaPath.c_str()).m_psz);\
update.bind("@status", static_cast<unsigned>(status));\
if (update.exec() != 1)\
{\
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(fail_event(siaPath, filePath, status, CA2W(update.getErrorMsg()).m_psz)));\
}\
else\
{\
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(success_event(siaPath, filePath)));\
statusUpdated = true;\
}\
}\
catch (SQLite::Exception e)\
{\
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DatabaseExceptionOccurred(e)));\
}
template <typename... Ts>
SString fmt(const SString &fmt, Ts... vs)
{
size_t required = _sntprintf(nullptr, 0, fmt.c_str(), vs...);
SString ret;
ret.resize(required);
_sntprintf(&ret[0], required, fmt.c_str(), vs...);
return ret;
}
static void CreateTableIfNotFound(SQLite::Database* database, const SString& tableName, const SString& columns)
{
SString sqlCreate = fmt(TABLE_CREATE, &tableName[0], &columns[0]);
database->exec(CW2A(sqlCreate.c_str()));
}
SString CUploadManager::UploadStatusToString(const UploadStatus& uploadStatus)
{
switch (uploadStatus)
{
case UploadStatus::Complete:
return L"Complete";
case UploadStatus::Copying:
return L"Copying";
case UploadStatus::Error:
return L"Error";
case UploadStatus::Modified:
return L"Modified";
case UploadStatus::NotFound:
return L"Not Found";
case UploadStatus::Queued:
return L"Queued";
case UploadStatus::Remove:
return L"Remove";
case UploadStatus::Uploading:
return L"Uploading";
default:
return L"!!Not Defined!!";
}
}
CUploadManager::CUploadManager(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig) :
CAutoThread(siaCurl, siaDriveConfig),
_uploadDatabase(siaDriveConfig->GetRenter_UploadDbFilePath(), SQLite::OPEN_CREATE | SQLite::OPEN_READWRITE),
_fileThread(siaCurl, siaDriveConfig, [this](const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig) { this->FileThreadCallback(siaCurl, siaDriveConfig); })
{
CreateTableIfNotFound(&_uploadDatabase, UPLOAD_TABLE, UPLOAD_TABLE_COLUMNS);
// Clean-up cache folder
if (!RecurDeleteFilesByExtentsion(CA2W(siaDriveConfig->GetCacheFolder().c_str()).m_psz, L".siadrive"))
{
throw StartupException(L"Failed to remove '.siadrive' files");
}
if (!RecurDeleteFilesByExtentsion(CA2W(siaDriveConfig->GetCacheFolder().c_str()).m_psz, L".siadrive.temp"))
{
throw StartupException(L"Failed to remove '.siadrive.temp' files");
}
// Re-add files to file action queue
UpdateFileQueueOnStartup();
// Detect files that have been removed since last startup
DeleteFilesRemovedFromSia(siaCurl, siaDriveConfig, true);
// Begin normal processing
StartAutoThread();
_fileThread.StartAutoThread();
}
CUploadManager::~CUploadManager()
{
// Stop all processing
_fileThread.StopAutoThread();
StopAutoThread();
}
void CUploadManager::UpdateFileQueueOnStartup()
{
try
{
// Re-add Copying and Remove
SQLite::Statement query(_uploadDatabase, QUERY_UPLOADS_BY_SIA_PATH_AND_2_STATUS);
query.bind("@status1", static_cast<unsigned>(UploadStatus::Copying));
query.bind("@status1", static_cast<unsigned>(UploadStatus::Remove));
while (query.executeStep())
{
SString siaPath = CA2W(query.getColumn(query.getColumnIndex("sia_path"))).m_psz;
SString filePath = CA2W(query.getColumn(query.getColumnIndex("file_path"))).m_psz;
SString siaDriveFilePath = CA2W(query.getColumn(query.getColumnIndex("sd_file_path"))).m_psz;
UploadStatus uploadStatus = static_cast<UploadStatus>(query.getColumn(query.getColumnIndex("status")).getUInt());
SString temp = filePath;
::PathRemoveFileSpec(&temp[0]);
SString rootPath = temp;
// Strip drive specification (i.e. C:\)
// TODO If mount to folder is ever enabled, this will need to change
SString siaDriveFileName = GenerateSha256(&filePath[3]) + L".siadrive";
SString tempSourcePath;
tempSourcePath.resize(MAX_PATH + 1);
PathCombine(&tempSourcePath[0], rootPath.c_str(), (siaDriveFileName + L".temp").c_str());
std::lock_guard<std::mutex> l(_fileQueueMutex);
if (uploadStatus == UploadStatus::Remove)
{
_fileQueue.push_back([=]() { this->FileAction(CSiaCurl(GetHostConfig()), siaPath, filePath, nullptr, siaDriveFilePath, true); });
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(FileRemoveAdded(siaPath)));
}
else
{
_fileQueue.push_back([=]() { this->FileAction(CSiaCurl(GetHostConfig()), siaPath, filePath, tempSourcePath, siaDriveFilePath, false); });
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(NewFileAdded(siaPath, filePath, siaDriveFilePath)));
}
}
}
catch (SQLite::Exception e)
{
throw StartupException(e.getErrorStr());
}
}
void CUploadManager::DeleteFilesRemovedFromSia(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig, const bool& isStartup)
{
CSiaFileTreePtr fileTree(new CSiaFileTree(siaCurl, siaDriveConfig));
json result;
SiaCurlError cerror = siaCurl.Get(L"/renter/files", result);
if (ApiSuccess(cerror))
{
fileTree->BuildTree(result);
auto fileList = fileTree->GetFileList();
}
else
{
if (isStartup)
{
throw StartupException(L"Failed to get Sia files");
}
}
}
void CUploadManager::FileThreadCallback(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig)
{
std::function<void()> nextFile = nullptr;
{
std::lock_guard<std::mutex> l(_fileQueueMutex);
if (_fileQueue.size())
{
nextFile = _fileQueue.front();
_fileQueue.pop_front();
}
}
if (nextFile)
{
nextFile();
}
}
void CUploadManager::HandleFileRemove(const CSiaCurl& siaCurl, const SString& siaPath, const SString& siaDriveFilePath)
{
try
{
std::lock_guard<std::mutex> l(_uploadMutex);
SQLite::Statement query(_uploadDatabase, QUERY_STATUS);
query.bind("@sia_path", CW2A(siaPath.c_str()).m_psz);
if (query.executeStep())
{
SString removeFilePath = CA2W(query.getColumn(query.getColumnIndex("file_path"))).m_psz;
UploadStatus uploadStatus = static_cast<UploadStatus>(static_cast<unsigned>(query.getColumn(query.getColumnIndex("status"))));
// Make sure status is still remove
if (uploadStatus == UploadStatus::Remove)
{
bool deleteFromDb = true;
if (::PathFileExists(removeFilePath.c_str()))
{
if (RetryDeleteFileIfExists(removeFilePath.c_str()))
{
if (!RetryDeleteFileIfExists(siaDriveFilePath))
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DeleteSiaDriveFileFailed(siaPath, removeFilePath, siaDriveFilePath)));
}
}
else
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(RemoveFileFailed(siaPath, removeFilePath)));
deleteFromDb = false;
}
}
if (deleteFromDb)
{
json response;
SiaCurlError cerror = siaCurl.Post(SString(L"/renter/delete") + siaPath, {}, response);
if (ApiSuccess(cerror))
{
// TODO validate response
SQLite::Statement del(_uploadDatabase, DELETE_UPLOAD);
del.bind("@sia_path", CW2A(siaPath.c_str()));
if (del.exec() == 1)
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(FileRemoved(siaPath, removeFilePath)));
}
else
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DatabaseDeleteFailed(siaPath, removeFilePath, CA2W(del.getErrorMsg()).m_psz)));
}
}
else
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(FailedToDeleteFromSia(siaPath, removeFilePath, cerror)));
}
}
}
}
}
catch (SQLite::Exception e)
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DatabaseExceptionOccurred(e)));
}
}
void CUploadManager::HandleAddFile(const SString& siaPath, const SString& filePath, const SString& tempSourcePath, const SString& siaDriveFilePath)
{
// Check for retry condition
if (!::PathFileExists(tempSourcePath.c_str()) && ::PathFileExists(siaDriveFilePath.c_str()))
{
try
{
std::lock_guard<std::mutex> l(_uploadMutex);
SQLite::Statement query(_uploadDatabase, QUERY_STATUS);
query.bind("@sia_path", CW2A(siaPath.c_str()).m_psz);
if (query.executeStep())
{
UploadStatus uploadStatus = static_cast<UploadStatus>(static_cast<unsigned>(query.getColumn(query.getColumnIndex("status"))));
if (uploadStatus == UploadStatus::Copying)
{
if (RetryDeleteFileIfExists(siaDriveFilePath))
{
SET_STATUS(UploadStatus::Queued, UploadAddedToQueue, ModifyUploadStatusFailed)
}
else
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DeleteSiaDriveFileFailed(siaPath, filePath, siaDriveFilePath)));
}
}
}
}
catch (SQLite::Exception e)
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DatabaseExceptionOccurred(e)));
}
}
else
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(CreatingTemporarySiaDriveFile(siaPath, filePath, tempSourcePath)));
if (RetryableAction(::CopyFile(filePath.c_str(), tempSourcePath.c_str(), FALSE), DEFAULT_RETRY_COUNT, DEFAULT_RETRY_DELAY_MS))
{
// Delete existing '.siadrive' file, if found
// !!Should never come here. If so, there was a problem with startup clean-up
if (!RetryDeleteFileIfExists(siaDriveFilePath))
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DeleteSiaDriveFileFailed(siaPath, filePath, siaDriveFilePath)));
}
// Rename '.siadrive.temp' to '.siadrive'
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(RenamingTemporarySiaDriveFile(siaPath, filePath, tempSourcePath, siaDriveFilePath)));
if (RetryableAction(::MoveFile(tempSourcePath.c_str(), siaDriveFilePath.c_str()), DEFAULT_RETRY_COUNT, DEFAULT_RETRY_DELAY_MS))
{
if (!RetryDeleteFileIfExists(tempSourcePath))
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DeleteTemporarySiaDriveFileFailed(siaPath, filePath, tempSourcePath)));
}
try
{
std::lock_guard<std::mutex> l(_uploadMutex);
SQLite::Statement query(_uploadDatabase, QUERY_STATUS);
query.bind("@sia_path", CW2A(siaPath.c_str()).m_psz);
if (query.executeStep())
{
UploadStatus uploadStatus = static_cast<UploadStatus>(static_cast<unsigned>(query.getColumn(query.getColumnIndex("status"))));
if (uploadStatus == UploadStatus::Copying)
{
SET_STATUS(UploadStatus::Queued, UploadAddedToQueue, ModifyUploadStatusFailed)
}
}
}
catch (SQLite::Exception e)
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DatabaseExceptionOccurred(e)));
}
}
else
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(RenamingTemporarySiaDriveFileFailed(siaPath, filePath, tempSourcePath, siaDriveFilePath)));
if (!RetryDeleteFileIfExists(tempSourcePath))
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DeleteTemporarySiaDriveFileFailed(siaPath, filePath, tempSourcePath)));
}
if (!RetryDeleteFileIfExists(siaDriveFilePath))
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DeleteSiaDriveFileFailed(siaPath, filePath, siaDriveFilePath)));
}
// Requeued
}
}
else
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(CreatingTemporarySiaDriveFileFailed(siaPath, filePath, tempSourcePath)));
// If temp copy fails, try to delete
// If partial copy and file is unable to be deleted, log warning
if (!RetryDeleteFileIfExists(tempSourcePath))
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DeleteTemporarySiaDriveFileFailed(siaPath, filePath, tempSourcePath)));
}
// Requeued
}
}
}
void CUploadManager::FileAction(const CSiaCurl& siaCurl, const SString& siaPath, const SString& filePath, const SString& tempSourcePath, const SString& siaDriveFilePath, const bool& remove)
{
{
std::lock_guard<std::mutex> l(_fileActionMutex);
_activeSiaPath = siaPath;
}
if (remove)
{
HandleFileRemove(siaCurl, siaPath, siaDriveFilePath);
}
else
{
HandleAddFile(siaPath, filePath, tempSourcePath, siaDriveFilePath);
}
{
std::lock_guard<std::mutex> l(_fileActionMutex);
_activeSiaPath.empty();
}
}
void CUploadManager::AutoThreadCallback(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig)
{
bool processNext = true;
try
{
CSiaFileTreePtr fileTree(new CSiaFileTree(siaCurl, siaDriveConfig));
json result;
if (ApiSuccess(siaCurl.Get(L"/renter/files", result)))
{
// Lock here - if file is modified again before a prior upload is complete, delete it and
// start again later
std::lock_guard<std::mutex> l(_uploadMutex);
SQLite::Statement query(_uploadDatabase, QUERY_UPLOADS_BY_2_STATUS);
query.bind("@status1", static_cast<unsigned>(UploadStatus::Uploading));
query.bind("@status2", static_cast<unsigned>(UploadStatus::Modified));
fileTree->BuildTree(result);
if (query.executeStep())
{
SString siaPath = CA2W(query.getColumn(query.getColumnIndex("sia_path"))).m_psz;
SString filePath = CA2W(query.getColumn(query.getColumnIndex("file_path"))).m_psz;
SString siaDriveFilePath = CA2W(query.getColumn(query.getColumnIndex("sd_file_path"))).m_psz;
UploadStatus uploadStatus = static_cast<UploadStatus>(query.getColumn(query.getColumnIndex("status")).getUInt());
auto fileList = fileTree->GetFileList();
auto it = std::find_if(fileList.begin(), fileList.end(), [&](const CSiaFilePtr& ptr)
{
return ptr->GetSiaPath() == siaPath;
});
// Removed by another client
if (it == fileList.end())
{
SET_STATUS(UploadStatus::Remove, ExternallyRemovedFileDetected, ModifyUploadStatusFailed)
if (statusUpdated)
{
std::lock_guard<std::mutex> l2(_fileQueueMutex);
_fileQueue.push_back([=]() { this->FileAction(CSiaCurl(GetHostConfig()), siaPath, filePath, nullptr, siaDriveFilePath, true); });
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(FileRemoveAdded(siaPath)));
}
}
// Changed file detected
else if (uploadStatus == UploadStatus::Modified)
{
json response;
SiaCurlError cerror = siaCurl.Post(SString(L"/renter/delete") + siaPath, {}, response);
if (ApiSuccess(cerror))
{
// TODO validate response
SET_STATUS(UploadStatus::Queued, ModifiedUploadQueued, ModifyUploadStatusFailed)
}
else
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(FailedToDeleteFromSia(siaPath, filePath, cerror)));
}
}
// Upload is complete
else if ((*it)->GetUploadProgress() >= 100)
{
SET_STATUS(UploadStatus::Complete, UploadComplete, ModifyUploadStatusFailed)
if (!RetryDeleteFileIfExists(siaDriveFilePath))
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DeleteSiaDriveFileFailed(siaPath, filePath, siaDriveFilePath)));
}
}
// Upload still active, don't process another file
else
{
processNext = false;
}
}
}
else
{
// error condition - host down?
processNext = false;
}
}
catch (const SQLite::Exception& e)
{
// error condition - database not initialized (i.e. no table)?
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DatabaseExceptionOccurred(e)));
processNext = false;
}
if (processNext)
{
try
{
std::lock_guard<std::mutex> l(_uploadMutex);
SQLite::Statement query(_uploadDatabase, QUERY_UPLOADS_BY_STATUS);
query.bind("@status", static_cast<unsigned>(UploadStatus::Queued));
// Lock here - if file is modified again before a prior upload is complete, delete it and
// start again later
if (query.executeStep())
{
SString siaPath = CA2W(query.getColumn(query.getColumnIndex("sia_path"))).m_psz;
SString filePath = CA2W(query.getColumn(query.getColumnIndex("file_path"))).m_psz;
// TODO Validate response
json response;
SiaCurlError cerror = siaCurl.Post(SString(L"/renter/upload") + siaPath, { {L"source", filePath} }, response);
if (ApiSuccess(cerror))
{
SET_STATUS(UploadStatus::Uploading, UploadToSiaStarted, ModifyUploadStatusFailed)
}
}
}
catch (const SQLite::Exception& e)
{
// error condition
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DatabaseExceptionOccurred(e)));
}
}
}
UploadStatus CUploadManager::GetUploadStatus(const SString& siaPath)
{
UploadStatus uploadStatus = UploadStatus::NotFound;
SQLite::Statement query(_uploadDatabase, QUERY_UPLOADS_BY_SIA_PATH);
query.bind("@sia_path", CW2A(siaPath.c_str()).m_psz);
if (query.executeStep())
{
uploadStatus = static_cast<UploadStatus>(static_cast<unsigned>(query.getColumn(query.getColumnIndex("status"))));
}
return uploadStatus;
}
// The real source file is copied to a hidden file. The hidden filename is constructed by generating an SHA25 hash of the
// source path (all lowercase). '.siadrive.temp' will be used as the extension. After copy is successful, the extension
// is renamed to '.siadrive'. '.siadrive' files will be hidden/system and used by the Dokan API for file i/o until Sia upload
// is complete. If a change occurs, the file will be deleted from Sia (cancelling the in-progress upload) and renamed to the
// real source path. The process will then start over again as if the file was new.
// If the file has been fully uploaded, the hidden file should not exist, so rename will not occur; however, the file
// will be deleted from Sia and treated as new.
// Uploads will always use the real file. User i/o will occur against the hidden file temporarily. Since upload will take
// longer to handle than a normal file copy, this seems to be the best compromise for performance.
//
// Error Scenarios:
// Crash before db update to status copying - file will be re-uploaded automatically, if complete; otherwise, deleted
// Crash before copy begins - on startup, check for copying status with no .siadrive
// Crash during copy - on startup, check for copying status and delete .siadrive
// Crash after copy but before db update - on startup, check for copying status and delete .siadrive
UploadError CUploadManager::AddOrUpdate(const SString& siaPath, SString filePath)
{
UploadError ret = UploadError::Success;
// Relative to absolute and grab parent folder of source
SString rootPath;
{
SString temp;
if (::PathIsRelative(filePath.c_str()))
{
temp.resize(MAX_PATH + 1);
filePath = _wfullpath(&temp[0], filePath.c_str(), MAX_PATH);
}
temp = filePath;
::PathRemoveFileSpec(&temp[0]);
rootPath = temp;
}
if (::PathFileExists(filePath.c_str()))
{
// Lock here - if file is modified again before a prior upload is complete, delete it and
// start again later
std::lock_guard<std::mutex> l(_uploadMutex);
try
{
SQLite::Statement query(_uploadDatabase, QUERY_UPLOADS_BY_SIA_PATH_AND_2_STATUS);
query.bind("@sia_path", CW2A(siaPath.c_str()).m_psz);
query.bind("@status1", static_cast<unsigned>(UploadStatus::Uploading));
query.bind("@status2", static_cast<unsigned>(UploadStatus::Modified));
// Check copying
if (query.executeStep())
{
UploadStatus uploadStatus = static_cast<UploadStatus>(static_cast<unsigned>(query.getColumn(query.getColumnIndex("status"))));
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(ExistingUploadFound(siaPath, filePath, uploadStatus)));
if (uploadStatus == UploadStatus::Uploading)
{
SET_STATUS(UploadStatus::Modified, UploadStatusSetToModified, ModifyUploadStatusFailed)
}
}
else
{
// Strip drive specification (i.e. C:\)
// TODO If mount to folder is ever enabled, this will need to change
// TODO Case sensative file names? Going to be a bit of an issue.
SString siaDriveFileName = GenerateSha256(&filePath[3]) + L".siadrive";
SString siaDriveFilePath;
siaDriveFilePath.resize(MAX_PATH + 1);
PathCombine(&siaDriveFilePath[0], rootPath.c_str(), siaDriveFileName.c_str());
SString tempSourcePath;
tempSourcePath.resize(MAX_PATH + 1);
PathCombine(&tempSourcePath[0], rootPath.c_str(), (siaDriveFileName + L".temp").c_str());
// Add to db
try
{
SQLite::Statement insert(_uploadDatabase, INSERT_UPLOAD);
insert.bind("@sia_path", CW2A(siaPath.c_str()).m_psz);
insert.bind("@file_path", CW2A(filePath.c_str()).m_psz);
insert.bind("@sd_file_path", CW2A(siaDriveFileName.c_str()).m_psz);
insert.bind("@status", static_cast<unsigned>(UploadStatus::Copying));
if (insert.exec() == 1)
{
// Queue file upload operation
std::lock_guard<std::mutex> l2(_fileQueueMutex);
_fileQueue.push_back([=]() { this->FileAction(CSiaCurl(GetHostConfig()), siaPath, filePath, tempSourcePath, siaDriveFilePath, false); });
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(NewFileAdded(siaPath, filePath, siaDriveFilePath)));
}
else
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DatabaseInsertFailed(siaPath, filePath, CA2W(insert.getErrorMsg()).m_psz)));
ret = UploadError::DatabaseError;
}
}
catch (SQLite::Exception e)
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DatabaseExceptionOccurred(e)));
ret = UploadError::DatabaseError;
}
}
}
catch (SQLite::Exception e)
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DatabaseExceptionOccurred(e)));
ret = UploadError::DatabaseError;
}
}
else
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(SourceFileNotFound(siaPath, filePath)));
ret = UploadError::SourceFileNotFound;
}
return ret;
}
UploadError CUploadManager::Remove(const SString& siaPath)
{
UploadError ret = UploadError::Success;
std::lock_guard<std::mutex> l(_uploadMutex);
try
{
bool remove = false;
SQLite::Statement query(_uploadDatabase, QUERY_STATUS);
query.bind("@sia_path", CW2A(siaPath.c_str()).m_psz);
if (query.executeStep())
{
SString filePath = CA2W(query.getColumn(query.getColumnIndex("file_path"))).m_psz;
SString siaDriveFilePath = CA2W(query.getColumn(query.getColumnIndex("sd_file_path"))).m_psz;
UploadStatus uploadStatus = static_cast<UploadStatus>(static_cast<unsigned>(query.getColumn(query.getColumnIndex("status"))));
switch (uploadStatus)
{
case UploadStatus::Complete:
case UploadStatus::Queued:
case UploadStatus::Modified:
case UploadStatus::Copying:
case UploadStatus::Uploading:
{
SET_STATUS(UploadStatus::Remove, UploadStatusSetToRemoved, ModifyUploadStatusFailed)
remove = statusUpdated;
if (!statusUpdated)
{
ret = UploadError::DatabaseError;
}
}
break;
default:
{
remove = false;
}
break;
}
if (remove)
{
std::lock_guard<std::mutex> l2(_fileQueueMutex);
_fileQueue.push_back([=]() { this->FileAction(CSiaCurl(GetHostConfig()), siaPath, filePath, nullptr, siaDriveFilePath, true); });
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(FileRemoveAdded(siaPath)));
}
}
}
catch (SQLite::Exception e)
{
CEventSystem::EventSystem.NotifyEvent(CreateSystemEvent(DatabaseExceptionOccurred(e)));
ret = UploadError::DatabaseError;
}
return ret;
}