BlockStorage/siadrive
Archived
1
0
Fork 0
Code Activity

Upload manager changes

Browse Source
This commit is contained in:
Scott E. Graves
2017-02-22 19:25:04 -06:00
parent 6c144c060e
commit b7cfb942a2
9 changed files with 85 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
SiaDrive.Api/SiaCommon.cpp
View File

@@ -1,36 +1,44 @@
#include "stdafx.h"
#include "SiaCommon.h"
#include <ttmath/ttmath.h>
#include <Wincrypt.h>
using namespace Sia::Api;
NS_BEGIN(Sia)
NS_BEGIN(Api)
String AFX_EXT_API GenerateSha256(const String& str)
{
String ret;
HCRYPTPROV hCryptProv = 0;
HCRYPTHASH hHash = 0;
BOOL ok = CryptAcquireContext(&hCryptProv, NULL, NULL, PROV_RSA_AES, 0);
ok = ok && CryptCreateHash(hCryptProv, CALG_SHA_256, 0, 0, &hHash);
ok = ok && CryptHashData(hHash, reinterpret_cast<const BYTE*>(&str[0]), str.length() * sizeof(TCHAR), 0);
if (ok)
{
DWORD dwHashLen;
DWORD dwCount = sizeof(DWORD);
if (CryptGetHashParam(hHash, HP_HASHSIZE, reinterpret_cast<BYTE *>(&dwHashLen), &dwCount, 0))
{
std::vector<unsigned char> hash(dwHashLen);
if (CryptGetHashParam(hHash, HP_HASHVAL, reinterpret_cast<BYTE *>(&hash[0]), &dwHashLen, 0))
{
std::ostringstream ss;
ss << std::hex << std::uppercase << std::setfill('0');
for (int c : hash)
{
ss << std::setw(2) << c;
}
std::string s = ss.str();
ret = CA2W(s.c_str()).m_psz;
}
}
}
if (hHash) CryptDestroyHash(hHash);
if (hCryptProv) CryptReleaseContext(hCryptProv, 0);
/*
// avgHostMetric computes the average of the metric given by `metric` on the
// list of `hosts`.
const avgHostMetric = (hosts, metric) = >
hosts.reduce((sum, host) = > sum.add(host[metric]), new BigNumber(0))
.dividedBy(hosts.size)
// avgStorageCost returns the average storage cost from a list of hosts given a
// period (blocks) and redundancy.
const avgStorageCost = (hosts, period, redundancy) = >
avgHostMetric(hosts, 'storageprice')
.times(period)
.plus(avgHostMetric(hosts, 'uploadbandwidthprice'))
.times(redundancy)
.plus(avgHostMetric(hosts, 'downloadbandwidthprice'))
// Compute an estimated amount of storage from an amount of funds (Hastings)
// and a list of hosts.
export const estimatedStorage = (funds, hosts) = > {
const validHosts = List(hosts).take(28)
const avgStorage = avgStorageCost(validHosts, allowancePeriod, baseRedundancy)
let fee = SiaAPI.siacoinsToHastings(baseFee)
fee = fee.plus(avgHostMetric(validHosts, 'contractprice').times(ncontracts))
fee = fee.plus(funds.minus(fee).times(siafundRate))
return '~' + readableFilesize(Math.max(0, funds.minus(fee).dividedBy(avgStorage).toNumber().toPrecision(1)))
return ret;
}
*/
NS_END(2)

2
SiaDrive.Api/SiaCommon.h
View File

@@ -138,4 +138,6 @@ static T& ReplaceStringInPlace(T& subject, typename T::value_type* search, typen
return ReplaceStringInPlace(subject, T(search), T(replace));
}
String AFX_EXT_API GenerateSha256(const String& str);
NS_END(2)

8
SiaDrive.Api/SiaDriveConfig.cpp
View File

@@ -11,6 +11,9 @@ CSiaDriveConfig::CSiaDriveConfig() :
CSiaDriveConfig::CSiaDriveConfig(const String& filePath) :
_FilePath(filePath)
{
#ifdef DEBUG
DeleteFile(filePath.c_str());
#endif
Load();
}
@@ -23,6 +26,11 @@ void CSiaDriveConfig::LoadDefaults()
{
SetUI_Main_TabIndex(0);
SetRenter_UploadDbFilePath("./Config/renter_upload.db3");
std::string tempFolder;
tempFolder.resize(MAX_PATH + 1);
::GetTempPathA(MAX_PATH, &tempFolder[0]);
SetTempFolder(tempFolder);
}
void CSiaDriveConfig::Load( )

1
SiaDrive.Api/SiaDriveConfig.h
View File

@@ -17,6 +17,7 @@ public:
Property(String, FilePath, public, private)
JProperty(std::uint8_t, UI_Main_TabIndex, public, private, _configDocument)
JProperty(std::string, Renter_UploadDbFilePath, public, private, _configDocument)
JProperty(std::string, TempFolder, public, private, _configDocument)
private:
json _configDocument;

28
SiaDrive.Api/UploadManager.cpp
View File

@@ -7,7 +7,7 @@ using namespace Sia::Api;
#define TABLE_CREATE L"create table if not exists %s (%s);"
#define UPLOAD_TABLE L"upload_table"
#define UPLOAD_TABLE_COLUMNS L"id integer primary key autoincrement, sia_path text unique not null, cache_path text unique not null, status integer not null"
#define UPLOAD_TABLE_COLUMNS L"id integer primary key autoincrement, sia_path text unique not null, file_path text unique not null, temp_path text unique not null, status integer not null"
#define QUERY_UPLOADS_BY_STATUS "select id, sia_path, status from upload_table where status=@status order by id desc limit 1;"
#define QUERY_UPLOADS_BY_2_STATUS "select id, sia_path, status from upload_table where (status=@status1 or status=@status2) order by id desc limit 1;"
#define QUERY_UPLOADS_BY_SIA_PATH "select id, sia_path, status from upload_table where sia_path=@sia_path order by id desc limit 1;"
@@ -33,7 +33,8 @@ static void CreateTableIfNotFound(SQLite::Database* database, const String& tabl
CUploadManager::CUploadManager(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig) :
CAutoThread(siaCurl, siaDriveConfig),
_uploadDatabase(siaDriveConfig->GetRenter_UploadDbFilePath(), SQLite::OPEN_CREATE | SQLite::OPEN_READWRITE)
_uploadDatabase(siaDriveConfig->GetRenter_UploadDbFilePath(), SQLite::OPEN_CREATE | SQLite::OPEN_READWRITE),
_siaDriveConfig(siaDriveConfig)
{
CreateTableIfNotFound(&_uploadDatabase, UPLOAD_TABLE, UPLOAD_TABLE_COLUMNS);
StartAutoThread();
@@ -54,11 +55,13 @@ void CUploadManager::AutoThreadCallback(const CSiaCurl& siaCurl, CSiaDriveConfig
json result;
if (ApiSuccess(siaCurl.Get(L"/renter/files", result)))
{
// Lock here - if file is modified again before a prior upload is complete, delete it and
// start again later
std::lock_guard<std::mutex> l(_uploadMutex);
SQLite::Statement query(_uploadDatabase, QUERY_UPLOADS_BY_2_STATUS);
query.bind("@status1", static_cast<unsigned>(UploadStatus::Uploading));
query.bind("@status2", static_cast<unsigned>(UploadStatus::Modified));
// TODO Lock here - if file is modified again before a prior upload is complete, delete it and start again later
fileTree->BuildTree(result);
if (query.executeStep())
{
@@ -110,7 +113,9 @@ void CUploadManager::AutoThreadCallback(const CSiaCurl& siaCurl, CSiaDriveConfig
{
SQLite::Statement query(_uploadDatabase, QUERY_UPLOADS_BY_STATUS);
query.bind("@status", static_cast<unsigned>(UploadStatus::Queued));
// TODO Lock here - if file is modified again before a prior upload is complete, delete it and start again later
// Lock here - if file is modified again before a prior upload is complete, delete it and
// start again later
if (query.executeStep())
{
String siaPath = CA2W(query.getColumn(1)).m_psz;
@@ -149,12 +154,14 @@ UploadStatus CUploadManager::GetUploadStatus(const String& siaPath)
void CUploadManager::AddOrUpdate(const String& siaPath, const String& filePath)
{
// Lock here - if file is modified again before a prior upload is complete, delete it and
// start again later
std::lock_guard<std::mutex> l(_uploadMutex);
SQLite::Statement query(_uploadDatabase, QUERY_UPLOADS_BY_SIA_PATH_AND_2_STATUS);
query.bind("@sia_path", CW2A(siaPath.c_str()).m_psz);
query.bind("@status1", static_cast<unsigned>(UploadStatus::Uploading));
query.bind("@status2", static_cast<unsigned>(UploadStatus::Modified));
// TODO Lock here - if file is modified again before a prior upload is complete, delete it and start again later
if (query.executeStep())
{
if (static_cast<UploadStatus>(static_cast<unsigned>(query.getColumn(2))) == UploadStatus::Uploading)
@@ -166,6 +173,15 @@ void CUploadManager::AddOrUpdate(const String& siaPath, const String& filePath)
else
{
String tempPath;
tempPath.resize(MAX_PATH + 1);
PathCombine(&tempPath[0], CA2W(_siaDriveConfig->GetTempFolder().c_str()), GenerateSha256(filePath).c_str());
SQLite::Statement addOrUpdate(_uploadDatabase, ADD_UPDATE_UPLOAD);
addOrUpdate.bind("@sia_path", CW2A(siaPath.c_str()).m_psz);
addOrUpdate.bind("@file_path", CW2A(filePath.c_str()).m_psz);
addOrUpdate.bind("@temp_path", CW2A(tempPath.c_str()).m_psz);
addOrUpdate.bind("@status", static_cast<unsigned>(UploadStatus::Queued));
tempPath = L"";
}
}

4
SiaDrive.Api/UploadManager.h
View File

@@ -25,7 +25,8 @@ private:
{
std::uint64_t Id;
String SiaPath;
String CachePath;
String FilePath;
String TempPath;
_UploadStatus Status;
} UploadData;
@@ -38,6 +39,7 @@ public:
private:
SQLite::Database _uploadDatabase;
std::mutex _uploadMutex;
CSiaDriveConfig* _siaDriveConfig;
protected:
virtual void AutoThreadCallback(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig) override;

12
SiaDrive.sln
View File

@@ -734,8 +734,7 @@ Global
{B3DF927F-A1CE-4F50-A621-A4C3A06E4F8A}.RelWithDebInfo|x64.Build.0 = Release|x64
{B3DF927F-A1CE-4F50-A621-A4C3A06E4F8A}.RelWithDebInfo|x86.ActiveCfg = Release|Win32
{B3DF927F-A1CE-4F50-A621-A4C3A06E4F8A}.RelWithDebInfo|x86.Build.0 = Release|Win32
{BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.Debug|x64.ActiveCfg = Debug|x64
{BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.Debug|x64.Build.0 = Debug|x64
{BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.Debug|x64.ActiveCfg = Debug|Win32
{BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.Debug|x86.ActiveCfg = Debug|Win32
{BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.Debug|x86.Build.0 = Debug|Win32
{BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.DLL Debug - DLL OpenSSL - DLL LibSSH2|x64.ActiveCfg = Release|Win32
@@ -853,15 +852,13 @@ Global
{BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.MinSizeRel|x64.ActiveCfg = MinSizeRel|Win32
{BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Win32
{BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.MinSizeRel|x86.Build.0 = MinSizeRel|Win32
{BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.Release|x64.ActiveCfg = Release|x64
{BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.Release|x64.Build.0 = Release|x64
{BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.Release|x64.ActiveCfg = Release|Win32
{BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.Release|x86.ActiveCfg = Release|Win32
{BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.Release|x86.Build.0 = Release|Win32
{BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.RelWithDebInfo|x64.ActiveCfg = RelWithDebInfo|Win32
{BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Win32
{BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Win32
{92EF9CAE-3F0C-31D5-9556-62586CC5072D}.Debug|x64.ActiveCfg = Debug|x64
{92EF9CAE-3F0C-31D5-9556-62586CC5072D}.Debug|x64.Build.0 = Debug|x64
{92EF9CAE-3F0C-31D5-9556-62586CC5072D}.Debug|x64.ActiveCfg = Debug|Win32
{92EF9CAE-3F0C-31D5-9556-62586CC5072D}.Debug|x86.ActiveCfg = Debug|Win32
{92EF9CAE-3F0C-31D5-9556-62586CC5072D}.Debug|x86.Build.0 = Debug|Win32
{92EF9CAE-3F0C-31D5-9556-62586CC5072D}.DLL Debug - DLL OpenSSL - DLL LibSSH2|x64.ActiveCfg = Release|Win32
@@ -979,8 +976,7 @@ Global
{92EF9CAE-3F0C-31D5-9556-62586CC5072D}.MinSizeRel|x64.ActiveCfg = MinSizeRel|Win32
{92EF9CAE-3F0C-31D5-9556-62586CC5072D}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Win32
{92EF9CAE-3F0C-31D5-9556-62586CC5072D}.MinSizeRel|x86.Build.0 = MinSizeRel|Win32
{92EF9CAE-3F0C-31D5-9556-62586CC5072D}.Release|x64.ActiveCfg = Release|x64
{92EF9CAE-3F0C-31D5-9556-62586CC5072D}.Release|x64.Build.0 = Release|x64
{92EF9CAE-3F0C-31D5-9556-62586CC5072D}.Release|x64.ActiveCfg = Release|Win32
{92EF9CAE-3F0C-31D5-9556-62586CC5072D}.Release|x86.ActiveCfg = Release|Win32
{92EF9CAE-3F0C-31D5-9556-62586CC5072D}.Release|x86.Build.0 = Release|Win32
{92EF9CAE-3F0C-31D5-9556-62586CC5072D}.RelWithDebInfo|x64.ActiveCfg = RelWithDebInfo|Win32

6
UnitTests/SiaDriveConfigTests.cpp
View File

@@ -35,6 +35,12 @@ namespace UnitTests
json j = json::parse(s.begin(), s.end());
Assert::AreEqual(static_cast<std::uint8_t>(0), j["UI_Main_TabIndex"].get <std::uint8_t>());
std::string tempFolder;
tempFolder.resize(MAX_PATH + 1);
::GetTempPathA(MAX_PATH, &tempFolder[0]);
Assert::AreEqual(tempFolder, j["TempFolder"].get <std::string>());
Assert::AreEqual(std::string("./Config/renter_upload.db3"), j["Renter_UploadDbFilePath"].get <std::string>());
DeleteFile(cfg.GetFilePath().c_str());
}
};

3
UnitTests/UploadManagerTest.cpp
View File

@@ -20,6 +20,7 @@ namespace UnitTests
public:
TEST_METHOD_INITIALIZE(Initialize)
{
// Always delete DB before next test
CSiaDriveConfig config;
::DeleteFileA(config.GetRenter_UploadDbFilePath().c_str());
@@ -36,7 +37,7 @@ namespace UnitTests
siad->Stop();
}
TEST_METHOD(TestMethod1)
TEST_METHOD(AddOrUpdateNoExisting)
{
siad->Start(SiadTestType::UploadFile);
try