From b7cfb942a2944fb99b758553411856c342d26df2 Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Wed, 22 Feb 2017 19:25:04 -0600 Subject: [PATCH] Upload manager changes --- SiaDrive.Api/SiaCommon.cpp | 66 +++++++++++++++++-------------- SiaDrive.Api/SiaCommon.h | 2 + SiaDrive.Api/SiaDriveConfig.cpp | 8 ++++ SiaDrive.Api/SiaDriveConfig.h | 1 + SiaDrive.Api/UploadManager.cpp | 28 ++++++++++--- SiaDrive.Api/UploadManager.h | 4 +- SiaDrive.sln | 12 ++---- UnitTests/SiaDriveConfigTests.cpp | 6 +++ UnitTests/UploadManagerTest.cpp | 3 +- 9 files changed, 85 insertions(+), 45 deletions(-) diff --git a/SiaDrive.Api/SiaCommon.cpp b/SiaDrive.Api/SiaCommon.cpp index 7d353a3..74df551 100644 --- a/SiaDrive.Api/SiaCommon.cpp +++ b/SiaDrive.Api/SiaCommon.cpp @@ -1,36 +1,44 @@ #include "stdafx.h" #include "SiaCommon.h" #include +#include -using namespace Sia::Api; +NS_BEGIN(Sia) +NS_BEGIN(Api) +String AFX_EXT_API GenerateSha256(const String& str) +{ + String ret; + HCRYPTPROV hCryptProv = 0; + HCRYPTHASH hHash = 0; + BOOL ok = CryptAcquireContext(&hCryptProv, NULL, NULL, PROV_RSA_AES, 0); + ok = ok && CryptCreateHash(hCryptProv, CALG_SHA_256, 0, 0, &hHash); + ok = ok && CryptHashData(hHash, reinterpret_cast(&str[0]), str.length() * sizeof(TCHAR), 0); + if (ok) + { + DWORD dwHashLen; + DWORD dwCount = sizeof(DWORD); + if (CryptGetHashParam(hHash, HP_HASHSIZE, reinterpret_cast(&dwHashLen), &dwCount, 0)) + { + std::vector hash(dwHashLen); + if (CryptGetHashParam(hHash, HP_HASHVAL, reinterpret_cast(&hash[0]), &dwHashLen, 0)) + { + std::ostringstream ss; + ss << std::hex << std::uppercase << std::setfill('0'); + for (int c : hash) + { + ss << std::setw(2) << c; + } + std::string s = ss.str(); + ret = CA2W(s.c_str()).m_psz; + } + } + } + + if (hHash) CryptDestroyHash(hHash); + if (hCryptProv) CryptReleaseContext(hCryptProv, 0); -/* -// avgHostMetric computes the average of the metric given by `metric` on the -// list of `hosts`. -const avgHostMetric = (hosts, metric) = > -hosts.reduce((sum, host) = > sum.add(host[metric]), new BigNumber(0)) -.dividedBy(hosts.size) - -// avgStorageCost returns the average storage cost from a list of hosts given a -// period (blocks) and redundancy. -const avgStorageCost = (hosts, period, redundancy) = > -avgHostMetric(hosts, 'storageprice') -.times(period) -.plus(avgHostMetric(hosts, 'uploadbandwidthprice')) -.times(redundancy) -.plus(avgHostMetric(hosts, 'downloadbandwidthprice')) - -// Compute an estimated amount of storage from an amount of funds (Hastings) -// and a list of hosts. -export const estimatedStorage = (funds, hosts) = > { - const validHosts = List(hosts).take(28) - const avgStorage = avgStorageCost(validHosts, allowancePeriod, baseRedundancy) - - let fee = SiaAPI.siacoinsToHastings(baseFee) - fee = fee.plus(avgHostMetric(validHosts, 'contractprice').times(ncontracts)) - fee = fee.plus(funds.minus(fee).times(siafundRate)) - - return '~' + readableFilesize(Math.max(0, funds.minus(fee).dividedBy(avgStorage).toNumber().toPrecision(1))) + return ret; } -*/ \ No newline at end of file + +NS_END(2) \ No newline at end of file diff --git a/SiaDrive.Api/SiaCommon.h b/SiaDrive.Api/SiaCommon.h index 3d0f0a7..0ad0511 100644 --- a/SiaDrive.Api/SiaCommon.h +++ b/SiaDrive.Api/SiaCommon.h @@ -138,4 +138,6 @@ static T& ReplaceStringInPlace(T& subject, typename T::value_type* search, typen return ReplaceStringInPlace(subject, T(search), T(replace)); } +String AFX_EXT_API GenerateSha256(const String& str); + NS_END(2) \ No newline at end of file diff --git a/SiaDrive.Api/SiaDriveConfig.cpp b/SiaDrive.Api/SiaDriveConfig.cpp index f8ae942..cf6a8a4 100644 --- a/SiaDrive.Api/SiaDriveConfig.cpp +++ b/SiaDrive.Api/SiaDriveConfig.cpp @@ -11,6 +11,9 @@ CSiaDriveConfig::CSiaDriveConfig() : CSiaDriveConfig::CSiaDriveConfig(const String& filePath) : _FilePath(filePath) { +#ifdef DEBUG + DeleteFile(filePath.c_str()); +#endif Load(); } @@ -23,6 +26,11 @@ void CSiaDriveConfig::LoadDefaults() { SetUI_Main_TabIndex(0); SetRenter_UploadDbFilePath("./Config/renter_upload.db3"); + + std::string tempFolder; + tempFolder.resize(MAX_PATH + 1); + ::GetTempPathA(MAX_PATH, &tempFolder[0]); + SetTempFolder(tempFolder); } void CSiaDriveConfig::Load( ) diff --git a/SiaDrive.Api/SiaDriveConfig.h b/SiaDrive.Api/SiaDriveConfig.h index 952f549..aa6950c 100644 --- a/SiaDrive.Api/SiaDriveConfig.h +++ b/SiaDrive.Api/SiaDriveConfig.h @@ -17,6 +17,7 @@ public: Property(String, FilePath, public, private) JProperty(std::uint8_t, UI_Main_TabIndex, public, private, _configDocument) JProperty(std::string, Renter_UploadDbFilePath, public, private, _configDocument) + JProperty(std::string, TempFolder, public, private, _configDocument) private: json _configDocument; diff --git a/SiaDrive.Api/UploadManager.cpp b/SiaDrive.Api/UploadManager.cpp index 7987ae9..0e7ba55 100644 --- a/SiaDrive.Api/UploadManager.cpp +++ b/SiaDrive.Api/UploadManager.cpp @@ -7,7 +7,7 @@ using namespace Sia::Api; #define TABLE_CREATE L"create table if not exists %s (%s);" #define UPLOAD_TABLE L"upload_table" -#define UPLOAD_TABLE_COLUMNS L"id integer primary key autoincrement, sia_path text unique not null, cache_path text unique not null, status integer not null" +#define UPLOAD_TABLE_COLUMNS L"id integer primary key autoincrement, sia_path text unique not null, file_path text unique not null, temp_path text unique not null, status integer not null" #define QUERY_UPLOADS_BY_STATUS "select id, sia_path, status from upload_table where status=@status order by id desc limit 1;" #define QUERY_UPLOADS_BY_2_STATUS "select id, sia_path, status from upload_table where (status=@status1 or status=@status2) order by id desc limit 1;" #define QUERY_UPLOADS_BY_SIA_PATH "select id, sia_path, status from upload_table where sia_path=@sia_path order by id desc limit 1;" @@ -33,7 +33,8 @@ static void CreateTableIfNotFound(SQLite::Database* database, const String& tabl CUploadManager::CUploadManager(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig) : CAutoThread(siaCurl, siaDriveConfig), - _uploadDatabase(siaDriveConfig->GetRenter_UploadDbFilePath(), SQLite::OPEN_CREATE | SQLite::OPEN_READWRITE) + _uploadDatabase(siaDriveConfig->GetRenter_UploadDbFilePath(), SQLite::OPEN_CREATE | SQLite::OPEN_READWRITE), + _siaDriveConfig(siaDriveConfig) { CreateTableIfNotFound(&_uploadDatabase, UPLOAD_TABLE, UPLOAD_TABLE_COLUMNS); StartAutoThread(); @@ -54,11 +55,13 @@ void CUploadManager::AutoThreadCallback(const CSiaCurl& siaCurl, CSiaDriveConfig json result; if (ApiSuccess(siaCurl.Get(L"/renter/files", result))) { + // Lock here - if file is modified again before a prior upload is complete, delete it and + // start again later + std::lock_guard l(_uploadMutex); SQLite::Statement query(_uploadDatabase, QUERY_UPLOADS_BY_2_STATUS); query.bind("@status1", static_cast(UploadStatus::Uploading)); query.bind("@status2", static_cast(UploadStatus::Modified)); - // TODO Lock here - if file is modified again before a prior upload is complete, delete it and start again later fileTree->BuildTree(result); if (query.executeStep()) { @@ -110,7 +113,9 @@ void CUploadManager::AutoThreadCallback(const CSiaCurl& siaCurl, CSiaDriveConfig { SQLite::Statement query(_uploadDatabase, QUERY_UPLOADS_BY_STATUS); query.bind("@status", static_cast(UploadStatus::Queued)); - // TODO Lock here - if file is modified again before a prior upload is complete, delete it and start again later + + // Lock here - if file is modified again before a prior upload is complete, delete it and + // start again later if (query.executeStep()) { String siaPath = CA2W(query.getColumn(1)).m_psz; @@ -149,12 +154,14 @@ UploadStatus CUploadManager::GetUploadStatus(const String& siaPath) void CUploadManager::AddOrUpdate(const String& siaPath, const String& filePath) { + // Lock here - if file is modified again before a prior upload is complete, delete it and + // start again later + std::lock_guard l(_uploadMutex); + SQLite::Statement query(_uploadDatabase, QUERY_UPLOADS_BY_SIA_PATH_AND_2_STATUS); query.bind("@sia_path", CW2A(siaPath.c_str()).m_psz); query.bind("@status1", static_cast(UploadStatus::Uploading)); query.bind("@status2", static_cast(UploadStatus::Modified)); - - // TODO Lock here - if file is modified again before a prior upload is complete, delete it and start again later if (query.executeStep()) { if (static_cast(static_cast(query.getColumn(2))) == UploadStatus::Uploading) @@ -166,6 +173,15 @@ void CUploadManager::AddOrUpdate(const String& siaPath, const String& filePath) else { + String tempPath; + tempPath.resize(MAX_PATH + 1); + PathCombine(&tempPath[0], CA2W(_siaDriveConfig->GetTempFolder().c_str()), GenerateSha256(filePath).c_str()); + SQLite::Statement addOrUpdate(_uploadDatabase, ADD_UPDATE_UPLOAD); + addOrUpdate.bind("@sia_path", CW2A(siaPath.c_str()).m_psz); + addOrUpdate.bind("@file_path", CW2A(filePath.c_str()).m_psz); + addOrUpdate.bind("@temp_path", CW2A(tempPath.c_str()).m_psz); + addOrUpdate.bind("@status", static_cast(UploadStatus::Queued)); + tempPath = L""; } } diff --git a/SiaDrive.Api/UploadManager.h b/SiaDrive.Api/UploadManager.h index 7651e6c..4438de0 100644 --- a/SiaDrive.Api/UploadManager.h +++ b/SiaDrive.Api/UploadManager.h @@ -25,7 +25,8 @@ private: { std::uint64_t Id; String SiaPath; - String CachePath; + String FilePath; + String TempPath; _UploadStatus Status; } UploadData; @@ -38,6 +39,7 @@ public: private: SQLite::Database _uploadDatabase; std::mutex _uploadMutex; + CSiaDriveConfig* _siaDriveConfig; protected: virtual void AutoThreadCallback(const CSiaCurl& siaCurl, CSiaDriveConfig* siaDriveConfig) override; diff --git a/SiaDrive.sln b/SiaDrive.sln index e66672c..b63fc19 100644 --- a/SiaDrive.sln +++ b/SiaDrive.sln @@ -734,8 +734,7 @@ Global {B3DF927F-A1CE-4F50-A621-A4C3A06E4F8A}.RelWithDebInfo|x64.Build.0 = Release|x64 {B3DF927F-A1CE-4F50-A621-A4C3A06E4F8A}.RelWithDebInfo|x86.ActiveCfg = Release|Win32 {B3DF927F-A1CE-4F50-A621-A4C3A06E4F8A}.RelWithDebInfo|x86.Build.0 = Release|Win32 - {BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.Debug|x64.ActiveCfg = Debug|x64 - {BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.Debug|x64.Build.0 = Debug|x64 + {BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.Debug|x64.ActiveCfg = Debug|Win32 {BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.Debug|x86.ActiveCfg = Debug|Win32 {BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.Debug|x86.Build.0 = Debug|Win32 {BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.DLL Debug - DLL OpenSSL - DLL LibSSH2|x64.ActiveCfg = Release|Win32 @@ -853,15 +852,13 @@ Global {BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.MinSizeRel|x64.ActiveCfg = MinSizeRel|Win32 {BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Win32 {BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.MinSizeRel|x86.Build.0 = MinSizeRel|Win32 - {BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.Release|x64.ActiveCfg = Release|x64 - {BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.Release|x64.Build.0 = Release|x64 + {BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.Release|x64.ActiveCfg = Release|Win32 {BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.Release|x86.ActiveCfg = Release|Win32 {BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.Release|x86.Build.0 = Release|Win32 {BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.RelWithDebInfo|x64.ActiveCfg = RelWithDebInfo|Win32 {BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Win32 {BE7EE71D-6608-36DD-9687-D84AAE20C0A3}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Win32 - {92EF9CAE-3F0C-31D5-9556-62586CC5072D}.Debug|x64.ActiveCfg = Debug|x64 - {92EF9CAE-3F0C-31D5-9556-62586CC5072D}.Debug|x64.Build.0 = Debug|x64 + {92EF9CAE-3F0C-31D5-9556-62586CC5072D}.Debug|x64.ActiveCfg = Debug|Win32 {92EF9CAE-3F0C-31D5-9556-62586CC5072D}.Debug|x86.ActiveCfg = Debug|Win32 {92EF9CAE-3F0C-31D5-9556-62586CC5072D}.Debug|x86.Build.0 = Debug|Win32 {92EF9CAE-3F0C-31D5-9556-62586CC5072D}.DLL Debug - DLL OpenSSL - DLL LibSSH2|x64.ActiveCfg = Release|Win32 @@ -979,8 +976,7 @@ Global {92EF9CAE-3F0C-31D5-9556-62586CC5072D}.MinSizeRel|x64.ActiveCfg = MinSizeRel|Win32 {92EF9CAE-3F0C-31D5-9556-62586CC5072D}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Win32 {92EF9CAE-3F0C-31D5-9556-62586CC5072D}.MinSizeRel|x86.Build.0 = MinSizeRel|Win32 - {92EF9CAE-3F0C-31D5-9556-62586CC5072D}.Release|x64.ActiveCfg = Release|x64 - {92EF9CAE-3F0C-31D5-9556-62586CC5072D}.Release|x64.Build.0 = Release|x64 + {92EF9CAE-3F0C-31D5-9556-62586CC5072D}.Release|x64.ActiveCfg = Release|Win32 {92EF9CAE-3F0C-31D5-9556-62586CC5072D}.Release|x86.ActiveCfg = Release|Win32 {92EF9CAE-3F0C-31D5-9556-62586CC5072D}.Release|x86.Build.0 = Release|Win32 {92EF9CAE-3F0C-31D5-9556-62586CC5072D}.RelWithDebInfo|x64.ActiveCfg = RelWithDebInfo|Win32 diff --git a/UnitTests/SiaDriveConfigTests.cpp b/UnitTests/SiaDriveConfigTests.cpp index 0027143..0f879ec 100644 --- a/UnitTests/SiaDriveConfigTests.cpp +++ b/UnitTests/SiaDriveConfigTests.cpp @@ -35,6 +35,12 @@ namespace UnitTests json j = json::parse(s.begin(), s.end()); Assert::AreEqual(static_cast(0), j["UI_Main_TabIndex"].get ()); + std::string tempFolder; + tempFolder.resize(MAX_PATH + 1); + ::GetTempPathA(MAX_PATH, &tempFolder[0]); + Assert::AreEqual(tempFolder, j["TempFolder"].get ()); + Assert::AreEqual(std::string("./Config/renter_upload.db3"), j["Renter_UploadDbFilePath"].get ()); + DeleteFile(cfg.GetFilePath().c_str()); } }; diff --git a/UnitTests/UploadManagerTest.cpp b/UnitTests/UploadManagerTest.cpp index 86e3761..2808b68 100644 --- a/UnitTests/UploadManagerTest.cpp +++ b/UnitTests/UploadManagerTest.cpp @@ -20,6 +20,7 @@ namespace UnitTests public: TEST_METHOD_INITIALIZE(Initialize) { + // Always delete DB before next test CSiaDriveConfig config; ::DeleteFileA(config.GetRenter_UploadDbFilePath().c_str()); @@ -36,7 +37,7 @@ namespace UnitTests siad->Stop(); } - TEST_METHOD(TestMethod1) + TEST_METHOD(AddOrUpdateNoExisting) { siad->Start(SiadTestType::UploadFile); try