### Issues
* \#1 \[bug\] Unable to mount S3 due to 'item_not_found' exception
* \#2 Require bucket name for S3 mounts
* \#3 \[bug\] File size is not being updated in S3 mount
* \#4 Upgrade to libfuse-3.x.x
* \#5 Switch to renterd for Sia support
* \#6 Switch to cpp-httplib to further reduce dependencies
* \#7 Remove global_data and calculate used disk space per provider
* \#8 Switch to libcurl for S3 mount support
### Changes from v1.x.x
* Added read-only encrypt provider
* Pass-through mount point that transparently encrypts source data using `XChaCha20-Poly1305`
* Added S3 encryption support via `XChaCha20-Poly1305`
* Added replay protection to remote mounts
* Added support base64 writes in remote FUSE
* Created static linked Linux binaries for `amd64` and `aarch64` using `musl-libc`
* Removed legacy Sia renter support
* Removed Skynet support
* Fixed multiple remote mount WinFSP API issues on \*NIX servers
* Implemented chunked read and write
* Writes for non-cached files are performed in chunks of 8Mib
* Removed `repertory-ui` support
* Removed `FreeBSD` support
* Switched to `libsodium` over `CryptoPP`
* Switched to `XChaCha20-Poly1305` for remote mounts
* Updated `GoogleTest` to v1.14.0
* Updated `JSON for Modern C++` to v3.11.2
* Updated `OpenSSL` to v1.1.1w
* Updated `RocksDB` to v8.5.3
* Updated `WinFSP` to 2023
* Updated `boost` to v1.78.0
* Updated `cURL` to v8.3.0
* Updated `zlib` to v1.3
* Use `upload_manager` for all providers
* Adds a delay to uploads to prevent excessive API calls
* Supports re-upload after mount restart for incomplete uploads
* NOTE: Uploads for all providers are full file (no resume support)
* Multipart upload support is planned for S3
Reviewed-on: #9
176 lines
3.7 KiB
C++
176 lines
3.7 KiB
C++
#include "allocator.hpp"
|
|
|
|
#include <string.h>
|
|
#include <assert.h>
|
|
#include <stdlib.h>
|
|
|
|
// Address sanitizer
|
|
#if defined(__has_feature)
|
|
# define ADDRESS_SANITIZER __has_feature(address_sanitizer)
|
|
#else
|
|
# if defined(__SANITIZE_ADDRESS__)
|
|
# define ADDRESS_SANITIZER 1
|
|
# else
|
|
# define ADDRESS_SANITIZER 0
|
|
# endif
|
|
#endif
|
|
|
|
// Low-level allocation functions
|
|
#if defined(_WIN32) || defined(_WIN64)
|
|
# ifdef __MWERKS__
|
|
# pragma ANSI_strict off // disable ANSI strictness to include windows.h
|
|
# pragma cpp_extensions on // enable some extensions to include windows.h
|
|
# endif
|
|
|
|
# if defined(_MSC_VER)
|
|
# pragma warning(disable: 4201) // nonstandard extension used: nameless struct/union
|
|
# endif
|
|
|
|
# ifdef _XBOX_VER
|
|
# define NOD3D
|
|
# include <xtl.h>
|
|
# else
|
|
# include <windows.h>
|
|
# endif
|
|
|
|
namespace
|
|
{
|
|
const size_t page_size = 4096;
|
|
|
|
size_t align_to_page(size_t value)
|
|
{
|
|
return (value + page_size - 1) & ~(page_size - 1);
|
|
}
|
|
|
|
void* allocate_page_aligned(size_t size)
|
|
{
|
|
// We can't use VirtualAlloc because it has 64Kb granularity so we run out of address space quickly
|
|
// We can't use malloc because of occasional problems with CW on CRT termination
|
|
static HANDLE heap = HeapCreate(0, 0, 0);
|
|
|
|
void* result = HeapAlloc(heap, 0, size + page_size);
|
|
|
|
return reinterpret_cast<void*>(align_to_page(reinterpret_cast<size_t>(result)));
|
|
}
|
|
|
|
void* allocate(size_t size)
|
|
{
|
|
size_t aligned_size = align_to_page(size);
|
|
|
|
void* ptr = allocate_page_aligned(aligned_size + page_size);
|
|
if (!ptr) return 0;
|
|
|
|
char* end = static_cast<char*>(ptr) + aligned_size;
|
|
|
|
DWORD old_flags;
|
|
VirtualProtect(end, page_size, PAGE_NOACCESS, &old_flags);
|
|
|
|
return end - size;
|
|
}
|
|
|
|
void deallocate(void* ptr, size_t size)
|
|
{
|
|
size_t aligned_size = align_to_page(size);
|
|
|
|
void* rptr = static_cast<char*>(ptr) + size - aligned_size;
|
|
|
|
DWORD old_flags;
|
|
VirtualProtect(rptr, aligned_size + page_size, PAGE_NOACCESS, &old_flags);
|
|
}
|
|
}
|
|
#elif (defined(__APPLE__) || defined(__linux__)) && (defined(__i386) || defined(__x86_64)) && !ADDRESS_SANITIZER
|
|
# include <sys/mman.h>
|
|
|
|
namespace
|
|
{
|
|
const size_t page_size = 4096;
|
|
|
|
size_t align_to_page(size_t value)
|
|
{
|
|
return (value + page_size - 1) & ~(page_size - 1);
|
|
}
|
|
|
|
void* allocate_page_aligned(size_t size)
|
|
{
|
|
void* result = malloc(size + page_size);
|
|
|
|
return reinterpret_cast<void*>(align_to_page(reinterpret_cast<size_t>(result)));
|
|
}
|
|
|
|
void* allocate(size_t size)
|
|
{
|
|
size_t aligned_size = align_to_page(size);
|
|
|
|
void* ptr = allocate_page_aligned(aligned_size + page_size);
|
|
if (!ptr) return 0;
|
|
|
|
char* end = static_cast<char*>(ptr) + aligned_size;
|
|
|
|
int res = mprotect(end, page_size, PROT_NONE);
|
|
assert(res == 0);
|
|
(void)!res;
|
|
|
|
return end - size;
|
|
}
|
|
|
|
void deallocate(void* ptr, size_t size)
|
|
{
|
|
size_t aligned_size = align_to_page(size);
|
|
|
|
void* rptr = static_cast<char*>(ptr) + size - aligned_size;
|
|
|
|
int res = mprotect(rptr, aligned_size + page_size, PROT_NONE);
|
|
assert(res == 0);
|
|
(void)!res;
|
|
}
|
|
}
|
|
#else
|
|
namespace
|
|
{
|
|
void* allocate(size_t size)
|
|
{
|
|
return malloc(size);
|
|
}
|
|
|
|
void deallocate(void* ptr, size_t size)
|
|
{
|
|
(void)size;
|
|
|
|
free(ptr);
|
|
}
|
|
}
|
|
#endif
|
|
|
|
// High-level allocation functions
|
|
const size_t memory_alignment = sizeof(double) > sizeof(void*) ? sizeof(double) : sizeof(void*);
|
|
|
|
void* memory_allocate(size_t size)
|
|
{
|
|
void* result = allocate(size + memory_alignment);
|
|
if (!result) return 0;
|
|
|
|
memcpy(result, &size, sizeof(size_t));
|
|
|
|
return static_cast<char*>(result) + memory_alignment;
|
|
}
|
|
|
|
size_t memory_size(void* ptr)
|
|
{
|
|
assert(ptr);
|
|
|
|
size_t result;
|
|
memcpy(&result, static_cast<char*>(ptr) - memory_alignment, sizeof(size_t));
|
|
|
|
return result;
|
|
}
|
|
|
|
void memory_deallocate(void* ptr)
|
|
{
|
|
if (!ptr) return;
|
|
|
|
size_t size = memory_size(ptr);
|
|
|
|
deallocate(static_cast<char*>(ptr) - memory_alignment, size + memory_alignment);
|
|
}
|
|
|