v2.0.2-rc (#27)
Some checks reported errors
BlockStorage/repertory/pipeline/head Something is wrong with the build of this commit

## v2.0.2-rc

### BREAKING CHANGES

* Refactored `config.json` - will need to verify configuration settings prior to mounting

### Issues

* \#12 \[Unit Test\] Complete all providers unit tests
* \#14 \[Unit Test\] SQLite mini-ORM unit tests and cleanup
* \#16 Add support for bucket name in Sia provider
* \#17 Update to common c++ build system
  * A single 64-bit Linux Jenkins server is used to build all Linux and Windows versions
  * All dependency sources are now included
  * MSVC is no longer supported
  * MSYS2 is required for building Windows binaries on Windows
  * OS X support is temporarily disabled
* \#19 \[bug\] Rename file is broken for files that are existing
* \#23 \[bug\] Incorrect file size displayed while upload is pending
* \#24 RocksDB implementations should be transactional
* \#25 Writes should block when maximum cache size is reached
* \#26 Complete ring buffer and direct download support

### Changes from v2.0.1-rc

* Ability to choose between RocksDB and SQLite databases
* Added direct reads and implemented download fallback
* Corrected file times on S3 and Sia providers
* Corrected handling of `chown()` and `chmod()`
* Fixed erroneous download of chunks after resize

Reviewed-on: #27
This commit is contained in:
2024-12-28 15:56:40 -06:00
parent 1b8de3b097
commit 8dd46b8ad8
790 changed files with 49979 additions and 417734 deletions

View File

@ -1,273 +1,178 @@
cmake_minimum_required(VERSION 3.22)
project(repertory)
cmake_minimum_required(VERSION 3.27)
if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.24.0")
cmake_policy(SET CMP0135 NEW)
endif()
cmake_policy(SET CMP0135 NEW)
cmake_policy(SET CMP0144 NEW)
project(${PROJECT_NAME}
DESCRIPTION ${PROJECT_DESC}
HOMEPAGE_URL ${PROJECT_URL}
VERSION ${PROJECT_MAJOR_VERSION}.${PROJECT_MINOR_VERSION}.${PROJECT_REVISION_VERSION}
)
include(CheckIncludeFileCXX)
include(CheckIncludeFiles)
include(ExternalProject)
set(REPERTORY_MAJOR 2)
set(REPERTORY_MINOR 0)
set(REPERTORY_REV 1)
set(REPERTORY_RELEASE_NUM 0)
set(REPERTORY_RELEASE_ITER rc)
set(REPERTORY_VERSION ${REPERTORY_MAJOR}.${REPERTORY_MINOR}.${REPERTORY_REV}-${REPERTORY_RELEASE_ITER})
check_include_files(sys/xattr.h HAS_SETXATTR)
if(HAS_SETXATTR)
add_definitions(-DHAS_SETXATTR)
endif()
include(cmake/hashes.cmake)
include(cmake/arch_detection.cmake)
include(cmake/os_detection.cmake)
include(cmake/options.cmake)
include(cmake/settings.cmake)
include(cmake/flags.cmake)
include(cmake/versions.cmake)
include(cmake/arch.cmake)
include(cmake/os.cmake)
include(cmake/options.cmake)
if (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/3rd_party/cacert.pem")
file(REMOVE ${CMAKE_CURRENT_SOURCE_DIR}/3rd_party/cacert.pem)
option(PROJECT_REQUIRE_ALPINE "Require Alpine Linux as build system" OFF)
option(PROJECT_STATIC_LINK "Static link executables" OFF)
if(PROJECT_STATIC_LINK)
add_definitions(-DPROJECT_STATIC_LINK)
endif()
file(DOWNLOAD https://curl.haxx.se/ca/cacert.pem ${CMAKE_CURRENT_SOURCE_DIR}/3rd_party/cacert.pem)
unset(REPERTORY_GIT_REV CACHE)
execute_process(
COMMAND git rev-parse --short HEAD
WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
OUTPUT_VARIABLE REPERTORY_GIT_REV
ERROR_QUIET
OUTPUT_STRIP_TRAILING_WHITESPACE
)
if (UNIX)
check_include_files(sys/xattr.h HAS_SETXATTR)
if (HAS_SETXATTR)
set(REPERTORY_DEFINITIONS ${REPERTORY_DEFINITIONS} -DHAS_SETXATTR)
if(PROJECT_IS_MINGW)
add_definitions(-DPROJECT_IS_MINGW)
if(PROJECT_IS_MINGW_UNIX)
add_definitions(-DPROJECT_IS_MINGW_UNIX)
endif()
endif()
if (LINUX OR MINGW)
if (CMAKE_COMPILER_IS_GNUCC AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 8.0)
message(FATAL_ERROR "Require at least gcc-8.0")
if(PROJECT_REQUIRE_ALPINE)
add_definitions(-DPROJECT_REQUIRE_ALPINE)
endif()
if(PROJECT_IS_ARM64)
add_definitions(-DPROJECT_IS_ARM64)
endif()
if(PROJECT_IS_MINGW)
option(PROJECT_ENABLE_WIN32_LONG_PATH_NAMES "Enable path sizes of 32767 characters on Windows" OFF)
if(PROJECT_ENABLE_WIN32_LONG_PATH_NAMES)
add_definitions(-DPROJECT_ENABLE_WIN32_LONG_PATH_NAMES)
endif()
endif()
include(cmake/settings.cmake)
include(cmake/flags.cmake)
include(cmake/functions.cmake)
include(cmake/libraries.cmake)
if(PROJECT_BUILD)
file(WRITE ${CMAKE_CURRENT_SOURCE_DIR}/${PROJECT_NAME}/version.cpp "")
if (PROJECT_IS_MINGW)
set(PROJECT_WINDOWS_VERSION_RC ${CMAKE_CURRENT_SOURCE_DIR}/${PROJECT_NAME}/version.rc)
file(WRITE ${PROJECT_WINDOWS_VERSION_RC} "")
endif()
set(ENV{PKG_CONFIG_PATH}
"${EXTERNAL_BUILD_ROOT}/lib/pkgconfig:${EXTERNAL_BUILD_ROOT}/share/pkgconfig:$ENV{PKG_CONFIG_PATH}"
file(GLOB_RECURSE ADDITIONAL_SOURCES
${CMAKE_CURRENT_SOURCE_DIR}/${PROJECT_NAME}/version.cpp
${PROJECT_SUPPORT_DIR}/src/*.c
${PROJECT_SUPPORT_DIR}/src/*.cpp
)
find_package(PkgConfig REQUIRED)
list(APPEND PROJECT_ADDITIONAL_SOURCES
${ADDITIONAL_SOURCES}
)
set(Boost_USE_STATIC_LIBS ON)
include(project.cmake)
if (NOT MINGW)
include_directories(/usr/local/include)
link_directories(
/usr/local/lib
/usr/local/lib64
)
endif()
string(REPLACE "\\<" "<" PROJECT_COMPANY_NAME "${PROJECT_COMPANY_NAME}")
string(REPLACE "\\<" "<" PROJECT_COPYRIGHT "${PROJECT_COPYRIGHT}")
string(REPLACE "\\<" "<" PROJECT_DESC "${PROJECT_DESC}")
string(REPLACE "\\>" ">" PROJECT_COMPANY_NAME "${PROJECT_COMPANY_NAME}")
string(REPLACE "\\>" ">" PROJECT_COPYRIGHT "${PROJECT_COPYRIGHT}")
string(REPLACE "\\>" ">" PROJECT_DESC "${PROJECT_DESC}")
if (REPERTORY_MUSL)
find_library(LIBFUSE3 NO_CACHE NAMES libfuse3.a)
else()
if (NOT MINGW)
find_library(LIBDL_LIBRARY NO_CACHE NAMES dl)
find_library(LIBFUSE3 NO_CACHE NAMES fuse3)
endif()
endif()
if (REPERTORY_MUSL OR MINGW)
find_library(LIBPTHREAD_LIBRARY NO_CACHE NAMES libpthread.a)
else()
find_library(LIBPTHREAD_LIBRARY NO_CACHE NAMES pthread)
endif()
if (LIBFUSE3 AND NOT MINGW)
pkg_check_modules(LIBFUSE3 REQUIRED fuse3>=3.0.0)
set(LIBFUSE3_LIBRARIES ${LIBFUSE3})
set(REPERTORY_DEFINITIONS ${REPERTORY_DEFINITIONS} -DFUSE_USE_VERSION=30)
endif()
if(NOT LIBFUSE3_LIBRARIES AND NOT MINGW)
pkg_check_modules(LIBFUSE2 REQUIRED fuse>=2.9.0)
set(REPERTORY_DEFINITIONS ${REPERTORY_DEFINITIONS} -DFUSE_USE_VERSION=29)
endif()
if (NOT LIBPTHREAD_LIBRARY)
message(FATAL_ERROR "'libpthread' not found")
endif()
if (REPERTORY_MUSL)
if (LIBFUSE3_LIBRARIES)
set(LIBFUSE3_LIBRARIES libfuse3.a)
if (PROJECT_IS_MINGW)
if ("${PROJECT_RELEASE_ITER}" STREQUAL "release")
set(PROJECT_PRERELEASE 0)
else()
set(LIBFUSE2_LIBRARIES libfuse.a)
endif()
set(REPERTORY_LINK_LIBRARIES
${REPERTORY_LINK_LIBRARIES}
${LIBPTHREAD_LIBRARY}
)
else()
if (NOT MINGW AND NOT LIBDL_LIBRARY)
message(FATAL_ERROR "'libdl' not found")
set(PROJECT_PRERELEASE VS_FF_PRERELEASE)
endif()
set(REPERTORY_LINK_LIBRARIES
${REPERTORY_LINK_LIBRARIES}
${LIBPTHREAD_LIBRARY}
${LIBDL_LIBRARY}
configure_file(
${CMAKE_CURRENT_SOURCE_DIR}/${PROJECT_NAME}/version.rc.in
${CMAKE_CURRENT_SOURCE_DIR}/${PROJECT_NAME}/version.rc
@ONLY
)
endif()
elseif (FREEBSD)
message(FATAL_ERROR "FreeBSD is currently not supported")
elseif (MACOS)
set(CMAKE_OSX_ARCHITECTURES x86_64)
find_library(LIBDL_LIBRARY NO_CACHE NAMES dl)
if (NOT LIBDL_LIBRARY)
message(FATAL_ERROR "'libdl' not found")
endif()
find_library(LIBPTHREAD_LIBRARY NO_CACHE NAMES pthread)
if (NOT LIBPTHREAD_LIBRARY)
message(FATAL_ERROR "'libpthread' not found")
endif()
set(REPERTORY_DEFINITIONS ${REPERTORY_DEFINITIONS} -DFUSE_USE_VERSION=29 -DBOOST_ASIO_HAS_STD_STRING_VIEW)
include_directories(/usr/local/include)
find_library(OSXFUSE NO_CACHE NAMES OSXFUSE)
if (NOT OSXFUSE)
message(FATAL_ERROR "FUSE for macOS not found (https://osxfuse.github.io/)")
endif ()
set(LIBFUSE2_LIBRARIES fuse)
set(REPERTORY_LINK_LIBRARIES
${REPERTORY_LINK_LIBRARIES}
${LIBDL_LIBRARY}
${LIBPTHREAD_LIBRARY}
"-L/usr/local/lib"
"-framework Foundation"
"-framework AppKit"
"-framework Security"
"-framework SystemConfiguration"
configure_file(
${CMAKE_CURRENT_SOURCE_DIR}/${PROJECT_NAME}/version.cpp.in
${CMAKE_CURRENT_SOURCE_DIR}/${PROJECT_NAME}/version.cpp
@ONLY
)
elseif (MSVC)
message(FATAL_ERROR "MSVC is currently not supported [MinGW-64 should be used]")
else()
message(STATUS "-=[CMake Settings]=-")
message(STATUS " C standard: ${CMAKE_C_STANDARD}")
message(STATUS " C++ standard: ${CMAKE_CXX_STANDARD}")
message(STATUS " CPU architecture: ${PROJECT_MARCH}")
if(PROJECT_ENABLE_FUSE)
message(STATUS " FUSE version: ${PROJECT_FUSE}")
endif()
if (MINGW)
set(WINFSP_LIBRARY_BASENAME winfsp-x64)
string(REPLACE "<" "\\<" PROJECT_COMPANY_NAME "${PROJECT_COMPANY_NAME}")
string(REPLACE "<" "\\<" PROJECT_COPYRIGHT "${PROJECT_COPYRIGHT}")
string(REPLACE "<" "\\<" PROJECT_DESC "${PROJECT_DESC}")
string(REPLACE ">" "\\>" PROJECT_COMPANY_NAME "${PROJECT_COMPANY_NAME}")
string(REPLACE ">" "\\>" PROJECT_COPYRIGHT "${PROJECT_COPYRIGHT}")
string(REPLACE ">" "\\>" PROJECT_DESC "${PROJECT_DESC}")
include_directories(SYSTEM ${CMAKE_SOURCE_DIR}/3rd_party/winfsp-${WINFSP_VERSION}/inc)
link_directories(${CMAKE_SOURCE_DIR}/3rd_party/winfsp-${WINFSP_VERSION}/lib)
set(REPERTORY_LINK_LIBRARIES
${REPERTORY_LINK_LIBRARIES}
${WINFSP_LIBRARY_BASENAME}.lib
mswsock
)
set(REPERTORY_VER_FILEVERSION ${REPERTORY_MAJOR},${REPERTORY_MINOR},${REPERTORY_REV},${REPERTORY_RELEASE_NUM})
set(REPERTORY_VER_FILEVERSION_STR ${REPERTORY_MAJOR}.${REPERTORY_MINOR}.${REPERTORY_REV}.${REPERTORY_RELEASE_NUM})
set(REPERTORY_VER_PRODUCTVERSION ${REPERTORY_MAJOR},${REPERTORY_MINOR},${REPERTORY_REV},${REPERTORY_RELEASE_NUM})
set(REPERTORY_VER_PRODUCTVERSION_STR ${REPERTORY_MAJOR}.${REPERTORY_MINOR}.${REPERTORY_REV}.${REPERTORY_RELEASE_NUM})
set(REPERTORY_VER_PRERELEASE VS_FF_PRERELEASE)
set(REPERTORY_VER_COMPANYNAME_STR "https://git.fifthgrid.com/blockstorage")
set(REPERTORY_VER_LEGALCOPYRIGHT_STR "Copyright 2018-2023 <MIT License> <https://git.fifthgrid.com/blockstorage/repertory>")
set(REPERTORY_VER_FILEDESCRIPTION_STR "Mount utility for Sia and S3")
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/version.rc.in ${CMAKE_CURRENT_SOURCE_DIR}/src/version.rc @ONLY)
set(WINDOWS_VERSION_RC ${CMAKE_CURRENT_SOURCE_DIR}/src/version.rc)
set(REPERTORY_LINK_LIBRARIES
${REPERTORY_LINK_LIBRARIES}
advapi32${LIB_EXT}
bcrypt${LIB_EXT}
comdlg32${LIB_EXT}
crypt32${LIB_EXT}
gdi32${LIB_EXT}
httpapi${LIB_EXT}
kernel32${LIB_EXT}
ncrypt${LIB_EXT}
ole32${LIB_EXT}
oleaut32${LIB_EXT}
rpcrt4${LIB_EXT}
secur32${LIB_EXT}
shell32${LIB_EXT}
shlwapi${LIB_EXT}
user32${LIB_EXT}
userenv${LIB_EXT}
uuid${LIB_EXT}
version${LIB_EXT}
winhttp${LIB_EXT}
wininet${LIB_EXT}
winspool${LIB_EXT}
ws2_32${LIB_EXT}
add_custom_target(project ALL
DEPENDS ${PROJECT_DEPENDENCIES}
COMMAND cd build && cmake
${CMAKE_CURRENT_SOURCE_DIR}
-DCMAKE_GENERATOR=${CMAKE_GENERATOR}
-DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}
-DPROJECT_3RD_PARTY_DIR=${PROJECT_3RD_PARTY_DIR}
-DPROJECT_BUILD=ON
-DPROJECT_BUILD_ARCH=${PROJECT_BUILD_ARCH}
-DPROJECT_BUILD_DIR=${PROJECT_BUILD_DIR}
-DPROJECT_BUILD_SHARED_LIBS=${PROJECT_BUILD_SHARED_LIBS}
-DPROJECT_CMAKE_BUILD_TYPE=${PROJECT_CMAKE_BUILD_TYPE}
-DPROJECT_COMPANY_NAME=${PROJECT_COMPANY_NAME}
-DPROJECT_COPYRIGHT=${PROJECT_COPYRIGHT}
-DPROJECT_DESC=${PROJECT_DESC}
-DPROJECT_DIST_DIR=${PROJECT_DIST_DIR}
-DPROJECT_ENABLE_WIN32_LONG_PATH_NAMES=${PROJECT_ENABLE_WIN32_LONG_PATH_NAMES}
-DPROJECT_ENABLE_BOOST=${PROJECT_ENABLE_BOOST}
-DPROJECT_ENABLE_CPP_HTTPLIB=${PROJECT_ENABLE_CPP_HTTPLIB}
-DPROJECT_ENABLE_CURL=${PROJECT_ENABLE_CURL}
-DPROJECT_ENABLE_FUSE=${PROJECT_ENABLE_FUSE}
-DPROJECT_ENABLE_JSON=${PROJECT_ENABLE_JSON}
-DPROJECT_ENABLE_LIBSODIUM=${PROJECT_ENABLE_LIBSODIUM}
-DPROJECT_ENABLE_OPENSSL=${PROJECT_ENABLE_OPENSSL}
-DPROJECT_ENABLE_PUGIXML=${PROJECT_ENABLE_PUGIXML}
-DPROJECT_ENABLE_ROCKSDB=${PROJECT_ENABLE_ROCKSDB}
-DPROJECT_ENABLE_SPDLOG=${PROJECT_ENABLE_SPDLOG}
-DPROJECT_ENABLE_SQLITE=${PROJECT_ENABLE_SQLITE}
-DPROJECT_ENABLE_STDUUID=${PROJECT_ENABLE_STDUUID}
-DPROJECT_ENABLE_TESTING=${PROJECT_ENABLE_TESTING}
-DPROJECT_ENABLE_WINFSP=${PROJECT_ENABLE_WINFSP}
-DPROJECT_EXTERNAL_BUILD_ROOT=${PROJECT_EXTERNAL_BUILD_ROOT}
-DPROJECT_FUSE=${PROJECT_FUSE}
-DPROJECT_FUSE_INCLUDE_DIRS=${PROJECT_FUSE_INCLUDE_DIRS}
-DPROJECT_GIT_REV=${PROJECT_GIT_REV}
-DPROJECT_IS_ALPINE=${PROJECT_IS_ALPINE}
-DPROJECT_IS_ARM64=${PROJECT_IS_ARM64}
-DPROJECT_IS_MINGW=${PROJECT_IS_MINGW}
-DPROJECT_IS_MINGW_UNIX=${PROJECT_IS_MINGW_UNIX}
-DPROJECT_MAJOR_VERSION=${PROJECT_MAJOR_VERSION}
-DPROJECT_MINOR_VERSION=${PROJECT_MINOR_VERSION}
-DPROJECT_NAME=${PROJECT_NAME}
-DPROJECT_RELEASE_ITER=${PROJECT_RELEASE_ITER}
-DPROJECT_RELEASE_NUM=${PROJECT_RELEASE_NUM}
-DPROJECT_REQUIRE_ALPINE=${PROJECT_REQUIRE_ALPINE}
-DPROJECT_REVISION_VERSION=${PROJECT_REVISION_VERSION}
-DPROJECT_STATIC_LINK=${PROJECT_STATIC_LINK}
-DPROJECT_SUPPORT_DIR=${PROJECT_SUPPORT_DIR}
-DPROJECT_TOOLCHAIN_FILE_CMAKE=${PROJECT_TOOLCHAIN_FILE_CMAKE}
-DPROJECT_TOOLCHAIN_FILE_MESON=${PROJECT_TOOLCHAIN_FILE_MESON}
-DPROJECT_URL=${PROJECT_URL}
)
endif()
include_directories(SYSTEM
${EXTERNAL_BUILD_ROOT}/include
)
link_directories(
${EXTERNAL_BUILD_ROOT}/lib
)
include(cmake/zlib.cmake)
include(cmake/openssl.cmake)
include(cmake/curl.cmake)
include(cmake/boost.cmake)
include(cmake/libsodium.cmake)
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/src/common.cpp.in ${CMAKE_CURRENT_SOURCE_DIR}/src/common.cpp @ONLY)
include_directories(include)
include_directories(SYSTEM
${Boost_INCLUDE_DIR}
${CMAKE_CURRENT_SOURCE_DIR}/3rd_party/cpp-httplib
${CMAKE_CURRENT_SOURCE_DIR}/3rd_party/stduuid
${CMAKE_CURRENT_SOURCE_DIR}/3rd_party/json
${CMAKE_CURRENT_SOURCE_DIR}/3rd_party/pugixml/src
${CMAKE_CURRENT_SOURCE_DIR}/3rd_party/sqlite/include
${CURL_INCLUDE_DIRS}
${LIBFUSE2_INCLUDE_DIRS}
${LIBFUSE3_INCLUDE_DIRS}
${OPENSSL_INCLUDE_DIR}
)
set(REPERTORY_LINK_LIBRARIES
${LIBFUSE2_LIBRARIES}
${LIBFUSE3_LIBRARIES}
${Boost_LIBRARIES}
${CURL_LIBRARIES}
${OPENSSL_LIBRARIES}
${ZLIB_LIBRARIES}
${LIBSODIUM_LIBRARIES}
${REPERTORY_LINK_LIBRARIES}
)
file(GLOB_RECURSE REPERTORY_HEADERS
${CMAKE_CURRENT_SOURCE_DIR}/include/*.h
${CMAKE_CURRENT_SOURCE_DIR}/include/*.hh
${CMAKE_CURRENT_SOURCE_DIR}/include/*.hpp
${CMAKE_CURRENT_SOURCE_DIR}/include/**/*.h
${CMAKE_CURRENT_SOURCE_DIR}/include/**/*.hh
${CMAKE_CURRENT_SOURCE_DIR}/include/**/*.hpp
${CMAKE_CURRENT_SOURCE_DIR}/3rd_party/pugixml/src/*.hpp
${CMAKE_CURRENT_SOURCE_DIR}/3rd_party/sqlite/include/*.h
)
file(GLOB_RECURSE REPERTORY_SOURCES
${CMAKE_CURRENT_SOURCE_DIR}/src/*.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/**/*.cpp
${CMAKE_CURRENT_SOURCE_DIR}/3rd_party/pugixml/src/*.cpp
${CMAKE_CURRENT_SOURCE_DIR}/3rd_party/sqlite/src/*.c
)
list(REMOVE_ITEM REPERTORY_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/src/main.cpp)
include(cmake/helpers.cmake)
include(cmake/librepertory.cmake)
include(cmake/repertory.cmake)
include(cmake/unittests.cmake)
include(cmake/install.cmake)
message(STATUS "Repertory Version: ${REPERTORY_VERSION}")
message(STATUS "Git Revision: ${REPERTORY_GIT_REV}")