This commit is contained in:
Scott E. Graves 2023-11-30 09:18:10 -06:00
parent 8436f2e2bb
commit 97487cf0c4
380 changed files with 144184 additions and 144135 deletions

0
3rd_party/pugixml/scripts/cocoapods_push.sh vendored Executable file → Normal file
View File

0
3rd_party/pugixml/tests/fuzz_setup.sh vendored Executable file → Normal file
View File

View File

@ -50,7 +50,7 @@ if (LINUX OR MINGW)
endif()
set(ENV{PKG_CONFIG_PATH}
"${EXTERNAL_BUILD_ROOT}/lib/pkgconfig:$ENV{PKG_CONFIG_PATH}"
"${EXTERNAL_BUILD_ROOT}/lib/pkgconfig:${EXTERNAL_BUILD_ROOT}/share/pkgconfig:$ENV{PKG_CONFIG_PATH}"
)
find_package(PkgConfig REQUIRED)

View File

@ -39,10 +39,12 @@ set (BOOST_COMMON_ARGS
)
if (MINGW)
set(BOOST_COMMON_ARGS
${BOOST_COMMON_ARGS}
--user-config=./user-config.jam
)
if (NOT WIN32)
set(BOOST_COMMON_ARGS
${BOOST_COMMON_ARGS}
--user-config=./user-config.jam
)
endif()
set(BOOST_TARGET_OS target-os=windows)
endif()
@ -75,7 +77,9 @@ if (MINGW)
set(BOOST_GCC_VERSION ${CMAKE_CXX_COMPILER_VERSION})
string(REPLACE "." ";" BOOST_GCC_VERSION_LIST ${BOOST_GCC_VERSION})
list(GET BOOST_GCC_VERSION_LIST 0 BOOST_GCC_MAJOR_VERSION)
# set(BOOST_LIB_EXTRA "-mgw${BOOST_GCC_MAJOR_VERSION}-mt-x64-${BOOST_MAJOR_VERSION}_${BOOST_MINOR_VERSION}")
if(WIN32)
set(BOOST_LIB_EXTRA "-mgw${BOOST_GCC_MAJOR_VERSION}-mt${DEBUG_EXTRA2}-x64-${BOOST_MAJOR_VERSION}_${BOOST_MINOR_VERSION}")
endif()
endif()
set(Boost_LIBRARIES
@ -91,3 +95,6 @@ set(Boost_LIBRARIES
)
add_dependencies(boost_project zlib_project)
if (WIN32)
include_directories(SYSTEM ${EXTERNAL_BUILD_ROOT}/include/boost-${BOOST_MAJOR_VERSION}_${BOOST_MINOR_VERSION})
endif()

View File

@ -38,18 +38,10 @@ if (CMAKE_TOOLCHAIN_FILE)
)
endif()
if (MINGW)
set(CURL_CMAKE_ARGS
${CURL_CMAKE_ARGS}
-DCURL_USE_OPENSSL=OFF
-DUSE_WIN32_CRYPTO=ON
)
else()
set(CURL_CMAKE_ARGS
${CURL_CMAKE_ARGS}
-DCURL_USE_OPENSSL=ON
)
endif()
set(CURL_CMAKE_ARGS
${CURL_CMAKE_ARGS}
-DCURL_USE_OPENSSL=ON
)
ExternalProject_Add(curl_project
DOWNLOAD_NO_PROGRESS 1

View File

@ -18,23 +18,21 @@ if (UNIX OR MINGW)
)
endif()
if (CMAKE_BUILD_TYPE STREQUAL "Debug")
set(REPERTORY_COMMON_FLAG_LIST
${REPERTORY_COMMON_FLAG_LIST}
-DDEBUG
-D_DEBUG
-Og
-fno-omit-frame-pointer
-g
-gdwarf-4
)
else()
set(REPERTORY_COMMON_FLAG_LIST
${REPERTORY_COMMON_FLAG_LIST}
-O3
-DNDEBUG
)
endif()
set(REPERTORY_COMMON_FLAG_LIST_DEBUG
${REPERTORY_COMMON_FLAG_LIST_DEBUG}
-DDEBUG
-D_DEBUG
-Og
-fno-omit-frame-pointer
-g
-gdwarf-4
)
set(REPERTORY_COMMON_FLAG_LIST_RELEASE
${REPERTORY_COMMON_FLAG_LIST_RELEASE}
-O3
-DNDEBUG
)
if (NOT IS_CLANG_COMPILER)
set(REPERTORY_GCC_FLAGS
@ -75,12 +73,30 @@ if (UNIX OR MINGW)
)
endif()
list(JOIN REPERTORY_COMMON_FLAG_LIST_DEBUG " " REPERTORY_COMMON_FLAG_LIST_DEBUG)
list(JOIN REPERTORY_COMMON_FLAG_LIST_RELEASE " " REPERTORY_COMMON_FLAG_LIST_RELEASE)
list(JOIN REPERTORY_CXX_FLAGS_LIST " " REPERTORY_CXX_FLAGS_LIST)
list(JOIN REPERTORY_C_FLAGS_LIST " " REPERTORY_C_FLAGS_LIST)
list(JOIN REPERTORY_SHARED_LINKER_FLAGS_LIST " " REPERTORY_SHARED_LINKER_FLAGS_LIST)
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${REPERTORY_C_FLAGS_LIST}")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${REPERTORY_CXX_FLAGS_LIST}")
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${REPERTORY_C_FLAGS_LIST}")
if (CMAKE_BUILD_TYPE STREQUAL "Debug")
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${REPERTORY_COMMON_FLAG_LIST_DEBUG}")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${REPERTORY_COMMON_FLAG_LIST_DEBUG}")
else()
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${REPERTORY_COMMON_FLAG_LIST_RELEASE}")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${REPERTORY_COMMON_FLAG_LIST_RELEASE}")
endif()
set (REPERTORY_CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS} ${REPERTORY_COMMON_FLAG_LIST_DEBUG}")
set (REPERTORY_CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS} ${REPERTORY_COMMON_FLAG_LIST_DEBUG}")
set (REPERTORY_CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS} ${REPERTORY_COMMON_FLAG_LIST_RELEASE}")
set (REPERTORY_CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS} ${REPERTORY_COMMON_FLAG_LIST_RELEASE}")
set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${REPERTORY_SHARED_LINKER_FLAGS_LIST}")
if (ALPINE_FOUND OR MINGW)
set(CMAKE_EXE_LINKER_FLAGS "-static-libgcc -static-libstdc++ -static")

View File

@ -1,14 +1,17 @@
set(ROCKSDB_PROJECT_NAME rocksdb_${ROCKSDB_VERSION})
set(ROCKSDB_BUILD_ROOT ${EXTERNAL_BUILD_ROOT}/builds/${ROCKSDB_PROJECT_NAME})
set(ROCKSDB_CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
if (MACOS)
set(ROCKSDB_CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=hidden")
else()
set(ROCKSDB_CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
set(ROCKSDB_CMAKE_CXX_FLAGS "${ROCKSDB_CMAKE_CXX_FLAGS} -fvisibility=hidden")
endif()
if (CMAKE_BUILD_TYPE STREQUAL "Debug")
set(ROCKSDB_CMAKE_CXX_FLAGS "${ROCKSDB_CMAKE_CXX_FLAGS} -DXXH_NO_INLINE_HINTS")
endif()
set(ROCKSDB_CMAKE_ARGS
-DCMAKE_BUILD_TYPE=${EXTERNAL_BUILD_TYPE}
-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
-DCMAKE_CXX_FLAGS=${ROCKSDB_CMAKE_CXX_FLAGS}
-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
@ -21,6 +24,7 @@ set(ROCKSDB_CMAKE_ARGS
-DPORTABLE=1
-DROCKSDB_BUILD_SHARED=OFF
-DWITH_BENCHMARK_TOOLS=OFF
-DWITH_GFLAGS=OFF
-DWITH_LIBURING=OFF
-DWITH_TESTS=OFF
-DWITH_TOOLS=OFF
@ -38,7 +42,7 @@ if (MINGW)
DOWNLOAD_NO_PROGRESS 1
URL https://github.com/facebook/rocksdb/archive/v${ROCKSDB_VERSION}.tar.gz
PREFIX ${ROCKSDB_BUILD_ROOT}
CMAKE_ARGS ${ROCKSDB_CMAKE_ARGS} -DWITH_GFLAGS=OFF
CMAKE_ARGS ${ROCKSDB_CMAKE_ARGS}
INSTALL_COMMAND ${CMAKE_COMMAND} -E echo "Skipping install step."
)
@ -49,7 +53,7 @@ else()
DOWNLOAD_NO_PROGRESS 1
URL https://github.com/facebook/rocksdb/archive/v${ROCKSDB_VERSION}.tar.gz
PREFIX ${ROCKSDB_BUILD_ROOT}
CMAKE_ARGS ${ROCKSDB_CMAKE_ARGS} -DWITH_GFLAGS=OFF
CMAKE_ARGS ${ROCKSDB_CMAKE_ARGS}
)
if (MACOS)

View File

@ -5,13 +5,13 @@ WORKDIR /mnt
ENV MINGW=/mingw
ARG PKG_CONFIG_VERSION=0.29.2
ARG CMAKE_VERSION=3.27.1
ARG BINUTILS_VERSION=2.41
ARG MINGW_VERSION=11.0.1
ARG CMAKE_VERSION=3.27.1
ARG GCC_VERSION=13.2.0
ARG MINGW_VERSION=11.0.1
ARG NASM_VERSION=2.16.01
ARG NVCC_VERSION=12.2.1
ARG PKG_CONFIG_VERSION=0.29.2
SHELL [ "/bin/bash", "-c" ]
@ -49,14 +49,18 @@ RUN set -ex \
yasm \
wget \
zip \
git \
git
RUN set -ex \
\
&& wget -q https://pkg-config.freedesktop.org/releases/pkg-config-${PKG_CONFIG_VERSION}.tar.gz -O - | tar -xz \
&& wget -q https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}.tar.gz -O - | tar -xz \
&& wget -q https://ftp.gnu.org/gnu/binutils/binutils-${BINUTILS_VERSION}.tar.xz -O - | tar -xJ \
&& wget -q https://sourceforge.net/projects/mingw-w64/files/mingw-w64/mingw-w64-release/mingw-w64-v${MINGW_VERSION}.tar.bz2 -O - | tar -xj \
&& wget -q https://ftp.gnu.org/gnu/gcc/gcc-${GCC_VERSION}/gcc-${GCC_VERSION}.tar.xz -O - | tar -xJ \
&& wget -q https://www.nasm.us/pub/nasm/releasebuilds/${NASM_VERSION}/nasm-${NASM_VERSION}.tar.xz -O - | tar -xJ \
&& wget -q https://www.nasm.us/pub/nasm/releasebuilds/${NASM_VERSION}/nasm-${NASM_VERSION}.tar.xz -O - | tar -xJ
RUN set -ex \
\
&& mkdir -p ${MINGW}/include ${MINGW}/lib/pkgconfig \
&& chmod 0777 -R /mnt ${MINGW} \
@ -69,16 +73,18 @@ RUN set -ex \
--disable-shared \
--disable-nls \
&& make -j`nproc` \
&& make install \
&& cd .. \
&& make install
RUN set -ex \
\
&& cd cmake-${CMAKE_VERSION} \
&& ./configure \
--prefix=/usr/local \
--parallel=`nproc` \
&& make -j`nproc` \
&& make install \
&& cd .. \
&& make install
RUN set -ex \
\
&& cd binutils-${BINUTILS_VERSION} \
&& ./configure \
@ -93,8 +99,9 @@ RUN set -ex \
--disable-werror \
--with-system-zlib \
&& make -j`nproc` \
&& make install \
&& cd .. \
&& make install
RUN set -ex \
\
&& mkdir mingw-w64 \
&& cd mingw-w64 \
@ -102,8 +109,9 @@ RUN set -ex \
--prefix=/usr/local/x86_64-w64-mingw32 \
--host=x86_64-w64-mingw32 \
--enable-sdk=all \
&& make install \
&& cd .. \
&& make install
RUN set -ex \
\
&& mkdir gcc \
&& cd gcc \
@ -125,8 +133,9 @@ RUN set -ex \
--disable-nls \
--disable-werror \
&& make -j`nproc` all-gcc \
&& make install-gcc \
&& cd .. \
&& make install-gcc
RUN set -ex \
\
&& cd mingw-w64 \
&& ../mingw-w64-v${MINGW_VERSION}/mingw-w64-crt/configure \
@ -136,8 +145,9 @@ RUN set -ex \
--disable-lib32 \
--enable-lib64 \
&& (make || make || make || make) \
&& make install \
&& cd .. \
&& make install
RUN set -ex \
\
&& cd mingw-w64 \
&& ../mingw-w64-v${MINGW_VERSION}/mingw-w64-libraries/winpthreads/configure \
@ -146,38 +156,45 @@ RUN set -ex \
--enable-static \
--disable-shared \
&& make -j`nproc` \
&& make install \
&& cd .. \
&& make install
RUN set -ex \
\
&& cd gcc \
&& make -j`nproc` \
&& make install \
&& cd .. \
&& make install
RUN set -ex \
\
&& cd nasm-${NASM_VERSION} \
&& ./configure --prefix=/usr/local \
&& make -j`nproc` \
&& make install \
&& cd .. \
&& make install
RUN set -ex \
\
&& rm -r pkg-config-${PKG_CONFIG_VERSION} \
&& rm -r cmake-${CMAKE_VERSION} \
&& rm -r binutils-${BINUTILS_VERSION} \
&& rm -r mingw-w64 mingw-w64-v${MINGW_VERSION} \
&& rm -r gcc gcc-${GCC_VERSION} \
&& rm -r nasm-${NASM_VERSION} \
&& rm -r nasm-${NASM_VERSION}
RUN set -ex \
\
&& apt-get remove --purge -y file gcc g++ zlib1g-dev libssl-dev libgmp-dev libmpfr-dev libmpc-dev libisl-dev \
\
&& apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/3bf863cc.pub \
&& echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/ /" > /etc/apt/sources.list.d/cuda.list \
&& apt-get update \
\
&& DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
cuda-nvcc-${NVCC_VERSION:0:2}-${NVCC_VERSION:3:1} \
cuda-nvcc-${NVCC_VERSION:0:2}-${NVCC_VERSION:3:1}
RUN set -ex \
\
&& ln -s /usr/bin/gcc /usr/local/cuda/bin/gcc \
&& ln -s /usr/bin/g++ /usr/local/cuda/bin/g++ \
&& ln -s /usr/bin/g++ /usr/local/cuda/bin/g++
RUN set -ex \
\
&& apt-get remove --purge -y gnupg \
&& apt-get autoremove --purge -y \

View File

@ -180,6 +180,10 @@ public:
return fsi_.source_path;
}
[[nodiscard]] auto has_handle(std::uint64_t handle) const -> bool override {
return open_data_.find(handle) != open_data_.end();
}
[[nodiscard]] auto is_directory() const -> bool override {
return fsi_.directory;
}
@ -453,8 +457,6 @@ private:
mutable std::recursive_mutex open_file_mtx_;
std::unordered_map<std::string, std::shared_ptr<i_closeable_open_file>>
open_file_lookup_;
std::unordered_map<std::uint64_t, i_closeable_open_file *>
open_handle_lookup_;
stop_type stop_requested_ = false;
rocksdb::ColumnFamilyHandle *upload_family_{};
rocksdb::ColumnFamilyHandle *upload_active_family_{};
@ -466,6 +468,9 @@ private:
private:
void close_timed_out_files();
auto get_open_file_by_handle(std::uint64_t handle) const
-> std::shared_ptr<i_closeable_open_file>;
auto get_open_file_count(const std::string &api_path) const -> std::size_t;
auto open(const std::string &api_path, bool directory,

View File

@ -55,6 +55,8 @@ public:
[[nodiscard]] virtual auto is_directory() const -> bool = 0;
[[nodiscard]] virtual auto has_handle(std::uint64_t handle) const -> bool = 0;
[[nodiscard]] virtual auto
native_operation(const native_operation_callback &callback) -> api_error = 0;

0
scripts/create_containers.sh Executable file → Normal file
View File

0
scripts/make_mingw64.sh Executable file → Normal file
View File

0
scripts/make_package.sh Executable file → Normal file
View File

0
scripts/make_unix.sh Executable file → Normal file
View File

0
scripts/make_unix_docker.sh Executable file → Normal file
View File

0
scripts/run_docker_shell.sh Executable file → Normal file
View File

0
scripts/setup_msys2.sh Executable file → Normal file
View File

View File

@ -87,17 +87,12 @@ file_manager::~file_manager() {
void file_manager::close(std::uint64_t handle) {
unique_recur_mutex_lock file_lock(open_file_mtx_);
auto iter = open_handle_lookup_.find(handle);
if (iter != open_handle_lookup_.end()) {
auto *cur_file = iter->second;
open_handle_lookup_.erase(handle);
cur_file->remove(handle);
auto closeable_file = get_open_file_by_handle(handle);
if (closeable_file) {
closeable_file->remove(handle);
if (cur_file->can_close()) {
const auto api_path = cur_file->get_api_path();
cur_file = nullptr;
auto closeable_file = open_file_lookup_.at(api_path);
if (closeable_file->can_close()) {
auto api_path = closeable_file->get_api_path();
open_file_lookup_.erase(api_path);
file_lock.unlock();
@ -118,12 +113,13 @@ void file_manager::close_all(const std::string &api_path) {
for (auto &handle : handles) {
open_file_lookup_[api_path]->remove(handle);
open_handle_lookup_.erase(handle);
}
auto file = open_file_lookup_.at(api_path);
open_file_lookup_.erase(api_path);
file_lock.unlock();
file->close();
}
void file_manager::close_timed_out_files() {
@ -231,9 +227,17 @@ auto file_manager::get_next_handle() -> std::uint64_t {
return next_handle_;
}
auto file_manager::get_open_file_by_handle(std::uint64_t handle) const
-> std::shared_ptr<i_closeable_open_file> {
auto iter = std::find_if(open_file_lookup_.begin(), open_file_lookup_.end(),
[&handle](const auto &item) -> bool {
return item.second->has_handle(handle);
});
return iter == open_file_lookup_.end() ? nullptr : iter->second;
}
auto file_manager::get_open_file_count(const std::string &api_path) const
-> std::size_t {
recur_mutex_lock open_lock(open_file_mtx_);
auto iter = open_file_lookup_.find(api_path);
if (iter != open_file_lookup_.end()) {
return iter->second->get_open_file_count();
@ -245,26 +249,25 @@ auto file_manager::get_open_file_count(const std::string &api_path) const
auto file_manager::get_open_file(std::uint64_t handle, bool write_supported,
std::shared_ptr<i_open_file> &file) -> bool {
recur_mutex_lock open_lock(open_file_mtx_);
auto iter = open_handle_lookup_.find(handle);
if (iter == open_handle_lookup_.end()) {
auto file_ptr = get_open_file_by_handle(handle);
if (not file_ptr) {
return false;
}
auto current_file = open_file_lookup_.at(iter->second->get_api_path());
if (write_supported && not current_file->is_write_supported()) {
if (write_supported && not file_ptr->is_write_supported()) {
auto writeable_file = std::make_shared<open_file>(
utils::encryption::encrypting_reader::get_data_chunk_size(),
config_.get_enable_chunk_download_timeout()
? config_.get_chunk_downloader_timeout_secs()
: 0U,
current_file->get_filesystem_item(), current_file->get_open_data(),
provider_, *this);
open_file_lookup_[current_file->get_api_path()] = writeable_file;
file_ptr->get_filesystem_item(), file_ptr->get_open_data(), provider_,
*this);
open_file_lookup_[file_ptr->get_api_path()] = writeable_file;
file = writeable_file;
return true;
}
file = current_file;
file = file_ptr;
return true;
}
@ -287,7 +290,11 @@ auto file_manager::get_open_files() const
auto file_manager::get_open_handle_count() const -> std::size_t {
recur_mutex_lock open_lock(open_file_mtx_);
return open_handle_lookup_.size();
return std::accumulate(
open_file_lookup_.begin(), open_file_lookup_.end(), std::size_t(0U),
[](std::size_t count, const auto &item) -> std::size_t {
return count + item.second->get_open_file_count();
});
}
auto file_manager::get_stored_downloads() const -> std::vector<json> {
@ -315,8 +322,7 @@ auto file_manager::handle_file_rename(const std::string &from_api_path,
}
auto file_manager::has_no_open_file_handles() const -> bool {
recur_mutex_lock open_lock(open_file_mtx_);
return open_handle_lookup_.empty();
return get_open_handle_count() == 0U;
}
auto file_manager::is_processing(const std::string &api_path) const -> bool {
@ -353,6 +359,7 @@ auto file_manager::is_processing(const std::string &api_path) const -> bool {
auto file_manager::open(const std::string &api_path, bool directory,
const open_file_data &ofd, std::uint64_t &handle,
std::shared_ptr<i_open_file> &file) -> api_error {
recur_mutex_lock open_lock(open_file_mtx_);
return open(api_path, directory, ofd, handle, file, nullptr);
}
@ -365,11 +372,9 @@ auto file_manager::open(const std::string &api_path, bool directory,
[&](std::shared_ptr<i_closeable_open_file> cur_file) {
handle = get_next_handle();
cur_file->add(handle, ofd);
open_handle_lookup_[handle] = cur_file.get();
file = cur_file;
};
recur_mutex_lock open_lock(open_file_mtx_);
auto iter = open_file_lookup_.find(api_path);
if (iter != open_file_lookup_.end()) {
create_and_add_handle(iter->second);
@ -766,13 +771,13 @@ void file_manager::start() {
std::uint64_t file_size{};
if (utils::file::get_file_size(fsi.source_path, file_size)) {
if (file_size == fsi.size) {
auto file = std::make_shared<open_file>(
auto closeable_file = std::make_shared<open_file>(
chunk_size,
config_.get_enable_chunk_download_timeout()
? config_.get_chunk_downloader_timeout_secs()
: 0U,
fsi, provider_, read_state, *this);
open_file_lookup_[api_path] = file;
open_file_lookup_[api_path] = closeable_file;
event_system::instance().raise<download_restored>(
fsi.api_path, fsi.source_path);
} else {
@ -825,7 +830,6 @@ void file_manager::stop() {
}
open_file_lookup_.clear();
open_handle_lookup_.clear();
upload_lock.lock();
for (auto &item : upload_lookup_) {

View File

@ -48,6 +48,8 @@ public:
MOCK_METHOD(std::string, get_source_path, (), (const, override));
MOCK_METHOD(bool, has_handle, (std::uint64_t handle), (const, override));
MOCK_METHOD(bool, is_directory, (), (const, override));
MOCK_METHOD(api_error, native_operation,