new_build_system (#18)
All checks were successful
BlockStorage/repertory/pipeline/head This commit looks good

Reviewed-on: #18
This commit is contained in:
2024-09-06 15:05:48 +00:00
parent 9d3e4b8767
commit a7239558bd
191 changed files with 10683 additions and 10598 deletions

View File

@@ -21,227 +21,309 @@
*/
#include "utils/file.hpp"
#include "utils/common.hpp"
#include "utils/encryption.hpp"
#include "utils/error.hpp"
#include "utils/path.hpp"
#include "utils/string.hpp"
#include "utils/time.hpp"
#include "utils/unix.hpp"
#include "utils/windows.hpp"
namespace repertory::utils::file {
auto file::open_file(std::filesystem::path path) -> file {
path = utils::path::absolute(path.string());
if (not is_file(path.string())) {
throw std::runtime_error("file not found: " + path.string());
}
auto stream = std::fstream{
path,
std::ios_base::binary | std::ios_base::in | std::ios_base::out,
};
return {
std::move(stream),
path,
};
}
auto file::open_or_create_file(std::filesystem::path path) -> file {
path = utils::path::absolute(path.string());
auto stream = std::fstream{
path.string().c_str(),
std::ios_base::binary | std::ios_base::trunc | std::ios_base::in |
std::ios_base::out,
};
return {
std::move(stream),
path,
};
}
void file::close() {
if (stream_.is_open()) {
stream_.close();
}
}
auto file::move_to(std::filesystem::path new_path) -> bool {
new_path = utils::path::absolute(new_path.string());
auto reopen{false};
if (stream_.is_open()) {
reopen = true;
close();
}
std::filesystem::rename(path_, new_path, error_);
if (not error_) {
path_ = new_path;
if (reopen) {
*this = open_file(path_);
}
return true;
}
if (reopen) {
*this = open_file(path_);
}
return false;
}
auto file::read_(unsigned char *data, std::size_t to_read, std::uint64_t offset,
std::size_t *total_read) -> bool {
auto change_to_process_directory() -> bool {
static constexpr const std::string_view function_name{
static_cast<const char *>(__FUNCTION__),
};
if (total_read != nullptr) {
(*total_read) = 0U;
try {
#if defined(_WIN32)
std::string file_name;
file_name.resize(MAX_PATH + 1U);
::GetModuleFileNameA(nullptr, file_name.data(),
static_cast<DWORD>(file_name.size() - 1U));
auto path = utils::path::strip_to_file_name(file_name.c_str());
::SetCurrentDirectoryA(path.c_str());
#else // !defined(_WIN32)
std::string path;
path.resize(PATH_MAX + 1);
#if defined(__APPLE__)
proc_pidpath(getpid(), path.c_str(), path.size());
#else // !defined(__APPLE__)
auto res = readlink("/proc/self/exe", path.data(), path.size());
if (res == -1) {
throw std::runtime_error("failed to readlink|" + path + '|' +
std::to_string(errno));
}
#endif // defined(__APPLE__)
path = utils::path::get_parent_path(path);
res = chdir(path.c_str());
if (res != 0) {
throw std::runtime_error("failed to chdir|" + path + '|' +
std::to_string(errno));
}
#endif // defined(_WIN32)
return true;
} catch (const std::exception &e) {
utils::error::handle_exception(function_name, e);
} catch (...) {
utils::error::handle_exception(function_name);
}
try {
stream_.seekg(static_cast<std::streamoff>(offset));
return false;
}
auto before = stream_.tellg();
if (before == -1) {
throw std::runtime_error("failed to tellg() before read");
auto create_temp_name(std::string_view file_part) -> std::string {
std::array<std::uint8_t, 8U> data{
utils::generate_random_between<std::uint8_t>(0U, 9U),
utils::generate_random_between<std::uint8_t>(0U, 9U),
utils::generate_random_between<std::uint8_t>(0U, 9U),
utils::generate_random_between<std::uint8_t>(0U, 9U),
utils::generate_random_between<std::uint8_t>(0U, 9U),
utils::generate_random_between<std::uint8_t>(0U, 9U),
utils::generate_random_between<std::uint8_t>(0U, 9U),
utils::generate_random_between<std::uint8_t>(0U, 9U),
};
return std::accumulate(data.begin(), data.end(), std::string{file_part} + '_',
[](auto &&name, auto &&val) -> auto {
return name + std::to_string(val);
});
}
auto create_temp_name(std::wstring_view file_part) -> std::wstring {
return utils::string::from_utf8(
create_temp_name(utils::string::to_utf8(file_part)));
}
auto get_free_drive_space(std::string_view path)
-> std::optional<std::uint64_t> {
static constexpr const std::string_view function_name{
static_cast<const char *>(__FUNCTION__),
};
try {
#if defined(_WIN32)
ULARGE_INTEGER li{};
if (not ::GetDiskFreeSpaceEx(std::string{path}.c_str(), &li, nullptr,
nullptr)) {
throw std::runtime_error("failed to get free disk space|" +
std::string{path} + '|' +
std::to_string(utils::get_last_error_code()));
}
stream_.read(reinterpret_cast<char *>(data),
static_cast<std::streamoff>(to_read));
return li.QuadPart;
#endif // defined(_WIN32)
#if defined(__linux__)
struct statfs64 st {};
if (statfs64(std::string{path}.c_str(), &st) != 0) {
throw std::runtime_error("failed to get free disk space|" +
std::string{path} + '|' +
std::to_string(utils::get_last_error_code()));
}
return st.f_bfree * static_cast<std::uint64_t>(st.f_bsize);
#endif // defined(__linux__)
#if defined(__APPLE__)
struct statvfs st {};
if (statvfs(path.c_str(), &st) != 0) {
throw std::runtime_error("failed to get free disk space|" +
std::string{path} + '|' +
std::to_string(utils::get_last_error_code()));
}
return st.f_bfree * static_cast<std::uint64_t>(st.f_frsize);
#endif // defined(__APPLE__)
} catch (const std::exception &e) {
utils::error::handle_exception(function_name, e);
} catch (...) {
utils::error::handle_exception(function_name);
}
return std::nullopt;
}
auto get_free_drive_space(std::wstring_view path)
-> std::optional<std::uint64_t> {
return get_free_drive_space(utils::string::to_utf8(path));
}
auto get_time(std::string_view path,
time_type type) -> std::optional<std::uint64_t> {
auto times = get_times(path);
if (times.has_value()) {
return times->get(type);
}
return std::nullopt;
}
auto get_time(std::wstring_view path,
time_type type) -> std::optional<std::uint64_t> {
return get_time(utils::string::to_utf8(path), type);
}
auto get_times(std::string_view path) -> std::optional<file_times> {
static constexpr const std::string_view function_name{
static_cast<const char *>(__FUNCTION__),
};
try {
file_times ret{};
#if defined(_WIN32)
auto file_handle =
::CreateFileA(std::string{path}.c_str(), GENERIC_READ,
FILE_SHARE_DELETE | FILE_SHARE_READ | FILE_SHARE_WRITE,
nullptr, OPEN_EXISTING, 0U, nullptr);
if (file_handle == INVALID_HANDLE_VALUE) {
throw std::runtime_error("failed to get file times|" + std::string{path} +
'|' +
std::to_string(utils::get_last_error_code()));
}
std::array<FILETIME, 3U> times{};
auto res =
::GetFileTime(file_handle, &times.at(0U), &times.at(1U), &times.at(2U));
::CloseHandle(file_handle);
if (not res) {
throw std::runtime_error("failed to get file times|" + std::string{path} +
'|' +
std::to_string(utils::get_last_error_code()));
}
ret.accessed = utils::time::windows_file_time_to_unix_time(times.at(1U));
ret.created = utils::time::windows_file_time_to_unix_time(times.at(0U));
ret.modified = utils::time::windows_file_time_to_unix_time(times.at(2U));
ret.written = utils::time::windows_file_time_to_unix_time(times.at(2U));
#else // !defined(_WIN32)
struct stat64 st {};
if (stat64(std::string{path}.c_str(), &st) != 0) {
throw std::runtime_error("failed to get file times|" + std::string{path} +
'|' + std::to_string(errno));
}
ret.accessed = static_cast<std::uint64_t>(st.st_atim.tv_nsec) +
static_cast<std::uint64_t>(st.st_atim.tv_sec) *
utils::time::NANOS_PER_SECOND;
ret.created = static_cast<std::uint64_t>(st.st_ctim.tv_nsec) +
static_cast<std::uint64_t>(st.st_ctim.tv_sec) *
utils::time::NANOS_PER_SECOND;
ret.modified = static_cast<std::uint64_t>(st.st_mtim.tv_nsec) +
static_cast<std::uint64_t>(st.st_mtim.tv_sec) *
utils::time::NANOS_PER_SECOND;
ret.written = static_cast<std::uint64_t>(st.st_mtim.tv_nsec) +
static_cast<std::uint64_t>(st.st_mtim.tv_sec) *
utils::time::NANOS_PER_SECOND;
#endif // defined(_WIN32)
return ret;
} catch (const std::exception &e) {
utils::error::handle_exception(function_name, e);
} catch (...) {
utils::error::handle_exception(function_name);
}
return std::nullopt;
}
auto get_times(std::wstring_view path) -> std::optional<file_times> {
return get_times(utils::string::to_utf8(path));
}
auto get_total_drive_space(std::string_view path)
-> std::optional<std::uint64_t> {
static constexpr const std::string_view function_name{
static_cast<const char *>(__FUNCTION__),
};
try {
#if defined(_WIN32)
ULARGE_INTEGER li{};
if (not ::GetDiskFreeSpaceEx(std::string{path}.c_str(), nullptr, &li,
nullptr)) {
throw std::runtime_error("failed to get total disk space|" +
std::string{path} + '|' +
std::to_string(utils::get_last_error_code()));
}
return li.QuadPart;
#endif // defined(_WIN32)
#if defined(__linux__)
struct statfs64 st {};
if (statfs64(std::string{path}.c_str(), &st) != 0) {
throw std::runtime_error("failed to get total disk space|" +
std::string{path} + '|' +
std::to_string(utils::get_last_error_code()));
}
return st.f_blocks * static_cast<std::uint64_t>(st.f_bsize);
#endif // defined(__linux__)
#if defined(__APPLE__)
struct statvfs st {};
if (statvfs(path.c_str(), &st) != 0) {
throw std::runtime_error("failed to get total disk space|" +
std::string{path} + '|' +
std::to_string(utils::get_last_error_code()));
}
return st.f_blocks * static_cast<std::uint64_t>(st.f_frsize);
#endif // defined(__APPLE__)
} catch (const std::exception &e) {
utils::error::handle_exception(function_name, e);
} catch (...) {
utils::error::handle_exception(function_name);
}
return std::nullopt;
}
auto get_total_drive_space(std::wstring_view path)
-> std::optional<std::uint64_t> {
return get_total_drive_space(utils::string::to_utf8(path));
}
auto i_fs_item::get_time(time_type type) const -> std::optional<std::uint64_t> {
return utils::file::get_time(get_path(), type);
}
auto i_file::read_all(data_buffer &data, std::uint64_t offset,
std::size_t *total_read) -> bool {
data_buffer buffer;
buffer.resize(get_read_buffer_size());
std::size_t current_read{};
while (read(reinterpret_cast<unsigned char *>(buffer.data()),
buffer.size() * sizeof(data_buffer::value_type), offset,
&current_read)) {
if (total_read != nullptr) {
auto after = stream_.tellg();
if (after >= 0) {
(*total_read) = static_cast<std::size_t>(after - before);
} else if (after == -1 && not stream_.eof()) {
throw std::runtime_error("failed to tellg() after read");
}
*total_read += current_read;
}
if (current_read != 0U) {
offset += current_read;
data.insert(
data.end(), buffer.begin(),
std::next(buffer.begin(),
static_cast<std::int64_t>(
current_read / sizeof(data_buffer::value_type))));
continue;
}
return true;
} catch (const std::exception &e) {
utils::error::handle_exception(function_name, e);
} catch (...) {
utils::error::handle_exception(function_name);
}
return false;
}
auto file::remove() -> bool {
close();
return std::filesystem::remove(path_, error_);
}
auto file::truncate(std::size_t size) -> bool {
auto reopen{false};
if (stream_.is_open()) {
reopen = true;
close();
}
std::filesystem::resize_file(path_, size, error_);
if (reopen) {
*this = open_file(path_);
}
return not error_;
}
auto file::write_(const unsigned char *data, std::size_t to_write,
std::size_t offset, std::size_t *total_written) -> bool {
static constexpr const std::string_view function_name{
static_cast<const char *>(__FUNCTION__),
};
if (total_written != nullptr) {
(*total_written) = 0U;
}
try {
stream_.seekp(static_cast<std::streamoff>(offset));
auto before = stream_.tellp();
if (before == -1) {
throw std::runtime_error("failed to tellp() before write");
}
stream_.write(reinterpret_cast<const char *>(data),
static_cast<std::streamoff>(to_write));
auto after = stream_.tellp();
if (after == -1) {
throw std::runtime_error("failed to tellp() after write");
}
if (total_written != nullptr) {
(*total_written) = static_cast<std::size_t>(after - before);
}
return true;
} catch (const std::exception &e) {
utils::error::handle_exception(function_name, e);
} catch (...) {
utils::error::handle_exception(function_name);
}
return false;
}
auto get_file_size(std::string_view path, std::uint64_t &file_size) -> bool {
auto abs_path = utils::path::absolute(path);
file_size = 0U;
#if defined(_WIN32)
struct _stat64 st {};
if (_stat64(abs_path.c_str(), &st) != 0) {
#else // !defined(_WIN32)
struct stat st {};
if (stat(abs_path.c_str(), &st) != 0) {
#endif // defined(_WIN32)
return false;
}
if (st.st_size >= 0) {
file_size = static_cast<std::uint64_t>(st.st_size);
}
return (st.st_size >= 0);
}
auto get_file_size(std::wstring_view path, std::uint64_t &file_size) -> bool {
return get_file_size(utils::string::to_utf8(path), file_size);
}
auto is_directory(std::string_view path) -> bool {
auto abs_path = utils::path::absolute(path);
#if defined(_WIN32)
return ::PathIsDirectoryA(abs_path.c_str()) != 0;
#else // !defined(_WIN32)
struct stat st {};
return (stat(abs_path.c_str(), &st) == 0 && S_ISDIR(st.st_mode));
#endif // defined(_WIN32)
}
auto is_directory(std::wstring_view path) -> bool {
return is_directory(utils::string::to_utf8(path));
}
auto is_file(std::string_view path) -> bool {
auto abs_path = utils::path::absolute(path);
#if defined(_WIN32)
return (::PathFileExistsA(abs_path.c_str()) &&
not ::PathIsDirectoryA(abs_path.c_str()));
#else // !defined(_WIN32)
struct stat st {};
return (stat(abs_path.c_str(), &st) == 0 && not S_ISDIR(st.st_mode));
#endif // defined(_WIN32)
}
auto is_file(std::wstring_view path) -> bool {
return is_file(utils::string::to_utf8(path));
}
#if defined(PROJECT_ENABLE_JSON)
#if defined(PROJECT_ENABLE_LIBSODIUM) && defined(PROJECT_ENABLE_BOOST)
auto read_json_file(std::string_view path, nlohmann::json &data,
@@ -255,44 +337,42 @@ auto read_json_file(std::string_view path, nlohmann::json &data) -> bool {
try {
auto abs_path = utils::path::absolute(path);
if (not is_file(abs_path)) {
return true;
auto file = file::open_file(abs_path);
if (not *file) {
return false;
}
std::ifstream file_stream{
abs_path.c_str(),
std::ios_base::binary | std::ios::in,
};
if (not file_stream.is_open()) {
throw std::runtime_error("failed to open file: " + abs_path);
}
auto ret{true};
try {
std::stringstream stream;
stream << file_stream.rdbuf();
data_buffer buffer{};
if (not file->read_all(buffer, 0U)) {
return false;
}
auto json_text = stream.str();
#if defined(PROJECT_ENABLE_LIBSODIUM) && defined(PROJECT_ENABLE_BOOST)
if (password.has_value()) {
auto decrypted_data =
utils::encryption::decrypt_data(*password, json_text);
json_text = {decrypted_data.begin(), decrypted_data.end()};
data_buffer decrypted_data{};
if (not utils::encryption::decrypt_data(*password, buffer,
decrypted_data)) {
return false;
}
buffer = decrypted_data;
}
#endif // defined(PROJECT_ENABLE_LIBSODIUM) && defined(PROJECT_ENABLE_BOOST)
if (not json_text.empty()) {
data = nlohmann::json::parse(json_text.c_str());
std::string json_str(buffer.begin(), buffer.end());
if (not json_str.empty()) {
data = nlohmann::json::parse(json_str);
}
} catch (const std::exception &e) {
utils::error::handle_exception(function_name, e);
ret = false;
return false;
} catch (...) {
utils::error::handle_exception(function_name);
ret = false;
return false;
}
file_stream.close();
return ret;
return true;
} catch (const std::exception &e) {
utils::error::handle_exception(function_name, e);
} catch (...) {
@@ -315,19 +395,26 @@ auto write_json_file(std::string_view path,
try {
auto file = file::open_or_create_file(path);
if (not file.truncate()) {
throw std::runtime_error("failed to truncate file: " +
file.get_error_code().message());
if (not file->truncate()) {
throw std::runtime_error("failed to truncate file");
}
#if defined(PROJECT_ENABLE_LIBSODIUM) && defined(PROJECT_ENABLE_BOOST)
if (password.has_value()) {
return file.write(utils::encryption::encrypt_data(*password, data.dump()),
0U);
const auto str_data = data.dump();
data_buffer encrypted_data{};
utils::encryption::encrypt_data(
*password, reinterpret_cast<const unsigned char *>(str_data.c_str()),
str_data.size(), encrypted_data);
return file->write(encrypted_data, 0U);
}
#endif // defined(PROJECT_ENABLE_LIBSODIUM) && defined(PROJECT_ENABLE_BOOST)
return file.write(data, 0U);
auto json_str = data.dump();
return file->write(
reinterpret_cast<const unsigned char *>(json_str.c_str()),
json_str.size(), 0U);
} catch (const std::exception &e) {
utils::error::handle_exception(function_name, e);
} catch (...) {
@@ -368,4 +455,170 @@ auto write_json_file(std::wstring_view path,
}
#endif // defined(PROJECT_ENABLE_LIBSODIUM) && defined(PROJECT_ENABLE_BOOST)
#endif // defined(PROJECT_ENABLE_JSON)
#if defined(PROJECT_ENABLE_LIBDSM)
static constexpr const auto validate_smb_path =
[](std::string_view path) -> bool {
return (not utils::string::begins_with(path, "///") &&
utils::string::begins_with(path, "//") &&
not utils::string::contains(path, " ") &&
std::count(path.begin(), path.end(), '/') >= 3U);
};
auto smb_create_smb_path(std::string_view smb_path,
std::string_view rel_path) -> std::string {
if (not validate_smb_path(smb_path)) {
throw std::runtime_error("invalid smb path|" + std::string{smb_path});
}
std::string path{rel_path};
utils::path::format_path(path, "/", "\\");
utils::string::left_trim(path, '/');
auto old_parts =
repertory::utils::string::split(smb_path.substr(2U), '/', false);
old_parts.erase(std::next(old_parts.begin(), 2U), old_parts.end());
auto new_parts = repertory::utils::string::split(path, '/', false);
old_parts.insert(old_parts.end(), new_parts.begin(), new_parts.end());
path = utils::string::join(old_parts, '/');
path = "//" + utils::path::format_path(path, "/", "\\");
if (not validate_smb_path(path)) {
throw std::runtime_error("invalid smb path|" + std::string{path});
}
return path;
}
auto smb_create_and_validate_relative_path(
std::string_view smb_path, std::string_view path) -> std::string {
if (not validate_smb_path(smb_path)) {
throw std::runtime_error("invalid smb path|" + std::string{smb_path});
}
std::string dir_path;
if (utils::string::begins_with(path, "//")) {
if (not utils::file::smb_parent_is_same(smb_path, path)) {
throw std::runtime_error("failed to validate path|" +
std::string{smb_path} + '|' + std::string{path} +
"|parent paths are not the same");
}
return utils::file::smb_create_relative_path(path);
}
return utils::file::smb_create_relative_path(std::string{smb_path} + '/' +
std::string{path});
}
auto smb_create_relative_path(std::string_view smb_path) -> std::string {
if (not validate_smb_path(smb_path)) {
throw std::runtime_error("invalid smb path|" + std::string{smb_path});
}
std::string path{smb_path};
utils::path::format_path(path, "\\", "/");
utils::string::left_trim(path, '\\');
auto parts = repertory::utils::string::split(path, '\\', false);
parts.erase(parts.begin(), std::next(parts.begin(), 2U));
return "\\" + utils::string::join(parts, '\\');
}
auto smb_create_search_path(std::string_view smb_path) -> std::string {
if (not validate_smb_path(smb_path)) {
throw std::runtime_error("invalid smb path|" + std::string{smb_path});
}
std::string path{smb_path};
utils::string::left_trim(path, '/');
auto parts = repertory::utils::string::split(path, '/', false);
parts.erase(parts.begin(), std::next(parts.begin(), 2U));
auto search_path = repertory::utils::string::join(parts, '\\');
return search_path.empty() ? "\\*" : "\\" + search_path + "\\*";
}
auto smb_get_parent_path(std::string_view smb_path) -> std::string {
if (not validate_smb_path(smb_path)) {
throw std::runtime_error("invalid smb path|" + std::string{smb_path});
}
auto parts = repertory::utils::string::split(smb_path.substr(2U), '/', false);
if (parts.size() > 2U) {
parts.erase(std::prev(parts.end()), parts.end());
}
auto parent_smb_path = "//" + utils::string::join(parts, '/');
if (not validate_smb_path(parent_smb_path)) {
throw std::runtime_error("invalid smb path|" + parent_smb_path);
}
return parent_smb_path;
}
auto smb_get_root_path(std::string_view smb_path) -> std::string {
if (not validate_smb_path(smb_path)) {
throw std::runtime_error("invalid smb path|" + std::string{smb_path});
}
auto parts = repertory::utils::string::split(smb_path.substr(2U), '/', false);
if (parts.size() > 2U) {
parts.erase(std::next(parts.begin(), 2U), parts.end());
}
return "//" + utils::string::join(parts, '/');
}
auto smb_get_unc_path(std::string_view smb_path) -> std::string {
if (not validate_smb_path(smb_path)) {
throw std::runtime_error("invalid smb path|" + std::string{smb_path});
}
std::string unc_path{smb_path};
utils::path::format_path(unc_path, "\\", "/");
return '\\' + unc_path;
}
auto smb_get_uri_path(std::string_view smb_path) -> std::string {
if (not validate_smb_path(smb_path)) {
throw std::runtime_error("invalid smb path|" + std::string{smb_path});
}
return "smb:" + std::string{smb_path};
}
auto smb_get_uri_path(std::string_view smb_path, std::string_view user,
std::string_view password) -> std::string {
if (not validate_smb_path(smb_path)) {
throw std::runtime_error("invalid smb path|" + std::string{smb_path});
}
return "smb://" + std::string{user} + ':' + std::string{password} + '@' +
std::string{smb_path.substr(2U)};
}
auto smb_parent_is_same(std::string_view smb_path1,
std::string_view smb_path2) -> bool {
if (not(validate_smb_path(smb_path1) && validate_smb_path(smb_path2))) {
return false;
}
auto parts1 = utils::string::split(smb_path1.substr(2U), "/", false);
auto parts2 = utils::string::split(smb_path2.substr(2U), "/", false);
if (parts1.size() < 2U || parts2.size() < 2U) {
return false;
}
if (parts2.at(1U).empty() || parts1.at(1U).empty()) {
return false;
}
return std::equal(parts1.begin(), std::next(parts1.begin(), 2U),
parts2.begin());
}
#endif // defined(PROJECT_ENABLE_LIBDSM)
} // namespace repertory::utils::file