updated build system

support/include/utils/config.hpp

@@ -468,7 +468,7 @@ struct http_range final {
   std::uint64_t end{};
 };
 
-using http_headers = std::unordered_map<std::string, std::string>;
+using http_headers = std::map<std::string, std::string>;
 using http_query_parameters = std::map<std::string, std::string>;
 using http_ranges = std::vector<http_range>;
 #endif // defined(PROJECT_ENABLE_CURL)

support/include/utils/encrypting_reader.hpp

@@ -87,16 +87,16 @@ private:
   auto reader_function(char *buffer, size_t size, size_t nitems) -> size_t;
 
 public:
-  [[nodiscard]] static auto
-  calculate_decrypted_size(std::uint64_t total_size) -> std::uint64_t;
+  [[nodiscard]] static auto calculate_decrypted_size(std::uint64_t total_size)
+      -> std::uint64_t;
 
   [[nodiscard]] static auto
   calculate_encrypted_size(std::string_view source_path) -> std::uint64_t;
 
   [[nodiscard]] auto create_iostream() const -> std::shared_ptr<iostream>;
 
-  [[nodiscard]] static constexpr auto
-  get_encrypted_chunk_size() -> std::size_t {
+  [[nodiscard]] static constexpr auto get_encrypted_chunk_size()
+      -> std::size_t {
     return encrypted_chunk_size_;
   }
 
@@ -116,9 +116,12 @@ public:
     return error_return_;
   }
 
-  [[nodiscard]] auto get_iv_list()
-      -> std::vector<std::array<unsigned char,
-                                crypto_aead_xchacha20poly1305_IETF_NPUBBYTES>> {
+  [[nodiscard]] static constexpr auto get_header_size() -> std::size_t {
+    return header_size_;
+  }
+
+  [[nodiscard]] auto get_iv_list() -> std::vector<
+      std::array<unsigned char, crypto_aead_xchacha20poly1305_IETF_NPUBBYTES>> {
     return iv_list_;
   }
 
@@ -131,8 +134,8 @@ public:
   }
 
   [[nodiscard]] static auto reader_function(char *buffer, size_t size,
-                                            size_t nitems,
-                                            void *instream) -> size_t {
+                                            size_t nitems, void *instream)
+      -> size_t {
     return reinterpret_cast<encrypting_reader *>(instream)->reader_function(
         buffer, size, nitems);
   }

support/include/utils/encryption.hpp

@@ -56,8 +56,8 @@ inline auto generate_key(
 template <typename result_t, typename arr_t, std::size_t arr_size>
 [[nodiscard]] inline auto decrypt_data(const std::array<arr_t, arr_size> &key,
                                        const unsigned char *buffer,
-                                       std::size_t buffer_size,
-                                       result_t &res) -> bool {
+                                       std::size_t buffer_size, result_t &res)
+    -> bool {
   if (buffer_size > encryption_header_size) {
     const std::uint32_t size =
         boost::endian::native_to_big(static_cast<std::uint32_t>(buffer_size));
@@ -76,8 +76,8 @@ template <typename result_t, typename arr_t, std::size_t arr_size>
 template <typename buffer_t, typename result_t, typename arr_t,
           std::size_t arr_size>
 [[nodiscard]] inline auto decrypt_data(const std::array<arr_t, arr_size> &key,
-                                       const buffer_t &buf,
-                                       result_t &res) -> bool {
+                                       const buffer_t &buf, result_t &res)
+    -> bool {
   return decrypt_data<result_t>(
       key, reinterpret_cast<const unsigned char *>(buf.data()), buf.size(),
       res);
@@ -195,6 +195,11 @@ read_encrypted_range(const http_range &range,
                      const utils::encryption::hash_256_t &key,
                      reader_func_t reader_func, std::uint64_t total_size,
                      data_buffer &data) -> bool;
+
+[[nodiscard]] auto read_encrypted_range(
+    const http_range &range, const utils::encryption::hash_256_t &key,
+    reader_func_t reader_func, std::uint64_t total_size, unsigned char *data,
+    std::size_t size, std::size_t &bytes_read) -> bool;
 #endif // defined(PROJECT_ENABLE_CURL)
 #endif // defined(PROJECT_ENABLE_BOOST)
 

support/include/utils/file_enc_file.hpp

@@ -45,12 +45,13 @@ public:
 
 private:
   fs_file_t file_;
+  std::string encryption_token_;
 
 public:
   void close() override;
 
-  [[nodiscard]] auto copy_to(std::string_view new_path,
-                             bool overwrite) const -> bool override;
+  [[nodiscard]] auto copy_to(std::string_view new_path, bool overwrite) const
+      -> bool override;
 
   [[nodiscard]] auto exists() const -> bool override { return file_->exists(); }
 
@@ -68,8 +69,8 @@ public:
     return file_->get_read_buffer_size();
   }
 
-  [[nodiscard]] auto
-  get_time(time_type type) const -> std::optional<std::uint64_t> override {
+  [[nodiscard]] auto get_time(time_type type) const
+      -> std::optional<std::uint64_t> override {
     return file_->get_time(type);
   }
 
@@ -97,9 +98,10 @@ public:
 
   [[nodiscard]] auto truncate(std::size_t size) -> bool override;
 
-  [[nodiscard]] auto
-  write(const unsigned char *data, std::size_t to_write, std::size_t offset,
-        std::size_t *total_written = nullptr) -> bool override;
+  [[nodiscard]] auto write(const unsigned char *data, std::size_t to_write,
+                           std::size_t offset,
+                           std::size_t *total_written = nullptr)
+      -> bool override;
 
 public:
   [[nodiscard]] operator bool() const override {

support/include/utils/file_thread_file.hpp

@@ -23,7 +23,6 @@
 #define REPERTORY_INCLUDE_UTILS_FILE_THREAD_FILE_HPP_
 
 #include "utils/file.hpp"
-#include <memory>
 
 namespace repertory::utils::file {
 class thread_file final : public i_file {

support/src/utils/encryption.cpp

@@ -65,21 +65,20 @@ auto read_encrypted_range(const http_range &range,
                           const utils::encryption::hash_256_t &key,
                           reader_func_t reader_func, std::uint64_t total_size,
                           data_buffer &data) -> bool {
-  const auto encrypted_chunk_size =
+  auto encrypted_chunk_size =
       utils::encryption::encrypting_reader::get_encrypted_chunk_size();
-  const auto data_chunk_size =
+  auto data_chunk_size =
       utils::encryption::encrypting_reader::get_data_chunk_size();
 
-  const auto start_chunk =
-      static_cast<std::size_t>(range.begin / data_chunk_size);
-  const auto end_chunk = static_cast<std::size_t>(range.end / data_chunk_size);
+  auto start_chunk = static_cast<std::size_t>(range.begin / data_chunk_size);
+  auto end_chunk = static_cast<std::size_t>(range.end / data_chunk_size);
   auto remain = range.end - range.begin + 1U;
   auto source_offset = static_cast<std::size_t>(range.begin % data_chunk_size);
 
   for (std::size_t chunk = start_chunk; chunk <= end_chunk; chunk++) {
     data_buffer cypher;
-    const auto start_offset = chunk * encrypted_chunk_size;
-    const auto end_offset = std::min(
+    auto start_offset = chunk * encrypted_chunk_size;
+    auto end_offset = std::min(
         start_offset + (total_size - (chunk * data_chunk_size)) +
             encryption_header_size - 1U,
         static_cast<std::uint64_t>(start_offset + encrypted_chunk_size - 1U));
@@ -94,7 +93,7 @@ auto read_encrypted_range(const http_range &range,
     }
     cypher.clear();
 
-    const auto data_size = static_cast<std::size_t>(std::min(
+    auto data_size = static_cast<std::size_t>(std::min(
         remain, static_cast<std::uint64_t>(data_chunk_size - source_offset)));
     std::copy(std::next(source_buffer.begin(),
                         static_cast<std::int64_t>(source_offset)),
@@ -107,6 +106,58 @@ auto read_encrypted_range(const http_range &range,
 
   return true;
 }
+
+auto read_encrypted_range(const http_range &range,
+                          const utils::encryption::hash_256_t &key,
+                          reader_func_t reader_func, std::uint64_t total_size,
+                          unsigned char *data, std::size_t size,
+                          std::size_t &bytes_read) -> bool {
+  bytes_read = 0U;
+
+  auto encrypted_chunk_size =
+      utils::encryption::encrypting_reader::get_encrypted_chunk_size();
+  auto data_chunk_size =
+      utils::encryption::encrypting_reader::get_data_chunk_size();
+
+  auto start_chunk = static_cast<std::size_t>(range.begin / data_chunk_size);
+  auto end_chunk = static_cast<std::size_t>(range.end / data_chunk_size);
+  auto remain = range.end - range.begin + 1U;
+  auto source_offset = static_cast<std::size_t>(range.begin % data_chunk_size);
+
+  std::span dest_buffer(data, size);
+  for (std::size_t chunk = start_chunk; chunk <= end_chunk; chunk++) {
+    data_buffer cypher;
+    auto start_offset = chunk * encrypted_chunk_size;
+    auto end_offset = std::min(
+        start_offset + (total_size - (chunk * data_chunk_size)) +
+            encryption_header_size - 1U,
+        static_cast<std::uint64_t>(start_offset + encrypted_chunk_size - 1U));
+
+    if (not reader_func(cypher, start_offset, end_offset)) {
+      return false;
+    }
+
+    data_buffer source_buffer;
+    if (not utils::encryption::decrypt_data(key, cypher, source_buffer)) {
+      return false;
+    }
+    cypher.clear();
+
+    auto data_size = static_cast<std::size_t>(std::min(
+        remain, static_cast<std::uint64_t>(data_chunk_size - source_offset)));
+    std::copy(
+        std::next(source_buffer.begin(),
+                  static_cast<std::int64_t>(source_offset)),
+        std::next(source_buffer.begin(),
+                  static_cast<std::int64_t>(source_offset + data_size)),
+        std::next(dest_buffer.begin(), static_cast<std::int64_t>(bytes_read)));
+    remain -= data_size;
+    bytes_read += data_size;
+    source_offset = 0U;
+  }
+
+  return true;
+}
 #endif // defined(PROJECT_ENABLE_CURL)
 } // namespace repertory::utils::encryption
 

support/src/utils/file_enc_file.cpp

@@ -23,31 +23,166 @@
 
 #include "utils/file_enc_file.hpp"
 
+#include "utils/common.hpp"
+#include "utils/encrypting_reader.hpp"
+#include "utils/encryption.hpp"
+
 namespace repertory::utils::file {
-auto enc_file::attach_file(fs_file_t file) -> fs_file_t {}
+auto enc_file::attach_file(fs_file_t file) -> fs_file_t {
+  return fs_file_t{
+      new enc_file(std::move(file)),
+  };
+}
 
 enc_file::enc_file(fs_file_t file) : file_(std::move(file)) {}
 
-void enc_file::close() {}
+void enc_file::close() { file_->close(); }
 
-auto enc_file::copy_to(std::string_view new_path,
-                       bool overwrite) const -> bool {}
+auto enc_file::copy_to(std::string_view new_path, bool overwrite) const
+    -> bool {
+  return file_->copy_to(new_path, overwrite);
+}
 
-void enc_file::flush() const {}
+void enc_file::flush() const { return file_->flush(); }
 
-auto enc_file::move_to(std::string_view path) -> bool {}
+auto enc_file::move_to(std::string_view path) -> bool {
+  return file_->move_to(path);
+}
 
 auto enc_file::read(unsigned char *data, std::size_t to_read,
-                    std::uint64_t offset, std::size_t *total_read) -> bool {}
+                    std::uint64_t offset, std::size_t *total_read) -> bool {
+  if (total_read != nullptr) {
+    *total_read = 0U;
+  }
 
-auto enc_file::remove() -> bool {}
+  auto file_size{size()};
+  if (not file_size.has_value()) {
+    return false;
+  }
 
-auto enc_file::truncate(std::size_t size) -> bool {}
+  to_read = utils::calculate_read_size(file_size.value(), to_read, offset);
+  if (to_read == 0U) {
+    return true;
+  }
+
+  std::size_t bytes_read{};
+  auto ret{
+      utils::encryption::read_encrypted_range(
+          {offset, offset + to_read - 1U},
+          utils::encryption::generate_key<utils::encryption::hash_256_t>(
+              encryption_token_),
+          [&](auto &&ct_buffer, auto &&start_offset,
+              auto &&end_offset) -> bool {
+            ct_buffer.resize(end_offset - start_offset + 1U);
+            return file_->read(ct_buffer, start_offset);
+          },
+          file_size.value(), data, to_read, bytes_read),
+  };
+  if (ret && total_read != nullptr) {
+    *total_read = bytes_read;
+  }
+
+  return ret;
+}
+
+auto enc_file::remove() -> bool { return file_->remove(); }
+
+auto enc_file::truncate(std::size_t size) -> bool {
+  if (size == 0U) {
+    return file_->truncate(size);
+  }
+
+  auto file_size{this->size()};
+  if (not file_size.has_value()) {
+    return false;
+  }
+
+  if (size == file_size.value()) {
+    return true;
+  }
+
+  auto chunks{
+      size / utils::encryption::encrypting_reader::get_data_chunk_size(),
+  };
+  auto real_size{
+      (chunks * utils::encryption::encrypting_reader::get_data_chunk_size()) +
+          (chunks * utils::encryption::encrypting_reader::get_header_size()),
+  };
+  auto remain{
+      size % utils::encryption::encrypting_reader::get_data_chunk_size(),
+  };
+  if (remain > 0U) {
+    real_size +=
+        (remain + utils::encryption::encrypting_reader::get_header_size());
+  }
+
+  if (size < file_size.value()) {
+    if (remain == 0U) {
+      return file_->truncate(real_size);
+    }
+
+    auto begin_chunk{
+        size / utils::encryption::encrypting_reader::get_data_chunk_size(),
+    };
+
+    auto offset{
+        begin_chunk *
+            utils::encryption::encrypting_reader::get_data_chunk_size(),
+    };
+
+    std::size_t total_read{};
+    data_buffer data(
+        utils::encryption::encrypting_reader::get_data_chunk_size());
+    if (not read(data, offset), &total_read) {
+      return false;
+    }
+    data.resize(remain);
+
+    if (not file_->truncate(real_size)) {
+      return false;
+    }
+
+    return write(data, offset);
+  }
+
+  auto begin_chunk{
+      file_size.value() /
+          utils::encryption::encrypting_reader::get_data_chunk_size(),
+  };
+  auto end_chunk{
+      utils::divide_with_ceiling(
+          file_size.value(),
+          utils::encryption::encrypting_reader::get_data_chunk_size()),
+  };
+
+  return false;
+}
 
 auto enc_file::write(const unsigned char *data, std::size_t to_write,
-                     std::size_t offset, std::size_t *total_written) -> bool {}
+                     std::size_t offset, std::size_t *total_written) -> bool {
+  auto file_size{size()};
+  if (not file_size.has_value()) {
+    return false;
+  }
 
-auto enc_file::size() const -> std::optional<std::uint64_t> {}
+  if ((offset + to_write) > file_size.value()) {
+    if (not truncate((offset + to_write) - file_size.value())) {
+      return false;
+    }
+  }
+
+  return false;
+}
+
+auto enc_file::size() const -> std::optional<std::uint64_t> {
+  auto file_size = file_->size();
+  if (not file_size.has_value()) {
+    return std::nullopt;
+  }
+
+  return utils::encryption::encrypting_reader::calculate_decrypted_size(
+      file_size.value());
+}
 } // namespace repertory::utils::file
 
 #endif // defined(PROJECT_ENABLE_LIBSODIUM) && defined(PROJECT_ENABLE_BOOST)

support/test/src/utils/file_test.cpp

@@ -22,9 +22,49 @@
 #include "test.hpp"
 
 namespace {
-static constexpr const auto file_type_count{2U};
+#if defined(PROJECT_ENABLE_LIBSODIUM) && defined(PROJECT_ENABLE_BOOST)
+#include "utils/file_enc_file.hpp"
+constexpr const auto file_type_count{3U};
+#else
+constexpr const auto file_type_count{2U};
+#endif
+
+[[nodiscard]] auto create_file(auto idx, auto path,
+                               bool read_only = false) -> auto {
+  switch (idx) {
+  case 0U:
+    return repertory::utils::file::file::open_or_create_file(path, read_only);
+  case 1U:
+    return repertory::utils::file::thread_file::open_or_create_file(path,
+                                                                    read_only);
+#if defined(PROJECT_ENABLE_LIBSODIUM) && defined(PROJECT_ENABLE_BOOST)
+  case 2U:
+    return repertory::utils::file::enc_file::attach_file(
+        repertory::utils::file::file::open_or_create_file(path, read_only));
+#endif
+  default:
+    throw std::runtime_error("not supported");
+  }
 }
 
+[[nodiscard]] auto open_file(auto idx, auto path,
+                             bool read_only = false) -> auto {
+  switch (idx) {
+  case 0U:
+    return repertory::utils::file::file::open_file(path, read_only);
+  case 1U:
+    return repertory::utils::file::thread_file::open_file(path, read_only);
+#if defined(PROJECT_ENABLE_LIBSODIUM) && defined(PROJECT_ENABLE_BOOST)
+  case 2U:
+    return repertory::utils::file::enc_file::attach_file(
+        repertory::utils::file::file::open_file(path, read_only));
+#endif
+  default:
+    throw std::runtime_error("not supported");
+  }
+}
+} // namespace
+
 namespace repertory {
 TEST(utils_file, can_create_and_remove_file) {
   for (auto idx = 0U; idx < file_type_count; ++idx) {
@@ -32,8 +72,7 @@ TEST(utils_file, can_create_and_remove_file) {
     EXPECT_FALSE(utils::file::file(path).exists() ||
                  utils::file::directory(path).exists());
 
-    auto file = idx == 0U ? utils::file::file::open_or_create_file(path)
-                          : utils::file::thread_file::open_or_create_file(path);
+    auto file{create_file(idx, path)};
     EXPECT_TRUE(*file);
 
     EXPECT_TRUE(utils::file::file(path).exists());
@@ -51,15 +90,12 @@ TEST(utils_file, can_open_file) {
     auto path = test::generate_test_file_name("utils_file");
 
     {
-      auto file = idx == 0U
-                      ? utils::file::file::open_or_create_file(path)
-                      : utils::file::thread_file::open_or_create_file(path);
+      auto file{create_file(idx, path)};
       EXPECT_TRUE(*file);
     }
 
     {
-      auto file = idx == 0U ? utils::file::file::open_file(path)
-                            : utils::file::thread_file::open_file(path);
+      auto file{create_file(idx, path)};
       EXPECT_TRUE(*file);
     }
   }
@@ -69,8 +105,7 @@ TEST(utils_file, open_file_fails_if_not_found) {
   for (auto idx = 0U; idx < file_type_count; ++idx) {
     auto path = test::generate_test_file_name("utils_file");
 
-    auto file = idx == 0U ? utils::file::file::open_file(path)
-                          : utils::file::thread_file::open_file(path);
+    auto file{open_file(idx, path)};
     EXPECT_FALSE(*file);
   }
 }
@@ -79,9 +114,7 @@ TEST(utils_file, write_fails_for_read_only_file) {
   for (auto idx = 0U; idx < file_type_count; ++idx) {
     auto path = test::generate_test_file_name("utils_file");
 
-    auto file = idx == 0U
-                    ? utils::file::file::open_or_create_file(path, true)
-                    : utils::file::thread_file::open_or_create_file(path, true);
+    auto file{create_file(idx, path, true)};
     EXPECT_TRUE(utils::file::file(path).exists());
     EXPECT_TRUE(*file);
     std::size_t bytes_written{};
@@ -188,8 +221,8 @@ TEST(utils_file, read_and_write_json_file_encrypted) {
 
 #if defined(PROJECT_ENABLE_LIBDSM)
 TEST(utils_file, smb_create_smb_path) {
-  auto path = "//server/share";
-  auto rel_path = "test/test.txt";
+  const auto *path = "//server/share";
+  const auto *rel_path = "test/test.txt";
   auto smb_path = utils::file::smb_create_smb_path(path, rel_path);
   EXPECT_STREQ("//server/share/test/test.txt", smb_path.c_str());
 
@@ -207,7 +240,7 @@ TEST(utils_file, smb_create_smb_path) {
 }
 
 TEST(utils_file, smb_create_relative_path) {
-  auto path = "//server/share/test.txt";
+  const auto *path = "//server/share/test.txt";
   auto rel_path = utils::file::smb_create_relative_path(path);
   EXPECT_STREQ("\\test.txt", rel_path.c_str());
 
@@ -225,7 +258,7 @@ TEST(utils_file, smb_create_relative_path) {
 }
 
 TEST(utils_file, smb_create_search_path) {
-  auto path = "//server/share";
+  const auto *path = "//server/share";
   auto search_path = utils::file::smb_create_search_path(path);
   EXPECT_STREQ("\\*", search_path.c_str());
 
@@ -251,8 +284,8 @@ TEST(utils_file, smb_create_search_path) {
 }
 
 TEST(utils_file, smb_parent_is_same) {
-  auto path1 = "//server/share";
-  auto path2 = "//server/share";
+  const auto *path1 = "//server/share";
+  const auto *path2 = "//server/share";
   EXPECT_TRUE(utils::file::smb_parent_is_same(path1, path2));
 
   path1 = "//server/share/";
@@ -269,8 +302,8 @@ TEST(utils_file, smb_parent_is_same) {
 }
 
 TEST(utils_file, smb_parent_is_not_same) {
-  auto path1 = "server/share";
-  auto path2 = "//server/share";
+  const auto *path1 = "server/share";
+  const auto *path2 = "//server/share";
   EXPECT_FALSE(utils::file::smb_parent_is_same(path1, path2));
 
   path1 = "server/share/";