Implement secure key via KDF for transparent data encryption/decryption #60

.cspell/words.txt

@@ -27,6 +27,7 @@ cpptrace
 cppvsdbg
 create_notraverse
 crypto_aead_xchacha20poly1305_ietf_npubbytes
+cspan
 cstdint
 curl_zstd
 curle_couldnt_resolve_host
@@ -197,6 +198,7 @@ pugi
 pugixml_project
 puint32
 pvoid
+pwhash
 pwstr
 rdrw
 remote_winfsp

repertory/librepertory/include/providers/s3/s3_provider.hpp

@@ -50,9 +50,11 @@ public:
 
 private:
   s3_config s3_config_;
+
+private:
   bool legacy_bucket_{true};
-  utils::encryption::kdf_config master_kdf_cfg_;
+  utils::encryption::kdf_config master_kdf_cfg_{};
-  utils::hash::hash_256_t master_key_;
+  utils::hash::hash_256_t master_key_{};
 
 private:
   [[nodiscard]] auto add_if_not_found(api_file &file,
@@ -71,8 +73,8 @@ private:
                                        api_meta_map &meta)
       -> api_error override;
 
-  [[nodiscard]] auto decrypt_object_name(std::string &object_name,
+  [[nodiscard]] auto decrypt_object_name(std::string &object_name) const
-                                         bool &uses_kdf) const -> api_error;
+      -> api_error;
 
   [[nodiscard]] auto get_last_modified(bool directory,
                                        const std::string &api_path,
@@ -95,11 +97,6 @@ private:
     return s3_config_;
   }
 
-  [[nodiscard]] auto read_file_bytes(const std::string &api_path,
-                                     std::size_t size, std::uint64_t offset,
-                                     data_buffer &data, bool encrypted,
-                                     stop_type &stop_requested) -> api_error;
-
   [[nodiscard]] auto search_keys_for_kdf(std::string_view encryption_token)
       -> bool;
 

repertory/librepertory/src/providers/s3/s3_provider.cpp

@@ -32,6 +32,7 @@
 #include "types/repertory.hpp"
 #include "types/s3.hpp"
 #include "utils/collection.hpp"
+#include "utils/common.hpp"
 #include "utils/config.hpp"
 #include "utils/encrypting_reader.hpp"
 #include "utils/encryption.hpp"
@@ -240,8 +241,16 @@ auto s3_provider::create_file_extra(const std::string &api_path,
 
 auto s3_provider::decrypt_object_name(std::string &object_name) const
     -> api_error {
-  if (utils::encryption::decrypt_file_path(get_s3_config().encryption_token,
+  if (legacy_bucket_) {
-                                           object_name)) {
+    if (utils::encryption::decrypt_file_path(get_s3_config().encryption_token,
+                                             object_name)) {
+      return api_error::success;
+    }
+
+    return api_error::decryption_error;
+  }
+
+  if (utils::encryption::decrypt_file_path(master_key_, object_name)) {
     return api_error::success;
   }
 
@@ -278,8 +287,7 @@ auto s3_provider::get_directory_item_count(const std::string &api_path) const
     std::string token{};
     std::uint64_t total_count{};
     while (grab_more) {
-      if (not get_object_list(response_data, response_code, "/", prefix,
+      if (not get_object_list(response_data, response_code, "/", "", token)) {
-                              token)) {
         return total_count;
       }
 
@@ -936,7 +944,7 @@ auto s3_provider::search_keys_for_kdf(std::string_view encryption_token)
   while (grab_more) {
     std::string response_data{};
     long response_code{};
-    if (not get_object_list(response_data, response_code, "/", prefix, token)) {
+    if (not get_object_list(response_data, response_code, "/", "", token)) {
       throw utils::error::create_exception(function_name,
                                            {"failed to get object list"});
     }
@@ -969,12 +977,12 @@ auto s3_provider::search_keys_for_kdf(std::string_view encryption_token)
                   .as_string();
     }
 
-    node_list = doc.select_nodes("/ListBucketResult/Contents");
+    auto node_list = doc.select_nodes("/ListBucketResult/Contents");
     for (const auto &node : node_list) {
-      auto object_name{
+      std::string object_name{
-              node.node().select_node("Key").node().text().as_string()),
+          node.node().select_node("Key").node().text().as_string(),
       };
-      if (object_name == "/")) {
+      if (object_name == "/") {
         continue;
       }
 
@@ -983,8 +991,8 @@ auto s3_provider::search_keys_for_kdf(std::string_view encryption_token)
         continue;
       }
 
-      kdf_config cfg;
+      utils::encryption::kdf_config cfg;
-      if (not kdf_config::from_header(buffer, cfg)) {
+      if (not utils::encryption::kdf_config::from_header(buffer, cfg)) {
         continue;
       }
 
@@ -1040,13 +1048,15 @@ auto s3_provider::set_meta_key(const std::string &api_path, api_meta_map &meta)
         cfg.encryption_token,
         *(utils::string::split(api_path, '/', false).end() - 1U), result);
   } else {
-    auto [key, cfg] = master_kdf_cfg_.create_subkey(
+    auto [path_key, path_cfg] = master_kdf_cfg_.create_subkey(
-        utils::encryption::kdf_context::path, id, master_key_);
+        utils::encryption::kdf_context::path,
+        utils::generate_secure_random<std::uint64_t>(), master_key_);
 
     utils::encryption::encrypt_data(
-        key, *(utils::string::split(api_path, '/', false).end() - 1U), result);
+        path_key, *(utils::string::split(api_path, '/', false).end() - 1U),
+        result);
 
-    auto hdr = cfg.to_header();
+    auto hdr = path_cfg.to_header();
     result.insert(result.begin(), hdr.begin(), hdr.end());
   }
 
@@ -1070,7 +1080,7 @@ auto s3_provider::start(api_item_added_callback api_item_added,
   if (ret && not cfg.encryption_token.empty()) {
     std::string kdf_str;
     auto res = get_item_meta("/", META_KDF, kdf_str);
-    ret == res == api_error::success;
+    ret = res == api_error::success;
     if (ret) {
       if (kdf_str.empty()) {
         try {
@@ -1082,9 +1092,9 @@ auto s3_provider::start(api_item_added_callback api_item_added,
                   utils::encryption::generate_key<utils::hash::hash_256_t>(
                       cfg.encryption_token, master_kdf_cfg_);
 
-              auto res = set_item_meta("/", META_KDF,
+              res = set_item_meta("/", META_KDF,
-                                       nlohmann::json(master_kdf_cfg_).dump());
+                                  nlohmann::json(master_kdf_cfg_).dump());
-              ret == res == api_error::success;
+              ret = res == api_error::success;
               if (not ret) {
                 utils::error::raise_api_path_error(function_name, "/", res,
                                                    "set kdf in meta failed");
@@ -1212,19 +1222,11 @@ auto s3_provider::upload_file_impl(const std::string &api_path,
 auto s3_provider::read_file_bytes(const std::string &api_path, std::size_t size,
                                   std::uint64_t offset, data_buffer &data,
                                   stop_type &stop_requested) -> api_error {
-  return read_file_bytes(api_path, size, offset, data,
-                         not get_s3_config().encryption_token.empty(),
-                         stop_requested);
-}
-
-auto s3_provider::read_file_bytes(const std::string &api_path, std::size_t size,
-                                  std::uint64_t offset, data_buffer &data,
-                                  bool encrypted, stop_type &stop_requested)
-    -> api_error {
   REPERTORY_USES_FUNCTION_NAME();
 
   try {
     const auto &cfg{get_s3_config()};
+    bool encrypted{not cfg.encryption_token.empty()};
 
     std::string key;
     if (encrypted) {
@@ -1313,9 +1315,9 @@ auto s3_provider::read_file_bytes(const std::string &api_path, std::size_t size,
       return res;
     }
 
-    utils::hash::hash_256_t key;
+    utils::hash::hash_256_t data_key;
     if (legacy_bucket_) {
-      key = utils::encryption::generate_key<utils::hash::hash_256_t>(
+      data_key = utils::encryption::generate_key<utils::hash::hash_256_t>(
           cfg.encryption_token);
     } else {
       res = get_item_meta(api_path, META_KDF, temp);
@@ -1323,19 +1325,15 @@ auto s3_provider::read_file_bytes(const std::string &api_path, std::size_t size,
         return res;
       }
 
-      if (not utils::encryption::recreate_key_argon2id(
+      auto data_cfg =
-              cfg.encryption_token,
+          nlohmann::json::parse(temp).get<utils::encryption::kdf_config>();
-              nlohmann::json::parse(temp).get<utils::encryption::kdf_config>(),
+      data_key = data_cfg.recreate_subkey(utils::encryption::kdf_context::data,
-              key)) {
+                                          master_key_);
-        throw utils::error::create_exception(
-            function_name, {"failed to recreate data key from kdf"});
-        return api_error::error;
-      }
     }
 
     auto total_size{utils::string::to_uint64(temp)};
     return utils::encryption::read_encrypted_range(
-               {.begin = offset, .end = offset + size - 1U}, key,
+               {.begin = offset, .end = offset + size - 1U}, data_key,
                not legacy_bucket_,
                [&](data_buffer &ct_buffer, std::uint64_t start_offset,
                    std::uint64_t end_offset) -> bool {

support/include/utils/encryption.hpp

@@ -143,6 +143,16 @@ struct kdf_config final {
     return {sub_key, cfg};
   }
 
+  template <typename hash_t>
+  [[nodiscard]] auto recreate_subkey(kdf_context ctx,
+                                     const hash_t &master_key) const -> hash_t {
+    hash_t sub_key;
+    crypto_kdf_derive_from_key(sub_key.data(), sub_key.size(), unique_id,
+                               get_kdf_context_name(ctx).data(),
+                               master_key.data());
+    return sub_key;
+  }
+
   [[nodiscard]] static auto from_header(data_cspan data, kdf_config &cfg)
       -> bool;
 
@@ -201,19 +211,19 @@ template <typename string_t>
                                          utils::hash::hash_256_t &key) -> bool;
 
 template <typename hash_t, typename string_t>
-[[nodiscard]] inline bool
+[[nodiscard]] inline auto
 detect_and_recreate_key(string_t password, data_cspan header, hash_t &key,
-                        std::optional<kdf_config> &cfg);
+                        std::optional<kdf_config> &cfg) -> bool;
 
 template <typename hash_t>
-[[nodiscard]] inline bool
+[[nodiscard]] inline auto
 detect_and_recreate_key(std::string_view password, data_cspan header,
-                        hash_t &key, std::optional<kdf_config> &cfg);
+                        hash_t &key, std::optional<kdf_config> &cfg) -> bool;
 
 template <typename hash_t>
-[[nodiscard]] inline bool
+[[nodiscard]] inline auto
 detect_and_recreate_key(std::wstring_view password, data_cspan header,
-                        hash_t &key, std::optional<kdf_config> &cfg);
+                        hash_t &key, std::optional<kdf_config> &cfg) -> bool;
 
 [[nodiscard]] auto decrypt_file_name(std::string_view encryption_token,
                                      std::string &file_name) -> bool;
@@ -229,6 +239,12 @@ detect_and_recreate_key(std::wstring_view password, data_cspan header,
                                      const kdf_config &cfg,
                                      std::string &file_path) -> bool;
 
+[[nodiscard]] auto decrypt_file_name(const utils::hash::hash_256_t &master_key,
+                                     std::string &file_name) -> bool;
+
+[[nodiscard]] auto decrypt_file_path(const utils::hash::hash_256_t &master_key,
+                                     std::string &file_path) -> bool;
+
 template <typename result_t, typename arr_t, std::size_t arr_size>
 [[nodiscard]] inline auto decrypt_data(const std::array<arr_t, arr_size> &key,
                                        const unsigned char *buffer,
@@ -570,9 +586,9 @@ inline auto recreate_key(std::wstring_view password, const kdf_config &cfg)
 }
 
 template <typename hash_t, typename string_t>
-inline bool detect_and_recreate_key(string_t password, data_cspan header,
+inline auto detect_and_recreate_key(string_t password, data_cspan header,
-                                    hash_t &key,
+                                    hash_t &key, std::optional<kdf_config> &cfg)
-                                    std::optional<kdf_config> &cfg) {
+    -> bool {
   if (header.size() >= kdf_config::size()) {
     kdf_config tmp{};
     if (kdf_config::from_header(header.first(kdf_config::size()), tmp)) {
@@ -587,17 +603,17 @@ inline bool detect_and_recreate_key(string_t password, data_cspan header,
 }
 
 template <typename hash_t>
-inline bool detect_and_recreate_key(std::string_view password,
+inline auto detect_and_recreate_key(std::string_view password,
                                     data_cspan header, hash_t &key,
-                                    std::optional<kdf_config> &cfg) {
+                                    std::optional<kdf_config> &cfg) -> bool {
   return detect_and_recreate_key<hash_t, std::string_view>(password, header,
                                                            key, cfg);
 }
 
 template <typename hash_t>
-inline bool detect_and_recreate_key(std::wstring_view password,
+inline auto detect_and_recreate_key(std::wstring_view password,
                                     data_cspan header, hash_t &key,
-                                    std::optional<kdf_config> &cfg) {
+                                    std::optional<kdf_config> &cfg) -> bool {
   return detect_and_recreate_key<hash_t, std::wstring_view>(password, header,
                                                             key, cfg);
 }

support/src/utils/encryption.cpp

@@ -126,6 +126,27 @@ auto decrypt_file_path(std::string_view encryption_token, const kdf_config &cfg,
   return true;
 }
 
+auto decrypt_file_path(const utils::hash::hash_256_t &master_key,
+                       std::string &file_path) -> bool {
+  std::vector<std::string> decrypted_parts;
+  for (const auto &part : std::filesystem::path(file_path)) {
+    auto file_name = part.string();
+    if (file_name == "/") {
+      continue;
+    }
+
+    if (not decrypt_file_name(master_key, file_name)) {
+      return false;
+    }
+
+    decrypted_parts.push_back(file_name);
+  }
+
+  file_path =
+      utils::path::create_api_path(utils::string::join(decrypted_parts, '/'));
+  return true;
+}
+
 auto decrypt_file_name(std::string_view encryption_token,
                        std::string &file_name) -> bool {
   data_buffer buffer;
@@ -149,6 +170,26 @@ auto decrypt_file_name(std::string_view encryption_token, const kdf_config &cfg,
                                          file_name);
 }
 
+auto decrypt_file_name(const utils::hash::hash_256_t &master_key,
+                       std::string &file_name) -> bool {
+  data_buffer buffer;
+  if (not utils::collection::from_hex_string(file_name, buffer)) {
+    return false;
+  }
+
+  utils::encryption::kdf_config path_cfg;
+  if (not utils::encryption::kdf_config::from_header(buffer, path_cfg)) {
+    return false;
+  }
+
+  auto path_key = path_cfg.recreate_subkey(utils::encryption::kdf_context::path,
+                                           master_key);
+
+  file_name.clear();
+  return utils::encryption::decrypt_data(
+      path_key, &buffer[utils::encryption::kdf_config::size()], file_name);
+}
+
 template <typename data_t>
 [[nodiscard]] auto
 read_encrypted_range(http_range range, const utils::hash::hash_256_t &key,