refactor s3 provider

include/types/repertory.hpp

@@ -169,7 +169,7 @@ struct directory_item {
   std::string api_parent;
   bool directory{false};
   std::uint64_t size{};
-  api_meta_map meta;
+  api_meta_map meta{};
   bool resolved{false};
 
   [[nodiscard]] static auto from_json(const json &item) -> directory_item {

src/comm/curl/requests/http_put_file.cpp

@@ -27,40 +27,42 @@ namespace repertory::curl::requests {
 auto http_put_file::set_method(CURL *curl, stop_type &stop_requested) const
     -> bool {
   curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "PUT");
+  curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L);
 
-  if (not source_path.empty()) {
+  if (source_path.empty()) {
-    curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L);
+    curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, 0L);
-    if (reader) {
+    return true;
-      curl_easy_setopt(curl, CURLOPT_READDATA, reader.get());
-      curl_easy_setopt(
-          curl, CURLOPT_READFUNCTION,
-          static_cast<curl_read_callback>(
-              utils::encryption::encrypting_reader::reader_function));
-      curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE,
-                       reader->get_total_size());
-      return true;
-    }
-
-    read_info = std::make_shared<read_file_info>(read_file_info{
-        stop_requested,
-    });
-
-    if (native_file::open(source_path, read_info->nf) != api_error::success) {
-      return false;
-    }
-
-    read_info->nf->set_auto_close(true);
-
-    std::uint64_t file_size{};
-    if (not read_info->nf->get_file_size(file_size)) {
-      return false;
-    }
-
-    curl_easy_setopt(curl, CURLOPT_READDATA, read_info.get());
-    curl_easy_setopt(curl, CURLOPT_READFUNCTION, read_file_data);
-    curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, file_size);
   }
 
+  if (reader) {
+    curl_easy_setopt(curl, CURLOPT_READDATA, reader.get());
+    curl_easy_setopt(
+        curl, CURLOPT_READFUNCTION,
+        static_cast<curl_read_callback>(
+            utils::encryption::encrypting_reader::reader_function));
+    curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, reader->get_total_size());
+    return true;
+  }
+
+  read_info = std::make_shared<read_file_info>(read_file_info{
+      stop_requested,
+  });
+
+  if (native_file::open(source_path, read_info->nf) != api_error::success) {
+    return false;
+  }
+
+  read_info->nf->set_auto_close(true);
+
+  std::uint64_t file_size{};
+  if (not read_info->nf->get_file_size(file_size)) {
+    return false;
+  }
+
+  curl_easy_setopt(curl, CURLOPT_READDATA, read_info.get());
+  curl_easy_setopt(curl, CURLOPT_READFUNCTION, read_file_data);
+  curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, file_size);
+
   return true;
 }
 } // namespace repertory::curl::requests

src/providers/s3/s3_provider.cpp

@@ -132,24 +132,36 @@ auto s3_provider::create_directory(const std::string &api_path,
   try {
     const auto cfg = config_.get_s3_config();
     const auto is_encrypted = not cfg.encryption_token.empty();
+    stop_type stop_requested{false};
 
-    std::string key;
     if (is_encrypted) {
-      res = get_item_meta(api_path, META_KEY, key);
+      std::string encrypted_file_path;
+      res = get_item_meta(utils::path::get_parent_api_path(api_path), META_KEY,
+                          encrypted_file_path);
       if (res != api_error::success) {
+        utils::error::raise_api_path_error(__FUNCTION__, api_path, res,
+                                           "failed to create file");
         return res;
       }
+
+      data_buffer result;
+      utils::encryption::encrypt_data(
+          cfg.encryption_token,
+          *(utils::string::split(api_path, '/', false).end() - 1U), result);
+
+      meta[META_KEY] = utils::path::create_api_path(utils::path::combine(
+          utils::path::create_api_path(encrypted_file_path),
+          {utils::to_hex_string(result)}));
     }
 
     const auto object_name =
-        utils::path::create_api_path(is_encrypted ? key : api_path);
+        utils::path::create_api_path(is_encrypted ? meta[META_KEY] : api_path);
 
     curl::requests::http_put_file put_file{};
     put_file.allow_timeout = true;
     put_file.aws_service = "aws:amz:" + cfg.region + ":s3";
     put_file.path = object_name + '/';
 
-    stop_type stop_requested{false};
     long response_code{};
     if (not comm_.make_request(put_file, response_code, stop_requested)) {
       utils::error::raise_api_path_error(__FUNCTION__, api_path,
@@ -249,8 +261,29 @@ auto s3_provider::create_file(const std::string &api_path, api_meta_map &meta)
     return api_error::item_exists;
   }
 
+  stop_type stop_requested{false};
   try {
     meta[META_DIRECTORY] = utils::string::from_bool(false);
+    if (not config_.get_s3_config().encryption_token.empty()) {
+      std::string encrypted_file_path;
+      res = get_item_meta(utils::path::get_parent_api_path(api_path), META_KEY,
+                          encrypted_file_path);
+      if (res != api_error::success) {
+        utils::error::raise_api_path_error(__FUNCTION__, api_path, res,
+                                           "failed to create file");
+        return res;
+      }
+
+      data_buffer result;
+      utils::encryption::encrypt_data(
+          config_.get_s3_config().encryption_token,
+          *(utils::string::split(api_path, '/', false).end() - 1U), result);
+
+      meta[META_KEY] = utils::path::create_api_path(utils::path::combine(
+          utils::path::create_api_path(encrypted_file_path),
+          {utils::to_hex_string(result)}));
+    }
+
     res = set_item_meta(api_path, meta);
     if (res != api_error::success) {
       utils::error::raise_api_path_error(__FUNCTION__, api_path, res,
@@ -258,8 +291,12 @@ auto s3_provider::create_file(const std::string &api_path, api_meta_map &meta)
       return res;
     }
 
-    stop_type stop_requested{false};
+    res = upload_file(api_path, meta[META_SOURCE], stop_requested);
-    return upload_file(api_path, meta[META_SOURCE], stop_requested);
+    if (res != api_error::success) {
+      db_->Delete(rocksdb::WriteOptions(), api_path);
+    }
+
+    return res;
   } catch (const std::exception &e) {
     utils::error::raise_api_path_error(__FUNCTION__, api_path, e,
                                        "failed to create file");
@@ -359,11 +396,135 @@ auto s3_provider::get_api_path_from_source(const std::string &source_path,
 }
 
 auto s3_provider::get_directory_item_count(const std::string &api_path) const
-    -> std::uint64_t {}
+    -> std::uint64_t {
+  // TODO implement this
+  return 0U;
+}
 
 auto s3_provider::get_directory_items(const std::string &api_path,
                                       directory_item_list &list) const
-    -> api_error {}
+    -> api_error {
+  try {
+    const auto cfg = config_.get_s3_config();
+    const auto is_encrypted = not cfg.encryption_token.empty();
+
+    std::string key;
+    if (is_encrypted) {
+      auto res = get_item_meta(api_path, META_KEY, key);
+      if (res != api_error::success) {
+        return res;
+      }
+    }
+
+    const auto object_name =
+        api_path == "/"
+            ? ""
+            : utils::path::create_api_path(is_encrypted ? key : api_path);
+
+    std::string response_data{};
+    long response_code{};
+    auto prefix = object_name.empty() ? object_name : object_name + "/";
+
+    if (not get_object_list(comm_, config_.get_s3_config(), response_data,
+                            response_code, "/", prefix)) {
+      return api_error::comm_error;
+    }
+
+    if (response_code == http_error_codes::not_found) {
+      return api_error::directory_not_found;
+    }
+
+    if (response_code != http_error_codes::ok) {
+      return api_error::comm_error;
+    }
+
+    pugi::xml_document doc;
+    auto res = doc.load_string(response_data.c_str());
+    if (res.status != pugi::xml_parse_status::status_ok) {
+      return api_error::error;
+    }
+
+    const auto add_directory_item =
+        [&](bool directory, const std::string &name,
+            std::function<std::uint64_t(const directory_item &)> get_size)
+        -> api_error {
+      auto child_api_path =
+          utils::path::create_api_path(utils::path::combine("/", {name}));
+      std::string child_object_name;
+      if (is_encrypted) {
+        child_object_name = child_api_path;
+        auto res = utils::encryption::decrypt_file_path(cfg.encryption_token,
+                                                        child_api_path);
+        if (res != api_error::success) {
+          return res;
+        }
+      }
+
+      directory_item dir_item{};
+      dir_item.api_path = child_api_path;
+      dir_item.api_parent = utils::path::get_parent_api_path(dir_item.api_path);
+      dir_item.directory = directory;
+      dir_item.size = get_size(dir_item);
+      auto res = get_item_meta(child_api_path, dir_item.meta);
+      if (not directory && res == api_error::item_not_found) {
+        api_file file{};
+        res = get_file(child_api_path, file);
+      }
+      if (res != api_error::success) {
+        return res;
+      }
+      list.push_back(std::move(dir_item));
+      return api_error::success;
+    };
+
+    auto node_list =
+        doc.select_nodes("/ListBucketResult/CommonPrefixes/Prefix");
+    for (const auto &node : node_list) {
+      add_directory_item(
+          true, node.node().text().as_string(),
+          [](const directory_item &) -> std::uint64_t { return 0U; });
+    }
+
+    node_list = doc.select_nodes("/ListBucketResult/Contents");
+    for (const auto &node : node_list) {
+      auto child_object_name =
+          std::string{node.node().select_node("Key").node().text().as_string()};
+      if (child_object_name != prefix) {
+        auto size = node.node().select_node("Size").node().text().as_ullong();
+        add_directory_item(
+            false, child_object_name,
+            [&is_encrypted, &size](const directory_item &) -> std::uint64_t {
+              return is_encrypted ? utils::encryption::encrypting_reader::
+                                        calculate_decrypted_size(size)
+                                  : size;
+            });
+      }
+    }
+
+    std::sort(list.begin(), list.end(),
+              [](const auto &item1, const auto &item2) -> bool {
+                return (item1.directory && not item2.directory) ||
+                       (not(item2.directory && not item1.directory) &&
+                        (item1.api_path.compare(item2.api_path) < 0));
+              });
+
+    list.insert(list.begin(), directory_item{
+                                  "..",
+                                  "",
+                                  true,
+                              });
+    list.insert(list.begin(), directory_item{
+                                  ".",
+                                  "",
+                                  true,
+                              });
+    return api_error::success;
+  } catch (const std::exception &e) {
+    utils::error::raise_error(__FUNCTION__, e, "exception occurred");
+  }
+
+  return api_error::error;
+}
 
 auto s3_provider::get_file(const std::string &api_path, api_file &file) const
     -> api_error {
@@ -906,9 +1067,13 @@ void s3_provider::remove_deleted_files() {
   }
 }
 
-auto s3_provider::remove_directory(const std::string &api_path) -> api_error {}
+auto s3_provider::remove_directory(const std::string &api_path) -> api_error {
+  return api_error::not_implemented;
+}
 
-auto s3_provider::remove_file(const std::string &api_path) -> api_error {}
+auto s3_provider::remove_file(const std::string &api_path) -> api_error {
+  return api_error::not_implemented;
+}
 
 auto s3_provider::remove_item_meta(const std::string &api_path,
                                    const std::string &key) -> api_error {
@@ -1055,6 +1220,12 @@ auto s3_provider::upload_file(const std::string &api_path,
     return error;
   };
   try {
+    std::uint64_t file_size{};
+    if (utils::file::is_file(source_path) &&
+        not utils::file::get_file_size(source_path, file_size)) {
+      return notify_end(api_error::comm_error);
+    }
+
     const auto cfg = config_.get_s3_config();
     const auto is_encrypted = not cfg.encryption_token.empty();
 
@@ -1074,7 +1245,7 @@ auto s3_provider::upload_file(const std::string &api_path,
     put_file.path = object_name;
     put_file.source_path = source_path;
 
-    if (is_encrypted) {
+    if (is_encrypted && file_size > 0U) {
       static stop_type no_stop{false};
 
       put_file.reader = std::make_shared<utils::encryption::encrypting_reader>(

src/utils/encrypting_reader.cpp

@@ -172,7 +172,7 @@ encrypting_reader::encrypting_reader(
     : key_(utils::encryption::generate_key(token)),
       stop_requested_(stop_requested),
       error_return_(error_return) {
-  const auto res = native_file::open(
+  const auto res = native_file::create_or_open(
       source_path, not relative_parent_path.has_value(), source_file_);
   if (res != api_error::success) {
     throw std::runtime_error("file open failed|src|" + source_path + '|' +
@@ -225,7 +225,8 @@ encrypting_reader::encrypting_reader(const std::string &encrypted_file_path,
     : key_(utils::encryption::generate_key(token)),
       stop_requested_(stop_requested),
       error_return_(error_return) {
-  const auto res = native_file::open(source_path, false, source_file_);
+  const auto res =
+      native_file::create_or_open(source_path, false, source_file_);
   if (res != api_error::success) {
     throw std::runtime_error("file open failed|src|" + source_path + '|' +
                              api_error_to_string(res));
@@ -266,7 +267,8 @@ encrypting_reader::encrypting_reader(
     : key_(utils::encryption::generate_key(token)),
       stop_requested_(stop_requested),
       error_return_(error_return) {
-  const auto res = native_file::open(source_path, false, source_file_);
+  const auto res =
+      native_file::create_or_open(source_path, false, source_file_);
   if (res != api_error::success) {
     throw std::runtime_error("file open failed|src|" + source_path + '|' +
                              api_error_to_string(res));