s3 fixes

repertory/librepertory/src/providers/s3/s3_provider.cpp

@@ -22,7 +22,6 @@
 #include "providers/s3/s3_provider.hpp"
 
 #include "app_config.hpp"
-#include "comm/curl/curl_comm.hpp"
 #include "comm/i_http_comm.hpp"
 #include "file_manager/i_file_manager.hpp"
 #include "types/repertory.hpp"
@@ -243,35 +242,52 @@ auto s3_provider::get_directory_item_count(const std::string &api_path) const
     long response_code{};
     auto prefix = object_name.empty() ? object_name : object_name + "/";
 
-    if (not get_object_list(response_data, response_code, "/", prefix)) {
-      return 0U;
+    auto grab_more{true};
+    std::string token{};
+    std::uint64_t total_count{};
+    while (grab_more) {
+      if (not get_object_list(response_data, response_code, "/", prefix,
+                              token)) {
+        return total_count;
+      }
+
+      if (response_code == http_error_codes::not_found) {
+        return total_count;
+      }
+
+      if (response_code != http_error_codes::ok) {
+        return total_count;
+      }
+
+      pugi::xml_document doc;
+      auto res = doc.load_string(response_data.c_str());
+      if (res.status != pugi::xml_parse_status::status_ok) {
+        return total_count;
+      }
+
+      grab_more = doc.select_node("/ListBucketResult/IsTruncated")
+                      .node()
+                      .text()
+                      .as_bool();
+      if (grab_more) {
+        token = doc.select_node("/ListBucketResult/NextContinuationToken")
+                    .node()
+                    .text()
+                    .as_string();
+      }
+
+      auto node_list =
+          doc.select_nodes("/ListBucketResult/CommonPrefixes/Prefix");
+      total_count += node_list.size();
+
+      node_list = doc.select_nodes("/ListBucketResult/Contents");
+      total_count += node_list.size();
+      if (not prefix.empty()) {
+        --total_count;
+      }
     }
 
-    if (response_code == http_error_codes::not_found) {
-      return 0U;
-    }
-
-    if (response_code != http_error_codes::ok) {
-      return 0U;
-    }
-
-    pugi::xml_document doc;
-    auto res = doc.load_string(response_data.c_str());
-    if (res.status != pugi::xml_parse_status::status_ok) {
-      return 0U;
-    }
-
-    auto node_list =
-        doc.select_nodes("/ListBucketResult/CommonPrefixes/Prefix");
-    std::uint64_t ret = node_list.size();
-
-    node_list = doc.select_nodes("/ListBucketResult/Contents");
-    ret += node_list.size();
-    if (not prefix.empty()) {
-      --ret;
-    }
-
-    return ret;
+    return total_count;
   } catch (const std::exception &e) {
     utils::error::raise_error(function_name, e, "exception occurred");
   }
@@ -434,63 +450,67 @@ auto s3_provider::get_file(const std::string &api_path,
 auto s3_provider::get_file_list(api_file_list &list) const -> api_error {
   REPERTORY_USES_FUNCTION_NAME();
 
-  std::string response_data;
-  long response_code{};
-  if (not get_object_list(response_data, response_code)) {
-    return api_error::comm_error;
-  }
+  auto grab_more{true};
+  std::string token{};
+  while (grab_more) {
+    std::string response_data;
+    long response_code{};
+    if (not get_object_list(response_data, response_code, std::nullopt,
+                            std::nullopt, token)) {
+      return api_error::comm_error;
+    }
 
-  if (response_code != http_error_codes::ok) {
-    utils::error::raise_error(function_name, response_code,
-                              "failed to get file list");
-    return api_error::comm_error;
-  }
+    if (response_code != http_error_codes::ok) {
+      utils::error::raise_error(function_name, response_code,
+                                "failed to get file list");
+      return api_error::comm_error;
+    }
 
-  pugi::xml_document doc;
-  auto res = doc.load_string(response_data.c_str());
-  if (res.status != pugi::xml_parse_status::status_ok) {
-    utils::error::raise_error(function_name, res.status,
-                              "failed to parse xml document");
-    return api_error::comm_error;
-  }
+    pugi::xml_document doc;
+    auto res = doc.load_string(response_data.c_str());
+    if (res.status != pugi::xml_parse_status::status_ok) {
+      utils::error::raise_error(function_name, res.status,
+                                "failed to parse xml document");
+      return api_error::comm_error;
+    }
 
-  auto node_list = doc.select_nodes("/ListBucketResult/Contents");
-  for (auto &&node : node_list) {
-    auto api_path =
-        std::string{node.node().select_node("Key").node().text().as_string()};
-    if (not utils::string::ends_with(api_path, "/")) {
-      auto is_encrypted =
-          not get_config().get_s3_config().encryption_token.empty();
-      if (is_encrypted) {
-        auto err = decrypt_object_name(api_path);
+    auto node_list = doc.select_nodes("/ListBucketResult/Contents");
+    for (auto &&node : node_list) {
+      auto api_path =
+          std::string{node.node().select_node("Key").node().text().as_string()};
+      if (not utils::string::ends_with(api_path, "/")) {
+        auto is_encrypted =
+            not get_config().get_s3_config().encryption_token.empty();
+        if (is_encrypted) {
+          auto err = decrypt_object_name(api_path);
+          if (err != api_error::success) {
+            return err;
+          }
+        }
+
+        auto size = node.node().select_node("Size").node().text().as_ullong();
+
+        api_file file{};
+        file.api_path = utils::path::create_api_path(api_path);
+        file.api_parent = utils::path::get_parent_api_path(file.api_path);
+        file.accessed_date = file.changed_date = file.creation_date =
+            file.modified_date =
+                convert_api_date(node.node()
+                                     .select_node("LastModified")
+                                     .node()
+                                     .text()
+                                     .as_string());
+        file.file_size = is_encrypted ? utils::encryption::encrypting_reader::
+                                            calculate_decrypted_size(size)
+                                      : size;
+        file.key = is_encrypted ? utils::path::create_api_path(api_path) : "";
+        auto err = add_if_not_found(file, api_path);
         if (err != api_error::success) {
           return err;
         }
+
+        list.push_back(std::move(file));
       }
-
-      auto size = node.node().select_node("Size").node().text().as_ullong();
-
-      api_file file{};
-      file.api_path = utils::path::create_api_path(api_path);
-      file.api_parent = utils::path::get_parent_api_path(file.api_path);
-      file.accessed_date = file.changed_date = file.creation_date =
-          file.modified_date = convert_api_date(node.node()
-                                                    .select_node("LastModified")
-                                                    .node()
-                                                    .text()
-                                                    .as_string());
-      file.file_size =
-          is_encrypted
-              ? utils::encryption::encrypting_reader::calculate_decrypted_size(
-                    size)
-              : size;
-      file.key = is_encrypted ? utils::path::create_api_path(api_path) : "";
-      auto err = add_if_not_found(file, api_path);
-      if (err != api_error::success) {
-        return err;
-      }
-
-      list.push_back(std::move(file));
     }
   }