s3 fixes

repertory/librepertory/src/providers/s3/s3_provider.cpp

@@ -22,7 +22,6 @@
 #include "providers/s3/s3_provider.hpp"
 
 #include "app_config.hpp"
-#include "comm/curl/curl_comm.hpp"
 #include "comm/i_http_comm.hpp"
 #include "file_manager/i_file_manager.hpp"
 #include "types/repertory.hpp"
@@ -243,35 +242,52 @@ auto s3_provider::get_directory_item_count(const std::string &api_path) const
     long response_code{};
     auto prefix = object_name.empty() ? object_name : object_name + "/";
 
-    if (not get_object_list(response_data, response_code, "/", prefix)) {
-      return 0U;
+    auto grab_more{true};
+    std::string token{};
+    std::uint64_t total_count{};
+    while (grab_more) {
+      if (not get_object_list(response_data, response_code, "/", prefix,
+                              token)) {
+        return total_count;
       }
 
       if (response_code == http_error_codes::not_found) {
-      return 0U;
+        return total_count;
       }
 
       if (response_code != http_error_codes::ok) {
-      return 0U;
+        return total_count;
       }
 
       pugi::xml_document doc;
       auto res = doc.load_string(response_data.c_str());
       if (res.status != pugi::xml_parse_status::status_ok) {
-      return 0U;
+        return total_count;
+      }
+
+      grab_more = doc.select_node("/ListBucketResult/IsTruncated")
+                      .node()
+                      .text()
+                      .as_bool();
+      if (grab_more) {
+        token = doc.select_node("/ListBucketResult/NextContinuationToken")
+                    .node()
+                    .text()
+                    .as_string();
       }
 
       auto node_list =
           doc.select_nodes("/ListBucketResult/CommonPrefixes/Prefix");
-    std::uint64_t ret = node_list.size();
+      total_count += node_list.size();
 
       node_list = doc.select_nodes("/ListBucketResult/Contents");
-    ret += node_list.size();
+      total_count += node_list.size();
       if (not prefix.empty()) {
-      --ret;
+        --total_count;
+      }
     }
 
-    return ret;
+    return total_count;
   } catch (const std::exception &e) {
     utils::error::raise_error(function_name, e, "exception occurred");
   }
@@ -434,9 +450,13 @@ auto s3_provider::get_file(const std::string &api_path,
 auto s3_provider::get_file_list(api_file_list &list) const -> api_error {
   REPERTORY_USES_FUNCTION_NAME();
 
+  auto grab_more{true};
+  std::string token{};
+  while (grab_more) {
     std::string response_data;
     long response_code{};
-  if (not get_object_list(response_data, response_code)) {
+    if (not get_object_list(response_data, response_code, std::nullopt,
+                            std::nullopt, token)) {
       return api_error::comm_error;
     }
 
@@ -474,15 +494,14 @@ auto s3_provider::get_file_list(api_file_list &list) const -> api_error {
         file.api_path = utils::path::create_api_path(api_path);
         file.api_parent = utils::path::get_parent_api_path(file.api_path);
         file.accessed_date = file.changed_date = file.creation_date =
-          file.modified_date = convert_api_date(node.node()
+            file.modified_date =
+                convert_api_date(node.node()
                                      .select_node("LastModified")
                                      .node()
                                      .text()
                                      .as_string());
-      file.file_size =
-          is_encrypted
-              ? utils::encryption::encrypting_reader::calculate_decrypted_size(
-                    size)
+        file.file_size = is_encrypted ? utils::encryption::encrypting_reader::
+                                            calculate_decrypted_size(size)
                                       : size;
         file.key = is_encrypted ? utils::path::create_api_path(api_path) : "";
         auto err = add_if_not_found(file, api_path);
@@ -493,6 +512,7 @@ auto s3_provider::get_file_list(api_file_list &list) const -> api_error {
         list.push_back(std::move(file));
       }
     }
+  }
 
   return api_error::success;
 }