refactor

repertory/librepertory/src/providers/s3/s3_provider.cpp

@@ -27,6 +27,7 @@
 #include "types/repertory.hpp"
 #include "types/s3.hpp"
 #include "utils/collection.hpp"
+#include "utils/config.hpp"
 #include "utils/encrypting_reader.hpp"
 #include "utils/encryption.hpp"
 #include "utils/error_utils.hpp"
@@ -35,7 +36,6 @@
 #include "utils/polling.hpp"
 #include "utils/string.hpp"
 #include "utils/time.hpp"
-#include <utils/config.hpp>
 
 namespace repertory {
 s3_provider::s3_provider(app_config &config, i_http_comm &comm)
@@ -320,7 +320,10 @@ auto s3_provider::get_directory_items_impl(
   long response_code{};
   auto prefix = object_name.empty() ? object_name : object_name + "/";
 
-  if (not get_object_list(response_data, response_code, "/", prefix)) {
+  auto grab_more{true};
+  std::string token{};
+  while (grab_more) {
+    if (not get_object_list(response_data, response_code, "/", prefix, token)) {
       return api_error::comm_error;
     }
 
@@ -340,8 +343,20 @@ auto s3_provider::get_directory_items_impl(
       return api_error::error;
     }
 
+    grab_more = doc.select_node("/ListBucketResult/IsTruncated")
+                    .node()
+                    .text()
+                    .as_bool();
+    if (grab_more) {
+      token = doc.select_node("/ListBucketResult/NextContinuationToken")
+                  .node()
+                  .text()
+                  .as_string();
+    }
+
     const auto add_directory_item =
-      [&](bool directory, const std::string &name, std::uint64_t last_modified,
+        [&](bool directory, const std::string &name,
+            std::uint64_t last_modified,
             std::function<std::uint64_t(const directory_item &)> get_size)
         -> api_error {
       auto child_api_path =
@@ -387,7 +402,8 @@ auto s3_provider::get_directory_items_impl(
       return api_error::success;
     };
 
-  auto node_list = doc.select_nodes("/ListBucketResult/CommonPrefixes/Prefix");
+    auto node_list =
+        doc.select_nodes("/ListBucketResult/CommonPrefixes/Prefix");
     for (auto &&node : node_list) {
       add_directory_item(
           true, node.node().text().as_string(), 0U,
@@ -411,6 +427,7 @@ auto s3_provider::get_directory_items_impl(
             });
       }
     }
+  }
 
   return ret;
 }
@@ -474,6 +491,17 @@ auto s3_provider::get_file_list(api_file_list &list) const -> api_error {
       return api_error::comm_error;
     }
 
+    grab_more = doc.select_node("/ListBucketResult/IsTruncated")
+                    .node()
+                    .text()
+                    .as_bool();
+    if (grab_more) {
+      token = doc.select_node("/ListBucketResult/NextContinuationToken")
+                  .node()
+                  .text()
+                  .as_string();
+    }
+
     auto node_list = doc.select_nodes("/ListBucketResult/Contents");
     for (auto &&node : node_list) {
       auto api_path =