extract common behavior

include/providers/base_provider.hpp

@@ -83,6 +83,10 @@ protected:
     return config_;
   }
 
+  [[nodiscard]] virtual auto
+  get_directory_items_impl(const std::string &api_path,
+                           directory_item_list &list) const -> api_error = 0;
+
   [[nodiscard]] auto get_db() const -> rocksdb::DB * { return db_.get(); }
 
   [[nodiscard]] auto get_file_mgr() -> i_file_manager * { return fm_; }
@@ -107,6 +111,10 @@ public:
                                               std::string &api_path) const
       -> api_error override;
 
+  [[nodiscard]] auto get_directory_items(const std::string &api_path,
+                                         directory_item_list &list) const
+      -> api_error override;
+
   [[nodiscard]] auto get_file_size(const std::string &api_path,
                                    std::uint64_t &file_size) const
       -> api_error override;

include/providers/s3/s3_provider.hpp

@@ -49,10 +49,6 @@ private:
                                       const std::string &object_name) const
       -> api_error;
 
-  [[nodiscard]] auto create_directory_impl(const std::string &api_path,
-                                           api_meta_map &meta)
-      -> api_error override;
-
   [[nodiscard]] auto create_file_extra(const std::string &api_path,
                                        api_meta_map &meta)
       -> api_error override;
@@ -75,14 +71,19 @@ private:
                   std::optional<std::string> prefix = std::nullopt) const
       -> bool;
 
+protected:
+  [[nodiscard]] auto create_directory_impl(const std::string &api_path,
+                                           api_meta_map &meta)
+      -> api_error override;
+
+  [[nodiscard]] auto get_directory_items_impl(const std::string &api_path,
+                                              directory_item_list &list) const
+      -> api_error override;
+
 public:
   [[nodiscard]] auto get_directory_item_count(const std::string &api_path) const
       -> std::uint64_t override;
 
-  [[nodiscard]] auto get_directory_items(const std::string &api_path,
-                                         directory_item_list &list) const
-      -> api_error override;
-
   [[nodiscard]] auto get_file(const std::string &api_path, api_file &file) const
       -> api_error override;
 

include/providers/sia/sia_provider.hpp

@@ -54,14 +54,14 @@ protected:
                                            api_meta_map &meta)
       -> api_error override;
 
+  [[nodiscard]] auto get_directory_items_impl(const std::string &api_path,
+                                              directory_item_list &list) const
+      -> api_error override;
+
 public:
   [[nodiscard]] auto get_directory_item_count(const std::string &api_path) const
       -> std::uint64_t override;
 
-  [[nodiscard]] auto get_directory_items(const std::string &api_path,
-                                         directory_item_list &list) const
-      -> api_error override;
-
   [[nodiscard]] auto get_file(const std::string &api_path, api_file &file) const
       -> api_error override;
 

src/providers/base_provider.cpp

@@ -246,6 +246,49 @@ auto base_provider::get_api_path_from_source(const std::string &source_path,
   return api_error::item_not_found;
 }
 
+auto base_provider::get_directory_items(const std::string &api_path,
+                                        directory_item_list &list) const
+    -> api_error {
+  bool exists{};
+  auto res = is_directory(api_path, exists);
+  if (res != api_error::success) {
+    return res;
+  }
+  if (not exists) {
+    return api_error::directory_not_found;
+  }
+
+  try {
+    res = get_directory_items_impl(api_path, list);
+    if (res != api_error::success) {
+      return res;
+    }
+  } catch (const std::exception &e) {
+    utils::error::raise_api_path_error(__FUNCTION__, api_path, e,
+                                       "failed to get directory items");
+    return api_error::error;
+  }
+
+  std::sort(list.begin(), list.end(),
+            [](const auto &item1, const auto &item2) -> bool {
+              return (item1.directory && not item2.directory) ||
+                     (not(item2.directory && not item1.directory) &&
+                      (item1.api_path.compare(item2.api_path) < 0));
+            });
+
+  list.insert(list.begin(), directory_item{
+                                "..",
+                                "",
+                                true,
+                            });
+  list.insert(list.begin(), directory_item{
+                                ".",
+                                "",
+                                true,
+                            });
+  return api_error::success;
+}
+
 auto base_provider::get_file_size(const std::string &api_path,
                                   std::uint64_t &file_size) const -> api_error {
   bool exists{};

src/providers/s3/s3_provider.cpp

@@ -254,151 +254,120 @@ auto s3_provider::get_directory_item_count(const std::string &api_path) const
   return 0U;
 }
 
-auto s3_provider::get_directory_items(const std::string &api_path,
+auto s3_provider::get_directory_items_impl(const std::string &api_path,
-                                      directory_item_list &list) const
+                                           directory_item_list &list) const
     -> api_error {
-  try {
+  const auto cfg = get_config().get_s3_config();
-    bool exists{};
+  const auto is_encrypted = not cfg.encryption_token.empty();
-    auto res = is_directory(api_path, exists);
+
-    if (res != api_error::success) {
+  auto ret = api_error::success;
-      return res;
+  std::string key;
+  if (is_encrypted) {
+    ret = get_item_meta(api_path, META_KEY, key);
+    if (ret != api_error::success) {
+      return ret;
     }
-    if (not exists) {
-      return api_error::directory_not_found;
-    }
-
-    const auto cfg = get_config().get_s3_config();
-    const auto is_encrypted = not cfg.encryption_token.empty();
-    std::string key;
-    if (is_encrypted) {
-      res = get_item_meta(api_path, META_KEY, key);
-      if (res != api_error::success) {
-        return res;
-      }
-    }
-
-    const auto object_name =
-        api_path == "/"
-            ? ""
-            : utils::path::create_api_path(is_encrypted ? key : api_path);
-
-    std::string response_data{};
-    long response_code{};
-    auto prefix = object_name.empty() ? object_name : object_name + "/";
-
-    if (not get_object_list(response_data, response_code, "/", prefix)) {
-      return api_error::comm_error;
-    }
-
-    if (response_code == http_error_codes::not_found) {
-      return api_error::directory_not_found;
-    }
-
-    if (response_code != http_error_codes::ok) {
-      return api_error::comm_error;
-    }
-
-    pugi::xml_document doc;
-    auto parse_res = doc.load_string(response_data.c_str());
-    if (parse_res.status != pugi::xml_parse_status::status_ok) {
-      return api_error::error;
-    }
-
-    const auto add_directory_item =
-        [&](bool directory, const std::string &name,
-            std::function<std::uint64_t(const directory_item &)> get_size)
-        -> api_error {
-      auto child_api_path =
-          utils::path::create_api_path(utils::path::combine("/", {name}));
-      std::string child_object_name;
-      if (is_encrypted) {
-        child_object_name = child_api_path;
-        res = utils::encryption::decrypt_file_path(cfg.encryption_token,
-                                                   child_api_path);
-        if (res != api_error::success) {
-          return res;
-        }
-      }
-
-      directory_item dir_item{};
-      dir_item.api_path = child_api_path;
-      dir_item.api_parent = utils::path::get_parent_api_path(dir_item.api_path);
-      dir_item.directory = directory;
-      dir_item.size = get_size(dir_item);
-      res = get_item_meta(child_api_path, dir_item.meta);
-      if (res == api_error::item_not_found) {
-        if (directory) {
-          res = create_path_directories(child_api_path, child_object_name);
-          if (res != api_error::success) {
-            return res;
-          }
-        } else {
-          auto file =
-              create_api_file(child_api_path, child_object_name, dir_item.size);
-          res = add_if_not_found(file, child_object_name);
-          if (res != api_error::success) {
-            return res;
-          }
-        }
-
-        res = get_item_meta(child_api_path, dir_item.meta);
-      }
-
-      if (res != api_error::success) {
-        return res;
-      }
-
-      list.push_back(std::move(dir_item));
-      return api_error::success;
-    };
-
-    auto node_list =
-        doc.select_nodes("/ListBucketResult/CommonPrefixes/Prefix");
-    for (const auto &node : node_list) {
-      add_directory_item(
-          true, node.node().text().as_string(),
-          [](const directory_item &) -> std::uint64_t { return 0U; });
-    }
-
-    node_list = doc.select_nodes("/ListBucketResult/Contents");
-    for (const auto &node : node_list) {
-      auto child_object_name = utils::path::create_api_path(
-          node.node().select_node("Key").node().text().as_string());
-      if (child_object_name != utils::path::create_api_path(prefix)) {
-        auto size = node.node().select_node("Size").node().text().as_ullong();
-        add_directory_item(
-            false, child_object_name,
-            [&is_encrypted, &size](const directory_item &) -> std::uint64_t {
-              return is_encrypted ? utils::encryption::encrypting_reader::
-                                        calculate_decrypted_size(size)
-                                  : size;
-            });
-      }
-    }
-
-    std::sort(list.begin(), list.end(),
-              [](const auto &item1, const auto &item2) -> bool {
-                return (item1.directory && not item2.directory) ||
-                       (not(item2.directory && not item1.directory) &&
-                        (item1.api_path.compare(item2.api_path) < 0));
-              });
-
-    list.insert(list.begin(), directory_item{
-                                  "..",
-                                  "",
-                                  true,
-                              });
-    list.insert(list.begin(), directory_item{
-                                  ".",
-                                  "",
-                                  true,
-                              });
-    return api_error::success;
-  } catch (const std::exception &e) {
-    utils::error::raise_error(__FUNCTION__, e, "exception occurred");
   }
 
-  return api_error::error;
+  const auto object_name =
+      api_path == "/"
+          ? ""
+          : utils::path::create_api_path(is_encrypted ? key : api_path);
+
+  std::string response_data{};
+  long response_code{};
+  auto prefix = object_name.empty() ? object_name : object_name + "/";
+
+  if (not get_object_list(response_data, response_code, "/", prefix)) {
+    return api_error::comm_error;
+  }
+
+  if (response_code == http_error_codes::not_found) {
+    return api_error::directory_not_found;
+  }
+
+  if (response_code != http_error_codes::ok) {
+    return api_error::comm_error;
+  }
+
+  pugi::xml_document doc;
+  auto parse_res = doc.load_string(response_data.c_str());
+  if (parse_res.status != pugi::xml_parse_status::status_ok) {
+    return api_error::error;
+  }
+
+  const auto add_directory_item =
+      [&](bool directory, const std::string &name,
+          std::function<std::uint64_t(const directory_item &)> get_size)
+      -> api_error {
+    auto child_api_path =
+        utils::path::create_api_path(utils::path::combine("/", {name}));
+    std::string child_object_name;
+    if (is_encrypted) {
+      child_object_name = child_api_path;
+      ret = utils::encryption::decrypt_file_path(cfg.encryption_token,
+                                                 child_api_path);
+      if (ret != api_error::success) {
+        return ret;
+      }
+    }
+
+    directory_item dir_item{};
+    dir_item.api_path = child_api_path;
+    dir_item.api_parent = utils::path::get_parent_api_path(dir_item.api_path);
+    dir_item.directory = directory;
+    dir_item.size = get_size(dir_item);
+    ret = get_item_meta(child_api_path, dir_item.meta);
+    if (ret == api_error::item_not_found) {
+      if (directory) {
+        ret = create_path_directories(child_api_path, child_object_name);
+        if (ret != api_error::success) {
+          return ret;
+        }
+      } else {
+        auto file =
+            create_api_file(child_api_path, child_object_name, dir_item.size);
+        ret = add_if_not_found(file, child_object_name);
+        if (ret != api_error::success) {
+          return ret;
+        }
+      }
+
+      ret = get_item_meta(child_api_path, dir_item.meta);
+    }
+
+    if (ret != api_error::success) {
+      return ret;
+    }
+
+    list.push_back(std::move(dir_item));
+    return api_error::success;
+  };
+
+  auto node_list = doc.select_nodes("/ListBucketResult/CommonPrefixes/Prefix");
+  for (const auto &node : node_list) {
+    add_directory_item(
+        true, node.node().text().as_string(),
+        [](const directory_item &) -> std::uint64_t { return 0U; });
+  }
+
+  node_list = doc.select_nodes("/ListBucketResult/Contents");
+  for (const auto &node : node_list) {
+    auto child_object_name = utils::path::create_api_path(
+        node.node().select_node("Key").node().text().as_string());
+    if (child_object_name != utils::path::create_api_path(prefix)) {
+      auto size = node.node().select_node("Size").node().text().as_ullong();
+      add_directory_item(
+          false, child_object_name,
+          [&is_encrypted, &size](const directory_item &) -> std::uint64_t {
+            return is_encrypted ? utils::encryption::encrypting_reader::
+                                      calculate_decrypted_size(size)
+                                : size;
+          });
+    }
+  }
+
+  return ret;
 }
 
 auto s3_provider::get_file(const std::string &api_path, api_file &file) const

src/providers/sia/sia_provider.cpp

@@ -99,94 +99,61 @@ auto sia_provider::get_directory_item_count(const std::string &api_path) const
   return 0U;
 }
 
-auto sia_provider::get_directory_items(const std::string &api_path,
+auto sia_provider::get_directory_items_impl(const std::string &api_path,
-                                       directory_item_list &list) const
+                                            directory_item_list &list) const
     -> api_error {
-  bool exists{};
+  json object_list{};
-  auto res = is_directory(api_path, exists);
+  if (not get_object_list(api_path, object_list)) {
-  if (res != api_error::success) {
+    return api_error::comm_error;
-    return res;
-  }
-  if (not exists) {
-    return api_error::directory_not_found;
   }
 
-  try {
+  if (object_list.contains("entries")) {
-    json object_list{};
+    for (const auto &entry : object_list.at("entries")) {
-    if (not get_object_list(api_path, object_list)) {
+      try {
-      return api_error::comm_error;
+        auto name = entry.at("name").get<std::string>();
-    }
+        auto entry_api_path = utils::path::create_api_path(name);
 
-    if (object_list.contains("entries")) {
+        auto directory = utils::string::ends_with(name, "/");
-      for (const auto &entry : object_list.at("entries")) {
+        if (directory && (entry_api_path == api_path)) {
-        try {
+          continue;
-          auto name = entry.at("name").get<std::string>();
+        }
-          auto entry_api_path = utils::path::create_api_path(name);
 
-          auto directory = utils::string::ends_with(name, "/");
+        api_file file{};
-          if (directory && (entry_api_path == api_path)) {
+
+        api_meta_map meta{};
+        if (get_item_meta(entry_api_path, meta) == api_error::item_not_found) {
+          file = create_api_file(
+              entry_api_path, "",
+              directory ? 0U : entry["size"].get<std::uint64_t>());
+          get_api_item_added()(directory, file);
+          auto res = get_item_meta(entry_api_path, meta);
+          if (res != api_error::success) {
+            utils::error::raise_error(__FUNCTION__, res,
+                                      "failed to get item meta");
             continue;
           }
-
+        } else {
-          api_file file{};
+          file = create_api_file(
-
+              entry_api_path,
-          api_meta_map meta{};
+              directory ? 0U : entry["size"].get<std::uint64_t>(), meta);
-          if (get_item_meta(entry_api_path, meta) ==
-              api_error::item_not_found) {
-            file = create_api_file(
-                entry_api_path, "",
-                directory ? 0U : entry["size"].get<std::uint64_t>());
-            get_api_item_added()(directory, file);
-            res = get_item_meta(entry_api_path, meta);
-            if (res != api_error::success) {
-              utils::error::raise_error(__FUNCTION__, res,
-                                        "failed to get item meta");
-              continue;
-            }
-          } else {
-            file = create_api_file(
-                entry_api_path,
-                directory ? 0U : entry["size"].get<std::uint64_t>(), meta);
-          }
-
-          directory_item dir_item{};
-          dir_item.api_parent = file.api_parent;
-          dir_item.api_path = file.api_path;
-          dir_item.directory = directory;
-          dir_item.meta = meta;
-          dir_item.resolved = true;
-          dir_item.size = file.file_size;
-          list.emplace_back(std::move(dir_item));
-        } catch (const std::exception &e) {
-          utils::error::raise_api_path_error(__FUNCTION__, api_path, e,
-                                             "failed to process entry|" +
-                                                 entry.dump());
         }
+
+        directory_item dir_item{};
+        dir_item.api_parent = file.api_parent;
+        dir_item.api_path = file.api_path;
+        dir_item.directory = directory;
+        dir_item.meta = meta;
+        dir_item.resolved = true;
+        dir_item.size = file.file_size;
+        list.emplace_back(std::move(dir_item));
+      } catch (const std::exception &e) {
+        utils::error::raise_api_path_error(__FUNCTION__, api_path, e,
+                                           "failed to process entry|" +
+                                               entry.dump());
       }
     }
-  } catch (const std::exception &e) {
-    utils::error::raise_api_path_error(__FUNCTION__, api_path, e,
-                                       "failed to get directory items");
-    return api_error::error;
   }
 
-  std::sort(list.begin(), list.end(),
-            [](const auto &item1, const auto &item2) -> bool {
-              return (item1.directory && not item2.directory) ||
-                     (not(item2.directory && not item1.directory) &&
-                      (item1.api_path.compare(item2.api_path) < 0));
-            });
-
-  list.insert(list.begin(), directory_item{
-                                "..",
-                                "",
-                                true,
-                            });
-  list.insert(list.begin(), directory_item{
-                                ".",
-                                "",
-                                true,
-                            });
   return api_error::success;
 }