cleanup

.cspell/words.txt

@@ -26,6 +26,7 @@ cppvsdbg
 create_notraverse
 crypto_aead_xchacha20poly1305_ietf_npubbytes
 cstdint
+curlopt_aws_sigv4
 cxxflags
 cxxstd
 d_largefile64_source

CHANGELOG.md

@@ -8,8 +8,10 @@
 * ~~\#20 Add support to evict open files~~
 * ~~\#21 \[Unit Test \] Complete WinFSP unit tests~~
 * ~~\#22 \[Unit Test\] Complete FUSE unit tests~~
-* ~~\#28 \[bug\] Address slow directory responses in S3 mounts for deep nested directories~~
-* ~~\#29 \[bug\] S3 error responses are not being logged~~
+* ~~\#28 \[bug\] Address slow directory responses in S3 mounts for deeply nested directories~~
+* \#29 \[bug\] S3 error responses are not being logged
+* \#30 \[bug\] Sia provider error responses are not logged
+* ~~\#31 \[bug\] S3 provider should limit max key size to 1024~~
 
 ### Changes from v2.0.2-rc
 
@@ -17,6 +19,7 @@
 * Updated build system to MinGW-w64 12.0.0
 * Updated copyright to 2018-2025
 * Fixed invalid directory nullptr error on remote mounts
+* Fixed http headers not being added for requests
 
 ## v2.0.2-rc
 

repertory/librepertory/include/comm/curl/curl_comm.hpp

@@ -61,13 +61,14 @@ public:
   [[nodiscard]] static auto reset_curl(CURL *curl_handle) -> CURL *;
 
 public:
-  [[nodiscard]] static auto
-  construct_url(CURL *curl, const std::string &relative_path,
-                const host_config &cfg) -> std::string;
+  [[nodiscard]] static auto construct_url(CURL *curl,
+                                          const std::string &relative_path,
+                                          const host_config &cfg)
+      -> std::string;
 
-  [[nodiscard]] static auto
-  create_host_config(const s3_config &cfg,
-                     bool use_s3_path_style) -> host_config;
+  [[nodiscard]] static auto create_host_config(const s3_config &cfg,
+                                               bool use_s3_path_style)
+      -> host_config;
 
   [[nodiscard]] static auto url_encode(CURL *curl, const std::string &data,
                                        bool allow_slash) -> std::string;
@@ -75,8 +76,8 @@ public:
   template <typename request_type>
   [[nodiscard]] static auto
   make_encrypted_request(const host_config &cfg, const request_type &request,
-                         long &response_code,
-                         stop_type &stop_requested) -> bool {
+                         long &response_code, stop_type &stop_requested)
+      -> bool {
     response_code = 0;
 
     if (not request.decryption_token.has_value() ||
@@ -193,6 +194,16 @@ public:
                        request.aws_service.value().c_str());
     }
 
+    curl_slist *header_list{nullptr};
+    if (not request.headers.empty()) {
+      for (const auto &header : request.headers) {
+        header_list = curl_slist_append(
+            header_list,
+            fmt::format("{}: {}", header.first, header.second).c_str());
+      }
+      curl_easy_setopt(curl, CURLOPT_HTTPHEADER, header_list);
+    }
+
     auto url = construct_url(curl, request.get_path(), cfg) + parameters;
     curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
 
@@ -200,6 +211,11 @@ public:
 
     CURLcode curl_code{};
     curl_request.get_result(curl_code, response_code);
+
+    if (header_list != nullptr) {
+      curl_slist_free_all(header_list);
+    }
+
     if (curl_code != CURLE_OK) {
       event_system::instance().raise<curl_error>(url, curl_code);
       return false;
@@ -215,26 +231,30 @@ public:
 public:
   void enable_s3_path_style(bool enable) override;
 
-  [[nodiscard]] auto
-  make_request(const curl::requests::http_delete &del, long &response_code,
-               stop_type &stop_requested) const -> bool override;
-
-  [[nodiscard]] auto
-  make_request(const curl::requests::http_get &get, long &response_code,
-               stop_type &stop_requested) const -> bool override;
-
-  [[nodiscard]] auto
-  make_request(const curl::requests::http_head &head, long &response_code,
-               stop_type &stop_requested) const -> bool override;
-
-  [[nodiscard]] auto
-  make_request(const curl::requests::http_post &post_file, long &response_code,
-               stop_type &stop_requested) const -> bool override;
-
-  [[nodiscard]] auto
-  make_request(const curl::requests::http_put_file &put_file,
+  [[nodiscard]] auto make_request(const curl::requests::http_delete &del,
                                   long &response_code,
-               stop_type &stop_requested) const -> bool override;
+                                  stop_type &stop_requested) const
+      -> bool override;
+
+  [[nodiscard]] auto make_request(const curl::requests::http_get &get,
+                                  long &response_code,
+                                  stop_type &stop_requested) const
+      -> bool override;
+
+  [[nodiscard]] auto make_request(const curl::requests::http_head &head,
+                                  long &response_code,
+                                  stop_type &stop_requested) const
+      -> bool override;
+
+  [[nodiscard]] auto make_request(const curl::requests::http_post &post_file,
+                                  long &response_code,
+                                  stop_type &stop_requested) const
+      -> bool override;
+
+  [[nodiscard]] auto make_request(const curl::requests::http_put_file &put_file,
+                                  long &response_code,
+                                  stop_type &stop_requested) const
+      -> bool override;
 };
 } // namespace repertory
 

repertory/librepertory/include/comm/curl/requests/http_post.hpp

@@ -32,15 +32,15 @@ struct http_post final : http_request_base {
   auto operator=(const http_post &) -> http_post & = default;
   auto operator=(http_post &&) -> http_post & = default;
 
-  ~http_post() override;
+  ~http_post() override = default;
 
   std::optional<nlohmann::json> json;
 
-  [[nodiscard]] auto
-  set_method(CURL *curl, stop_type & /*stop_requested*/) const -> bool override;
+  [[nodiscard]] auto set_method(CURL *curl,
+                                stop_type & /*stop_requested*/) const
+      -> bool override;
 
 private:
-  mutable curl_slist *headers{nullptr};
   mutable std::optional<std::string> json_str;
 };
 } // namespace repertory::curl::requests

repertory/librepertory/include/comm/curl/requests/http_request_base.hpp

@@ -51,7 +51,7 @@ struct http_request_base {
   bool allow_timeout{};
   std::optional<std::string> aws_service;
   std::optional<std::string> decryption_token{};
-  http_headers headers{};
+  mutable http_headers headers{};
   std::string path{};
   http_query_parameters query{};
   std::optional<http_range> range{};

repertory/librepertory/include/providers/s3/s3_provider.hpp

@@ -54,14 +54,14 @@ private:
                                       const std::string &object_name) const
       -> api_error;
 
+  [[nodiscard]] auto create_directory_paths(const std::string &api_path,
+                                            const std::string &key) const
+      -> api_error;
+
   [[nodiscard]] auto create_file_extra(const std::string &api_path,
                                        api_meta_map &meta)
       -> api_error override;
 
-  [[nodiscard]] auto create_path_directories(const std::string &api_path,
-                                             const std::string &key) const
-      -> api_error;
-
   [[nodiscard]] auto decrypt_object_name(std::string &object_name) const
       -> api_error;
 

repertory/librepertory/include/rpc/server/server.hpp

@@ -41,9 +41,6 @@ private:
   std::atomic<bool> started_ = false;
 
 private:
-  // [[nodiscard]] bool check_authorization(const httpserver::http_request
-  // &request);
-
   [[nodiscard]] auto check_authorization(const httplib::Request &req) -> bool;
 
   void handle_get_config(const httplib::Request &req, httplib::Response &res);

repertory/librepertory/include/types/repertory.hpp

@@ -41,6 +41,7 @@ constexpr const auto default_task_wait_ms{100U};
 constexpr const auto default_timeout_ms{60000U};
 constexpr const auto max_orphaned_file_retention_days{std::uint16_t(31U)};
 constexpr const auto max_ring_buffer_file_size{std::uint16_t(1024U)};
+constexpr const auto max_s3_object_name_length{1024U};
 constexpr const auto min_cache_size_bytes{
     std::uint64_t(100UL * 1024UL * 1024UL)};
 constexpr const auto min_download_timeout_secs{std::uint8_t(5U)};
@@ -200,6 +201,7 @@ enum class api_error {
   item_exists,
   item_not_found,
   more_data,
+  name_too_long,
   no_disk_space,
   not_implemented,
   not_supported,
@@ -216,31 +218,33 @@ enum class api_error {
 
 [[nodiscard]] auto api_error_from_string(std::string_view str) -> api_error;
 
-[[nodiscard]] auto
-api_error_to_string(const api_error &error) -> const std::string &;
+[[nodiscard]] auto api_error_to_string(const api_error &error)
+    -> const std::string &;
 
 enum class database_type {
   rocksdb,
   sqlite,
 };
-[[nodiscard]] auto database_type_from_string(
-    std::string type,
-    database_type default_type = database_type::rocksdb) -> database_type;
-
 [[nodiscard]] auto
-database_type_to_string(const database_type &type) -> std::string;
+database_type_from_string(std::string type,
+                          database_type default_type = database_type::rocksdb)
+    -> database_type;
+
+[[nodiscard]] auto database_type_to_string(const database_type &type)
+    -> std::string;
 
 enum class download_type {
   default_,
   direct,
   ring_buffer,
 };
-[[nodiscard]] auto download_type_from_string(
-    std::string type,
-    download_type default_type = download_type::default_) -> download_type;
-
 [[nodiscard]] auto
-download_type_to_string(const download_type &type) -> std::string;
+download_type_from_string(std::string type,
+                          download_type default_type = download_type::default_)
+    -> download_type;
+
+[[nodiscard]] auto download_type_to_string(const download_type &type)
+    -> std::string;
 
 enum class exit_code : std::int32_t {
   success = 0,

repertory/librepertory/src/comm/curl/requests/http_post.cpp

@@ -22,18 +22,11 @@
 #include "comm/curl/requests/http_post.hpp"
 
 namespace repertory::curl::requests {
-http_post::~http_post() {
-  if (headers != nullptr) {
-    curl_slist_free_all(headers);
-  }
-}
-
 auto http_post::set_method(CURL *curl, stop_type & /*stop_requested*/) const
     -> bool {
   curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "POST");
   if (json.has_value()) {
-    headers = curl_slist_append(headers, "content-type: application/json");
-    curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
+    headers["content-type"] = "application/json";
 
     json_str = json->dump();
     curl_easy_setopt(curl, CURLOPT_POSTFIELDS, json_str->c_str());

repertory/librepertory/src/drives/fuse/fuse_drive.cpp

@@ -81,8 +81,8 @@ auto fuse_drive::chown_impl(std::string api_path, uid_t uid, gid_t gid,
                             struct fuse_file_info * /*file_info*/)
     -> api_error {
 #else
-auto fuse_drive::chown_impl(std::string api_path, uid_t uid,
-                            gid_t gid) -> api_error {
+auto fuse_drive::chown_impl(std::string api_path, uid_t uid, gid_t gid)
+    -> api_error {
 #endif
   return check_and_perform(
       api_path, X_OK, [&](api_meta_map &meta) -> api_error {
@@ -363,13 +363,8 @@ auto fuse_drive::fgetattr_impl(std::string api_path, struct stat *unix_st,
     return res;
   }
 
-  bool directory{};
-  res = provider_.is_directory(api_path, directory);
-  if (res != api_error::success) {
-    return res;
-  }
   fuse_drive_base::populate_stat(api_path, open_file->get_file_size(), meta,
-                                 directory, provider_, unix_st);
+                                 open_file->is_directory(), provider_, unix_st);
 
   return api_error::success;
 }
@@ -481,8 +476,8 @@ auto fuse_drive::getattr_impl(std::string api_path, struct stat *unix_st,
                               struct fuse_file_info * /*file_info*/)
     -> api_error {
 #else
-auto fuse_drive::getattr_impl(std::string api_path,
-                              struct stat *unix_st) -> api_error {
+auto fuse_drive::getattr_impl(std::string api_path, struct stat *unix_st)
+    -> api_error {
 #endif
   auto parent = utils::path::get_parent_api_path(api_path);
 
@@ -508,14 +503,9 @@ auto fuse_drive::getattr_impl(std::string api_path,
       return res;
     }
 
-    bool directory{};
-    res = provider_.is_directory(api_path, directory);
-    if (res != api_error::success) {
-      return res;
-    }
-    fuse_drive_base::populate_stat(api_path,
-                                   utils::string::to_uint64(meta[META_SIZE]),
-                                   meta, directory, provider_, unix_st);
+    fuse_drive_base::populate_stat(
+        api_path, utils::string::to_uint64(meta[META_SIZE]), meta,
+        utils::string::to_bool(meta[META_DIRECTORY]), provider_, unix_st);
   }
 
   return api_error::success;
@@ -565,8 +555,8 @@ auto fuse_drive::getxtimes_impl(std::string api_path, struct timespec *bkuptime,
 #endif // __APPLE__
 
 #if FUSE_USE_VERSION >= 30
-auto fuse_drive::init_impl(struct fuse_conn_info *conn,
-                           struct fuse_config *cfg) -> void * {
+auto fuse_drive::init_impl(struct fuse_conn_info *conn, struct fuse_config *cfg)
+    -> void * {
 #else
 void *fuse_drive::init_impl(struct fuse_conn_info *conn) {
 #endif
@@ -803,8 +793,9 @@ auto fuse_drive::release_impl(std::string /*api_path*/,
   return api_error::success;
 }
 
-auto fuse_drive::releasedir_impl(
-    std::string /*api_path*/, struct fuse_file_info *file_info) -> api_error {
+auto fuse_drive::releasedir_impl(std::string /*api_path*/,
+                                 struct fuse_file_info *file_info)
+    -> api_error {
   auto iter = directory_cache_->get_directory(file_info->fh);
   if (iter == nullptr) {
     return api_error::invalid_handle;
@@ -822,8 +813,8 @@ auto fuse_drive::rename_directory(const std::string &from_api_path,
 }
 
 auto fuse_drive::rename_file(const std::string &from_api_path,
-                             const std::string &to_api_path,
-                             bool overwrite) -> int {
+                             const std::string &to_api_path, bool overwrite)
+    -> int {
   auto res = fm_->rename_file(from_api_path, to_api_path, overwrite);
   errno = std::abs(utils::from_api_error(res));
   return (res == api_error::success) ? 0 : -1;
@@ -833,8 +824,8 @@ auto fuse_drive::rename_file(const std::string &from_api_path,
 auto fuse_drive::rename_impl(std::string from_api_path, std::string to_api_path,
                              unsigned int /*flags*/) -> api_error {
 #else
-auto fuse_drive::rename_impl(std::string from_api_path,
-                             std::string to_api_path) -> api_error {
+auto fuse_drive::rename_impl(std::string from_api_path, std::string to_api_path)
+    -> api_error {
 #endif
   auto res = check_parent_access(to_api_path, W_OK | X_OK);
   if (res != api_error::success) {
@@ -948,15 +939,15 @@ auto fuse_drive::getxattr_impl(std::string api_path, const char *name,
 }
 #else  // __APPLE__
 auto fuse_drive::getxattr_impl(std::string api_path, const char *name,
-                               char *value, size_t size,
-                               int &attribute_size) -> api_error {
+                               char *value, size_t size, int &attribute_size)
+    -> api_error {
   return getxattr_common(api_path, name, value, size, attribute_size, nullptr);
 }
 #endif // __APPLE__
 
 auto fuse_drive::listxattr_impl(std::string api_path, char *buffer, size_t size,
-                                int &required_size,
-                                bool &return_size) -> api_error {
+                                int &required_size, bool &return_size)
+    -> api_error {
   auto check_size = (size == 0);
 
   auto res = check_parent_access(api_path, X_OK);
@@ -996,8 +987,8 @@ auto fuse_drive::listxattr_impl(std::string api_path, char *buffer, size_t size,
   return res;
 }
 
-auto fuse_drive::removexattr_impl(std::string api_path,
-                                  const char *name) -> api_error {
+auto fuse_drive::removexattr_impl(std::string api_path, const char *name)
+    -> api_error {
   std::string attribute_name;
 #if defined(__APPLE__)
   auto res = parse_xattr_parameters(name, 0, attribute_name, api_path);
@@ -1025,8 +1016,8 @@ auto fuse_drive::setxattr_impl(std::string api_path, const char *name,
                                uint32_t position) -> api_error {
 #else // __APPLE__
 auto fuse_drive::setxattr_impl(std::string api_path, const char *name,
-                               const char *value, size_t size,
-                               int flags) -> api_error {
+                               const char *value, size_t size, int flags)
+    -> api_error {
 #endif
   std::string attribute_name;
 #if defined(__APPLE__)
@@ -1104,8 +1095,8 @@ void fuse_drive::set_item_meta(const std::string &api_path,
 }
 
 #if defined(__APPLE__)
-auto fuse_drive::setattr_x_impl(std::string api_path,
-                                struct setattr_x *attr) -> api_error {
+auto fuse_drive::setattr_x_impl(std::string api_path, struct setattr_x *attr)
+    -> api_error {
   bool exists{};
   auto res = provider_.is_file(api_path, exists);
   if (res != api_error::success) {
@@ -1159,7 +1150,7 @@ auto fuse_drive::setattr_x_impl(std::string api_path,
       ts[0].tv_sec = attr->acctime.tv_sec;
       ts[0].tv_nsec = attr->acctime.tv_nsec;
     } else {
-      struct timeval tv {};
+      struct timeval tv{};
       gettimeofday(&tv, NULL);
       ts[0].tv_sec = tv.tv_sec;
       ts[0].tv_nsec = tv.tv_usec * 1000;
@@ -1204,8 +1195,9 @@ auto fuse_drive::setattr_x_impl(std::string api_path,
   return api_error::success;
 }
 
-auto fuse_drive::setbkuptime_impl(
-    std::string api_path, const struct timespec *bkuptime) -> api_error {
+auto fuse_drive::setbkuptime_impl(std::string api_path,
+                                  const struct timespec *bkuptime)
+    -> api_error {
   return check_and_perform(
       api_path, X_OK, [&](api_meta_map &meta) -> api_error {
         auto nanos = bkuptime->tv_nsec +
@@ -1241,8 +1233,8 @@ auto fuse_drive::setvolname_impl(const char * /*volname*/) -> api_error {
   return api_error::success;
 }
 
-auto fuse_drive::statfs_x_impl(std::string /*api_path*/,
-                               struct statfs *stbuf) -> api_error {
+auto fuse_drive::statfs_x_impl(std::string /*api_path*/, struct statfs *stbuf)
+    -> api_error {
   if (statfs(&config_.get_cache_directory()[0], stbuf) != 0) {
     return api_error::os_error;
   }
@@ -1267,8 +1259,8 @@ auto fuse_drive::statfs_x_impl(std::string /*api_path*/,
   return api_error::success;
 }
 #else  // __APPLE__
-auto fuse_drive::statfs_impl(std::string /*api_path*/,
-                             struct statvfs *stbuf) -> api_error {
+auto fuse_drive::statfs_impl(std::string /*api_path*/, struct statvfs *stbuf)
+    -> api_error {
   if (statvfs(config_.get_cache_directory().data(), stbuf) != 0) {
     return api_error::os_error;
   }
@@ -1352,8 +1344,8 @@ auto fuse_drive::utimens_impl(std::string api_path, const struct timespec tv[2],
                               struct fuse_file_info * /*file_info*/)
     -> api_error {
 #else
-auto fuse_drive::utimens_impl(std::string api_path,
-                              const struct timespec tv[2]) -> api_error {
+auto fuse_drive::utimens_impl(std::string api_path, const struct timespec tv[2])
+    -> api_error {
 #endif
   api_meta_map meta;
   auto res = provider_.get_item_meta(api_path, meta);

repertory/librepertory/src/drives/winfsp/winfsp_drive.cpp

@@ -896,10 +896,7 @@ auto winfsp_drive::ReadDirectory(PVOID /*file_node*/, PVOID file_desc,
     std::shared_ptr<i_open_file> file;
     if (fm_->get_open_file(handle, false, file)) {
       api_path = file->get_api_path();
-      bool exists{};
-      error = provider_.is_directory(api_path, exists);
-      if (error == api_error::success) {
-        if (exists) {
+      if (file->is_directory()) {
         directory_item_list list{};
         error = provider_.get_directory_items(api_path, list);
         if (error == api_error::success) {
@@ -917,8 +914,8 @@ auto winfsp_drive::ReadDirectory(PVOID /*file_node*/, PVOID file_desc,
                     : iterator.get_next_directory_offset(
                           utils::path::create_api_path(utils::path::combine(
                               api_path, {utils::string::to_utf8(marker)})));
-              while ((error = iterator.get_directory_item(
-                          offset++, dir_item)) == api_error::success) {
+            while ((error = iterator.get_directory_item(offset++, dir_item)) ==
+                   api_error::success) {
               if (dir_item.api_path == ".") {
                 auto res = provider_.get_item_meta(api_path, dir_item.meta);
                 if (res != api_error::success) {
@@ -931,8 +928,7 @@ auto winfsp_drive::ReadDirectory(PVOID /*file_node*/, PVOID file_desc,
               } else if (dir_item.api_path == "..") {
                 // TODO handle '/' parent
                 auto res = provider_.get_item_meta(
-                      utils::path::get_parent_api_path(api_path),
-                      dir_item.meta);
+                    utils::path::get_parent_api_path(api_path), dir_item.meta);
                 if (res != api_error::success) {
                   error = res;
                   utils::error::raise_api_path_error(function_name,
@@ -971,8 +967,8 @@ auto winfsp_drive::ReadDirectory(PVOID /*file_node*/, PVOID file_desc,
               ::wcscpy_s(&directory_info->FileNameBuf[0U],
                          repertory::max_path_length, display_name.data());
 
-                FspFileSystemFillDirectoryBuffer(
-                    directory_buffer, directory_info, &status_result);
+              FspFileSystemFillDirectoryBuffer(directory_buffer, directory_info,
+                                               &status_result);
             }
 
             FspFileSystemReleaseDirectoryBuffer(directory_buffer);
@@ -980,8 +976,7 @@ auto winfsp_drive::ReadDirectory(PVOID /*file_node*/, PVOID file_desc,
 
           if (status_result == STATUS_SUCCESS) {
             FspFileSystemReadDirectoryBuffer(directory_buffer, marker, buffer,
-                                               buffer_length,
-                                               bytes_transferred);
+                                             buffer_length, bytes_transferred);
             if (error == api_error::directory_end_of_files) {
               error = api_error::success;
             }
@@ -995,7 +990,6 @@ auto winfsp_drive::ReadDirectory(PVOID /*file_node*/, PVOID file_desc,
       }
     }
   }
-  }
 
   auto ret = utils::from_api_error(error);
   RAISE_WINFSP_EVENT(function_name, api_path, ret);

repertory/librepertory/src/providers/s3/s3_provider.cpp

@@ -39,24 +39,39 @@
 #include "utils/string.hpp"
 #include "utils/time.hpp"
 
+namespace {
+[[nodiscard]] auto set_request_path(auto &request,
+                                    const std::string &object_name)
+    -> repertory::api_error {
+  request.path = object_name;
+  if (request.path.substr(1U).size() > repertory::max_s3_object_name_length) {
+    return repertory::api_error::name_too_long;
+  }
+
+  return repertory::api_error::success;
+}
+} // namespace
+
 namespace repertory {
 s3_provider::s3_provider(app_config &config, i_http_comm &comm)
     : base_provider(config, comm) {}
 
-auto s3_provider::add_if_not_found(
-    api_file &file, const std::string &object_name) const -> api_error {
+auto s3_provider::add_if_not_found(api_file &file,
+                                   const std::string &object_name) const
+    -> api_error {
   api_meta_map meta{};
-  if (get_item_meta(file.api_path, meta) == api_error::item_not_found) {
-    auto err = create_path_directories(
-        file.api_parent, utils::path::get_parent_api_path(object_name));
-    if (err != api_error::success) {
-      return err;
+  auto res = get_item_meta(file.api_path, meta);
+  if (res == api_error::item_not_found) {
+    res = create_directory_paths(file.api_parent,
+                                 utils::path::get_parent_api_path(object_name));
+    if (res != api_error::success) {
+      return res;
     }
 
     get_api_item_added()(false, file);
   }
 
-  return api_error::success;
+  return res;
 }
 
 auto s3_provider::convert_api_date(std::string_view date) -> std::uint64_t {
@@ -70,7 +85,7 @@ auto s3_provider::convert_api_date(std::string_view date) -> std::uint64_t {
                 utils::string::split(date_parts.at(1U), 'Z', true).at(0U)) *
                 1000000UL;
 
-  struct tm tm1 {};
+  struct tm tm1{};
 #if defined(_WIN32)
   utils::time::strptime(date_time.c_str(), "%Y-%m-%dT%T", &tm1);
   return nanos + utils::time::windows_time_t_to_unix_time(_mkgmtime(&tm1));
@@ -113,17 +128,22 @@ auto s3_provider::create_directory_impl(const std::string &api_path,
       utils::path::create_api_path(is_encrypted ? meta[META_KEY] : api_path);
 
   std::string response_data;
-  curl::requests::http_put_file put_file{};
-  put_file.allow_timeout = true;
-  put_file.aws_service = "aws:amz:" + cfg.region + ":s3";
-  put_file.path = object_name + '/';
-  put_file.response_handler = [&response_data](auto &&data,
+  curl::requests::http_put_file put_dir{};
+  put_dir.allow_timeout = true;
+  put_dir.aws_service = "aws:amz:" + cfg.region + ":s3";
+  put_dir.response_handler = [&response_data](auto &&data,
+
                                               long /*response_code*/) {
     response_data = std::string(data.begin(), data.end());
   };
 
+  auto res = set_request_path(put_dir, object_name + '/');
+  if (res != api_error::success) {
+    return res;
+  }
+
   long response_code{};
-  if (not get_comm().make_request(put_file, response_code, stop_requested)) {
+  if (not get_comm().make_request(put_dir, response_code, stop_requested)) {
     utils::error::raise_api_path_error(function_name, api_path,
                                        api_error::comm_error,
                                        "failed to create directory");
@@ -140,36 +160,9 @@ auto s3_provider::create_directory_impl(const std::string &api_path,
   return api_error::success;
 }
 
-auto s3_provider::create_file_extra(const std::string &api_path,
-                                    api_meta_map &meta) -> api_error {
-  REPERTORY_USES_FUNCTION_NAME();
-
-  const auto &cfg = get_s3_config();
-  if (not cfg.encryption_token.empty()) {
-    std::string encrypted_file_path;
-    auto res = get_item_meta(utils::path::get_parent_api_path(api_path),
-                             META_KEY, encrypted_file_path);
-    if (res != api_error::success) {
-      utils::error::raise_api_path_error(function_name, api_path, res,
-                                         "failed to create file");
-      return res;
-    }
-
-    data_buffer result;
-    utils::encryption::encrypt_data(
-        cfg.encryption_token,
-        *(utils::string::split(api_path, '/', false).end() - 1U), result);
-
-    meta[META_KEY] = utils::path::create_api_path(
-        utils::path::combine(utils::path::create_api_path(encrypted_file_path),
-                             {utils::collection::to_hex_string(result)}));
-  }
-
-  return api_error::success;
-}
-
-auto s3_provider::create_path_directories(
-    const std::string &api_path, const std::string &key) const -> api_error {
+auto s3_provider::create_directory_paths(const std::string &api_path,
+                                         const std::string &key) const
+    -> api_error {
   REPERTORY_USES_FUNCTION_NAME();
 
   if (api_path == "/") {
@@ -215,18 +208,23 @@ auto s3_provider::create_path_directories(
 
       if (not exists) {
         std::string response_data;
-        curl::requests::http_put_file put_file{};
-        put_file.allow_timeout = true;
-        put_file.aws_service = "aws:amz:" + cfg.region + ":s3";
-        put_file.path = (is_encrypted ? cur_key : cur_path) + '/';
-        put_file.response_handler = [&response_data](auto &&data,
+        curl::requests::http_put_file put_dir{};
+        put_dir.allow_timeout = true;
+        put_dir.aws_service = "aws:amz:" + cfg.region + ":s3";
+        put_dir.response_handler = [&response_data](auto &&data,
                                                     long /*response_code*/) {
           response_data = std::string(data.begin(), data.end());
         };
 
+        res = set_request_path(put_dir,
+                               (is_encrypted ? cur_key : cur_path) + '/');
+        if (res != api_error::success) {
+          return res;
+        }
+
         stop_type stop_requested{false};
         long response_code{};
-        if (not get_comm().make_request(put_file, response_code,
+        if (not get_comm().make_request(put_dir, response_code,
                                         stop_requested)) {
           utils::error::raise_api_path_error(
               function_name, cur_path, api_error::comm_error,
@@ -256,6 +254,34 @@ auto s3_provider::create_path_directories(
   return api_error::success;
 }
 
+auto s3_provider::create_file_extra(const std::string &api_path,
+                                    api_meta_map &meta) -> api_error {
+  REPERTORY_USES_FUNCTION_NAME();
+
+  const auto &cfg = get_s3_config();
+  if (not cfg.encryption_token.empty()) {
+    std::string encrypted_file_path;
+    auto res = get_item_meta(utils::path::get_parent_api_path(api_path),
+                             META_KEY, encrypted_file_path);
+    if (res != api_error::success) {
+      utils::error::raise_api_path_error(function_name, api_path, res,
+                                         "failed to create file");
+      return res;
+    }
+
+    data_buffer result;
+    utils::encryption::encrypt_data(
+        cfg.encryption_token,
+        *(utils::string::split(api_path, '/', false).end() - 1U), result);
+
+    meta[META_KEY] = utils::path::create_api_path(
+        utils::path::combine(utils::path::create_api_path(encrypted_file_path),
+                             {utils::collection::to_hex_string(result)}));
+  }
+
+  return api_error::success;
+}
+
 auto s3_provider::decrypt_object_name(std::string &object_name) const
     -> api_error {
   auto parts = utils::string::split(object_name, '/', false);
@@ -347,34 +373,99 @@ auto s3_provider::get_directory_item_count(const std::string &api_path) const
   return 0U;
 }
 
-auto s3_provider::get_directory_items_impl(
-    const std::string &api_path, directory_item_list &list) const -> api_error {
+auto s3_provider::get_directory_items_impl(const std::string &api_path,
+                                           directory_item_list &list) const
+    -> api_error {
   REPERTORY_USES_FUNCTION_NAME();
 
-  const auto &cfg = get_s3_config();
-  auto is_encrypted = not cfg.encryption_token.empty();
+  const auto &cfg{get_s3_config()};
+  auto is_encrypted{not cfg.encryption_token.empty()};
+
+  const auto add_diretory_item = [this, &cfg, &is_encrypted, &list](
+                                     const std::string &child_object_name,
+                                     bool directory, auto node) -> api_error {
+    auto res{api_error::success};
+
+    directory_item dir_item{};
+    dir_item.api_path = child_object_name;
+    dir_item.directory = directory;
+    if (is_encrypted) {
+      if (not utils::encryption::decrypt_file_path(cfg.encryption_token,
+                                                   dir_item.api_path)) {
+        return api_error::decryption_error;
+      }
+    }
+    dir_item.api_path = utils::path::create_api_path(dir_item.api_path);
+    dir_item.api_parent = utils::path::get_parent_api_path(dir_item.api_path);
+
+    if (directory) {
+      res = get_item_meta(dir_item.api_path, dir_item.meta);
+      if (res == api_error::item_not_found) {
+        res = create_directory_paths(dir_item.api_path, child_object_name);
+      }
+    } else {
+      std::string size_str;
+      if (get_item_meta(dir_item.api_path, META_SIZE, size_str) ==
+          api_error::success) {
+        dir_item.size = utils::string::to_uint64(size_str);
+      } else {
+        auto size = node.select_node("Size").node().text().as_ullong();
+
+        dir_item.size = is_encrypted ? utils::encryption::encrypting_reader::
+                                           calculate_decrypted_size(size)
+                                     : size;
+      }
+
+      res = get_item_meta(dir_item.api_path, dir_item.meta);
+      if (res == api_error::item_not_found) {
+        auto last_modified = convert_api_date(
+            node.select_node("LastModified").node().text().as_string());
+
+        api_file file{};
+        file.api_path = dir_item.api_path;
+        file.api_parent = dir_item.api_parent;
+        file.accessed_date = file.changed_date = file.creation_date =
+            file.modified_date = last_modified;
+        file.file_size = dir_item.size;
+        if (is_encrypted) {
+          file.key = child_object_name;
+        }
+
+        res = add_if_not_found(file, child_object_name);
+      }
+    }
+
+    if (res != api_error::success) {
+      return res;
+    }
+
+    list.push_back(std::move(dir_item));
+    return api_error::success;
+  };
 
-  auto ret = api_error::success;
   std::string key;
   if (is_encrypted) {
-    ret = get_item_meta(api_path, META_KEY, key);
-    if (ret != api_error::success) {
-      return ret;
+    auto res = get_item_meta(api_path, META_KEY, key);
+    if (res != api_error::success) {
+      return res;
     }
   }
 
-  auto object_name =
+  auto object_name{
       api_path == "/"
           ? ""
-          : utils::path::create_api_path(is_encrypted ? key : api_path);
+          : utils::path::create_api_path(is_encrypted ? key : api_path),
+  };
 
-  std::string response_data{};
-  long response_code{};
-  auto prefix = object_name.empty() ? object_name : object_name + "/";
+  auto prefix{
+      object_name.empty() ? object_name : object_name + "/",
+  };
 
   auto grab_more{true};
   std::string token{};
   while (grab_more) {
+    std::string response_data{};
+    long response_code{};
     if (not get_object_list(response_data, response_code, "/", prefix, token)) {
       return api_error::comm_error;
     }
@@ -411,34 +502,10 @@ auto s3_provider::get_directory_items_impl(
     for (const auto &node : node_list) {
       auto child_object_name = utils::path::create_api_path(
           utils::path::combine("/", {node.node().text().as_string()}));
-      directory_item dir_item{};
-      dir_item.api_path = child_object_name;
-      if (is_encrypted) {
-        if (not utils::encryption::decrypt_file_path(cfg.encryption_token,
-                                                     dir_item.api_path)) {
-          return api_error::decryption_error;
-        }
-      }
-      dir_item.api_path = utils::path::create_api_path(dir_item.api_path);
-      dir_item.api_parent = utils::path::get_parent_api_path(dir_item.api_path);
-      dir_item.directory = true;
-      dir_item.size = 0U;
-
-      auto res = get_item_meta(dir_item.api_path, dir_item.meta);
-      if (res == api_error::item_not_found) {
-        res = create_path_directories(dir_item.api_path, child_object_name);
+      auto res = add_diretory_item(child_object_name, true, node.node());
       if (res != api_error::success) {
         return res;
       }
-
-        res = get_item_meta(dir_item.api_path, dir_item.meta);
-      }
-
-      if (res != api_error::success) {
-        return res;
-      }
-
-      list.push_back(std::move(dir_item));
     }
 
     node_list = doc.select_nodes("/ListBucketResult/Contents");
@@ -449,66 +516,18 @@ auto s3_provider::get_directory_items_impl(
         continue;
       }
 
-      directory_item dir_item{};
-      dir_item.api_path = child_object_name;
-      if (is_encrypted) {
-        if (not utils::encryption::decrypt_file_path(cfg.encryption_token,
-                                                     dir_item.api_path)) {
-          return api_error::decryption_error;
-        }
-      }
-      dir_item.api_path = utils::path::create_api_path(dir_item.api_path);
-      dir_item.api_parent = utils::path::get_parent_api_path(dir_item.api_path);
-      dir_item.directory = false;
-
-      std::string size_str;
-      if (get_item_meta(dir_item.api_path, META_SIZE, size_str) ==
-          api_error::success) {
-        dir_item.size = utils::string::to_uint64(size_str);
-      } else {
-        auto size = node.node().select_node("Size").node().text().as_ullong();
-
-        dir_item.size = is_encrypted ? utils::encryption::encrypting_reader::
-                                           calculate_decrypted_size(size)
-                                     : size;
-      }
-
-      auto res = get_item_meta(dir_item.api_path, dir_item.meta);
-      if (res == api_error::item_not_found) {
-        auto last_modified = convert_api_date(
-            node.node().select_node("LastModified").node().text().as_string());
-
-        api_file file{};
-        file.api_path = dir_item.api_path;
-        file.api_parent = dir_item.api_parent;
-        file.accessed_date = file.changed_date = file.creation_date =
-            file.modified_date = last_modified;
-        file.file_size = dir_item.size;
-        if (is_encrypted) {
-          file.key = child_object_name;
-        }
-
-        res = add_if_not_found(file, child_object_name);
+      auto res = add_diretory_item(child_object_name, false, node.node());
       if (res != api_error::success) {
         return res;
       }
-
-        res = get_item_meta(dir_item.api_path, dir_item.meta);
-      }
-
-      if (res != api_error::success) {
-        return res;
-      }
-
-      list.push_back(std::move(dir_item));
     }
   }
 
-  return ret;
+  return api_error::success;
 }
 
-auto s3_provider::get_file(const std::string &api_path,
-                           api_file &file) const -> api_error {
+auto s3_provider::get_file(const std::string &api_path, api_file &file) const
+    -> api_error {
   REPERTORY_USES_FUNCTION_NAME();
 
   try {
@@ -547,10 +566,11 @@ auto s3_provider::get_file(const std::string &api_path,
   return api_error::error;
 }
 
-auto s3_provider::get_file_list(api_file_list &list,
-                                std::string &marker) const -> api_error {
+auto s3_provider::get_file_list(api_file_list &list, std::string &marker) const
+    -> api_error {
   REPERTORY_USES_FUNCTION_NAME();
 
+  try {
     std::string response_data;
     long response_code{};
     if (not get_object_list(response_data, response_code, std::nullopt,
@@ -565,15 +585,17 @@ auto s3_provider::get_file_list(api_file_list &list,
     }
 
     pugi::xml_document doc;
-  auto res = doc.load_string(response_data.c_str());
-  if (res.status != pugi::xml_parse_status::status_ok) {
-    utils::error::raise_error(function_name, res.status,
+    auto result = doc.load_string(response_data.c_str());
+    if (result.status != pugi::xml_parse_status::status_ok) {
+      utils::error::raise_error(function_name, result.status,
                                 "failed to parse xml document");
       return api_error::comm_error;
     }
 
-  auto grab_more =
-      doc.select_node("/ListBucketResult/IsTruncated").node().text().as_bool();
+    auto grab_more = doc.select_node("/ListBucketResult/IsTruncated")
+                         .node()
+                         .text()
+                         .as_bool();
     if (grab_more) {
       marker = doc.select_node("/ListBucketResult/NextContinuationToken")
                    .node()
@@ -592,9 +614,9 @@ auto s3_provider::get_file_list(api_file_list &list,
 
       auto is_encrypted = not get_s3_config().encryption_token.empty();
       if (is_encrypted) {
-      auto err = decrypt_object_name(api_path);
-      if (err != api_error::success) {
-        return err;
+        auto res = decrypt_object_name(api_path);
+        if (res != api_error::success) {
+          return res;
         }
       }
 
@@ -604,27 +626,36 @@ auto s3_provider::get_file_list(api_file_list &list,
       file.api_path = utils::path::create_api_path(api_path);
       file.api_parent = utils::path::get_parent_api_path(file.api_path);
       file.accessed_date = file.changed_date = file.creation_date =
-        file.modified_date = convert_api_date(
-            node.node().select_node("LastModified").node().text().as_string());
+          file.modified_date = convert_api_date(node.node()
+                                                    .select_node("LastModified")
+                                                    .node()
+                                                    .text()
+                                                    .as_string());
       file.file_size =
           is_encrypted
               ? utils::encryption::encrypting_reader::calculate_decrypted_size(
                     size)
               : size;
       file.key = is_encrypted ? utils::path::create_api_path(object_name) : "";
-    auto err = add_if_not_found(file, file.key);
-    if (err != api_error::success) {
-      return err;
+      auto res = add_if_not_found(file, file.key);
+      if (res != api_error::success) {
+        return res;
       }
 
       list.push_back(std::move(file));
     }
 
     return grab_more ? api_error::more_data : api_error::success;
+  } catch (const std::exception &e) {
+    utils::error::raise_error(function_name, e, "exception occurred");
+  }
+
+  return api_error::error;
 }
 
-auto s3_provider::get_last_modified(
-    bool directory, const std::string &api_path) const -> std::uint64_t {
+auto s3_provider::get_last_modified(bool directory,
+                                    const std::string &api_path) const
+    -> std::uint64_t {
   bool is_encrypted{};
   std::string object_name;
   head_object_result result{};
@@ -634,9 +665,10 @@ auto s3_provider::get_last_modified(
              : utils::time::get_time_now();
 }
 
-auto s3_provider::get_object_info(
-    bool directory, const std::string &api_path, bool &is_encrypted,
-    std::string &object_name, head_object_result &result) const -> api_error {
+auto s3_provider::get_object_info(bool directory, const std::string &api_path,
+                                  bool &is_encrypted, std::string &object_name,
+                                  head_object_result &result) const
+    -> api_error {
   REPERTORY_USES_FUNCTION_NAME();
 
   try {
@@ -657,12 +689,16 @@ auto s3_provider::get_object_info(
     curl::requests::http_head head{};
     head.allow_timeout = true;
     head.aws_service = "aws:amz:" + cfg.region + ":s3";
-    head.path = directory ? object_name + '/' : object_name;
     head.response_headers = http_headers{};
     head.response_handler = [&response_data](auto &&data,
                                              long /*response_code*/) {
       response_data = std::string(data.begin(), data.end());
     };
+    auto res =
+        set_request_path(head, directory ? object_name + '/' : object_name);
+    if (res != api_error::success) {
+      return res;
+    }
 
     stop_type stop_requested{false};
     long response_code{};
@@ -691,10 +727,12 @@ auto s3_provider::get_object_info(
   return api_error::error;
 }
 
-auto s3_provider::get_object_list(
-    std::string &response_data, long &response_code,
-    std::optional<std::string> delimiter, std::optional<std::string> prefix,
-    std::optional<std::string> token) const -> bool {
+auto s3_provider::get_object_list(std::string &response_data,
+                                  long &response_code,
+                                  std::optional<std::string> delimiter,
+                                  std::optional<std::string> prefix,
+                                  std::optional<std::string> token) const
+    -> bool {
   curl::requests::http_get get{};
   get.allow_timeout = true;
   get.aws_service = "aws:amz:" + get_s3_config().region + ":s3";
@@ -722,17 +760,17 @@ auto s3_provider::get_total_drive_space() const -> std::uint64_t {
   return std::numeric_limits<std::int64_t>::max() / std::int64_t(2);
 }
 
-auto s3_provider::is_directory(const std::string &api_path,
-                               bool &exists) const -> api_error {
+auto s3_provider::is_directory(const std::string &api_path, bool &exists) const
+    -> api_error {
   REPERTORY_USES_FUNCTION_NAME();
 
+  try {
     exists = false;
     if (api_path == "/") {
       exists = true;
       return api_error::success;
     }
 
-  try {
     bool is_encrypted{};
     std::string object_name;
     head_object_result result{};
@@ -750,16 +788,16 @@ auto s3_provider::is_directory(const std::string &api_path,
   return api_error::error;
 }
 
-auto s3_provider::is_file(const std::string &api_path,
-                          bool &exists) const -> api_error {
+auto s3_provider::is_file(const std::string &api_path, bool &exists) const
+    -> api_error {
   REPERTORY_USES_FUNCTION_NAME();
 
+  try {
     exists = false;
     if (api_path == "/") {
       return api_error::success;
     }
 
-  try {
     bool is_encrypted{};
     std::string object_name;
     head_object_result result{};
@@ -813,7 +851,6 @@ auto s3_provider::read_file_bytes(const std::string &api_path, std::size_t size,
         curl::requests::http_get get{};
         get.aws_service = "aws:amz:" + cfg.region + ":s3";
         get.headers["response-content-type"] = "binary/octet-stream";
-        get.path = object_name;
         get.range = {{
             read_offset,
             read_offset + read_size - 1U,
@@ -822,6 +859,10 @@ auto s3_provider::read_file_bytes(const std::string &api_path, std::size_t size,
                                               long /*response_code*/) {
           read_buffer = response_data;
         };
+        res = set_request_path(get, object_name);
+        if (res != api_error::success) {
+          return res;
+        }
 
         long response_code{};
         const auto notify_retry = [&]() {
@@ -909,17 +950,22 @@ auto s3_provider::remove_directory_impl(const std::string &api_path)
       utils::path::create_api_path(is_encrypted ? key : api_path);
 
   std::string response_data;
-  curl::requests::http_delete del{};
-  del.allow_timeout = true;
-  del.aws_service = "aws:amz:" + cfg.region + ":s3";
-  del.path = object_name + '/';
-  del.response_handler = [&response_data](auto &&data, long /*response_code*/) {
+  curl::requests::http_delete del_dir{};
+  del_dir.allow_timeout = true;
+  del_dir.aws_service = "aws:amz:" + cfg.region + ":s3";
+  del_dir.response_handler = [&response_data](auto &&data,
+                                              long /*response_code*/) {
     response_data = std::string(data.begin(), data.end());
   };
 
+  auto res = set_request_path(del_dir, object_name + '/');
+  if (res != api_error::success) {
+    return res;
+  }
+
   long response_code{};
   stop_type stop_requested{};
-  if (not get_comm().make_request(del, response_code, stop_requested)) {
+  if (not get_comm().make_request(del_dir, response_code, stop_requested)) {
     utils::error::raise_api_path_error(function_name, api_path,
                                        api_error::comm_error,
                                        "failed to remove directory");
@@ -955,14 +1001,22 @@ auto s3_provider::remove_file_impl(const std::string &api_path) -> api_error {
   auto object_name =
       utils::path::create_api_path(is_encrypted ? key : api_path);
 
-  curl::requests::http_delete del{};
-  del.allow_timeout = true;
-  del.aws_service = "aws:amz:" + cfg.region + ":s3";
-  del.path = object_name;
+  std::string response_data;
+  curl::requests::http_delete del_file{};
+  del_file.allow_timeout = true;
+  del_file.aws_service = "aws:amz:" + cfg.region + ":s3";
+  del_file.response_handler = [&response_data](auto &&data,
+                                               long /*response_code*/) {
+    response_data = std::string(data.begin(), data.end());
+  };
+  auto res = set_request_path(del_file, object_name);
+  if (res != api_error::success) {
+    return res;
+  }
 
   long response_code{};
   stop_type stop_requested{};
-  if (not get_comm().make_request(del, response_code, stop_requested)) {
+  if (not get_comm().make_request(del_file, response_code, stop_requested)) {
     utils::error::raise_api_path_error(function_name, api_path,
                                        api_error::comm_error,
                                        "failed to remove file");
@@ -972,8 +1026,9 @@ auto s3_provider::remove_file_impl(const std::string &api_path) -> api_error {
   if ((response_code < http_error_codes::ok ||
        response_code >= http_error_codes::multiple_choices) &&
       response_code != http_error_codes::not_found) {
-    utils::error::raise_api_path_error(function_name, api_path, response_code,
-                                       "failed to remove file");
+    utils::error::raise_api_path_error(
+        function_name, api_path, response_code,
+        fmt::format("failed to remove file|response|{}", response_data));
     return api_error::comm_error;
   }
 
@@ -1031,13 +1086,17 @@ auto s3_provider::upload_file_impl(const std::string &api_path,
   std::string response_data;
   curl::requests::http_put_file put_file{};
   put_file.aws_service = "aws:amz:" + cfg.region + ":s3";
-  put_file.path = object_name;
   put_file.response_handler = [&response_data](auto &&data,
                                                long /*response_code*/) {
     response_data = std::string(data.begin(), data.end());
   };
   put_file.source_path = source_path;
 
+  auto res = set_request_path(put_file, object_name);
+  if (res != api_error::success) {
+    return res;
+  }
+
   if (is_encrypted && file_size > 0U) {
     static stop_type no_stop{false};
 

repertory/librepertory/src/providers/sia/sia_provider.cpp

@@ -69,6 +69,15 @@ auto sia_provider::create_directory_impl(const std::string &api_path,
   put_file.path = "/api/worker/objects" + api_path + "/";
   put_file.query["bucket"] = get_bucket(get_sia_config());
 
+  std::string error_data;
+  put_file.response_handler = [&error_data](auto &&data, long response_code) {
+    if (response_code == http_error_codes::ok) {
+      return;
+    }
+
+    error_data = std::string(data.begin(), data.end());
+  };
+
   long response_code{};
   stop_type stop_requested{};
   if (not get_comm().make_request(put_file, response_code, stop_requested)) {
@@ -79,8 +88,9 @@ auto sia_provider::create_directory_impl(const std::string &api_path,
   }
 
   if (response_code != http_error_codes::ok) {
-    utils::error::raise_api_path_error(function_name, api_path, response_code,
-                                       "failed to create directory");
+    utils::error::raise_api_path_error(
+        function_name, api_path, response_code,
+        fmt::format("failed to create directory|response|{}", error_data));
     return api_error::comm_error;
   }
 
@@ -156,8 +166,8 @@ auto sia_provider::get_directory_items_impl(const std::string &api_path,
           get_api_item_added()(directory, file);
           auto res = get_item_meta(entry_api_path, meta);
           if (res != api_error::success) {
-            utils::error::raise_error(function_name, res,
-                                      "failed to get item meta");
+            utils::error::raise_api_path_error(function_name, entry_api_path,
+                                               res, "failed to get item meta");
             continue;
           }
         } else {
@@ -187,6 +197,9 @@ auto sia_provider::get_directory_items_impl(const std::string &api_path,
 
 auto sia_provider::get_file(const std::string &api_path, api_file &file) const
     -> api_error {
+  REPERTORY_USES_FUNCTION_NAME();
+
+  try {
     json file_data{};
     auto res = get_object_info(api_path, file_data);
     if (res != api_error::success) {
@@ -194,8 +207,8 @@ auto sia_provider::get_file(const std::string &api_path, api_file &file) const
     }
 
     auto slabs = file_data["object"]["Slabs"];
-  auto size =
-      std::accumulate(slabs.begin(), slabs.end(), std::uint64_t(0U),
+    auto size = std::accumulate(
+        slabs.begin(), slabs.end(), std::uint64_t(0U),
         [](std::uint64_t total_size,
            const nlohmann::json &slab) -> std::uint64_t {
           return total_size + slab["Length"].get<std::uint64_t>();
@@ -211,6 +224,12 @@ auto sia_provider::get_file(const std::string &api_path, api_file &file) const
     }
 
     return api_error::success;
+  } catch (const std::exception &e) {
+    utils::error::raise_api_path_error(function_name, api_path, e,
+                                       "failed to get file");
+  }
+
+  return api_error::error;
 }
 
 auto sia_provider::get_file_list(api_file_list &list,
@@ -270,8 +289,8 @@ auto sia_provider::get_file_list(api_file_list &list,
 
       return api_error::success;
     } catch (const std::exception &e) {
-      utils::error::raise_error(function_name, e,
-                                "failed to process directory|" + api_path);
+      utils::error::raise_api_path_error(function_name, api_path, e,
+                                         "failed to process directory");
     }
 
     return api_error::error;
@@ -290,11 +309,15 @@ auto sia_provider::get_object_info(const std::string &api_path,
     get.path = "/api/bus/objects" + api_path;
     get.query["bucket"] = get_bucket(get_sia_config());
 
-    get.response_handler = [&object_info](const data_buffer &data,
+    std::string error_data;
+    get.response_handler = [&error_data, &object_info](auto &&data,
                                                        long response_code) {
       if (response_code == http_error_codes::ok) {
         object_info = nlohmann::json::parse(data.begin(), data.end());
+        return;
       }
+
+      error_data = std::string(data.begin(), data.end());
     };
 
     long response_code{};
@@ -308,8 +331,9 @@ auto sia_provider::get_object_info(const std::string &api_path,
     }
 
     if (response_code != http_error_codes::ok) {
-      utils::error::raise_api_path_error(function_name, api_path, response_code,
-                                         "failed to get object info");
+      utils::error::raise_api_path_error(
+          function_name, api_path, response_code,
+          fmt::format("failed to get object info|response|{}", error_data));
       return api_error::comm_error;
     }
 
@@ -326,16 +350,21 @@ auto sia_provider::get_object_list(const std::string &api_path,
                                    nlohmann::json &object_list) const -> bool {
   REPERTORY_USES_FUNCTION_NAME();
 
+  try {
     curl::requests::http_get get{};
     get.allow_timeout = true;
     get.path = "/api/bus/objects" + api_path + "/";
     get.query["bucket"] = get_bucket(get_sia_config());
 
-  get.response_handler = [&object_list](const data_buffer &data,
+    std::string error_data;
+    get.response_handler = [&error_data, &object_list](auto &&data,
                                                        long response_code) {
       if (response_code == http_error_codes::ok) {
         object_list = nlohmann::json::parse(data.begin(), data.end());
+        return;
       }
+
+      error_data = std::string(data.begin(), data.end());
     };
 
     long response_code{};
@@ -348,12 +377,19 @@ auto sia_provider::get_object_list(const std::string &api_path,
     }
 
     if (response_code != http_error_codes::ok) {
-    utils::error::raise_api_path_error(function_name, api_path, response_code,
-                                       "failed to get object list");
+      utils::error::raise_api_path_error(
+          function_name, api_path, response_code,
+          fmt::format("failed to get object list|response|{}", error_data));
       return false;
     }
 
     return true;
+  } catch (const std::exception &e) {
+    utils::error::raise_api_path_error(function_name, api_path, e,
+                                       "failed to get object list");
+  }
+
+  return false;
 }
 
 auto sia_provider::get_total_drive_space() const -> std::uint64_t {
@@ -365,12 +401,16 @@ auto sia_provider::get_total_drive_space() const -> std::uint64_t {
     get.path = "/api/autopilot/config";
     get.query["bucket"] = get_bucket(get_sia_config());
 
-    json config_data{};
-    get.response_handler = [&config_data](const data_buffer &data,
+    json config_data;
+    std::string error_data;
+    get.response_handler = [&error_data, &config_data](auto &&data,
                                                        long response_code) {
       if (response_code == http_error_codes::ok) {
         config_data = nlohmann::json::parse(data.begin(), data.end());
+        return;
       }
+
+      error_data = std::string(data.begin(), data.end());
     };
 
     long response_code{};
@@ -380,8 +420,10 @@ auto sia_provider::get_total_drive_space() const -> std::uint64_t {
     }
 
     if (response_code != http_error_codes::ok) {
-      utils::error::raise_error(function_name, response_code,
-                                "failed to get total drive space");
+      utils::error::raise_error(
+          function_name, response_code,
+          fmt::format("failed to get total drive space|response|{}",
+                      error_data));
       return 0U;
     }
 
@@ -398,6 +440,7 @@ auto sia_provider::is_directory(const std::string &api_path, bool &exists) const
     -> api_error {
   REPERTORY_USES_FUNCTION_NAME();
 
+  try {
     if (api_path == "/") {
       exists = true;
       return api_error::success;
@@ -405,7 +448,6 @@ auto sia_provider::is_directory(const std::string &api_path, bool &exists) const
 
     exists = false;
 
-  try {
     json object_list{};
     if (not get_object_list(utils::path::get_parent_api_path(api_path),
                             object_list)) {
@@ -420,8 +462,8 @@ auto sia_provider::is_directory(const std::string &api_path, bool &exists) const
                           }) != object_list.at("entries").end();
     return api_error::success;
   } catch (const std::exception &e) {
-    utils::error::raise_api_path_error(function_name, api_path, e,
-                                       "failed to determine path is directory");
+    utils::error::raise_api_path_error(
+        function_name, api_path, e, "failed to determine if path is directory");
   }
 
   return api_error::error;
@@ -431,12 +473,12 @@ auto sia_provider::is_file(const std::string &api_path, bool &exists) const
     -> api_error {
   REPERTORY_USES_FUNCTION_NAME();
 
+  try {
     exists = false;
     if (api_path == "/") {
       return api_error::success;
     }
 
-  try {
     json file_data{};
     auto res = get_object_info(api_path, file_data);
     if (res == api_error::item_not_found) {
@@ -451,7 +493,7 @@ auto sia_provider::is_file(const std::string &api_path, bool &exists) const
     return api_error::success;
   } catch (const std::exception &e) {
     utils::error::raise_api_path_error(function_name, api_path, e,
-                                       "failed to determine path is directory");
+                                       "failed to determine if path is file");
   }
 
   return api_error::error;
@@ -466,12 +508,16 @@ auto sia_provider::is_online() const -> bool {
     get.path = "/api/bus/consensus/state";
     get.query["bucket"] = get_bucket(get_sia_config());
 
-    json state_data{};
-    get.response_handler = [&state_data](const data_buffer &data,
+    std::string error_data;
+    json state_data;
+    get.response_handler = [&error_data, &state_data](auto &&data,
                                                       long response_code) {
       if (response_code == http_error_codes::ok) {
         state_data = nlohmann::json::parse(data.begin(), data.end());
+        return;
       }
+
+      error_data = std::string(data.begin(), data.end());
     };
 
     long response_code{};
@@ -483,8 +529,10 @@ auto sia_provider::is_online() const -> bool {
     }
 
     if (response_code != http_error_codes::ok) {
-      utils::error::raise_error(function_name, response_code,
-                                "failed to determine if provider is online");
+      utils::error::raise_error(
+          function_name, response_code,
+          fmt::format("failed to determine if provider is online|response|{}",
+                      error_data));
       return false;
     }
 
@@ -503,6 +551,7 @@ auto sia_provider::read_file_bytes(const std::string &api_path,
                                    stop_type &stop_requested) -> api_error {
   REPERTORY_USES_FUNCTION_NAME();
 
+  try {
     curl::requests::http_get get{};
     get.path = "/api/worker/objects" + api_path;
     get.query["bucket"] = get_bucket(get_sia_config());
@@ -510,10 +559,11 @@ auto sia_provider::read_file_bytes(const std::string &api_path,
         offset,
         offset + size - 1U,
     }};
-  get.response_handler = [&buffer](const data_buffer &data,
-                                   long /*response_code*/) { buffer = data; };
+    get.response_handler = [&buffer](auto &&data, long /* response_code */) {
+      buffer = data;
+    };
 
-  auto res = api_error::comm_error;
+    auto res{api_error::comm_error};
     for (std::uint32_t idx = 0U;
          not stop_requested && res != api_error::success &&
          idx < get_config().get_retry_read_count() + 1U;
@@ -551,6 +601,12 @@ auto sia_provider::read_file_bytes(const std::string &api_path,
     }
 
     return res;
+  } catch (const std::exception &e) {
+    utils::error::raise_api_path_error(function_name, api_path, e,
+                                       "failed to read file bytes");
+  }
+
+  return api_error::error;
 }
 
 auto sia_provider::remove_directory_impl(const std::string &api_path)
@@ -562,6 +618,15 @@ auto sia_provider::remove_directory_impl(const std::string &api_path)
   del.path = "/api/bus/objects" + api_path + "/";
   del.query["bucket"] = get_bucket(get_sia_config());
 
+  std::string error_data;
+  del.response_handler = [&error_data](auto &&data, long response_code) {
+    if (response_code == http_error_codes::ok) {
+      return;
+    }
+
+    error_data = std::string(data.begin(), data.end());
+  };
+
   long response_code{};
   stop_type stop_requested{};
   if (not get_comm().make_request(del, response_code, stop_requested)) {
@@ -572,8 +637,9 @@ auto sia_provider::remove_directory_impl(const std::string &api_path)
   }
 
   if (response_code != http_error_codes::ok) {
-    utils::error::raise_api_path_error(function_name, api_path, response_code,
-                                       "failed to remove directory");
+    utils::error::raise_api_path_error(
+        function_name, api_path, response_code,
+        fmt::format("failed to remove directory|response|{}", error_data));
     return api_error::comm_error;
   }
 
@@ -588,6 +654,15 @@ auto sia_provider::remove_file_impl(const std::string &api_path) -> api_error {
   del.path = "/api/bus/objects" + api_path;
   del.query["bucket"] = get_bucket(get_sia_config());
 
+  std::string error_data;
+  del.response_handler = [&error_data](auto &&data, long response_code) {
+    if (response_code == http_error_codes::ok) {
+      return;
+    }
+
+    error_data = std::string(data.begin(), data.end());
+  };
+
   long response_code{};
   stop_type stop_requested{};
   if (not get_comm().make_request(del, response_code, stop_requested)) {
@@ -599,8 +674,9 @@ auto sia_provider::remove_file_impl(const std::string &api_path) -> api_error {
 
   if (response_code != http_error_codes::ok &&
       response_code != http_error_codes::not_found) {
-    utils::error::raise_api_path_error(function_name, api_path, response_code,
-                                       "failed to remove file");
+    utils::error::raise_api_path_error(
+        function_name, api_path, response_code,
+        fmt::format("failed to remove file|response|{}", error_data));
     return api_error::comm_error;
   }
 
@@ -611,6 +687,7 @@ auto sia_provider::rename_file(const std::string &from_api_path,
                                const std::string &to_api_path) -> api_error {
   REPERTORY_USES_FUNCTION_NAME();
 
+  try {
     curl::requests::http_post post{};
     post.json = nlohmann::json({
         {"from", from_api_path},
@@ -620,24 +697,41 @@ auto sia_provider::rename_file(const std::string &from_api_path,
     post.path = "/api/bus/objects/rename";
     post.query["bucket"] = get_bucket(get_sia_config());
 
+    std::string error_data;
+    post.response_handler = [&error_data](auto &&data, long response_code) {
+      if (response_code == http_error_codes::ok) {
+        return;
+      }
+
+      error_data = std::string(data.begin(), data.end());
+    };
+
     long response_code{};
     stop_type stop_requested{};
     if (not get_comm().make_request(post, response_code, stop_requested)) {
       utils::error::raise_api_path_error(
-        function_name, from_api_path + '|' + to_api_path, api_error::comm_error,
-        "failed to rename file");
+          function_name, fmt::format("{}|{}", from_api_path, to_api_path),
+          api_error::comm_error, "failed to rename file");
       return api_error::comm_error;
     }
 
     if (response_code < http_error_codes::ok ||
         response_code >= http_error_codes::multiple_choices) {
       utils::error::raise_api_path_error(
-        function_name, from_api_path + '|' + to_api_path, response_code,
-        "failed to rename file file");
+          function_name, fmt::format("{}|{}", from_api_path, to_api_path),
+          response_code,
+          fmt::format("failed to rename file file|response|{}", error_data));
       return api_error::comm_error;
     }
 
     return get_db().rename_item_meta(from_api_path, to_api_path);
+  } catch (const std::exception &e) {
+    utils::error::raise_api_path_error(
+        function_name, fmt::format("{}|{}", from_api_path, to_api_path), e,
+        "failed to rename file file|response");
+  }
+
+  return api_error::error;
 }
 
 auto sia_provider::start(api_item_added_callback api_item_added,
@@ -663,6 +757,15 @@ auto sia_provider::upload_file_impl(const std::string &api_path,
   put_file.query["bucket"] = get_bucket(get_sia_config());
   put_file.source_path = source_path;
 
+  std::string error_data;
+  put_file.response_handler = [&error_data](auto &&data, long response_code) {
+    if (response_code == http_error_codes::ok) {
+      return;
+    }
+
+    error_data = std::string(data.begin(), data.end());
+  };
+
   long response_code{};
   if (not get_comm().make_request(put_file, response_code, stop_requested)) {
     utils::error::raise_api_path_error(function_name, api_path, source_path,
@@ -672,8 +775,9 @@ auto sia_provider::upload_file_impl(const std::string &api_path,
   }
 
   if (response_code != http_error_codes::ok) {
-    utils::error::raise_api_path_error(function_name, api_path, source_path,
-                                       response_code, "failed to upload file");
+    utils::error::raise_api_path_error(
+        function_name, api_path, response_code,
+        fmt::format("failed to upload file|response|{}", error_data));
     return api_error::comm_error;
   }
 

repertory/librepertory/src/types/repertory.cpp

@@ -25,8 +25,8 @@
 #include "utils/string.hpp"
 
 namespace repertory {
-auto database_type_from_string(std::string type,
-                               database_type default_type) -> database_type {
+auto database_type_from_string(std::string type, database_type default_type)
+    -> database_type {
   type = utils::string::to_lower(utils::string::trim(type));
   if (type == "rocksdb") {
     return database_type::rocksdb;
@@ -50,8 +50,8 @@ auto database_type_to_string(const database_type &type) -> std::string {
   }
 }
 
-auto download_type_from_string(std::string type,
-                               download_type default_type) -> download_type {
+auto download_type_from_string(std::string type, download_type default_type)
+    -> download_type {
   type = utils::string::to_lower(utils::string::trim(type));
   if (type == "default") {
     return download_type::default_;
@@ -112,6 +112,7 @@ static const std::unordered_map<api_error, std::string> LOOKUP = {
     {api_error::item_exists, "item_exists"},
     {api_error::item_not_found, "item_not_found"},
     {api_error::more_data, "more_data"},
+    {api_error::name_too_long, "name_too_long"},
     {api_error::no_disk_space, "no_disk_space"},
     {api_error::not_implemented, "not_implemented"},
     {api_error::not_supported, "not_supported"},

repertory/librepertory/src/utils/unix/unix_utils.cpp

@@ -58,6 +58,8 @@ auto from_api_error(const api_error &err) -> int {
     return -EINVAL;
   case api_error::item_not_found:
     return -ENOENT;
+  case api_error::name_too_long:
+    return -ENAMETOOLONG;
   case api_error::out_of_memory:
     return -ENOMEM;
   case api_error::no_disk_space:
@@ -128,6 +130,8 @@ auto to_api_error(int err) -> api_error {
     return api_error::file_in_use;
   case EINVAL:
     return api_error::invalid_operation;
+  case ENAMETOOLONG:
+    return api_error::name_too_long;
   case ENOENT:
     return api_error::item_not_found;
   case ENOMEM:

repertory/librepertory/src/utils/windows/windows_utils.cpp

@@ -65,6 +65,8 @@ auto from_api_error(const api_error &e) -> NTSTATUS {
     return STATUS_CLIENT_SERVER_PARAMETERS_INVALID;
   case api_error::item_not_found:
     return STATUS_OBJECT_NAME_NOT_FOUND;
+  case api_error::name_too_long:
+    return STATUS_NAME_TOO_LONG;
   case api_error::no_disk_space:
     return STATUS_DEVICE_INSUFFICIENT_RESOURCES;
   case api_error::os_error:

support/include/utils/config.hpp

@@ -468,7 +468,7 @@ struct http_range final {
   std::uint64_t end{};
 };
 
-using http_headers = std::unordered_map<std::string, std::string>;
+using http_headers = std::map<std::string, std::string>;
 using http_query_parameters = std::map<std::string, std::string>;
 using http_ranges = std::vector<http_range>;
 #endif // defined(PROJECT_ENABLE_CURL)