Address compiler warnings #10

.cspell/words.txt

@@ -70,7 +70,6 @@ duse_libidn2
 dwith_gflags
 dwith_liburing
 dwith_tools
-dxxh_no_inline_hints
 dylib
 endfunction
 endmacro
@@ -244,7 +243,6 @@ wcast
 wconversion
 wdouble
 wduplicated
-wfloat
 wformat
 windres
 winfsp

.gitignore

@@ -15,4 +15,3 @@ tags
 src/common.cpp
 cspell.json
 dviml
-*.log

.jenkins_linux_builds

@@ -86,9 +86,9 @@ pipeline {
       agent any
 
       steps {
-        sh 'scripts/make_package.sh alpine /home/sgraves/cert build/alpine_arm64 . /mnt/storj 1'
-        sh 'scripts/make_package.sh alpine /home/sgraves/cert build/alpine . /mnt/storj'
-        sh 'scripts/make_package.sh mingw64 /home/sgraves/cert build/mingw64 . /mnt/storj'
+        sh 'scripts/make_package.sh alpine /home/sgraves/cert build/alpine_arm64 . /mnt/filebase 1'
+        sh 'scripts/make_package.sh alpine /home/sgraves/cert build/alpine . /mnt/filebase'
+        sh 'scripts/make_package.sh mingw64 /home/sgraves/cert build/mingw64 . /mnt/filebase'
       }
     }
   }

.jenkins_osx

@@ -47,7 +47,7 @@ pipeline {
       }
 
       steps {
-        sh 'scripts/make_package.sh darwin /Users/sgraves/cert build . /Users/sgraves/mnt/storj'
+        sh 'scripts/make_package.sh darwin /Users/sgraves/cert build . /Users/sgraves/mnt/filebase'
       }
     }
   }

3rd_party/cpp-httplib/httplib.h

@@ -8,7 +8,7 @@
 #ifndef CPPHTTPLIB_HTTPLIB_H
 #define CPPHTTPLIB_HTTPLIB_H
 
-#define CPPHTTPLIB_VERSION "0.14.2"
+#define CPPHTTPLIB_VERSION "0.14.0"
 
 /*
  * Configuration
@@ -247,6 +247,7 @@ using socket_t = int;
 
 #ifdef _MSC_VER
 #pragma comment(lib, "crypt32.lib")
+#pragma comment(lib, "cryptui.lib")
 #endif
 #elif defined(CPPHTTPLIB_USE_CERTS_FROM_MACOSX_KEYCHAIN) && defined(__APPLE__)
 #include <TargetConditionals.h>
@@ -330,7 +331,7 @@ struct scope_exit {
   explicit scope_exit(std::function<void(void)> &&f)
       : exit_function(std::move(f)), execute_on_destruction{true} {}
 
-  scope_exit(scope_exit &&rhs) noexcept
+  scope_exit(scope_exit &&rhs)
       : exit_function(std::move(rhs.exit_function)),
         execute_on_destruction{rhs.execute_on_destruction} {
     rhs.release();
@@ -384,7 +385,6 @@ public:
   DataSink &operator=(DataSink &&) = delete;
 
   std::function<bool(const char *data, size_t data_len)> write;
-  std::function<bool()> is_writable;
   std::function<void()> done;
   std::function<void(const Headers &trailer)> done_with_trailer;
   std::ostream os;
@@ -395,7 +395,7 @@ private:
     explicit data_sink_streambuf(DataSink &sink) : sink_(sink) {}
 
   protected:
-    std::streamsize xsputn(const char *s, std::streamsize n) override {
+    std::streamsize xsputn(const char *s, std::streamsize n) {
       sink_.write(s, static_cast<size_t>(n));
       return n;
     }
@@ -873,15 +873,15 @@ private:
   bool routing(Request &req, Response &res, Stream &strm);
   bool handle_file_request(const Request &req, Response &res,
                            bool head = false);
-  bool dispatch_request(Request &req, Response &res,
-                        const Handlers &handlers) const;
-  bool dispatch_request_for_content_reader(
-      Request &req, Response &res, ContentReader content_reader,
-      const HandlersForContentReader &handlers) const;
+  bool dispatch_request(Request &req, Response &res, const Handlers &handlers);
+  bool
+  dispatch_request_for_content_reader(Request &req, Response &res,
+                                      ContentReader content_reader,
+                                      const HandlersForContentReader &handlers);
 
-  bool parse_request_line(const char *s, Request &req) const;
+  bool parse_request_line(const char *s, Request &req);
   void apply_ranges(const Request &req, Response &res,
-                    std::string &content_type, std::string &boundary) const;
+                    std::string &content_type, std::string &boundary);
   bool write_response(Stream &strm, bool close_connection, const Request &req,
                       Response &res);
   bool write_response_with_content(Stream &strm, bool close_connection,
@@ -901,7 +901,7 @@ private:
   bool read_content_core(Stream &strm, Request &req, Response &res,
                          ContentReceiver receiver,
                          MultipartContentHeader multipart_header,
-                         ContentReceiver multipart_receiver) const;
+                         ContentReceiver multipart_receiver);
 
   virtual bool process_and_close_socket(socket_t sock);
 
@@ -967,7 +967,7 @@ enum class Error {
   SSLPeerCouldBeClosed_,
 };
 
-std::string to_string(Error error);
+std::string to_string(const Error error);
 
 std::ostream &operator<<(std::ostream &os, const Error &obj);
 
@@ -1226,7 +1226,7 @@ public:
   void set_ca_cert_path(const std::string &ca_cert_file_path,
                         const std::string &ca_cert_dir_path = std::string());
   void set_ca_cert_store(X509_STORE *ca_cert_store);
-  X509_STORE *create_ca_cert_store(const char *ca_cert, std::size_t size) const;
+  X509_STORE *create_ca_cert_store(const char *ca_cert, std::size_t size);
 #endif
 
 #ifdef CPPHTTPLIB_OPENSSL_SUPPORT
@@ -1255,14 +1255,14 @@ protected:
   // Also, shutdown_ssl and close_socket should also NOT be called concurrently
   // with a DIFFERENT thread sending requests using that socket.
   virtual void shutdown_ssl(Socket &socket, bool shutdown_gracefully);
-  void shutdown_socket(Socket &socket) const;
+  void shutdown_socket(Socket &socket);
   void close_socket(Socket &socket);
 
   bool process_request(Stream &strm, Request &req, Response &res,
                        bool close_connection, Error &error);
 
   bool write_content_with_provider(Stream &strm, const Request &req,
-                                   Error &error) const;
+                                   Error &error);
 
   void copy_settings(const ClientImpl &rhs);
 
@@ -1353,8 +1353,7 @@ private:
   Result send_(Request &&req);
 
   socket_t create_client_socket(Error &error) const;
-  bool read_response_line(Stream &strm, const Request &req,
-                          Response &res) const;
+  bool read_response_line(Stream &strm, const Request &req, Response &res);
   bool write_request(Stream &strm, Request &req, bool close_connection,
                      Error &error);
   bool redirect(Request &req, Response &res, Error &error);
@@ -1373,7 +1372,7 @@ private:
       const std::string &content_type);
   ContentProviderWithoutLength get_multipart_content_provider(
       const std::string &boundary, const MultipartFormDataItems &items,
-      const MultipartFormDataProviderItems &provider_items) const;
+      const MultipartFormDataProviderItems &provider_items);
 
   std::string adjust_host_string(const std::string &host) const;
 
@@ -1686,7 +1685,7 @@ public:
 private:
   bool create_and_connect_socket(Socket &socket, Error &error) override;
   void shutdown_ssl(Socket &socket, bool shutdown_gracefully) override;
-  void shutdown_ssl_impl(Socket &socket, bool shutdown_gracefully);
+  void shutdown_ssl_impl(Socket &socket, bool shutdown_socket);
 
   bool process_socket(const Socket &socket,
                       std::function<bool(Stream &strm)> callback) override;
@@ -2076,9 +2075,6 @@ std::string trim_copy(const std::string &s);
 void split(const char *b, const char *e, char d,
            std::function<void(const char *, const char *)> fn);
 
-void split(const char *b, const char *e, char d, size_t m,
-           std::function<void(const char *, const char *)> fn);
-
 bool process_client_socket(socket_t sock, time_t read_timeout_sec,
                            time_t read_timeout_usec, time_t write_timeout_sec,
                            time_t write_timeout_usec,
@@ -2155,7 +2151,7 @@ public:
 
 class nocompressor : public compressor {
 public:
-  ~nocompressor() override = default;
+  virtual ~nocompressor() = default;
 
   bool compress(const char *data, size_t data_length, bool /*last*/,
                 Callback callback) override;
@@ -2165,7 +2161,7 @@ public:
 class gzip_compressor : public compressor {
 public:
   gzip_compressor();
-  ~gzip_compressor() override;
+  ~gzip_compressor();
 
   bool compress(const char *data, size_t data_length, bool last,
                 Callback callback) override;
@@ -2178,7 +2174,7 @@ private:
 class gzip_decompressor : public decompressor {
 public:
   gzip_decompressor();
-  ~gzip_decompressor() override;
+  ~gzip_decompressor();
 
   bool is_valid() const override;
 
@@ -2321,7 +2317,7 @@ inline std::string from_i_to_hex(size_t n) {
 
 inline size_t to_utf8(int code, char *buff) {
   if (code < 0x0080) {
-    buff[0] = static_cast<char>(code & 0x7F);
+    buff[0] = (code & 0x7F);
     return 1;
   } else if (code < 0x0800) {
     buff[0] = static_cast<char>(0xC0 | ((code >> 6) & 0x1F));
@@ -2587,23 +2583,16 @@ inline std::string trim_double_quotes_copy(const std::string &s) {
 
 inline void split(const char *b, const char *e, char d,
                   std::function<void(const char *, const char *)> fn) {
-  return split(b, e, d, std::numeric_limits<size_t>::max(), fn);
-}
-
-inline void split(const char *b, const char *e, char d, size_t m,
-                  std::function<void(const char *, const char *)> fn) {
   size_t i = 0;
   size_t beg = 0;
-  size_t count = 1;
 
   while (e ? (b + i < e) : (b[i] != '\0')) {
-    if (b[i] == d && count < m) {
+    if (b[i] == d) {
       auto r = trim(b, e, beg, i);
       if (r.first < r.second) {
         fn(&b[r.first], &b[r.second]);
       }
       beg = i + 1;
-      count++;
     }
     i++;
   }
@@ -2694,7 +2683,9 @@ inline mmap::mmap(const char *path)
       ,
       size_(0),
       addr_(nullptr) {
-  open(path);
+  if (!open(path)) {
+    std::runtime_error("");
+  }
 }
 
 inline mmap::~mmap() { close(); }
@@ -2983,7 +2974,7 @@ private:
   size_t read_buff_off_ = 0;
   size_t read_buff_content_size_ = 0;
 
-  static const size_t read_buff_size_ = 1024l * 4;
+  static const size_t read_buff_size_ = 1024 * 4;
 };
 
 #ifdef CPPHTTPLIB_OPENSSL_SUPPORT
@@ -3118,9 +3109,8 @@ socket_t create_socket(const std::string &host, const std::string &ip, int port,
 #ifndef _WIN32
   if (hints.ai_family == AF_UNIX) {
     const auto addrlen = host.length();
-    if (addrlen > sizeof(sockaddr_un::sun_path)) {
+    if (addrlen > sizeof(sockaddr_un::sun_path))
       return INVALID_SOCKET;
-    }
 
     auto sock = socket(hints.ai_family, hints.ai_socktype, hints.ai_protocol);
     if (sock != INVALID_SOCKET) {
@@ -3334,7 +3324,7 @@ inline socket_t create_client_socket(
           if (ip_from_if.empty()) {
             ip_from_if = intf;
           }
-          if (!bind_ip_address(sock2, ip_from_if)) {
+          if (!bind_ip_address(sock2, ip_from_if.c_str())) {
             error = Error::BindIPAddress;
             return false;
           }
@@ -3496,7 +3486,7 @@ find_content_type(const std::string &path,
 
   auto it = user_data.find(ext);
   if (it != user_data.end()) {
-    return it->second;
+    return it->second.c_str();
   }
 
   using udl::operator""_t;
@@ -3762,9 +3752,8 @@ inline bool gzip_decompressor::decompress(const char *data, size_t data_length,
       }
     }
 
-    if (ret != Z_OK && ret != Z_STREAM_END) {
+    if (ret != Z_OK && ret != Z_STREAM_END)
       return false;
-    }
 
   } while (data_length > 0);
 
@@ -4080,7 +4069,7 @@ inline bool read_content_chunked(Stream &strm, T &x,
       return false;
     }
 
-    if (strcmp(line_reader.ptr(), "\r\n") != 0) {
+    if (strcmp(line_reader.ptr(), "\r\n")) {
       return false;
     }
 
@@ -4096,7 +4085,7 @@ inline bool read_content_chunked(Stream &strm, T &x,
     return false;
   }
 
-  while (strcmp(line_reader.ptr(), "\r\n") != 0) {
+  while (strcmp(line_reader.ptr(), "\r\n")) {
     if (line_reader.size() > CPPHTTPLIB_HEADER_MAX_LENGTH) {
       return false;
     }
@@ -4252,8 +4241,6 @@ inline bool write_content(Stream &strm, const ContentProvider &content_provider,
     return ok;
   };
 
-  data_sink.is_writable = [&]() -> bool { return strm.is_writable(); };
-
   while (offset < end_offset && !is_shutting_down()) {
     if (!strm.is_writable()) {
       error = Error::Write;
@@ -4300,8 +4287,6 @@ write_content_without_length(Stream &strm,
     return ok;
   };
 
-  data_sink.is_writable = [&]() -> bool { return strm.is_writable(); };
-
   data_sink.done = [&](void) { data_available = false; };
 
   while (data_available && !is_shutting_down()) {
@@ -4352,8 +4337,6 @@ write_content_chunked(Stream &strm, const ContentProvider &content_provider,
     return ok;
   };
 
-  data_sink.is_writable = [&]() -> bool { return strm.is_writable(); };
-
   auto done_with_trailer = [&](const Headers *trailer) {
     if (!ok) {
       return;
@@ -4455,9 +4438,8 @@ inline bool redirect(T &cli, Request &req, Response &res,
     req = new_req;
     res = new_res;
 
-    if (res.location.empty()) {
+    if (res.location.empty())
       res.location = location;
-    }
   }
   return ret;
 }
@@ -4552,9 +4534,8 @@ inline bool parse_range_header(const std::string &s, Ranges &ranges) try {
     auto len = static_cast<size_t>(m.length(1));
     auto all_valid_ranges = true;
     split(&s[pos], &s[pos + len], ',', [&](const char *b, const char *e) {
-      if (!all_valid_ranges) {
+      if (!all_valid_ranges)
         return;
-      }
       static auto re_another_range = std::regex(R"(\s*(\d*)-(\d*))");
       std::cmatch cm;
       if (std::regex_match(b, e, cm, re_another_range)) {
@@ -4941,9 +4922,8 @@ serialize_multipart_formdata(const MultipartFormDataItems &items,
     body += item.content + serialize_multipart_formdata_item_end();
   }
 
-  if (finish) {
+  if (finish)
     body += serialize_multipart_formdata_finish(boundary);
-  }
 
   return body;
 }
@@ -5396,7 +5376,7 @@ inline std::string random_string(size_t length) {
                            "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
                            "abcdefghijklmnopqrstuvwxyz";
     const size_t max_index = (sizeof(charset) - 1);
-    return charset[static_cast<size_t>(std::rand()) % max_index];
+    return charset[repertory_rand<size_t>() % max_index];
   };
   std::string str(length, 0);
   std::generate_n(str.begin(), length, randchar);
@@ -5691,7 +5671,7 @@ inline SocketStream::SocketStream(socket_t sock, time_t read_timeout_sec,
       write_timeout_usec_(write_timeout_usec),
       read_buff_(read_buff_size_, 0) {}
 
-inline SocketStream::~SocketStream() = default;
+inline SocketStream::~SocketStream() {}
 
 inline bool SocketStream::is_readable() const {
   return select_read(sock_, read_timeout_sec_, read_timeout_usec_) > 0;
@@ -5917,7 +5897,7 @@ inline Server::Server()
 #endif
 }
 
-inline Server::~Server() = default;
+inline Server::~Server() {}
 
 inline std::unique_ptr<detail::MatcherBase>
 Server::make_matcher(const std::string &pattern) {
@@ -5929,60 +5909,66 @@ Server::make_matcher(const std::string &pattern) {
 }
 
 inline Server &Server::Get(const std::string &pattern, Handler handler) {
-  get_handlers_.emplace_back(make_matcher(pattern), std::move(handler));
+  get_handlers_.push_back(
+      std::make_pair(make_matcher(pattern), std::move(handler)));
   return *this;
 }
 
 inline Server &Server::Post(const std::string &pattern, Handler handler) {
-  post_handlers_.emplace_back(make_matcher(pattern), std::move(handler));
+  post_handlers_.push_back(
+      std::make_pair(make_matcher(pattern), std::move(handler)));
   return *this;
 }
 
 inline Server &Server::Post(const std::string &pattern,
                             HandlerWithContentReader handler) {
-  post_handlers_for_content_reader_.emplace_back(make_matcher(pattern),
-                                                 std::move(handler));
+  post_handlers_for_content_reader_.push_back(
+      std::make_pair(make_matcher(pattern), std::move(handler)));
   return *this;
 }
 
 inline Server &Server::Put(const std::string &pattern, Handler handler) {
-  put_handlers_.emplace_back(make_matcher(pattern), std::move(handler));
+  put_handlers_.push_back(
+      std::make_pair(make_matcher(pattern), std::move(handler)));
   return *this;
 }
 
 inline Server &Server::Put(const std::string &pattern,
                            HandlerWithContentReader handler) {
-  put_handlers_for_content_reader_.emplace_back(make_matcher(pattern),
-                                                std::move(handler));
+  put_handlers_for_content_reader_.push_back(
+      std::make_pair(make_matcher(pattern), std::move(handler)));
   return *this;
 }
 
 inline Server &Server::Patch(const std::string &pattern, Handler handler) {
-  patch_handlers_.emplace_back(make_matcher(pattern), std::move(handler));
+  patch_handlers_.push_back(
+      std::make_pair(make_matcher(pattern), std::move(handler)));
   return *this;
 }
 
 inline Server &Server::Patch(const std::string &pattern,
                              HandlerWithContentReader handler) {
-  patch_handlers_for_content_reader_.emplace_back(make_matcher(pattern),
-                                                  std::move(handler));
+  patch_handlers_for_content_reader_.push_back(
+      std::make_pair(make_matcher(pattern), std::move(handler)));
   return *this;
 }
 
 inline Server &Server::Delete(const std::string &pattern, Handler handler) {
-  delete_handlers_.emplace_back(make_matcher(pattern), std::move(handler));
+  delete_handlers_.push_back(
+      std::make_pair(make_matcher(pattern), std::move(handler)));
   return *this;
 }
 
 inline Server &Server::Delete(const std::string &pattern,
                               HandlerWithContentReader handler) {
-  delete_handlers_for_content_reader_.emplace_back(make_matcher(pattern),
-                                                   std::move(handler));
+  delete_handlers_for_content_reader_.push_back(
+      std::make_pair(make_matcher(pattern), std::move(handler)));
   return *this;
 }
 
 inline Server &Server::Options(const std::string &pattern, Handler handler) {
-  options_handlers_.emplace_back(make_matcher(pattern), std::move(handler));
+  options_handlers_.push_back(
+      std::make_pair(make_matcher(pattern), std::move(handler)));
   return *this;
 }
 
@@ -6130,7 +6116,9 @@ inline Server &Server::set_payload_max_length(size_t length) {
 
 inline bool Server::bind_to_port(const std::string &host, int port,
                                  int socket_flags) {
-  return bind_internal(host, port, socket_flags) >= 0;
+  if (bind_internal(host, port, socket_flags) < 0)
+    return false;
+  return true;
 }
 inline int Server::bind_to_any_port(const std::string &host, int socket_flags) {
   return bind_internal(host, 0, socket_flags);
@@ -6164,7 +6152,7 @@ inline void Server::stop() {
   }
 }
 
-inline bool Server::parse_request_line(const char *s, Request &req) const {
+inline bool Server::parse_request_line(const char *s, Request &req) {
   auto len = strlen(s);
   if (len < 2 || s[len - 2] != '\r' || s[len - 1] != '\n') {
     return false;
@@ -6220,7 +6208,7 @@ inline bool Server::parse_request_line(const char *s, Request &req) const {
     size_t count = 0;
 
     detail::split(req.target.data(), req.target.data() + req.target.size(), '?',
-                  2, [&](const char *b, const char *e) {
+                  [&](const char *b, const char *e) {
                     switch (count) {
                     case 0:
                       req.path = detail::decode_url(std::string(b, e), false);
@@ -6445,11 +6433,10 @@ inline bool Server::read_content_with_content_receiver(
                            std::move(multipart_receiver));
 }
 
-inline bool
-Server::read_content_core(Stream &strm, Request &req, Response &res,
-                          ContentReceiver receiver,
-                          MultipartContentHeader multipart_header,
-                          ContentReceiver multipart_receiver) const {
+inline bool Server::read_content_core(Stream &strm, Request &req, Response &res,
+                                      ContentReceiver receiver,
+                                      MultipartContentHeader multipart_header,
+                                      ContentReceiver multipart_receiver) {
   detail::MultipartFormDataParser multipart_form_data_parser;
   ContentReceiverWithProgress out;
 
@@ -6515,7 +6502,7 @@ inline bool Server::handle_file_request(const Request &req, Response &res,
 
         if (detail::is_file(path)) {
           for (const auto &kv : entry.headers) {
-            res.set_header(kv.first, kv.second);
+            res.set_header(kv.first.c_str(), kv.second);
           }
 
           auto mm = std::make_shared<detail::mmap>(path.c_str());
@@ -6752,7 +6739,7 @@ inline bool Server::routing(Request &req, Response &res, Stream &strm) {
 }
 
 inline bool Server::dispatch_request(Request &req, Response &res,
-                                     const Handlers &handlers) const {
+                                     const Handlers &handlers) {
   for (const auto &x : handlers) {
     const auto &matcher = x.first;
     const auto &handler = x.second;
@@ -6767,7 +6754,7 @@ inline bool Server::dispatch_request(Request &req, Response &res,
 
 inline void Server::apply_ranges(const Request &req, Response &res,
                                  std::string &content_type,
-                                 std::string &boundary) const {
+                                 std::string &boundary) {
   if (req.ranges.size() > 1) {
     boundary = detail::make_multipart_data_boundary();
 
@@ -6879,7 +6866,7 @@ inline void Server::apply_ranges(const Request &req, Response &res,
 
 inline bool Server::dispatch_request_for_content_reader(
     Request &req, Response &res, ContentReader content_reader,
-    const HandlersForContentReader &handlers) const {
+    const HandlersForContentReader &handlers) {
   for (const auto &x : handlers) {
     const auto &matcher = x.first;
     const auto &handler = x.second;
@@ -7138,9 +7125,8 @@ inline socket_t ClientImpl::create_client_socket(Error &error) const {
   // Check is custom IP specified for host_
   std::string ip;
   auto it = addr_map_.find(host_);
-  if (it != addr_map_.end()) {
+  if (it != addr_map_.end())
     ip = it->second;
-  }
 
   return detail::create_client_socket(
       host_, ip, port_, address_family_, tcp_nodelay_, socket_options_,
@@ -7167,7 +7153,7 @@ inline void ClientImpl::shutdown_ssl(Socket & /*socket*/,
          socket_requests_are_from_thread_ == std::this_thread::get_id());
 }
 
-inline void ClientImpl::shutdown_socket(Socket &socket) const {
+inline void ClientImpl::shutdown_socket(Socket &socket) {
   if (socket.sock == INVALID_SOCKET) {
     return;
   }
@@ -7196,7 +7182,7 @@ inline void ClientImpl::close_socket(Socket &socket) {
 }
 
 inline bool ClientImpl::read_response_line(Stream &strm, const Request &req,
-                                           Response &res) const {
+                                           Response &res) {
   std::array<char, 2048> buf{};
 
   detail::stream_line_reader line_reader(strm, buf.data(), buf.size());
@@ -7490,7 +7476,7 @@ inline bool ClientImpl::redirect(Request &req, Response &res, Error &error) {
   } else {
     if (next_scheme == "https") {
 #ifdef CPPHTTPLIB_OPENSSL_SUPPORT
-      SSLClient cli(next_host, next_port);
+      SSLClient cli(next_host.c_str(), next_port);
       cli.copy_settings(*this);
       if (ca_cert_store_) {
         cli.set_ca_cert_store(ca_cert_store_);
@@ -7500,7 +7486,7 @@ inline bool ClientImpl::redirect(Request &req, Response &res, Error &error) {
       return false;
 #endif
     } else {
-      ClientImpl cli(next_host, next_port);
+      ClientImpl cli(next_host.c_str(), next_port);
       cli.copy_settings(*this);
       return detail::redirect(cli, req, res, path, location, error);
     }
@@ -7509,7 +7495,7 @@ inline bool ClientImpl::redirect(Request &req, Response &res, Error &error) {
 
 inline bool ClientImpl::write_content_with_provider(Stream &strm,
                                                     const Request &req,
-                                                    Error &error) const {
+                                                    Error &error) {
   auto is_shutting_down = []() { return false; };
 
   if (req.is_chunked_content_provider_) {
@@ -7857,14 +7843,13 @@ inline bool ClientImpl::process_request(Stream &strm, Request &req,
 
 inline ContentProviderWithoutLength ClientImpl::get_multipart_content_provider(
     const std::string &boundary, const MultipartFormDataItems &items,
-    const MultipartFormDataProviderItems &provider_items) const {
-  size_t cur_item = 0;
-  size_t cur_start = 0;
+    const MultipartFormDataProviderItems &provider_items) {
+  size_t cur_item = 0, cur_start = 0;
   // cur_item and cur_start are copied to within the std::function and maintain
   // state between successive calls
   return [&, cur_item, cur_start](size_t offset,
                                   DataSink &sink) mutable -> bool {
-    if (!offset && !items.empty()) {
+    if (!offset && items.size()) {
       sink.os << detail::serialize_multipart_formdata(items, boundary, false);
       return true;
     } else if (cur_item < provider_items.size()) {
@@ -7881,9 +7866,8 @@ inline ContentProviderWithoutLength ClientImpl::get_multipart_content_provider(
       cur_sink.write = sink.write;
       cur_sink.done = [&]() { has_data = false; };
 
-      if (!provider_items[cur_item].provider(offset - cur_start, cur_sink)) {
+      if (!provider_items[cur_item].provider(offset - cur_start, cur_sink))
         return false;
-      }
 
       if (!has_data) {
         sink.os << detail::serialize_multipart_formdata_item_end();
@@ -8004,7 +7988,7 @@ inline Result ClientImpl::Get(const std::string &path, const Params &params,
   }
 
   std::string path_with_query = append_query_params(path, params);
-  return Get(path_with_query, headers, progress);
+  return Get(path_with_query.c_str(), headers, progress);
 }
 
 inline Result ClientImpl::Get(const std::string &path, const Params &params,
@@ -8024,8 +8008,8 @@ inline Result ClientImpl::Get(const std::string &path, const Params &params,
   }
 
   std::string path_with_query = append_query_params(path, params);
-  return Get(path_with_query, headers, response_handler, content_receiver,
-             progress);
+  return Get(path_with_query.c_str(), headers, response_handler,
+             content_receiver, progress);
 }
 
 inline Result ClientImpl::Head(const std::string &path) {
@@ -8127,7 +8111,7 @@ inline Result ClientImpl::Post(const std::string &path, const Headers &headers,
   const auto &content_type =
       detail::serialize_multipart_formdata_get_content_type(boundary);
   const auto &body = detail::serialize_multipart_formdata(items, boundary);
-  return Post(path, headers, body, content_type);
+  return Post(path, headers, body, content_type.c_str());
 }
 
 inline Result ClientImpl::Post(const std::string &path, const Headers &headers,
@@ -8140,7 +8124,7 @@ inline Result ClientImpl::Post(const std::string &path, const Headers &headers,
   const auto &content_type =
       detail::serialize_multipart_formdata_get_content_type(boundary);
   const auto &body = detail::serialize_multipart_formdata(items, boundary);
-  return Post(path, headers, body, content_type);
+  return Post(path, headers, body, content_type.c_str());
 }
 
 inline Result
@@ -8524,11 +8508,10 @@ inline void ClientImpl::set_ca_cert_store(X509_STORE *ca_cert_store) {
 }
 
 inline X509_STORE *ClientImpl::create_ca_cert_store(const char *ca_cert,
-                                                    std::size_t size) const {
+                                                    std::size_t size) {
   auto mem = BIO_new_mem_buf(ca_cert, static_cast<int>(size));
-  if (!mem) {
+  if (!mem)
     return nullptr;
-  }
 
   auto inf = PEM_X509_INFO_read_bio(mem, nullptr, nullptr, nullptr);
   if (!inf) {
@@ -8693,7 +8676,7 @@ inline SSLSocketStream::SSLSocketStream(socket_t sock, SSL *ssl,
   SSL_clear_mode(ssl, SSL_MODE_AUTO_RETRY);
 }
 
-inline SSLSocketStream::~SSLSocketStream() = default;
+inline SSLSocketStream::~SSLSocketStream() {}
 
 inline bool SSLSocketStream::is_readable() const {
   return detail::select_read(sock_, read_timeout_sec_, read_timeout_usec_) > 0;
@@ -8915,7 +8898,7 @@ inline SSLClient::SSLClient(const std::string &host, int port,
 
   detail::split(&host_[0], &host_[host_.size()], '.',
                 [&](const char *b, const char *e) {
-                  host_components_.emplace_back(b, e);
+                  host_components_.emplace_back(std::string(b, e));
                 });
 
   if (!client_cert_path.empty() && !client_key_path.empty()) {
@@ -8936,7 +8919,7 @@ inline SSLClient::SSLClient(const std::string &host, int port,
 
   detail::split(&host_[0], &host_[host_.size()], '.',
                 [&](const char *b, const char *e) {
-                  host_components_.emplace_back(b, e);
+                  host_components_.emplace_back(std::string(b, e));
                 });
 
   if (client_cert != nullptr && client_key != nullptr) {
@@ -9214,8 +9197,8 @@ SSLClient::verify_host_with_subject_alt_name(X509 *server_cert) const {
 
   auto type = GEN_DNS;
 
-  struct in6_addr addr6 {};
-  struct in_addr addr {};
+  struct in6_addr addr6;
+  struct in_addr addr;
   size_t addr_len = 0;
 
 #ifndef __MINGW32__
@@ -9296,7 +9279,7 @@ inline bool SSLClient::check_host_name(const char *pattern,
   std::vector<std::string> pattern_components;
   detail::split(&pattern[0], &pattern[pattern_len], '.',
                 [&](const char *b, const char *e) {
-                  pattern_components.emplace_back(b, e);
+                  pattern_components.emplace_back(std::string(b, e));
                 });
 
   if (host_components_.size() != pattern_components.size()) {
@@ -9381,7 +9364,7 @@ inline Client::Client(const std::string &host, int port,
     : cli_(detail::make_unique<ClientImpl>(host, port, client_cert_path,
                                            client_key_path)) {}
 
-inline Client::~Client() = default;
+inline Client::~Client() {}
 
 inline bool Client::is_valid() const {
   return cli_ != nullptr && cli_->is_valid();

CHANGELOG.md

@@ -5,20 +5,15 @@
 ### Issues
 
 * \#10 Address compiler warnings
-* \#11 Switch to SQLite over RocksDB
 
 ### Changes from v2.0.0-rc
 
-* Refactored Sia, S3 and base provider
-* Fixed intermittent deadlock on file close
-* Removed MSVC compilation support (MinGW-64 should be used)
 * Require `c++20`
-* Switched to Storj over Filebase for hosting binaries
-* Updated `boost` to v1.83.0
-* Updated `cpp-httplib` to v0.14.2
-* Updated `curl` to v8.4.0
-* Updated `libsodium` to v1.0.19
-* Updated `OpenSSL` to v3.2.0
+* Removed MSVC compilation support (MinGW-64 should be used)
+* Upgraded `boost` to v1.83.0
+* Upgraded `curl` to v8.4.0
+* Upgraded `libsodium` to v1.0.19
+* Upgraded `rocksdb` to v8.6.7
 
 ## 2.0.0-rc
 

CMakeLists.txt

@@ -50,7 +50,7 @@ if (LINUX OR MINGW)
   endif()
 
   set(ENV{PKG_CONFIG_PATH}
-    "${EXTERNAL_BUILD_ROOT}/lib/pkgconfig:${EXTERNAL_BUILD_ROOT}/share/pkgconfig:$ENV{PKG_CONFIG_PATH}"
+    "${EXTERNAL_BUILD_ROOT}/lib/pkgconfig:${EXTERNAL_BUILD_ROOT}/lib64/pkgconfig:$ENV{PKG_CONFIG_PATH}"
   )
 
   find_package(PkgConfig REQUIRED)
@@ -83,12 +83,12 @@ if (LINUX OR MINGW)
   if (LIBFUSE3 AND NOT MINGW)
     pkg_check_modules(LIBFUSE3 REQUIRED fuse3>=3.0.0)
     set(LIBFUSE3_LIBRARIES ${LIBFUSE3})
-    set(REPERTORY_DEFINITIONS ${REPERTORY_DEFINITIONS} -DFUSE_USE_VERSION=30)
+    add_definitions(-DFUSE_USE_VERSION=30)
   endif()
 
   if(NOT LIBFUSE3_LIBRARIES AND NOT MINGW)
     pkg_check_modules(LIBFUSE2 REQUIRED fuse>=2.9.0)
-    set(REPERTORY_DEFINITIONS ${REPERTORY_DEFINITIONS} -DFUSE_USE_VERSION=29)
+    add_definitions(-DFUSE_USE_VERSION=29)
   endif()
 
   if (NOT LIBPTHREAD_LIBRARY) 
@@ -132,7 +132,8 @@ elseif (MACOS)
     message(FATAL_ERROR "'libpthread' not found")
   endif()
 
-  set(REPERTORY_DEFINITIONS ${REPERTORY_DEFINITIONS} -DFUSE_USE_VERSION=29 -DBOOST_ASIO_HAS_STD_STRING_VIEW)
+  set(REPERTORY_DEFINITIONS ${REPERTORY_DEFINITIONS} -DBOOST_ASIO_HAS_STD_STRING_VIEW)
+  add_definitions(-DFUSE_USE_VERSION=29)
 
   include_directories(/usr/local/include)
   find_library(OSXFUSE NO_CACHE NAMES OSXFUSE)
@@ -209,12 +210,14 @@ include_directories(SYSTEM
 
 link_directories(
   ${EXTERNAL_BUILD_ROOT}/lib
+  ${EXTERNAL_BUILD_ROOT}/lib64
 )
 
 include(cmake/zlib.cmake)
 include(cmake/openssl.cmake)
 include(cmake/curl.cmake)
 include(cmake/boost.cmake)
+include(cmake/rocksdb.cmake)
 include(cmake/libsodium.cmake)
 
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/src/common.cpp.in ${CMAKE_CURRENT_SOURCE_DIR}/src/common.cpp @ONLY)
@@ -226,14 +229,15 @@ include_directories(SYSTEM
   ${CMAKE_CURRENT_SOURCE_DIR}/3rd_party/stduuid
   ${CMAKE_CURRENT_SOURCE_DIR}/3rd_party/json
   ${CMAKE_CURRENT_SOURCE_DIR}/3rd_party/pugixml/src
-  ${CMAKE_CURRENT_SOURCE_DIR}/3rd_party/sqlite/include
   ${CURL_INCLUDE_DIRS}
   ${LIBFUSE2_INCLUDE_DIRS}
   ${LIBFUSE3_INCLUDE_DIRS}
   ${OPENSSL_INCLUDE_DIR}
+  ${ROCKSDB_INCLUDE_DIRS}
 )
 
 set(REPERTORY_LINK_LIBRARIES
+  ${ROCKSDB_LIBRARIES}
   ${LIBFUSE2_LIBRARIES}
   ${LIBFUSE3_LIBRARIES}
   ${Boost_LIBRARIES}
@@ -252,14 +256,12 @@ file(GLOB_RECURSE REPERTORY_HEADERS
   ${CMAKE_CURRENT_SOURCE_DIR}/include/**/*.hh
   ${CMAKE_CURRENT_SOURCE_DIR}/include/**/*.hpp
   ${CMAKE_CURRENT_SOURCE_DIR}/3rd_party/pugixml/src/*.hpp
-  ${CMAKE_CURRENT_SOURCE_DIR}/3rd_party/sqlite/include/*.h
 )
 
 file(GLOB_RECURSE REPERTORY_SOURCES
   ${CMAKE_CURRENT_SOURCE_DIR}/src/*.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/src/**/*.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/3rd_party/pugixml/src/*.cpp
-  ${CMAKE_CURRENT_SOURCE_DIR}/3rd_party/sqlite/src/*.c
 )
 list(REMOVE_ITEM REPERTORY_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/src/main.cpp)
 

README.md

@@ -16,11 +16,13 @@ on Windows.
 
 * [Sia renterd](https://github.com/SiaFoundation/renterd/releases) v0.4.0+ for Sia support
 * Only 64-bit operating systems are supported
-  * Linux requires `fusermount3` or `repertory` must be manually compiled with `libfuse2` support
+  * Linux requires the following dependencies:
+    * `libfuse3`
   * OS X requires the following dependency to be installed:
     * [FUSE for macOS v4.5.0](https://github.com/osxfuse/osxfuse/releases/download/macfuse-4.5.0/macfuse-4.5.0.dmg)
   * Windows requires the following dependencies to be installed:
     * [WinFSP 2023](https://github.com/winfsp/winfsp/releases/download/v2.0/winfsp-2.0.23075.msi)
+    * [Microsoft Visual C++ Redistributable for Visual Studio 2015, 2017 and 2019](https://aka.ms/vs/16/release/vc_redist.x64.exe)
 
 ## Supported Operating Systems
 
@@ -34,17 +36,18 @@ on Windows.
 * [boost c++ libraries](https://www.boost.org/)
 * [cpp-httplib](https://github.com/yhirose/cpp-httplib)
 * [curl](https://curl.haxx.se/)
+* [Filebase](https://filebase.com/)
 * [FUSE for macOS](https://osxfuse.github.io/)
 * [Google Test](https://github.com/google/googletest)
 * [JSON for Modern C++](https://github.com/nlohmann/json)
 * [libfuse](https://github.com/libfuse/libfuse)
 * [libsodium](https://doc.libsodium.org/)
 * [OpenSSL](https://www.openssl.org/)
+* [OSSP uuid](http://www.ossp.org/pkg/lib/uuid/)
+* [RocksDB](https://rocksdb.org/)
 * [ScPrime](https://scpri.me/)
 * [Sia Decentralized Cloud Storage](https://sia.tech/)
-* [SQLite](https://www.sqlite.org)
 * [stduuid](https://github.com/mariusbancila/stduuid)
-* [Storj](https://storj.io/)
 * [WinFSP - FUSE for Windows](https://github.com/billziss-gh/winfsp)
 * [zlib](https://zlib.net/)
 
@@ -77,3 +80,4 @@ mMH3segHBkRj0xJcfOxceRLj1a+ULIIR3xL/3f8s5Id25TDo/nqBoCvu5PeCpo6L
 9wIDAQAB
 -----END PUBLIC KEY-----
 ```
+

cmake/boost.cmake

@@ -1,10 +1,9 @@
-set(BOOST_VERSION ${BOOST_MAJOR_VERSION}.${BOOST_MINOR_VERSION}.${BOOST_REVISION})
-set(BOOST_VERSION2 ${BOOST_MAJOR_VERSION}_${BOOST_MINOR_VERSION}_${BOOST_REVISION})
-
 set(BOOST_PROJECT_NAME boost_${BOOST_VERSION})
 set(BOOST_BUILD_ROOT ${EXTERNAL_BUILD_ROOT}/builds/${BOOST_PROJECT_NAME})
 
 set(BOOST_ADDRESS_MODEL 64)
+set(BOOST_VERSION ${BOOST_MAJOR_VERSION}.${BOOST_MINOR_VERSION}.${BOOST_REVISION})
+set(BOOST_VERSION2 ${BOOST_MAJOR_VERSION}_${BOOST_MINOR_VERSION}_${BOOST_REVISION})
 
 set(BOOST_DOWNLOAD_URL https://boostorg.jfrog.io/artifactory/main/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION2}.tar.gz)
 
@@ -39,12 +38,10 @@ set (BOOST_COMMON_ARGS
 )
 
 if (MINGW)
-  if (NOT CMAKE_HOST_WIN32)
-    set(BOOST_COMMON_ARGS 
-      ${BOOST_COMMON_ARGS}
-      --user-config=./user-config.jam
-    )
-  endif()
+  set(BOOST_COMMON_ARGS 
+    ${BOOST_COMMON_ARGS}
+    --user-config=./user-config.jam
+  )
   set(BOOST_TARGET_OS target-os=windows)
 endif()
 
@@ -72,11 +69,12 @@ ExternalProject_Add(boost_project
 
 add_dependencies(boost_project openssl_project)
 
-if (MINGW AND CMAKE_HOST_WIN32)
+set(BOOST_ROOT ${BOOST_BUILD_ROOT}/src/boost_project)
+if (MINGW)
   set(BOOST_GCC_VERSION ${CMAKE_CXX_COMPILER_VERSION})
   string(REPLACE "." ";" BOOST_GCC_VERSION_LIST ${BOOST_GCC_VERSION})
   list(GET BOOST_GCC_VERSION_LIST 0 BOOST_GCC_MAJOR_VERSION)
-  set(BOOST_LIB_EXTRA "-mgw${BOOST_GCC_MAJOR_VERSION}-mt${DEBUG_EXTRA2}-x64-${BOOST_MAJOR_VERSION}_${BOOST_MINOR_VERSION}")
+  # set(BOOST_LIB_EXTRA "-mgw${BOOST_GCC_MAJOR_VERSION}-mt-x64-${BOOST_MAJOR_VERSION}_${BOOST_MINOR_VERSION}")
 endif()
 
 set(Boost_LIBRARIES
@@ -92,6 +90,3 @@ set(Boost_LIBRARIES
 )
 
 add_dependencies(boost_project zlib_project)
-if (CMAKE_HOST_WIN32)
-  include_directories(SYSTEM ${EXTERNAL_BUILD_ROOT}/include/boost-${BOOST_MAJOR_VERSION}_${BOOST_MINOR_VERSION})
-endif()

cmake/flags.cmake

@@ -18,21 +18,23 @@ if (UNIX OR MINGW)
     )
   endif()
 
-  set(REPERTORY_COMMON_FLAG_LIST_DEBUG
-    ${REPERTORY_COMMON_FLAG_LIST_DEBUG}
-    -DDEBUG
-    -D_DEBUG
-    -Og
-    -fno-omit-frame-pointer
-    -g
-    -gdwarf-4
-  )
-
-  set(REPERTORY_COMMON_FLAG_LIST_RELEASE
-    ${REPERTORY_COMMON_FLAG_LIST_RELEASE}
-    -O3
-    -DNDEBUG
-  )
+  if (CMAKE_BUILD_TYPE STREQUAL "Debug")
+    set(REPERTORY_COMMON_FLAG_LIST
+      ${REPERTORY_COMMON_FLAG_LIST}
+      -DDEBUG
+      -D_DEBUG
+      -Og
+      -fno-omit-frame-pointer
+      -g
+      -gdwarf-4
+    )
+  else()
+    set(REPERTORY_COMMON_FLAG_LIST
+      ${REPERTORY_COMMON_FLAG_LIST}
+      -O3
+      -DNDEBUG
+    )
+  endif()
 
   if (NOT IS_CLANG_COMPILER)
     set(REPERTORY_GCC_FLAGS 
@@ -57,7 +59,7 @@ if (UNIX OR MINGW)
       -Wnon-virtual-dtor
       -Wold-style-cast
       -Woverloaded-virtual
-      -Wno-useless-cast
+      -Wuseless-cast
     )
   endif()
 
@@ -73,30 +75,12 @@ if (UNIX OR MINGW)
   )
 endif()
 
-list(JOIN REPERTORY_COMMON_FLAG_LIST_DEBUG " " REPERTORY_COMMON_FLAG_LIST_DEBUG)
-list(JOIN REPERTORY_COMMON_FLAG_LIST_RELEASE " " REPERTORY_COMMON_FLAG_LIST_RELEASE)
-
 list(JOIN REPERTORY_CXX_FLAGS_LIST " " REPERTORY_CXX_FLAGS_LIST)
 list(JOIN REPERTORY_C_FLAGS_LIST " " REPERTORY_C_FLAGS_LIST)
-
 list(JOIN REPERTORY_SHARED_LINKER_FLAGS_LIST " " REPERTORY_SHARED_LINKER_FLAGS_LIST)
 
-set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${REPERTORY_CXX_FLAGS_LIST}")
 set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${REPERTORY_C_FLAGS_LIST}")
-
-if (CMAKE_BUILD_TYPE STREQUAL "Debug")
-  set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${REPERTORY_COMMON_FLAG_LIST_DEBUG}")
-  set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${REPERTORY_COMMON_FLAG_LIST_DEBUG}")
-else()
-  set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${REPERTORY_COMMON_FLAG_LIST_RELEASE}")
-  set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${REPERTORY_COMMON_FLAG_LIST_RELEASE}")
-endif()
-
-set (REPERTORY_CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS} ${REPERTORY_COMMON_FLAG_LIST_DEBUG}")
-set (REPERTORY_CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS} ${REPERTORY_COMMON_FLAG_LIST_DEBUG}")
-set (REPERTORY_CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS} ${REPERTORY_COMMON_FLAG_LIST_RELEASE}")
-set (REPERTORY_CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS} ${REPERTORY_COMMON_FLAG_LIST_RELEASE}")
-
+set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${REPERTORY_CXX_FLAGS_LIST}")
 set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${REPERTORY_SHARED_LINKER_FLAGS_LIST}")
 if (ALPINE_FOUND OR MINGW)
   set(CMAKE_EXE_LINKER_FLAGS "-static-libgcc -static-libstdc++ -static")

cmake/librepertory.cmake

@@ -17,5 +17,6 @@ add_dependencies(librepertory
   curl_project
   libsodium_project
   openssl_project
+  rocksdb_project
   zlib_project
 )

cmake/openssl.cmake

@@ -20,7 +20,7 @@ set(OPENSSL_PROJECT_NAME openssl_${OPENSSL_VERSION})
 set(OPENSSL_BUILD_ROOT ${EXTERNAL_BUILD_ROOT}/builds/${OPENSSL_PROJECT_NAME})
 ExternalProject_Add(openssl_project
   DOWNLOAD_NO_PROGRESS 1
-  URL https://github.com/openssl/openssl/releases/download/openssl-${OPENSSL_VERSION}/openssl-${OPENSSL_VERSION}.tar.gz 
+  URL https://github.com/openssl/openssl/archive/refs/tags/OpenSSL_${OPENSSL_VERSION}.tar.gz 
   PREFIX ${OPENSSL_BUILD_ROOT}
   BUILD_IN_SOURCE 1
   CONFIGURE_COMMAND ./Configure no-shared ${OPENSSL_COMPILE_TYPE} --openssldir=${EXTERNAL_BUILD_ROOT}/ssl --prefix=${EXTERNAL_BUILD_ROOT}

cmake/rocksdb.cmake

@@ -0,0 +1,66 @@
+set(ROCKSDB_PROJECT_NAME rocksdb_${ROCKSDB_VERSION})
+set(ROCKSDB_BUILD_ROOT ${EXTERNAL_BUILD_ROOT}/builds/${ROCKSDB_PROJECT_NAME})
+
+if (MACOS)
+  set(ROCKSDB_CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=hidden")
+else()
+  set(ROCKSDB_CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
+endif()
+
+set(ROCKSDB_CMAKE_ARGS
+  -DCMAKE_BUILD_TYPE=${EXTERNAL_BUILD_TYPE}
+  -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
+  -DCMAKE_CXX_FLAGS=${ROCKSDB_CMAKE_CXX_FLAGS}
+  -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
+  -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}
+  -DCMAKE_GENERATOR=${CMAKE_GENERATOR}
+  -DCMAKE_INSTALL_PREFIX=${EXTERNAL_BUILD_ROOT}
+  -DCMAKE_POSITION_INDEPENDENT_CODE=${CMAKE_POSITION_INDEPENDENT_CODE}
+  -DCMAKE_SHARED_LINKER_FLAGS=${CMAKE_SHARED_LINKER_FLAGS}
+  -DFAIL_ON_WARNINGS=OFF
+  -DPORTABLE=1
+  -DROCKSDB_BUILD_SHARED=OFF
+  -DWITH_BENCHMARK_TOOLS=OFF
+  -DWITH_LIBURING=OFF
+  -DWITH_TESTS=OFF
+  -DWITH_TOOLS=OFF
+)
+
+if (MINGW)
+  if (CMAKE_TOOLCHAIN_FILE)
+    set(ROCKSDB_CMAKE_ARGS
+      ${ROCKSDB_CMAKE_ARGS}
+      -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} 
+    )
+  endif()
+
+  ExternalProject_Add(rocksdb_project
+    DOWNLOAD_NO_PROGRESS 1
+    URL https://github.com/facebook/rocksdb/archive/v${ROCKSDB_VERSION}.tar.gz
+    PREFIX ${ROCKSDB_BUILD_ROOT}
+    CMAKE_ARGS ${ROCKSDB_CMAKE_ARGS} -DWITH_GFLAGS=OFF
+    INSTALL_COMMAND ${CMAKE_COMMAND} -E echo "Skipping install step."
+  )
+
+  set(ROCKSDB_LIBRARIES ${ROCKSDB_BUILD_ROOT}/src/rocksdb_project-build/librocksdb.a)
+  include_directories(SYSTEM ${ROCKSDB_BUILD_ROOT}/src/rocksdb_project/include)
+else()
+  ExternalProject_Add(rocksdb_project
+    DOWNLOAD_NO_PROGRESS 1
+    URL https://github.com/facebook/rocksdb/archive/v${ROCKSDB_VERSION}.tar.gz
+    PREFIX ${ROCKSDB_BUILD_ROOT}
+    CMAKE_ARGS ${ROCKSDB_CMAKE_ARGS} -DWITH_GFLAGS=OFF
+  )
+
+  if (MACOS)
+    set(ROCKSDB_LIBRARIES ${EXTERNAL_BUILD_ROOT}/lib/librocksdb.a)
+  else()
+    set(ROCKSDB_LIBRARIES librocksdb.a)
+  endif()
+endif()
+
+if (LINUX OR MINGW)
+  add_dependencies(rocksdb_project curl_project)
+endif()
+
+add_dependencies(rocksdb_project zlib_project)

cmake/settings.cmake

@@ -32,8 +32,3 @@ set(REPERTORY_OUTPUT_DIR ${CMAKE_BINARY_DIR})
 
 set(EXTERNAL_BUILD_ROOT ${CMAKE_BINARY_DIR}/external)
 set(EXTERNAL_BUILD_TYPE ${CMAKE_BUILD_TYPE})
-
-if (UNIX OR MINGW)
-  file(MAKE_DIRECTORY ${EXTERNAL_BUILD_ROOT}/lib)
-  file(CREATE_LINK ${EXTERNAL_BUILD_ROOT}/lib ${EXTERNAL_BUILD_ROOT}/lib64 SYMBOLIC)
-endif()

cmake/versions.cmake

@@ -1,9 +1,11 @@
 set(BOOST_MAJOR_VERSION 1)
 set(BOOST_MINOR_VERSION 83)
 set(BOOST_REVISION 0)
-set(CURL_VERSION 8_5_0)
+set(CURL_VERSION 8_4_0)
 set(GTEST_VERSION v1.14.0)
 set(LIBSODIUM_VERSION 1.0.19)
-set(OPENSSL_VERSION 3.2.0)
+set(LIBUUID_VERSION 1.6.2)
+set(OPENSSL_VERSION 1_1_1w)
+set(ROCKSDB_VERSION 8.6.7)
 set(WINFSP_VERSION 2.0)
 set(ZLIB_VERSION v1.3)

docker/64_bit/mingw64

@@ -5,13 +5,13 @@ WORKDIR /mnt
 
 ENV MINGW=/mingw
 
-ARG BINUTILS_VERSION=2.41
+ARG PKG_CONFIG_VERSION=0.29.2
 ARG CMAKE_VERSION=3.27.1
-ARG GCC_VERSION=13.2.0
+ARG BINUTILS_VERSION=2.41
 ARG MINGW_VERSION=11.0.1
+ARG GCC_VERSION=13.2.0
 ARG NASM_VERSION=2.16.01
 ARG NVCC_VERSION=12.2.1
-ARG PKG_CONFIG_VERSION=0.29.2
 
 SHELL [ "/bin/bash", "-c" ]
 
@@ -49,18 +49,14 @@ RUN set -ex \
         yasm \
         wget \
         zip \
-        git
-
-RUN set -ex \
+        git \
     \
     && wget -q https://pkg-config.freedesktop.org/releases/pkg-config-${PKG_CONFIG_VERSION}.tar.gz -O - | tar -xz \
     && wget -q https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}.tar.gz -O - | tar -xz \
     && wget -q https://ftp.gnu.org/gnu/binutils/binutils-${BINUTILS_VERSION}.tar.xz -O - | tar -xJ \
     && wget -q https://sourceforge.net/projects/mingw-w64/files/mingw-w64/mingw-w64-release/mingw-w64-v${MINGW_VERSION}.tar.bz2 -O - | tar -xj \
     && wget -q https://ftp.gnu.org/gnu/gcc/gcc-${GCC_VERSION}/gcc-${GCC_VERSION}.tar.xz -O - | tar -xJ \
-    && wget -q https://www.nasm.us/pub/nasm/releasebuilds/${NASM_VERSION}/nasm-${NASM_VERSION}.tar.xz -O - | tar -xJ
-
-RUN set -ex \
+    && wget -q https://www.nasm.us/pub/nasm/releasebuilds/${NASM_VERSION}/nasm-${NASM_VERSION}.tar.xz -O - | tar -xJ \
     \
     && mkdir -p ${MINGW}/include ${MINGW}/lib/pkgconfig \
     && chmod 0777 -R /mnt ${MINGW} \
@@ -73,18 +69,16 @@ RUN set -ex \
         --disable-shared \
         --disable-nls \
     && make -j`nproc` \
-    && make install
-
-RUN set -ex \
+    && make install \
+    && cd .. \
     \
     && cd cmake-${CMAKE_VERSION} \
     && ./configure \
         --prefix=/usr/local \
         --parallel=`nproc` \
     && make -j`nproc` \
-    && make install
-
-RUN set -ex \
+    && make install \
+    && cd .. \
     \
     && cd binutils-${BINUTILS_VERSION} \
     && ./configure \
@@ -99,9 +93,8 @@ RUN set -ex \
         --disable-werror \
         --with-system-zlib \
     && make -j`nproc` \
-    && make install
-
-RUN set -ex \
+    && make install \
+    && cd .. \
     \
     && mkdir mingw-w64 \
     && cd mingw-w64 \
@@ -109,9 +102,8 @@ RUN set -ex \
         --prefix=/usr/local/x86_64-w64-mingw32 \
         --host=x86_64-w64-mingw32 \
         --enable-sdk=all \
-    && make install
-
-RUN set -ex \
+    && make install \
+    && cd .. \
     \
     && mkdir gcc \
     && cd gcc \
@@ -133,9 +125,8 @@ RUN set -ex \
         --disable-nls \
         --disable-werror \
     && make -j`nproc` all-gcc \
-    && make install-gcc 
-
-RUN set -ex \
+    && make install-gcc \
+    && cd .. \
     \
     && cd mingw-w64 \
     && ../mingw-w64-v${MINGW_VERSION}/mingw-w64-crt/configure \
@@ -145,9 +136,8 @@ RUN set -ex \
         --disable-lib32 \
         --enable-lib64 \
     && (make || make || make || make) \
-    && make install
-
-RUN set -ex \
+    && make install \
+    && cd .. \
     \
     && cd mingw-w64 \
     && ../mingw-w64-v${MINGW_VERSION}/mingw-w64-libraries/winpthreads/configure \
@@ -156,45 +146,38 @@ RUN set -ex \
         --enable-static \
         --disable-shared \
     && make -j`nproc` \
-    && make install
-
-RUN set -ex \
+    && make install \
+    && cd .. \
     \
     && cd gcc \
     && make -j`nproc` \
-    && make install
-
-RUN set -ex \
+    && make install \
+    && cd .. \
     \
     && cd nasm-${NASM_VERSION} \
     && ./configure --prefix=/usr/local \
     && make -j`nproc` \
-    && make install
-
-RUN set -ex \
+    && make install \
+    && cd .. \
     \
     && rm -r pkg-config-${PKG_CONFIG_VERSION} \
     && rm -r cmake-${CMAKE_VERSION} \
     && rm -r binutils-${BINUTILS_VERSION} \
     && rm -r mingw-w64 mingw-w64-v${MINGW_VERSION} \
     && rm -r gcc gcc-${GCC_VERSION} \
-    && rm -r nasm-${NASM_VERSION}
-
-RUN set -ex \
+    && rm -r nasm-${NASM_VERSION} \
     \
     && apt-get remove --purge -y file gcc g++ zlib1g-dev libssl-dev libgmp-dev libmpfr-dev libmpc-dev libisl-dev \
+    \
     && apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/3bf863cc.pub \
     && echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/ /" > /etc/apt/sources.list.d/cuda.list \
     && apt-get update \
+    \
     && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
-        cuda-nvcc-${NVCC_VERSION:0:2}-${NVCC_VERSION:3:1}
-
-RUN set -ex \
+        cuda-nvcc-${NVCC_VERSION:0:2}-${NVCC_VERSION:3:1} \
     \
     && ln -s /usr/bin/gcc /usr/local/cuda/bin/gcc \
-    && ln -s /usr/bin/g++ /usr/local/cuda/bin/g++
-
-RUN set -ex \
+    && ln -s /usr/bin/g++ /usr/local/cuda/bin/g++ \
     \
     && apt-get remove --purge -y gnupg \
     && apt-get autoremove --purge -y \

include/app_config.hpp

@@ -30,42 +30,39 @@
 namespace repertory {
 class app_config final {
 public:
-  [[nodiscard]] static auto default_agent_name(const provider_type &prov)
+  [[nodiscard]] static auto default_agent_name(const provider_type &pt)
       -> std::string;
 
-  [[nodiscard]] static auto default_api_port(const provider_type &prov)
+  [[nodiscard]] static auto default_api_port(const provider_type &pt)
       -> std::uint16_t;
 
-  [[nodiscard]] static auto default_data_directory(const provider_type &prov)
+  [[nodiscard]] static auto default_data_directory(const provider_type &pt)
       -> std::string;
 
-  [[nodiscard]] static auto default_remote_port(const provider_type &prov)
+  [[nodiscard]] static auto default_rpc_port(const provider_type &pt)
       -> std::uint16_t;
 
-  [[nodiscard]] static auto default_rpc_port(const provider_type &prov)
-      -> std::uint16_t;
-
-  [[nodiscard]] static auto get_provider_api_password(const provider_type &prov)
+  [[nodiscard]] static auto get_provider_api_password(const provider_type &pt)
       -> std::string;
 
-  [[nodiscard]] static auto get_provider_display_name(const provider_type &prov)
+  [[nodiscard]] static auto get_provider_display_name(const provider_type &pt)
       -> std::string;
 
-  [[nodiscard]] static auto get_provider_name(const provider_type &prov)
+  [[nodiscard]] static auto get_provider_name(const provider_type &pt)
       -> std::string;
 
 public:
-  app_config(const provider_type &prov, const std::string &data_directory = "");
+  app_config(const provider_type &pt, const std::string &data_directory = "");
 
   ~app_config() { save(); }
 
 private:
-  provider_type prov_;
+  const provider_type pt_;
   std::string api_auth_;
   std::uint16_t api_port_;
   std::string api_user_;
   bool config_changed_;
-  std::string data_directory_;
+  const std::string data_directory_;
   std::uint8_t download_timeout_secs_;
   bool enable_chunk_downloader_timeout_;
   bool enable_comm_duration_events_;
@@ -103,8 +100,8 @@ private:
   s3_config s3_config_;
   std::uint64_t version_ = REPERTORY_CONFIG_VERSION;
   std::string log_directory_;
-  mutable std::recursive_mutex read_write_mutex_;
-  mutable std::recursive_mutex remote_mount_mutex_;
+  std::recursive_mutex read_write_mutex_;
+  std::recursive_mutex remote_mount_mutex_;
 
 private:
   [[nodiscard]] auto load() -> bool;
@@ -132,7 +129,7 @@ private:
   template <typename dest, typename source>
   auto set_value(dest &dst, const source &src) -> bool {
     auto ret = false;
-    recur_mutex_lock lock(read_write_mutex_);
+    recur_mutex_lock l(read_write_mutex_);
     if (dst != src) {
       dst = src;
       config_changed_ = true;
@@ -247,9 +244,7 @@ public:
                                             download_type::fallback);
   }
 
-  [[nodiscard]] auto get_provider_type() const -> provider_type {
-    return prov_;
-  }
+  [[nodiscard]] auto get_provider_type() const -> provider_type { return pt_; }
 
   [[nodiscard]] auto get_read_ahead_count() const -> std::uint8_t {
     return std::max(static_cast<std::uint8_t>(1U), read_ahead_count_);
@@ -363,17 +358,9 @@ public:
   }
 
 #ifdef REPERTORY_TESTING
-  void set_host_config(host_config hc) {
-    config_changed_ = true;
-    hc_ = std::move(hc);
-    save();
-  }
+  void set_host_config(host_config hc) { hc_ = std::move(hc); }
 
-  void set_s3_config(s3_config s3) {
-    config_changed_ = true;
-    s3_config_ = std::move(s3);
-    save();
-  }
+  void set_s3_config(s3_config s3) { s3_config_ = std::move(s3); }
 #endif
 
   void set_is_remote_mount(bool is_remote_mount);

include/cli/actions.hpp

@@ -23,12 +23,14 @@
 #define INCLUDE_CLI_ACTIONS_HPP_
 
 #include "cli/check_version.hpp"
+#include "cli/create_directory.hpp"
 #include "cli/display_config.hpp"
 #include "cli/drive_information.hpp"
 #include "cli/get.hpp"
 #include "cli/get_directory_items.hpp"
 #include "cli/get_pinned_files.hpp"
 #include "cli/help.hpp"
+#include "cli/list_objects.hpp"
 #include "cli/mount.hpp"
 #include "cli/open_files.hpp"
 #include "cli/pin_file.hpp"
@@ -42,12 +44,12 @@
 
 namespace repertory::cli::actions {
 using action = std::function<exit_code(
-    std::vector<const char *>, const std::string &, const provider_type &,
+    int, char **, const std::string &, const provider_type &,
     const std::string &, std::string, std::string)>;
 
 struct option_hasher {
   auto operator()(const utils::cli::option &opt) const -> std::size_t {
-    return std::hash<std::string>()(opt[0U] + '|' + opt[1U]);
+    return std::hash<std::string>()(opt[0u] + '|' + opt[1u]);
   }
 };
 
@@ -55,6 +57,11 @@ static const std::unordered_map<utils::cli::option, action, option_hasher>
     option_actions = {
         {utils::cli::options::check_version_option,
          cli::actions::check_version},
+#if defined(REPERTORY_ENABLE_S3)
+        {utils::cli::options::create_directory_option,
+         cli::actions::create_directory},
+        {utils::cli::options::list_objects_option, cli::actions::list_objects},
+#endif
         {utils::cli::options::display_config_option,
          cli::actions::display_config},
         {utils::cli::options::drive_information_option,
@@ -75,14 +82,14 @@ static const std::unordered_map<utils::cli::option, action, option_hasher>
 };
 
 [[nodiscard]] inline auto
-perform_action(const utils::cli::option &opt, std::vector<const char *> args,
-               const std::string &data_directory, const provider_type &prov,
+perform_action(const utils::cli::option &opt, int argc, char *argv[],
+               const std::string &data_directory, const provider_type &pt,
                const std::string &unique_id, std::string user,
                std::string password) -> exit_code {
-  if (utils::cli::has_option(args, opt)) {
+  if (utils::cli::has_option(argc, argv, opt)) {
     if (option_actions.find(opt) != option_actions.end()) {
-      return option_actions.at(opt)(args, data_directory, prov, unique_id, user,
-                                    password);
+      return option_actions.at(opt)(argc, argv, data_directory, pt, unique_id,
+                                    user, password);
     }
   }
 

include/cli/check_version.hpp

@@ -27,10 +27,9 @@
 
 namespace repertory::cli::actions {
 [[nodiscard]] inline auto
-check_version(std::vector<const char *> /* args */,
-              const std::string & /* data_directory */,
-              const provider_type & /* pt */, const std::string & /*unique_id*/,
-              std::string /*user*/, std::string /*password*/) -> exit_code {
+check_version(int, char *[], const std::string & /* data_directory */,
+              const provider_type & /* pt */, const std::string &, std::string,
+              std::string) -> exit_code {
   auto ret = exit_code::success;
 
   // TODO need to updated way to check version

include/cli/create_directory.hpp

@@ -0,0 +1,80 @@
+/*
+  Copyright <2018-2023> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all
+  copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  SOFTWARE.
+*/
+#ifndef INCLUDE_CLI_CREATE_DIRECTORY_HPP_
+#define INCLUDE_CLI_CREATE_DIRECTORY_HPP_
+#if defined(REPERTORY_ENABLE_S3)
+
+#include "app_config.hpp"
+#include "comm/s3/s3_comm.hpp"
+#include "platform/platform.hpp"
+#include "providers/i_provider.hpp"
+#include "providers/provider.hpp"
+#include "rpc/client/client.hpp"
+#include "types/repertory.hpp"
+#include "utils/cli_utils.hpp"
+
+namespace repertory::cli::actions {
+[[nodiscard]] inline auto
+create_directory(int argc, char *argv[], const std::string &data_directory,
+                 const provider_type &pt, const std::string &unique_id,
+                 std::string /* user */, std::string /* password */)
+    -> exit_code {
+  auto ret = exit_code::invalid_provider_type;
+
+  if (pt == provider_type::s3) {
+    std::string api_path;
+    if ((ret = utils::cli::parse_string_option(
+             argc, argv, utils::cli::options::create_directory_option,
+             api_path)) == exit_code::success) {
+      lock_data lock(pt, unique_id);
+      const auto lock_res = lock.grab_lock(1u);
+      /* if (res == lock_result::locked) { */
+      /* auto port = app_config::default_api_port(pt); */
+      /* utils::cli::get_api_authentication_data(user, password, port, pt,
+       * data_directory); */
+      /* const auto response = */
+      /*     client({"localhost", password, port,
+       * user}).create_directory(api_path); */
+      /* std::cout << static_cast<int>(response.response_type) << std::endl; */
+      /* std::cout << response.data.dump(2) << std::endl; */
+      /* } else */
+      if (lock_res == lock_result::success) {
+        std::cout << "creating directory: '" << api_path << "'" << std::endl;
+        app_config config(pt, data_directory);
+        s3_comm comm(config);
+        const auto res = comm.create_directory(api_path);
+        std::cout << api_error_to_string(res) << std::endl;
+        ret = exit_code::success;
+      } else {
+        std::cout << "failed to grab lock: '" << static_cast<int>(lock_res)
+                  << "'" << std::endl;
+        ret = exit_code::lock_failed;
+      }
+    }
+  }
+
+  return ret;
+}
+} // namespace repertory::cli::actions
+
+#endif // REPERTORY_ENABLE_S3
+#endif // INCLUDE_CLI_CREATE_DIRECTORY_HPP_

include/cli/display_config.hpp

@@ -30,22 +30,20 @@
 #include "utils/cli_utils.hpp"
 
 namespace repertory::cli::actions {
-[[nodiscard]] inline auto display_config(std::vector<const char *> /* args */,
-                                         const std::string &data_directory,
-                                         const provider_type &prov,
-                                         const std::string &unique_id,
-                                         std::string user, std::string password)
-    -> exit_code {
-  lock_data lock(prov, unique_id);
-  const auto res = lock.grab_lock(1U);
+[[nodiscard]] inline auto
+display_config(int, char *[], const std::string &data_directory,
+               const provider_type &pt, const std::string &unique_id,
+               std::string user, std::string password) -> exit_code {
+  lock_data lock(pt, unique_id);
+  const auto res = lock.grab_lock(1u);
   if (res == lock_result::success) {
-    app_config config(prov, data_directory);
+    app_config config(pt, data_directory);
     const auto cfg = config.get_json();
     std::cout << 0 << std::endl;
     std::cout << cfg.dump(2) << std::endl;
   } else if (res == lock_result::locked) {
-    auto port = app_config::default_api_port(prov);
-    utils::cli::get_api_authentication_data(user, password, port, prov,
+    auto port = app_config::default_api_port(pt);
+    utils::cli::get_api_authentication_data(user, password, port, pt,
                                             data_directory);
     const auto response =
         client({"localhost", password, port, user}).get_config();

include/cli/drive_information.hpp

@@ -31,25 +31,24 @@
 
 namespace repertory::cli::actions {
 [[nodiscard]] inline auto
-drive_information(std::vector<const char *> /* args */,
-                  const std::string &data_directory, const provider_type &prov,
-                  const std::string &unique_id, std::string user,
-                  std::string password) -> exit_code {
+drive_information(int, char *[], const std::string &data_directory,
+                  const provider_type &pt, const std::string &unique_id,
+                  std::string user, std::string password) -> exit_code {
   auto ret = exit_code::success;
 
-  lock_data lock(prov, unique_id);
-  const auto res = lock.grab_lock(1U);
+  lock_data lock(pt, unique_id);
+  const auto res = lock.grab_lock(1u);
   if (res == lock_result::locked) {
-    auto port = app_config::default_api_port(prov);
-    utils::cli::get_api_authentication_data(user, password, port, prov,
+    auto port = app_config::default_api_port(pt);
+    utils::cli::get_api_authentication_data(user, password, port, pt,
                                             data_directory);
     const auto response =
         client({"localhost", password, port, user}).get_drive_information();
     std::cout << static_cast<int>(response.response_type) << std::endl;
     std::cout << response.data.dump(2) << std::endl;
   } else {
-    std::cerr << app_config::get_provider_display_name(prov)
-              << " is not mounted." << std::endl;
+    std::cerr << app_config::get_provider_display_name(pt) << " is not mounted."
+              << std::endl;
     ret = exit_code::not_mounted;
   }
 

include/cli/get.hpp

@@ -30,19 +30,19 @@
 #include "utils/cli_utils.hpp"
 
 namespace repertory::cli::actions {
-[[nodiscard]] inline auto get(std::vector<const char *> args,
+[[nodiscard]] inline auto get(int argc, char *argv[],
                               const std::string &data_directory,
-                              const provider_type &prov,
+                              const provider_type &pt,
                               const std::string &unique_id, std::string user,
                               std::string password) -> exit_code {
   std::string data;
   auto ret = utils::cli::parse_string_option(
-      args, repertory::utils::cli::options::get_option, data);
+      argc, argv, repertory::utils::cli::options::get_option, data);
   if (ret == exit_code::success) {
-    lock_data lock(prov, unique_id);
+    lock_data lock(pt, unique_id);
     const auto res = lock.grab_lock(1);
     if (res == lock_result::success) {
-      app_config config(prov, data_directory);
+      app_config config(pt, data_directory);
       const auto value = config.get_value_by_name(data);
       std::cout << (value.empty()
                         ? static_cast<int>(
@@ -51,8 +51,8 @@ namespace repertory::cli::actions {
                 << std::endl;
       std::cout << json({{"value", value}}).dump(2) << std::endl;
     } else if (res == lock_result::locked) {
-      auto port = app_config::default_api_port(prov);
-      utils::cli::get_api_authentication_data(user, password, port, prov,
+      auto port = app_config::default_api_port(pt);
+      utils::cli::get_api_authentication_data(user, password, port, pt,
                                               data_directory);
       const auto response = client({"localhost", password, port, user})
                                 .get_config_value_by_name(data);

include/cli/get_directory_items.hpp

@@ -29,16 +29,17 @@
 #include "utils/cli_utils.hpp"
 
 namespace repertory::cli::actions {
-[[nodiscard]] inline auto get_directory_items(
-    std::vector<const char *> args, const std::string &data_directory,
-    const provider_type &prov, const std::string & /* unique_id */,
-    std::string user, std::string password) -> exit_code {
+[[nodiscard]] inline auto
+get_directory_items(int argc, char *argv[], const std::string &data_directory,
+                    const provider_type &pt, const std::string &,
+                    std::string user, std::string password) -> exit_code {
   std::string data;
   auto ret = utils::cli::parse_string_option(
-      args, repertory::utils::cli::options::get_directory_items_option, data);
+      argc, argv, repertory::utils::cli::options::get_directory_items_option,
+      data);
   if (ret == exit_code::success) {
-    auto port = app_config::default_api_port(prov);
-    utils::cli::get_api_authentication_data(user, password, port, prov,
+    auto port = app_config::default_api_port(pt);
+    utils::cli::get_api_authentication_data(user, password, port, pt,
                                             data_directory);
     const auto response =
         client({"localhost", password, port, user}).get_directory_items(data);

include/cli/get_pinned_files.hpp

@@ -29,15 +29,13 @@
 #include "utils/cli_utils.hpp"
 
 namespace repertory::cli::actions {
-[[nodiscard]] inline auto get_pinned_files(std::vector<const char *> /* args */,
-                                           const std::string &data_directory,
-                                           const provider_type &prov,
-                                           const std::string & /* unique_id */,
-                                           std::string user,
-                                           std::string password) -> exit_code {
+[[nodiscard]] inline auto
+get_pinned_files(int, char *[], const std::string &data_directory,
+                 const provider_type &pt, const std::string &, std::string user,
+                 std::string password) -> exit_code {
   auto ret = exit_code::success;
-  auto port = app_config::default_api_port(prov);
-  utils::cli::get_api_authentication_data(user, password, port, prov,
+  auto port = app_config::default_api_port(pt);
+  utils::cli::get_api_authentication_data(user, password, port, pt,
                                           data_directory);
   const auto response =
       client({"localhost", password, port, user}).get_pinned_files();

include/cli/help.hpp

@@ -23,8 +23,8 @@
 #define INCLUDE_CLI_HELP_HPP_
 
 namespace repertory::cli::actions {
-template <typename drive> inline void help(std::vector<const char *> args) {
-  drive::display_options(args);
+template <typename drive> inline void help(int argc, char *argv[]) {
+  drive::display_options(argc, argv);
   std::cout << "Repertory options:" << std::endl;
   std::cout << "    -cv,--check_version               Check daemon version "
                "compatibility"
@@ -42,6 +42,11 @@ template <typename drive> inline void help(std::vector<const char *> args) {
   std::cout << "    -na,--name                        Unique name for S3 "
                "instance [Required]"
             << std::endl;
+  std::cout
+      << "    -cd,--create_directory [API path] Create directory object in S3"
+      << std::endl;
+  std::cout << "    -lo,--list_objects                List all S3 objects"
+            << std::endl;
 #endif // defined(REPERTORY_ENABLE_S3)
   std::cout
       << "    -gc,--generate_config             Generate initial configuration"

include/cli/list_objects.hpp

@@ -0,0 +1,71 @@
+/*
+  Copyright <2018-2023> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all
+  copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  SOFTWARE.
+*/
+#ifndef INCLUDE_CLI_LIST_OBJECTS_HPP_
+#define INCLUDE_CLI_LIST_OBJECTS_HPP_
+#if defined(REPERTORY_ENABLE_S3)
+
+#include "app_config.hpp"
+#include "comm/s3/s3_comm.hpp"
+#include "platform/platform.hpp"
+#include "providers/i_provider.hpp"
+#include "providers/provider.hpp"
+#include "rpc/client/client.hpp"
+#include "types/repertory.hpp"
+#include "utils/cli_utils.hpp"
+
+namespace repertory::cli::actions {
+[[nodiscard]] inline auto
+list_objects(int /* argc */, char * /* argv */[],
+             const std::string &data_directory, const provider_type &pt,
+             const std::string &unique_id, std::string /* user */,
+             std::string /* password */) -> exit_code {
+  auto ret = exit_code::invalid_provider_type;
+
+  if (pt == provider_type::s3) {
+    lock_data lock(pt, unique_id);
+    auto lock_res = lock.grab_lock(1u);
+    if (lock_res == lock_result::success) {
+      app_config config(pt, data_directory);
+      s3_comm comm(config);
+      std::vector<directory_item> list{};
+      const auto res = comm.get_object_list(list);
+      if (res == api_error::success) {
+        for (const auto &di : list) {
+          std::cout << di.to_json().dump(2) << std::endl;
+        }
+        ret = exit_code::success;
+      } else {
+        std::cout << api_error_to_string(res) << std::endl;
+      }
+    } else {
+      std::cout << "failed to grab lock: '" << static_cast<int>(lock_res) << "'"
+                << std::endl;
+      ret = exit_code::lock_failed;
+    }
+  }
+
+  return ret;
+}
+} // namespace repertory::cli::actions
+
+#endif // REPERTORY_ENABLE_S3
+#endif // INCLUDE_CLI_LIST_OBJECTS_HPP_

include/cli/mount.hpp

@@ -54,30 +54,30 @@ using remote_instance = repertory::remote_fuse::i_remote_instance;
 
 namespace repertory::cli::actions {
 [[nodiscard]] inline auto
-mount(std::vector<const char *> args, std::string data_directory,
-      int &mount_result, provider_type prov, const std::string &remote_host,
+mount(int argc, char *argv[], std::string data_directory, int &mount_result,
+      provider_type pt, const std::string &remote_host,
       std::uint16_t remote_port, const std::string &unique_id) -> exit_code {
   auto ret = exit_code::success;
 
-  lock_data lock(prov, unique_id);
+  lock_data lock(pt, unique_id);
   const auto res = lock.grab_lock();
   if (res == lock_result::locked) {
     ret = exit_code::mount_active;
-    std::cerr << app_config::get_provider_display_name(prov)
+    std::cerr << app_config::get_provider_display_name(pt)
               << " mount is already active" << std::endl;
   } else if (res == lock_result::success) {
     const auto generate_config = utils::cli::has_option(
-        args, utils::cli::options::generate_config_option);
+        argc, argv, utils::cli::options::generate_config_option);
     if (generate_config) {
-      app_config config(prov, data_directory);
-      if (prov == provider_type::remote) {
+      app_config config(pt, data_directory);
+      if (pt == provider_type::remote) {
         config.set_enable_remote_mount(false);
         config.set_is_remote_mount(true);
         config.set_remote_host_name_or_ip(remote_host);
         config.set_remote_port(remote_port);
         config.save();
       }
-      std::cout << "Generated " << app_config::get_provider_display_name(prov)
+      std::cout << "Generated " << app_config::get_provider_display_name(pt)
                 << " Configuration" << std::endl;
       std::cout << config.get_config_file_path() << std::endl;
       ret = utils::file::is_file(config.get_config_file_path())
@@ -85,13 +85,14 @@ mount(std::vector<const char *> args, std::string data_directory,
                 : exit_code::file_creation_failed;
     } else {
 #ifdef _WIN32
-      if (utils::cli::has_option(args, utils::cli::options::hidden_option)) {
+      if (utils::cli::has_option(argc, argv,
+                                 utils::cli::options::hidden_option)) {
         ::ShowWindow(::GetConsoleWindow(), SW_HIDE);
       }
 #endif
       const auto drive_args =
-          utils::cli::parse_drive_options(args, prov, data_directory);
-      app_config config(prov, data_directory);
+          utils::cli::parse_drive_options(argc, argv, pt, data_directory);
+      app_config config(pt, data_directory);
 #ifdef _WIN32
       if (config.get_enable_mount_manager() &&
           not utils::is_process_elevated()) {
@@ -101,8 +102,8 @@ mount(std::vector<const char *> args, std::string data_directory,
         }
         lock.release();
 
-        mount_result = utils::run_process_elevated(args);
-        lock_data lock2(prov, unique_id);
+        mount_result = utils::run_process_elevated(argc, argv);
+        lock_data lock2(pt, unique_id);
         if (lock2.grab_lock() == lock_result::success) {
           if (not lock2.set_mount_state(false, "", -1)) {
             std::cerr << "failed to set mount state" << std::endl;
@@ -113,16 +114,15 @@ mount(std::vector<const char *> args, std::string data_directory,
         return exit_code::mount_result;
       }
 #endif
-      std::cout << "Initializing "
-                << app_config::get_provider_display_name(prov)
+      std::cout << "Initializing " << app_config::get_provider_display_name(pt)
                 << (unique_id.empty() ? ""
-                    : (prov == provider_type::s3)
+                    : (pt == provider_type::s3)
                         ? " [" + unique_id + ']'
                         : " [" + remote_host + ':' +
                               std::to_string(remote_port) + ']')
                 << " Drive" << std::endl;
-      if (prov == provider_type::remote) {
-        std::uint16_t port{0U};
+      if (pt == provider_type::remote) {
+        std::uint16_t port = 0u;
         if (utils::get_next_available_port(config.get_api_port(), port)) {
           config.set_remote_host_name_or_ip(remote_host);
           config.set_remote_port(remote_port);
@@ -155,7 +155,7 @@ mount(std::vector<const char *> args, std::string data_directory,
         config.set_is_remote_mount(false);
 
         try {
-          auto provider = create_provider(prov, config);
+          auto provider = create_provider(pt, config);
           repertory_drive drive(config, lock, *provider);
           if (not lock.set_mount_state(true, "", -1)) {
             std::cerr << "failed to set mount state" << std::endl;

include/cli/open_files.hpp

@@ -31,24 +31,23 @@
 
 namespace repertory::cli::actions {
 [[nodiscard]] inline auto
-open_files(std::vector<const char *> /* args */,
-           const std::string &data_directory, const provider_type &prov,
-           const std::string &unique_id, std::string user, std::string password)
-    -> exit_code {
+open_files(int, char *[], const std::string &data_directory,
+           const provider_type &pt, const std::string &unique_id,
+           std::string user, std::string password) -> exit_code {
   auto ret = exit_code::success;
-  lock_data lock(prov, unique_id);
-  const auto res = lock.grab_lock(1U);
+  lock_data lock(pt, unique_id);
+  const auto res = lock.grab_lock(1u);
   if (res == lock_result::locked) {
-    auto port = app_config::default_api_port(prov);
-    utils::cli::get_api_authentication_data(user, password, port, prov,
+    auto port = app_config::default_api_port(pt);
+    utils::cli::get_api_authentication_data(user, password, port, pt,
                                             data_directory);
     const auto response =
         client({"localhost", password, port, user}).get_open_files();
     std::cout << static_cast<int>(response.response_type) << std::endl;
     std::cout << response.data.dump(2) << std::endl;
   } else {
-    std::cerr << app_config::get_provider_display_name(prov)
-              << " is not mounted." << std::endl;
+    std::cerr << app_config::get_provider_display_name(pt) << " is not mounted."
+              << std::endl;
     ret = exit_code::not_mounted;
   }
 

include/cli/pin_file.hpp

@@ -29,16 +29,17 @@
 #include "utils/cli_utils.hpp"
 
 namespace repertory::cli::actions {
-[[nodiscard]] inline auto
-pin_file(std::vector<const char *> args, const std::string &data_directory,
-         const provider_type &prov, const std::string & /* unique_id */,
-         std::string user, std::string password) -> exit_code {
+[[nodiscard]] inline auto pin_file(int argc, char *argv[],
+                                   const std::string &data_directory,
+                                   const provider_type &pt, const std::string &,
+                                   std::string user, std::string password)
+    -> exit_code {
   std::string data;
   auto ret = utils::cli::parse_string_option(
-      args, repertory::utils::cli::options::pin_file_option, data);
+      argc, argv, repertory::utils::cli::options::pin_file_option, data);
   if (ret == exit_code::success) {
-    auto port = app_config::default_api_port(prov);
-    utils::cli::get_api_authentication_data(user, password, port, prov,
+    auto port = app_config::default_api_port(pt);
+    utils::cli::get_api_authentication_data(user, password, port, pt,
                                             data_directory);
     const auto response =
         client({"localhost", password, port, user}).pin_file(data);

include/cli/pinned_status.hpp

@@ -29,16 +29,17 @@
 #include "utils/cli_utils.hpp"
 
 namespace repertory::cli::actions {
-[[nodiscard]] inline auto
-pinned_status(std::vector<const char *> args, const std::string &data_directory,
-              const provider_type &prov, const std::string & /*unique_id*/,
-              std::string user, std::string password) -> exit_code {
+[[nodiscard]] inline auto pinned_status(int argc, char *argv[],
+                                        const std::string &data_directory,
+                                        const provider_type &pt,
+                                        const std::string &, std::string user,
+                                        std::string password) -> exit_code {
   std::string data;
   auto ret = utils::cli::parse_string_option(
-      args, repertory::utils::cli::options::pinned_status_option, data);
+      argc, argv, repertory::utils::cli::options::pinned_status_option, data);
   if (ret == exit_code::success) {
-    auto port = app_config::default_api_port(prov);
-    utils::cli::get_api_authentication_data(user, password, port, prov,
+    auto port = app_config::default_api_port(pt);
+    utils::cli::get_api_authentication_data(user, password, port, pt,
                                             data_directory);
     const auto response =
         client({"localhost", password, port, user}).pinned_status(data);

include/cli/set.hpp

@@ -30,27 +30,27 @@
 #include "utils/cli_utils.hpp"
 
 namespace repertory::cli::actions {
-[[nodiscard]] inline auto set(std::vector<const char *> args,
+[[nodiscard]] inline auto set(int argc, char *argv[],
                               const std::string &data_directory,
-                              const provider_type &prov,
+                              const provider_type &pt,
                               const std::string &unique_id, std::string user,
                               std::string password) -> exit_code {
   auto ret = exit_code::success;
-  auto data = utils::cli::parse_option(args, "-set", 2U);
+  auto data = utils::cli::parse_option(argc, argv, "-set", 2u);
   if (data.empty()) {
-    data = utils::cli::parse_option(args, "--set", 2U);
+    data = utils::cli::parse_option(argc, argv, "--set", 2u);
     if (data.empty()) {
       ret = exit_code::invalid_syntax;
       std::cerr << "Invalid syntax for '-set'" << std::endl;
     }
   }
   if (ret == exit_code::success) {
-    lock_data lock(prov, unique_id);
-    const auto res = lock.grab_lock(1U);
+    lock_data lock(pt, unique_id);
+    const auto res = lock.grab_lock(1u);
     if (res == lock_result::success) {
-      app_config config(prov, data_directory);
-      const auto value = config.set_value_by_name(data[0U], data[1U]);
-      const auto notFound = value.empty() && not data[1U].empty();
+      app_config config(pt, data_directory);
+      const auto value = config.set_value_by_name(data[0u], data[1u]);
+      const auto notFound = value.empty() && not data[1u].empty();
       ret = notFound ? exit_code::set_option_not_found : exit_code::success;
       std::cout << (notFound ? static_cast<int>(
                                    rpc_response_type::config_value_not_found)
@@ -58,11 +58,11 @@ namespace repertory::cli::actions {
                 << std::endl;
       std::cout << json({{"value", value}}).dump(2) << std::endl;
     } else if (res == lock_result::locked) {
-      auto port = app_config::default_api_port(prov);
-      utils::cli::get_api_authentication_data(user, password, port, prov,
+      auto port = app_config::default_api_port(pt);
+      utils::cli::get_api_authentication_data(user, password, port, pt,
                                               data_directory);
       const auto response = client({"localhost", password, port, user})
-                                .set_config_value_by_name(data[0U], data[1U]);
+                                .set_config_value_by_name(data[0u], data[1u]);
       std::cout << static_cast<int>(response.response_type) << std::endl;
       std::cout << response.data.dump(2) << std::endl;
       ret = response.response_type == rpc_response_type::config_value_not_found

include/cli/status.hpp

@@ -26,15 +26,13 @@
 #include "types/repertory.hpp"
 
 namespace repertory::cli::actions {
-[[nodiscard]] inline auto status(std::vector<const char *> /* args */,
-                                 const std::string & /*data_directory*/,
-                                 const provider_type &prov,
-                                 const std::string &unique_id,
-                                 std::string /* user */,
-                                 std::string /* password */) -> exit_code {
+[[nodiscard]] inline auto status(int, char *[], const std::string &,
+                                 const provider_type &pt,
+                                 const std::string &unique_id, std::string,
+                                 std::string) -> exit_code {
   auto ret = exit_code::success;
-  lock_data lock(prov, unique_id);
-  [[maybe_unused]] auto status = lock.grab_lock(10U);
+  lock_data lock(pt, unique_id);
+  [[maybe_unused]] auto status = lock.grab_lock(10u);
   json mount_state;
   if (lock.get_mount_state(mount_state)) {
     std::cout << mount_state.dump(2) << std::endl;

include/cli/unmount.hpp

@@ -29,13 +29,14 @@
 #include "utils/cli_utils.hpp"
 
 namespace repertory::cli::actions {
-[[nodiscard]] inline auto
-unmount(std::vector<const char *> /* args */, const std::string &data_directory,
-        const provider_type &prov, const std::string & /*unique_id*/,
-        std::string user, std::string password) -> exit_code {
+[[nodiscard]] inline auto unmount(int, char *[],
+                                  const std::string &data_directory,
+                                  const provider_type &pt, const std::string &,
+                                  std::string user, std::string password)
+    -> exit_code {
   auto ret = exit_code::success;
-  auto port = app_config::default_api_port(prov);
-  utils::cli::get_api_authentication_data(user, password, port, prov,
+  auto port = app_config::default_api_port(pt);
+  utils::cli::get_api_authentication_data(user, password, port, pt,
                                           data_directory);
   const auto response = client({"localhost", password, port, user}).unmount();
   std::cout << static_cast<int>(response.response_type) << std::endl;

include/cli/unpin_file.hpp

@@ -29,16 +29,17 @@
 #include "utils/cli_utils.hpp"
 
 namespace repertory::cli::actions {
-[[nodiscard]] inline auto
-unpin_file(std::vector<const char *> args, const std::string &data_directory,
-           const provider_type &prov, const std::string & /*unique_id*/,
-           std::string user, std::string password) -> exit_code {
+[[nodiscard]] inline auto unpin_file(int argc, char *argv[],
+                                     const std::string &data_directory,
+                                     const provider_type &pt,
+                                     const std::string &, std::string user,
+                                     std::string password) -> exit_code {
   std::string data;
   auto ret = utils::cli::parse_string_option(
-      args, repertory::utils::cli::options::unpin_file_option, data);
+      argc, argv, repertory::utils::cli::options::unpin_file_option, data);
   if (ret == exit_code::success) {
-    auto port = app_config::default_api_port(prov);
-    utils::cli::get_api_authentication_data(user, password, port, prov,
+    auto port = app_config::default_api_port(pt);
+    utils::cli::get_api_authentication_data(user, password, port, pt,
                                             data_directory);
     const auto response =
         client({"localhost", password, port, user}).unpin_file(data);

include/cli/version.hpp

@@ -23,12 +23,12 @@
 #define INCLUDE_CLI_VERSION_HPP_
 
 namespace repertory::cli::actions {
-template <typename drive> inline void version(std::vector<const char *> args) {
+template <typename drive> inline void version(int argc, char *argv[]) {
   std::cout << "Repertory core version: " << get_repertory_version()
             << std::endl;
   std::cout << "Repertory Git revision: " << get_repertory_git_revision()
             << std::endl;
-  drive::display_version_information(args);
+  drive::display_version_information(argc, argv);
 }
 } // namespace repertory::cli::actions
 

include/comm/curl/curl_comm.hpp

@@ -24,8 +24,6 @@
 
 #include "comm/curl/multi_request.hpp"
 #include "comm/i_http_comm.hpp"
-#include "events/event_system.hpp"
-#include "events/events.hpp"
 #include "utils/encryption.hpp"
 #include "utils/utils.hpp"
 
@@ -198,7 +196,6 @@ public:
     CURLcode curl_code{};
     curl_request.get_result(curl_code, response_code);
     if (curl_code != CURLE_OK) {
-      event_system::instance().raise<curl_error>(url, curl_code);
       return false;
     }
 
@@ -227,11 +224,6 @@ public:
                                   stop_type &stop_requested) const
       -> bool override;
 
-  [[nodiscard]] auto make_request(const curl::requests::http_post &post_file,
-                                  long &response_code,
-                                  stop_type &stop_requested) const
-      -> bool override;
-
   [[nodiscard]] auto make_request(const curl::requests::http_put_file &put_file,
                                   long &response_code,
                                   stop_type &stop_requested) const

include/comm/curl/requests/http_put_file.hpp

@@ -27,17 +27,14 @@
 
 namespace repertory::curl::requests {
 struct http_put_file final : http_request_base {
-  http_put_file() = default;
-  http_put_file(const http_put_file &) = default;
-  http_put_file(http_put_file &&) = default;
-
-  auto operator=(const http_put_file &) -> http_put_file & = default;
-  auto operator=(http_put_file &&) -> http_put_file & = default;
-
   ~http_put_file() override = default;
 
-  std::shared_ptr<utils::encryption::encrypting_reader> reader;
-  std::string source_path;
+  std::optional<std::string> encryption_token{};
+  std::string file_name{};
+  mutable std::shared_ptr<utils::encryption::encrypting_reader> reader{};
+  std::string source_path{};
+
+  [[nodiscard]] auto get_path() const -> std::string override;
 
   [[nodiscard]] auto set_method(CURL *curl, stop_type &stop_requested) const
       -> bool override;

include/comm/curl/requests/http_request_base.hpp

@@ -54,10 +54,8 @@ struct http_request_base {
   http_request_base() = default;
   http_request_base(const http_request_base &) = default;
   http_request_base(http_request_base &&) = default;
-
   auto operator=(const http_request_base &) -> http_request_base & = default;
   auto operator=(http_request_base &&) -> http_request_base & = default;
-
   virtual ~http_request_base() = default;
 
   bool allow_timeout{};

include/comm/i_http_comm.hpp

@@ -25,7 +25,6 @@
 #include "comm/curl/requests/http_delete.hpp"
 #include "comm/curl/requests/http_get.hpp"
 #include "comm/curl/requests/http_head.hpp"
-#include "comm/curl/requests/http_post.hpp"
 #include "comm/curl/requests/http_put_file.hpp"
 #include "types/repertory.hpp"
 
@@ -50,11 +49,6 @@ public:
                                           stop_type &stop_requested) const
       -> bool = 0;
 
-  [[nodiscard]] virtual auto make_request(const curl::requests::http_post &post,
-                                          long &response_code,
-                                          stop_type &stop_requested) const
-      -> bool = 0;
-
   [[nodiscard]] virtual auto
   make_request(const curl::requests::http_put_file &put_file,
                long &response_code, stop_type &stop_requested) const

include/comm/i_s3_comm.hpp

@@ -0,0 +1,109 @@
+/*
+  Copyright <2018-2023> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all
+  copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  SOFTWARE.
+*/
+#ifndef INCLUDE_COMM_I_S3_COMM_HPP_
+#define INCLUDE_COMM_I_S3_COMM_HPP_
+#if defined(REPERTORY_ENABLE_S3)
+
+#include "types/repertory.hpp"
+#include "types/s3.hpp"
+
+namespace repertory {
+class i_provider;
+
+class i_s3_comm {
+  INTERFACE_SETUP(i_s3_comm);
+
+public:
+  [[nodiscard]] virtual auto create_directory(const std::string &api_path)
+      -> api_error = 0;
+
+  [[nodiscard]] virtual auto directory_exists(const std::string &api_path) const
+      -> api_error = 0;
+
+  [[nodiscard]] virtual auto file_exists(const std::string &api_path,
+                                         const get_key_callback &get_key) const
+      -> api_error = 0;
+
+  [[nodiscard]] virtual auto
+  get_directory_item_count(const std::string &api_path,
+                           meta_provider_callback meta_provider) const
+      -> std::size_t = 0;
+
+  [[nodiscard]] virtual auto
+  get_directory_items(const std::string &api_path,
+                      meta_provider_callback meta_provider,
+                      directory_item_list &list) const -> api_error = 0;
+
+  [[nodiscard]] virtual auto get_directory_list(api_file_list &list) const
+      -> api_error = 0;
+
+  [[nodiscard]] virtual auto get_file(const std::string &api_path,
+                                      const get_key_callback &get_key,
+                                      const get_name_callback &get_name,
+                                      const get_token_callback &get_token,
+                                      api_file &file) const -> api_error = 0;
+
+  [[nodiscard]] virtual auto
+  get_file_list(const get_api_file_token_callback &get_api_file_token,
+                const get_name_callback &get_name, api_file_list &list) const
+      -> api_error = 0;
+
+  [[nodiscard]] virtual auto
+  get_object_list(std::vector<directory_item> &list) const -> api_error = 0;
+
+  [[nodiscard]] virtual auto
+  get_object_name(const std::string &api_path,
+                  const get_key_callback &get_key) const -> std::string = 0;
+
+  [[nodiscard]] virtual auto get_s3_config() -> s3_config = 0;
+
+  [[nodiscard]] virtual auto get_s3_config() const -> s3_config = 0;
+
+  [[nodiscard]] virtual auto is_online() const -> bool = 0;
+
+  [[nodiscard]] virtual auto read_file_bytes(
+      const std::string &api_path, std::size_t size, std::uint64_t offset,
+      data_buffer &data, const get_key_callback &get_key,
+      const get_size_callback &get_size, const get_token_callback &get_token,
+      stop_type &stop_requested) const -> api_error = 0;
+
+  [[nodiscard]] virtual auto remove_directory(const std::string &api_path)
+      -> api_error = 0;
+
+  [[nodiscard]] virtual auto remove_file(const std::string &api_path,
+                                         const get_key_callback &get_key)
+      -> api_error = 0;
+
+  [[nodiscard]] virtual auto rename_file(const std::string &api_path,
+                                         const std::string &new_api_path)
+      -> api_error = 0;
+
+  [[nodiscard]] virtual auto
+  upload_file(const std::string &api_path, const std::string &source_path,
+              const std::string &encryption_token,
+              const get_key_callback &get_key, const set_key_callback &set_key,
+              stop_type &stop_requested) -> api_error = 0;
+};
+} // namespace repertory
+
+#endif // REPERTORY_ENABLE_S3
+#endif // INCLUDE_COMM_I_S3_COMM_HPP_

include/comm/s3/s3_comm.hpp

@@ -0,0 +1,177 @@
+/*
+  Copyright <2018-2023> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all
+  copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  SOFTWARE.
+*/
+#ifndef INCLUDE_COMM_S3_S3_COMM_HPP_
+#define INCLUDE_COMM_S3_S3_COMM_HPP_
+#if defined(REPERTORY_ENABLE_S3)
+
+#include "comm/i_http_comm.hpp"
+#include "comm/i_s3_comm.hpp"
+#include "types/repertory.hpp"
+
+namespace repertory {
+class app_config;
+
+class s3_comm final : public i_s3_comm {
+public:
+  explicit s3_comm(const app_config &config);
+  s3_comm(s3_comm &&comm);
+
+  ~s3_comm() override;
+
+private:
+  struct cache_entry final {
+    std::chrono::system_clock::time_point expiration;
+    directory_item_list items;
+
+    void reset_timeout(std::chrono::seconds timeout) {
+      timeout = std::max(std::chrono::seconds(5u), timeout);
+      expiration = std::chrono::system_clock::now() + timeout;
+    }
+  };
+
+private:
+  const app_config &config_;
+  s3_config s3_config_;
+  std::unique_ptr<i_http_comm> s3_client_;
+
+private:
+  mutable std::recursive_mutex cached_directories_mutex_;
+  mutable std::unordered_map<std::string, cache_entry> cached_directories_;
+
+protected:
+  bool active_ = true;
+
+private:
+  void clear_expired_directories();
+
+  [[nodiscard]] auto
+  get_cached_directory_item_count(const std::string &api_path,
+                                  std::size_t &count) const -> bool;
+
+  [[nodiscard]] auto
+  get_cached_directory_items(const std::string &api_path,
+                             meta_provider_callback meta_provider,
+                             directory_item_list &list) const -> bool;
+
+  [[nodiscard]] auto get_cached_file_exists(const std::string &api_path) const
+      -> bool;
+
+  [[nodiscard]] auto grab_directory_items(const std::string &api_path,
+                                          meta_provider_callback meta_provider,
+                                          directory_item_list &list) const
+      -> api_error;
+
+  void raise_begin(const std::string &function_name,
+                   const std::string &api_path) const;
+
+  [[nodiscard]] auto raise_end(const std::string &function_name,
+                               const std::string &api_path,
+                               const api_error &error, long code) const
+      -> api_error;
+
+  void remove_cached_directory(const std::string &api_path);
+
+  void set_cached_directory_items(const std::string &api_path,
+                                  directory_item_list list) const;
+
+public:
+  [[nodiscard]] auto create_directory(const std::string &api_path)
+      -> api_error override;
+
+  [[nodiscard]] auto directory_exists(const std::string &api_path) const
+      -> api_error override;
+
+  [[nodiscard]] auto file_exists(const std::string &api_path,
+                                 const get_key_callback &get_key) const
+      -> api_error override;
+
+  [[nodiscard]] auto
+  get_directory_item_count(const std::string &api_path,
+                           meta_provider_callback meta_provider) const
+      -> std::size_t override;
+
+  [[nodiscard]] auto get_directory_items(const std::string &api_path,
+                                         meta_provider_callback meta_provider,
+                                         directory_item_list &list) const
+      -> api_error override;
+
+  [[nodiscard]] auto get_directory_list(api_file_list &list) const
+      -> api_error override;
+
+  [[nodiscard]] auto get_file(const std::string &api_path,
+                              const get_key_callback &get_key,
+                              const get_name_callback &get_name,
+                              const get_token_callback &get_token,
+                              api_file &file) const -> api_error override;
+
+  [[nodiscard]] auto
+  get_file_list(const get_api_file_token_callback &get_api_file_token,
+                const get_name_callback &get_name, api_file_list &list) const
+      -> api_error override;
+
+  [[nodiscard]] auto get_object_list(std::vector<directory_item> &list) const
+      -> api_error override;
+
+  [[nodiscard]] auto get_object_name(const std::string &api_path,
+                                     const get_key_callback &getKey) const
+      -> std::string override;
+
+  [[nodiscard]] auto get_s3_config() -> s3_config override {
+    return s3_config_;
+  }
+
+  [[nodiscard]] auto get_s3_config() const -> s3_config override {
+    return s3_config_;
+  }
+
+  [[nodiscard]] auto is_online() const -> bool override {
+    // TODO implement this
+    return true;
+  }
+
+  [[nodiscard]] auto read_file_bytes(
+      const std::string &api_path, std::size_t size, std::uint64_t offset,
+      data_buffer &data, const get_key_callback &get_key,
+      const get_size_callback &get_size, const get_token_callback &get_token,
+      stop_type &stop_requested) const -> api_error override;
+
+  [[nodiscard]] auto remove_directory(const std::string &api_path)
+      -> api_error override;
+
+  [[nodiscard]] auto remove_file(const std::string &api_path,
+                                 const get_key_callback &get_key)
+      -> api_error override;
+
+  [[nodiscard]] auto rename_file(const std::string &api_path,
+                                 const std::string &new_api_path)
+      -> api_error override;
+
+  [[nodiscard]] auto
+  upload_file(const std::string &api_path, const std::string &source_path,
+              const std::string &encryption_token,
+              const get_key_callback &get_key, const set_key_callback &set_key,
+              stop_type &stop_requested) -> api_error override;
+};
+} // namespace repertory
+
+#endif // REPERTORY_ENABLE_S3
+#endif // INCLUDE_COMM_S3_S3_COMM_HPP_

include/comm/s3/s3_requests.hpp

@@ -0,0 +1,117 @@
+/*
+  Copyright <2018-2023> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all
+  copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  SOFTWARE.
+*/
+#ifndef INCLUDE_COMM_S3_S3_REQUESTS_HPP_
+#define INCLUDE_COMM_S3_S3_REQUESTS_HPP_
+#if defined(REPERTORY_ENABLE_S3)
+
+#include "comm/s3/s3_requests_curl.hpp"
+
+#include "types/repertory.hpp"
+#include "types/s3.hpp"
+#include "utils/string_utils.hpp"
+
+namespace repertory {
+template <typename client_type>
+[[nodiscard]] inline auto
+create_directory_object_request(client_type &client, const s3_config &config,
+                                const std::string &object_name,
+                                long &response_code) -> bool {
+  return create_directory_object_request_impl(client, config, object_name,
+                                              response_code);
+}
+
+template <typename client_type>
+[[nodiscard]] inline auto delete_object_request(client_type &client,
+                                                const s3_config &config,
+                                                const std::string &object_name,
+                                                long &response_code) -> bool {
+  return delete_object_request_impl(client, config, object_name, response_code);
+}
+
+template <typename client_type>
+[[nodiscard]] inline auto
+head_object_request(client_type &client, const s3_config &config,
+                    const std::string &object_name, head_object_result &result,
+                    long &response_code) -> bool {
+  return head_object_request_impl(client, config, object_name, result,
+                                  response_code);
+}
+
+template <typename client_type>
+[[nodiscard]] inline auto
+list_directories_request(client_type &client, const s3_config &config,
+                         list_directories_result &result, long &response_code)
+    -> bool {
+  return list_directories_request_impl(client, config, result, response_code);
+}
+
+template <typename client_type>
+[[nodiscard]] inline auto
+list_files_request(client_type &client, const s3_config &config,
+                   const get_api_file_token_callback &get_api_file_token,
+                   const get_name_callback &get_name, list_files_result &result,
+                   long &response_code) -> bool {
+  return list_files_request_impl(client, config, get_api_file_token, get_name,
+                                 result, response_code);
+}
+
+template <typename client_type>
+[[nodiscard]] inline auto list_objects_in_directory_request(
+    client_type &client, const s3_config &config,
+    const std::string &object_name, meta_provider_callback meta_provider,
+    list_objects_result &result, long &response_code) -> bool {
+  return list_objects_in_directory_request_impl(
+      client, config, object_name, meta_provider, result, response_code);
+}
+
+template <typename client_type>
+[[nodiscard]] inline auto
+list_objects_request(client_type &client, const s3_config &config,
+                     list_objects_result &result, long &response_code) -> bool {
+  return list_objects_request_impl(client, config, result, response_code);
+}
+
+template <typename client_type>
+[[nodiscard]] inline auto
+put_object_request(client_type &client, const s3_config &config,
+                   std::string object_name, const std::string &source_path,
+                   const std::string &encryption_token,
+                   get_key_callback get_key, set_key_callback set_key,
+                   long &response_code, stop_type &stop_requested) -> bool {
+  return put_object_request_impl(client, config, object_name, source_path,
+                                 encryption_token, get_key, set_key,
+                                 response_code, stop_requested);
+}
+
+template <typename client_type>
+[[nodiscard]] inline auto
+read_object_request(client_type &client, const s3_config &config,
+                    const std::string &object_name, std::size_t size,
+                    std::uint64_t offset, data_buffer &data,
+                    long &response_code, stop_type &stop_requested) -> bool {
+  return read_object_request_impl(client, config, object_name, size, offset,
+                                  data, response_code, stop_requested);
+}
+} // namespace repertory
+
+#endif
+#endif // INCLUDE_COMM_S3_S3_REQUESTS_HPP_

include/comm/s3/s3_requests_curl.hpp

@@ -0,0 +1,83 @@
+/*
+  Copyright <2018-2023> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all
+  copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  SOFTWARE.
+*/
+#ifndef INCLUDE_COMM_S3_S3_REQUESTS_CURL_HPP_
+#define INCLUDE_COMM_S3_S3_REQUESTS_CURL_HPP_
+#if defined(REPERTORY_ENABLE_S3)
+
+#include "comm/i_http_comm.hpp"
+#include "types/repertory.hpp"
+#include "types/s3.hpp"
+#include "utils/string_utils.hpp"
+#include "utils/utils.hpp"
+
+namespace repertory {
+[[nodiscard]] auto create_directory_object_request_impl(
+    i_http_comm &client, const s3_config &config,
+    const std::string &object_name, long &response_code) -> bool;
+
+[[nodiscard]] auto delete_object_request_impl(i_http_comm &client,
+                                              const s3_config &config,
+                                              const std::string &object_name,
+                                              long &response_code) -> bool;
+
+[[nodiscard]] auto head_object_request_impl(i_http_comm &client,
+                                            const s3_config &config,
+                                            const std::string &object_name,
+                                            head_object_result &result,
+                                            long &response_code) -> bool;
+
+[[nodiscard]] auto
+list_directories_request_impl(i_http_comm &client, const s3_config &config,
+                              list_directories_result &result,
+                              long &response_code) -> bool;
+
+[[nodiscard]] auto
+list_files_request_impl(i_http_comm &client, const s3_config &config,
+                        const get_api_file_token_callback &get_api_file_token,
+                        const get_name_callback &get_name,
+                        list_files_result &result, long &response_code) -> bool;
+
+[[nodiscard]] auto list_objects_in_directory_request_impl(
+    i_http_comm &client, const s3_config &config,
+    const std::string &object_name, meta_provider_callback meta_provider,
+    list_objects_result &result, long &response_code) -> bool;
+
+[[nodiscard]] auto list_objects_request_impl(i_http_comm &client,
+                                             const s3_config &config,
+                                             list_objects_result &result,
+                                             long &response_code) -> bool;
+
+[[nodiscard]] auto
+put_object_request_impl(i_http_comm &client, const s3_config &config,
+                        std::string object_name, const std::string &source_path,
+                        const std::string &encryption_token,
+                        get_key_callback get_key, set_key_callback set_key,
+                        long &response_code, stop_type &stop_requested) -> bool;
+
+[[nodiscard]] auto read_object_request_impl(
+    i_http_comm &client, const s3_config &config,
+    const std::string &object_name, std::size_t size, std::uint64_t offset,
+    data_buffer &data, long &response_code, stop_type &stop_requested) -> bool;
+} // namespace repertory
+
+#endif // REPERTORY_ENABLE_S3
+#endif // INCLUDE_COMM_S3_S3_REQUESTS_CURL_HPP_

include/common.hpp

@@ -22,32 +22,6 @@
 #ifndef INCLUDE_COMMON_HPP_
 #define INCLUDE_COMMON_HPP_
 
-#if defined(__GNUC__)
-// clang-format off
-#define REPERTORY_IGNORE_WARNINGS_ENABLE()                                            \
-  _Pragma("GCC diagnostic push")                                               \
-  _Pragma("GCC diagnostic ignored \"-Wconversion\"")                           \
-  _Pragma("GCC diagnostic ignored \"-Wdouble-promotion\"")                     \
-  _Pragma("GCC diagnostic ignored \"-Wduplicated-branches\"")                  \
-  _Pragma("GCC diagnostic ignored \"-Wfloat-conversion\"")                     \
-  _Pragma("GCC diagnostic ignored \"-Wimplicit-fallthrough\"")                 \
-  _Pragma("GCC diagnostic ignored \"-Wnull-dereference\"")                     \
-  _Pragma("GCC diagnostic ignored \"-Wsign-conversion\"")                      \
-  _Pragma("GCC diagnostic ignored \"-Wunused-but-set-variable\"")              \
-  _Pragma("GCC diagnostic ignored \"-Wunused-parameter\"")                     \
-  _Pragma("GCC diagnostic ignored \"-Wuseless-cast\"")
-
-#define REPERTORY_IGNORE_WARNINGS_DISABLE()                                    \
-  _Pragma("GCC diagnostic pop")
-#else
-#define REPERTORY_IGNORE_WARNINGS_ENABLE() 
-#define REPERTORY_IGNORE_WARNINGS_DISABLE()                                    
-#endif // defined(__GNUC__)
-// clang-format on
-
-#ifdef __cplusplus
-REPERTORY_IGNORE_WARNINGS_ENABLE()
-
 #ifdef _WIN32
 #include <winsock2.h>
 #include <ws2tcpip.h>
@@ -129,7 +103,7 @@ template <typename data_type>
 #include <curl/curl.h>
 #include <curl/multi.h>
 #include <json.hpp>
-#include <sqlite3.h>
+#include <rocksdb/db.h>
 #include <stduuid.h>
 
 #ifdef _WIN32
@@ -150,28 +124,6 @@ template <typename data_type>
 #define CPPHTTPLIB_OPENSSL_SUPPORT
 #include <httplib.h>
 
-REPERTORY_IGNORE_WARNINGS_DISABLE()
-
-struct sqlite3_deleter {
-  void operator()(sqlite3 *db3) {
-    if (db3 != nullptr) {
-      sqlite3_close_v2(db3);
-    }
-  }
-};
-
-using db3_t = std::unique_ptr<sqlite3, sqlite3_deleter>;
-
-struct sqlite3_statement_deleter {
-  void operator()(sqlite3_stmt *stmt) {
-    if (stmt != nullptr) {
-      sqlite3_finalize(stmt);
-    }
-  }
-};
-
-using db3_stmt_t = std::unique_ptr<sqlite3_stmt, sqlite3_statement_deleter>;
-
 using namespace std::chrono_literals;
 using json = nlohmann::json;
 
@@ -404,17 +356,17 @@ enum {
 };
 
 struct FSP_FSCTL_FILE_INFO {
-  UINT32 FileAttributes{};
-  UINT32 ReparseTag{};
-  UINT64 AllocationSize{};
-  UINT64 FileSize{};
-  UINT64 CreationTime{};
-  UINT64 LastAccessTime{};
-  UINT64 LastWriteTime{};
-  UINT64 ChangeTime{};
-  UINT64 IndexNumber{};
-  UINT32 HardLinks{};
-  UINT32 EaSize{};
+  UINT32 FileAttributes;
+  UINT32 ReparseTag;
+  UINT64 AllocationSize;
+  UINT64 FileSize;
+  UINT64 CreationTime;
+  UINT64 LastAccessTime;
+  UINT64 LastWriteTime;
+  UINT64 ChangeTime;
+  UINT64 IndexNumber;
+  UINT32 HardLinks;
+  UINT32 EaSize;
 };
 
 using FileInfo = FSP_FSCTL_FILE_INFO;
@@ -430,13 +382,6 @@ void repertory_init();
 void repertory_shutdown();
 } // namespace repertory
 
-namespace {
-template <class... Ts> struct overloaded : Ts... {
-  using Ts::operator()...;
-};
-template <class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
-} // namespace
-
 #define INTERFACE_SETUP(name)                                                  \
 public:                                                                        \
   name(const name &) noexcept = delete;                                        \
@@ -450,5 +395,4 @@ protected:                                                                     \
 public:                                                                        \
   virtual ~name() = default
 
-#endif // __cplusplus
 #endif // INCLUDE_COMMON_HPP_

include/database/db_common.hpp

@@ -1,217 +0,0 @@
-/*
-  Copyright <2018-2023> <scott.e.graves@protonmail.com>
-
-  Permission is hereby granted, free of charge, to any person obtaining a copy
-  of this software and associated documentation files (the "Software"), to deal
-  in the Software without restriction, including without limitation the rights
-  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-  copies of the Software, and to permit persons to whom the Software is
-  furnished to do so, subject to the following conditions:
-
-  The above copyright notice and this permission notice shall be included in all
-  copies or substantial portions of the Software.
-
-  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-  SOFTWARE.
-*/
-#ifndef INCLUDE_DATABASE_DB_COMMON_HPP_
-#define INCLUDE_DATABASE_DB_COMMON_HPP_
-
-#include "utils/error_utils.hpp"
-
-namespace repertory::db {
-using db_types_t = std::variant<std::int64_t, std::string>;
-
-struct comp_data_t final {
-  std::string column_name;
-  std::string op_type;
-  db_types_t value;
-};
-
-class db_column final {
-public:
-  db_column() noexcept = default;
-  db_column(const db_column &) = default;
-  db_column(db_column &&column) noexcept = default;
-  ~db_column() = default;
-
-  auto operator=(const db_column &) -> db_column & = default;
-  auto operator=(db_column &&) -> db_column & = default;
-
-  db_column(std::int32_t index, std::string name, db_types_t value) noexcept
-      : index_(index), name_(std::move(name)), value_(std::move(value)) {}
-
-private:
-  std::int32_t index_{};
-  std::string name_;
-  db_types_t value_;
-
-public:
-  [[nodiscard]] auto get_index() const -> std::int32_t { return index_; }
-
-  [[nodiscard]] auto get_name() const -> std::string { return name_; }
-
-  template <typename data_type>
-  [[nodiscard]] auto get_value() const -> data_type {
-    return std::visit(
-        overloaded{
-            [](const data_type &value) -> data_type { return value; },
-            [](auto &&) -> data_type {
-              throw std::runtime_error("data type not supported");
-            },
-        },
-        value_);
-  }
-
-  [[nodiscard]] auto get_value_as_json() const -> nlohmann::json {
-    return std::visit(
-        overloaded{
-            [this](std::int64_t value) -> auto {
-              return nlohmann::json({{name_, value}});
-            },
-            [](auto &&value) -> auto { return nlohmann::json::parse(value); },
-        },
-        value_);
-  }
-};
-
-template <typename context_t> class db_row final {
-public:
-  db_row(std::shared_ptr<context_t> context) {
-    auto column_count = sqlite3_column_count(context->stmt.get());
-    for (std::int32_t col = 0; col < column_count; col++) {
-      std::string name{sqlite3_column_name(context->stmt.get(), col)};
-      auto column_type = sqlite3_column_type(context->stmt.get(), col);
-
-      db_types_t value;
-      switch (column_type) {
-      case SQLITE_INTEGER: {
-        value = sqlite3_column_int64(context->stmt.get(), col);
-      } break;
-
-      case SQLITE_TEXT: {
-        const auto *text = reinterpret_cast<const char *>(
-            sqlite3_column_text(context->stmt.get(), col));
-        value = std::string(text == nullptr ? "" : text);
-      } break;
-
-      default:
-        throw std::runtime_error("column type not implemented|" + name + '|' +
-                                 std::to_string(column_type));
-      }
-
-      columns_[name] = db_column{col, name, value};
-    }
-  }
-
-private:
-  std::map<std::string, db_column> columns_;
-
-public:
-  [[nodiscard]] auto get_columns() const -> std::vector<db_column> {
-    std::vector<db_column> ret;
-    for (const auto &item : columns_) {
-      ret.push_back(item.second);
-    }
-    return ret;
-  }
-
-  [[nodiscard]] auto get_column(std::int32_t index) const -> db_column {
-    auto iter = std::find_if(columns_.begin(), columns_.end(),
-                             [&index](auto &&col) -> bool {
-                               return col.second.get_index() == index;
-                             });
-    if (iter == columns_.end()) {
-      throw std::out_of_range("");
-    }
-
-    return iter->second;
-  }
-
-  [[nodiscard]] auto get_column(std::string name) const -> db_column {
-    return columns_.at(name);
-  }
-};
-
-template <typename context_t> struct db_result final {
-  db_result(std::shared_ptr<context_t> context, std::int32_t res)
-      : context_(std::move(context)), res_(res) {
-    if (res == SQLITE_OK) {
-      set_res(sqlite3_step(context_->stmt.get()), __FUNCTION__);
-    }
-  }
-
-private:
-  std::shared_ptr<context_t> context_;
-  mutable std::int32_t res_;
-
-private:
-  void set_res(std::int32_t res, std::string function) const {
-    if (res != SQLITE_OK && res != SQLITE_DONE && res != SQLITE_ROW) {
-      utils::error::raise_error(function, "failed to step|" +
-                                              std::to_string(res) + '|' +
-                                              sqlite3_errstr(res));
-    }
-    res_ = res;
-  }
-
-public:
-  [[nodiscard]] auto ok() const -> bool {
-    return res_ == SQLITE_DONE || res_ == SQLITE_ROW;
-  }
-
-  [[nodiscard]] auto get_error() const -> std::int32_t { return res_; }
-
-  [[nodiscard]] auto get_error_str() const -> std::string {
-    return sqlite3_errstr(res_);
-  }
-
-  [[nodiscard]] auto get_row(std::optional<db_row<context_t>> &row) const
-      -> bool {
-    row.reset();
-    if (has_row()) {
-      row = db_row{context_};
-      set_res(sqlite3_step(context_->stmt.get()), __FUNCTION__);
-      return true;
-    }
-    return false;
-  }
-
-  [[nodiscard]] auto has_row() const -> bool { return res_ == SQLITE_ROW; }
-
-  void next_row() const {
-    if (has_row()) {
-      set_res(sqlite3_step(context_->stmt.get()), __FUNCTION__);
-    }
-  }
-};
-
-inline void set_journal_mode(sqlite3 &db3) {
-  sqlite3_exec(&db3, "PRAGMA journal_mode = WAL;", nullptr, nullptr, nullptr);
-}
-
-[[nodiscard]] inline auto execute_sql(sqlite3 &db3, const std::string &sql,
-                                      std::string &err) -> bool {
-  char *err_msg{nullptr};
-  auto res = sqlite3_exec(&db3, sql.c_str(), nullptr, nullptr, &err_msg);
-  if (err_msg != nullptr) {
-    err = err_msg;
-    sqlite3_free(err_msg);
-    err_msg = nullptr;
-  }
-  if (res != SQLITE_OK) {
-    err = "failed to execute sql|" + sql + "|" + std::to_string(res) + '|' +
-          (err.empty() ? sqlite3_errstr(res) : err);
-    return false;
-  }
-
-  return true;
-}
-} // namespace repertory::db
-
-#endif // INCLUDE_DATABASE_DB_COMMON_HPP_

include/database/db_select.hpp

@@ -1,125 +0,0 @@
-/*
-  Copyright <2018-2023> <scott.e.graves@protonmail.com>
-
-  Permission is hereby granted, free of charge, to any person obtaining a copy
-  of this software and associated documentation files (the "Software"), to deal
-  in the Software without restriction, including without limitation the rights
-  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-  copies of the Software, and to permit persons to whom the Software is
-  furnished to do so, subject to the following conditions:
-
-  The above copyright notice and this permission notice shall be included in all
-  copies or substantial portions of the Software.
-
-  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-  SOFTWARE.
-*/
-#ifndef INCLUDE_DATABASE_DB_SELECT_HPP_
-#define INCLUDE_DATABASE_DB_SELECT_HPP_
-
-#include "database/db_common.hpp"
-#include "utils/error_utils.hpp"
-
-namespace repertory::db {
-class db_select final {
-public:
-  struct context final {
-    context(sqlite3 &db3_, std::string table_name_)
-        : db3(db3_), table_name(std::move(table_name_)) {}
-
-    sqlite3 &db3;
-    std::string table_name;
-
-    std::vector<comp_data_t> ands{};
-    std::vector<std::string> columns{};
-    std::map<std::string, std::string> count_columns{};
-    bool delete_query{false};
-    std::optional<std::int32_t> limit;
-    std::optional<std::pair<std::string, bool>> order_by;
-    db3_stmt_t stmt{nullptr};
-  };
-
-  using row = db_row<context>;
-
-public:
-  db_select(sqlite3 &db3, std::string table_name)
-      : context_(std::make_shared<context>(db3, table_name)) {}
-
-  db_select(std::shared_ptr<context> ctx) : context_(std::move(ctx)) {}
-
-public:
-  struct db_where final {
-    db_where(std::shared_ptr<context> ctx, std::string column_name)
-        : context_(std::move(ctx)), column_name_(std::move(column_name)) {}
-
-  public:
-    struct db_where_next final {
-      db_where_next(std::shared_ptr<context> ctx) : context_(std::move(ctx)) {}
-
-    private:
-      std::shared_ptr<context> context_;
-
-    public:
-      [[nodiscard]] auto and_where(std::string column_name) const -> db_where {
-        return db_where{context_, column_name};
-      }
-
-      [[nodiscard]] auto dump() const -> std::string {
-        return db_select{context_}.dump();
-      }
-
-      [[nodiscard]] auto go() const -> db_result<context> {
-        return db_select{context_}.go();
-      }
-
-      [[nodiscard]] auto limit(std::int32_t value) const -> db_select {
-        return db_select{context_}.limit(value);
-      }
-
-      [[nodiscard]] auto order_by(std::string column_name, bool ascending) const
-          -> db_select {
-        return db_select{context_}.order_by(column_name, ascending);
-      }
-    };
-
-  private:
-    std::shared_ptr<context> context_;
-    std::string column_name_;
-
-  public:
-    [[nodiscard]] auto equals(db_types_t value) const -> db_where_next {
-      context_->ands.emplace_back(comp_data_t{column_name_, "=", value});
-      return db_where_next{context_};
-    }
-  };
-
-private:
-  std::shared_ptr<context> context_;
-
-public:
-  [[nodiscard]] auto column(std::string column_name) -> db_select &;
-
-  [[nodiscard]] auto count(std::string column_name, std::string as_column_name)
-      -> db_select &;
-
-  [[nodiscard]] auto delete_query() -> db_select &;
-
-  [[nodiscard]] auto dump() const -> std::string;
-
-  [[nodiscard]] auto go() const -> db_result<context>;
-
-  [[nodiscard]] auto limit(std::int32_t value) -> db_select &;
-
-  [[nodiscard]] auto order_by(std::string column_name, bool ascending)
-      -> db_select &;
-
-  [[nodiscard]] auto where(std::string column_name) const -> db_where;
-};
-} // namespace repertory::db
-
-#endif // INCLUDE_DATABASE_DB_SELECT_HPP_

include/db/directory_db.hpp

@@ -0,0 +1,117 @@
+/*
+  Copyright <2018-2023> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all
+  copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  SOFTWARE.
+*/
+#ifndef INCLUDE_DB_DIRECTORY_DB_HPP_
+#define INCLUDE_DB_DIRECTORY_DB_HPP_
+
+#include "app_config.hpp"
+#include "types/repertory.hpp"
+#include "utils/rocksdb_utils.hpp"
+
+namespace repertory {
+class directory_db final {
+private:
+  class directory_tree final {
+  private:
+    std::unordered_map<std::string, std::vector<std::string>>
+        sub_directory_lookup_;
+
+  public:
+    void add_path(const std::string &api_path,
+                  const std::vector<std::string> &files, rocksdb::DB &db);
+
+    [[nodiscard]] auto get_count(const std::string &api_path) const
+        -> std::size_t;
+
+    [[nodiscard]] auto get_directories() const -> std::vector<std::string>;
+
+    [[nodiscard]] auto get_sub_directories(const std::string &api_path) const
+        -> std::vector<std::string>;
+
+    [[nodiscard]] auto is_directory(const std::string &api_path) const -> bool;
+
+    void remove_directory(const std::string &api_path, rocksdb::DB &db,
+                          bool allow_remove_root = false);
+  };
+
+public:
+  explicit directory_db(const app_config &config);
+
+public:
+  ~directory_db();
+
+private:
+  mutable std::recursive_mutex directory_mutex_;
+  std::unique_ptr<rocksdb::DB> db_;
+  directory_tree tree_;
+  const std::string DIRDB_NAME = "directory_db";
+
+private:
+  [[nodiscard]] auto get_directory_data(const std::string &api_path) const
+      -> json;
+
+public:
+  [[nodiscard]] auto create_directory(const std::string &api_path,
+                                      bool create_always = false) -> api_error;
+
+  [[nodiscard]] auto create_file(const std::string &api_path) -> api_error;
+
+  [[nodiscard]] auto get_directory_item_count(const std::string &api_path) const
+      -> std::uint64_t;
+
+  [[nodiscard]] auto
+  get_file(const std::string &api_path, api_file &file,
+           api_file_provider_callback api_file_provider) const -> api_error;
+
+  [[nodiscard]] auto
+  get_file_list(api_file_list &list,
+                api_file_provider_callback api_file_provider) const
+      -> api_error;
+
+  [[nodiscard]] auto get_sub_directory_count(const std::string &api_path) const
+      -> std::size_t;
+
+  [[nodiscard]] auto get_total_item_count() const -> std::uint64_t;
+
+  [[nodiscard]] auto is_directory(const std::string &api_path) const -> bool;
+
+  [[nodiscard]] auto is_file(const std::string &api_path) const -> bool;
+
+  void populate_directory_files(const std::string &api_path,
+                                meta_provider_callback meta_provider,
+                                directory_item_list &list) const;
+
+  void populate_sub_directories(const std::string &api_path,
+                                meta_provider_callback meta_provider,
+                                directory_item_list &list) const;
+
+  [[nodiscard]] auto remove_directory(const std::string &api_path,
+                                      bool allow_remove_root = false)
+      -> api_error;
+
+  [[nodiscard]] auto remove_file(const std::string &api_path) -> bool;
+
+  [[nodiscard]] auto rename_file(const std::string &from_api_path,
+                                 const std::string &to_api_path) -> api_error;
+};
+} // namespace repertory
+
+#endif // INCLUDE_DB_DIRECTORY_DB_HPP_

include/db/meta_db.hpp

@@ -19,55 +19,77 @@
   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
   SOFTWARE.
 */
-#ifndef INCLUDE_PROVIDERS_META_DB_HPP_
-#define INCLUDE_PROVIDERS_META_DB_HPP_
+#ifndef INCLUDE_DB_META_DB_HPP_
+#define INCLUDE_DB_META_DB_HPP_
 
+#include "app_config.hpp"
 #include "types/repertory.hpp"
+#include "utils/rocksdb_utils.hpp"
 
 namespace repertory {
-class app_config;
-
 class meta_db final {
 public:
-  meta_db(const app_config &cfg);
-  ~meta_db();
-
-  meta_db(const meta_db &) = delete;
-  meta_db(meta_db &&) = delete;
-  auto operator=(const meta_db &) -> meta_db & = delete;
-  auto operator=(meta_db &&) -> meta_db & = delete;
-
-private:
-  db3_t db_;
-  constexpr static const auto table_name = "meta";
-
-private:
-  [[nodiscard]] auto update_item_meta(const std::string &api_path,
-                                      api_meta_map meta) -> api_error;
+  explicit meta_db(const app_config &config);
 
 public:
-  [[nodiscard]] auto get_api_path(const std::string &source_path,
-                                  std::string &api_path) -> api_error;
+  ~meta_db();
 
-  [[nodiscard]] auto get_api_path_list() -> std::vector<std::string>;
+private:
+  std::unique_ptr<rocksdb::DB> db_;
+  rocksdb::ColumnFamilyHandle *default_family_{};
+  rocksdb::ColumnFamilyHandle *source_family_{};
+  rocksdb::ColumnFamilyHandle *keys_family_{};
+  const std::string METADB_NAME = "meta_db";
+
+private:
+  [[nodiscard]] auto
+  perform_action(const std::string &function_name,
+                 const std::function<rocksdb::Status()> &action) const
+      -> api_error;
+
+  [[nodiscard]] auto get_item_meta_json(const std::string &api_path,
+                                        json &json_data) const -> api_error;
+
+  [[nodiscard]] auto store_item_meta(const std::string &api_path,
+                                     const std::string &key,
+                                     const std::string &value) -> api_error;
+
+public:
+  [[nodiscard]] auto create_iterator(bool source_family) const
+      -> std::shared_ptr<rocksdb::Iterator>;
+
+  [[nodiscard]] auto get_api_path_from_key(const std::string &key,
+                                           std::string &api_path) const
+      -> api_error;
+
+  [[nodiscard]] auto get_api_path_from_source(const std::string &source_path,
+                                              std::string &api_path) const
+      -> api_error;
 
   [[nodiscard]] auto get_item_meta(const std::string &api_path,
-                                   api_meta_map &meta) -> api_error;
+                                   api_meta_map &meta) const -> api_error;
 
   [[nodiscard]] auto get_item_meta(const std::string &api_path,
                                    const std::string &key,
                                    std::string &value) const -> api_error;
 
-  [[nodiscard]] auto get_pinned_files() const -> std::vector<std::string>;
+  [[nodiscard]] auto get_item_meta_exists(const std::string &api_path) const
+      -> bool;
 
   [[nodiscard]] auto get_total_item_count() const -> std::uint64_t;
 
-  void remove_api_path(const std::string &api_path);
+  [[nodiscard]] auto get_pinned_files() const -> std::vector<std::string>;
+
+  [[nodiscard]] auto
+  get_source_path_exists(const std::string &source_path) const -> bool;
+
+  [[nodiscard]] auto remove_item_meta(const std::string &api_path) -> api_error;
 
   [[nodiscard]] auto remove_item_meta(const std::string &api_path,
                                       const std::string &key) -> api_error;
 
-  [[nodiscard]] auto rename_item_meta(const std::string &from_api_path,
+  [[nodiscard]] auto rename_item_meta(const std::string &source_path,
+                                      const std::string &from_api_path,
                                       const std::string &to_api_path)
       -> api_error;
 
@@ -77,7 +99,11 @@ public:
 
   [[nodiscard]] auto set_item_meta(const std::string &api_path,
                                    const api_meta_map &meta) -> api_error;
+
+  [[nodiscard]] auto set_source_path(const std::string &api_path,
+                                     const std::string &source_path)
+      -> api_error;
 };
 } // namespace repertory
 
-#endif // INCLUDE_PROVIDERS_META_DB_HPP_
+#endif // INCLUDE_DB_META_DB_HPP_

include/drives/directory_cache.hpp

@@ -33,19 +33,15 @@ public:
 
 private:
   struct open_directory {
-    directory_iterator *iterator{nullptr};
-    std::chrono::system_clock::time_point last_update{
-        std::chrono::system_clock::now()};
+    directory_iterator *iterator;
+    std::chrono::system_clock::time_point last_update =
+        std::chrono::system_clock::now();
   };
 
 public:
   directory_cache() : single_thread_service_base("directory_cache") {}
-  ~directory_cache() override = default;
 
-  directory_cache(const directory_cache &) = delete;
-  directory_cache(directory_cache &&) = delete;
-  auto operator=(const directory_cache &) -> directory_cache & = delete;
-  auto operator=(directory_cache &&) -> directory_cache & = delete;
+  ~directory_cache() override = default;
 
 private:
   std::unordered_map<std::string, open_directory> directory_lookup_;

include/drives/fuse/fuse_base.hpp

@@ -38,12 +38,6 @@ public:
 
   virtual ~fuse_base();
 
-public:
-  fuse_base(const fuse_base &) = delete;
-  fuse_base(fuse_base &&) = delete;
-  auto operator=(const fuse_base &) -> fuse_base & = delete;
-  auto operator=(fuse_base &&) -> fuse_base & = delete;
-
 protected:
   app_config &config_;
 
@@ -315,7 +309,7 @@ protected:
     return api_error::not_implemented;
   }
 
-  virtual void destroy_impl(void * /*ptr*/);
+  virtual void destroy_impl(void * /*ptr*/) { return; }
 
   [[nodiscard]] virtual auto
   fallocate_impl(std::string /*api_path*/, int /*mode*/, off_t /*offset*/,
@@ -599,9 +593,9 @@ protected:
   virtual void shutdown();
 
 public:
-  static void display_options(std::vector<const char *> args);
+  static void display_options(int argc, char *argv[]);
 
-  static void display_version_information(std::vector<const char *> args);
+  static void display_version_information(int argc, char *argv[]);
 
   static auto unmount(const std::string &mount_location) -> int;
 

include/drives/fuse/fuse_drive.hpp

@@ -316,6 +316,9 @@ public:
   [[nodiscard]] auto is_processing(const std::string &api_path) const
       -> bool override;
 
+  void populate_stat(const directory_item &dir_item,
+                     struct stat &st) const override;
+
   [[nodiscard]] auto rename_directory(const std::string &from_api_path,
                                       const std::string &to_api_path)
       -> int override;

include/drives/fuse/fuse_drive_base.hpp

@@ -36,12 +36,6 @@ public:
 
   ~fuse_drive_base() override = default;
 
-public:
-  fuse_drive_base(const fuse_drive_base &) = delete;
-  fuse_drive_base(fuse_drive_base &&) = delete;
-  auto operator=(const fuse_drive_base &) -> fuse_drive_base & = delete;
-  auto operator=(fuse_drive_base &&) -> fuse_drive_base & = delete;
-
 protected:
   [[nodiscard]] auto access_impl(std::string api_path, int mask)
       -> api_error override;

include/drives/fuse/i_fuse_drive.hpp

@@ -69,6 +69,9 @@ public:
   [[nodiscard]] virtual auto is_processing(const std::string &api_path) const
       -> bool = 0;
 
+  virtual void populate_stat(const directory_item &dir_item,
+                             struct stat &st) const = 0;
+
   [[nodiscard]] virtual auto rename_directory(const std::string &from_api_path,
                                               const std::string &to_api_path)
       -> int = 0;

include/drives/fuse/remotefuse/i_remote_instance.hpp

@@ -112,14 +112,14 @@ public:
       -> packet::error_type = 0;
 
   [[nodiscard]] virtual auto fuse_read(const char *path, char *buffer,
-                                       const remote::file_size &read_size,
-                                       const remote::file_offset &read_offset,
+                                       const remote::file_size &readSize,
+                                       const remote::file_offset &readOffset,
                                        const remote::file_handle &handle)
       -> packet::error_type = 0;
 
   [[nodiscard]] virtual auto
   fuse_readdir(const char *path, const remote::file_offset &offset,
-               const remote::file_handle &handle, std::string &item_path)
+               const remote::file_handle &handle, std::string &itemPath)
       -> packet::error_type = 0;
 
   [[nodiscard]] virtual auto fuse_release(const char *path,

include/drives/fuse/remotefuse/remote_fuse_drive.hpp

@@ -53,8 +53,7 @@ private:
   bool was_mounted_ = false;
 
 private:
-  void populate_stat(const remote::stat &r_stat, bool directory,
-                     struct stat &unix_st);
+  void populate_stat(const remote::stat &r, bool directory, struct stat &st);
 
 protected:
   [[nodiscard]] auto access_impl(std::string api_path, int mask)
@@ -67,7 +66,7 @@ protected:
 
 #if FUSE_USE_VERSION >= 30
   [[nodiscard]] auto chmod_impl(std::string api_path, mode_t mode,
-                                struct fuse_file_info *f_info)
+                                struct fuse_file_info *fi)
       -> api_error override;
 #else
   [[nodiscard]] auto chmod_impl(std::string api_path, mode_t mode)
@@ -76,7 +75,7 @@ protected:
 
 #if FUSE_USE_VERSION >= 30
   [[nodiscard]] auto chown_impl(std::string api_path, uid_t uid, gid_t gid,
-                                struct fuse_file_info *f_info)
+                                struct fuse_file_info *fi)
       -> api_error override;
 #else
   [[nodiscard]] auto chown_impl(std::string api_path, uid_t uid, gid_t gid)
@@ -84,38 +83,37 @@ protected:
 #endif
 
   [[nodiscard]] auto create_impl(std::string api_path, mode_t mode,
-                                 struct fuse_file_info *f_info)
+                                 struct fuse_file_info *fi)
       -> api_error override;
-
   void destroy_impl(void * /*ptr*/) override;
 
-  [[nodiscard]] auto fgetattr_impl(std::string api_path, struct stat *unix_st,
-                                   struct fuse_file_info *f_info)
+  [[nodiscard]] auto fgetattr_impl(std::string api_path, struct stat *st,
+                                   struct fuse_file_info *fi)
       -> api_error override;
 
 #ifdef __APPLE__
   [[nodiscard]] auto fsetattr_x_impl(std::string api_path,
                                      struct setattr_x *attr,
-                                     struct fuse_file_info *f_info)
+                                     struct fuse_file_info *fi)
       -> api_error override;
 #endif // __APPLE__
 
   [[nodiscard]] auto fsync_impl(std::string api_path, int datasync,
-                                struct fuse_file_info *f_info)
+                                struct fuse_file_info *fi)
       -> api_error override;
 
 #if FUSE_USE_VERSION < 30
   [[nodiscard]] auto ftruncate_impl(std::string api_path, off_t size,
-                                    struct fuse_file_info *f_info)
+                                    struct fuse_file_info *fi)
       -> api_error override;
 #endif
 
 #if FUSE_USE_VERSION >= 30
-  [[nodiscard]] auto getattr_impl(std::string api_path, struct stat *unix_st,
-                                  struct fuse_file_info *f_info)
+  [[nodiscard]] auto getattr_impl(std::string api_path, struct stat *st,
+                                  struct fuse_file_info *fi)
       -> api_error override;
 #else
-  [[nodiscard]] auto getattr_impl(std::string api_path, struct stat *unix_st)
+  [[nodiscard]] auto getattr_impl(std::string api_path, struct stat *st)
       -> api_error override;
 #endif
 
@@ -138,38 +136,37 @@ protected:
 
   void notify_fuse_main_exit(int &ret) override;
 
-  [[nodiscard]] auto open_impl(std::string api_path,
-                               struct fuse_file_info *f_info)
+  [[nodiscard]] auto open_impl(std::string api_path, struct fuse_file_info *fi)
       -> api_error override;
 
   [[nodiscard]] auto opendir_impl(std::string api_path,
-                                  struct fuse_file_info *f_info)
+                                  struct fuse_file_info *fi)
       -> api_error override;
 
   [[nodiscard]] auto read_impl(std::string api_path, char *buffer,
                                size_t read_size, off_t read_offset,
-                               struct fuse_file_info *f_info,
+                               struct fuse_file_info *fi,
                                std::size_t &bytes_read) -> api_error override;
 
 #if FUSE_USE_VERSION >= 30
   [[nodiscard]] auto readdir_impl(std::string api_path, void *buf,
                                   fuse_fill_dir_t fuse_fill_dir, off_t offset,
-                                  struct fuse_file_info *f_info,
+                                  struct fuse_file_info *fi,
                                   fuse_readdir_flags flags)
       -> api_error override;
 #else
   [[nodiscard]] auto readdir_impl(std::string api_path, void *buf,
                                   fuse_fill_dir_t fuse_fill_dir, off_t offset,
-                                  struct fuse_file_info *f_info)
+                                  struct fuse_file_info *fi)
       -> api_error override;
 #endif
 
   [[nodiscard]] auto release_impl(std::string api_path,
-                                  struct fuse_file_info *f_info)
+                                  struct fuse_file_info *fi)
       -> api_error override;
 
   [[nodiscard]] auto releasedir_impl(std::string api_path,
-                                     struct fuse_file_info *f_info)
+                                     struct fuse_file_info *fi)
       -> api_error override;
 
 #if FUSE_USE_VERSION >= 30
@@ -213,7 +210,7 @@ protected:
 
 #if FUSE_USE_VERSION >= 30
   [[nodiscard]] auto truncate_impl(std::string api_path, off_t size,
-                                   struct fuse_file_info *f_info)
+                                   struct fuse_file_info *fi)
       -> api_error override;
 #else
   [[nodiscard]] auto truncate_impl(std::string api_path, off_t size)
@@ -225,7 +222,7 @@ protected:
 #if FUSE_USE_VERSION >= 30
   [[nodiscard]] auto utimens_impl(std::string api_path,
                                   const struct timespec tv[2],
-                                  struct fuse_file_info *f_info)
+                                  struct fuse_file_info *fi)
       -> api_error override;
 #else
   [[nodiscard]] auto utimens_impl(std::string api_path,
@@ -235,7 +232,7 @@ protected:
 
   [[nodiscard]] auto write_impl(std::string api_path, const char *buffer,
                                 size_t write_size, off_t write_offset,
-                                struct fuse_file_info *f_info,
+                                struct fuse_file_info *fi,
                                 std::size_t &bytes_written)
       -> api_error override;
 };

include/drives/remote/remote_server_base.hpp

@@ -341,7 +341,7 @@ public:
            ret = this->winfsp_read(file_desc, buffer.data(), offset, length,
                                    &bytes_transferred);
            if (ret == STATUS_SUCCESS) {
-             response.encode(bytes_transferred);
+             response.encode(static_cast<UINT32>(bytes_transferred));
              if (bytes_transferred != 0U) {
                response.encode(buffer.data(), bytes_transferred);
              }
@@ -1237,7 +1237,7 @@ public:
              return -ERANGE;
            }
 
-           data_buffer buffer(write_size);
+           data_buffer buffer(static_cast<std::size_t>(write_size));
            if ((ret = request->decode(buffer.data(), buffer.size())) == 0) {
              remote::file_offset write_offset{};
              DECODE_OR_RETURN(request, write_offset);
@@ -1267,7 +1267,7 @@ public:
              return -ERANGE;
            }
 
-           data_buffer buffer(write_size);
+           data_buffer buffer(static_cast<std::size_t>(write_size));
            if ((ret = request->decode(buffer.data(), buffer.size())) == 0) {
              buffer = macaron::Base64::Decode(
                  std::string(buffer.begin(), buffer.end()));

include/drives/winfsp/winfsp_drive.hpp

@@ -193,9 +193,9 @@ public:
 
   void shutdown();
 
-  static void display_options(std::vector<const char *> args) {}
+  static void display_options(int argc, char *argv[]) {}
 
-  static void display_version_information(std::vector<const char *> args) {}
+  static void display_version_information(int argc, char *argv[]) {}
 };
 } // namespace repertory
 

include/events/event_system.hpp

@@ -60,7 +60,6 @@ using event_consumer = event_system::event_consumer;
     return ss;                                                                 \
   })(d)
 #define E_STRING(t) t
-#define E_FROM_CURL_CODE(t) std::string(curl_easy_strerror(t))
 #define E_FROM_UINT8(t) std::to_string(t)
 #define E_FROM_UINT32(t) std::to_string(t)
 #define E_FROM_UINT64(t) std::to_string(t)
@@ -246,17 +245,16 @@ private:                                                                       \
 
 #define E_SUBSCRIBE(name, callback)                                            \
   event_consumers_.emplace_back(std::make_shared<repertory::event_consumer>(   \
-      #name, [this](const event &evt) { callback(evt); }))
+      #name, [this](const event &e) { callback(e); }))
 
 #define E_SUBSCRIBE_EXACT(name, callback)                                      \
   event_consumers_.emplace_back(std::make_shared<repertory::event_consumer>(   \
-      #name, [this](const event &evt) {                                        \
-        callback(dynamic_cast<const name &>(evt));                             \
-      }))
+      #name,                                                                   \
+      [this](const event &e) { callback(dynamic_cast<const name &>(e)); }))
 
 #define E_SUBSCRIBE_ALL(callback)                                              \
   event_consumers_.emplace_back(std::make_shared<repertory::event_consumer>(   \
-      [this](const event &evt) { callback(evt); }))
+      [this](const event &e) { callback(e); }))
 } // namespace repertory
 
 #endif // INCLUDE_EVENTS_EVENT_SYSTEM_HPP_

include/events/events.hpp

@@ -28,9 +28,145 @@
 
 namespace repertory {
 // clang-format off
-E_SIMPLE2(curl_error, normal, true,
+E_SIMPLE3(cache_file_processed, normal, true,
+  std::string, api_path, ap, E_STRING,
+  std::string, source, src, E_STRING,
+  api_error, result, res, E_FROM_API_FILE_ERROR
+);
+
+E_SIMPLE2(comm_auth_begin, normal, true,
   std::string, url, url, E_STRING,
-  CURLcode, res, res, E_FROM_CURL_CODE 
+  std::string, user, user, E_STRING
+);
+
+E_SIMPLE4(comm_auth_end, normal, true,
+  std::string, url, url, E_STRING,
+  std::string, user, user, E_STRING,
+  CURLcode, curl, curl, E_FROM_INT32,
+  int, http, http, E_FROM_INT32
+);
+
+E_SIMPLE2(comm_auth_logout_begin, normal, true,
+  std::string, url, url, E_STRING,
+  std::string, user, user, E_STRING
+);
+
+E_SIMPLE4(comm_auth_logout_end, normal, true,
+  std::string, url, url, E_STRING,
+  std::string, user, user, E_STRING,
+  CURLcode, curl, curl, E_FROM_INT32,
+  int, http, http, E_FROM_INT32
+);
+
+E_SIMPLE1(comm_del_begin, verbose, true,
+  std::string, url, url, E_STRING
+);
+
+E_SIMPLE4(comm_del_end, verbose, true,
+  std::string, url, url, E_STRING,
+  CURLcode, curl, curl, E_FROM_INT32,
+  int, http, http, E_FROM_INT32,
+  std::string, result, res, E_STRING
+);
+
+E_SIMPLE1(comm_get_begin, verbose, true,
+  std::string, url, url, E_STRING
+);
+
+E_SIMPLE4(comm_get_end, verbose, true,
+  std::string, url, url, E_STRING,
+  CURLcode, curl, curl, E_FROM_INT32,
+  int, http, http, E_FROM_INT32,
+  std::string, result, res, E_STRING
+);
+
+E_SIMPLE1(comm_get_range_begin, verbose, true,
+  std::string, url, url, E_STRING
+);
+
+E_SIMPLE4(comm_get_range_end, verbose, true,
+  std::string, url, url, E_STRING,
+  CURLcode, curl, curl, E_FROM_INT32,
+  int, http, http, E_FROM_INT32,
+  std::string, error, err, E_STRING
+);
+
+E_SIMPLE1(comm_get_raw_begin, verbose, true,
+  std::string, url, url, E_STRING
+);
+
+E_SIMPLE4(comm_get_raw_end, verbose, true,
+  std::string, url, url, E_STRING,
+  CURLcode, curl, curl, E_FROM_INT32,
+  int, http, http, E_FROM_INT32,
+  std::string, error, err, E_STRING
+);
+
+E_SIMPLE2(comm_post_begin, verbose, true,
+  std::string, url, url, E_STRING,
+  std::string, fields, fields, E_STRING
+);
+
+E_SIMPLE4(comm_post_end, verbose, true,
+  std::string, url, url, E_STRING,
+  CURLcode, curl, curl, E_FROM_INT32,
+  int, http, http, E_FROM_INT32,
+  std::string, result, res, E_STRING
+);
+
+E_SIMPLE2(comm_duration, normal, true,
+  std::string, url, url, E_STRING,
+  std::string, duration, duration, E_STRING
+);
+
+E_SIMPLE1(comm_post_file_begin, debug, true,
+  std::string, url, url, E_STRING
+);
+
+E_SIMPLE4(comm_post_file_end, debug, true,
+  std::string, url, url, E_STRING,
+  CURLcode, curl, curl, E_FROM_INT32,
+  int, http, http, E_FROM_INT32,
+  std::string, error, err, E_STRING
+);
+
+E_SIMPLE1(comm_post_multi_part_file_begin, debug, true,
+  std::string, url, url, E_STRING
+);
+
+E_SIMPLE4(comm_post_multi_part_file_end, debug, true,
+  std::string, url, url, E_STRING,
+  CURLcode, curl, curl, E_FROM_INT32,
+  int, http, http, E_FROM_INT32,
+  std::string, error, err, E_STRING
+);
+
+E_SIMPLE4(comm_tus_upload_begin, debug, true,
+  std::string, file_name, fn, E_STRING,
+  std::string, url, url, E_STRING,
+  std::uint64_t, remain, rem, E_FROM_UINT64,
+  std::uint64_t, offset, off, E_FROM_UINT64
+);
+
+E_SIMPLE6(comm_tus_upload_end, debug, true,
+  std::string, file_name, fn, E_STRING,
+  std::string, url, url, E_STRING,
+  std::uint64_t, remain, rem, E_FROM_UINT64,
+  std::uint64_t, offset, OFF, E_FROM_UINT64,
+  CURLcode, curl, curl, E_FROM_INT32,
+  int, http, http, E_FROM_INT32
+);
+
+E_SIMPLE2(comm_tus_upload_create_begin, debug, true,
+  std::string, file_name, fn, E_STRING,
+  std::string, url, url, E_STRING
+);
+
+E_SIMPLE4(comm_tus_upload_create_end, debug, true,
+  std::string, file_name, fn, E_STRING,
+  std::string, url, url, E_STRING,
+  CURLcode, curl, curl, E_FROM_INT32,
+  int, http, http, E_FROM_INT32
 );
 
 E_SIMPLE3(debug_log, debug, true,
@@ -205,8 +341,9 @@ E_SIMPLE2(file_upload_queued, normal, true,
   std::string, source, src, E_STRING
 );
 
-E_SIMPLE1(file_upload_removed, debug, true,
-  std::string, api_path, ap, E_STRING
+E_SIMPLE2(file_upload_removed, debug, true,
+  std::string, api_path, ap, E_STRING,
+  std::string, source, src, E_STRING
 );
 
 E_SIMPLE3(file_upload_retry, normal, true,

include/file_manager/events.hpp

@@ -93,7 +93,7 @@ E_SIMPLE3(download_stored_failed, error, true,
   std::string, error, err, E_STRING
 );
 
-E_SIMPLE1(item_timeout, normal, true,
+E_SIMPLE1(download_timeout, warn, true,
   std::string, api_path, ap, E_STRING
 );
 // clang-format on

include/file_manager/file_manager.hpp

@@ -117,23 +117,22 @@ public:
     i_provider &provider_;
 
   private:
-    api_error error_{api_error::success};
+    api_error error_ = api_error::success;
     mutable std::mutex error_mtx_;
-    stop_type io_stop_requested_{false};
+    stop_type io_stop_requested_ = false;
     std::unique_ptr<std::thread> io_thread_;
 
   protected:
     std::unordered_map<std::size_t, std::shared_ptr<download>>
         active_downloads_;
     mutable std::recursive_mutex file_mtx_;
-    std::atomic<std::chrono::system_clock::time_point> last_access_{
-        std::chrono::system_clock::now()};
-    bool modified_{false};
+    std::atomic<std::chrono::system_clock::time_point> last_access_ =
+        std::chrono::system_clock::now();
+    bool modified_ = false;
     native_file_ptr nf_;
     mutable std::mutex io_thread_mtx_;
     std::condition_variable io_thread_notify_;
     std::deque<std::shared_ptr<io_item>> io_thread_queue_;
-    bool removed_{false};
 
   private:
     void file_io_thread();
@@ -181,10 +180,6 @@ public:
       return fsi_.source_path;
     }
 
-    [[nodiscard]] auto has_handle(std::uint64_t handle) const -> bool override {
-      return open_data_.find(handle) != open_data_.end();
-    }
-
     [[nodiscard]] auto is_directory() const -> bool override {
       return fsi_.directory;
     }
@@ -247,8 +242,7 @@ public:
                         std::size_t end_chunk_index_inclusive,
                         bool should_reset);
 
-    void set_modified();
-
+  private:
     void update_background_reader(std::size_t read_chunk);
 
   protected:
@@ -451,14 +445,19 @@ public:
 private:
   app_config &config_;
   i_provider &provider_;
+  std::unique_ptr<rocksdb::DB> db_;
 
 private:
-  db3_t db_{nullptr};
-  std::uint64_t next_handle_{0U};
+  rocksdb::ColumnFamilyHandle *default_family_{};
+  std::uint64_t next_handle_ = 0u;
   mutable std::recursive_mutex open_file_mtx_;
   std::unordered_map<std::string, std::shared_ptr<i_closeable_open_file>>
       open_file_lookup_;
-  stop_type stop_requested_{false};
+  std::unordered_map<std::uint64_t, i_closeable_open_file *>
+      open_handle_lookup_;
+  stop_type stop_requested_ = false;
+  rocksdb::ColumnFamilyHandle *upload_family_{};
+  rocksdb::ColumnFamilyHandle *upload_active_family_{};
   std::unordered_map<std::string, std::unique_ptr<upload>> upload_lookup_;
   mutable std::mutex upload_mtx_;
   std::condition_variable upload_notify_;
@@ -467,15 +466,12 @@ private:
 private:
   void close_timed_out_files();
 
-  auto get_open_file_by_handle(std::uint64_t handle) const
-      -> std::shared_ptr<i_closeable_open_file>;
-
   auto get_open_file_count(const std::string &api_path) const -> std::size_t;
 
   auto open(const std::string &api_path, bool directory,
             const open_file_data &ofd, std::uint64_t &handle,
-            std::shared_ptr<i_open_file> &file,
-            std::shared_ptr<i_closeable_open_file> closeable_file) -> api_error;
+            std::shared_ptr<i_open_file> &f,
+            std::shared_ptr<i_closeable_open_file> of) -> api_error;
 
   void queue_upload(const std::string &api_path, const std::string &source_path,
                     bool no_lock);
@@ -484,7 +480,7 @@ private:
 
   void swap_renamed_items(std::string from_api_path, std::string to_api_path);
 
-  void upload_completed(const file_upload_completed &evt);
+  void upload_completed(const file_upload_completed &e);
 
   void upload_handler();
 
@@ -494,14 +490,14 @@ public:
   auto handle_file_rename(const std::string &from_api_path,
                           const std::string &to_api_path) -> api_error;
 
-  void queue_upload(const i_open_file &file) override;
+  void queue_upload(const i_open_file &o) override;
 
   void remove_resume(const std::string &api_path,
                      const std::string &source_path) override;
 
   void remove_upload(const std::string &api_path) override;
 
-  void store_resume(const i_open_file &file) override;
+  void store_resume(const i_open_file &o) override;
 
 public:
   void close(std::uint64_t handle);
@@ -510,7 +506,7 @@ public:
 
   [[nodiscard]] auto create(const std::string &api_path, api_meta_map &meta,
                             open_file_data ofd, std::uint64_t &handle,
-                            std::shared_ptr<i_open_file> &file) -> api_error;
+                            std::shared_ptr<i_open_file> &f) -> api_error;
 
   [[nodiscard]] auto evict_file(const std::string &api_path) -> bool override;
 
@@ -518,7 +514,7 @@ public:
       -> directory_item_list override;
 
   [[nodiscard]] auto get_open_file(std::uint64_t handle, bool write_supported,
-                                   std::shared_ptr<i_open_file> &file) -> bool;
+                                   std::shared_ptr<i_open_file> &f) -> bool;
 
   [[nodiscard]] auto get_open_file_count() const -> std::size_t;
 
@@ -537,11 +533,14 @@ public:
 #ifdef REPERTORY_TESTING
   [[nodiscard]] auto open(std::shared_ptr<i_closeable_open_file> of,
                           const open_file_data &ofd, std::uint64_t &handle,
-                          std::shared_ptr<i_open_file> &file) -> api_error;
+                          std::shared_ptr<i_open_file> &f) -> api_error;
 #endif
   [[nodiscard]] auto open(const std::string &api_path, bool directory,
                           const open_file_data &ofd, std::uint64_t &handle,
-                          std::shared_ptr<i_open_file> &file) -> api_error;
+                          std::shared_ptr<i_open_file> &f) -> api_error;
+
+  auto perform_locked_operation(locked_operation_callback locked_operation)
+      -> bool override;
 
   [[nodiscard]] auto remove_file(const std::string &api_path) -> api_error;
 

include/file_manager/i_file_manager.hpp

@@ -30,6 +30,9 @@ class i_provider;
 class i_file_manager {
   INTERFACE_SETUP(i_file_manager);
 
+public:
+  using locked_operation_callback = std::function<bool(i_provider &)>;
+
 public:
   [[nodiscard]] virtual auto evict_file(const std::string &api_path)
       -> bool = 0;
@@ -46,6 +49,10 @@ public:
   [[nodiscard]] virtual auto is_processing(const std::string &api_path) const
       -> bool = 0;
 
+  virtual auto
+  perform_locked_operation(locked_operation_callback locked_operation)
+      -> bool = 0;
+
   virtual void update_used_space(std::uint64_t &used_space) const = 0;
 };
 } // namespace repertory

include/file_manager/i_open_file.hpp

@@ -40,9 +40,6 @@ public:
 
   [[nodiscard]] virtual auto get_filesystem_item() const -> filesystem_item = 0;
 
-  [[nodiscard]] virtual auto get_open_data() const
-      -> std::map<std::uint64_t, open_file_data> = 0;
-
   [[nodiscard]] virtual auto get_open_data(std::uint64_t handle) const
       -> open_file_data = 0;
 
@@ -58,8 +55,6 @@ public:
 
   [[nodiscard]] virtual auto is_directory() const -> bool = 0;
 
-  [[nodiscard]] virtual auto has_handle(std::uint64_t handle) const -> bool = 0;
-
   [[nodiscard]] virtual auto
   native_operation(const native_operation_callback &callback) -> api_error = 0;
 
@@ -94,6 +89,9 @@ public:
   [[nodiscard]] virtual auto get_handles() const
       -> std::vector<std::uint64_t> = 0;
 
+  [[nodiscard]] virtual auto get_open_data() const
+      -> std::map<std::uint64_t, open_file_data> = 0;
+
   [[nodiscard]] virtual auto is_complete() const -> bool = 0;
 
   [[nodiscard]] virtual auto is_modified() const -> bool = 0;

include/platform/unix_platform.hpp

@@ -68,10 +68,11 @@ public:
 [[nodiscard]] auto create_meta_attributes(
     std::uint64_t accessed_date, std::uint32_t attributes,
     std::uint64_t changed_date, std::uint64_t creation_date, bool directory,
-    std::uint32_t gid, const std::string &key, std::uint32_t mode,
-    std::uint64_t modified_date, std::uint32_t osx_backup,
-    std::uint32_t osx_flags, std::uint64_t size, const std::string &source_path,
-    std::uint32_t uid, std::uint64_t written_date) -> api_meta_map;
+    const std::string &encryption_token, std::uint32_t gid,
+    const std::string &key, std::uint32_t mode, std::uint64_t modified_date,
+    std::uint32_t osx_backup, std::uint32_t osx_flags, std::uint64_t size,
+    const std::string &source_path, std::uint32_t uid,
+    std::uint64_t written_date) -> api_meta_map;
 
 [[nodiscard]] auto provider_meta_handler(i_provider &provider, bool directory,
                                          const api_file &file) -> api_error;

include/platform/win32_platform.hpp

@@ -73,10 +73,11 @@ public:
 [[nodiscard]] auto create_meta_attributes(
     std::uint64_t accessed_date, std::uint32_t attributes,
     std::uint64_t changed_date, std::uint64_t creation_date, bool directory,
-    std::uint32_t gid, const std::string &key, std::uint32_t mode,
-    std::uint64_t modified_date, std::uint32_t osx_backup,
-    std::uint32_t osx_flags, std::uint64_t size, const std::string &source_path,
-    std::uint32_t uid, std::uint64_t written_date) -> api_meta_map;
+    const std::string &encryption_token, std::uint32_t gid,
+    const std::string &key, std::uint32_t mode, std::uint64_t modified_date,
+    std::uint32_t osx_backup, std::uint32_t osx_flags, std::uint64_t size,
+    const std::string &source_path, std::uint32_t uid,
+    std::uint64_t written_date) -> api_meta_map;
 
 [[nodiscard]] auto provider_meta_handler(i_provider &provider, bool directory,
                                          const api_file &file) -> api_error;

include/providers/base_provider.hpp

@@ -22,92 +22,95 @@
 #ifndef INCLUDE_PROVIDERS_BASE_PROVIDER_HPP_
 #define INCLUDE_PROVIDERS_BASE_PROVIDER_HPP_
 
+#include "db/meta_db.hpp"
 #include "providers/i_provider.hpp"
-#include "providers/meta_db.hpp"
-#include "types/repertory.hpp"
 
 namespace repertory {
 class app_config;
 class i_file_manager;
-class i_http_comm;
 
 class base_provider : public i_provider {
 public:
-  base_provider(app_config &config, i_http_comm &comm)
-      : config_(config), comm_(comm) {}
+  explicit base_provider(app_config &config);
+
+  ~base_provider() override = default;
 
 private:
   app_config &config_;
-  i_http_comm &comm_;
-
-private:
-  api_item_added_callback api_item_added_;
-  std::unique_ptr<meta_db> db3_;
-  i_file_manager *fm_{};
-
-private:
-  void remove_deleted_files();
+  std::atomic<std::uint64_t> used_space_{0U};
 
 protected:
-  [[nodiscard]] static auto create_api_file(std::string path, std::string key,
-                                            std::uint64_t size) -> api_file;
+  api_item_added_callback api_item_added_;
+  std::unique_ptr<meta_db> meta_db_;
+  mutable std::recursive_mutex notify_added_mutex_;
+  i_file_manager *fm_ = nullptr;
+  stop_type stop_requested_ = false;
 
-  [[nodiscard]] static auto create_api_file(std::string path,
-                                            std::uint64_t size,
-                                            api_meta_map &meta) -> api_file;
-
-  [[nodiscard]] virtual auto create_directory_impl(const std::string &api_path,
-                                                   api_meta_map &meta)
-      -> api_error = 0;
+protected:
+  void calculate_used_drive_space(bool add_missing);
 
   [[nodiscard]] virtual auto
-  create_file_extra(const std::string & /* api_path */,
-                    api_meta_map & /* meta */) -> api_error {
-    return api_error::success;
-  }
+  check_file_exists(const std::string &api_path) const -> api_error = 0;
 
-  [[nodiscard]] auto get_api_item_added() -> api_item_added_callback & {
-    return api_item_added_;
-  }
-
-  [[nodiscard]] auto get_api_item_added() const
-      -> const api_item_added_callback & {
-    return api_item_added_;
-  }
-
-  [[nodiscard]] auto get_comm() const -> i_http_comm & { return comm_; }
+  void cleanup();
 
   [[nodiscard]] auto get_config() -> app_config & { return config_; }
 
-  [[nodiscard]] auto get_config() const -> const app_config & {
-    return config_;
-  }
-
-  [[nodiscard]] auto get_db() -> meta_db & { return *db3_; }
+  [[nodiscard]] auto get_config() const -> app_config & { return config_; }
 
   [[nodiscard]] virtual auto
-  get_directory_items_impl(const std::string &api_path,
-                           directory_item_list &list) const -> api_error = 0;
+  handle_rename_file(const std::string &from_api_path,
+                     const std::string &to_api_path,
+                     const std::string &source_path) -> api_error = 0;
 
-  [[nodiscard]] auto get_file_mgr() -> i_file_manager * { return fm_; }
-
-  [[nodiscard]] auto get_file_mgr() const -> const i_file_manager * {
-    return fm_;
+  [[nodiscard]] auto notify_directory_added(const std::string &api_path,
+                                            const std::string &api_parent) const
+      -> api_error {
+    return const_cast<base_provider *>(this)->notify_directory_added(
+        api_path, api_parent);
   }
 
-  [[nodiscard]] virtual auto get_used_drive_space_impl() const
-      -> std::uint64_t = 0;
+  [[nodiscard]] virtual auto
+  notify_directory_added(const std::string &api_path,
+                         const std::string &api_parent) -> api_error;
 
-  [[nodiscard]] virtual auto remove_directory_impl(const std::string &api_path)
+  [[nodiscard]] auto notify_file_added(const std::string &api_path,
+                                       const std::string &api_parent,
+                                       std::uint64_t size) const -> api_error {
+    return const_cast<base_provider *>(this)->notify_file_added(
+        api_path, api_parent, size);
+  }
+
+  [[nodiscard]] virtual auto notify_file_added(const std::string &api_path,
+                                               const std::string &api_parent,
+                                               std::uint64_t size)
       -> api_error = 0;
 
-  [[nodiscard]] virtual auto remove_file_impl(const std::string &api_path)
+  [[nodiscard]] virtual auto
+  populate_directory_items(const std::string &api_path,
+                           directory_item_list &list) const -> api_error = 0;
+
+  [[nodiscard]] virtual auto populate_file(const std::string &api_path,
+                                           api_file &file) const
       -> api_error = 0;
 
-  [[nodiscard]] virtual auto upload_file_impl(const std::string &api_path,
-                                              const std::string &source_path,
-                                              stop_type &stop_requested)
-      -> api_error = 0;
+  auto processed_orphaned_file(const std::string &source_path,
+                               const std::string &api_path = "") const -> bool;
+
+  void remove_deleted_files();
+
+  void remove_expired_orphaned_files();
+
+  void remove_unknown_source_files();
+
+  [[nodiscard]] auto remove_item_meta(const std::string &api_path)
+      -> api_error {
+    return meta_db_->remove_item_meta(api_path);
+  }
+
+  void update_filesystem_item(bool directory, const api_error &error,
+                              const std::string &api_path,
+                              filesystem_item &fsi) const;
 
 public:
   [[nodiscard]] auto
@@ -115,9 +118,6 @@ public:
                                      const std::string &api_path)
       -> api_error override;
 
-  [[nodiscard]] auto create_directory(const std::string &api_path,
-                                      api_meta_map &meta) -> api_error override;
-
   [[nodiscard]] auto create_file(const std::string &api_path,
                                  api_meta_map &meta) -> api_error override;
 
@@ -129,6 +129,9 @@ public:
                                          directory_item_list &list) const
       -> api_error override;
 
+  [[nodiscard]] auto get_file(const std::string &api_path, api_file &file) const
+      -> api_error override;
+
   [[nodiscard]] auto get_file_size(const std::string &api_path,
                                    std::uint64_t &file_size) const
       -> api_error override;
@@ -139,7 +142,7 @@ public:
       -> api_error override;
 
   [[nodiscard]] auto get_filesystem_item_and_file(const std::string &api_path,
-                                                  api_file &f,
+                                                  api_file &file,
                                                   filesystem_item &fsi) const
       -> api_error override;
 
@@ -158,43 +161,44 @@ public:
       -> api_error override;
 
   [[nodiscard]] auto get_pinned_files() const
-      -> std::vector<std::string> override;
-
-  [[nodiscard]] auto get_total_item_count() const -> std::uint64_t override;
+      -> std::vector<std::string> override {
+    return meta_db_->get_pinned_files();
+  }
 
   [[nodiscard]] auto get_used_drive_space() const -> std::uint64_t override;
 
-  [[nodiscard]] auto is_file_writeable(const std::string &api_path) const
-      -> bool override;
-
-  [[nodiscard]] auto remove_directory(const std::string &api_path)
-      -> api_error override;
-
-  [[nodiscard]] auto remove_file(const std::string &api_path)
-      -> api_error override;
+  [[nodiscard]] auto is_file_writeable(const std::string &) const
+      -> bool override {
+    return true;
+  }
 
   [[nodiscard]] auto remove_item_meta(const std::string &api_path,
                                       const std::string &key)
+      -> api_error override {
+    return meta_db_->remove_item_meta(api_path, key);
+  }
+
+  [[nodiscard]] auto rename_file(const std::string &from_api_path,
+                                 const std::string &to_api_path)
       -> api_error override;
 
   [[nodiscard]] auto set_item_meta(const std::string &api_path,
                                    const std::string &key,
                                    const std::string &value)
-      -> api_error override;
+      -> api_error override {
+    return meta_db_->set_item_meta(api_path, key, value);
+  }
 
   [[nodiscard]] auto set_item_meta(const std::string &api_path,
                                    const api_meta_map &meta)
-      -> api_error override;
+      -> api_error override {
+    return meta_db_->set_item_meta(api_path, meta);
+  }
 
   [[nodiscard]] auto start(api_item_added_callback api_item_added,
-                           i_file_manager *mgr) -> bool override;
+                           i_file_manager *fm) -> bool override;
 
   void stop() override;
-
-  [[nodiscard]] auto upload_file(const std::string &api_path,
-                                 const std::string &source_path,
-                                 stop_type &stop_requested)
-      -> api_error override;
 };
 } // namespace repertory
 

include/providers/encrypt/encrypt_provider.hpp

@@ -33,12 +33,6 @@ public:
 
   ~encrypt_provider() override = default;
 
-public:
-  encrypt_provider(const encrypt_provider &) = delete;
-  encrypt_provider(encrypt_provider &&) = delete;
-  auto operator=(const encrypt_provider &) -> encrypt_provider & = delete;
-  auto operator=(encrypt_provider &&) -> encrypt_provider & = delete;
-
 private:
   struct reader_info final {
     std::chrono::system_clock::time_point last_access_time =
@@ -49,7 +43,11 @@ private:
 
 private:
   app_config &config_;
-  db3_t db_;
+  std::unique_ptr<rocksdb::DB> db_;
+  rocksdb::ColumnFamilyHandle *dir_family_{};
+  rocksdb::ColumnFamilyHandle *file_family_{};
+  rocksdb::ColumnFamilyHandle *source_family_{};
+  const std::string DB_NAME = "meta_db";
 
 private:
   i_file_manager *fm_ = nullptr;
@@ -58,7 +56,7 @@ private:
   std::recursive_mutex reader_lookup_mtx_{};
 
 private:
-  static auto create_api_file(const std::string &api_path, bool directory,
+  static auto create_api_file(const std::string api_path, bool directory,
                               const std::string &source_path) -> api_file;
 
   static void create_item_meta(api_meta_map &meta, bool directory,
@@ -203,12 +201,13 @@ public:
   }
 
   [[nodiscard]] auto start(api_item_added_callback api_item_added,
-                           i_file_manager *mgr) -> bool override;
+                           i_file_manager *fm) -> bool override;
 
   void stop() override;
 
   [[nodiscard]] auto upload_file(const std::string & /*api_path*/,
                                  const std::string & /*source_path*/,
+                                 const std::string & /*encryption_token*/,
                                  stop_type & /*stop_requested*/)
       -> api_error override {
     return api_error::not_implemented;

include/providers/i_provider.hpp

@@ -143,13 +143,13 @@ public:
       -> api_error = 0;
 
   [[nodiscard]] virtual auto start(api_item_added_callback api_item_added,
-                                   i_file_manager *mgr) -> bool = 0;
+                                   i_file_manager *fm) -> bool = 0;
 
   virtual void stop() = 0;
 
-  [[nodiscard]] virtual auto upload_file(const std::string &api_path,
-                                         const std::string &source_path,
-                                         stop_type &stop_requested)
+  [[nodiscard]] virtual auto
+  upload_file(const std::string &api_path, const std::string &source_path,
+              const std::string &encryption_token, stop_type &stop_requested)
       -> api_error = 0;
 };
 } // namespace repertory

include/providers/provider.hpp

@@ -28,8 +28,7 @@ namespace repertory {
 class app_config;
 class i_provider;
 
-[[nodiscard]] auto create_provider(const provider_type &prov,
-                                   app_config &config)
+[[nodiscard]] auto create_provider(const provider_type &pt, app_config &config)
     -> std::unique_ptr<i_provider>;
 } // namespace repertory
 

include/providers/s3/s3_provider.hpp

@@ -23,93 +23,87 @@
 #define INCLUDE_PROVIDERS_S3_S3_PROVIDER_HPP_
 #if defined(REPERTORY_ENABLE_S3)
 
+#include "db/directory_db.hpp"
 #include "providers/base_provider.hpp"
 #include "types/repertory.hpp"
 
 namespace repertory {
 class app_config;
 class i_file_manager;
-class i_http_comm;
-struct head_object_result;
+class i_s3_comm;
 
 class s3_provider final : public base_provider {
 public:
-  s3_provider(app_config &config, i_http_comm &comm);
+  s3_provider(app_config &config, i_s3_comm &s3_comm);
 
   ~s3_provider() override = default;
 
-public:
-  s3_provider(const s3_provider &) = delete;
-  s3_provider(s3_provider &&) = delete;
-  auto operator=(const s3_provider &) -> s3_provider & = delete;
-  auto operator=(s3_provider &&) -> s3_provider & = delete;
+private:
+  i_s3_comm &s3_comm_;
 
 private:
-  [[nodiscard]] auto add_if_not_found(api_file &file,
-                                      const std::string &object_name) const
-      -> api_error;
+  std::unique_ptr<directory_db> directory_db_;
+  std::unique_ptr<std::thread> background_thread_;
 
-  [[nodiscard]] auto create_file_extra(const std::string &api_path,
-                                       api_meta_map &meta)
-      -> api_error override;
+private:
+  void create_directories();
 
-  [[nodiscard]] auto create_path_directories(const std::string &api_path,
-                                             const std::string &key) const
-      -> api_error;
+  void create_parent_directories(const api_file &file, bool directory);
 
-  [[nodiscard]] auto decrypt_object_name(std::string &object_name) const
-      -> api_error;
-
-  [[nodiscard]] auto
-  get_object_info(bool directory, const std::string &api_path,
-                  bool &is_encrypted, std::string &object_name,
-                  head_object_result &result) const -> api_error;
-
-  [[nodiscard]] auto
-  get_object_list(std::string &response_data, long &response_code,
-                  std::optional<std::string> delimiter = std::nullopt,
-                  std::optional<std::string> prefix = std::nullopt) const
-      -> bool;
+  void update_item_meta(directory_item &di) const;
 
 protected:
-  [[nodiscard]] auto create_directory_impl(const std::string &api_path,
-                                           api_meta_map &meta)
+  [[nodiscard]] auto check_file_exists(const std::string &api_path) const
       -> api_error override;
 
-  [[nodiscard]] auto get_directory_items_impl(const std::string &api_path,
+  [[nodiscard]] auto handle_rename_file(const std::string &from_api_path,
+                                        const std::string &to_api_path,
+                                        const std::string &source_path)
+      -> api_error override;
+
+  [[nodiscard]] auto notify_directory_added(const std::string &api_path,
+                                            const std::string &api_parent)
+      -> api_error override;
+
+  [[nodiscard]] auto notify_file_added(const std::string &api_path,
+                                       const std::string &api_parent,
+                                       std::uint64_t size)
+      -> api_error override;
+
+  [[nodiscard]] auto populate_directory_items(const std::string &api_path,
                                               directory_item_list &list) const
       -> api_error override;
 
-  [[nodiscard]] auto get_used_drive_space_impl() const
-      -> std::uint64_t override;
-
-  [[nodiscard]] auto remove_directory_impl(const std::string &api_path)
-      -> api_error override;
-
-  [[nodiscard]] auto remove_file_impl(const std::string &api_path)
-      -> api_error override;
-
-  [[nodiscard]] auto upload_file_impl(const std::string &api_path,
-                                      const std::string &source_path,
-                                      stop_type &stop_requested)
-      -> api_error override;
+  [[nodiscard]] auto populate_file(const std::string &api_path,
+                                   api_file &file) const -> api_error override;
 
 public:
+#ifdef REPERTORY_TESTING
+  void set_callback(api_item_added_callback cb) { api_item_added_ = cb; }
+#endif // REPERTORY_TESTING
+
+  [[nodiscard]] auto create_directory(const std::string &api_path,
+                                      api_meta_map &meta) -> api_error override;
+
+  [[nodiscard]] auto create_file(const std::string &api_path,
+                                 api_meta_map &meta) -> api_error override;
+
   [[nodiscard]] auto get_directory_item_count(const std::string &api_path) const
       -> std::uint64_t override;
 
-  [[nodiscard]] auto get_file(const std::string &api_path, api_file &file) const
-      -> api_error override;
-
   [[nodiscard]] auto get_file_list(api_file_list &list) const
       -> api_error override;
 
-  [[nodiscard]] auto get_total_drive_space() const -> std::uint64_t override;
-
   [[nodiscard]] auto get_provider_type() const -> provider_type override {
     return provider_type::s3;
   }
 
+  [[nodiscard]] auto get_total_drive_space() const -> std::uint64_t override {
+    return std::numeric_limits<std::int64_t>::max() / std::int64_t(2);
+  }
+
+  [[nodiscard]] auto get_total_item_count() const -> std::uint64_t override;
+
   [[nodiscard]] auto is_direct_only() const -> bool override { return false; }
 
   [[nodiscard]] auto is_directory(const std::string &api_path,
@@ -122,7 +116,7 @@ public:
 
   [[nodiscard]] auto is_rename_supported() const -> bool override {
     return false;
-  };
+  }
 
   [[nodiscard]] auto read_file_bytes(const std::string &api_path,
                                      std::size_t size, std::uint64_t offset,
@@ -130,14 +124,21 @@ public:
                                      stop_type &stop_requested)
       -> api_error override;
 
-  [[nodiscard]] auto rename_file(const std::string &from_api_path,
-                                 const std::string &to_api_path)
+  [[nodiscard]] auto remove_directory(const std::string &api_path)
+      -> api_error override;
+
+  [[nodiscard]] auto remove_file(const std::string &api_path)
       -> api_error override;
 
   [[nodiscard]] auto start(api_item_added_callback api_item_added,
-                           i_file_manager *mgr) -> bool override;
+                           i_file_manager *fm) -> bool override;
 
   void stop() override;
+
+  [[nodiscard]] auto
+  upload_file(const std::string &api_path, const std::string &source_path,
+              const std::string &encryption_token, stop_type &stop_requested)
+      -> api_error override;
 };
 } // namespace repertory
 

include/providers/sia/sia_provider.hpp

@@ -22,7 +22,7 @@
 #ifndef INCLUDE_PROVIDERS_SIA_SIA_PROVIDER_HPP_
 #define INCLUDE_PROVIDERS_SIA_SIA_PROVIDER_HPP_
 
-#include "providers/base_provider.hpp"
+#include "providers/i_provider.hpp"
 #include "types/repertory.hpp"
 
 namespace repertory {
@@ -30,64 +30,108 @@ class app_config;
 class i_file_manager;
 class i_http_comm;
 
-class sia_provider : public base_provider {
+class sia_provider : public i_provider {
 public:
   sia_provider(app_config &config, i_http_comm &comm);
 
   ~sia_provider() override = default;
 
-public:
-  sia_provider(const sia_provider &) = delete;
-  sia_provider(sia_provider &&) = delete;
-  auto operator=(const sia_provider &) -> sia_provider & = delete;
-  auto operator=(sia_provider &&) -> sia_provider & = delete;
+private:
+  app_config &config_;
+  i_http_comm &comm_;
 
 private:
+  const std::string DB_NAME = "meta_db";
+  api_item_added_callback api_item_added_;
+  std::unique_ptr<rocksdb::DB> db_;
+  i_file_manager *fm_ = nullptr;
+
+private:
+  [[nodiscard]] static auto create_api_file(std::string path,
+                                            std::uint64_t size) -> api_file;
+
+  [[nodiscard]] static auto create_api_file(std::string path,
+                                            std::uint64_t size,
+                                            api_meta_map &meta) -> api_file;
+
   [[nodiscard]] auto get_object_info(const std::string &api_path,
                                      json &object_info) const -> api_error;
 
-  [[nodiscard]] auto get_object_list(const std::string &api_path,
+  [[nodiscard]] auto get_object_list(const std::string api_path,
                                      nlohmann::json &object_list) const -> bool;
 
-protected:
-  [[nodiscard]] auto create_directory_impl(const std::string &api_path,
-                                           api_meta_map &meta)
-      -> api_error override;
-
-  [[nodiscard]] auto get_directory_items_impl(const std::string &api_path,
-                                              directory_item_list &list) const
-      -> api_error override;
-
-  [[nodiscard]] auto get_used_drive_space_impl() const
-      -> std::uint64_t override;
-
-  [[nodiscard]] auto remove_directory_impl(const std::string &api_path)
-      -> api_error override;
-
-  [[nodiscard]] auto remove_file_impl(const std::string &api_path)
-      -> api_error override;
-
-  [[nodiscard]] auto upload_file_impl(const std::string &api_path,
-                                      const std::string &source_path,
-                                      stop_type &stop_requested)
-      -> api_error override;
+  void remove_deleted_files();
 
 public:
+  [[nodiscard]] auto create_directory(const std::string &api_path,
+                                      api_meta_map &meta) -> api_error override;
+
+  [[nodiscard]] auto
+  create_directory_clone_source_meta(const std::string &source_api_path,
+                                     const std::string &api_path)
+      -> api_error override;
+
   [[nodiscard]] auto get_directory_item_count(const std::string &api_path) const
       -> std::uint64_t override;
 
+  [[nodiscard]] auto create_file(const std::string &api_path,
+                                 api_meta_map &meta) -> api_error override;
+
+  [[nodiscard]] auto get_api_path_from_source(const std::string &source_path,
+                                              std::string &api_path) const
+      -> api_error override;
+
+  [[nodiscard]] auto get_directory_items(const std::string &api_path,
+                                         directory_item_list &list) const
+      -> api_error override;
+
   [[nodiscard]] auto get_file(const std::string &api_path, api_file &file) const
       -> api_error override;
 
   [[nodiscard]] auto get_file_list(api_file_list &list) const
       -> api_error override;
 
+  [[nodiscard]] auto get_file_size(const std::string &api_path,
+                                   std::uint64_t &file_size) const
+      -> api_error override;
+
+  [[nodiscard]] auto get_filesystem_item(const std::string &api_path,
+                                         bool directory,
+                                         filesystem_item &fsi) const
+      -> api_error override;
+
+  [[nodiscard]] auto get_filesystem_item_and_file(const std::string &api_path,
+                                                  api_file &f,
+                                                  filesystem_item &fsi) const
+      -> api_error override;
+
+  [[nodiscard]] auto
+  get_filesystem_item_from_source_path(const std::string &source_path,
+                                       filesystem_item &fsi) const
+      -> api_error override;
+
+  [[nodiscard]] auto get_item_meta(const std::string &api_path,
+                                   api_meta_map &meta) const
+      -> api_error override;
+
+  [[nodiscard]] auto get_item_meta(const std::string &api_path,
+                                   const std::string &key,
+                                   std::string &value) const
+      -> api_error override;
+
+  [[nodiscard]] auto get_pinned_files() const
+      -> std::vector<std::string> override;
+
   [[nodiscard]] auto get_provider_type() const -> provider_type override {
     return provider_type::sia;
   }
 
   [[nodiscard]] auto get_total_drive_space() const -> std::uint64_t override;
 
+  [[nodiscard]] auto get_total_item_count() const -> std::uint64_t override;
+
+  [[nodiscard]] auto get_used_drive_space() const -> std::uint64_t override;
+
   [[nodiscard]] auto is_direct_only() const -> bool override { return false; }
 
   [[nodiscard]] auto is_directory(const std::string &api_path,
@@ -96,10 +140,13 @@ public:
   [[nodiscard]] auto is_file(const std::string &api_path, bool &exists) const
       -> api_error override;
 
+  [[nodiscard]] auto is_file_writeable(const std::string &api_path) const
+      -> bool override;
+
   [[nodiscard]] auto is_online() const -> bool override;
 
   [[nodiscard]] auto is_rename_supported() const -> bool override {
-    return true;
+    return false;
   }
 
   [[nodiscard]] auto read_file_bytes(const std::string &api_path,
@@ -108,14 +155,39 @@ public:
                                      stop_type &stop_requested)
       -> api_error override;
 
+  [[nodiscard]] auto remove_directory(const std::string &api_path)
+      -> api_error override;
+
+  [[nodiscard]] auto remove_file(const std::string &api_path)
+      -> api_error override;
+
+  [[nodiscard]] auto remove_item_meta(const std::string &api_path,
+                                      const std::string &key)
+      -> api_error override;
+
   [[nodiscard]] auto rename_file(const std::string &from_api_path,
                                  const std::string &to_api_path)
       -> api_error override;
 
+  [[nodiscard]] auto set_item_meta(const std::string &api_path,
+                                   const std::string &key,
+                                   const std::string &value)
+      -> api_error override;
+
+  [[nodiscard]] auto set_item_meta(const std::string &api_path,
+                                   const api_meta_map &meta)
+      -> api_error override;
+
   [[nodiscard]] auto start(api_item_added_callback api_item_added,
-                           i_file_manager *mgr) -> bool override;
+                           i_file_manager *fm) -> bool override;
 
   void stop() override;
+
+  [[nodiscard]] auto upload_file(const std::string &api_path,
+                                 const std::string &source_path,
+                                 const std::string & /* encryption_token */,
+                                 stop_type &stop_requested)
+      -> api_error override;
 };
 } // namespace repertory
 

include/types/repertory.hpp

@@ -29,7 +29,9 @@ const std::string META_BACKUP = "backup";
 const std::string META_CHANGED = "changed";
 const std::string META_CREATION = "creation";
 const std::string META_DIRECTORY = "directory";
+const std::string META_ENCRYPTION_TOKEN = "token";
 const std::string META_GID = "gid";
+const std::string META_ID = "id";
 const std::string META_KEY = "key";
 const std::string META_MODE = "mode";
 const std::string META_MODIFIED = "modified";
@@ -41,10 +43,24 @@ const std::string META_UID = "uid";
 const std::string META_WRITTEN = "written";
 
 const std::vector<std::string> META_USED_NAMES = {
-    META_ACCESSED, META_ATTRIBUTES, META_BACKUP,   META_CHANGED,
-    META_CREATION, META_DIRECTORY,  META_GID,      META_KEY,
-    META_MODE,     META_MODIFIED,   META_OSXFLAGS, META_PINNED,
-    META_SIZE,     META_SOURCE,     META_UID,      META_WRITTEN,
+    META_ACCESSED,
+    META_ATTRIBUTES,
+    META_BACKUP,
+    META_CHANGED,
+    META_CREATION,
+    META_DIRECTORY,
+    META_ENCRYPTION_TOKEN,
+    META_GID,
+    META_ID,
+    META_KEY,
+    META_MODE,
+    META_MODIFIED,
+    META_OSXFLAGS,
+    META_PINNED,
+    META_SIZE,
+    META_SOURCE,
+    META_UID,
+    META_WRITTEN,
 };
 
 using api_meta_map = std::map<std::string, std::string>;
@@ -125,7 +141,6 @@ enum class exit_code : std::int32_t {
 
 enum http_error_codes : std::int32_t {
   ok = 200,
-  multiple_choices = 300,
   not_found = 404,
 };
 
@@ -139,37 +154,39 @@ enum class provider_type : std::size_t {
   sia,
   remote,
   s3,
+  passthrough,
   encrypt,
   unknown,
 };
 
 #ifdef _WIN32
 struct open_file_data {
-  void *directory_buffer{};
+  void *directory_buffer = nullptr;
 };
 #else
 using open_file_data = int;
 #endif
 
 struct api_file {
-  std::string api_path;
-  std::string api_parent;
-  std::uint64_t accessed_date{};
-  std::uint64_t changed_date{};
-  std::uint64_t creation_date{};
-  std::uint64_t file_size{};
-  std::string key;
-  std::uint64_t modified_date{};
-  std::string source_path;
+  std::string api_path{};
+  std::string api_parent{};
+  std::uint64_t accessed_date = 0u;
+  std::uint64_t changed_date = 0u;
+  std::uint64_t creation_date = 0u;
+  std::string encryption_token{};
+  std::uint64_t file_size = 0u;
+  std::string key{};
+  std::uint64_t modified_date = 0u;
+  std::string source_path{};
 };
 
 struct directory_item {
-  std::string api_path;
-  std::string api_parent;
-  bool directory{false};
-  std::uint64_t size{};
+  std::string api_path{};
+  std::string api_parent{};
+  bool directory = false;
+  std::uint64_t size = 0u;
   api_meta_map meta{};
-  bool resolved{false};
+  bool resolved = false;
 
   [[nodiscard]] static auto from_json(const json &item) -> directory_item {
     directory_item ret{};
@@ -191,22 +208,27 @@ struct directory_item {
 };
 
 struct filesystem_item {
-  std::string api_path;
-  std::string api_parent;
-  bool directory{false};
-  std::uint64_t size{};
-  std::string source_path;
+  std::string api_path{};
+  std::string api_parent{};
+  bool directory = false;
+  std::string encryption_token{};
+  std::uint64_t size = 0u;
+  std::string source_path{};
+
+  [[nodiscard]] auto is_encrypted() const -> bool {
+    return not encryption_token.empty();
+  }
 };
 
 struct host_config {
-  std::string agent_string;
-  std::string api_password;
-  std::string api_user;
-  std::uint16_t api_port{};
-  std::string host_name_or_ip{"localhost"};
+  std::string agent_string{};
+  std::string api_password{};
+  std::string api_user{};
+  std::uint16_t api_port = 0u;
+  std::string host_name_or_ip = "localhost";
   std::string path{};
-  std::string protocol{"http"};
-  std::uint32_t timeout_ms{60000U};
+  std::string protocol = "http";
+  std::uint32_t timeout_ms = 60000u;
 
   auto operator==(const host_config &hc) const noexcept -> bool {
     if (&hc != this) {
@@ -258,25 +280,29 @@ from_json(const json &j, host_config &hc) {
 }
 
 struct http_range {
-  std::uint64_t begin;
-  std::uint64_t end;
+  std::uint64_t begin = 0u;
+  std::uint64_t end = 0u;
+};
+
+struct passthrough_config {
+  std::string location{};
+  std::string name{};
 };
 
 struct encrypt_config {
-  std::string encryption_token;
-  std::string path;
+  std::string encryption_token{};
+  std::string path{};
 };
 
 struct s3_config {
-  std::string access_key;
-  std::string bucket;
-  std::uint16_t cache_timeout_secs{60U};
-  std::string encryption_token;
+  std::string access_key{};
+  std::string bucket{};
+  std::uint16_t cache_timeout_secs = 60u;
+  std::string encryption_token{};
   std::string region = "any";
-  std::string secret_key;
-  std::uint32_t timeout_ms{60000U};
+  std::string secret_key{};
+  std::uint32_t timeout_ms = 60000u;
   std::string url;
-  bool use_path_style{false};
   bool use_region_in_url{false};
 };
 

include/utils/cli_utils.hpp

@@ -34,6 +34,8 @@ static const option data_directory_option = {"-dd", "--data_directory"};
 static const option encrypt_option = {"-en", "--encrypt"};
 static const option drive_information_option = {"-di", "--drive_information"};
 #if defined(REPERTORY_ENABLE_S3)
+static const option create_directory_option = {"-cd", "--create_directory"};
+static const option list_objects_option = {"-lo", "--list_objects"};
 static const option name_option = {"-na", "--name"};
 static const option s3_option = {"-s3", "--s3"};
 #endif // defined(REPERTORY_ENABLE_S3)
@@ -65,6 +67,8 @@ static const std::vector<option> option_list = {
 #if defined(REPERTORY_ENABLE_S3)
     s3_option,
     name_option,
+    create_directory_option,
+    list_objects_option,
 #endif // defined(REPERTORY_ENABLE_S3)
     generate_config_option,
     get_option,
@@ -88,28 +92,26 @@ static const std::vector<option> option_list = {
 
 // Prototypes
 void get_api_authentication_data(std::string &user, std::string &password,
-                                 std::uint16_t &port, const provider_type &prov,
+                                 std::uint16_t &port, const provider_type &pt,
                                  const std::string &data_directory);
 
-[[nodiscard]] auto get_provider_type_from_args(std::vector<const char *> args)
+[[nodiscard]] auto get_provider_type_from_args(int argc, char *argv[])
     -> provider_type;
 
-[[nodiscard]] auto has_option(std::vector<const char *> args,
+[[nodiscard]] auto has_option(int argc, char *argv[],
                               const std::string &option_name) -> bool;
 
-[[nodiscard]] auto has_option(std::vector<const char *> args, const option &opt)
+[[nodiscard]] auto has_option(int argc, char *argv[], const option &opt)
     -> bool;
 
-[[nodiscard]] auto parse_option(std::vector<const char *> args,
+[[nodiscard]] auto parse_option(int argc, char *argv[],
                                 const std::string &option_name,
                                 std::uint8_t count) -> std::vector<std::string>;
 
-[[nodiscard]] auto parse_string_option(std::vector<const char *> args,
-                                       const option &opt, std::string &value)
-    -> exit_code;
+[[nodiscard]] auto parse_string_option(int argc, char **argv, const option &opt,
+                                       std::string &value) -> exit_code;
 
-[[nodiscard]] auto parse_drive_options(std::vector<const char *> args,
-                                       provider_type &prov,
+[[nodiscard]] auto parse_drive_options(int argc, char **argv, provider_type &pt,
                                        std::string &data_directory)
     -> std::vector<std::string>;
 } // namespace repertory::utils::cli

include/utils/encrypting_reader.hpp

@@ -30,15 +30,11 @@ using key_type = std::array<unsigned char, 32U>;
 
 class encrypting_reader final {
 public:
-  encrypting_reader(const std::string &file_name,
-                    const std::string &source_path, stop_type &stop_requested,
-                    const std::string &token,
-                    std::optional<std::string> relative_parent_path,
-                    std::size_t error_return = 0U);
-
-  encrypting_reader(const std::string &encrypted_file_path,
-                    const std::string &source_path, stop_type &stop_requested,
-                    const std::string &token, std::size_t error_return = 0U);
+  encrypting_reader(
+      const std::string &file_name, const std::string &source_path,
+      stop_type &stop_requested, const std::string &token,
+      std::optional<std::string> relative_parent_path = std::nullopt,
+      const size_t error_return = 0);
 
   encrypting_reader(
       const std::string &encrypted_file_path, const std::string &source_path,
@@ -46,13 +42,9 @@ public:
       std::vector<std::array<unsigned char,
                              crypto_aead_xchacha20poly1305_IETF_NPUBBYTES>>
           iv_list,
-      std::size_t error_return = 0U);
+      const size_t error_return = 0);
 
-  encrypting_reader(const encrypting_reader &reader);
-  encrypting_reader(encrypting_reader &&) = delete;
-
-  auto operator=(const encrypting_reader &) -> encrypting_reader & = delete;
-  auto operator=(encrypting_reader &&) -> encrypting_reader & = delete;
+  encrypting_reader(const encrypting_reader &r);
 
   ~encrypting_reader();
 
@@ -70,11 +62,11 @@ private:
   std::vector<
       std::array<unsigned char, crypto_aead_xchacha20poly1305_IETF_NPUBBYTES>>
       iv_list_;
-  std::size_t last_data_chunk_{};
-  std::size_t last_data_chunk_size_{};
-  std::uint64_t read_offset_{};
+  std::size_t last_data_chunk_ = 0u;
+  std::size_t last_data_chunk_size_ = 0u;
+  std::uint64_t read_offset_ = 0u;
   native_file_ptr source_file_;
-  std::uint64_t total_size_{};
+  std::uint64_t total_size_ = 0u;
 
 private:
   static const std::size_t header_size_;

include/utils/encryption.hpp

@@ -26,9 +26,6 @@
 #include "utils/encrypting_reader.hpp"
 
 namespace repertory::utils::encryption {
-using reader_func = std::function<api_error(data_buffer &cypher_text,
-                                            std::uint64_t start_offset,
-                                            std::uint64_t end_offset)>;
 // Prototypes
 [[nodiscard]] auto decrypt_file_path(const std::string &encryption_token,
                                      std::string &file_path) -> api_error;
@@ -39,10 +36,11 @@ using reader_func = std::function<api_error(data_buffer &cypher_text,
 [[nodiscard]] auto generate_key(const std::string &encryption_token)
     -> key_type;
 
-[[nodiscard]] auto read_encrypted_range(const http_range &range,
-                                        const key_type &key, reader_func reader,
-                                        std::uint64_t total_size,
-                                        data_buffer &data) -> api_error;
+[[nodiscard]] auto read_encrypted_range(
+    const http_range &range, const key_type &key,
+    const std::function<api_error(data_buffer &ct, std::uint64_t start_offset,
+                                  std::uint64_t end_offset)> &reader,
+    std::uint64_t total_size, data_buffer &data) -> api_error;
 
 // Implementations
 template <typename result>

include/utils/error_utils.hpp

@@ -25,60 +25,60 @@
 #include "types/repertory.hpp"
 
 namespace repertory::utils::error {
-void raise_error(std::string function, std::string_view msg);
+void raise_error(std::string_view function, std::string_view msg);
 
-void raise_error(std::string function, const api_error &err,
+void raise_error(std::string_view function, const api_error &e,
                  std::string_view msg);
 
-void raise_error(std::string function, const std::exception &exception,
+void raise_error(std::string_view function, const std::exception &e,
                  std::string_view msg);
 
-void raise_error(std::string function, std::int64_t err, std::string_view msg);
+void raise_error(std::string_view function, std::int64_t e,
+                 std::string_view msg);
 
-void raise_error(std::string function, const json &err, std::string_view msg);
+void raise_error(std::string_view function, const json &e,
+                 std::string_view msg);
 
-void raise_error(std::string function, const api_error &err,
+void raise_error(std::string_view function, const api_error &e,
                  std::string_view file_path, std::string_view msg);
 
-void raise_error(std::string function, std::int64_t err,
+void raise_error(std::string_view function, std::int64_t e,
                  std::string_view file_path, std::string_view msg);
 
-void raise_error(std::string function, const std::exception &exception,
+void raise_error(std::string_view function, const std::exception &e,
                  std::string_view file_path, std::string_view msg);
 
-void raise_api_path_error(std::string function, std::string_view api_path,
-                          const api_error &err, std::string_view msg);
+void raise_api_path_error(std::string_view function, std::string_view api_path,
+                          const api_error &e, std::string_view msg);
 
-void raise_api_path_error(std::string function, std::string_view api_path,
-                          const std::exception &exception,
+void raise_api_path_error(std::string_view function, std::string_view api_path,
+                          const std::exception &e, std::string_view msg);
+
+void raise_api_path_error(std::string_view function, std::string_view api_path,
+                          std::int64_t e, std::string_view msg);
+
+void raise_api_path_error(std::string_view function, std::string_view api_path,
+                          const json &e, std::string_view msg);
+
+void raise_api_path_error(std::string_view function, std::string_view api_path,
+                          std::string_view source_path, const api_error &e,
                           std::string_view msg);
 
-void raise_api_path_error(std::string function, std::string_view api_path,
-                          std::int64_t err, std::string_view msg);
-
-void raise_api_path_error(std::string function, std::string_view api_path,
-                          const json &err, std::string_view msg);
-
-void raise_api_path_error(std::string function, std::string_view api_path,
-                          std::string_view source_path, const api_error &err,
+void raise_api_path_error(std::string_view function, std::string_view api_path,
+                          std::string_view source_path, std::int64_t e,
                           std::string_view msg);
 
-void raise_api_path_error(std::string function, std::string_view api_path,
-                          std::string_view source_path, std::int64_t err,
+void raise_api_path_error(std::string_view function, std::string_view api_path,
+                          std::string_view source_path, const std::exception &e,
                           std::string_view msg);
 
-void raise_api_path_error(std::string function, std::string_view api_path,
-                          std::string_view source_path,
-                          const std::exception &exception,
-                          std::string_view msg);
+void raise_url_error(std::string_view function, std::string_view url,
+                     CURLcode e, std::string_view msg);
 
-void raise_url_error(std::string function, std::string_view url, CURLcode err,
+void raise_url_error(std::string_view function, std::string_view url,
+                     std::string_view source_path, const std::exception &e,
                      std::string_view msg);
 
-void raise_url_error(std::string function, std::string_view url,
-                     std::string_view source_path,
-                     const std::exception &exception, std::string_view msg);
-
 } // namespace repertory::utils::error
 
 #endif // INCLUDE_UTILS_ERROR_UTILS_HPP_

include/utils/rocksdb_utils.hpp

@@ -19,27 +19,22 @@
   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
   SOFTWARE.
 */
-#include "comm/curl/requests/http_post.hpp"
+#ifndef INCLUDE_UTILS_ROCKSDB_UTILS_HPP_
+#define INCLUDE_UTILS_ROCKSDB_UTILS_HPP_
 
-namespace repertory::curl::requests {
-http_post::~http_post() {
-  if (headers != nullptr) {
-    curl_slist_free_all(headers);
-  }
-}
+namespace repertory {
+class app_config;
 
-auto http_post::set_method(CURL *curl, stop_type & /*stop_requested*/) const
-    -> bool {
-  curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "POST");
-  if (json.has_value()) {
-    headers = curl_slist_append(headers, "content-type: application/json");
-    curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
+namespace utils::db {
+void create_rocksdb(const app_config &config, const std::string &name,
+                    std::unique_ptr<rocksdb::DB> &db);
 
-    json_str = json->dump();
-    curl_easy_setopt(curl, CURLOPT_POSTFIELDS, json_str->c_str());
-    curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, -1L);
-  }
+void create_rocksdb(
+    const app_config &config, const std::string &name,
+    const std::vector<rocksdb::ColumnFamilyDescriptor> &families,
+    std::vector<rocksdb::ColumnFamilyHandle *> &handles,
+    std::unique_ptr<rocksdb::DB> &db);
+} // namespace utils::db
+} // namespace repertory
 
-  return true;
-}
-} // namespace repertory::curl::requests
+#endif // INCLUDE_UTILS_ROCKSDB_UTILS_HPP_

include/utils/unix/unix_utils.hpp

@@ -28,7 +28,7 @@
 
 namespace repertory::utils {
 #if __linux__
-inline const std::array<std::string, 4U> attribute_namespaces = {
+inline const std::array<std::string, 4u> attribute_namespaces = {
     "security",
     "system",
     "trusted",
@@ -38,12 +38,12 @@ inline const std::array<std::string, 4U> attribute_namespaces = {
 
 #if __APPLE__
 template <typename t>
-[[nodiscard]] auto convert_to_uint64(const t *ptr) -> std::uint64_t;
+[[nodiscard]] auto convert_to_uint64(const t *v) -> std::uint64_t;
 #else
-[[nodiscard]] auto convert_to_uint64(const pthread_t &thread) -> std::uint64_t;
+[[nodiscard]] auto convert_to_uint64(const pthread_t &t) -> std::uint64_t;
 #endif
 
-[[nodiscard]] auto from_api_error(const api_error &err) -> int;
+[[nodiscard]] auto from_api_error(const api_error &e) -> int;
 
 [[nodiscard]] auto get_last_error_code() -> int;
 
@@ -54,20 +54,20 @@ template <typename t>
 
 void set_last_error_code(int error_code);
 
-[[nodiscard]] auto to_api_error(int err) -> api_error;
+[[nodiscard]] auto to_api_error(int e) -> api_error;
 
-[[nodiscard]] auto unix_error_to_windows(int err) -> std::int32_t;
+[[nodiscard]] auto unix_error_to_windows(int e) -> std::int32_t;
 
-[[nodiscard]] auto unix_time_to_windows_time(const remote::file_time &file_time)
+[[nodiscard]] auto unix_time_to_windows_time(const remote::file_time &ts)
     -> UINT64;
 
-void use_getpwuid(uid_t uid, std::function<void(struct passwd *pass)> callback);
+void use_getpwuid(uid_t uid, std::function<void(struct passwd *pw)> fn);
 
 void windows_create_to_unix(const UINT32 &create_options,
                             const UINT32 &granted_access, std::uint32_t &flags,
                             remote::file_mode &mode);
 
-[[nodiscard]] auto windows_time_to_unix_time(std::uint64_t win_time)
+[[nodiscard]] auto windows_time_to_unix_time(std::uint64_t t)
     -> remote::file_time;
 
 // template implementations

include/utils/utils.hpp

@@ -151,7 +151,7 @@ template <typename t>
 template <typename data_type>
 [[nodiscard]] auto random_between(const data_type &begin, const data_type &end)
     -> data_type {
-  return begin + repertory_rand<data_type>() % ((end + data_type{1}) - begin);
+  return begin + repertory_rand<data_type>() % ((end + 1) - begin);
 }
 
 template <typename collection_t>

include/utils/windows/windows_utils.hpp

@@ -50,7 +50,7 @@ namespace repertory::utils {
 
 [[nodiscard]] auto is_process_elevated() -> bool;
 
-[[nodiscard]] auto run_process_elevated(std::vector<const char *> args) -> int;
+[[nodiscard]] auto run_process_elevated(int argc, char *argv[]) -> int;
 
 void set_last_error_code(DWORD errorCode);
 

scripts/make_unix.sh

@@ -12,8 +12,6 @@ if [ -z "${BUILD_ARCH}" ]; then
   BUILD_ARCH=64_bit
 fi
 
-ln -sf ${SOURCE_DIR}/build/compile_commands.json ${SOURCE_DIR}/compile_commands.json
-
 docker stop repertory_${NAME}
 docker rm repertory_${NAME}
 docker build -t repertory:${NAME} - < ${SOURCE_DIR}/docker/${BUILD_ARCH}/${NAME} &&

scripts/make_unix_docker.sh

@@ -41,15 +41,6 @@ cmake ../.. ${CMAKE_ADDITIONAL_OPTS} \
   -DREPERTORY_ENABLE_S3=ON \
   -DREPERTORY_ENABLE_S3_TESTING=ON || exit 1
 
-pushd ..
-ln -sf ${BUILD_FOLDER}/compile_commands.json .
-ln -sf ${BUILD_FOLDER}/repertory${EXE_EXT} .
-ln -sf ${BUILD_FOLDER}/unittests${EXE_EXT} .
-if [ "${IS_MINGW}" == "1" ]; then
-  ln -sf ${BUILD_FOLDER}/winfsp-x64.dll .
-fi
-popd
-
 if [ "${BUILD_CLEAN}" == "clean" ]; then
   make clean
 fi

src/app_config.cpp

@@ -28,91 +28,70 @@
 #include "utils/unix/unix_utils.hpp"
 #include "utils/utils.hpp"
 
-namespace {
-constexpr const auto default_api_auth_size = 48U;
-constexpr const auto default_download_timeout_ces = 30U;
-constexpr const auto default_eviction_delay_mins = 10U;
-constexpr const auto default_high_freq_interval_secs = 30U;
-constexpr const auto default_low_freq_interval_secs = 60U * 60U;
-constexpr const auto default_max_cache_size_bytes =
-    20ULL * 1024ULL * 1024ULL * 1024ULL;
-constexpr const auto default_max_upload_count = 5U;
-constexpr const auto default_min_download_timeout_secs = 5U;
-constexpr const auto default_online_check_retry_secs = 60U;
-constexpr const auto default_orphaned_file_retention_days = 15U;
-constexpr const auto default_read_ahead_count = 4U;
-constexpr const auto default_remote_client_pool_size = 10U;
-constexpr const auto default_remote_host_name_or_ip = "";
-constexpr const auto default_remote_max_connections = 20U;
-constexpr const auto default_remote_receive_timeout_secs = 120U;
-constexpr const auto default_remote_send_timeout_secs = 30U;
-constexpr const auto default_remote_token = "";
-constexpr const auto default_retry_read_count = 6U;
-constexpr const auto default_ring_buffer_file_size = 512U;
-constexpr const auto retry_save_count = 5U;
-} // namespace
-
 namespace repertory {
-app_config::app_config(const provider_type &prov,
+app_config::app_config(const provider_type &pt,
                        const std::string &data_directory)
-    : prov_(prov),
-      api_auth_(utils::generate_random_string(default_api_auth_size)),
-      api_port_(default_rpc_port(prov)),
-      api_user_(REPERTORY),
+    : pt_(pt),
+      api_auth_(utils::generate_random_string(48u)),
+      api_port_(default_rpc_port(pt)),
+      api_user_("repertory"),
       config_changed_(false),
-      data_directory_(data_directory.empty()
-                          ? default_data_directory(prov)
-                          : utils::path::absolute(data_directory)),
-      download_timeout_secs_(default_download_timeout_ces),
+      data_directory_(
+          data_directory.empty() ? default_data_directory(pt)
+          : ((pt == provider_type::remote) || (pt == provider_type::s3))
+              ? utils::path::absolute(data_directory)
+              : utils::path::absolute(utils::path::combine(
+                    data_directory, {get_provider_name(pt)}))),
+      download_timeout_secs_(30),
       enable_chunk_downloader_timeout_(true),
       enable_comm_duration_events_(false),
       enable_drive_events_(false),
-      enable_max_cache_size_(false),
+      enable_max_cache_size_(true),
 #ifdef _WIN32
       enable_mount_manager_(false),
 #endif
       enable_remote_mount_(false),
       event_level_(event_level::normal),
-      eviction_delay_mins_(default_eviction_delay_mins),
+      eviction_delay_mins_(30),
       eviction_uses_accessed_time_(false),
-      high_freq_interval_secs_(default_high_freq_interval_secs),
+      high_freq_interval_secs_(30),
       is_remote_mount_(false),
-      low_freq_interval_secs_(default_low_freq_interval_secs),
-      max_cache_size_bytes_(default_max_cache_size_bytes),
-      max_upload_count_(default_max_upload_count),
-      min_download_timeout_secs_(default_min_download_timeout_secs),
-      online_check_retry_secs_(default_online_check_retry_secs),
-      orphaned_file_retention_days_(default_orphaned_file_retention_days),
+      low_freq_interval_secs_(60 * 60),
+      max_cache_size_bytes_(20 * 1024 * 1024 * 1024ULL),
+      max_upload_count_(5u),
+      min_download_timeout_secs_(5),
+      online_check_retry_secs_(60),
+      orphaned_file_retention_days_(15),
       preferred_download_type_(
           utils::download_type_to_string(download_type::fallback)),
-      read_ahead_count_(default_read_ahead_count),
-      remote_client_pool_size_(default_remote_client_pool_size),
-      remote_host_name_or_ip_(default_remote_host_name_or_ip),
-      remote_max_connections_(default_remote_max_connections),
-      remote_port_(default_remote_port(prov)),
-      remote_receive_timeout_secs_(default_remote_receive_timeout_secs),
-      remote_send_timeout_secs_(default_remote_send_timeout_secs),
-      remote_token_(default_remote_token),
-      retry_read_count_(default_retry_read_count),
-      ring_buffer_file_size_(default_ring_buffer_file_size) {
+      read_ahead_count_(4),
+      remote_client_pool_size_(10),
+      remote_host_name_or_ip_(""),
+      remote_max_connections_(20),
+      remote_port_((pt == provider_type::sia)       ? 20000
+                   : (pt == provider_type::s3)      ? 20001
+                   : (pt == provider_type::encrypt) ? 20002
+                                                    : 20003),
+      remote_receive_timeout_secs_(120),
+      remote_send_timeout_secs_(30),
+      remote_token_(""),
+      retry_read_count_(6),
+      ring_buffer_file_size_(512) {
   cache_directory_ = utils::path::combine(data_directory_, {"cache"});
   log_directory_ = utils::path::combine(data_directory_, {"logs"});
 
-  hc_.agent_string = default_agent_name(prov_);
-  hc_.api_password = get_provider_api_password(prov_);
-  hc_.api_port = default_api_port(prov_);
+  hc_.agent_string = default_agent_name(pt_);
+  hc_.api_password = get_provider_api_password(pt_);
+  hc_.api_port = default_api_port(pt_);
 
-  if (not utils::file::create_full_directory_path(data_directory_)) {
+  if (not utils::file::create_full_directory_path(data_directory_))
     throw startup_exception("unable to create: " + data_directory_);
-  }
 
-  if (not utils::file::create_full_directory_path(cache_directory_)) {
+  if (not utils::file::create_full_directory_path(cache_directory_))
     throw startup_exception("unable to create: " + cache_directory_);
-  }
 
-  if (not utils::file::create_full_directory_path(log_directory_)) {
+  if (not utils::file::create_full_directory_path(log_directory_))
     throw startup_exception("unable to create: " + log_directory_);
-  }
 
   if (not load()) {
     save();
@@ -123,73 +102,48 @@ auto app_config::get_config_file_path() const -> std::string {
   return utils::path::combine(data_directory_, {"config.json"});
 }
 
-auto app_config::default_agent_name(const provider_type &prov) -> std::string {
+auto app_config::default_agent_name(const provider_type &pt) -> std::string {
   static const std::array<std::string,
                           static_cast<std::size_t>(provider_type::unknown)>
-      PROVIDER_AGENT_NAMES = {
-          "Sia-Agent",
-          "",
-          "",
-          "",
-      };
+      PROVIDER_AGENT_NAMES = {"Sia-Agent", "", "", "", ""};
 
-  return PROVIDER_AGENT_NAMES.at(static_cast<std::size_t>(prov));
+  return PROVIDER_AGENT_NAMES[static_cast<std::size_t>(pt)];
 }
 
-auto app_config::default_api_port(const provider_type &prov) -> std::uint16_t {
+auto app_config::default_api_port(const provider_type &pt) -> std::uint16_t {
   static const std::array<std::uint16_t,
                           static_cast<std::size_t>(provider_type::unknown)>
-      PROVIDER_API_PORTS = {
-          9980U,
-          0U,
-          0U,
-          0U,
-      };
-  return PROVIDER_API_PORTS.at(static_cast<std::size_t>(prov));
+      PROVIDER_API_PORTS = {9980u, 0u, 0u, 0u, 0u};
+  return PROVIDER_API_PORTS[static_cast<std::size_t>(pt)];
 }
 
-auto app_config::default_data_directory(const provider_type &prov)
+auto app_config::default_data_directory(const provider_type &pt)
     -> std::string {
 #ifdef _WIN32
   auto data_directory = utils::path::combine(
       utils::get_local_app_data_directory(),
-      {REPERTORY_DATA_NAME, app_config::get_provider_name(prov)});
+      {REPERTORY_DATA_NAME, app_config::get_provider_name(pt)});
 #else
 #ifdef __APPLE__
   auto data_directory = utils::path::resolve(
       std::string("~/Library/Application Support/") + REPERTORY_DATA_NAME +
-      '/' + app_config::get_provider_name(prov));
+      '/' + app_config::get_provider_name(pt));
 #else
   auto data_directory =
       utils::path::resolve(std::string("~/.local/") + REPERTORY_DATA_NAME +
-                           '/' + app_config::get_provider_name(prov));
+                           '/' + app_config::get_provider_name(pt));
 #endif
 #endif
   return data_directory;
 }
 
-auto app_config::default_remote_port(const provider_type &prov)
-    -> std::uint16_t {
-  static const std::array<std::uint16_t,
-                          static_cast<std::size_t>(provider_type::unknown)>
-      PROVIDER_REMOTE_PORTS = {
-          20000U,
-          20010U,
-          20100U,
-          20001U,
-      };
-  return PROVIDER_REMOTE_PORTS.at(static_cast<std::size_t>(prov));
-}
-auto app_config::default_rpc_port(const provider_type &prov) -> std::uint16_t {
+auto app_config::default_rpc_port(const provider_type &pt) -> std::uint16_t {
   static const std::array<std::uint16_t,
                           static_cast<std::size_t>(provider_type::unknown)>
       PROVIDER_RPC_PORTS = {
-          10000U,
-          10010U,
-          10100U,
-          10002U,
+          11101u, 11102u, 11103u, 11104u, 11105u,
       };
-  return PROVIDER_RPC_PORTS.at(static_cast<std::size_t>(prov));
+  return PROVIDER_RPC_PORTS[static_cast<std::size_t>(pt)];
 }
 
 auto app_config::get_json() const -> json {
@@ -255,12 +209,11 @@ auto app_config::get_json() const -> json {
            {"SecretKey", s3_config_.secret_key},
            {"TimeoutMs", s3_config_.timeout_ms},
            {"URL", s3_config_.url},
-           {"UsePathStyle", s3_config_.use_path_style},
            {"UseRegionInURL", s3_config_.use_region_in_url},
        }},
       {"Version", version_}};
 
-  if (prov_ == provider_type::encrypt) {
+  if (pt_ == provider_type::encrypt) {
     ret.erase("ChunkDownloaderTimeoutSeconds");
     ret.erase("EnableChunkDownloaderTimeout");
     ret.erase("EnableMaxCacheSize");
@@ -276,13 +229,13 @@ auto app_config::get_json() const -> json {
     ret.erase("RetryReadCount");
     ret.erase("RingBufferFileSize");
     ret.erase("S3Config");
-  } else if (prov_ == provider_type::s3) {
+  } else if (pt_ == provider_type::s3) {
     ret.erase("EncryptConfig");
     ret.erase("HostConfig");
-  } else if (prov_ == provider_type::sia) {
+  } else if (pt_ == provider_type::sia) {
     ret.erase("EncryptConfig");
     ret.erase("S3Config");
-  } else if (prov_ == provider_type::remote) {
+  } else if (pt_ == provider_type::remote) {
     ret.erase("ChunkDownloaderTimeoutSeconds");
     ret.erase("EnableChunkDownloaderTimeout");
     ret.erase("EnableChunkDownloaderTimeout");
@@ -308,56 +261,50 @@ auto app_config::get_json() const -> json {
 
 auto app_config::get_max_cache_size_bytes() const -> std::uint64_t {
   const auto max_space =
-      std::max(static_cast<std::uint64_t>(100ULL * 1024ULL * 1024ULL),
+      std::max(static_cast<std::uint64_t>(100ull * 1024ull * 1024ull),
                max_cache_size_bytes_);
   return std::min(utils::file::get_free_drive_space(get_cache_directory()),
                   max_space);
 }
 
-auto app_config::get_provider_api_password(const provider_type &prov)
+auto app_config::get_provider_api_password(const provider_type &pt)
     -> std::string {
 #ifdef _WIN32
   auto api_file =
       utils::path::combine(utils::get_local_app_data_directory(),
-                           {get_provider_display_name(prov), "apipassword"});
+                           {get_provider_display_name(pt), "apipassword"});
 #else
 #ifdef __APPLE__
   auto api_file =
       utils::path::combine(utils::path::resolve("~"),
                            {"/Library/Application Support",
-                            get_provider_display_name(prov), "apipassword"});
+                            get_provider_display_name(pt), "apipassword"});
 #else
-  auto api_file = utils::path::combine(
-      utils::path::resolve("~/."), {get_provider_name(prov), "apipassword"});
+  auto api_file = utils::path::combine(utils::path::resolve("~/."),
+                                       {get_provider_name(pt), "apipassword"});
 #endif
 #endif
   auto lines = utils::file::read_file_lines(api_file);
-  return lines.empty() ? "" : utils::string::trim(lines[0U]);
+  return lines.empty() ? "" : utils::string::trim(lines[0]);
 }
 
-auto app_config::get_provider_display_name(const provider_type &prov)
+auto app_config::get_provider_display_name(const provider_type &pt)
     -> std::string {
   static const std::array<std::string,
                           static_cast<std::size_t>(provider_type::unknown)>
       PROVIDER_DISPLAY_NAMES = {
-          "Sia",
-          "Remote",
-          "S3",
-          "Encrypt",
+          "Sia", "Remote", "S3", "Passthrough", "Encrypt",
       };
-  return PROVIDER_DISPLAY_NAMES.at(static_cast<std::size_t>(prov));
+  return PROVIDER_DISPLAY_NAMES[static_cast<std::size_t>(pt)];
 }
 
-auto app_config::get_provider_name(const provider_type &prov) -> std::string {
+auto app_config::get_provider_name(const provider_type &pt) -> std::string {
   static const std::array<std::string,
                           static_cast<std::size_t>(provider_type::unknown)>
       PROVIDER_NAMES = {
-          "sia",
-          "remote",
-          "s3",
-          "encrypt",
+          "sia", "remote", "s3", "passthrough", "encrypt",
       };
-  return PROVIDER_NAMES.at(static_cast<std::size_t>(prov));
+  return PROVIDER_NAMES[static_cast<std::size_t>(pt)];
 }
 
 auto app_config::get_value_by_name(const std::string &name) -> std::string {
@@ -375,16 +322,16 @@ auto app_config::get_value_by_name(const std::string &name) -> std::string {
       return std::to_string(get_chunk_downloader_timeout_secs());
     }
     if (name == "EnableChunkDownloaderTimeout") {
-      return utils::string::from_bool(get_enable_chunk_download_timeout());
+      return std::to_string(get_enable_chunk_download_timeout());
     }
     if (name == "GetEnableCommDurationEvents") {
-      return utils::string::from_bool(get_enable_comm_duration_events());
+      return std::to_string(get_enable_comm_duration_events());
     }
     if (name == "EnableDriveEvents") {
-      return utils::string::from_bool(get_enable_drive_events());
+      return std::to_string(get_enable_drive_events());
     }
     if (name == "EnableMaxCacheSize") {
-      return utils::string::from_bool(get_enable_max_cache_size());
+      return std::to_string(get_enable_max_cache_size());
 #ifdef _WIN32
     }
     if (name == "EnableMountManager") {
@@ -404,7 +351,7 @@ auto app_config::get_value_by_name(const std::string &name) -> std::string {
       return std::to_string(get_eviction_delay_mins());
     }
     if (name == "EvictionUsesAccessedTime") {
-      return utils::string::from_bool(get_eviction_uses_accessed_time());
+      return std::to_string(get_eviction_uses_accessed_time());
     }
     if (name == "HighFreqIntervalSeconds") {
       return std::to_string(get_high_frequency_interval_secs());
@@ -447,10 +394,10 @@ auto app_config::get_value_by_name(const std::string &name) -> std::string {
       return std::to_string(get_read_ahead_count());
     }
     if (name == "RemoteMount.EnableRemoteMount") {
-      return utils::string::from_bool(get_enable_remote_mount());
+      return std::to_string(get_enable_remote_mount());
     }
     if (name == "RemoteMount.IsRemoteMount") {
-      return utils::string::from_bool(get_is_remote_mount());
+      return std::to_string(get_is_remote_mount());
     }
     if (name == "RemoteMount.RemoteClientPoolSize") {
       return std::to_string(get_remote_client_pool_size());
@@ -500,11 +447,8 @@ auto app_config::get_value_by_name(const std::string &name) -> std::string {
     if (name == "S3Config.URL") {
       return s3_config_.url;
     }
-    if (name == "S3Config.UsePathStyle") {
-      return utils::string::from_bool(s3_config_.use_path_style);
-    }
     if (name == "S3Config.UseRegionInURL") {
-      return utils::string::from_bool(s3_config_.use_region_in_url);
+      return std::to_string(s3_config_.use_region_in_url);
     }
     if (name == "S3Config.TimeoutMs") {
       return std::to_string(s3_config_.timeout_ms);
@@ -519,17 +463,16 @@ auto app_config::load() -> bool {
   auto ret = false;
 
   const auto config_file_path = get_config_file_path();
-  recur_mutex_lock lock(read_write_mutex_);
+  recur_mutex_lock l(read_write_mutex_);
   if (utils::file::is_file(config_file_path)) {
     try {
-      std::ifstream config_file(config_file_path.data());
+      std::ifstream config_file(&config_file_path[0]);
       if (config_file.is_open()) {
-        std::stringstream stream;
-        stream << config_file.rdbuf();
-        const auto json_text = stream.str();
+        std::stringstream ss;
+        ss << config_file.rdbuf();
+        const auto json_text = ss.str();
         config_file.close();
-        ret = not json_text.empty();
-        if (ret) {
+        if ((ret = not json_text.empty())) {
           const auto json_document = json::parse(json_text);
 
           get_value(json_document, "ApiAuth", api_auth_, ret);
@@ -567,20 +510,20 @@ auto app_config::load() -> bool {
 
           if (json_document.find("HostConfig") != json_document.end()) {
             auto host_config_json = json_document["HostConfig"];
-            auto cfg = hc_;
-            get_value(host_config_json, "AgentString", cfg.agent_string, ret);
-            get_value(host_config_json, "ApiPassword", cfg.api_password, ret);
-            get_value(host_config_json, "ApiPort", cfg.api_port, ret);
-            get_value(host_config_json, "HostNameOrIp", cfg.host_name_or_ip,
+            auto hc = hc_;
+            get_value(host_config_json, "AgentString", hc.agent_string, ret);
+            get_value(host_config_json, "ApiPassword", hc.api_password, ret);
+            get_value(host_config_json, "ApiPort", hc.api_port, ret);
+            get_value(host_config_json, "HostNameOrIp", hc.host_name_or_ip,
                       ret);
-            get_value(host_config_json, "TimeoutMs", cfg.timeout_ms, ret);
-            hc_ = cfg;
+            get_value(host_config_json, "TimeoutMs", hc.timeout_ms, ret);
+            hc_ = hc;
           } else {
             ret = false;
           }
 
           if (hc_.api_password.empty()) {
-            hc_.api_password = get_provider_api_password(prov_);
+            hc_.api_password = get_provider_api_password(pt_);
             if (hc_.api_password.empty()) {
               ret = false;
             }
@@ -588,22 +531,20 @@ auto app_config::load() -> bool {
 
           if (json_document.find("S3Config") != json_document.end()) {
             auto s3_config_json = json_document["S3Config"];
-            auto s3_cfg = s3_config_;
-            get_value(s3_config_json, "AccessKey", s3_cfg.access_key, ret);
-            get_value(s3_config_json, "Bucket", s3_cfg.bucket, ret);
+            auto s3 = s3_config_;
+            get_value(s3_config_json, "AccessKey", s3.access_key, ret);
+            get_value(s3_config_json, "Bucket", s3.bucket, ret);
             get_value(s3_config_json, "CacheTimeoutSeconds",
-                      s3_cfg.cache_timeout_secs, ret);
-            get_value(s3_config_json, "EncryptionToken",
-                      s3_cfg.encryption_token, ret);
-            get_value(s3_config_json, "Region", s3_cfg.region, ret);
-            get_value(s3_config_json, "SecretKey", s3_cfg.secret_key, ret);
-            get_value(s3_config_json, "TimeoutMs", s3_cfg.timeout_ms, ret);
-            get_value(s3_config_json, "URL", s3_cfg.url, ret);
-            get_value(s3_config_json, "UsePathStyle", s3_cfg.use_path_style,
+                      s3.cache_timeout_secs, ret);
+            get_value(s3_config_json, "EncryptionToken", s3.encryption_token,
                       ret);
-            get_value(s3_config_json, "UseRegionInURL",
-                      s3_cfg.use_region_in_url, ret);
-            s3_config_ = s3_cfg;
+            get_value(s3_config_json, "Region", s3.region, ret);
+            get_value(s3_config_json, "SecretKey", s3.secret_key, ret);
+            get_value(s3_config_json, "TimeoutMs", s3.timeout_ms, ret);
+            get_value(s3_config_json, "URL", s3.url, ret);
+            get_value(s3_config_json, "UseRegionInURL", s3.use_region_in_url,
+                      ret);
+            s3_config_ = s3;
           } else {
             ret = false;
           }
@@ -652,13 +593,13 @@ auto app_config::load() -> bool {
             ret = false;
           }
 
-          std::uint64_t version{};
+          std::uint64_t version = 0u;
           get_value(json_document, "Version", version, ret);
 
           // Handle configuration defaults for new config versions
           if (version != REPERTORY_CONFIG_VERSION) {
             if (version > REPERTORY_CONFIG_VERSION) {
-              version = 0U;
+              version = 0u;
             }
 
             version_ = version;
@@ -680,9 +621,9 @@ auto app_config::load() -> bool {
 }
 
 void app_config::save() {
-  const auto file_path = get_config_file_path();
-  recur_mutex_lock lock(read_write_mutex_);
-  if (config_changed_ || not utils::file::is_file(file_path)) {
+  const auto configFilePath = get_config_file_path();
+  recur_mutex_lock l(read_write_mutex_);
+  if (config_changed_ || not utils::file::is_file(configFilePath)) {
     if (not utils::file::is_directory(data_directory_)) {
       if (not utils::file::create_full_directory_path(data_directory_)) {
         utils::error::raise_error(
@@ -694,9 +635,8 @@ void app_config::save() {
     config_changed_ = false;
     json data = get_json();
     auto success = false;
-    for (auto i = 0U; not success && (i < retry_save_count); i++) {
-      success = utils::file::write_json_file(file_path, data);
-      if (not success) {
+    for (auto i = 0; not success && (i < 5); i++) {
+      if (not(success = utils::file::write_json_file(configFilePath, data))) {
         std::this_thread::sleep_for(1s);
       }
     }
@@ -744,19 +684,19 @@ auto app_config::set_value_by_name(const std::string &name,
     }
     if (name == "EnableChunkDownloaderTimeout") {
       set_enable_chunk_downloader_timeout(utils::string::to_bool(value));
-      return utils::string::from_bool(get_enable_chunk_download_timeout());
+      return std::to_string(get_enable_chunk_download_timeout());
     }
     if (name == "EnableCommDurationEvents") {
       set_enable_comm_duration_events(utils::string::to_bool(value));
-      return utils::string::from_bool(get_enable_comm_duration_events());
+      return std::to_string(get_enable_comm_duration_events());
     }
     if (name == "EnableDriveEvents") {
       set_enable_drive_events(utils::string::to_bool(value));
-      return utils::string::from_bool(get_enable_drive_events());
+      return std::to_string(get_enable_drive_events());
     }
     if (name == "EnableMaxCacheSize") {
       set_enable_max_cache_size(utils::string::to_bool(value));
-      return utils::string::from_bool(get_enable_max_cache_size());
+      return std::to_string(get_enable_max_cache_size());
 #ifdef _WIN32
     }
     if (name == "EnableMountManager") {
@@ -782,7 +722,7 @@ auto app_config::set_value_by_name(const std::string &name,
     }
     if (name == "EvictionUsesAccessedTime") {
       set_eviction_uses_accessed_time(utils::string::to_bool(value));
-      return utils::string::from_bool(get_eviction_uses_accessed_time());
+      return std::to_string(get_eviction_uses_accessed_time());
     }
     if (name == "HighFreqIntervalSeconds") {
       set_high_frequency_interval_secs(utils::string::to_uint8(value));
@@ -839,11 +779,11 @@ auto app_config::set_value_by_name(const std::string &name,
     }
     if (name == "RemoteMount.EnableRemoteMount") {
       set_enable_remote_mount(utils::string::to_bool(value));
-      return utils::string::from_bool(get_enable_remote_mount());
+      return std::to_string(get_enable_remote_mount());
     }
     if (name == "RemoteMount.IsRemoteMount") {
       set_is_remote_mount(utils::string::to_bool(value));
-      return utils::string::from_bool(get_is_remote_mount());
+      return std::to_string(get_is_remote_mount());
     }
     if (name == "RemoteMount.RemoteClientPoolSize") {
       set_remote_client_pool_size(utils::string::to_uint8(value));
@@ -891,7 +831,7 @@ auto app_config::set_value_by_name(const std::string &name,
     }
     if (name == "S3Config.CacheTimeoutSeconds") {
       const auto timeout =
-          std::max(std::uint16_t(5U), utils::string::to_uint16(value));
+          std::max(std::uint16_t(5u), utils::string::to_uint16(value));
       set_value(s3_config_.cache_timeout_secs, timeout);
       return std::to_string(s3_config_.cache_timeout_secs);
     }
@@ -907,13 +847,9 @@ auto app_config::set_value_by_name(const std::string &name,
       set_value(s3_config_.url, value);
       return s3_config_.url;
     }
-    if (name == "S3Config.UsePathStyle") {
-      set_value(s3_config_.use_path_style, utils::string::to_bool(value));
-      return utils::string::from_bool(s3_config_.use_path_style);
-    }
     if (name == "S3Config.UseRegionInURL") {
       set_value(s3_config_.use_region_in_url, utils::string::to_bool(value));
-      return utils::string::from_bool(s3_config_.use_region_in_url);
+      return std::to_string(s3_config_.use_region_in_url);
     }
     if (name == "S3Config.TimeoutMs") {
       set_value(s3_config_.timeout_ms, utils::string::to_uint32(value));

src/comm/curl/curl_comm.cpp

@@ -154,16 +154,6 @@ auto curl_comm::make_request(const curl::requests::http_head &head,
       head, response_code, stop_requested);
 }
 
-auto curl_comm::make_request(const curl::requests::http_post &post,
-                             long &response_code,
-                             stop_type &stop_requested) const -> bool {
-  return make_request(
-      s3_config_.has_value()
-          ? create_host_config(s3_config_.value(), use_s3_path_style_)
-          : host_config_.value_or(host_config{}),
-      post, response_code, stop_requested);
-}
-
 auto curl_comm::make_request(const curl::requests::http_put_file &put_file,
                              long &response_code,
                              stop_type &stop_requested) const -> bool {

src/comm/curl/requests/http_put_file.cpp

@@ -24,46 +24,57 @@
 #include "utils/string_utils.hpp"
 
 namespace repertory::curl::requests {
+auto http_put_file::get_path() const -> std::string {
+  if (reader) {
+    auto updated_path = path;
+    return utils::string::replace(updated_path, file_name,
+                                  reader->get_encrypted_file_name());
+  }
+
+  return http_request_base::get_path();
+}
+
 auto http_put_file::set_method(CURL *curl, stop_type &stop_requested) const
     -> bool {
   curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "PUT");
-  curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L);
 
-  if (source_path.empty()) {
-    curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, 0L);
-    return true;
+  if (not source_path.empty()) {
+    curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L);
+    if (not encryption_token.value_or("").empty()) {
+      if (not reader) {
+        reader = std::make_shared<utils::encryption::encrypting_reader>(
+            file_name, source_path, stop_requested, encryption_token.value());
+      }
+      curl_easy_setopt(curl, CURLOPT_READDATA, reader.get());
+      curl_easy_setopt(
+          curl, CURLOPT_READFUNCTION,
+          static_cast<curl_read_callback>(
+              utils::encryption::encrypting_reader::reader_function));
+      curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE,
+                       reader->get_total_size());
+      return true;
+    }
+
+    read_info = std::make_shared<read_file_info>(read_file_info{
+        stop_requested,
+    });
+
+    if (native_file::open(source_path, read_info->nf) != api_error::success) {
+      return false;
+    }
+
+    read_info->nf->set_auto_close(true);
+
+    std::uint64_t file_size{};
+    if (not read_info->nf->get_file_size(file_size)) {
+      return false;
+    }
+
+    curl_easy_setopt(curl, CURLOPT_READDATA, read_info.get());
+    curl_easy_setopt(curl, CURLOPT_READFUNCTION, read_file_data);
+    curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, file_size);
   }
 
-  if (reader) {
-    curl_easy_setopt(curl, CURLOPT_READDATA, reader.get());
-    curl_easy_setopt(
-        curl, CURLOPT_READFUNCTION,
-        static_cast<curl_read_callback>(
-            utils::encryption::encrypting_reader::reader_function));
-    curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, reader->get_total_size());
-    return true;
-  }
-
-  read_info = std::make_shared<read_file_info>(read_file_info{
-      stop_requested,
-  });
-
-  if (native_file::create_or_open(source_path, read_info->nf) !=
-      api_error::success) {
-    return false;
-  }
-
-  read_info->nf->set_auto_close(true);
-
-  std::uint64_t file_size{};
-  if (not read_info->nf->get_file_size(file_size)) {
-    return false;
-  }
-
-  curl_easy_setopt(curl, CURLOPT_READDATA, read_info.get());
-  curl_easy_setopt(curl, CURLOPT_READFUNCTION, read_file_data);
-  curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, file_size);
-
   return true;
 }
 } // namespace repertory::curl::requests

src/comm/packet/client_pool.cpp

@@ -148,8 +148,8 @@ void client_pool::shutdown() {
     unique_mutex_lock pool_lock(pool_mutex_);
     if (not shutdown_) {
       shutdown_ = true;
-      for (auto &pool_entry : pool_lookup_) {
-        pool_entry.second->shutdown();
+      for (auto &pool : pool_lookup_) {
+        pool.second->shutdown();
       }
       pool_lookup_.clear();
     }

src/comm/s3/s3_comm.cpp

@@ -0,0 +1,577 @@
+/*
+  Copyright <2018-2023> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all
+  copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  SOFTWARE.
+*/
+#if defined(REPERTORY_ENABLE_S3)
+
+#include "comm/s3/s3_comm.hpp"
+
+#include "app_config.hpp"
+#include "comm/curl/curl_comm.hpp"
+#include "comm/s3/s3_requests.hpp"
+#include "events/event_system.hpp"
+#include "events/events.hpp"
+#include "providers/i_provider.hpp"
+#include "types/repertory.hpp"
+#include "types/s3.hpp"
+#include "utils/encryption.hpp"
+#include "utils/error_utils.hpp"
+#include "utils/path_utils.hpp"
+#include "utils/polling.hpp"
+#include "utils/string_utils.hpp"
+
+namespace repertory {
+static const get_key_callback empty_key = []() { return ""; };
+
+s3_comm::s3_comm(const app_config &config)
+    : config_(config), s3_config_(config.get_s3_config()) {
+  s3_config_.bucket = utils::string::trim(s3_config_.bucket);
+
+  // TODO make configurable
+  const auto enable_path_style =
+      utils::string::begins_with(s3_config_.url,
+                                 "https://gateway.storjshare.io") ||
+      utils::string::begins_with(s3_config_.url, "http://localhost") ||
+      utils::string::begins_with(s3_config_.url, "https://localhost") ||
+      utils::string::begins_with(s3_config_.url, "http://127.0.0.1") ||
+      utils::string::begins_with(s3_config_.url, "https://127.0.0.1");
+
+  s3_client_ = std::make_unique<curl_comm>(s3_config_);
+  s3_client_->enable_s3_path_style(enable_path_style);
+
+  polling::instance().set_callback(
+      {"s3_directory_cache", polling::frequency::high,
+       [this]() { this->clear_expired_directories(); }});
+}
+
+s3_comm::s3_comm(s3_comm &&comm)
+    : config_(std::move(comm.config_)),
+      s3_config_(std::move(comm.s3_config_)),
+      s3_client_(std::move(comm.s3_client_)) {
+  comm.active_ = false;
+
+  polling::instance().set_callback(
+      {"s3_directory_cache", polling::frequency::high,
+       [this]() { this->clear_expired_directories(); }});
+}
+
+s3_comm::~s3_comm() {
+  if (active_) {
+    polling::instance().remove_callback("s3_directory_cache");
+  }
+}
+
+void s3_comm::clear_expired_directories() {
+  recur_mutex_lock l(cached_directories_mutex_);
+  std::vector<std::string> expired_list;
+  for (const auto &kv : cached_directories_) {
+    if (kv.second.expiration <= std::chrono::system_clock::now()) {
+      expired_list.emplace_back(kv.first);
+    }
+  }
+
+  for (const auto &expired : expired_list) {
+    event_system::instance().raise<debug_log>(__FUNCTION__, expired, "expired");
+    cached_directories_.erase(expired);
+  }
+}
+
+auto s3_comm::create_directory(const std::string &api_path) -> api_error {
+  raise_begin(__FUNCTION__, api_path);
+
+  long response_code{};
+
+  auto object_name = get_object_name(api_path, empty_key) + '/';
+  if (not create_directory_object_request(*s3_client_, s3_config_, object_name,
+                                          response_code)) {
+    return raise_end(__FUNCTION__, api_path, api_error::comm_error,
+                     response_code);
+  }
+
+  return raise_end(__FUNCTION__, api_path,
+                   response_code == 200 ? api_error::success
+                                        : api_error::comm_error,
+                   response_code);
+}
+
+auto s3_comm::directory_exists(const std::string &api_path) const -> api_error {
+  raise_begin(__FUNCTION__, api_path);
+
+  auto object_name = get_object_name(api_path, empty_key) + "/";
+
+  head_object_result result{};
+  long response_code{};
+  if (head_object_request(*s3_client_, s3_config_, object_name, result,
+                          response_code)) {
+    if (response_code == 404) {
+      return raise_end(__FUNCTION__, api_path, api_error::directory_not_found,
+                       response_code);
+    }
+
+    return raise_end(__FUNCTION__, api_path, api_error::directory_exists,
+                     response_code);
+  }
+
+  return raise_end(__FUNCTION__, api_path, api_error::comm_error,
+                   response_code);
+}
+
+auto s3_comm::file_exists(const std::string &api_path,
+                          const get_key_callback &get_key) const -> api_error {
+  raise_begin(__FUNCTION__, api_path);
+
+  if (get_cached_file_exists(api_path)) {
+    return raise_end(__FUNCTION__, api_path, api_error::item_exists, 200);
+  }
+
+  auto object_name = get_object_name(api_path, get_key);
+
+  head_object_result result{};
+  long response_code{};
+  if (head_object_request(*s3_client_, s3_config_, object_name, result,
+                          response_code)) {
+    if (response_code == 404) {
+      return raise_end(__FUNCTION__, api_path, api_error::item_not_found,
+                       response_code);
+    }
+
+    return raise_end(__FUNCTION__, api_path, api_error::item_exists,
+                     response_code);
+  }
+
+  return raise_end(__FUNCTION__, api_path, api_error::directory_exists,
+                   response_code);
+}
+
+auto s3_comm::get_object_list(std::vector<directory_item> &list) const
+    -> api_error {
+  raise_begin(__FUNCTION__, "/");
+
+  long response_code{};
+  auto success =
+      list_objects_request(*s3_client_, s3_config_, list, response_code);
+  return raise_end(__FUNCTION__, "/",
+                   success ? api_error::success : api_error::comm_error,
+                   response_code);
+}
+
+auto s3_comm::get_object_name(const std::string &api_path,
+                              const get_key_callback &get_key) const
+    -> std::string {
+  auto object_name = utils::path::create_api_path(api_path).substr(1);
+
+  const auto key = get_key();
+  if (not key.empty()) {
+    auto parts = utils::string::split(object_name, '/', false);
+    parts[parts.size() - 1u] = key;
+    object_name = utils::string::join(parts, '/');
+  }
+
+  return object_name;
+}
+
+auto s3_comm::get_cached_directory_item_count(const std::string &api_path,
+                                              std::size_t &count) const
+    -> bool {
+  recur_mutex_lock l(cached_directories_mutex_);
+  if (cached_directories_.find(api_path) != cached_directories_.end()) {
+    count = cached_directories_.at(api_path).items.size();
+    return true;
+  }
+
+  return false;
+}
+
+auto s3_comm::get_cached_directory_items(const std::string &api_path,
+                                         meta_provider_callback meta_provider,
+                                         directory_item_list &list) const
+    -> bool {
+  unique_recur_mutex_lock l(cached_directories_mutex_);
+  if (cached_directories_.find(api_path) != cached_directories_.end()) {
+    auto &cachedEntry = cached_directories_.at(api_path);
+    list = cachedEntry.items;
+    cached_directories_[api_path].reset_timeout(
+        std::chrono::seconds(config_.get_s3_config().cache_timeout_secs));
+    l.unlock();
+
+    for (auto &item : list) {
+      meta_provider(item);
+    }
+    return true;
+  }
+  return false;
+}
+
+auto s3_comm::get_cached_file_exists(const std::string &api_path) const
+    -> bool {
+  unique_recur_mutex_lock l(cached_directories_mutex_);
+  const auto parent_api_path = utils::path::get_parent_api_path(api_path);
+  if (cached_directories_.find(parent_api_path) != cached_directories_.end()) {
+    auto &entry = cached_directories_.at(parent_api_path);
+    if (std::find_if(entry.items.begin(), entry.items.end(),
+                     [&api_path](const auto &item) -> bool {
+                       return not item.directory && (api_path == item.api_path);
+                     }) != entry.items.end()) {
+      cached_directories_[api_path].reset_timeout(
+          std::chrono::seconds(config_.get_s3_config().cache_timeout_secs));
+      return true;
+    }
+  }
+
+  return false;
+}
+
+auto s3_comm::get_directory_item_count(
+    const std::string &api_path, meta_provider_callback meta_provider) const
+    -> std::size_t {
+  raise_begin(__FUNCTION__, api_path);
+
+  std::size_t ret = 0u;
+  if (not get_cached_directory_item_count(api_path, ret)) {
+    directory_item_list list;
+    const auto res = grab_directory_items(api_path, meta_provider, list);
+    if (res != api_error::success) {
+      utils::error::raise_api_path_error(__FUNCTION__, api_path, res,
+                                         "failed to grab directory items");
+    }
+    return list.size();
+  }
+
+  if (config_.get_event_level() >= event_level::debug) {
+    event_system::instance().raise<debug_log>(__FUNCTION__, api_path,
+                                              "end|" + std::to_string(ret));
+  }
+
+  return ret;
+}
+
+auto s3_comm::get_directory_items(const std::string &api_path,
+                                  meta_provider_callback meta_provider,
+                                  directory_item_list &list) const
+    -> api_error {
+  raise_begin(__FUNCTION__, api_path);
+
+  auto ret = api_error::success;
+  if (not get_cached_directory_items(api_path, meta_provider, list)) {
+    ret = grab_directory_items(api_path, meta_provider, list);
+  }
+
+  if (config_.get_event_level() >= event_level::debug) {
+    event_system::instance().raise<debug_log>(
+        __FUNCTION__, api_path, "end|" + api_error_to_string(ret));
+  }
+  return ret;
+}
+
+auto s3_comm::get_directory_list(api_file_list &list) const -> api_error {
+  raise_begin(__FUNCTION__, "/");
+
+  long response_code{};
+  auto success =
+      list_directories_request(*s3_client_, s3_config_, list, response_code);
+  return raise_end(__FUNCTION__, "/",
+                   success ? api_error::success : api_error::comm_error,
+                   response_code);
+}
+
+auto s3_comm::get_file(const std::string &api_path,
+                       const get_key_callback &get_key,
+                       const get_name_callback &get_name,
+                       const get_token_callback &get_token,
+                       api_file &file) const -> api_error {
+  raise_begin(__FUNCTION__, api_path);
+
+  auto ret = api_error::success;
+
+  auto object_name = get_object_name(api_path, get_key);
+
+  head_object_result result{};
+  long response_code{};
+  if (head_object_request(*s3_client_, s3_config_, object_name, result,
+                          response_code)) {
+    const auto key = get_key();
+    object_name = get_name(key, object_name);
+    file.accessed_date = utils::get_file_time_now();
+    file.api_path = utils::path::create_api_path(object_name);
+    file.api_parent = utils::path::get_parent_api_path(file.api_path);
+    file.changed_date = utils::aws::format_time(result.last_modified);
+    file.creation_date = utils::aws::format_time(result.last_modified);
+    file.encryption_token = get_token();
+    file.file_size =
+        file.encryption_token.empty()
+            ? result.content_length
+            : utils::encryption::encrypting_reader::calculate_decrypted_size(
+                  result.content_length);
+    file.modified_date = utils::aws::format_time(result.last_modified);
+  } else {
+    utils::error::raise_api_path_error(__FUNCTION__, api_path, response_code,
+                                       "head object request failed");
+    ret = api_error::comm_error;
+  }
+
+  if (config_.get_event_level() >= event_level::debug) {
+    event_system::instance().raise<debug_log>(
+        __FUNCTION__, api_path, "end|" + api_error_to_string(ret));
+  }
+
+  return ret;
+}
+
+auto s3_comm::get_file_list(
+    const get_api_file_token_callback &get_api_file_token,
+    const get_name_callback &get_name, api_file_list &list) const -> api_error {
+  raise_begin(__FUNCTION__, "/");
+
+  long response_code{};
+  auto success = list_files_request(*s3_client_, s3_config_, get_api_file_token,
+                                    get_name, list, response_code);
+  return raise_end(__FUNCTION__, "/",
+                   success ? api_error::success : api_error::comm_error,
+                   response_code);
+}
+
+auto s3_comm::grab_directory_items(const std::string &api_path,
+                                   meta_provider_callback meta_provider,
+                                   directory_item_list &list) const
+    -> api_error {
+  auto object_name = get_object_name(api_path, empty_key);
+  long response_code{};
+  if (list_objects_in_directory_request(*s3_client_, s3_config_, object_name,
+                                        meta_provider, list, response_code)) {
+    if (response_code == 404) {
+      return api_error::directory_not_found;
+    }
+
+    if (response_code != 200) {
+      return api_error::comm_error;
+    }
+
+    set_cached_directory_items(api_path, list);
+    return api_error::success;
+  }
+
+  return api_error::comm_error;
+}
+
+void s3_comm::raise_begin(const std::string &function_name,
+                          const std::string &api_path) const {
+  if (config_.get_event_level() >= event_level::debug) {
+    event_system::instance().raise<debug_log>(function_name, api_path,
+                                              "begin|");
+  }
+}
+
+auto s3_comm::raise_end(const std::string &function_name,
+                        const std::string &api_path, const api_error &error,
+                        long code) const -> api_error {
+  if (config_.get_event_level() >= event_level::debug) {
+    event_system::instance().raise<debug_log>(
+        function_name, api_path,
+        "end|" + api_error_to_string(error) + '|' + std::to_string(code));
+  }
+
+  return error;
+}
+
+auto s3_comm::read_file_bytes(const std::string &api_path, std::size_t size,
+                              std::uint64_t offset, data_buffer &data,
+                              const get_key_callback &get_key,
+                              const get_size_callback &get_size,
+                              const get_token_callback &get_token,
+                              stop_type &stop_requested) const -> api_error {
+  data.clear();
+
+  auto object_name = get_object_name(api_path, get_key);
+
+  const auto encryption_token = get_token();
+  const auto data_size = get_size();
+  if (encryption_token.empty()) {
+    long response_code{};
+    if (not read_object_request(*s3_client_, s3_config_, object_name, size,
+                                offset, data, response_code, stop_requested)) {
+      auto res =
+          stop_requested ? api_error::download_stopped : api_error::comm_error;
+      utils::error::raise_api_path_error(__FUNCTION__, api_path, res,
+                                         "failed to read file bytes");
+      return res;
+    }
+
+    if (response_code < 200 || response_code >= 300) {
+      utils::error::raise_api_path_error(__FUNCTION__, api_path, response_code,
+                                         "failed to read file bytes");
+      return api_error::comm_error;
+    }
+
+    return api_error::success;
+  }
+
+  const auto key = utils::encryption::generate_key(encryption_token);
+  return utils::encryption::read_encrypted_range(
+      {offset, offset + size - 1}, key,
+      [&](data_buffer &ct, std::uint64_t start_offset,
+          std::uint64_t end_offset) -> api_error {
+        return read_file_bytes(
+            api_path, (end_offset - start_offset + 1u), start_offset, ct,
+            get_key, get_size, []() -> std::string { return ""; },
+            stop_requested);
+      },
+      data_size, data);
+}
+
+void s3_comm::remove_cached_directory(const std::string &api_path) {
+  recur_mutex_lock l(cached_directories_mutex_);
+  cached_directories_.erase(api_path);
+}
+
+auto s3_comm::remove_directory(const std::string &api_path) -> api_error {
+  raise_begin(__FUNCTION__, api_path);
+
+  auto object_name = get_object_name(api_path, empty_key) + "/";
+
+  long response_code{};
+  if (delete_object_request(*s3_client_, s3_config_, object_name,
+                            response_code)) {
+    if (response_code == 404) {
+      return raise_end(__FUNCTION__, api_path, api_error::directory_not_found,
+                       response_code);
+    }
+
+    if (response_code != 204) {
+      return raise_end(__FUNCTION__, api_path, api_error::comm_error,
+                       response_code);
+    }
+
+    remove_cached_directory(utils::path::get_parent_api_path(api_path));
+    remove_cached_directory(api_path);
+
+    return raise_end(__FUNCTION__, api_path, api_error::success, response_code);
+  }
+
+  return raise_end(__FUNCTION__, api_path, api_error::comm_error,
+                   response_code);
+}
+
+auto s3_comm::remove_file(const std::string &api_path,
+                          const get_key_callback &get_key) -> api_error {
+  raise_begin(__FUNCTION__, api_path);
+
+  auto object_name = get_object_name(api_path, get_key);
+
+  long response_code{};
+  if (delete_object_request(*s3_client_, s3_config_, object_name,
+                            response_code)) {
+    if (response_code == 404) {
+      return raise_end(__FUNCTION__, api_path, api_error::item_not_found,
+                       response_code);
+    }
+
+    if (response_code != 204) {
+      return raise_end(__FUNCTION__, api_path, api_error::comm_error,
+                       response_code);
+    }
+
+    remove_cached_directory(utils::path::get_parent_api_path(api_path));
+    return raise_end(__FUNCTION__, api_path, api_error::success, response_code);
+  }
+
+  return raise_end(__FUNCTION__, api_path, api_error::comm_error,
+                   response_code);
+}
+
+auto s3_comm::rename_file(const std::string & /*api_path*/,
+                          const std::string & /*new_api_path*/) -> api_error {
+  return api_error::not_implemented;
+  /*   if (config_.get_event_level() >= event_level::debug) { */
+  /*     event_system::instance().raise<debug_log>(__FUNCTION__, api_path,
+   * "begin"); */
+  /*   } */
+  /*   auto ret = api_error::success; */
+  /*  */
+  /*   std::string bucket_name, object_name; */
+  /*   get_object_name(api_path, bucket_name, object_name); */
+  /*  */
+  /*   std::string new_object_name; */
+  /*   get_object_name(new_api_path, bucket_name, new_object_name); */
+  /*  */
+  /*   Aws::S3::Model::CopyObjectRequest request{}; */
+  /*   request.SetBucket(bucket_name); */
+  /*   request.SetCopySource(bucket_name + '/' + object_name); */
+  /*   request.SetKey(new_object_name); */
+  /*  */
+  /*   const auto outcome = s3_client_->CopyObject(request); */
+  /*   if (outcome.IsSuccess()) { */
+  /*     ret = remove_file(api_path); */
+  /*   } else { */
+  /*     const auto &error = outcome.GetError(); */
+  /*     event_system::instance().raise<repertory_exception>(__FUNCTION__,
+   * error.GetExceptionName()
+   * +
+   * "|" + */
+  /*                                                                         error.GetMessage());
+   */
+  /*     ret = api_error::comm_error; */
+  /*   } */
+  /*  */
+  /*   if (config_.get_event_level() >= event_level::debug) { */
+  /*     event_system::instance().raise<debug_log>(__FUNCTION__, api_path, */
+  /*                                             "end|" +
+   * std::to_string(std::uint8_t(ret))); */
+  /*   } */
+  /*   return ret; */
+}
+
+void s3_comm::set_cached_directory_items(const std::string &api_path,
+                                         directory_item_list list) const {
+  recur_mutex_lock l(cached_directories_mutex_);
+  cached_directories_[api_path].items = std::move(list);
+  cached_directories_[api_path].reset_timeout(
+      std::chrono::seconds(config_.get_s3_config().cache_timeout_secs));
+}
+
+auto s3_comm::upload_file(const std::string &api_path,
+                          const std::string &source_path,
+                          const std::string &encryption_token,
+                          const get_key_callback &get_key,
+                          const set_key_callback &set_key,
+                          stop_type &stop_requested) -> api_error {
+  raise_begin(__FUNCTION__, api_path);
+
+  auto object_name = get_object_name(api_path, get_key);
+  long response_code{};
+  if (not put_object_request(*s3_client_, s3_config_, object_name, source_path,
+                             encryption_token, get_key, set_key, response_code,
+                             stop_requested)) {
+    return raise_end(__FUNCTION__, api_path,
+                     stop_requested ? api_error::upload_stopped
+                                    : api_error::upload_failed,
+                     response_code);
+  }
+
+  if (response_code != 200) {
+    return raise_end(__FUNCTION__, api_path, api_error::comm_error,
+                     response_code);
+  }
+
+  remove_cached_directory(utils::path::get_parent_api_path(api_path));
+  return raise_end(__FUNCTION__, api_path, api_error::success, response_code);
+}
+} // namespace repertory
+
+#endif // REPERTORY_ENABLE_S3

src/comm/s3/s3_requests_curl.cpp

@@ -0,0 +1,404 @@
+/*
+  Copyright <2018-2023> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all
+  copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  SOFTWARE.
+*/
+#if defined(REPERTORY_ENABLE_S3)
+
+#include "comm/s3/s3_requests.hpp"
+
+#include "comm/curl/curl_comm.hpp"
+#include "comm/curl/requests/http_get.hpp"
+#include "utils/encryption.hpp"
+#include "utils/error_utils.hpp"
+#include "utils/path_utils.hpp"
+
+namespace repertory {
+namespace {
+[[nodiscard]] auto
+get_object_list(i_http_comm &client, const s3_config &config,
+                std::string &response_data, long &response_code,
+                std::optional<std::string> delimiter = std::nullopt,
+                std::optional<std::string> prefix = std::nullopt) -> bool {
+  curl::requests::http_get get{};
+  get.allow_timeout = true;
+  get.aws_service = "aws:amz:" + config.region + ":s3";
+  get.path = '/';
+  get.query["list-type"] = "2";
+  if (delimiter.has_value() && not delimiter.value().empty()) {
+    get.query["delimiter"] = delimiter.value();
+  }
+  if (prefix.has_value() && not prefix.value().empty()) {
+    get.query["prefix"] = prefix.value();
+  }
+  get.response_handler = [&response_data](const data_buffer &data,
+                                          long /*response_code*/) {
+    response_data = std::string(data.begin(), data.end());
+  };
+
+  stop_type stop_requested{};
+  return client.make_request(get, response_code, stop_requested);
+}
+} // namespace
+
+auto create_directory_object_request_impl(i_http_comm &client,
+                                          const s3_config &config,
+                                          const std::string &object_name,
+                                          long &response_code) -> bool {
+  try {
+    curl::requests::http_put_file put_file{};
+    put_file.allow_timeout = true;
+    put_file.aws_service = "aws:amz:" + config.region + ":s3";
+    put_file.file_name =
+        *(utils::string::split(object_name, '/', false).end() - 1U);
+    put_file.path = '/' + object_name;
+
+    stop_type stop_requested{false};
+    return client.make_request(put_file, response_code, stop_requested);
+  } catch (const std::exception &e) {
+    utils::error::raise_error(__FUNCTION__, e, "exception occurred");
+  }
+
+  return false;
+}
+
+auto delete_object_request_impl(i_http_comm &client, const s3_config &config,
+                                const std::string &object_name,
+                                long &response_code) -> bool {
+  try {
+    head_object_result result{};
+    if (not head_object_request_impl(client, config, object_name, result,
+                                     response_code)) {
+      return false;
+    }
+
+    if (response_code == http_error_codes::not_found) {
+      return true;
+    }
+
+    curl::requests::http_delete del{};
+    del.allow_timeout = true;
+    del.aws_service = "aws:amz:" + config.region + ":s3";
+    del.path = '/' + object_name;
+
+    stop_type stop_requested{false};
+    return client.make_request(del, response_code, stop_requested);
+  } catch (const std::exception &e) {
+    utils::error::raise_error(__FUNCTION__, e, "exception occurred");
+  }
+
+  return false;
+}
+
+auto head_object_request_impl(i_http_comm &client, const s3_config &config,
+                              const std::string &object_name,
+                              head_object_result &result, long &response_code)
+    -> bool {
+  try {
+    curl::requests::http_head head{};
+    head.allow_timeout = true;
+    head.aws_service = "aws:amz:" + config.region + ":s3";
+    head.path = '/' + object_name;
+    head.response_headers = http_headers{};
+
+    stop_type stop_requested{false};
+    if (not client.make_request(head, response_code, stop_requested)) {
+      return false;
+    }
+
+    if (response_code == http_error_codes::ok) {
+      result.from_headers(head.response_headers.value());
+    }
+
+    return true;
+  } catch (const std::exception &e) {
+    utils::error::raise_error(__FUNCTION__, e, "exception occurred");
+  }
+
+  return false;
+}
+
+auto list_directories_request_impl(i_http_comm &client, const s3_config &config,
+                                   list_directories_result &result,
+                                   long &response_code) -> bool {
+  try {
+    std::string response_data{};
+    if (not get_object_list(client, config, response_data, response_code)) {
+      return false;
+    }
+
+    if (response_code != http_error_codes::ok) {
+      return false;
+    }
+
+    pugi::xml_document doc;
+    auto res = doc.load_string(response_data.c_str());
+    if (res.status != pugi::xml_parse_status::status_ok) {
+      return false;
+    }
+
+    auto node_list = doc.select_nodes("/ListBucketResult/Contents");
+    for (const auto &node : node_list) {
+      const auto *object_name =
+          node.node().select_node("Key").node().text().as_string();
+      if (utils::string::ends_with(object_name, "/")) {
+        api_file directory{};
+        directory.api_path = utils::path::create_api_path(object_name);
+        directory.api_parent =
+            utils::path::get_parent_api_path(directory.api_path);
+        directory.accessed_date = utils::get_file_time_now();
+        directory.changed_date = utils::convert_api_date(
+            node.node().select_node("LastModified").node().text().as_string());
+        directory.creation_date = directory.changed_date;
+        directory.modified_date = directory.changed_date;
+        result.emplace_back(std::move(directory));
+      }
+    }
+
+    return true;
+  } catch (const std::exception &e) {
+    utils::error::raise_error(__FUNCTION__, e, "exception occurred");
+  }
+
+  return false;
+}
+
+auto list_files_request_impl(
+    i_http_comm &client, const s3_config &config,
+    const get_api_file_token_callback &get_api_file_token,
+    const get_name_callback &get_name, list_files_result &result,
+    long &response_code) -> bool {
+  try {
+    std::string response_data{};
+    if (not get_object_list(client, config, response_data, response_code)) {
+      return false;
+    }
+
+    if (response_code != http_error_codes::ok) {
+      return false;
+    }
+
+    pugi::xml_document doc;
+    auto res = doc.load_string(response_data.c_str());
+    if (res.status != pugi::xml_parse_status::status_ok) {
+      return false;
+    }
+
+    auto node_list = doc.select_nodes("/ListBucketResult/Contents");
+    for (const auto &node : node_list) {
+      std::string object_name =
+          node.node().select_node("Key").node().text().as_string();
+      if (not utils::string::ends_with(object_name, "/")) {
+        api_file file{};
+        object_name = get_name(
+            *(utils::string::split(object_name, '/', false).end() - 1U),
+            object_name);
+        file.api_path = utils::path::create_api_path(object_name);
+        file.api_parent = utils::path::get_parent_api_path(file.api_path);
+        file.accessed_date = utils::get_file_time_now();
+        file.encryption_token = get_api_file_token(file.api_path);
+        auto size = node.node().select_node("Size").node().text().as_ullong();
+        file.file_size = file.encryption_token.empty()
+                             ? size
+                             : utils::encryption::encrypting_reader::
+                                   calculate_decrypted_size(size);
+        file.changed_date = utils::convert_api_date(
+            node.node().select_node("LastModified").node().text().as_string());
+        file.creation_date = file.changed_date;
+        file.modified_date = file.changed_date;
+        result.emplace_back(std::move(file));
+      }
+    }
+
+    return true;
+  } catch (const std::exception &e) {
+    utils::error::raise_error(__FUNCTION__, e, "exception occurred");
+  }
+
+  return false;
+}
+
+auto list_objects_in_directory_request_impl(
+    i_http_comm &client, const s3_config &config,
+    const std::string &object_name, meta_provider_callback meta_provider,
+    list_objects_result &result, long &response_code) -> bool {
+  try {
+    std::string response_data{};
+    auto prefix = object_name.empty() ? object_name : object_name + "/";
+    if (not get_object_list(client, config, response_data, response_code, "/",
+                            prefix)) {
+      return false;
+    }
+
+    if (response_code != http_error_codes::ok) {
+      return false;
+    }
+
+    pugi::xml_document doc;
+    auto res = doc.load_string(response_data.c_str());
+    if (res.status != pugi::xml_parse_status::status_ok) {
+      return false;
+    }
+
+    const auto add_directory_item =
+        [&](bool directory, const std::string &name,
+            std::function<std::uint64_t(const directory_item &)> get_size) {
+          directory_item dir_item{};
+          dir_item.api_path =
+              utils::path::create_api_path(utils::path::combine("/", {name}));
+          dir_item.api_parent =
+              utils::path::get_parent_api_path(dir_item.api_path);
+          dir_item.directory = directory;
+          dir_item.size = get_size(dir_item);
+          meta_provider(dir_item);
+          result.emplace_back(std::move(dir_item));
+        };
+
+    auto node_list =
+        doc.select_nodes("/ListBucketResult/CommonPrefixes/Prefix");
+    for (const auto &node : node_list) {
+      add_directory_item(
+          true, node.node().text().as_string(),
+          [](const directory_item &) -> std::uint64_t { return 0U; });
+    }
+
+    node_list = doc.select_nodes("/ListBucketResult/Contents");
+    for (const auto &node : node_list) {
+      const auto *child_object_name =
+          node.node().select_node("Key").node().text().as_string();
+      if (child_object_name != prefix) {
+        auto size = node.node().select_node("Size").node().text().as_ullong();
+        add_directory_item(
+            false, child_object_name,
+            [&size](const directory_item &) -> std::uint64_t { return size; });
+      }
+    }
+
+    return true;
+  } catch (const std::exception &e) {
+    utils::error::raise_error(__FUNCTION__, e, "exception occurred");
+  }
+
+  return false;
+}
+
+auto list_objects_request_impl(i_http_comm &client, const s3_config &config,
+                               list_objects_result &result, long &response_code)
+    -> bool {
+  try {
+    std::string response_data{};
+    if (not get_object_list(client, config, response_data, response_code)) {
+      return false;
+    }
+
+    if (response_code != http_error_codes::ok) {
+      return false;
+    }
+
+    pugi::xml_document doc;
+    auto res = doc.load_string(response_data.c_str());
+    if (res.status != pugi::xml_parse_status::status_ok) {
+      return false;
+    }
+
+    auto node_list = doc.select_nodes("/ListBucketResult/Contents");
+    for (const auto &node : node_list) {
+      const auto *object_name =
+          node.node().select_node("Key").node().text().as_string();
+      auto size = node.node().select_node("Size").node().text().as_ullong();
+      directory_item dir_item{};
+      dir_item.api_path = utils::path::create_api_path(object_name);
+      dir_item.api_parent = utils::path::get_parent_api_path(dir_item.api_path);
+      dir_item.directory = utils::string::ends_with(object_name, "/");
+      dir_item.size = dir_item.directory ? 0U : size;
+      dir_item.resolved = false;
+      result.emplace_back(std::move(dir_item));
+    }
+
+    return true;
+  } catch (const std::exception &e) {
+    utils::error::raise_error(__FUNCTION__, e, "exception occurred");
+  }
+
+  return false;
+}
+
+auto put_object_request_impl(i_http_comm &client, const s3_config &config,
+                             std::string object_name,
+                             const std::string &source_path,
+                             const std::string &encryption_token,
+                             get_key_callback get_key, set_key_callback set_key,
+                             long &response_code, stop_type &stop_requested)
+    -> bool {
+  try {
+    curl::requests::http_put_file put_file{};
+    put_file.aws_service = "aws:amz:" + config.region + ":s3";
+    put_file.encryption_token = encryption_token;
+    put_file.file_name =
+        *(utils::string::split(object_name, '/', false).end() - 1U);
+    put_file.path = '/' + object_name;
+    put_file.source_path = source_path;
+
+    if (not encryption_token.empty()) {
+      static stop_type no_stop{false};
+
+      put_file.reader = std::make_shared<utils::encryption::encrypting_reader>(
+          put_file.file_name, source_path, no_stop, encryption_token,
+          std::nullopt, -1);
+      auto key = get_key();
+      if (key.empty()) {
+        key = put_file.reader->get_encrypted_file_name();
+        set_key(key);
+      }
+    }
+
+    return client.make_request(put_file, response_code, stop_requested);
+  } catch (const std::exception &e) {
+    utils::error::raise_error(__FUNCTION__, e, "exception occurred");
+  }
+
+  return false;
+}
+
+auto read_object_request_impl(i_http_comm &client, const s3_config &config,
+                              const std::string &object_name, std::size_t size,
+                              std::uint64_t offset, data_buffer &data,
+                              long &response_code, stop_type &stop_requested)
+    -> bool {
+  try {
+    curl::requests::http_get get{};
+    get.aws_service = "aws:amz:" + config.region + ":s3";
+    get.headers["response-content-type"] = "binary/octet-stream";
+    get.path = '/' + object_name;
+    get.range = {{offset, offset + size - 1}};
+    get.response_handler = [&data](const data_buffer &response_data,
+                                   long /*response_code*/) {
+      data = response_data;
+    };
+
+    return client.make_request(get, response_code, stop_requested);
+  } catch (const std::exception &e) {
+    utils::error::raise_error(__FUNCTION__, e, "exception occurred");
+  }
+
+  return false;
+}
+} // namespace repertory
+
+#endif

src/common.cpp.in

@@ -49,11 +49,7 @@ void repertory_init() {
   }
 
   curl_global_init(CURL_GLOBAL_DEFAULT);
-  sqlite3_initialize();
 }
 
-void repertory_shutdown() {
-  curl_global_cleanup();
-  sqlite3_shutdown();
-}
+void repertory_shutdown() { curl_global_cleanup(); }
 } // namespace repertory

src/database/db_insert.cpp

@@ -1,96 +0,0 @@
-/*
-  Copyright <2018-2023> <scott.e.graves@protonmail.com>
-
-  Permission is hereby granted, free of charge, to any person obtaining a copy
-  of this software and associated documentation files (the "Software"), to deal
-  in the Software without restriction, including without limitation the rights
-  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-  copies of the Software, and to permit persons to whom the Software is
-  furnished to do so, subject to the following conditions:
-
-  The above copyright notice and this permission notice shall be included in all
-  copies or substantial portions of the Software.
-
-  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-  SOFTWARE.
-*/
-#include "database/db_insert.hpp"
-
-namespace repertory::db {
-auto db_insert::column_value(std::string column_name, db_types_t value)
-    -> db_insert & {
-  context_->values[column_name] = value;
-  return *this;
-}
-
-auto db_insert::dump() const -> std::string {
-  std::stringstream query;
-  query << "INSERT ";
-  if (context_->or_replace) {
-    query << "OR REPLACE ";
-  }
-  query << "INTO \"" << context_->table_name << "\" (";
-
-  for (std::int32_t idx = 0;
-       idx < static_cast<std::int32_t>(context_->values.size()); idx++) {
-    if (idx > 0) {
-      query << ", ";
-    }
-    query << '"' << std::next(context_->values.begin(), idx)->first << '"';
-  }
-
-  query << ") VALUES (";
-  for (std::int32_t idx = 0;
-       idx < static_cast<std::int32_t>(context_->values.size()); idx++) {
-    if (idx > 0) {
-      query << ", ";
-    }
-    query << "?" << (idx + 1);
-  }
-  query << ");";
-
-  return query.str();
-}
-
-auto db_insert::go() const -> db_result<context> {
-  sqlite3_stmt *stmt_ptr{nullptr};
-  auto query_str = dump();
-  auto res = sqlite3_prepare_v2(&context_->db3, query_str.c_str(), -1,
-                                &stmt_ptr, nullptr);
-  if (res != SQLITE_OK) {
-    utils::error::raise_error(__FUNCTION__, "failed to prepare|" +
-                                                std::to_string(res) + '|' +
-                                                sqlite3_errstr(res));
-    return {context_, res};
-  }
-  context_->stmt.reset(stmt_ptr);
-
-  for (std::int32_t idx = 0;
-       idx < static_cast<std::int32_t>(context_->values.size()); idx++) {
-    res = std::visit(
-        overloaded{
-            [this, &idx](std::int64_t data) -> std::int32_t {
-              return sqlite3_bind_int64(context_->stmt.get(), idx + 1, data);
-            },
-            [this, &idx](const std::string &data) -> std::int32_t {
-              return sqlite3_bind_text(context_->stmt.get(), idx + 1,
-                                       data.c_str(), -1, nullptr);
-            },
-        },
-        std::next(context_->values.begin(), idx)->second);
-    if (res != SQLITE_OK) {
-      utils::error::raise_error(__FUNCTION__, "failed to bind|" +
-                                                  std::to_string(res) + '|' +
-                                                  sqlite3_errstr(res));
-      return {context_, res};
-    }
-  }
-
-  return {context_, res};
-}
-} // namespace repertory::db

src/database/db_select.cpp

@@ -1,173 +0,0 @@
-/*
-  Copyright <2018-2023> <scott.e.graves@protonmail.com>
-
-  Permission is hereby granted, free of charge, to any person obtaining a copy
-  of this software and associated documentation files (the "Software"), to deal
-  in the Software without restriction, including without limitation the rights
-  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-  copies of the Software, and to permit persons to whom the Software is
-  furnished to do so, subject to the following conditions:
-
-  The above copyright notice and this permission notice shall be included in all
-  copies or substantial portions of the Software.
-
-  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-  SOFTWARE.
-*/
-#include "database/db_select.hpp"
-
-namespace repertory::db {
-
-auto db_select::column(std::string column_name) -> db_select & {
-  if (context_->delete_query) {
-    throw std::runtime_error("columns may not be specified for delete");
-  }
-
-  context_->columns.push_back(column_name);
-  return *this;
-}
-
-auto db_select::count(std::string column_name, std::string as_column_name)
-    -> db_select & {
-  context_->count_columns[column_name] = as_column_name;
-  return *this;
-}
-
-auto db_select::delete_query() -> db_select & {
-  if (not context_->columns.empty()) {
-    throw std::runtime_error("columns must be empty for delete");
-  }
-
-  context_->delete_query = true;
-  return *this;
-}
-
-auto db_select::dump() const -> std::string {
-  std::stringstream query;
-  query << (context_->delete_query ? "DELETE " : "SELECT ");
-  if (not context_->delete_query) {
-    bool has_column{false};
-    if (context_->columns.empty()) {
-      if (context_->count_columns.empty()) {
-        query << "*";
-        has_column = true;
-      }
-    } else {
-      has_column = not context_->columns.empty();
-      for (std::size_t idx = 0U; idx < context_->columns.size(); idx++) {
-        if (idx > 0U) {
-          query << ", ";
-        }
-        query << context_->columns.at(idx);
-      }
-    }
-
-    for (std::int32_t idx = 0U;
-         idx < static_cast<std::int32_t>(context_->count_columns.size());
-         idx++) {
-      if (has_column || idx > 0) {
-        query << ", ";
-      }
-      query << "COUNT(\"";
-      auto &count_column = *std::next(context_->count_columns.begin(), idx);
-      query << count_column.first << "\") AS \"" << count_column.second << '"';
-    }
-  }
-  query << " FROM \"" << context_->table_name << "\"";
-
-  if (not context_->ands.empty()) {
-    query << " WHERE (";
-    for (std::int32_t idx = 0;
-         idx < static_cast<std::int32_t>(context_->ands.size()); idx++) {
-      if (idx > 0) {
-        query << " AND ";
-      }
-
-      auto &item = context_->ands.at(static_cast<std::size_t>(idx));
-      query << '"' << item.column_name << '"' << item.op_type << "?"
-            << (idx + 1);
-    }
-    query << ")";
-  }
-
-  if (not context_->delete_query) {
-    if (context_->order_by.has_value()) {
-      query << " ORDER BY \"" << context_->order_by.value().first << "\" ";
-      query << (context_->order_by.value().second ? "ASC" : "DESC");
-    }
-
-    if (context_->limit.has_value()) {
-      query << " LIMIT " << context_->limit.value();
-    }
-  }
-
-  query << ';';
-
-  return query.str();
-}
-
-auto db_select::go() const -> db_result<context> {
-  sqlite3_stmt *stmt_ptr{nullptr};
-  auto query_str = dump();
-  auto res = sqlite3_prepare_v2(&context_->db3, query_str.c_str(), -1,
-                                &stmt_ptr, nullptr);
-  if (res != SQLITE_OK) {
-    utils::error::raise_error(__FUNCTION__,
-                              "failed to prepare|" + std::to_string(res) + '|' +
-                                  sqlite3_errstr(res) + '|' + query_str);
-    return {context_, res};
-  }
-  context_->stmt.reset(stmt_ptr);
-
-  for (std::int32_t idx = 0;
-       idx < static_cast<std::int32_t>(context_->ands.size()); idx++) {
-    res = std::visit(
-        overloaded{
-            [this, &idx](std::int64_t data) -> std::int32_t {
-              return sqlite3_bind_int64(context_->stmt.get(), idx + 1, data);
-            },
-            [this, &idx](const std::string &data) -> std::int32_t {
-              return sqlite3_bind_text(context_->stmt.get(), idx + 1,
-                                       data.c_str(), -1, nullptr);
-            },
-        },
-        context_->ands.at(static_cast<std::size_t>(idx)).value);
-    if (res != SQLITE_OK) {
-      utils::error::raise_error(__FUNCTION__,
-                                "failed to bind|" + std::to_string(res) + '|' +
-                                    sqlite3_errstr(res) + '|' + query_str);
-      return {context_, res};
-    }
-  }
-
-  return {context_, res};
-}
-
-auto db_select::limit(std::int32_t value) -> db_select & {
-  if (context_->delete_query) {
-    throw std::runtime_error("limit may not be specified for delete");
-  }
-
-  context_->limit = value;
-  return *this;
-}
-
-auto db_select::order_by(std::string column_name, bool ascending)
-    -> db_select & {
-  if (context_->delete_query) {
-    throw std::runtime_error("order_by may not be specified for delete");
-  }
-
-  context_->order_by = {column_name, ascending};
-  return *this;
-}
-
-auto db_select::where(std::string column_name) const -> db_where {
-  return db_where{context_, column_name};
-}
-} // namespace repertory::db

src/db/directory_db.cpp

@@ -0,0 +1,370 @@
+/*
+  Copyright <2018-2023> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all
+  copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  SOFTWARE.
+*/
+#include "db/directory_db.hpp"
+
+#include "utils/path_utils.hpp"
+
+namespace repertory {
+void directory_db::directory_tree::add_path(
+    const std::string &api_path, const std::vector<std::string> &files,
+    rocksdb::DB &db) {
+  const auto create_not_found = [&](const auto &create_path) {
+    std::string value;
+    if (not db.Get(rocksdb::ReadOptions(), create_path, &value).ok()) {
+      json directoryData = {
+          {"path", api_path},
+          {"files", files},
+      };
+      db.Put(rocksdb::WriteOptions(), create_path, directoryData.dump());
+    }
+  };
+
+  const auto parts = utils::string::split(api_path, '/', false);
+  std::string previous_directory;
+  for (const auto &directory_part : parts) {
+    if (directory_part.empty()) {
+      sub_directory_lookup_["/"];
+      previous_directory = "/";
+      create_not_found("/");
+    } else {
+      auto &sub_directories = sub_directory_lookup_[previous_directory];
+      if (std::find(sub_directories.begin(), sub_directories.end(),
+                    directory_part) == sub_directories.end()) {
+        sub_directories.emplace_back(directory_part);
+      }
+      previous_directory = utils::path::create_api_path(
+          utils::path::combine(previous_directory, {directory_part}));
+      sub_directory_lookup_[previous_directory];
+      create_not_found(previous_directory);
+    }
+  }
+}
+
+auto directory_db::directory_tree::get_count(const std::string &api_path) const
+    -> std::size_t {
+  return (sub_directory_lookup_.find(api_path) == sub_directory_lookup_.end())
+             ? 0
+             : sub_directory_lookup_.at(api_path).size();
+}
+
+auto directory_db::directory_tree::get_directories() const
+    -> std::vector<std::string> {
+  std::vector<std::string> ret;
+  std::transform(sub_directory_lookup_.begin(), sub_directory_lookup_.end(),
+                 std::back_inserter(ret),
+                 [](const auto &kv) { return kv.first; });
+
+  return ret;
+}
+
+auto directory_db::directory_tree::get_sub_directories(
+    const std::string &api_path) const -> std::vector<std::string> {
+  std::vector<std::string> ret;
+  if (sub_directory_lookup_.find(api_path) != sub_directory_lookup_.end()) {
+    const auto &lookup = sub_directory_lookup_.at(api_path);
+    std::transform(lookup.begin(), lookup.end(), std::back_inserter(ret),
+                   [&api_path](const auto &directory) {
+                     return utils::path::create_api_path(
+                         utils::path::combine(api_path, {directory}));
+                   });
+  }
+  return ret;
+}
+
+auto directory_db::directory_tree::is_directory(
+    const std::string &api_path) const -> bool {
+  return sub_directory_lookup_.find(api_path) != sub_directory_lookup_.end();
+}
+
+void directory_db::directory_tree::remove_directory(const std::string &api_path,
+                                                    rocksdb::DB &db,
+                                                    bool allow_remove_root) {
+  if ((allow_remove_root || (api_path != "/")) && is_directory(api_path)) {
+    sub_directory_lookup_.erase(api_path);
+    db.Delete(rocksdb::WriteOptions(), api_path);
+
+    const auto parent_api_path = utils::path::get_parent_api_path(api_path);
+    const auto parts = utils::string::split(api_path, '/', false);
+    utils::remove_element_from(sub_directory_lookup_[parent_api_path],
+                               parts[parts.size() - 1]);
+  }
+}
+
+directory_db::directory_db(const app_config &config) {
+  utils::db::create_rocksdb(config, DIRDB_NAME, db_);
+  auto iterator = std::unique_ptr<rocksdb::Iterator>(
+      db_->NewIterator(rocksdb::ReadOptions()));
+  for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
+    auto directory_data = json::parse(iterator->value().ToString());
+    tree_.add_path(directory_data["path"].get<std::string>(),
+                   directory_data["files"].get<std::vector<std::string>>(),
+                   *db_);
+  }
+}
+
+directory_db::~directory_db() { db_.reset(); }
+
+auto directory_db::create_directory(const std::string &api_path,
+                                    bool create_always) -> api_error {
+  recur_mutex_lock directory_lock(directory_mutex_);
+  auto ret = api_error::directory_exists;
+  if (not is_directory(api_path)) {
+    ret = api_error::directory_not_found;
+    if (create_always || (api_path == "/") ||
+        is_directory(utils::path::get_parent_api_path(api_path))) {
+      ret = api_error::item_exists;
+      if (not is_file(api_path)) {
+        tree_.add_path(api_path, {}, *db_);
+        ret = api_error::success;
+      }
+    }
+  }
+
+  return ret;
+}
+
+auto directory_db::create_file(const std::string &api_path) -> api_error {
+  recur_mutex_lock directory_lock(directory_mutex_);
+  if (is_directory(api_path)) {
+    return api_error::directory_exists;
+  }
+
+  const auto parent_api_path = utils::path::get_parent_api_path(api_path);
+  auto directory_data = get_directory_data(parent_api_path);
+  if (directory_data.empty()) {
+    return api_error::directory_not_found;
+  }
+
+  const auto file_name = utils::path::strip_to_file_name(api_path);
+  if (utils::collection_includes(directory_data["files"], file_name)) {
+    return api_error::item_exists;
+  }
+
+  directory_data["files"].emplace_back(file_name);
+  db_->Put(rocksdb::WriteOptions(), parent_api_path, directory_data.dump());
+  return api_error::success;
+}
+
+auto directory_db::get_directory_data(const std::string &api_path) const
+    -> json {
+  std::string data;
+  db_->Get(rocksdb::ReadOptions(), api_path, &data);
+  if (data.empty()) {
+    return {};
+  }
+
+  return json::parse(data);
+}
+
+auto directory_db::get_directory_item_count(const std::string &api_path) const
+    -> std::uint64_t {
+  auto directory_data = get_directory_data(api_path);
+  const auto sub_directory_count = get_sub_directory_count(api_path);
+  const auto file_count =
+      (directory_data.empty() ? 0 : directory_data["files"].size());
+  return sub_directory_count + file_count;
+}
+
+auto directory_db::get_file(const std::string &api_path, api_file &file,
+                            api_file_provider_callback api_file_provider) const
+    -> api_error {
+  const auto parent_api_path = utils::path::get_parent_api_path(api_path);
+  auto directory_data = get_directory_data(parent_api_path);
+  if (not directory_data.empty()) {
+    const auto file_name = utils::path::strip_to_file_name(api_path);
+    if (utils::collection_includes(directory_data["files"], file_name)) {
+      file.api_path = utils::path::create_api_path(
+          utils::path::combine(parent_api_path, {file_name})),
+      api_file_provider(file);
+      return api_error::success;
+    }
+  }
+
+  return api_error::item_not_found;
+}
+
+auto directory_db::get_file_list(
+    api_file_list &list, api_file_provider_callback api_file_provider) const
+    -> api_error {
+  auto iterator = std::unique_ptr<rocksdb::Iterator>(
+      db_->NewIterator(rocksdb::ReadOptions()));
+  for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
+    auto directory_data = json::parse(iterator->value().ToString());
+    for (const auto &directory_file : directory_data["files"]) {
+      api_file file{
+          utils::path::create_api_path(utils::path::combine(
+              iterator->key().ToString(), {directory_file.get<std::string>()})),
+      };
+      api_file_provider(file);
+      list.emplace_back(file);
+    }
+  }
+
+  return api_error::success;
+}
+
+auto directory_db::get_sub_directory_count(const std::string &api_path) const
+    -> std::size_t {
+  recur_mutex_lock directoryLock(directory_mutex_);
+  return tree_.get_count(api_path);
+}
+
+auto directory_db::get_total_item_count() const -> std::uint64_t {
+  unique_recur_mutex_lock directory_lock(directory_mutex_);
+  const auto directories = tree_.get_directories();
+  directory_lock.unlock();
+
+  return std::accumulate(
+      directories.begin(), directories.end(), std::uint64_t(directories.size()),
+      [this](std::uint64_t c, const std::string &directory) {
+        const auto dirData = this->get_directory_data(directory);
+        return c + (dirData.empty() ? 0 : dirData["files"].size());
+      });
+}
+
+auto directory_db::is_directory(const std::string &api_path) const -> bool {
+  recur_mutex_lock directory_lock(directory_mutex_);
+  return tree_.is_directory(api_path);
+}
+
+auto directory_db::is_file(const std::string &api_path) const -> bool {
+  auto directory_data =
+      get_directory_data(utils::path::get_parent_api_path(api_path));
+  if (directory_data.empty()) {
+    return false;
+  }
+
+  const auto file_name = utils::path::strip_to_file_name(api_path);
+  return utils::collection_includes(directory_data["files"], file_name);
+}
+
+void directory_db::populate_directory_files(
+    const std::string &api_path, meta_provider_callback meta_provider,
+    directory_item_list &list) const {
+  auto directory_data = get_directory_data(api_path);
+  if (not directory_data.empty()) {
+    for (const auto &directory_file : directory_data["files"]) {
+      directory_item di{};
+      di.api_path = utils::path::create_api_path(
+          utils::path::combine(api_path, {directory_file.get<std::string>()}));
+      di.directory = false;
+      meta_provider(di);
+      di.size = utils::string::to_uint64(di.meta[META_SIZE]);
+      list.emplace_back(std::move(di));
+    }
+  }
+}
+
+void directory_db::populate_sub_directories(
+    const std::string &api_path, meta_provider_callback meta_provider,
+    directory_item_list &list) const {
+  unique_recur_mutex_lock directory_lock(directory_mutex_);
+  const auto directories = tree_.get_sub_directories(api_path);
+  directory_lock.unlock();
+
+  std::size_t offset{};
+  for (const auto &directory : directories) {
+    if (std::find_if(list.begin(), list.end(),
+                     [&directory](const auto &di) -> bool {
+                       return directory == di.api_path;
+                     }) == list.end()) {
+      directory_item di{};
+      di.api_path = directory;
+      di.api_parent = utils::path::get_parent_api_path(directory);
+      di.directory = true;
+      di.size = get_sub_directory_count(directory);
+      meta_provider(di);
+
+      list.insert(list.begin() + static_cast<std::int64_t>(offset++),
+                  std::move(di));
+    }
+  }
+}
+
+auto directory_db::remove_directory(const std::string &api_path,
+                                    bool allow_remove_root) -> api_error {
+  recur_mutex_lock directory_lock(directory_mutex_);
+  if ((api_path == "/") && not allow_remove_root) {
+    return api_error::access_denied;
+  }
+
+  if (is_file(api_path) || not is_directory(api_path)) {
+    return api_error::directory_not_found;
+  }
+
+  if (tree_.get_count(api_path) == 0) {
+    auto directory_data = get_directory_data(api_path);
+    if (directory_data.empty() || directory_data["files"].empty()) {
+      tree_.remove_directory(api_path, *db_, allow_remove_root);
+      return api_error::success;
+    }
+  }
+
+  return api_error::directory_not_empty;
+}
+
+auto directory_db::remove_file(const std::string &api_path) -> bool {
+  recur_mutex_lock directory_lock(directory_mutex_);
+
+  if (is_directory(api_path)) {
+    return false;
+  }
+
+  const auto parent_api_path = utils::path::get_parent_api_path(api_path);
+  auto directory_data = get_directory_data(parent_api_path);
+  if (directory_data.empty()) {
+    return false;
+  }
+
+  const auto file_name = utils::path::strip_to_file_name(api_path);
+  if (utils::collection_excludes(directory_data["files"], file_name)) {
+    return false;
+  }
+
+  utils::remove_element_from(directory_data["files"], file_name);
+  db_->Put(rocksdb::WriteOptions(), parent_api_path, directory_data.dump());
+  return true;
+}
+
+auto directory_db::rename_file(const std::string &from_api_path,
+                               const std::string &to_api_path) -> api_error {
+  recur_mutex_lock directory_lock(directory_mutex_);
+
+  if (is_directory(from_api_path) || is_directory(to_api_path)) {
+    return api_error::directory_exists;
+  }
+
+  if (not is_directory(utils::path::get_parent_api_path(to_api_path))) {
+    return api_error::directory_not_found;
+  }
+
+  if (is_file(to_api_path)) {
+    return api_error::item_exists;
+  }
+
+  if (not remove_file(from_api_path)) {
+    return api_error::item_not_found;
+  }
+
+  return create_file(to_api_path);
+}
+} // namespace repertory

src/db/meta_db.cpp

@@ -0,0 +1,347 @@
+/*
+  Copyright <2018-2023> <scott.e.graves@protonmail.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all
+  copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  SOFTWARE.
+*/
+#include "db/meta_db.hpp"
+
+#include "types/repertory.hpp"
+#include "types/startup_exception.hpp"
+#include "utils/error_utils.hpp"
+#include "utils/file_utils.hpp"
+#include "utils/path_utils.hpp"
+#include "utils/utils.hpp"
+
+namespace repertory {
+meta_db::meta_db(const app_config &config) {
+  const auto create_resources = [this, &config](const std::string &name) {
+    auto families = std::vector<rocksdb::ColumnFamilyDescriptor>();
+    families.emplace_back(rocksdb::kDefaultColumnFamilyName,
+                          rocksdb::ColumnFamilyOptions());
+    families.emplace_back("keys", rocksdb::ColumnFamilyOptions());
+    families.emplace_back("source", rocksdb::ColumnFamilyOptions());
+
+    auto handles = std::vector<rocksdb::ColumnFamilyHandle *>();
+    utils::db::create_rocksdb(config, name, families, handles, db_);
+
+    std::size_t idx{};
+    default_family_ = handles[idx++];
+    keys_family_ = handles[idx++];
+    source_family_ = handles[idx++];
+  };
+
+  create_resources(METADB_NAME);
+}
+
+meta_db::~meta_db() { db_.reset(); }
+
+auto meta_db::create_iterator(bool source_family) const
+    -> std::shared_ptr<rocksdb::Iterator> {
+  return std::shared_ptr<rocksdb::Iterator>(
+      db_->NewIterator(rocksdb::ReadOptions(),
+                       source_family ? source_family_ : default_family_));
+}
+
+auto meta_db::get_api_path_from_key(const std::string &key,
+                                    std::string &api_path) const -> api_error {
+  if (key.empty()) {
+    return api_error::item_not_found;
+  }
+
+  return perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
+    return db_->Get(rocksdb::ReadOptions(), keys_family_, key, &api_path);
+  });
+}
+
+auto meta_db::get_api_path_from_source(const std::string &source_path,
+                                       std::string &api_path) const
+    -> api_error {
+  if (source_path.empty()) {
+    return api_error::item_not_found;
+  }
+
+  return perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
+    return db_->Get(rocksdb::ReadOptions(), source_family_, source_path,
+                    &api_path);
+  });
+}
+
+auto meta_db::get_item_meta_json(const std::string &api_path,
+                                 json &json_data) const -> api_error {
+  std::string value;
+  const auto res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
+    return db_->Get(rocksdb::ReadOptions(), default_family_, api_path, &value);
+  });
+  if (res != api_error::success) {
+    return res;
+  }
+
+  json_data = json::parse(value);
+  return api_error::success;
+}
+
+auto meta_db::get_item_meta(const std::string &api_path,
+                            api_meta_map &meta) const -> api_error {
+  json json_data;
+  const auto ret = get_item_meta_json(api_path, json_data);
+  if (ret == api_error::success) {
+    for (auto it = json_data.begin(); it != json_data.end(); it++) {
+      meta[it.key()] = it.value().get<std::string>();
+    }
+  }
+
+  return ret;
+}
+
+auto meta_db::get_item_meta(const std::string &api_path, const std::string &key,
+                            std::string &value) const -> api_error {
+  json json_data;
+  const auto ret = get_item_meta_json(api_path, json_data);
+  if (ret == api_error::success) {
+    if (json_data.find(key) != json_data.end()) {
+      value = json_data[key].get<std::string>();
+    }
+  }
+
+  return ret;
+}
+
+auto meta_db::get_item_meta_exists(const std::string &api_path) const -> bool {
+  std::string value;
+  return db_->Get(rocksdb::ReadOptions(), api_path, &value).ok();
+}
+
+auto meta_db::get_pinned_files() const -> std::vector<std::string> {
+  std::vector<std::string> ret;
+
+  auto iterator = const_cast<meta_db *>(this)->create_iterator(false);
+  for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
+    auto api_path = iterator->key().ToString();
+    std::string pinned;
+    const auto res = get_item_meta(api_path, META_PINNED, pinned);
+    if ((res == api_error::success) && not pinned.empty() &&
+        utils::string::to_bool(pinned)) {
+      ret.emplace_back(api_path);
+    }
+  }
+
+  return ret;
+}
+
+auto meta_db::get_source_path_exists(const std::string &source_path) const
+    -> bool {
+  std::string value;
+  return db_->Get(rocksdb::ReadOptions(), source_family_, source_path, &value)
+      .ok();
+}
+
+auto meta_db::perform_action(
+    const std::string &function_name,
+    const std::function<rocksdb::Status()> &action) const -> api_error {
+  const auto res = action();
+  if (res.ok()) {
+    return api_error::success;
+  }
+
+  if (not res.IsNotFound()) {
+    utils::error::raise_error(function_name, res.ToString());
+  }
+
+  return res.IsNotFound() ? api_error::item_not_found : api_error::error;
+}
+
+auto meta_db::get_total_item_count() const -> std::uint64_t {
+  std::uint64_t ret = 0u;
+  auto iter = create_iterator(false);
+  for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
+    ret++;
+  }
+
+  return ret;
+}
+
+auto meta_db::remove_item_meta(const std::string &api_path) -> api_error {
+  json json_data;
+  auto res = get_item_meta_json(api_path, json_data);
+  if (res != api_error::success) {
+    return res == api_error::item_not_found ? api_error::success : res;
+  }
+
+  if (not json_data[META_KEY].empty()) {
+    if ((res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
+           return db_->Delete(rocksdb::WriteOptions(), keys_family_,
+                              json_data[META_KEY].get<std::string>());
+         })) != api_error::success) {
+      return res;
+    }
+  }
+
+  if (not json_data[META_SOURCE].empty()) {
+    if ((res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
+           return db_->Delete(rocksdb::WriteOptions(), source_family_,
+                              json_data[META_SOURCE].get<std::string>());
+         })) != api_error::success) {
+      return res;
+    }
+  }
+
+  return perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
+    return db_->Delete(rocksdb::WriteOptions(), default_family_, api_path);
+  });
+}
+
+auto meta_db::remove_item_meta(const std::string &api_path,
+                               const std::string &key) -> api_error {
+  json json_data;
+  auto res = get_item_meta_json(api_path, json_data);
+  if (res != api_error::success) {
+    return res == api_error::item_not_found ? api_error::success : res;
+  }
+
+  if ((key == META_KEY) && not json_data[META_KEY].empty()) {
+    if ((res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
+           return db_->Delete(rocksdb::WriteOptions(), keys_family_,
+                              json_data[META_KEY].get<std::string>());
+         })) != api_error::success) {
+      return res;
+    }
+  }
+
+  if ((key == META_SOURCE) && not json_data[META_SOURCE].empty()) {
+    if ((res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
+           return db_->Delete(rocksdb::WriteOptions(), source_family_,
+                              json_data[META_SOURCE].get<std::string>());
+         })) != api_error::success) {
+      return res;
+    }
+  }
+
+  json_data.erase(key);
+  return perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
+    return db_->Put(rocksdb::WriteOptions(), default_family_, api_path,
+                    json_data.dump());
+  });
+}
+
+auto meta_db::rename_item_meta(const std::string &source_path,
+                               const std::string &from_api_path,
+                               const std::string &to_api_path) -> api_error {
+  api_meta_map meta{};
+  auto res = get_item_meta(from_api_path, meta);
+  if (res != api_error::success) {
+    return res;
+  }
+
+  if ((res = remove_item_meta(from_api_path)) != api_error::success) {
+    return res;
+  }
+
+  if (not source_path.empty()) {
+    meta[META_SOURCE] = source_path;
+  }
+
+  return set_item_meta(to_api_path, meta);
+}
+
+auto meta_db::set_item_meta(const std::string &api_path, const std::string &key,
+                            const std::string &value) -> api_error {
+  if (key == META_SOURCE) {
+    return set_source_path(api_path, value);
+  }
+
+  if (key == META_KEY) {
+    const auto res = remove_item_meta(api_path, META_KEY);
+    if ((res != api_error::success) && (res != api_error::item_not_found)) {
+      return res;
+    }
+  }
+
+  auto res = store_item_meta(api_path, key, value);
+  if (res != api_error::success) {
+    return res;
+  }
+
+  if (key == META_KEY) {
+    return perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
+      return db_->Put(rocksdb::WriteOptions(), keys_family_, value, api_path);
+    });
+  }
+
+  return api_error::success;
+}
+
+auto meta_db::set_item_meta(const std::string &api_path,
+                            const api_meta_map &meta) -> api_error {
+  auto ret = api_error::success;
+  auto it = meta.begin();
+  for (std::size_t i = 0u; (ret == api_error::success) && (i < meta.size());
+       i++) {
+    ret = set_item_meta(api_path, it->first, it->second);
+    it++;
+  }
+
+  return ret;
+}
+
+auto meta_db::set_source_path(const std::string &api_path,
+                              const std::string &source_path) -> api_error {
+  std::string current_source_path;
+  auto res = get_item_meta(api_path, META_SOURCE, current_source_path);
+  if ((res != api_error::success) && (res != api_error::item_not_found)) {
+    return res;
+  }
+
+  // TODO multiple db ops should be in transaction
+  if (not current_source_path.empty()) {
+    if ((res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
+           return db_->Delete(rocksdb::WriteOptions(), source_family_,
+                              current_source_path);
+         })) != api_error::success) {
+      return res;
+    }
+  }
+
+  if (not source_path.empty()) {
+    if ((res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
+           return db_->Put(rocksdb::WriteOptions(), source_family_, source_path,
+                           api_path);
+         })) != api_error::success) {
+      return res;
+    }
+  }
+
+  return store_item_meta(api_path, META_SOURCE, source_path);
+}
+
+auto meta_db::store_item_meta(const std::string &api_path,
+                              const std::string &key, const std::string &value)
+    -> api_error {
+  json json_data;
+  auto res = get_item_meta_json(api_path, json_data);
+  if ((res != api_error::success) && (res != api_error::item_not_found)) {
+    return res;
+  }
+
+  json_data[key] = value;
+  return perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
+    return db_->Put(rocksdb::WriteOptions(), default_family_, api_path,
+                    json_data.dump());
+  });
+}
+} // namespace repertory

src/drives/directory_iterator.cpp

@@ -54,12 +54,10 @@ auto directory_iterator::fill_buffer(const remote::file_offset &offset,
       }
 
 #if FUSE_USE_VERSION >= 30
-      if (filler_function(buffer, item_name.data(), pst,
-                          static_cast<off_t>(offset + 1),
+      if (filler_function(buffer, &item_name[0], pst, offset + 1,
                           FUSE_FILL_DIR_PLUS) != 0) {
 #else
-      if (filler_function(buffer, item_name.data(), pst,
-                          static_cast<off_t>(offset + 1)) != 0) {
+      if (filler_function(buffer, &item_name[0], pst, offset + 1) != 0) {
 #endif
         errno = ENOMEM;
         return -1;
@@ -123,14 +121,13 @@ auto directory_iterator::get_json(std::size_t offset, json &item) -> int {
 
 auto directory_iterator::get_next_directory_offset(
     const std::string &api_path) const -> std::size_t {
-  const auto iter = std::find_if(items_.begin(), items_.end(),
-                                 [&api_path](const auto &dir_item) -> bool {
-                                   return api_path == dir_item.api_path;
-                                 });
+  const auto it = std::find_if(
+      items_.begin(), items_.end(),
+      [&api_path](const auto &di) -> bool { return api_path == di.api_path; });
 
-  return (iter == items_.end()) ? 0U
-                                : static_cast<std::size_t>(
-                                      std::distance(items_.begin(), iter) + 1);
+  return (it == items_.end())
+             ? 0
+             : std::distance(items_.begin(), it) + std::size_t(1u);
 }
 
 auto directory_iterator::operator=(const directory_iterator &iterator) noexcept

src/drives/eviction.cpp

@@ -41,12 +41,12 @@ auto eviction::check_minimum_requirements(const std::string &file_path)
   }
 
   auto ret = false;
-  if (file_size != 0U) {
+  if (file_size) {
     std::uint64_t reference_time{};
-    ret = config_.get_eviction_uses_accessed_time()
-              ? utils::file::get_accessed_time(file_path, reference_time)
-              : utils::file::get_modified_time(file_path, reference_time);
-    if (ret) {
+    if ((ret =
+             config_.get_eviction_uses_accessed_time()
+                 ? utils::file::get_accessed_time(file_path, reference_time)
+                 : utils::file::get_modified_time(file_path, reference_time))) {
 #ifdef _WIN32
       const auto now = std::chrono::system_clock::now();
       const auto delay =
@@ -57,7 +57,7 @@ auto eviction::check_minimum_requirements(const std::string &file_path)
       const auto now = utils::get_time_now();
       const auto delay =
           (config_.get_eviction_delay_mins() * 60L) * NANOS_PER_SECOND;
-      ret = ((reference_time + static_cast<std::uint64_t>(delay)) <= now);
+      ret = ((reference_time + delay) <= now);
 #endif
     }
   }

src/drives/fuse/fuse_base.cpp

@@ -19,7 +19,6 @@
   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
   SOFTWARE.
 */
-#include "common.hpp"
 #ifndef _WIN32
 
 #include "drives/fuse/fuse_base.hpp"
@@ -29,7 +28,6 @@
 #include "events/event_system.hpp"
 #include "events/events.hpp"
 #include "providers/i_provider.hpp"
-#include "utils/file_utils.hpp"
 #include "utils/path_utils.hpp"
 #include "utils/string_utils.hpp"
 #include "utils/utils.hpp"
@@ -162,27 +160,21 @@ void fuse_base::destroy_(void *ptr) {
   execute_void_callback(__FUNCTION__, [&]() { instance().destroy_impl(ptr); });
 }
 
-void fuse_base::destroy_impl(void * /* ptr */) { repertory_shutdown(); }
-
-void fuse_base::display_options(
-    [[maybe_unused]] std::vector<const char *> args) {
+void fuse_base::display_options([[maybe_unused]] int argc,
+                                [[maybe_unused]] char *argv[]) {
 #if FUSE_USE_VERSION >= 30
   fuse_cmdline_help();
 #else
   struct fuse_operations fuse_ops {};
-  fuse_main(args.size(),
-            reinterpret_cast<char **>(const_cast<char **>(args.data())),
-            &fuse_ops, nullptr);
+  fuse_main(argc, argv, &fuse_ops, nullptr);
 #endif
 
   std::cout << std::endl;
 }
 
-void fuse_base::display_version_information(std::vector<const char *> args) {
+void fuse_base::display_version_information(int argc, char *argv[]) {
   struct fuse_operations fuse_ops {};
-  fuse_main(static_cast<int>(args.size()),
-            reinterpret_cast<char **>(const_cast<char **>(args.data())),
-            &fuse_ops, nullptr);
+  fuse_main(argc, argv, &fuse_ops, nullptr);
 }
 
 auto fuse_base::execute_callback(
@@ -316,9 +308,6 @@ auto fuse_base::init_(struct fuse_conn_info *conn) -> void * {
 #if FUSE_USE_VERSION >= 30
 auto fuse_base::init_impl([[maybe_unused]] struct fuse_conn_info *conn,
                           struct fuse_config *cfg) -> void * {
-  utils::file::change_to_process_directory();
-  repertory_init();
-
 #ifdef __APPLE__
   conn->want |= FUSE_CAP_VOL_RENAME;
   conn->want |= FUSE_CAP_XTIMES;
@@ -331,9 +320,6 @@ auto fuse_base::init_impl([[maybe_unused]] struct fuse_conn_info *conn,
 }
 #else
 auto fuse_base::init_impl(struct fuse_conn_info *conn) -> void * {
-  utils::file::change_to_process_directory();
-  repertory_init();
-
 #ifdef __APPLE__
   conn->want |= FUSE_CAP_VOL_RENAME;
   conn->want |= FUSE_CAP_XTIMES;
@@ -361,11 +347,10 @@ auto fuse_base::mount(std::vector<std::string> args) -> int {
     }
 
     {
-      struct fuse_args fa = FUSE_ARGS_INIT(
-          static_cast<int>(fuse_argv.size()),
-          reinterpret_cast<char **>(const_cast<char **>(fuse_argv.data())));
+      struct fuse_args fa = FUSE_ARGS_INIT(static_cast<int>(fuse_argv.size()),
+                                           (char **)&fuse_argv[0]);
 
-      char *mount_location{nullptr};
+      char *mount_location = nullptr;
 #if FUSE_USE_VERSION >= 30
       struct fuse_cmdline_opts opts {};
       fuse_parse_cmdline(&fa, &opts);
@@ -385,10 +370,8 @@ auto fuse_base::mount(std::vector<std::string> args) -> int {
 #if FUSE_USE_VERSION < 30
     umask(0);
 #endif
-    ret = fuse_main(
-        static_cast<int>(fuse_argv.size()),
-        reinterpret_cast<char **>(const_cast<char **>(fuse_argv.data())),
-        &fuse_ops_, this);
+    ret = fuse_main(static_cast<int>(fuse_argv.size()), (char **)&fuse_argv[0],
+                    &fuse_ops_, this);
     notify_fuse_main_exit(ret);
   }