updated CHANGELOG.md

CHANGELOG.md

@@ -14,6 +14,7 @@
   * MSYS2 is required for building Windows binaries on Windows
   * OS X support is temporarily disabled
 * \#19 \[bug\] Rename file is broken for files that are existing
+* \#23 \[bug\] Incorrect file size displayed while upload is pending
 
 ### Changes from v2.0.1-rc
 
@@ -21,6 +22,7 @@
 * Corrected handling of `chown()` and `chmod()`
 * Fixed erroneous download of chunks after resize
 * Comprehensive WinFSP and FUSE unit tests, including remote testing
+* Ability to choose between RocksDB and SQLite databases
 
 ## v2.0.1-rc
 

repertory/librepertory/include/app_config.hpp

@@ -30,26 +30,26 @@
 namespace repertory {
 class app_config final {
 public:
-  [[nodiscard]] static auto
-  default_agent_name(const provider_type &prov) -> std::string;
+  [[nodiscard]] static auto default_agent_name(const provider_type &prov)
+      -> std::string;
 
-  [[nodiscard]] static auto
-  default_api_port(const provider_type &prov) -> std::uint16_t;
+  [[nodiscard]] static auto default_api_port(const provider_type &prov)
+      -> std::uint16_t;
 
-  [[nodiscard]] static auto
-  default_data_directory(const provider_type &prov) -> std::string;
+  [[nodiscard]] static auto default_data_directory(const provider_type &prov)
+      -> std::string;
 
-  [[nodiscard]] static auto
-  default_remote_port(const provider_type &prov) -> std::uint16_t;
+  [[nodiscard]] static auto default_remote_port(const provider_type &prov)
+      -> std::uint16_t;
 
-  [[nodiscard]] static auto
-  default_rpc_port(const provider_type &prov) -> std::uint16_t;
+  [[nodiscard]] static auto default_rpc_port(const provider_type &prov)
+      -> std::uint16_t;
 
-  [[nodiscard]] static auto
-  get_provider_display_name(const provider_type &prov) -> std::string;
+  [[nodiscard]] static auto get_provider_display_name(const provider_type &prov)
+      -> std::string;
 
-  [[nodiscard]] static auto
-  get_provider_name(const provider_type &prov) -> std::string;
+  [[nodiscard]] static auto get_provider_name(const provider_type &prov)
+      -> std::string;
 
 public:
   app_config(const provider_type &prov, std::string_view data_directory = "");
@@ -63,6 +63,7 @@ private:
   std::string api_user_;
   bool config_changed_;
   std::string data_directory_;
+  database_type db_type_{database_type::rocksdb};
   std::uint8_t download_timeout_secs_;
   bool enable_chunk_downloader_timeout_;
   bool enable_comm_duration_events_;
@@ -109,7 +110,8 @@ private:
   mutable std::recursive_mutex remote_mount_mutex_;
 
 private:
-  [[nodiscard]] auto load() -> bool;
+  auto get_database_value(const json &json_document, const std::string &name,
+                          database_type &dst, bool &success_flag) -> bool;
 
   template <typename dest>
   auto get_value(const json &json_document, const std::string &name, dest &dst,
@@ -133,6 +135,8 @@ private:
     return ret;
   }
 
+  [[nodiscard]] auto load() -> bool;
+
   template <typename dest, typename source>
   auto set_value(dest &dst, const source &src) -> bool {
     auto ret{false};
@@ -164,6 +168,10 @@ public:
 
   [[nodiscard]] auto get_config_file_path() const -> std::string;
 
+  [[nodiscard]] auto get_database_type() const -> database_type {
+    return db_type_;
+  }
+
   [[nodiscard]] auto get_data_directory() const -> std::string {
     return data_directory_;
   }
@@ -332,6 +340,10 @@ public:
     set_value(download_timeout_secs_, chunk_downloader_timeout_secs);
   }
 
+  void set_database_type(const database_type &type) {
+    set_value(db_type_, type);
+  }
+
   void
   set_enable_chunk_downloader_timeout(bool enable_chunk_downloader_timeout) {
     set_value(enable_chunk_downloader_timeout_,

repertory/librepertory/include/file_manager/file_manager.hpp

@@ -85,6 +85,9 @@ private:
   void queue_upload(const std::string &api_path, const std::string &source_path,
                     bool no_lock);
 
+  void remove_resume(const std::string &api_path,
+                     const std::string &source_path, bool no_lock);
+
   void remove_upload(const std::string &api_path, bool no_lock);
 
   void swap_renamed_items(std::string from_api_path, std::string to_api_path,
@@ -131,13 +134,13 @@ public:
 
   [[nodiscard]] auto get_open_handle_count() const -> std::size_t;
 
-  [[nodiscard]] auto
-  get_stored_downloads() const -> std::vector<i_file_mgr_db::resume_entry>;
+  [[nodiscard]] auto get_stored_downloads() const
+      -> std::vector<i_file_mgr_db::resume_entry>;
 
   [[nodiscard]] auto has_no_open_file_handles() const -> bool override;
 
-  [[nodiscard]] auto
-  is_processing(const std::string &api_path) const -> bool override;
+  [[nodiscard]] auto is_processing(const std::string &api_path) const
+      -> bool override;
 
 #if defined(PROJECT_TESTING)
   [[nodiscard]] auto open(std::shared_ptr<i_closeable_open_file> of,
@@ -150,13 +153,13 @@ public:
 
   [[nodiscard]] auto remove_file(const std::string &api_path) -> api_error;
 
-  [[nodiscard]] auto
-  rename_directory(const std::string &from_api_path,
-                   const std::string &to_api_path) -> api_error;
+  [[nodiscard]] auto rename_directory(const std::string &from_api_path,
+                                      const std::string &to_api_path)
+      -> api_error;
 
   [[nodiscard]] auto rename_file(const std::string &from_api_path,
-                                 const std::string &to_api_path,
-                                 bool overwrite) -> api_error;
+                                 const std::string &to_api_path, bool overwrite)
+      -> api_error;
 
   void start();
 

repertory/librepertory/include/file_manager/i_upload_manager.hpp

@@ -29,14 +29,14 @@ class i_upload_manager {
   INTERFACE_SETUP(i_upload_manager);
 
 public:
-  virtual void queue_upload(const i_open_file &o) = 0;
+  virtual void queue_upload(const i_open_file &file) = 0;
 
   virtual void remove_resume(const std::string &api_path,
                              const std::string &source_path) = 0;
 
   virtual void remove_upload(const std::string &api_path) = 0;
 
-  virtual void store_resume(const i_open_file &o) = 0;
+  virtual void store_resume(const i_open_file &file) = 0;
 };
 } // namespace repertory
 

repertory/librepertory/include/types/repertory.hpp

@@ -23,7 +23,9 @@
 #define REPERTORY_INCLUDE_TYPES_REPERTORY_HPP_
 
 namespace repertory {
-inline constexpr const auto max_time{std::numeric_limits<std::uint64_t>::max()};
+inline constexpr const auto max_time{
+    std::numeric_limits<std::uint64_t>::max(),
+};
 
 inline constexpr const std::string META_ACCESSED{"accessed"};
 inline constexpr const std::string META_ATTRIBUTES{"attributes"};
@@ -99,7 +101,22 @@ enum class api_error {
 [[nodiscard]] auto api_error_to_string(const api_error &error)
     -> const std::string &;
 
-enum class download_type { direct, fallback, ring_buffer };
+enum class database_type {
+  rocksdb,
+  sqlite,
+};
+[[nodiscard]] auto database_type_from_string(std::string type,
+                                             const database_type &default_type)
+    -> database_type;
+
+[[nodiscard]] auto database_type_to_string(const database_type &type)
+    -> std::string;
+
+enum class download_type {
+  direct,
+  fallback,
+  ring_buffer,
+};
 [[nodiscard]] auto download_type_from_string(std::string type,
                                              const download_type &default_type)
     -> download_type;
@@ -158,23 +175,23 @@ using open_file_data = int;
 #endif
 
 struct api_file final {
-  std::string api_path{};
-  std::string api_parent{};
+  std::string api_path;
+  std::string api_parent;
   std::uint64_t accessed_date{};
   std::uint64_t changed_date{};
   std::uint64_t creation_date{};
   std::uint64_t file_size{};
-  std::string key{};
+  std::string key;
   std::uint64_t modified_date{};
   std::string source_path;
 };
 
 struct directory_item final {
-  std::string api_path{};
-  std::string api_parent{};
+  std::string api_path;
+  std::string api_parent;
   bool directory{false};
   std::uint64_t size{};
-  api_meta_map meta{};
+  api_meta_map meta;
   bool resolved{false};
 
   [[nodiscard]] static auto from_json(const json &item) -> directory_item {
@@ -196,42 +213,42 @@ struct directory_item final {
 };
 
 struct encrypt_config final {
-  std::string encryption_token{};
-  std::string path{};
+  std::string encryption_token;
+  std::string path;
 };
 
 struct filesystem_item final {
-  std::string api_path{};
-  std::string api_parent{};
+  std::string api_path;
+  std::string api_parent;
   bool directory{false};
   std::uint64_t size{};
-  std::string source_path{};
+  std::string source_path;
 };
 
 struct host_config final {
-  std::string agent_string{};
-  std::string api_password{};
-  std::string api_user{};
-  std::uint16_t api_port{};
+  std::string agent_string;
+  std::string api_password;
+  std::string api_user;
+  std::uint16_t api_port;
   std::string host_name_or_ip{"localhost"};
-  std::string path{};
+  std::string path;
   std::string protocol{"http"};
   std::uint32_t timeout_ms{60000U};
 
-  auto operator==(const host_config &hc) const noexcept -> bool {
-    if (&hc != this) {
-      return agent_string == hc.agent_string &&
-             api_password == hc.api_password && api_user == hc.api_user &&
-             api_port == hc.api_port && host_name_or_ip == hc.host_name_or_ip &&
-             path == hc.path && protocol == hc.protocol &&
-             timeout_ms == hc.timeout_ms;
+  auto operator==(const host_config &cfg) const noexcept -> bool {
+    if (&cfg != this) {
+      return agent_string == cfg.agent_string &&
+             api_password == cfg.api_password && api_user == cfg.api_user &&
+             api_port == cfg.api_port &&
+             host_name_or_ip == cfg.host_name_or_ip && path == cfg.path &&
+             protocol == cfg.protocol && timeout_ms == cfg.timeout_ms;
     }
     return true;
   }
 
-  auto operator!=(const host_config &hc) const noexcept -> bool {
-    if (&hc != this) {
-      return not(hc == *this);
+  auto operator!=(const host_config &cfg) const noexcept -> bool {
+    if (&cfg != this) {
+      return not(cfg == *this);
     }
     return false;
   }
@@ -241,46 +258,48 @@ struct host_config final {
 __attribute__((unused))
 #endif
 static void
-to_json(json &j, const host_config &hc) {
-  j = json{{"AgentString", hc.agent_string},
-           {"ApiPassword", hc.api_password},
-           {"ApiPort", hc.api_port},
-           {"ApiUser", hc.api_user},
-           {"HostNameOrIp", hc.host_name_or_ip},
-           {"Path", hc.path},
-           {"Protocol", hc.protocol},
-           {"TimeoutMs", hc.timeout_ms}};
+to_json(json &data, const host_config &cfg) {
+  data = json{
+      {"AgentString", cfg.agent_string},
+      {"ApiPassword", cfg.api_password},
+      {"ApiPort", cfg.api_port},
+      {"ApiUser", cfg.api_user},
+      {"HostNameOrIp", cfg.host_name_or_ip},
+      {"Path", cfg.path},
+      {"Protocol", cfg.protocol},
+      {"TimeoutMs", cfg.timeout_ms},
+  };
 }
 
 #if defined(__GNUG__)
 __attribute__((unused))
 #endif
 static void
-from_json(const json &j, host_config &hc) {
-  j.at("AgentString").get_to(hc.agent_string);
-  j.at("ApiPassword").get_to(hc.api_password);
-  j.at("ApiPort").get_to(hc.api_port);
-  j.at("AuthUser").get_to(hc.api_user);
-  j.at("HostNameOrIp").get_to(hc.host_name_or_ip);
-  j.at("Path").get_to(hc.path);
-  j.at("Protocol").get_to(hc.protocol);
-  j.at("TimeoutMs").get_to(hc.timeout_ms);
+from_json(const json &data, host_config &cfg) {
+  data.at("AgentString").get_to(cfg.agent_string);
+  data.at("ApiPassword").get_to(cfg.api_password);
+  data.at("ApiPort").get_to(cfg.api_port);
+  data.at("AuthUser").get_to(cfg.api_user);
+  data.at("HostNameOrIp").get_to(cfg.host_name_or_ip);
+  data.at("Path").get_to(cfg.path);
+  data.at("Protocol").get_to(cfg.protocol);
+  data.at("TimeoutMs").get_to(cfg.timeout_ms);
 }
 
 struct s3_config final {
-  std::string access_key{};
-  std::string bucket{};
-  std::string encryption_token{};
+  std::string access_key;
+  std::string bucket;
+  std::string encryption_token;
   std::string region{"any"};
-  std::string secret_key{};
+  std::string secret_key;
   std::uint32_t timeout_ms{60000U};
-  std::string url{};
+  std::string url;
   bool use_path_style{false};
   bool use_region_in_url{false};
 };
 
 struct sia_config final {
-  std::string bucket{};
+  std::string bucket;
 };
 
 using api_file_list = std::vector<api_file>;

repertory/librepertory/src/app_config.cpp

@@ -127,6 +127,29 @@ auto app_config::get_config_file_path() const -> std::string {
   return utils::path::combine(data_directory_, {"config.json"});
 }
 
+auto app_config::get_database_value(const json &json_document,
+                                    const std::string &name, database_type &dst,
+                                    bool &success_flag) -> bool {
+  REPERTORY_USES_FUNCTION_NAME();
+
+  auto ret{false};
+  try {
+    if (json_document.find(name) != json_document.end()) {
+      dst = database_type_from_string(json_document[name].get<std::string>(),
+                                      database_type::rocksdb);
+      ret = true;
+    } else {
+      success_flag = false;
+    }
+  } catch (const json::exception &ex) {
+    utils::error::raise_error(function_name, ex, "exception occurred");
+    success_flag = false;
+    ret = false;
+  }
+
+  return ret;
+}
+
 auto app_config::default_agent_name(const provider_type &prov) -> std::string {
   static const std::array<std::string,
                           static_cast<std::size_t>(provider_type::unknown)>
@@ -212,6 +235,7 @@ auto app_config::get_json() const -> json {
       {"ApiPort", api_port_},
       {"ApiUser", api_user_},
       {"ChunkDownloaderTimeoutSeconds", download_timeout_secs_},
+      {"DatabaseType", database_type_to_string(db_type_)},
       {"EnableChunkDownloaderTimeout", enable_chunk_downloader_timeout_},
       {"EnableCommDurationEvents", enable_comm_duration_events_},
       {"EnableDriveEvents", enable_drive_events_},
@@ -305,6 +329,7 @@ auto app_config::get_json() const -> json {
     ret.erase("S3Config");
   } else if (prov_ == provider_type::remote) {
     ret.erase("ChunkDownloaderTimeoutSeconds");
+    ret.erase("DatabaseType");
     ret.erase("EnableChunkDownloaderTimeout");
     ret.erase("EnableChunkDownloaderTimeout");
     ret.erase("EnableMaxCacheSize");
@@ -376,6 +401,9 @@ auto app_config::get_value_by_name(const std::string &name) -> std::string {
     if (name == "ApiUser") {
       return api_user_;
     }
+    if (name == "DatabaseType") {
+      return database_type_to_string(get_database_type());
+    }
     if (name == "ChunkDownloaderTimeoutSeconds") {
       return std::to_string(get_chunk_downloader_timeout_secs());
     }
@@ -550,6 +578,7 @@ auto app_config::load() -> bool {
           get_value(json_document, "ApiUser", api_user_, ret);
           get_value(json_document, "ChunkDownloaderTimeoutSeconds",
                     download_timeout_secs_, ret);
+          get_database_value(json_document, "DatabaseType", db_type_, ret);
           get_value(json_document, "EvictionDelayMinutes", eviction_delay_mins_,
                     ret);
           get_value(json_document, "EvictionUsesAccessedTime",
@@ -762,6 +791,11 @@ auto app_config::set_value_by_name(const std::string &name,
       set_chunk_downloader_timeout_secs(utils::string::to_uint8(value));
       return std::to_string(get_chunk_downloader_timeout_secs());
     }
+    if (name == "DatabaseType") {
+      set_database_type(
+          database_type_from_string(value, database_type::rocksdb));
+      return database_type_to_string(db_type_);
+    }
     if (name == "EnableChunkDownloaderTimeout") {
       set_enable_chunk_downloader_timeout(utils::string::to_bool(value));
       return utils::string::from_bool(get_enable_chunk_download_timeout());

repertory/librepertory/src/db/file_mgr_db.cpp

@@ -29,6 +29,12 @@
 namespace repertory {
 auto create_file_mgr_db(const app_config &cfg)
     -> std::unique_ptr<i_file_mgr_db> {
+  switch (cfg.get_database_type()) {
+  case database_type::sqlite:
+    return std::make_unique<sqlite_file_mgr_db>(cfg);
+
+  default:
     return std::make_unique<rdb_file_mgr_db>(cfg);
   }
+}
 } // namespace repertory

repertory/librepertory/src/db/meta_db.cpp

@@ -27,6 +27,12 @@
 
 namespace repertory {
 auto create_meta_db(const app_config &cfg) -> std::unique_ptr<i_meta_db> {
+  switch (cfg.get_database_type()) {
+  case database_type::sqlite:
+    return std::make_unique<sqlite_meta_db>(cfg);
+
+  default:
     return std::make_unique<rdb_meta_db>(cfg);
   }
+}
 } // namespace repertory

repertory/librepertory/src/file_manager/file_manager.cpp

@@ -429,7 +429,7 @@ void file_manager::queue_upload(const std::string &api_path,
           api_path,
           source_path,
       })) {
-    remove_resume(api_path, source_path);
+    remove_resume(api_path, source_path, true);
     event_system::instance().raise<file_upload_queued>(api_path, source_path);
   } else {
     event_system::instance().raise<file_upload_failed>(
@@ -454,8 +454,10 @@ auto file_manager::remove_file(const std::string &api_path) -> api_error {
 
   close_all(api_path);
 
+  mutex_lock lock(upload_mtx_);
   remove_upload(api_path, true);
-  remove_resume(api_path, fsi.source_path);
+  remove_resume(api_path, fsi.source_path, true);
+  upload_notify_.notify_all();
 
   res = provider_.remove_file(api_path);
   if (res != api_error::success) {
@@ -473,14 +475,30 @@ auto file_manager::remove_file(const std::string &api_path) -> api_error {
 
 void file_manager::remove_resume(const std::string &api_path,
                                  const std::string &source_path) {
-  if (not mgr_db_->remove_resume(api_path)) {
+  return remove_resume(api_path, source_path, false);
+}
+
+void file_manager::remove_resume(const std::string &api_path,
+                                 const std::string &source_path, bool no_lock) {
+  if (provider_.is_read_only()) {
     return;
   }
 
+  std::unique_ptr<mutex_lock> lock;
+  if (not no_lock) {
+    lock = std::make_unique<mutex_lock>(upload_mtx_);
+  }
+
+  if (mgr_db_->remove_resume(api_path)) {
     event_system::instance().raise<download_resume_removed>(api_path,
                                                             source_path);
   }
 
+  if (not no_lock) {
+    upload_notify_.notify_all();
+  }
+}
+
 void file_manager::remove_upload(const std::string &api_path) {
   remove_upload(api_path, false);
 }

repertory/librepertory/src/types/repertory.cpp

@@ -25,6 +25,32 @@
 #include "utils/string.hpp"
 
 namespace repertory {
+auto database_type_from_string(std::string type,
+                               const database_type &default_type)
+    -> database_type {
+  type = utils::string::to_lower(utils::string::trim(type));
+  if (type == "rocksdb") {
+    return database_type::rocksdb;
+  }
+
+  if (type == "sqlite") {
+    return database_type::sqlite;
+  }
+
+  return default_type;
+}
+
+auto database_type_to_string(const database_type &type) -> std::string {
+  switch (type) {
+  case database_type::rocksdb:
+    return "rocksdb";
+  case database_type::sqlite:
+    return "sqlite";
+  default:
+    return "rocksdb";
+  }
+}
+
 auto download_type_from_string(std::string type,
                                const download_type &default_type)
     -> download_type {

repertory/repertory_test/src/config_test.cpp

@@ -60,6 +60,7 @@ const auto DEFAULT_SIA_CONFIG = "{\n"
                                 "  \"ApiPort\": 10000,\n"
                                 "  \"ApiUser\": \"repertory\",\n"
                                 "  \"ChunkDownloaderTimeoutSeconds\": 30,\n"
+                                "  \"DatabaseType\": \"rocksdb\",\n"
                                 "  \"EnableChunkDownloaderTimeout\": true,\n"
                                 "  \"EnableCommDurationEvents\": false,\n"
                                 "  \"EnableDriveEvents\": false,\n"
@@ -113,6 +114,7 @@ const auto DEFAULT_S3_CONFIG = "{\n"
                                "  \"ApiPort\": 10100,\n"
                                "  \"ApiUser\": \"repertory\",\n"
                                "  \"ChunkDownloaderTimeoutSeconds\": 30,\n"
+                               "  \"DatabaseType\": \"rocksdb\",\n"
                                "  \"EnableChunkDownloaderTimeout\": true,\n"
                                "  \"EnableCommDurationEvents\": false,\n"
                                "  \"EnableDriveEvents\": false,\n"
@@ -857,4 +859,18 @@ TEST_F(config_test, task_wait_ms_minimum_value) {
     EXPECT_EQ(50U, config.get_task_wait_ms());
   }
 }
+
+TEST_F(config_test, can_set_database_type) {
+  {
+    app_config config(provider_type::sia, sia_directory);
+    config.set_database_type(database_type::rocksdb);
+    EXPECT_EQ(database_type::rocksdb, config.get_database_type());
+
+    config.set_database_type(database_type::sqlite);
+    EXPECT_EQ(database_type::sqlite, config.get_database_type());
+
+    config.set_database_type(database_type::rocksdb);
+    EXPECT_EQ(database_type::rocksdb, config.get_database_type());
+  }
+}
 } // namespace repertory