BlockStorage/repertory
1
0
Fork 0
Code Issues 7 Pull Requests Releases 6 Wiki Activity

refactor s3 provider

Browse Source
This commit is contained in:
Scott E. Graves 2023-11-11 11:32:14 -06:00
parent f2c1f64f02
commit 68476cbc00
55 changed files with 1504 additions and 5050 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
include/app_config.hpp
View File

@ -30,39 +30,39 @@
namespace repertory {
class app_config final {
public:
[[nodiscard]] static auto default_agent_name(const provider_type &pt)
[[nodiscard]] static auto default_agent_name(const provider_type &prov)
-> std::string;
[[nodiscard]] static auto default_api_port(const provider_type &pt)
[[nodiscard]] static auto default_api_port(const provider_type &prov)
-> std::uint16_t;
[[nodiscard]] static auto default_data_directory(const provider_type &pt)
[[nodiscard]] static auto default_data_directory(const provider_type &prov)
-> std::string;
[[nodiscard]] static auto default_rpc_port(const provider_type &pt)
[[nodiscard]] static auto default_rpc_port(const provider_type &prov)
-> std::uint16_t;
[[nodiscard]] static auto get_provider_api_password(const provider_type &pt)
[[nodiscard]] static auto get_provider_api_password(const provider_type &prov)
-> std::string;
[[nodiscard]] static auto get_provider_display_name(const provider_type &pt)
[[nodiscard]] static auto get_provider_display_name(const provider_type &prov)
-> std::string;
[[nodiscard]] static auto get_provider_name(const provider_type &pt)
[[nodiscard]] static auto get_provider_name(const provider_type &prov)
-> std::string;
public:
app_config(const provider_type &pt, const std::string &data_directory = "");
app_config(const provider_type &prov, const std::string &data_directory = "");
~app_config() { save(); }
private:
const provider_type pt_;
provider_type prov_;
std::string api_auth_;
std::uint16_t api_port_;
std::string api_user_;
bool config_changed_;
const std::string data_directory_;
std::string data_directory_;
std::uint8_t download_timeout_secs_;
bool enable_chunk_downloader_timeout_;
bool enable_comm_duration_events_;
@ -129,7 +129,7 @@ private:
template <typename dest, typename source>
auto set_value(dest &dst, const source &src) -> bool {
auto ret = false;
recur_mutex_lock l(read_write_mutex_);
recur_mutex_lock lock(read_write_mutex_);
if (dst != src) {
dst = src;
config_changed_ = true;
@ -244,7 +244,9 @@ public:
download_type::fallback);
}
[[nodiscard]] auto get_provider_type() const -> provider_type { return pt_; }
[[nodiscard]] auto get_provider_type() const -> provider_type {
return prov_;
}
[[nodiscard]] auto get_read_ahead_count() const -> std::uint8_t {
return std::max(static_cast<std::uint8_t>(1U), read_ahead_count_);

7
include/cli/actions.hpp
View File

@ -23,14 +23,12 @@
#define INCLUDE_CLI_ACTIONS_HPP_
#include "cli/check_version.hpp"
#include "cli/create_directory.hpp"
#include "cli/display_config.hpp"
#include "cli/drive_information.hpp"
#include "cli/get.hpp"
#include "cli/get_directory_items.hpp"
#include "cli/get_pinned_files.hpp"
#include "cli/help.hpp"
#include "cli/list_objects.hpp"
#include "cli/mount.hpp"
#include "cli/open_files.hpp"
#include "cli/pin_file.hpp"
@ -57,11 +55,6 @@ static const std::unordered_map<utils::cli::option, action, option_hasher>
option_actions = {
{utils::cli::options::check_version_option,
cli::actions::check_version},
#if defined(REPERTORY_ENABLE_S3)
{utils::cli::options::create_directory_option,
cli::actions::create_directory},
{utils::cli::options::list_objects_option, cli::actions::list_objects},
#endif
{utils::cli::options::display_config_option,
cli::actions::display_config},
{utils::cli::options::drive_information_option,

80
include/cli/create_directory.hpp
View File

@ -1,80 +0,0 @@
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#ifndef INCLUDE_CLI_CREATE_DIRECTORY_HPP_
#define INCLUDE_CLI_CREATE_DIRECTORY_HPP_
#if defined(REPERTORY_ENABLE_S3)
#include "app_config.hpp"
#include "comm/s3/s3_comm.hpp"
#include "platform/platform.hpp"
#include "providers/i_provider.hpp"
#include "providers/provider.hpp"
#include "rpc/client/client.hpp"
#include "types/repertory.hpp"
#include "utils/cli_utils.hpp"
namespace repertory::cli::actions {
[[nodiscard]] inline auto
create_directory(int argc, char *argv[], const std::string &data_directory,
const provider_type &pt, const std::string &unique_id,
std::string /* user */, std::string /* password */)
-> exit_code {
auto ret = exit_code::invalid_provider_type;
if (pt == provider_type::s3) {
std::string api_path;
if ((ret = utils::cli::parse_string_option(
argc, argv, utils::cli::options::create_directory_option,
api_path)) == exit_code::success) {
lock_data lock(pt, unique_id);
const auto lock_res = lock.grab_lock(1u);
/* if (res == lock_result::locked) { */
/* auto port = app_config::default_api_port(pt); */
/* utils::cli::get_api_authentication_data(user, password, port, pt,
* data_directory); */
/* const auto response = */
/* client({"localhost", password, port,
* user}).create_directory(api_path); */
/* std::cout << static_cast<int>(response.response_type) << std::endl; */
/* std::cout << response.data.dump(2) << std::endl; */
/* } else */
if (lock_res == lock_result::success) {
std::cout << "creating directory: '" << api_path << "'" << std::endl;
app_config config(pt, data_directory);
s3_comm comm(config);
const auto res = comm.create_directory(api_path);
std::cout << api_error_to_string(res) << std::endl;
ret = exit_code::success;
} else {
std::cout << "failed to grab lock: '" << static_cast<int>(lock_res)
<< "'" << std::endl;
ret = exit_code::lock_failed;
}
}
}
return ret;
}
} // namespace repertory::cli::actions
#endif // REPERTORY_ENABLE_S3
#endif // INCLUDE_CLI_CREATE_DIRECTORY_HPP_

5
include/cli/help.hpp
View File

@ -42,11 +42,6 @@ template <typename drive> inline void help(int argc, char *argv[]) {
std::cout << " -na,--name Unique name for S3 "
"instance [Required]"
<< std::endl;
std::cout
<< " -cd,--create_directory [API path] Create directory object in S3"
<< std::endl;
std::cout << " -lo,--list_objects List all S3 objects"
<< std::endl;
#endif // defined(REPERTORY_ENABLE_S3)
std::cout
<< " -gc,--generate_config Generate initial configuration"

71
include/cli/list_objects.hpp
View File

@ -1,71 +0,0 @@
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#ifndef INCLUDE_CLI_LIST_OBJECTS_HPP_
#define INCLUDE_CLI_LIST_OBJECTS_HPP_
#if defined(REPERTORY_ENABLE_S3)
#include "app_config.hpp"
#include "comm/s3/s3_comm.hpp"
#include "platform/platform.hpp"
#include "providers/i_provider.hpp"
#include "providers/provider.hpp"
#include "rpc/client/client.hpp"
#include "types/repertory.hpp"
#include "utils/cli_utils.hpp"
namespace repertory::cli::actions {
[[nodiscard]] inline auto
list_objects(int /* argc */, char * /* argv */[],
const std::string &data_directory, const provider_type &pt,
const std::string &unique_id, std::string /* user */,
std::string /* password */) -> exit_code {
auto ret = exit_code::invalid_provider_type;
if (pt == provider_type::s3) {
lock_data lock(pt, unique_id);
auto lock_res = lock.grab_lock(1u);
if (lock_res == lock_result::success) {
app_config config(pt, data_directory);
s3_comm comm(config);
std::vector<directory_item> list{};
const auto res = comm.get_object_list(list);
if (res == api_error::success) {
for (const auto &di : list) {
std::cout << di.to_json().dump(2) << std::endl;
}
ret = exit_code::success;
} else {
std::cout << api_error_to_string(res) << std::endl;
}
} else {
std::cout << "failed to grab lock: '" << static_cast<int>(lock_res) << "'"
<< std::endl;
ret = exit_code::lock_failed;
}
}
return ret;
}
} // namespace repertory::cli::actions
#endif // REPERTORY_ENABLE_S3
#endif // INCLUDE_CLI_LIST_OBJECTS_HPP_

15
include/comm/curl/requests/http_put_file.hpp
View File

@ -27,14 +27,17 @@
namespace repertory::curl::requests {
struct http_put_file final : http_request_base {
http_put_file() = default;
http_put_file(const http_put_file &) = default;
http_put_file(http_put_file &&) = default;
auto operator=(const http_put_file &) -> http_put_file & = default;
auto operator=(http_put_file &&) -> http_put_file & = default;
~http_put_file() override = default;
std::optional<std::string> encryption_token{};
std::string file_name{};
mutable std::shared_ptr<utils::encryption::encrypting_reader> reader{};
std::string source_path{};
[[nodiscard]] auto get_path() const -> std::string override;
std::shared_ptr<utils::encryption::encrypting_reader> reader;
std::string source_path;
[[nodiscard]] auto set_method(CURL *curl, stop_type &stop_requested) const
-> bool override;

2
include/comm/curl/requests/http_request_base.hpp
View File

@ -54,8 +54,10 @@ struct http_request_base {
http_request_base() = default;
http_request_base(const http_request_base &) = default;
http_request_base(http_request_base &&) = default;
auto operator=(const http_request_base &) -> http_request_base & = default;
auto operator=(http_request_base &&) -> http_request_base & = default;
virtual ~http_request_base() = default;
bool allow_timeout{};

109
include/comm/i_s3_comm.hpp
View File

@ -1,109 +0,0 @@
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#ifndef INCLUDE_COMM_I_S3_COMM_HPP_
#define INCLUDE_COMM_I_S3_COMM_HPP_
#if defined(REPERTORY_ENABLE_S3)
#include "types/repertory.hpp"
#include "types/s3.hpp"
namespace repertory {
class i_provider;
class i_s3_comm {
INTERFACE_SETUP(i_s3_comm);
public:
[[nodiscard]] virtual auto create_directory(const std::string &api_path)
-> api_error = 0;
[[nodiscard]] virtual auto directory_exists(const std::string &api_path) const
-> api_error = 0;
[[nodiscard]] virtual auto file_exists(const std::string &api_path,
const get_key_callback &get_key) const
-> api_error = 0;
[[nodiscard]] virtual auto
get_directory_item_count(const std::string &api_path,
meta_provider_callback meta_provider) const
-> std::size_t = 0;
[[nodiscard]] virtual auto
get_directory_items(const std::string &api_path,
meta_provider_callback meta_provider,
directory_item_list &list) const -> api_error = 0;
[[nodiscard]] virtual auto get_directory_list(api_file_list &list) const
-> api_error = 0;
[[nodiscard]] virtual auto get_file(const std::string &api_path,
const get_key_callback &get_key,
const get_name_callback &get_name,
const get_token_callback &get_token,
api_file &file) const -> api_error = 0;
[[nodiscard]] virtual auto
get_file_list(const get_api_file_token_callback &get_api_file_token,
const get_name_callback &get_name, api_file_list &list) const
-> api_error = 0;
[[nodiscard]] virtual auto
get_object_list(std::vector<directory_item> &list) const -> api_error = 0;
[[nodiscard]] virtual auto
get_object_name(const std::string &api_path,
const get_key_callback &get_key) const -> std::string = 0;
[[nodiscard]] virtual auto get_s3_config() -> s3_config = 0;
[[nodiscard]] virtual auto get_s3_config() const -> s3_config = 0;
[[nodiscard]] virtual auto is_online() const -> bool = 0;
[[nodiscard]] virtual auto read_file_bytes(
const std::string &api_path, std::size_t size, std::uint64_t offset,
data_buffer &data, const get_key_callback &get_key,
const get_size_callback &get_size, const get_token_callback &get_token,
stop_type &stop_requested) const -> api_error = 0;
[[nodiscard]] virtual auto remove_directory(const std::string &api_path)
-> api_error = 0;
[[nodiscard]] virtual auto remove_file(const std::string &api_path,
const get_key_callback &get_key)
-> api_error = 0;
[[nodiscard]] virtual auto rename_file(const std::string &api_path,
const std::string &new_api_path)
-> api_error = 0;
[[nodiscard]] virtual auto
upload_file(const std::string &api_path, const std::string &source_path,
const std::string &encryption_token,
const get_key_callback &get_key, const set_key_callback &set_key,
stop_type &stop_requested) -> api_error = 0;
};
} // namespace repertory
#endif // REPERTORY_ENABLE_S3
#endif // INCLUDE_COMM_I_S3_COMM_HPP_

177
include/comm/s3/s3_comm.hpp
View File

@ -1,177 +0,0 @@
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#ifndef INCLUDE_COMM_S3_S3_COMM_HPP_
#define INCLUDE_COMM_S3_S3_COMM_HPP_
#if defined(REPERTORY_ENABLE_S3)
#include "comm/i_http_comm.hpp"
#include "comm/i_s3_comm.hpp"
#include "types/repertory.hpp"
namespace repertory {
class app_config;
class s3_comm final : public i_s3_comm {
public:
explicit s3_comm(const app_config &config);
s3_comm(s3_comm &&comm);
~s3_comm() override;
private:
struct cache_entry final {
std::chrono::system_clock::time_point expiration;
directory_item_list items;
void reset_timeout(std::chrono::seconds timeout) {
timeout = std::max(std::chrono::seconds(5u), timeout);
expiration = std::chrono::system_clock::now() + timeout;
}
};
private:
const app_config &config_;
s3_config s3_config_;
std::unique_ptr<i_http_comm> s3_client_;
private:
mutable std::recursive_mutex cached_directories_mutex_;
mutable std::unordered_map<std::string, cache_entry> cached_directories_;
protected:
bool active_ = true;
private:
void clear_expired_directories();
[[nodiscard]] auto
get_cached_directory_item_count(const std::string &api_path,
std::size_t &count) const -> bool;
[[nodiscard]] auto
get_cached_directory_items(const std::string &api_path,
meta_provider_callback meta_provider,
directory_item_list &list) const -> bool;
[[nodiscard]] auto get_cached_file_exists(const std::string &api_path) const
-> bool;
[[nodiscard]] auto grab_directory_items(const std::string &api_path,
meta_provider_callback meta_provider,
directory_item_list &list) const
-> api_error;
void raise_begin(const std::string &function_name,
const std::string &api_path) const;
[[nodiscard]] auto raise_end(const std::string &function_name,
const std::string &api_path,
const api_error &error, long code) const
-> api_error;
void remove_cached_directory(const std::string &api_path);
void set_cached_directory_items(const std::string &api_path,
directory_item_list list) const;
public:
[[nodiscard]] auto create_directory(const std::string &api_path)
-> api_error override;
[[nodiscard]] auto directory_exists(const std::string &api_path) const
-> api_error override;
[[nodiscard]] auto file_exists(const std::string &api_path,
const get_key_callback &get_key) const
-> api_error override;
[[nodiscard]] auto
get_directory_item_count(const std::string &api_path,
meta_provider_callback meta_provider) const
-> std::size_t override;
[[nodiscard]] auto get_directory_items(const std::string &api_path,
meta_provider_callback meta_provider,
directory_item_list &list) const
-> api_error override;
[[nodiscard]] auto get_directory_list(api_file_list &list) const
-> api_error override;
[[nodiscard]] auto get_file(const std::string &api_path,
const get_key_callback &get_key,
const get_name_callback &get_name,
const get_token_callback &get_token,
api_file &file) const -> api_error override;
[[nodiscard]] auto
get_file_list(const get_api_file_token_callback &get_api_file_token,
const get_name_callback &get_name, api_file_list &list) const
-> api_error override;
[[nodiscard]] auto get_object_list(std::vector<directory_item> &list) const
-> api_error override;
[[nodiscard]] auto get_object_name(const std::string &api_path,
const get_key_callback &getKey) const
-> std::string override;
[[nodiscard]] auto get_s3_config() -> s3_config override {
return s3_config_;
}
[[nodiscard]] auto get_s3_config() const -> s3_config override {
return s3_config_;
}
[[nodiscard]] auto is_online() const -> bool override {
// TODO implement this
return true;
}
[[nodiscard]] auto read_file_bytes(
const std::string &api_path, std::size_t size, std::uint64_t offset,
data_buffer &data, const get_key_callback &get_key,
const get_size_callback &get_size, const get_token_callback &get_token,
stop_type &stop_requested) const -> api_error override;
[[nodiscard]] auto remove_directory(const std::string &api_path)
-> api_error override;
[[nodiscard]] auto remove_file(const std::string &api_path,
const get_key_callback &get_key)
-> api_error override;
[[nodiscard]] auto rename_file(const std::string &api_path,
const std::string &new_api_path)
-> api_error override;
[[nodiscard]] auto
upload_file(const std::string &api_path, const std::string &source_path,
const std::string &encryption_token,
const get_key_callback &get_key, const set_key_callback &set_key,
stop_type &stop_requested) -> api_error override;
};
} // namespace repertory
#endif // REPERTORY_ENABLE_S3
#endif // INCLUDE_COMM_S3_S3_COMM_HPP_

117
include/db/directory_db.hpp
View File

@ -1,117 +0,0 @@
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#ifndef INCLUDE_DB_DIRECTORY_DB_HPP_
#define INCLUDE_DB_DIRECTORY_DB_HPP_
#include "app_config.hpp"
#include "types/repertory.hpp"
#include "utils/rocksdb_utils.hpp"
namespace repertory {
class directory_db final {
private:
class directory_tree final {
private:
std::unordered_map<std::string, std::vector<std::string>>
sub_directory_lookup_;
public:
void add_path(const std::string &api_path,
const std::vector<std::string> &files, rocksdb::DB &db);
[[nodiscard]] auto get_count(const std::string &api_path) const
-> std::size_t;
[[nodiscard]] auto get_directories() const -> std::vector<std::string>;
[[nodiscard]] auto get_sub_directories(const std::string &api_path) const
-> std::vector<std::string>;
[[nodiscard]] auto is_directory(const std::string &api_path) const -> bool;
void remove_directory(const std::string &api_path, rocksdb::DB &db,
bool allow_remove_root = false);
};
public:
explicit directory_db(const app_config &config);
public:
~directory_db();
private:
mutable std::recursive_mutex directory_mutex_;
std::unique_ptr<rocksdb::DB> db_;
directory_tree tree_;
const std::string DIRDB_NAME = "directory_db";
private:
[[nodiscard]] auto get_directory_data(const std::string &api_path) const
-> json;
public:
[[nodiscard]] auto create_directory(const std::string &api_path,
bool create_always = false) -> api_error;
[[nodiscard]] auto create_file(const std::string &api_path) -> api_error;
[[nodiscard]] auto get_directory_item_count(const std::string &api_path) const
-> std::uint64_t;
[[nodiscard]] auto
get_file(const std::string &api_path, api_file &file,
api_file_provider_callback api_file_provider) const -> api_error;
[[nodiscard]] auto
get_file_list(api_file_list &list,
api_file_provider_callback api_file_provider) const
-> api_error;
[[nodiscard]] auto get_sub_directory_count(const std::string &api_path) const
-> std::size_t;
[[nodiscard]] auto get_total_item_count() const -> std::uint64_t;
[[nodiscard]] auto is_directory(const std::string &api_path) const -> bool;
[[nodiscard]] auto is_file(const std::string &api_path) const -> bool;
void populate_directory_files(const std::string &api_path,
meta_provider_callback meta_provider,
directory_item_list &list) const;
void populate_sub_directories(const std::string &api_path,
meta_provider_callback meta_provider,
directory_item_list &list) const;
[[nodiscard]] auto remove_directory(const std::string &api_path,
bool allow_remove_root = false)
-> api_error;
[[nodiscard]] auto remove_file(const std::string &api_path) -> bool;
[[nodiscard]] auto rename_file(const std::string &from_api_path,
const std::string &to_api_path) -> api_error;
};
} // namespace repertory
#endif // INCLUDE_DB_DIRECTORY_DB_HPP_

109
include/db/meta_db.hpp
View File

@ -1,109 +0,0 @@
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#ifndef INCLUDE_DB_META_DB_HPP_
#define INCLUDE_DB_META_DB_HPP_
#include "app_config.hpp"
#include "types/repertory.hpp"
#include "utils/rocksdb_utils.hpp"
namespace repertory {
class meta_db final {
public:
explicit meta_db(const app_config &config);
public:
~meta_db();
private:
std::unique_ptr<rocksdb::DB> db_;
rocksdb::ColumnFamilyHandle *default_family_{};
rocksdb::ColumnFamilyHandle *source_family_{};
rocksdb::ColumnFamilyHandle *keys_family_{};
const std::string METADB_NAME = "meta_db";
private:
[[nodiscard]] auto
perform_action(const std::string &function_name,
const std::function<rocksdb::Status()> &action) const
-> api_error;
[[nodiscard]] auto get_item_meta_json(const std::string &api_path,
json &json_data) const -> api_error;
[[nodiscard]] auto store_item_meta(const std::string &api_path,
const std::string &key,
const std::string &value) -> api_error;
public:
[[nodiscard]] auto create_iterator(bool source_family) const
-> std::shared_ptr<rocksdb::Iterator>;
[[nodiscard]] auto get_api_path_from_key(const std::string &key,
std::string &api_path) const
-> api_error;
[[nodiscard]] auto get_api_path_from_source(const std::string &source_path,
std::string &api_path) const
-> api_error;
[[nodiscard]] auto get_item_meta(const std::string &api_path,
api_meta_map &meta) const -> api_error;
[[nodiscard]] auto get_item_meta(const std::string &api_path,
const std::string &key,
std::string &value) const -> api_error;
[[nodiscard]] auto get_item_meta_exists(const std::string &api_path) const
-> bool;
[[nodiscard]] auto get_total_item_count() const -> std::uint64_t;
[[nodiscard]] auto get_pinned_files() const -> std::vector<std::string>;
[[nodiscard]] auto
get_source_path_exists(const std::string &source_path) const -> bool;
[[nodiscard]] auto remove_item_meta(const std::string &api_path) -> api_error;
[[nodiscard]] auto remove_item_meta(const std::string &api_path,
const std::string &key) -> api_error;
[[nodiscard]] auto rename_item_meta(const std::string &source_path,
const std::string &from_api_path,
const std::string &to_api_path)
-> api_error;
[[nodiscard]] auto set_item_meta(const std::string &api_path,
const std::string &key,
const std::string &value) -> api_error;
[[nodiscard]] auto set_item_meta(const std::string &api_path,
const api_meta_map &meta) -> api_error;
[[nodiscard]] auto set_source_path(const std::string &api_path,
const std::string &source_path)
-> api_error;
};
} // namespace repertory
#endif // INCLUDE_DB_META_DB_HPP_

9
include/platform/unix_platform.hpp
View File

@ -68,11 +68,10 @@ public:
[[nodiscard]] auto create_meta_attributes(
std::uint64_t accessed_date, std::uint32_t attributes,
std::uint64_t changed_date, std::uint64_t creation_date, bool directory,
const std::string &encryption_token, std::uint32_t gid,
const std::string &key, std::uint32_t mode, std::uint64_t modified_date,
std::uint32_t osx_backup, std::uint32_t osx_flags, std::uint64_t size,
const std::string &source_path, std::uint32_t uid,
std::uint64_t written_date) -> api_meta_map;
std::uint32_t gid, const std::string &key, std::uint32_t mode,
std::uint64_t modified_date, std::uint32_t osx_backup,
std::uint32_t osx_flags, std::uint64_t size, const std::string &source_path,
std::uint32_t uid, std::uint64_t written_date) -> api_meta_map;
[[nodiscard]] auto provider_meta_handler(i_provider &provider, bool directory,
const api_file &file) -> api_error;

9
include/platform/win32_platform.hpp
View File

@ -73,11 +73,10 @@ public:
[[nodiscard]] auto create_meta_attributes(
std::uint64_t accessed_date, std::uint32_t attributes,
std::uint64_t changed_date, std::uint64_t creation_date, bool directory,
const std::string &encryption_token, std::uint32_t gid,
const std::string &key, std::uint32_t mode, std::uint64_t modified_date,
std::uint32_t osx_backup, std::uint32_t osx_flags, std::uint64_t size,
const std::string &source_path, std::uint32_t uid,
std::uint64_t written_date) -> api_meta_map;
std::uint32_t gid, const std::string &key, std::uint32_t mode,
std::uint64_t modified_date, std::uint32_t osx_backup,
std::uint32_t osx_flags, std::uint64_t size, const std::string &source_path,
std::uint32_t uid, std::uint64_t written_date) -> api_meta_map;
[[nodiscard]] auto provider_meta_handler(i_provider &provider, bool directory,
const api_file &file) -> api_error;

205
include/providers/base_provider.hpp
View File

@ -1,205 +0,0 @@
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#ifndef INCLUDE_PROVIDERS_BASE_PROVIDER_HPP_
#define INCLUDE_PROVIDERS_BASE_PROVIDER_HPP_
#include "db/meta_db.hpp"
#include "providers/i_provider.hpp"
namespace repertory {
class app_config;
class i_file_manager;
class base_provider : public i_provider {
public:
explicit base_provider(app_config &config);
~base_provider() override = default;
private:
app_config &config_;
std::atomic<std::uint64_t> used_space_{0U};
protected:
api_item_added_callback api_item_added_;
std::unique_ptr<meta_db> meta_db_;
mutable std::recursive_mutex notify_added_mutex_;
i_file_manager *fm_ = nullptr;
stop_type stop_requested_ = false;
protected:
void calculate_used_drive_space(bool add_missing);
[[nodiscard]] virtual auto
check_file_exists(const std::string &api_path) const -> api_error = 0;
void cleanup();
[[nodiscard]] auto get_config() -> app_config & { return config_; }
[[nodiscard]] auto get_config() const -> app_config & { return config_; }
[[nodiscard]] virtual auto
handle_rename_file(const std::string &from_api_path,
const std::string &to_api_path,
const std::string &source_path) -> api_error = 0;
[[nodiscard]] auto notify_directory_added(const std::string &api_path,
const std::string &api_parent) const
-> api_error {
return const_cast<base_provider *>(this)->notify_directory_added(
api_path, api_parent);
}
[[nodiscard]] virtual auto
notify_directory_added(const std::string &api_path,
const std::string &api_parent) -> api_error;
[[nodiscard]] auto notify_file_added(const std::string &api_path,
const std::string &api_parent,
std::uint64_t size) const -> api_error {
return const_cast<base_provider *>(this)->notify_file_added(
api_path, api_parent, size);
}
[[nodiscard]] virtual auto notify_file_added(const std::string &api_path,
const std::string &api_parent,
std::uint64_t size)
-> api_error = 0;
[[nodiscard]] virtual auto
populate_directory_items(const std::string &api_path,
directory_item_list &list) const -> api_error = 0;
[[nodiscard]] virtual auto populate_file(const std::string &api_path,
api_file &file) const
-> api_error = 0;
auto processed_orphaned_file(const std::string &source_path,
const std::string &api_path = "") const -> bool;
void remove_deleted_files();
void remove_expired_orphaned_files();
void remove_unknown_source_files();
[[nodiscard]] auto remove_item_meta(const std::string &api_path)
-> api_error {
return meta_db_->remove_item_meta(api_path);
}
void update_filesystem_item(bool directory, const api_error &error,
const std::string &api_path,
filesystem_item &fsi) const;
public:
[[nodiscard]] auto
create_directory_clone_source_meta(const std::string &source_api_path,
const std::string &api_path)
-> api_error override;
[[nodiscard]] auto create_file(const std::string &api_path,
api_meta_map &meta) -> api_error override;
[[nodiscard]] auto get_api_path_from_source(const std::string &source_path,
std::string &api_path) const
-> api_error override;
[[nodiscard]] auto get_directory_items(const std::string &api_path,
directory_item_list &list) const
-> api_error override;
[[nodiscard]] auto get_file(const std::string &api_path, api_file &file) const
-> api_error override;
[[nodiscard]] auto get_file_size(const std::string &api_path,
std::uint64_t &file_size) const
-> api_error override;
[[nodiscard]] auto get_filesystem_item(const std::string &api_path,
bool directory,
filesystem_item &fsi) const
-> api_error override;
[[nodiscard]] auto get_filesystem_item_and_file(const std::string &api_path,
api_file &file,
filesystem_item &fsi) const
-> api_error override;
[[nodiscard]] auto
get_filesystem_item_from_source_path(const std::string &source_path,
filesystem_item &fsi) const
-> api_error override;
[[nodiscard]] auto get_item_meta(const std::string &api_path,
api_meta_map &meta) const
-> api_error override;
[[nodiscard]] auto get_item_meta(const std::string &api_path,
const std::string &key,
std::string &value) const
-> api_error override;
[[nodiscard]] auto get_pinned_files() const
-> std::vector<std::string> override {
return meta_db_->get_pinned_files();
}
[[nodiscard]] auto get_used_drive_space() const -> std::uint64_t override;
[[nodiscard]] auto is_file_writeable(const std::string &) const
-> bool override {
return true;
}
[[nodiscard]] auto remove_item_meta(const std::string &api_path,
const std::string &key)
-> api_error override {
return meta_db_->remove_item_meta(api_path, key);
}
[[nodiscard]] auto rename_file(const std::string &from_api_path,
const std::string &to_api_path)
-> api_error override;
[[nodiscard]] auto set_item_meta(const std::string &api_path,
const std::string &key,
const std::string &value)
-> api_error override {
return meta_db_->set_item_meta(api_path, key, value);
}
[[nodiscard]] auto set_item_meta(const std::string &api_path,
const api_meta_map &meta)
-> api_error override {
return meta_db_->set_item_meta(api_path, meta);
}
[[nodiscard]] auto start(api_item_added_callback api_item_added,
i_file_manager *fm) -> bool override;
void stop() override;
};
} // namespace repertory
#endif // INCLUDE_PROVIDERS_BASE_PROVIDER_HPP_

11
include/providers/encrypt/encrypt_provider.hpp
View File

@ -33,6 +33,12 @@ public:
~encrypt_provider() override = default;
public:
encrypt_provider(const encrypt_provider &) = delete;
encrypt_provider(encrypt_provider &&) = delete;
auto operator=(const encrypt_provider &) -> encrypt_provider & = delete;
auto operator=(encrypt_provider &&) -> encrypt_provider & = delete;
private:
struct reader_info final {
std::chrono::system_clock::time_point last_access_time =
@ -47,7 +53,7 @@ private:
rocksdb::ColumnFamilyHandle *dir_family_{};
rocksdb::ColumnFamilyHandle *file_family_{};
rocksdb::ColumnFamilyHandle *source_family_{};
const std::string DB_NAME = "meta_db";
std::string DB_NAME = "meta_db";
private:
i_file_manager *fm_ = nullptr;
@ -56,7 +62,7 @@ private:
std::recursive_mutex reader_lookup_mtx_{};
private:
static auto create_api_file(const std::string api_path, bool directory,
static auto create_api_file(const std::string &api_path, bool directory,
const std::string &source_path) -> api_file;
static void create_item_meta(api_meta_map &meta, bool directory,
@ -207,7 +213,6 @@ public:
[[nodiscard]] auto upload_file(const std::string & /*api_path*/,
const std::string & /*source_path*/,
const std::string & /*encryption_token*/,
stop_type & /*stop_requested*/)
-> api_error override {
return api_error::not_implemented;

6
include/providers/i_provider.hpp
View File

@ -147,9 +147,9 @@ public:
virtual void stop() = 0;
[[nodiscard]] virtual auto
upload_file(const std::string &api_path, const std::string &source_path,
const std::string &encryption_token, stop_type &stop_requested)
[[nodiscard]] virtual auto upload_file(const std::string &api_path,
const std::string &source_path,
stop_type &stop_requested)
-> api_error = 0;
};
} // namespace repertory

155
include/providers/s3/s3_provider.hpp
View File

@ -23,86 +23,129 @@
#define INCLUDE_PROVIDERS_S3_S3_PROVIDER_HPP_
#if defined(REPERTORY_ENABLE_S3)
#include "db/directory_db.hpp"
#include "providers/base_provider.hpp"
#include "providers/i_provider.hpp"
#include "types/repertory.hpp"
namespace repertory {
class app_config;
class i_file_manager;
class i_s3_comm;
class i_http_comm;
struct head_object_result;
class s3_provider final : public base_provider {
class s3_provider final : public i_provider {
public:
s3_provider(app_config &config, i_s3_comm &s3_comm);
s3_provider(app_config &config, i_http_comm &comm);
~s3_provider() override = default;
private:
i_s3_comm &s3_comm_;
public:
s3_provider(const s3_provider &) = delete;
s3_provider(s3_provider &&) = delete;
auto operator=(const s3_provider &) -> s3_provider & = delete;
auto operator=(s3_provider &&) -> s3_provider & = delete;
private:
std::unique_ptr<directory_db> directory_db_;
std::unique_ptr<std::thread> background_thread_;
app_config &config_;
i_http_comm &comm_;
private:
void create_directories();
api_item_added_callback api_item_added_;
std::unique_ptr<rocksdb::DB> db_;
std::string DB_NAME = "meta_db";
i_file_manager *fm_{};
void create_parent_directories(const api_file &file, bool directory);
private:
[[nodiscard]] auto add_if_not_found(api_file &file,
const std::string &object_name) const
-> api_error;
void update_item_meta(directory_item &di) const;
[[nodiscard]] static auto create_api_file(const std::string &path,
const std::string &key,
std::uint64_t size) -> api_file;
protected:
[[nodiscard]] auto check_file_exists(const std::string &api_path) const
-> api_error override;
[[nodiscard]] auto create_path_directories(const std::string &api_path,
const std::string &key) const
-> api_error;
[[nodiscard]] auto handle_rename_file(const std::string &from_api_path,
const std::string &to_api_path,
const std::string &source_path)
-> api_error override;
[[nodiscard]] auto decrypt_object_name(std::string &object_name) const
-> api_error;
[[nodiscard]] auto notify_directory_added(const std::string &api_path,
const std::string &api_parent)
-> api_error override;
[[nodiscard]] auto
get_object_info(const std::string &api_path, bool &is_encrypted,
std::string &object_name, head_object_result &result) const
-> api_error;
[[nodiscard]] auto notify_file_added(const std::string &api_path,
const std::string &api_parent,
std::uint64_t size)
-> api_error override;
[[nodiscard]] auto populate_directory_items(const std::string &api_path,
directory_item_list &list) const
-> api_error override;
[[nodiscard]] auto populate_file(const std::string &api_path,
api_file &file) const -> api_error override;
void remove_deleted_files();
public:
#ifdef REPERTORY_TESTING
void set_callback(api_item_added_callback cb) { api_item_added_ = cb; }
#endif // REPERTORY_TESTING
[[nodiscard]] auto create_directory(const std::string &api_path,
api_meta_map &meta) -> api_error override;
[[nodiscard]] auto
create_directory_clone_source_meta(const std::string &source_api_path,
const std::string &api_path)
-> api_error override;
[[nodiscard]] auto create_file(const std::string &api_path,
api_meta_map &meta) -> api_error override;
[[nodiscard]] auto get_api_path_from_source(const std::string &source_path,
std::string &api_path) const
-> api_error override;
[[nodiscard]] auto get_directory_item_count(const std::string &api_path) const
-> std::uint64_t override;
[[nodiscard]] auto get_directory_items(const std::string &api_path,
directory_item_list &list) const
-> api_error override;
[[nodiscard]] auto get_file(const std::string &api_path, api_file &file) const
-> api_error override;
[[nodiscard]] auto get_file_list(api_file_list &list) const
-> api_error override;
[[nodiscard]] auto get_file_size(const std::string &api_path,
std::uint64_t &file_size) const
-> api_error override;
[[nodiscard]] auto get_filesystem_item(const std::string &api_path,
bool directory,
filesystem_item &fsi) const
-> api_error override;
[[nodiscard]] auto get_filesystem_item_and_file(const std::string &api_path,
api_file &file,
filesystem_item &fsi) const
-> api_error override;
[[nodiscard]] auto
get_filesystem_item_from_source_path(const std::string &source_path,
filesystem_item &fsi) const
-> api_error override;
[[nodiscard]] auto get_pinned_files() const
-> std::vector<std::string> override;
[[nodiscard]] auto get_item_meta(const std::string &api_path,
api_meta_map &meta) const
-> api_error override;
[[nodiscard]] auto get_item_meta(const std::string &api_path,
const std::string &key,
std::string &value) const
-> api_error override;
[[nodiscard]] auto get_total_drive_space() const -> std::uint64_t override;
[[nodiscard]] auto get_total_item_count() const -> std::uint64_t override;
[[nodiscard]] auto get_provider_type() const -> provider_type override {
return provider_type::s3;
}
[[nodiscard]] auto get_total_drive_space() const -> std::uint64_t override {
return std::numeric_limits<std::int64_t>::max() / std::int64_t(2);
}
[[nodiscard]] auto get_total_item_count() const -> std::uint64_t override;
[[nodiscard]] auto get_used_drive_space() const -> std::uint64_t override;
[[nodiscard]] auto is_direct_only() const -> bool override { return false; }
@ -112,11 +155,14 @@ public:
[[nodiscard]] auto is_file(const std::string &api_path, bool &exists) const
-> api_error override;
[[nodiscard]] auto is_file_writeable(const std::string &api_path) const
-> bool override;
[[nodiscard]] auto is_online() const -> bool override;
[[nodiscard]] auto is_rename_supported() const -> bool override {
return false;
}
};
[[nodiscard]] auto read_file_bytes(const std::string &api_path,
std::size_t size, std::uint64_t offset,
@ -130,14 +176,31 @@ public:
[[nodiscard]] auto remove_file(const std::string &api_path)
-> api_error override;
[[nodiscard]] auto remove_item_meta(const std::string &api_path,
const std::string &key)
-> api_error override;
[[nodiscard]] auto rename_file(const std::string &from_api_path,
const std::string &to_api_path)
-> api_error override;
[[nodiscard]] auto set_item_meta(const std::string &api_path,
const std::string &key,
const std::string &value)
-> api_error override;
[[nodiscard]] auto set_item_meta(const std::string &api_path,
const api_meta_map &meta)
-> api_error override;
[[nodiscard]] auto start(api_item_added_callback api_item_added,
i_file_manager *fm) -> bool override;
void stop() override;
[[nodiscard]] auto
upload_file(const std::string &api_path, const std::string &source_path,
const std::string &encryption_token, stop_type &stop_requested)
[[nodiscard]] auto upload_file(const std::string &api_path,
const std::string &source_path,
stop_type &stop_requested)
-> api_error override;
};
} // namespace repertory

13
include/providers/sia/sia_provider.hpp
View File

@ -36,15 +36,21 @@ public:
~sia_provider() override = default;
public:
sia_provider(const sia_provider &) = delete;
sia_provider(sia_provider &&) = delete;
auto operator=(const sia_provider &) -> sia_provider & = delete;
auto operator=(sia_provider &&) -> sia_provider & = delete;
private:
app_config &config_;
i_http_comm &comm_;
private:
const std::string DB_NAME = "meta_db";
api_item_added_callback api_item_added_;
std::unique_ptr<rocksdb::DB> db_;
i_file_manager *fm_ = nullptr;
std::string DB_NAME = "meta_db";
i_file_manager *fm_{};
private:
[[nodiscard]] static auto create_api_file(std::string path,
@ -57,7 +63,7 @@ private:
[[nodiscard]] auto get_object_info(const std::string &api_path,
json &object_info) const -> api_error;
[[nodiscard]] auto get_object_list(const std::string api_path,
[[nodiscard]] auto get_object_list(const std::string &api_path,
nlohmann::json &object_list) const -> bool;
void remove_deleted_files();
@ -185,7 +191,6 @@ public:
[[nodiscard]] auto upload_file(const std::string &api_path,
const std::string &source_path,
const std::string & /* encryption_token */,
stop_type &stop_requested)
-> api_error override;
};

113
include/types/repertory.hpp
View File

@ -29,7 +29,6 @@ const std::string META_BACKUP = "backup";
const std::string META_CHANGED = "changed";
const std::string META_CREATION = "creation";
const std::string META_DIRECTORY = "directory";
const std::string META_ENCRYPTION_TOKEN = "token";
const std::string META_GID = "gid";
const std::string META_ID = "id";
const std::string META_KEY = "key";
@ -43,24 +42,10 @@ const std::string META_UID = "uid";
const std::string META_WRITTEN = "written";
const std::vector<std::string> META_USED_NAMES = {
META_ACCESSED,
META_ATTRIBUTES,
META_BACKUP,
META_CHANGED,
META_CREATION,
META_DIRECTORY,
META_ENCRYPTION_TOKEN,
META_GID,
META_ID,
META_KEY,
META_MODE,
META_MODIFIED,
META_OSXFLAGS,
META_PINNED,
META_SIZE,
META_SOURCE,
META_UID,
META_WRITTEN,
META_ACCESSED, META_ATTRIBUTES, META_BACKUP, META_CHANGED, META_CREATION,
META_DIRECTORY, META_GID, META_ID, META_KEY, META_MODE,
META_MODIFIED, META_OSXFLAGS, META_PINNED, META_SIZE, META_SOURCE,
META_UID, META_WRITTEN,
};
using api_meta_map = std::map<std::string, std::string>;
@ -141,6 +126,7 @@ enum class exit_code : std::int32_t {
enum http_error_codes : std::int32_t {
ok = 200,
multiple_choices = 300,
not_found = 404,
};
@ -154,39 +140,37 @@ enum class provider_type : std::size_t {
sia,
remote,
s3,
passthrough,
encrypt,
unknown,
};
#ifdef _WIN32
struct open_file_data {
void *directory_buffer = nullptr;
void *directory_buffer{};
};
#else
using open_file_data = int;
#endif
struct api_file {
std::string api_path{};
std::string api_parent{};
std::uint64_t accessed_date = 0u;
std::uint64_t changed_date = 0u;
std::uint64_t creation_date = 0u;
std::string encryption_token{};
std::uint64_t file_size = 0u;
std::string key{};
std::uint64_t modified_date = 0u;
std::string source_path{};
std::string api_path;
std::string api_parent;
std::uint64_t accessed_date{};
std::uint64_t changed_date{};
std::uint64_t creation_date{};
std::uint64_t file_size{};
std::string key;
std::uint64_t modified_date{};
std::string source_path;
};
struct directory_item {
std::string api_path{};
std::string api_parent{};
bool directory = false;
std::uint64_t size = 0u;
api_meta_map meta{};
bool resolved = false;
std::string api_path;
std::string api_parent;
bool directory{false};
std::uint64_t size{};
api_meta_map meta;
bool resolved{false};
[[nodiscard]] static auto from_json(const json &item) -> directory_item {
directory_item ret{};
@ -208,27 +192,22 @@ struct directory_item {
};
struct filesystem_item {
std::string api_path{};
std::string api_parent{};
bool directory = false;
std::string encryption_token{};
std::uint64_t size = 0u;
std::string source_path{};
[[nodiscard]] auto is_encrypted() const -> bool {
return not encryption_token.empty();
}
std::string api_path;
std::string api_parent;
bool directory{false};
std::uint64_t size{};
std::string source_path;
};
struct host_config {
std::string agent_string{};
std::string api_password{};
std::string api_user{};
std::uint16_t api_port = 0u;
std::string host_name_or_ip = "localhost";
std::string agent_string;
std::string api_password;
std::string api_user;
std::uint16_t api_port{};
std::string host_name_or_ip{"localhost"};
std::string path{};
std::string protocol = "http";
std::uint32_t timeout_ms = 60000u;
std::string protocol{"http"};
std::uint32_t timeout_ms{60000U};
auto operator==(const host_config &hc) const noexcept -> bool {
if (&hc != this) {
@ -280,29 +259,25 @@ from_json(const json &j, host_config &hc) {
}
struct http_range {
std::uint64_t begin = 0u;
std::uint64_t end = 0u;
};
struct passthrough_config {
std::string location{};
std::string name{};
std::uint64_t begin;
std::uint64_t end;
};
struct encrypt_config {
std::string encryption_token{};
std::string path{};
std::string encryption_token;
std::string path;
};
struct s3_config {
std::string access_key{};
std::string bucket{};
std::uint16_t cache_timeout_secs = 60u;
std::string encryption_token{};
std::string access_key;
std::string bucket;
std::uint16_t cache_timeout_secs{60U};
std::string encryption_token;
std::string region = "any";
std::string secret_key{};
std::uint32_t timeout_ms = 60000u;
std::string secret_key;
std::uint32_t timeout_ms{60000U};
std::string url;
bool use_path_style{false};
bool use_region_in_url{false};
};

4
include/utils/cli_utils.hpp
View File

@ -34,8 +34,6 @@ static const option data_directory_option = {"-dd", "--data_directory"};
static const option encrypt_option = {"-en", "--encrypt"};
static const option drive_information_option = {"-di", "--drive_information"};
#if defined(REPERTORY_ENABLE_S3)
static const option create_directory_option = {"-cd", "--create_directory"};
static const option list_objects_option = {"-lo", "--list_objects"};
static const option name_option = {"-na", "--name"};
static const option s3_option = {"-s3", "--s3"};
#endif // defined(REPERTORY_ENABLE_S3)
@ -67,8 +65,6 @@ static const std::vector<option> option_list = {
#if defined(REPERTORY_ENABLE_S3)
s3_option,
name_option,
create_directory_option,
list_objects_option,
#endif // defined(REPERTORY_ENABLE_S3)
generate_config_option,
get_option,

30
include/utils/encrypting_reader.hpp
View File

@ -30,11 +30,15 @@ using key_type = std::array<unsigned char, 32U>;
class encrypting_reader final {
public:
encrypting_reader(
const std::string &file_name, const std::string &source_path,
stop_type &stop_requested, const std::string &token,
std::optional<std::string> relative_parent_path = std::nullopt,
const size_t error_return = 0);
encrypting_reader(const std::string &file_name,
const std::string &source_path, stop_type &stop_requested,
const std::string &token,
std::optional<std::string> relative_parent_path,
std::size_t error_return = 0U);
encrypting_reader(const std::string &encrypted_file_path,
const std::string &source_path, stop_type &stop_requested,
const std::string &token, std::size_t error_return = 0U);
encrypting_reader(
const std::string &encrypted_file_path, const std::string &source_path,
@ -42,9 +46,13 @@ public:
std::vector<std::array<unsigned char,
crypto_aead_xchacha20poly1305_IETF_NPUBBYTES>>
iv_list,
const size_t error_return = 0);
std::size_t error_return = 0U);
encrypting_reader(const encrypting_reader &r);
encrypting_reader(const encrypting_reader &reader);
encrypting_reader(encrypting_reader &&) = delete;
auto operator=(const encrypting_reader &) -> encrypting_reader & = delete;
auto operator=(encrypting_reader &&) -> encrypting_reader & = delete;
~encrypting_reader();
@ -62,11 +70,11 @@ private:
std::vector<
std::array<unsigned char, crypto_aead_xchacha20poly1305_IETF_NPUBBYTES>>
iv_list_;
std::size_t last_data_chunk_ = 0u;
std::size_t last_data_chunk_size_ = 0u;
std::uint64_t read_offset_ = 0u;
std::size_t last_data_chunk_{};
std::size_t last_data_chunk_size_{};
std::uint64_t read_offset_{};
native_file_ptr source_file_;
std::uint64_t total_size_ = 0u;
std::uint64_t total_size_{};
private:
static const std::size_t header_size_;

192
src/app_config.cpp
View File

@ -29,19 +29,19 @@
#include "utils/utils.hpp"
namespace repertory {
app_config::app_config(const provider_type &pt,
app_config::app_config(const provider_type &prov,
const std::string &data_directory)
: pt_(pt),
api_auth_(utils::generate_random_string(48u)),
api_port_(default_rpc_port(pt)),
: prov_(prov),
api_auth_(utils::generate_random_string(48U)),
api_port_(default_rpc_port(prov)),
api_user_("repertory"),
config_changed_(false),
data_directory_(
data_directory.empty() ? default_data_directory(pt)
: ((pt == provider_type::remote) || (pt == provider_type::s3))
data_directory.empty() ? default_data_directory(prov)
: ((prov == provider_type::remote) || (prov == provider_type::s3))
? utils::path::absolute(data_directory)
: utils::path::absolute(utils::path::combine(
data_directory, {get_provider_name(pt)}))),
data_directory, {get_provider_name(prov)}))),
download_timeout_secs_(30),
enable_chunk_downloader_timeout_(true),
enable_comm_duration_events_(false),
@ -58,7 +58,7 @@ app_config::app_config(const provider_type &pt,
is_remote_mount_(false),
low_freq_interval_secs_(60 * 60),
max_cache_size_bytes_(20 * 1024 * 1024 * 1024ULL),
max_upload_count_(5u),
max_upload_count_(5U),
min_download_timeout_secs_(5),
online_check_retry_secs_(60),
orphaned_file_retention_days_(15),
@ -68,10 +68,10 @@ app_config::app_config(const provider_type &pt,
remote_client_pool_size_(10),
remote_host_name_or_ip_(""),
remote_max_connections_(20),
remote_port_((pt == provider_type::sia) ? 20000
: (pt == provider_type::s3) ? 20001
: (pt == provider_type::encrypt) ? 20002
: 20003),
remote_port_((prov == provider_type::sia) ? 20000
: (prov == provider_type::s3) ? 20001
: (prov == provider_type::encrypt) ? 20002
: 20003),
remote_receive_timeout_secs_(120),
remote_send_timeout_secs_(30),
remote_token_(""),
@ -80,18 +80,21 @@ app_config::app_config(const provider_type &pt,
cache_directory_ = utils::path::combine(data_directory_, {"cache"});
log_directory_ = utils::path::combine(data_directory_, {"logs"});
hc_.agent_string = default_agent_name(pt_);
hc_.api_password = get_provider_api_password(pt_);
hc_.api_port = default_api_port(pt_);
hc_.agent_string = default_agent_name(prov_);
hc_.api_password = get_provider_api_password(prov_);
hc_.api_port = default_api_port(prov_);
if (not utils::file::create_full_directory_path(data_directory_))
if (not utils::file::create_full_directory_path(data_directory_)) {
throw startup_exception("unable to create: " + data_directory_);
}
if (not utils::file::create_full_directory_path(cache_directory_))
if (not utils::file::create_full_directory_path(cache_directory_)) {
throw startup_exception("unable to create: " + cache_directory_);
}
if (not utils::file::create_full_directory_path(log_directory_))
if (not utils::file::create_full_directory_path(log_directory_)) {
throw startup_exception("unable to create: " + log_directory_);
}
if (not load()) {
save();
@ -102,48 +105,51 @@ auto app_config::get_config_file_path() const -> std::string {
return utils::path::combine(data_directory_, {"config.json"});
}
auto app_config::default_agent_name(const provider_type &pt) -> std::string {
auto app_config::default_agent_name(const provider_type &prov) -> std::string {
static const std::array<std::string,
static_cast<std::size_t>(provider_type::unknown)>
PROVIDER_AGENT_NAMES = {"Sia-Agent", "", "", "", ""};
PROVIDER_AGENT_NAMES = {"Sia-Agent", "", "", ""};
return PROVIDER_AGENT_NAMES[static_cast<std::size_t>(pt)];
return PROVIDER_AGENT_NAMES[static_cast<std::size_t>(prov)];
}
auto app_config::default_api_port(const provider_type &pt) -> std::uint16_t {
auto app_config::default_api_port(const provider_type &prov) -> std::uint16_t {
static const std::array<std::uint16_t,
static_cast<std::size_t>(provider_type::unknown)>
PROVIDER_API_PORTS = {9980u, 0u, 0u, 0u, 0u};
return PROVIDER_API_PORTS[static_cast<std::size_t>(pt)];
PROVIDER_API_PORTS = {9980U, 0U, 0U, 0U};
return PROVIDER_API_PORTS[static_cast<std::size_t>(prov)];
}
auto app_config::default_data_directory(const provider_type &pt)
auto app_config::default_data_directory(const provider_type &prov)
-> std::string {
#ifdef _WIN32
auto data_directory = utils::path::combine(
utils::get_local_app_data_directory(),
{REPERTORY_DATA_NAME, app_config::get_provider_name(pt)});
{REPERTORY_DATA_NAME, app_config::get_provider_name(prov)});
#else
#ifdef __APPLE__
auto data_directory = utils::path::resolve(
std::string("~/Library/Application Support/") + REPERTORY_DATA_NAME +
'/' + app_config::get_provider_name(pt));
'/' + app_config::get_provider_name(prov));
#else
auto data_directory =
utils::path::resolve(std::string("~/.local/") + REPERTORY_DATA_NAME +
'/' + app_config::get_provider_name(pt));
'/' + app_config::get_provider_name(prov));
#endif
#endif
return data_directory;
}
auto app_config::default_rpc_port(const provider_type &pt) -> std::uint16_t {
auto app_config::default_rpc_port(const provider_type &prov) -> std::uint16_t {
static const std::array<std::uint16_t,
static_cast<std::size_t>(provider_type::unknown)>
PROVIDER_RPC_PORTS = {
11101u, 11102u, 11103u, 11104u, 11105u,
11101U,
11102U,
11103U,
11104U,
};
return PROVIDER_RPC_PORTS[static_cast<std::size_t>(pt)];
return PROVIDER_RPC_PORTS[static_cast<std::size_t>(prov)];
}
auto app_config::get_json() const -> json {
@ -209,11 +215,12 @@ auto app_config::get_json() const -> json {
{"SecretKey", s3_config_.secret_key},
{"TimeoutMs", s3_config_.timeout_ms},
{"URL", s3_config_.url},
{"UsePathStyle", s3_config_.use_path_style},
{"UseRegionInURL", s3_config_.use_region_in_url},
}},
{"Version", version_}};
if (pt_ == provider_type::encrypt) {
if (prov_ == provider_type::encrypt) {
ret.erase("ChunkDownloaderTimeoutSeconds");
ret.erase("EnableChunkDownloaderTimeout");
ret.erase("EnableMaxCacheSize");
@ -229,13 +236,13 @@ auto app_config::get_json() const -> json {
ret.erase("RetryReadCount");
ret.erase("RingBufferFileSize");
ret.erase("S3Config");
} else if (pt_ == provider_type::s3) {
} else if (prov_ == provider_type::s3) {
ret.erase("EncryptConfig");
ret.erase("HostConfig");
} else if (pt_ == provider_type::sia) {
} else if (prov_ == provider_type::sia) {
ret.erase("EncryptConfig");
ret.erase("S3Config");
} else if (pt_ == provider_type::remote) {
} else if (prov_ == provider_type::remote) {
ret.erase("ChunkDownloaderTimeoutSeconds");
ret.erase("EnableChunkDownloaderTimeout");
ret.erase("EnableChunkDownloaderTimeout");
@ -261,50 +268,56 @@ auto app_config::get_json() const -> json {
auto app_config::get_max_cache_size_bytes() const -> std::uint64_t {
const auto max_space =
std::max(static_cast<std::uint64_t>(100ull * 1024ull * 1024ull),
std::max(static_cast<std::uint64_t>(100ULL * 1024ULL * 1024ULL),
max_cache_size_bytes_);
return std::min(utils::file::get_free_drive_space(get_cache_directory()),
max_space);
}
auto app_config::get_provider_api_password(const provider_type &pt)
auto app_config::get_provider_api_password(const provider_type &prov)
-> std::string {
#ifdef _WIN32
auto api_file =
utils::path::combine(utils::get_local_app_data_directory(),
{get_provider_display_name(pt), "apipassword"});
{get_provider_display_name(prov), "apipassword"});
#else
#ifdef __APPLE__
auto api_file =
utils::path::combine(utils::path::resolve("~"),
{"/Library/Application Support",
get_provider_display_name(pt), "apipassword"});
get_provider_display_name(prov), "apipassword"});
#else
auto api_file = utils::path::combine(utils::path::resolve("~/."),
{get_provider_name(pt), "apipassword"});
auto api_file = utils::path::combine(
utils::path::resolve("~/."), {get_provider_name(prov), "apipassword"});
#endif
#endif
auto lines = utils::file::read_file_lines(api_file);
return lines.empty() ? "" : utils::string::trim(lines[0]);
}
auto app_config::get_provider_display_name(const provider_type &pt)
auto app_config::get_provider_display_name(const provider_type &prov)
-> std::string {
static const std::array<std::string,
static_cast<std::size_t>(provider_type::unknown)>
PROVIDER_DISPLAY_NAMES = {
"Sia", "Remote", "S3", "Passthrough", "Encrypt",
"Sia",
"Remote",
"S3",
"Encrypt",
};
return PROVIDER_DISPLAY_NAMES[static_cast<std::size_t>(pt)];
return PROVIDER_DISPLAY_NAMES[static_cast<std::size_t>(prov)];
}
auto app_config::get_provider_name(const provider_type &pt) -> std::string {
auto app_config::get_provider_name(const provider_type &prov) -> std::string {
static const std::array<std::string,
static_cast<std::size_t>(provider_type::unknown)>
PROVIDER_NAMES = {
"sia", "remote", "s3", "passthrough", "encrypt",
"sia",
"remote",
"s3",
"encrypt",
};
return PROVIDER_NAMES[static_cast<std::size_t>(pt)];
return PROVIDER_NAMES[static_cast<std::size_t>(prov)];
}
auto app_config::get_value_by_name(const std::string &name) -> std::string {
@ -447,8 +460,11 @@ auto app_config::get_value_by_name(const std::string &name) -> std::string {
if (name == "S3Config.URL") {
return s3_config_.url;
}
if (name == "S3Config.UsePathStyle") {
return utils::string::from_bool(s3_config_.use_path_style);
}
if (name == "S3Config.UseRegionInURL") {
return std::to_string(s3_config_.use_region_in_url);
return utils::string::from_bool(s3_config_.use_region_in_url);
}
if (name == "S3Config.TimeoutMs") {
return std::to_string(s3_config_.timeout_ms);
@ -463,14 +479,14 @@ auto app_config::load() -> bool {
auto ret = false;
const auto config_file_path = get_config_file_path();
recur_mutex_lock l(read_write_mutex_);
recur_mutex_lock lock(read_write_mutex_);
if (utils::file::is_file(config_file_path)) {
try {
std::ifstream config_file(&config_file_path[0]);
std::ifstream config_file(config_file_path.data());
if (config_file.is_open()) {
std::stringstream ss;
ss << config_file.rdbuf();
const auto json_text = ss.str();
std::stringstream stream;
stream << config_file.rdbuf();
const auto json_text = stream.str();
config_file.close();
if ((ret = not json_text.empty())) {
const auto json_document = json::parse(json_text);
@ -510,20 +526,20 @@ auto app_config::load() -> bool {
if (json_document.find("HostConfig") != json_document.end()) {
auto host_config_json = json_document["HostConfig"];
auto hc = hc_;
get_value(host_config_json, "AgentString", hc.agent_string, ret);
get_value(host_config_json, "ApiPassword", hc.api_password, ret);
get_value(host_config_json, "ApiPort", hc.api_port, ret);
get_value(host_config_json, "HostNameOrIp", hc.host_name_or_ip,
auto cfg = hc_;
get_value(host_config_json, "AgentString", cfg.agent_string, ret);
get_value(host_config_json, "ApiPassword", cfg.api_password, ret);
get_value(host_config_json, "ApiPort", cfg.api_port, ret);
get_value(host_config_json, "HostNameOrIp", cfg.host_name_or_ip,
ret);
get_value(host_config_json, "TimeoutMs", hc.timeout_ms, ret);
hc_ = hc;
get_value(host_config_json, "TimeoutMs", cfg.timeout_ms, ret);
hc_ = cfg;
} else {
ret = false;
}
if (hc_.api_password.empty()) {
hc_.api_password = get_provider_api_password(pt_);
hc_.api_password = get_provider_api_password(prov_);
if (hc_.api_password.empty()) {
ret = false;
}
@ -531,20 +547,22 @@ auto app_config::load() -> bool {
if (json_document.find("S3Config") != json_document.end()) {
auto s3_config_json = json_document["S3Config"];
auto s3 = s3_config_;
get_value(s3_config_json, "AccessKey", s3.access_key, ret);
get_value(s3_config_json, "Bucket", s3.bucket, ret);
auto s3_cfg = s3_config_;
get_value(s3_config_json, "AccessKey", s3_cfg.access_key, ret);
get_value(s3_config_json, "Bucket", s3_cfg.bucket, ret);
get_value(s3_config_json, "CacheTimeoutSeconds",
s3.cache_timeout_secs, ret);
get_value(s3_config_json, "EncryptionToken", s3.encryption_token,
s3_cfg.cache_timeout_secs, ret);
get_value(s3_config_json, "EncryptionToken",
s3_cfg.encryption_token, ret);
get_value(s3_config_json, "Region", s3_cfg.region, ret);
get_value(s3_config_json, "SecretKey", s3_cfg.secret_key, ret);
get_value(s3_config_json, "TimeoutMs", s3_cfg.timeout_ms, ret);
get_value(s3_config_json, "URL", s3_cfg.url, ret);
get_value(s3_config_json, "UsePathStyle", s3_cfg.use_path_style,
ret);
get_value(s3_config_json, "Region", s3.region, ret);
get_value(s3_config_json, "SecretKey", s3.secret_key, ret);
get_value(s3_config_json, "TimeoutMs", s3.timeout_ms, ret);
get_value(s3_config_json, "URL", s3.url, ret);
get_value(s3_config_json, "UseRegionInURL", s3.use_region_in_url,
ret);
s3_config_ = s3;
get_value(s3_config_json, "UseRegionInURL",
s3_cfg.use_region_in_url, ret);
s3_config_ = s3_cfg;
} else {
ret = false;
}
@ -593,13 +611,13 @@ auto app_config::load() -> bool {
ret = false;
}
std::uint64_t version = 0u;
std::uint64_t version{};
get_value(json_document, "Version", version, ret);
// Handle configuration defaults for new config versions
if (version != REPERTORY_CONFIG_VERSION) {
if (version > REPERTORY_CONFIG_VERSION) {
version = 0u;
version = 0U;
}
version_ = version;
@ -622,7 +640,7 @@ auto app_config::load() -> bool {
void app_config::save() {
const auto configFilePath = get_config_file_path();
recur_mutex_lock l(read_write_mutex_);
recur_mutex_lock lock(read_write_mutex_);
if (config_changed_ || not utils::file::is_file(configFilePath)) {
if (not utils::file::is_directory(data_directory_)) {
if (not utils::file::create_full_directory_path(data_directory_)) {
@ -684,19 +702,19 @@ auto app_config::set_value_by_name(const std::string &name,
}
if (name == "EnableChunkDownloaderTimeout") {
set_enable_chunk_downloader_timeout(utils::string::to_bool(value));
return std::to_string(get_enable_chunk_download_timeout());
return utils::string::from_bool(get_enable_chunk_download_timeout());
}
if (name == "EnableCommDurationEvents") {
set_enable_comm_duration_events(utils::string::to_bool(value));
return std::to_string(get_enable_comm_duration_events());
return utils::string::from_bool(get_enable_comm_duration_events());
}
if (name == "EnableDriveEvents") {
set_enable_drive_events(utils::string::to_bool(value));
return std::to_string(get_enable_drive_events());
return utils::string::from_bool(get_enable_drive_events());
}
if (name == "EnableMaxCacheSize") {
set_enable_max_cache_size(utils::string::to_bool(value));
return std::to_string(get_enable_max_cache_size());
return utils::string::from_bool(get_enable_max_cache_size());
#ifdef _WIN32
}
if (name == "EnableMountManager") {
@ -722,7 +740,7 @@ auto app_config::set_value_by_name(const std::string &name,
}
if (name == "EvictionUsesAccessedTime") {
set_eviction_uses_accessed_time(utils::string::to_bool(value));
return std::to_string(get_eviction_uses_accessed_time());
return utils::string::from_bool(get_eviction_uses_accessed_time());
}
if (name == "HighFreqIntervalSeconds") {
set_high_frequency_interval_secs(utils::string::to_uint8(value));
@ -779,11 +797,11 @@ auto app_config::set_value_by_name(const std::string &name,
}
if (name == "RemoteMount.EnableRemoteMount") {
set_enable_remote_mount(utils::string::to_bool(value));
return std::to_string(get_enable_remote_mount());
return utils::string::from_bool(get_enable_remote_mount());
}
if (name == "RemoteMount.IsRemoteMount") {
set_is_remote_mount(utils::string::to_bool(value));
return std::to_string(get_is_remote_mount());
return utils::string::from_bool(get_is_remote_mount());
}
if (name == "RemoteMount.RemoteClientPoolSize") {
set_remote_client_pool_size(utils::string::to_uint8(value));
@ -831,7 +849,7 @@ auto app_config::set_value_by_name(const std::string &name,
}
if (name == "S3Config.CacheTimeoutSeconds") {
const auto timeout =
std::max(std::uint16_t(5u), utils::string::to_uint16(value));
std::max(std::uint16_t(5U), utils::string::to_uint16(value));
set_value(s3_config_.cache_timeout_secs, timeout);
return std::to_string(s3_config_.cache_timeout_secs);
}
@ -847,9 +865,13 @@ auto app_config::set_value_by_name(const std::string &name,
set_value(s3_config_.url, value);
return s3_config_.url;
}
if (name == "S3Config.UsePathStyle") {
set_value(s3_config_.use_path_style, utils::string::to_bool(value));
return utils::string::from_bool(s3_config_.use_path_style);
}
if (name == "S3Config.UseRegionInURL") {
set_value(s3_config_.use_region_in_url, utils::string::to_bool(value));
return std::to_string(s3_config_.use_region_in_url);
return utils::string::from_bool(s3_config_.use_region_in_url);
}
if (name == "S3Config.TimeoutMs") {
set_value(s3_config_.timeout_ms, utils::string::to_uint32(value));

16
src/comm/curl/requests/http_put_file.cpp
View File

@ -24,27 +24,13 @@
#include "utils/string_utils.hpp"
namespace repertory::curl::requests {
auto http_put_file::get_path() const -> std::string {
if (reader) {
auto updated_path = path;
return utils::string::replace(updated_path, file_name,
reader->get_encrypted_file_name());
}
return http_request_base::get_path();
}
auto http_put_file::set_method(CURL *curl, stop_type &stop_requested) const
-> bool {
curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "PUT");
if (not source_path.empty()) {
curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L);
if (not encryption_token.value_or("").empty()) {
if (not reader) {
reader = std::make_shared<utils::encryption::encrypting_reader>(
file_name, source_path, stop_requested, encryption_token.value());
}
if (reader) {
curl_easy_setopt(curl, CURLOPT_READDATA, reader.get());
curl_easy_setopt(
curl, CURLOPT_READFUNCTION,

4
src/comm/packet/client_pool.cpp
View File

@ -148,8 +148,8 @@ void client_pool::shutdown() {
unique_mutex_lock pool_lock(pool_mutex_);
if (not shutdown_) {
shutdown_ = true;
for (auto &pool : pool_lookup_) {
pool.second->shutdown();
for (auto &pool_entry : pool_lookup_) {
pool_entry.second->shutdown();
}
pool_lookup_.clear();
}

577
src/comm/s3/s3_comm.cpp
View File

@ -1,577 +0,0 @@
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#if defined(REPERTORY_ENABLE_S3)
#include "comm/s3/s3_comm.hpp"
#include "app_config.hpp"
#include "comm/curl/curl_comm.hpp"
#include "comm/s3/s3_requests.hpp"
#include "events/event_system.hpp"
#include "events/events.hpp"
#include "providers/i_provider.hpp"
#include "types/repertory.hpp"
#include "types/s3.hpp"
#include "utils/encryption.hpp"
#include "utils/error_utils.hpp"
#include "utils/path_utils.hpp"
#include "utils/polling.hpp"
#include "utils/string_utils.hpp"
namespace repertory {
static const get_key_callback empty_key = []() { return ""; };
s3_comm::s3_comm(const app_config &config)
: config_(config), s3_config_(config.get_s3_config()) {
s3_config_.bucket = utils::string::trim(s3_config_.bucket);
// TODO make configurable
const auto enable_path_style =
utils::string::begins_with(s3_config_.url,
"https://gateway.storjshare.io") ||
utils::string::begins_with(s3_config_.url, "http://localhost") ||
utils::string::begins_with(s3_config_.url, "https://localhost") ||
utils::string::begins_with(s3_config_.url, "http://127.0.0.1") ||
utils::string::begins_with(s3_config_.url, "https://127.0.0.1");
s3_client_ = std::make_unique<curl_comm>(s3_config_);
s3_client_->enable_s3_path_style(enable_path_style);
polling::instance().set_callback(
{"s3_directory_cache", polling::frequency::high,
[this]() { this->clear_expired_directories(); }});
}
s3_comm::s3_comm(s3_comm &&comm)
: config_(std::move(comm.config_)),
s3_config_(std::move(comm.s3_config_)),
s3_client_(std::move(comm.s3_client_)) {
comm.active_ = false;
polling::instance().set_callback(
{"s3_directory_cache", polling::frequency::high,
[this]() { this->clear_expired_directories(); }});
}
s3_comm::~s3_comm() {
if (active_) {
polling::instance().remove_callback("s3_directory_cache");
}
}
void s3_comm::clear_expired_directories() {
recur_mutex_lock l(cached_directories_mutex_);
std::vector<std::string> expired_list;
for (const auto &kv : cached_directories_) {
if (kv.second.expiration <= std::chrono::system_clock::now()) {
expired_list.emplace_back(kv.first);
}
}
for (const auto &expired : expired_list) {
event_system::instance().raise<debug_log>(__FUNCTION__, expired, "expired");
cached_directories_.erase(expired);
}
}
auto s3_comm::create_directory(const std::string &api_path) -> api_error {
raise_begin(__FUNCTION__, api_path);
long response_code{};
auto object_name = get_object_name(api_path, empty_key) + '/';
if (not create_directory_object_request(*s3_client_, s3_config_, object_name,
response_code)) {
return raise_end(__FUNCTION__, api_path, api_error::comm_error,
response_code);
}
return raise_end(__FUNCTION__, api_path,
response_code == 200 ? api_error::success
: api_error::comm_error,
response_code);
}
auto s3_comm::directory_exists(const std::string &api_path) const -> api_error {
raise_begin(__FUNCTION__, api_path);
auto object_name = get_object_name(api_path, empty_key) + "/";
head_object_result result{};
long response_code{};
if (head_object_request(*s3_client_, s3_config_, object_name, result,
response_code)) {
if (response_code == 404) {
return raise_end(__FUNCTION__, api_path, api_error::directory_not_found,
response_code);
}
return raise_end(__FUNCTION__, api_path, api_error::directory_exists,
response_code);
}
return raise_end(__FUNCTION__, api_path, api_error::comm_error,
response_code);
}
auto s3_comm::file_exists(const std::string &api_path,
const get_key_callback &get_key) const -> api_error {
raise_begin(__FUNCTION__, api_path);
if (get_cached_file_exists(api_path)) {
return raise_end(__FUNCTION__, api_path, api_error::item_exists, 200);
}
auto object_name = get_object_name(api_path, get_key);
head_object_result result{};
long response_code{};
if (head_object_request(*s3_client_, s3_config_, object_name, result,
response_code)) {
if (response_code == 404) {
return raise_end(__FUNCTION__, api_path, api_error::item_not_found,
response_code);
}
return raise_end(__FUNCTION__, api_path, api_error::item_exists,
response_code);
}
return raise_end(__FUNCTION__, api_path, api_error::directory_exists,
response_code);
}
auto s3_comm::get_object_list(std::vector<directory_item> &list) const
-> api_error {
raise_begin(__FUNCTION__, "/");
long response_code{};
auto success =
list_objects_request(*s3_client_, s3_config_, list, response_code);
return raise_end(__FUNCTION__, "/",
success ? api_error::success : api_error::comm_error,
response_code);
}
auto s3_comm::get_object_name(const std::string &api_path,
const get_key_callback &get_key) const
-> std::string {
auto object_name = utils::path::create_api_path(api_path).substr(1);
const auto key = get_key();
if (not key.empty()) {
auto parts = utils::string::split(object_name, '/', false);
parts[parts.size() - 1u] = key;
object_name = utils::string::join(parts, '/');
}
return object_name;
}
auto s3_comm::get_cached_directory_item_count(const std::string &api_path,
std::size_t &count) const
-> bool {
recur_mutex_lock l(cached_directories_mutex_);
if (cached_directories_.find(api_path) != cached_directories_.end()) {
count = cached_directories_.at(api_path).items.size();
return true;
}
return false;
}
auto s3_comm::get_cached_directory_items(const std::string &api_path,
meta_provider_callback meta_provider,
directory_item_list &list) const
-> bool {
unique_recur_mutex_lock l(cached_directories_mutex_);
if (cached_directories_.find(api_path) != cached_directories_.end()) {
auto &cachedEntry = cached_directories_.at(api_path);
list = cachedEntry.items;
cached_directories_[api_path].reset_timeout(
std::chrono::seconds(config_.get_s3_config().cache_timeout_secs));
l.unlock();
for (auto &item : list) {
meta_provider(item);
}
return true;
}
return false;
}
auto s3_comm::get_cached_file_exists(const std::string &api_path) const
-> bool {
unique_recur_mutex_lock l(cached_directories_mutex_);
const auto parent_api_path = utils::path::get_parent_api_path(api_path);
if (cached_directories_.find(parent_api_path) != cached_directories_.end()) {
auto &entry = cached_directories_.at(parent_api_path);
if (std::find_if(entry.items.begin(), entry.items.end(),
[&api_path](const auto &item) -> bool {
return not item.directory && (api_path == item.api_path);
}) != entry.items.end()) {
cached_directories_[api_path].reset_timeout(
std::chrono::seconds(config_.get_s3_config().cache_timeout_secs));
return true;
}
}
return false;
}
auto s3_comm::get_directory_item_count(
const std::string &api_path, meta_provider_callback meta_provider) const
-> std::size_t {
raise_begin(__FUNCTION__, api_path);
std::size_t ret = 0u;
if (not get_cached_directory_item_count(api_path, ret)) {
directory_item_list list;
const auto res = grab_directory_items(api_path, meta_provider, list);
if (res != api_error::success) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, res,
"failed to grab directory items");
}
return list.size();
}
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(__FUNCTION__, api_path,
"end|" + std::to_string(ret));
}
return ret;
}
auto s3_comm::get_directory_items(const std::string &api_path,
meta_provider_callback meta_provider,
directory_item_list &list) const
-> api_error {
raise_begin(__FUNCTION__, api_path);
auto ret = api_error::success;
if (not get_cached_directory_items(api_path, meta_provider, list)) {
ret = grab_directory_items(api_path, meta_provider, list);
}
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(
__FUNCTION__, api_path, "end|" + api_error_to_string(ret));
}
return ret;
}
auto s3_comm::get_directory_list(api_file_list &list) const -> api_error {
raise_begin(__FUNCTION__, "/");
long response_code{};
auto success =
list_directories_request(*s3_client_, s3_config_, list, response_code);
return raise_end(__FUNCTION__, "/",
success ? api_error::success : api_error::comm_error,
response_code);
}
auto s3_comm::get_file(const std::string &api_path,
const get_key_callback &get_key,
const get_name_callback &get_name,
const get_token_callback &get_token,
api_file &file) const -> api_error {
raise_begin(__FUNCTION__, api_path);
auto ret = api_error::success;
auto object_name = get_object_name(api_path, get_key);
head_object_result result{};
long response_code{};
if (head_object_request(*s3_client_, s3_config_, object_name, result,
response_code)) {
const auto key = get_key();
object_name = get_name(key, object_name);
file.accessed_date = utils::get_file_time_now();
file.api_path = utils::path::create_api_path(object_name);
file.api_parent = utils::path::get_parent_api_path(file.api_path);
file.changed_date = utils::aws::format_time(result.last_modified);
file.creation_date = utils::aws::format_time(result.last_modified);
file.encryption_token = get_token();
file.file_size =
file.encryption_token.empty()
? result.content_length
: utils::encryption::encrypting_reader::calculate_decrypted_size(
result.content_length);
file.modified_date = utils::aws::format_time(result.last_modified);
} else {
utils::error::raise_api_path_error(__FUNCTION__, api_path, response_code,
"head object request failed");
ret = api_error::comm_error;
}
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(
__FUNCTION__, api_path, "end|" + api_error_to_string(ret));
}
return ret;
}
auto s3_comm::get_file_list(
const get_api_file_token_callback &get_api_file_token,
const get_name_callback &get_name, api_file_list &list) const -> api_error {
raise_begin(__FUNCTION__, "/");
long response_code{};
auto success = list_files_request(*s3_client_, s3_config_, get_api_file_token,
get_name, list, response_code);
return raise_end(__FUNCTION__, "/",
success ? api_error::success : api_error::comm_error,
response_code);
}
auto s3_comm::grab_directory_items(const std::string &api_path,
meta_provider_callback meta_provider,
directory_item_list &list) const
-> api_error {
auto object_name = get_object_name(api_path, empty_key);
long response_code{};
if (list_objects_in_directory_request(*s3_client_, s3_config_, object_name,
meta_provider, list, response_code)) {
if (response_code == 404) {
return api_error::directory_not_found;
}
if (response_code != 200) {
return api_error::comm_error;
}
set_cached_directory_items(api_path, list);
return api_error::success;
}
return api_error::comm_error;
}
void s3_comm::raise_begin(const std::string &function_name,
const std::string &api_path) const {
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(function_name, api_path,
"begin|");
}
}
auto s3_comm::raise_end(const std::string &function_name,
const std::string &api_path, const api_error &error,
long code) const -> api_error {
if (config_.get_event_level() >= event_level::debug) {
event_system::instance().raise<debug_log>(
function_name, api_path,
"end|" + api_error_to_string(error) + '|' + std::to_string(code));
}
return error;
}
auto s3_comm::read_file_bytes(const std::string &api_path, std::size_t size,
std::uint64_t offset, data_buffer &data,
const get_key_callback &get_key,
const get_size_callback &get_size,
const get_token_callback &get_token,
stop_type &stop_requested) const -> api_error {
data.clear();
auto object_name = get_object_name(api_path, get_key);
const auto encryption_token = get_token();
const auto data_size = get_size();
if (encryption_token.empty()) {
long response_code{};
if (not read_object_request(*s3_client_, s3_config_, object_name, size,
offset, data, response_code, stop_requested)) {
auto res =
stop_requested ? api_error::download_stopped : api_error::comm_error;
utils::error::raise_api_path_error(__FUNCTION__, api_path, res,
"failed to read file bytes");
return res;
}
if (response_code < 200 || response_code >= 300) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, response_code,
"failed to read file bytes");
return api_error::comm_error;
}
return api_error::success;
}
const auto key = utils::encryption::generate_key(encryption_token);
return utils::encryption::read_encrypted_range(
{offset, offset + size - 1}, key,
[&](data_buffer &ct, std::uint64_t start_offset,
std::uint64_t end_offset) -> api_error {
return read_file_bytes(
api_path, (end_offset - start_offset + 1u), start_offset, ct,
get_key, get_size, []() -> std::string { return ""; },
stop_requested);
},
data_size, data);
}
void s3_comm::remove_cached_directory(const std::string &api_path) {
recur_mutex_lock l(cached_directories_mutex_);
cached_directories_.erase(api_path);
}
auto s3_comm::remove_directory(const std::string &api_path) -> api_error {
raise_begin(__FUNCTION__, api_path);
auto object_name = get_object_name(api_path, empty_key) + "/";
long response_code{};
if (delete_object_request(*s3_client_, s3_config_, object_name,
response_code)) {
if (response_code == 404) {
return raise_end(__FUNCTION__, api_path, api_error::directory_not_found,
response_code);
}
if (response_code != 204) {
return raise_end(__FUNCTION__, api_path, api_error::comm_error,
response_code);
}
remove_cached_directory(utils::path::get_parent_api_path(api_path));
remove_cached_directory(api_path);
return raise_end(__FUNCTION__, api_path, api_error::success, response_code);
}
return raise_end(__FUNCTION__, api_path, api_error::comm_error,
response_code);
}
auto s3_comm::remove_file(const std::string &api_path,
const get_key_callback &get_key) -> api_error {
raise_begin(__FUNCTION__, api_path);
auto object_name = get_object_name(api_path, get_key);
long response_code{};
if (delete_object_request(*s3_client_, s3_config_, object_name,
response_code)) {
if (response_code == 404) {
return raise_end(__FUNCTION__, api_path, api_error::item_not_found,
response_code);
}
if (response_code != 204) {
return raise_end(__FUNCTION__, api_path, api_error::comm_error,
response_code);
}
remove_cached_directory(utils::path::get_parent_api_path(api_path));
return raise_end(__FUNCTION__, api_path, api_error::success, response_code);
}
return raise_end(__FUNCTION__, api_path, api_error::comm_error,
response_code);
}
auto s3_comm::rename_file(const std::string & /*api_path*/,
const std::string & /*new_api_path*/) -> api_error {
return api_error::not_implemented;
/* if (config_.get_event_level() >= event_level::debug) { */
/* event_system::instance().raise<debug_log>(__FUNCTION__, api_path,
* "begin"); */
/* } */
/* auto ret = api_error::success; */
/* */
/* std::string bucket_name, object_name; */
/* get_object_name(api_path, bucket_name, object_name); */
/* */
/* std::string new_object_name; */
/* get_object_name(new_api_path, bucket_name, new_object_name); */
/* */
/* Aws::S3::Model::CopyObjectRequest request{}; */
/* request.SetBucket(bucket_name); */
/* request.SetCopySource(bucket_name + '/' + object_name); */
/* request.SetKey(new_object_name); */
/* */
/* const auto outcome = s3_client_->CopyObject(request); */
/* if (outcome.IsSuccess()) { */
/* ret = remove_file(api_path); */
/* } else { */
/* const auto &error = outcome.GetError(); */
/* event_system::instance().raise<repertory_exception>(__FUNCTION__,
* error.GetExceptionName()
* +
* "|" + */
/* error.GetMessage());
*/
/* ret = api_error::comm_error; */
/* } */
/* */
/* if (config_.get_event_level() >= event_level::debug) { */
/* event_system::instance().raise<debug_log>(__FUNCTION__, api_path, */
/* "end|" +
* std::to_string(std::uint8_t(ret))); */
/* } */
/* return ret; */
}
void s3_comm::set_cached_directory_items(const std::string &api_path,
directory_item_list list) const {
recur_mutex_lock l(cached_directories_mutex_);
cached_directories_[api_path].items = std::move(list);
cached_directories_[api_path].reset_timeout(
std::chrono::seconds(config_.get_s3_config().cache_timeout_secs));
}
auto s3_comm::upload_file(const std::string &api_path,
const std::string &source_path,
const std::string &encryption_token,
const get_key_callback &get_key,
const set_key_callback &set_key,
stop_type &stop_requested) -> api_error {
raise_begin(__FUNCTION__, api_path);
auto object_name = get_object_name(api_path, get_key);
long response_code{};
if (not put_object_request(*s3_client_, s3_config_, object_name, source_path,
encryption_token, get_key, set_key, response_code,
stop_requested)) {
return raise_end(__FUNCTION__, api_path,
stop_requested ? api_error::upload_stopped
: api_error::upload_failed,
response_code);
}
if (response_code != 200) {
return raise_end(__FUNCTION__, api_path, api_error::comm_error,
response_code);
}
remove_cached_directory(utils::path::get_parent_api_path(api_path));
return raise_end(__FUNCTION__, api_path, api_error::success, response_code);
}
} // namespace repertory
#endif // REPERTORY_ENABLE_S3

19
src/comm/s3/s3_requests_curl.cpp
View File

@ -65,8 +65,6 @@ auto create_directory_object_request_impl(i_http_comm &client,
curl::requests::http_put_file put_file{};
put_file.allow_timeout = true;
put_file.aws_service = "aws:amz:" + config.region + ":s3";
put_file.file_name =
*(utils::string::split(object_name, '/', false).end() - 1U);
put_file.path = '/' + object_name;
stop_type stop_requested{false};
@ -212,12 +210,12 @@ auto list_files_request_impl(
file.api_path = utils::path::create_api_path(object_name);
file.api_parent = utils::path::get_parent_api_path(file.api_path);
file.accessed_date = utils::get_file_time_now();
file.encryption_token = get_api_file_token(file.api_path);
// file.encryption_token = get_api_file_token(file.api_path);
auto size = node.node().select_node("Size").node().text().as_ullong();
file.file_size = file.encryption_token.empty()
? size
: utils::encryption::encrypting_reader::
calculate_decrypted_size(size);
// file.file_size = file.encryption_token.empty()
// ? size
// : utils::encryption::encrypting_reader::
// calculate_decrypted_size(size);
file.changed_date = utils::convert_api_date(
node.node().select_node("LastModified").node().text().as_string());
file.creation_date = file.changed_date;
@ -349,9 +347,6 @@ auto put_object_request_impl(i_http_comm &client, const s3_config &config,
try {
curl::requests::http_put_file put_file{};
put_file.aws_service = "aws:amz:" + config.region + ":s3";
put_file.encryption_token = encryption_token;
put_file.file_name =
*(utils::string::split(object_name, '/', false).end() - 1U);
put_file.path = '/' + object_name;
put_file.source_path = source_path;
@ -359,8 +354,8 @@ auto put_object_request_impl(i_http_comm &client, const s3_config &config,
static stop_type no_stop{false};
put_file.reader = std::make_shared<utils::encryption::encrypting_reader>(
put_file.file_name, source_path, no_stop, encryption_token,
std::nullopt, -1);
*(utils::string::split(object_name, '/', false).end() - 1U),
source_path, no_stop, encryption_token, std::nullopt, -1);
auto key = get_key();
if (key.empty()) {
key = put_file.reader->get_encrypted_file_name();

370
src/db/directory_db.cpp
View File

@ -1,370 +0,0 @@
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "db/directory_db.hpp"
#include "utils/path_utils.hpp"
namespace repertory {
void directory_db::directory_tree::add_path(
const std::string &api_path, const std::vector<std::string> &files,
rocksdb::DB &db) {
const auto create_not_found = [&](const auto &create_path) {
std::string value;
if (not db.Get(rocksdb::ReadOptions(), create_path, &value).ok()) {
json directoryData = {
{"path", api_path},
{"files", files},
};
db.Put(rocksdb::WriteOptions(), create_path, directoryData.dump());
}
};
const auto parts = utils::string::split(api_path, '/', false);
std::string previous_directory;
for (const auto &directory_part : parts) {
if (directory_part.empty()) {
sub_directory_lookup_["/"];
previous_directory = "/";
create_not_found("/");
} else {
auto &sub_directories = sub_directory_lookup_[previous_directory];
if (std::find(sub_directories.begin(), sub_directories.end(),
directory_part) == sub_directories.end()) {
sub_directories.emplace_back(directory_part);
}
previous_directory = utils::path::create_api_path(
utils::path::combine(previous_directory, {directory_part}));
sub_directory_lookup_[previous_directory];
create_not_found(previous_directory);
}
}
}
auto directory_db::directory_tree::get_count(const std::string &api_path) const
-> std::size_t {
return (sub_directory_lookup_.find(api_path) == sub_directory_lookup_.end())
? 0
: sub_directory_lookup_.at(api_path).size();
}
auto directory_db::directory_tree::get_directories() const
-> std::vector<std::string> {
std::vector<std::string> ret;
std::transform(sub_directory_lookup_.begin(), sub_directory_lookup_.end(),
std::back_inserter(ret),
[](const auto &kv) { return kv.first; });
return ret;
}
auto directory_db::directory_tree::get_sub_directories(
const std::string &api_path) const -> std::vector<std::string> {
std::vector<std::string> ret;
if (sub_directory_lookup_.find(api_path) != sub_directory_lookup_.end()) {
const auto &lookup = sub_directory_lookup_.at(api_path);
std::transform(lookup.begin(), lookup.end(), std::back_inserter(ret),
[&api_path](const auto &directory) {
return utils::path::create_api_path(
utils::path::combine(api_path, {directory}));
});
}
return ret;
}
auto directory_db::directory_tree::is_directory(
const std::string &api_path) const -> bool {
return sub_directory_lookup_.find(api_path) != sub_directory_lookup_.end();
}
void directory_db::directory_tree::remove_directory(const std::string &api_path,
rocksdb::DB &db,
bool allow_remove_root) {
if ((allow_remove_root || (api_path != "/")) && is_directory(api_path)) {
sub_directory_lookup_.erase(api_path);
db.Delete(rocksdb::WriteOptions(), api_path);
const auto parent_api_path = utils::path::get_parent_api_path(api_path);
const auto parts = utils::string::split(api_path, '/', false);
utils::remove_element_from(sub_directory_lookup_[parent_api_path],
parts[parts.size() - 1]);
}
}
directory_db::directory_db(const app_config &config) {
utils::db::create_rocksdb(config, DIRDB_NAME, db_);
auto iterator = std::unique_ptr<rocksdb::Iterator>(
db_->NewIterator(rocksdb::ReadOptions()));
for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
auto directory_data = json::parse(iterator->value().ToString());
tree_.add_path(directory_data["path"].get<std::string>(),
directory_data["files"].get<std::vector<std::string>>(),
*db_);
}
}
directory_db::~directory_db() { db_.reset(); }
auto directory_db::create_directory(const std::string &api_path,
bool create_always) -> api_error {
recur_mutex_lock directory_lock(directory_mutex_);
auto ret = api_error::directory_exists;
if (not is_directory(api_path)) {
ret = api_error::directory_not_found;
if (create_always || (api_path == "/") ||
is_directory(utils::path::get_parent_api_path(api_path))) {
ret = api_error::item_exists;
if (not is_file(api_path)) {
tree_.add_path(api_path, {}, *db_);
ret = api_error::success;
}
}
}
return ret;
}
auto directory_db::create_file(const std::string &api_path) -> api_error {
recur_mutex_lock directory_lock(directory_mutex_);
if (is_directory(api_path)) {
return api_error::directory_exists;
}
const auto parent_api_path = utils::path::get_parent_api_path(api_path);
auto directory_data = get_directory_data(parent_api_path);
if (directory_data.empty()) {
return api_error::directory_not_found;
}
const auto file_name = utils::path::strip_to_file_name(api_path);
if (utils::collection_includes(directory_data["files"], file_name)) {
return api_error::item_exists;
}
directory_data["files"].emplace_back(file_name);
db_->Put(rocksdb::WriteOptions(), parent_api_path, directory_data.dump());
return api_error::success;
}
auto directory_db::get_directory_data(const std::string &api_path) const
-> json {
std::string data;
db_->Get(rocksdb::ReadOptions(), api_path, &data);
if (data.empty()) {
return {};
}
return json::parse(data);
}
auto directory_db::get_directory_item_count(const std::string &api_path) const
-> std::uint64_t {
auto directory_data = get_directory_data(api_path);
const auto sub_directory_count = get_sub_directory_count(api_path);
const auto file_count =
(directory_data.empty() ? 0 : directory_data["files"].size());
return sub_directory_count + file_count;
}
auto directory_db::get_file(const std::string &api_path, api_file &file,
api_file_provider_callback api_file_provider) const
-> api_error {
const auto parent_api_path = utils::path::get_parent_api_path(api_path);
auto directory_data = get_directory_data(parent_api_path);
if (not directory_data.empty()) {
const auto file_name = utils::path::strip_to_file_name(api_path);
if (utils::collection_includes(directory_data["files"], file_name)) {
file.api_path = utils::path::create_api_path(
utils::path::combine(parent_api_path, {file_name})),
api_file_provider(file);
return api_error::success;
}
}
return api_error::item_not_found;
}
auto directory_db::get_file_list(
api_file_list &list, api_file_provider_callback api_file_provider) const
-> api_error {
auto iterator = std::unique_ptr<rocksdb::Iterator>(
db_->NewIterator(rocksdb::ReadOptions()));
for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
auto directory_data = json::parse(iterator->value().ToString());
for (const auto &directory_file : directory_data["files"]) {
api_file file{
utils::path::create_api_path(utils::path::combine(
iterator->key().ToString(), {directory_file.get<std::string>()})),
};
api_file_provider(file);
list.emplace_back(file);
}
}
return api_error::success;
}
auto directory_db::get_sub_directory_count(const std::string &api_path) const
-> std::size_t {
recur_mutex_lock directoryLock(directory_mutex_);
return tree_.get_count(api_path);
}
auto directory_db::get_total_item_count() const -> std::uint64_t {
unique_recur_mutex_lock directory_lock(directory_mutex_);
const auto directories = tree_.get_directories();
directory_lock.unlock();
return std::accumulate(
directories.begin(), directories.end(), std::uint64_t(directories.size()),
[this](std::uint64_t c, const std::string &directory) {
const auto dirData = this->get_directory_data(directory);
return c + (dirData.empty() ? 0 : dirData["files"].size());
});
}
auto directory_db::is_directory(const std::string &api_path) const -> bool {
recur_mutex_lock directory_lock(directory_mutex_);
return tree_.is_directory(api_path);
}
auto directory_db::is_file(const std::string &api_path) const -> bool {
auto directory_data =
get_directory_data(utils::path::get_parent_api_path(api_path));
if (directory_data.empty()) {
return false;
}
const auto file_name = utils::path::strip_to_file_name(api_path);
return utils::collection_includes(directory_data["files"], file_name);
}
void directory_db::populate_directory_files(
const std::string &api_path, meta_provider_callback meta_provider,
directory_item_list &list) const {
auto directory_data = get_directory_data(api_path);
if (not directory_data.empty()) {
for (const auto &directory_file : directory_data["files"]) {
directory_item di{};
di.api_path = utils::path::create_api_path(
utils::path::combine(api_path, {directory_file.get<std::string>()}));
di.directory = false;
meta_provider(di);
di.size = utils::string::to_uint64(di.meta[META_SIZE]);
list.emplace_back(std::move(di));
}
}
}
void directory_db::populate_sub_directories(
const std::string &api_path, meta_provider_callback meta_provider,
directory_item_list &list) const {
unique_recur_mutex_lock directory_lock(directory_mutex_);
const auto directories = tree_.get_sub_directories(api_path);
directory_lock.unlock();
std::size_t offset{};
for (const auto &directory : directories) {
if (std::find_if(list.begin(), list.end(),
[&directory](const auto &di) -> bool {
return directory == di.api_path;
}) == list.end()) {
directory_item di{};
di.api_path = directory;
di.api_parent = utils::path::get_parent_api_path(directory);
di.directory = true;
di.size = get_sub_directory_count(directory);
meta_provider(di);
list.insert(list.begin() + static_cast<std::int64_t>(offset++),
std::move(di));
}
}
}
auto directory_db::remove_directory(const std::string &api_path,
bool allow_remove_root) -> api_error {
recur_mutex_lock directory_lock(directory_mutex_);
if ((api_path == "/") && not allow_remove_root) {
return api_error::access_denied;
}
if (is_file(api_path) || not is_directory(api_path)) {
return api_error::directory_not_found;
}
if (tree_.get_count(api_path) == 0) {
auto directory_data = get_directory_data(api_path);
if (directory_data.empty() || directory_data["files"].empty()) {
tree_.remove_directory(api_path, *db_, allow_remove_root);
return api_error::success;
}
}
return api_error::directory_not_empty;
}
auto directory_db::remove_file(const std::string &api_path) -> bool {
recur_mutex_lock directory_lock(directory_mutex_);
if (is_directory(api_path)) {
return false;
}
const auto parent_api_path = utils::path::get_parent_api_path(api_path);
auto directory_data = get_directory_data(parent_api_path);
if (directory_data.empty()) {
return false;
}
const auto file_name = utils::path::strip_to_file_name(api_path);
if (utils::collection_excludes(directory_data["files"], file_name)) {
return false;
}
utils::remove_element_from(directory_data["files"], file_name);
db_->Put(rocksdb::WriteOptions(), parent_api_path, directory_data.dump());
return true;
}
auto directory_db::rename_file(const std::string &from_api_path,
const std::string &to_api_path) -> api_error {
recur_mutex_lock directory_lock(directory_mutex_);
if (is_directory(from_api_path) || is_directory(to_api_path)) {
return api_error::directory_exists;
}
if (not is_directory(utils::path::get_parent_api_path(to_api_path))) {
return api_error::directory_not_found;
}
if (is_file(to_api_path)) {
return api_error::item_exists;
}
if (not remove_file(from_api_path)) {
return api_error::item_not_found;
}
return create_file(to_api_path);
}
} // namespace repertory

347
src/db/meta_db.cpp
View File

@ -1,347 +0,0 @@
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "db/meta_db.hpp"
#include "types/repertory.hpp"
#include "types/startup_exception.hpp"
#include "utils/error_utils.hpp"
#include "utils/file_utils.hpp"
#include "utils/path_utils.hpp"
#include "utils/utils.hpp"
namespace repertory {
meta_db::meta_db(const app_config &config) {
const auto create_resources = [this, &config](const std::string &name) {
auto families = std::vector<rocksdb::ColumnFamilyDescriptor>();
families.emplace_back(rocksdb::kDefaultColumnFamilyName,
rocksdb::ColumnFamilyOptions());
families.emplace_back("keys", rocksdb::ColumnFamilyOptions());
families.emplace_back("source", rocksdb::ColumnFamilyOptions());
auto handles = std::vector<rocksdb::ColumnFamilyHandle *>();
utils::db::create_rocksdb(config, name, families, handles, db_);
std::size_t idx{};
default_family_ = handles[idx++];
keys_family_ = handles[idx++];
source_family_ = handles[idx++];
};
create_resources(METADB_NAME);
}
meta_db::~meta_db() { db_.reset(); }
auto meta_db::create_iterator(bool source_family) const
-> std::shared_ptr<rocksdb::Iterator> {
return std::shared_ptr<rocksdb::Iterator>(
db_->NewIterator(rocksdb::ReadOptions(),
source_family ? source_family_ : default_family_));
}
auto meta_db::get_api_path_from_key(const std::string &key,
std::string &api_path) const -> api_error {
if (key.empty()) {
return api_error::item_not_found;
}
return perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
return db_->Get(rocksdb::ReadOptions(), keys_family_, key, &api_path);
});
}
auto meta_db::get_api_path_from_source(const std::string &source_path,
std::string &api_path) const
-> api_error {
if (source_path.empty()) {
return api_error::item_not_found;
}
return perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
return db_->Get(rocksdb::ReadOptions(), source_family_, source_path,
&api_path);
});
}
auto meta_db::get_item_meta_json(const std::string &api_path,
json &json_data) const -> api_error {
std::string value;
const auto res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
return db_->Get(rocksdb::ReadOptions(), default_family_, api_path, &value);
});
if (res != api_error::success) {
return res;
}
json_data = json::parse(value);
return api_error::success;
}
auto meta_db::get_item_meta(const std::string &api_path,
api_meta_map &meta) const -> api_error {
json json_data;
const auto ret = get_item_meta_json(api_path, json_data);
if (ret == api_error::success) {
for (auto it = json_data.begin(); it != json_data.end(); it++) {
meta[it.key()] = it.value().get<std::string>();
}
}
return ret;
}
auto meta_db::get_item_meta(const std::string &api_path, const std::string &key,
std::string &value) const -> api_error {
json json_data;
const auto ret = get_item_meta_json(api_path, json_data);
if (ret == api_error::success) {
if (json_data.find(key) != json_data.end()) {
value = json_data[key].get<std::string>();
}
}
return ret;
}
auto meta_db::get_item_meta_exists(const std::string &api_path) const -> bool {
std::string value;
return db_->Get(rocksdb::ReadOptions(), api_path, &value).ok();
}
auto meta_db::get_pinned_files() const -> std::vector<std::string> {
std::vector<std::string> ret;
auto iterator = const_cast<meta_db *>(this)->create_iterator(false);
for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
auto api_path = iterator->key().ToString();
std::string pinned;
const auto res = get_item_meta(api_path, META_PINNED, pinned);
if ((res == api_error::success) && not pinned.empty() &&
utils::string::to_bool(pinned)) {
ret.emplace_back(api_path);
}
}
return ret;
}
auto meta_db::get_source_path_exists(const std::string &source_path) const
-> bool {
std::string value;
return db_->Get(rocksdb::ReadOptions(), source_family_, source_path, &value)
.ok();
}
auto meta_db::perform_action(
const std::string &function_name,
const std::function<rocksdb::Status()> &action) const -> api_error {
const auto res = action();
if (res.ok()) {
return api_error::success;
}
if (not res.IsNotFound()) {
utils::error::raise_error(function_name, res.ToString());
}
return res.IsNotFound() ? api_error::item_not_found : api_error::error;
}
auto meta_db::get_total_item_count() const -> std::uint64_t {
std::uint64_t ret = 0u;
auto iter = create_iterator(false);
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
ret++;
}
return ret;
}
auto meta_db::remove_item_meta(const std::string &api_path) -> api_error {
json json_data;
auto res = get_item_meta_json(api_path, json_data);
if (res != api_error::success) {
return res == api_error::item_not_found ? api_error::success : res;
}
if (not json_data[META_KEY].empty()) {
if ((res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
return db_->Delete(rocksdb::WriteOptions(), keys_family_,
json_data[META_KEY].get<std::string>());
})) != api_error::success) {
return res;
}
}
if (not json_data[META_SOURCE].empty()) {
if ((res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
return db_->Delete(rocksdb::WriteOptions(), source_family_,
json_data[META_SOURCE].get<std::string>());
})) != api_error::success) {
return res;
}
}
return perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
return db_->Delete(rocksdb::WriteOptions(), default_family_, api_path);
});
}
auto meta_db::remove_item_meta(const std::string &api_path,
const std::string &key) -> api_error {
json json_data;
auto res = get_item_meta_json(api_path, json_data);
if (res != api_error::success) {
return res == api_error::item_not_found ? api_error::success : res;
}
if ((key == META_KEY) && not json_data[META_KEY].empty()) {
if ((res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
return db_->Delete(rocksdb::WriteOptions(), keys_family_,
json_data[META_KEY].get<std::string>());
})) != api_error::success) {
return res;
}
}
if ((key == META_SOURCE) && not json_data[META_SOURCE].empty()) {
if ((res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
return db_->Delete(rocksdb::WriteOptions(), source_family_,
json_data[META_SOURCE].get<std::string>());
})) != api_error::success) {
return res;
}
}
json_data.erase(key);
return perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
return db_->Put(rocksdb::WriteOptions(), default_family_, api_path,
json_data.dump());
});
}
auto meta_db::rename_item_meta(const std::string &source_path,
const std::string &from_api_path,
const std::string &to_api_path) -> api_error {
api_meta_map meta{};
auto res = get_item_meta(from_api_path, meta);
if (res != api_error::success) {
return res;
}
if ((res = remove_item_meta(from_api_path)) != api_error::success) {
return res;
}
if (not source_path.empty()) {
meta[META_SOURCE] = source_path;
}
return set_item_meta(to_api_path, meta);
}
auto meta_db::set_item_meta(const std::string &api_path, const std::string &key,
const std::string &value) -> api_error {
if (key == META_SOURCE) {
return set_source_path(api_path, value);
}
if (key == META_KEY) {
const auto res = remove_item_meta(api_path, META_KEY);
if ((res != api_error::success) && (res != api_error::item_not_found)) {
return res;
}
}
auto res = store_item_meta(api_path, key, value);
if (res != api_error::success) {
return res;
}
if (key == META_KEY) {
return perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
return db_->Put(rocksdb::WriteOptions(), keys_family_, value, api_path);
});
}
return api_error::success;
}
auto meta_db::set_item_meta(const std::string &api_path,
const api_meta_map &meta) -> api_error {
auto ret = api_error::success;
auto it = meta.begin();
for (std::size_t i = 0u; (ret == api_error::success) && (i < meta.size());
i++) {
ret = set_item_meta(api_path, it->first, it->second);
it++;
}
return ret;
}
auto meta_db::set_source_path(const std::string &api_path,
const std::string &source_path) -> api_error {
std::string current_source_path;
auto res = get_item_meta(api_path, META_SOURCE, current_source_path);
if ((res != api_error::success) && (res != api_error::item_not_found)) {
return res;
}
// TODO multiple db ops should be in transaction
if (not current_source_path.empty()) {
if ((res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
return db_->Delete(rocksdb::WriteOptions(), source_family_,
current_source_path);
})) != api_error::success) {
return res;
}
}
if (not source_path.empty()) {
if ((res = perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
return db_->Put(rocksdb::WriteOptions(), source_family_, source_path,
api_path);
})) != api_error::success) {
return res;
}
}
return store_item_meta(api_path, META_SOURCE, source_path);
}
auto meta_db::store_item_meta(const std::string &api_path,
const std::string &key, const std::string &value)
-> api_error {
json json_data;
auto res = get_item_meta_json(api_path, json_data);
if ((res != api_error::success) && (res != api_error::item_not_found)) {
return res;
}
json_data[key] = value;
return perform_action(__FUNCTION__, [&]() -> rocksdb::Status {
return db_->Put(rocksdb::WriteOptions(), default_family_, api_path,
json_data.dump());
});
}
} // namespace repertory

9
src/drives/fuse/fuse_drive.cpp
View File

@ -170,8 +170,7 @@ auto fuse_drive::create_impl(std::string api_path, mode_t mode,
auto meta = create_meta_attributes(
now, FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_ARCHIVE, now, now,
is_directory_op, "", get_effective_gid(), "", mode, now, 0U, osx_flags,
0U,
is_directory_op, get_effective_gid(), "", mode, now, 0U, osx_flags, 0U,
utils::path::combine(config_.get_cache_directory(),
{utils::create_uuid_string()}),
get_effective_uid(), now);
@ -589,9 +588,9 @@ auto fuse_drive::mkdir_impl(std::string api_path, mode_t mode) -> api_error {
}
const auto now = utils::get_file_time_now();
auto meta = create_meta_attributes(
now, FILE_ATTRIBUTE_DIRECTORY, now, now, true, "", get_effective_gid(),
"", mode, now, 0U, 0U, 0U, "", get_effective_uid(), now);
auto meta = create_meta_attributes(now, FILE_ATTRIBUTE_DIRECTORY, now, now,
true, get_effective_gid(), "", mode, now,
0U, 0U, 0U, "", get_effective_uid(), now);
if ((res = provider_.create_directory(api_path, meta)) !=
api_error::success) {
return res;

4
src/drives/winfsp/winfsp_drive.cpp
View File

@ -281,8 +281,8 @@ auto winfsp_drive::Create(PWSTR file_name, UINT32 create_options,
const auto now = utils::get_file_time_now();
auto meta = create_meta_attributes(
now, attributes, now, now, attributes & FILE_ATTRIBUTE_DIRECTORY, "", 0U,
"", 0U, now, 0U, 0U, 0U,
now, attributes, now, now, attributes & FILE_ATTRIBUTE_DIRECTORY, 0U, "",
0U, now, 0U, 0U, 0U,
(attributes & FILE_ATTRIBUTE_DIRECTORY)
? ""
: utils::path::combine(config_.get_cache_directory(),

4
src/file_manager/file_manager_upload.cpp
View File

@ -48,8 +48,8 @@ void file_manager::upload::cancel() {
void file_manager::upload::stop() { stop_requested_ = true; }
void file_manager::upload::upload_thread() {
error_ = provider_.upload_file(fsi_.api_path, fsi_.source_path,
fsi_.encryption_token, stop_requested_);
error_ =
provider_.upload_file(fsi_.api_path, fsi_.source_path, stop_requested_);
if (not utils::file::reset_modified_time(fsi_.source_path)) {
utils::error::raise_api_path_error(
__FUNCTION__, fsi_.api_path, fsi_.source_path,

13
src/platform/unix_platform.cpp
View File

@ -185,11 +185,10 @@ auto lock_data::wait_for_lock(int fd, std::uint8_t retry_count) -> int {
auto create_meta_attributes(
std::uint64_t accessed_date, std::uint32_t attributes,
std::uint64_t changed_date, std::uint64_t creation_date, bool directory,
const std::string &encryption_token, std::uint32_t gid,
const std::string &key, std::uint32_t mode, std::uint64_t modified_date,
std::uint32_t osx_backup, std::uint32_t osx_flags, std::uint64_t size,
const std::string &source_path, std::uint32_t uid,
std::uint64_t written_date) -> api_meta_map {
std::uint32_t gid, const std::string &key, std::uint32_t mode,
std::uint64_t modified_date, std::uint32_t osx_backup,
std::uint32_t osx_flags, std::uint64_t size, const std::string &source_path,
std::uint32_t uid, std::uint64_t written_date) -> api_meta_map {
return {
{META_ACCESSED, std::to_string(accessed_date)},
{META_ATTRIBUTES, std::to_string(attributes)},
@ -197,7 +196,6 @@ auto create_meta_attributes(
{META_CHANGED, std::to_string(changed_date)},
{META_CREATION, std::to_string(creation_date)},
{META_DIRECTORY, utils::string::from_bool(directory)},
{META_ENCRYPTION_TOKEN, encryption_token},
{META_GID, std::to_string(gid)},
{META_KEY, key},
{META_MODE, std::to_string(mode)},
@ -217,8 +215,7 @@ auto provider_meta_handler(i_provider &provider, bool directory,
file.accessed_date,
directory ? FILE_ATTRIBUTE_DIRECTORY
: FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_ARCHIVE,
file.changed_date, file.creation_date, directory, file.encryption_token,
getgid(), file.key,
file.changed_date, file.creation_date, directory, getgid(), file.key,
directory ? S_IFDIR | S_IRUSR | S_IWUSR | S_IXUSR
: S_IFREG | S_IRUSR | S_IWUSR,
file.modified_date, 0u, 0u, file.file_size, file.source_path, getuid(),

16
src/platform/win32_platform.cpp
View File

@ -166,11 +166,10 @@ auto lock_data::set_mount_state(bool active, const std::string &mount_location,
auto create_meta_attributes(
std::uint64_t accessed_date, std::uint32_t attributes,
std::uint64_t changed_date, std::uint64_t creation_date, bool directory,
const std::string &encryption_token, std::uint32_t gid,
const std::string &key, std::uint32_t mode, std::uint64_t modified_date,
std::uint32_t osx_backup, std::uint32_t osx_flags, std::uint64_t size,
const std::string &source_path, std::uint32_t uid,
std::uint64_t written_date) -> api_meta_map {
std::uint32_t gid, const std::string &key, std::uint32_t mode,
std::uint64_t modified_date, std::uint32_t osx_backup,
std::uint32_t osx_flags, std::uint64_t size, const std::string &source_path,
std::uint32_t uid, std::uint64_t written_date) -> api_meta_map {
return {
{META_ACCESSED, std::to_string(accessed_date)},
{META_ATTRIBUTES, std::to_string(attributes)},
@ -178,7 +177,6 @@ auto create_meta_attributes(
{META_CHANGED, std::to_string(changed_date)},
{META_CREATION, std::to_string(creation_date)},
{META_DIRECTORY, utils::string::from_bool(directory)},
{META_ENCRYPTION_TOKEN, encryption_token},
{META_GID, std::to_string(gid)},
{META_KEY, key},
{META_MODE, std::to_string(mode)},
@ -198,9 +196,9 @@ auto provider_meta_handler(i_provider &provider, bool directory,
file.accessed_date,
directory ? FILE_ATTRIBUTE_DIRECTORY
: FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_ARCHIVE,
file.changed_date, file.creation_date, directory, file.encryption_token,
0u, file.key, directory ? S_IFDIR : S_IFREG, file.modified_date, 0u, 0u,
file.file_size, file.source_path, 0u, file.modified_date);
file.changed_date, file.creation_date, directory, 0u, file.key,
directory ? S_IFDIR : S_IFREG, file.modified_date, 0u, 0u, file.file_size,
file.source_path, 0u, file.modified_date);
auto res = provider.set_item_meta(file.api_path, meta);
if (res == api_error::success) {
event_system::instance().raise<filesystem_item_added>(

546
src/providers/base_provider.cpp
View File

@ -1,546 +0,0 @@
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "providers/base_provider.hpp"
#include "app_config.hpp"
#include "file_manager/i_file_manager.hpp"
#include "types/repertory.hpp"
#include "types/startup_exception.hpp"
#include "utils/error_utils.hpp"
#include "utils/file_utils.hpp"
#include "utils/native_file.hpp"
#include "utils/path_utils.hpp"
namespace repertory {
base_provider::base_provider(app_config &config) : config_(config) {}
void base_provider::calculate_used_drive_space(bool add_missing) {
api_file_list list{};
if (get_file_list(list) != api_error::success) {
return;
}
used_space_ = std::accumulate(
list.begin(), list.end(), std::uint64_t(0U),
[this, &add_missing](std::uint64_t total_size, const auto &file) {
if (add_missing && not meta_db_->get_item_meta_exists(file.api_path)) {
[[maybe_unused]] auto res = this->notify_file_added(
file.api_path, utils::path::get_parent_api_path(file.api_path),
0);
}
return total_size + file.file_size;
});
}
void base_provider::cleanup() {
remove_deleted_files();
remove_unknown_source_files();
remove_expired_orphaned_files();
}
auto base_provider::create_directory_clone_source_meta(
const std::string &source_api_path, const std::string &api_path)
-> api_error {
api_meta_map meta{};
auto ret = get_item_meta(source_api_path, meta);
if (ret == api_error::success) {
ret = create_directory(api_path, meta);
}
return ret == api_error::item_not_found ? api_error::directory_not_found
: ret;
}
auto base_provider::create_file(const std::string &api_path, api_meta_map &meta)
-> api_error {
bool exists{};
auto res = is_directory(api_path, exists);
if (res != api_error::success) {
return res;
}
if (exists) {
return api_error::directory_exists;
}
res = is_file(api_path, exists);
if (res != api_error::success) {
return res;
}
if (exists) {
return api_error::item_exists;
}
if ((res = meta_db_->set_item_meta(api_path, meta)) != api_error::success) {
return res;
}
{
native_file_ptr nf;
res = native_file::create_or_open(meta[META_SOURCE], nf);
if (res != api_error::success) {
return res;
}
nf->close();
}
stop_type stop_requested = false;
return upload_file(api_path, meta[META_SOURCE], meta[META_ENCRYPTION_TOKEN],
stop_requested);
}
auto base_provider::get_api_path_from_source(const std::string &source_path,
std::string &api_path) const
-> api_error {
return meta_db_->get_api_path_from_source(source_path, api_path);
}
auto base_provider::get_directory_items(const std::string &api_path,
directory_item_list &list) const
-> api_error {
auto res = populate_directory_items(api_path, list);
if (res != api_error::success) {
return res;
}
std::sort(list.begin(), list.end(), [](const auto &a, const auto &b) -> bool {
return (a.directory && not b.directory) ||
(not(b.directory && not a.directory) &&
(a.api_path.compare(b.api_path) < 0));
});
list.insert(list.begin(), directory_item{
"..",
"",
true,
});
list.insert(list.begin(), directory_item{
".",
"",
true,
});
return api_error::success;
}
auto base_provider::get_file(const std::string &api_path, api_file &file) const
-> api_error {
auto ret = api_error::success;
try {
if ((ret = populate_file(api_path, file)) != api_error::success) {
event_system::instance().raise<file_get_failed>(api_path,
api_error_to_string(ret));
}
std::string sz;
if ((ret = get_item_meta(api_path, META_SIZE, sz)) != api_error::success) {
return ret;
}
file.file_size = utils::string::to_uint64(sz);
return ret;
} catch (const std::exception &e) {
event_system::instance().raise<file_get_failed>(
api_path, e.what() ? e.what() : "failed to get file");
}
return api_error::error;
}
auto base_provider::get_file_size(const std::string &api_path,
std::uint64_t &file_size) const -> api_error {
api_file file{};
const auto ret = get_file(api_path, file);
if (ret == api_error::success) {
file_size = file.file_size;
} else {
event_system::instance().raise<file_get_size_failed>(
api_path, api_error_to_string(ret));
}
return ret;
}
auto base_provider::get_filesystem_item(const std::string &api_path,
bool directory,
filesystem_item &fsi) const
-> api_error {
auto ret = api_error::error;
if (directory) {
bool exists{};
ret = is_directory(api_path, exists);
if (ret != api_error::success) {
return ret;
}
ret = exists ? api_error::success : api_error::item_not_found;
update_filesystem_item(true, ret, api_path, fsi);
} else {
api_file file{};
ret = get_filesystem_item_and_file(api_path, file, fsi);
}
return ret;
}
auto base_provider::get_filesystem_item_and_file(const std::string &api_path,
api_file &file,
filesystem_item &fsi) const
-> api_error {
auto ret = get_item_meta(api_path, META_SOURCE, fsi.source_path);
if (ret == api_error::success) {
ret = get_file(api_path, file);
if (ret == api_error::success) {
fsi.encryption_token = file.encryption_token;
fsi.size = file.file_size;
} else {
bool exists{};
ret = is_file(api_path, exists);
if (ret != api_error::success) {
return ret;
}
if (not exists) {
ret = api_error::item_not_found;
}
}
}
update_filesystem_item(false, ret, api_path, fsi);
return ret;
}
auto base_provider::get_filesystem_item_from_source_path(
const std::string &source_path, filesystem_item &fsi) const -> api_error {
auto ret = api_error::item_not_found;
if (not source_path.empty()) {
std::string api_path;
if ((ret = get_api_path_from_source(source_path, api_path)) ==
api_error::success) {
ret = get_filesystem_item(api_path, false, fsi);
}
}
return ret;
}
auto base_provider::get_item_meta(const std::string &api_path,
api_meta_map &meta) const -> api_error {
auto ret = meta_db_->get_item_meta(api_path, meta);
if (ret == api_error::item_not_found) {
auto get_meta = false;
bool exists{};
ret = is_directory(api_path, exists);
if (ret != api_error::success) {
return ret;
}
if (exists) {
ret = notify_directory_added(api_path,
utils::path::get_parent_api_path(api_path));
if (ret == api_error::success) {
get_meta = true;
}
} else {
ret = is_file(api_path, exists);
if (ret != api_error::success) {
return ret;
}
if (exists) {
std::uint64_t file_size{};
if ((ret = get_file_size(api_path, file_size)) == api_error::success) {
get_meta = ((ret = notify_file_added(
api_path, utils::path::get_parent_api_path(api_path),
file_size)) == api_error::success);
}
}
}
ret = get_meta ? meta_db_->get_item_meta(api_path, meta)
: api_error::item_not_found;
}
return ret;
}
auto base_provider::get_item_meta(const std::string &api_path,
const std::string &key,
std::string &value) const -> api_error {
auto ret = meta_db_->get_item_meta(api_path, key, value);
if (ret == api_error::item_not_found) {
auto get_meta = false;
bool exists{};
ret = is_directory(api_path, exists);
if (ret != api_error::success) {
return ret;
}
if (exists) {
ret = notify_directory_added(api_path,
utils::path::get_parent_api_path(api_path));
if (ret == api_error::success) {
get_meta = true;
}
} else {
ret = is_file(api_path, exists);
if (ret != api_error::success) {
return ret;
}
if (exists) {
std::uint64_t file_size{};
if ((ret = get_file_size(api_path, file_size)) == api_error::success) {
get_meta = ((ret = notify_file_added(
api_path, utils::path::get_parent_api_path(api_path),
file_size)) == api_error::success);
}
}
}
ret = get_meta ? meta_db_->get_item_meta(api_path, key, value)
: api_error::item_not_found;
}
return ret;
}
auto base_provider::get_used_drive_space() const -> std::uint64_t {
std::uint64_t used_space = used_space_;
fm_->update_used_space(used_space);
return used_space;
}
auto base_provider::notify_directory_added(const std::string &api_path,
const std::string &api_parent)
-> api_error {
recur_mutex_lock l(notify_added_mutex_);
const auto now = utils::get_file_time_now();
api_file file{};
file.api_path = api_path;
file.api_parent = api_parent;
file.accessed_date = now;
file.changed_date = now;
file.creation_date = now;
file.file_size = 0U;
file.modified_date = now;
return api_item_added_(true, file);
}
auto base_provider::processed_orphaned_file(const std::string &source_path,
const std::string &api_path) const
-> bool {
const auto orphaned_directory =
utils::path::combine(get_config().get_data_directory(), {"orphaned"});
if (utils::file::create_full_directory_path(orphaned_directory)) {
event_system::instance().raise<orphaned_file_detected>(source_path);
const auto parts = utils::string::split(api_path, '/', false);
const auto orphaned_file = utils::path::combine(
orphaned_directory, {utils::path::strip_to_file_name(source_path) +
'_' + parts[parts.size() - 1U]});
if (utils::file::reset_modified_time(source_path) &&
utils::file::move_file(source_path, orphaned_file)) {
event_system::instance().raise<orphaned_file_processed>(source_path,
orphaned_file);
return true;
}
event_system::instance().raise<orphaned_file_processing_failed>(
source_path, orphaned_file,
std::to_string(utils::get_last_error_code()));
return false;
}
utils::error::raise_error(
__FUNCTION__, std::to_string(utils::get_last_error_code()),
"failed to create orphaned director|sp|" + orphaned_directory);
return false;
}
void base_provider::remove_deleted_files() {
std::vector<std::string> removed_files{};
api_file_list list{};
if (get_file_list(list) == api_error::success) {
if (not list.empty()) {
auto iterator = meta_db_->create_iterator(false);
for (iterator->SeekToFirst(); not stop_requested_ && iterator->Valid();
iterator->Next()) {
const auto meta_api_path = iterator->key().ToString();
if (meta_api_path.empty()) {
const auto res = meta_db_->remove_item_meta(meta_api_path);
if (res != api_error::success) {
utils::error::raise_api_path_error(__FUNCTION__, meta_api_path, res,
"failed to remove item meta");
}
} else {
auto api_path = meta_api_path;
const auto it = std::find_if(list.begin(), list.end(),
[&api_path](const auto &file) -> bool {
return file.api_path == api_path;
});
if (it == list.end()) {
removed_files.emplace_back(api_path);
}
}
}
}
}
while (not stop_requested_ && not removed_files.empty()) {
const auto api_path = removed_files.back();
removed_files.pop_back();
bool exists{};
if (is_directory(api_path, exists) != api_error::success) {
continue;
}
std::string source_path;
if (not exists &&
(check_file_exists(api_path) == api_error::item_not_found) &&
(meta_db_->get_item_meta(api_path, META_SOURCE, source_path) ==
api_error::success)) {
if (not source_path.empty()) {
fm_->perform_locked_operation(
[this, &api_path, &source_path](i_provider &) -> bool {
if (fm_->has_no_open_file_handles()) {
auto res = meta_db_->remove_item_meta(api_path);
if (res == api_error::success) {
event_system::instance().raise<file_removed_externally>(
api_path, source_path);
processed_orphaned_file(source_path, api_path);
} else {
utils::error::raise_api_path_error(
__FUNCTION__, api_path, source_path, res,
"failed to remove item meta for externally removed file");
}
}
return true;
});
}
}
}
}
void base_provider::remove_expired_orphaned_files() {
const auto orphaned_directory =
utils::path::combine(get_config().get_data_directory(), {"orphaned"});
const auto files = utils::file::get_directory_files(orphaned_directory, true);
for (const auto &file : files) {
if (utils::file::is_modified_date_older_than(
file, std::chrono::hours(
get_config().get_orphaned_file_retention_days() * 24))) {
if (utils::file::retry_delete_file(file)) {
event_system::instance().raise<orphaned_file_deleted>(file);
}
}
if (stop_requested_) {
break;
}
}
}
void base_provider::remove_unknown_source_files() {
auto files = utils::file::get_directory_files(
get_config().get_cache_directory(), true);
while (not stop_requested_ && not files.empty()) {
const auto file = files.front();
files.pop_front();
std::string api_path;
if (not meta_db_->get_source_path_exists(file)) {
processed_orphaned_file(file);
}
}
}
auto base_provider::rename_file(const std::string &from_api_path,
const std::string &to_api_path) -> api_error {
std::string source_path;
auto ret = get_item_meta(from_api_path, META_SOURCE, source_path);
if (ret != api_error::success) {
return ret;
}
std::string encryption_token;
ret = get_item_meta(from_api_path, META_ENCRYPTION_TOKEN, encryption_token);
if (ret != api_error::success) {
return ret;
}
ret = handle_rename_file(from_api_path, to_api_path, source_path);
return ret;
}
auto base_provider::start(api_item_added_callback api_item_added,
i_file_manager *fm) -> bool {
meta_db_ = std::make_unique<meta_db>(config_);
api_item_added_ = api_item_added;
fm_ = fm;
auto unmount_requested = false;
{
repertory::event_consumer ec(
"unmount_requested",
[&unmount_requested](const event &) { unmount_requested = true; });
for (std::uint16_t i = 0U; not unmount_requested && not is_online() &&
(i < get_config().get_online_check_retry_secs());
i++) {
event_system::instance().raise<provider_offline>(
get_config().get_host_config().host_name_or_ip,
get_config().get_host_config().api_port);
std::this_thread::sleep_for(1s);
}
}
auto ret = not unmount_requested && is_online();
if (ret) {
// Force root creation
api_meta_map meta{};
auto res = get_item_meta("/", meta);
if (res != api_error::success) {
throw startup_exception("failed to create root|err|" +
api_error_to_string(res));
}
calculate_used_drive_space(false);
}
return ret;
}
void base_provider::stop() { meta_db_.reset(); }
void base_provider::update_filesystem_item(bool directory,
const api_error &error,
const std::string &api_path,
filesystem_item &fsi) const {
if (error == api_error::success) {
fsi.directory = directory;
fsi.api_path = api_path;
fsi.api_parent = utils::path::get_parent_api_path(api_path);
} else {
event_system::instance().raise<filesystem_item_get_failed>(
api_path, std::to_string(static_cast<int>(error)));
}
}
} // namespace repertory

2
src/providers/encrypt/encrypt_provider.cpp
View File

@ -33,7 +33,7 @@
namespace repertory {
encrypt_provider::encrypt_provider(app_config &config) : config_(config) {}
auto encrypt_provider::create_api_file(const std::string api_path,
auto encrypt_provider::create_api_file(const std::string &api_path,
bool directory,
const std::string &source_path)
-> api_file {

10
src/providers/provider.cpp
View File

@ -24,8 +24,6 @@
#include "app_config.hpp"
#include "comm/curl/curl_comm.hpp"
#include "comm/i_http_comm.hpp"
#include "comm/i_s3_comm.hpp"
#include "comm/s3/s3_comm.hpp"
#include "events/events.hpp"
#include "providers/encrypt/encrypt_provider.hpp"
#include "providers/s3/s3_provider.hpp"
@ -51,9 +49,6 @@ auto create_provider(const provider_type &pt, app_config &config)
mutex_lock lock(mutex);
static std::unique_ptr<i_http_comm> comm;
#if defined(REPERTORY_ENABLE_S3)
static std::unique_ptr<i_s3_comm> s3_comm_;
#endif // defined(REPERTORY_ENABLE_S3)
switch (pt) {
case provider_type::sia: {
@ -64,9 +59,10 @@ auto create_provider(const provider_type &pt, app_config &config)
}
#if defined(REPERTORY_ENABLE_S3)
case provider_type::s3: {
create_comm<i_s3_comm, s3_comm, app_config>(s3_comm_, config);
create_comm<i_http_comm, curl_comm, s3_config>(comm,
config.get_s3_config());
return std::unique_ptr<i_provider>(
dynamic_cast<i_provider *>(new s3_provider(config, *s3_comm_)));
dynamic_cast<i_provider *>(new s3_provider(config, *comm)));
}
#endif // defined(REPERTORY_ENABLE_S3)
case provider_type::encrypt: {

1480
src/providers/s3/s3_provider.cpp
View File

File diff suppressed because it is too large Load Diff

83
src/providers/sia/sia_provider.cpp
View File

@ -23,7 +23,6 @@
#include "app_config.hpp"
#include "comm/i_http_comm.hpp"
#include "db/meta_db.hpp"
#include "events/events.hpp"
#include "file_manager/i_file_manager.hpp"
#include "types/repertory.hpp"
@ -31,6 +30,7 @@
#include "utils/file_utils.hpp"
#include "utils/path_utils.hpp"
#include "utils/polling.hpp"
#include "utils/rocksdb_utils.hpp"
#include "utils/string_utils.hpp"
#include "utils/utils.hpp"
@ -47,7 +47,7 @@ auto sia_provider::get_object_info(const std::string &api_path,
get.response_handler = [&object_info](const data_buffer &data,
long response_code) {
if (response_code == 200) {
if (response_code == http_error_codes::ok) {
object_info = nlohmann::json::parse(data.begin(), data.end());
}
};
@ -58,11 +58,11 @@ auto sia_provider::get_object_info(const std::string &api_path,
return api_error::comm_error;
}
if (response_code == 404) {
if (response_code == http_error_codes::not_found) {
return api_error::item_not_found;
}
if (response_code != 200) {
if (response_code != http_error_codes::ok) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, response_code,
"failed to get object info");
return api_error::comm_error;
@ -77,7 +77,7 @@ auto sia_provider::get_object_info(const std::string &api_path,
return api_error::error;
}
auto sia_provider::get_object_list(const std::string api_path,
auto sia_provider::get_object_list(const std::string &api_path,
nlohmann::json &object_list) const -> bool {
curl::requests::http_get get{};
get.allow_timeout = true;
@ -85,7 +85,7 @@ auto sia_provider::get_object_list(const std::string api_path,
get.response_handler = [&object_list](const data_buffer &data,
long response_code) {
if (response_code == 200) {
if (response_code == http_error_codes::ok) {
object_list = nlohmann::json::parse(data.begin(), data.end());
}
};
@ -99,7 +99,7 @@ auto sia_provider::get_object_list(const std::string api_path,
return false;
}
if (response_code != 200) {
if (response_code != http_error_codes::ok) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, response_code,
"failed to get object list");
return false;
@ -166,8 +166,6 @@ auto sia_provider::create_directory(const std::string &api_path,
try {
curl::requests::http_put_file put_file{};
put_file.file_name =
*(utils::string::split(api_path, '/', false).end() - 1u);
put_file.path = "/api/worker/objects" + api_path + "/";
long response_code{};
@ -179,7 +177,7 @@ auto sia_provider::create_directory(const std::string &api_path,
return api_error::comm_error;
}
if (response_code != 200) {
if (response_code != http_error_codes::ok) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, response_code,
"failed to create directory");
return api_error::comm_error;
@ -272,8 +270,6 @@ auto sia_provider::create_file(const std::string &api_path, api_meta_map &meta)
try {
curl::requests::http_put_file put_file{};
put_file.file_name =
*(utils::string::split(api_path, '/', false).end() - 1u);
put_file.path = "/api/worker/objects" + api_path;
long response_code{};
@ -285,7 +281,7 @@ auto sia_provider::create_file(const std::string &api_path, api_meta_map &meta)
return api_error::comm_error;
}
if (response_code != 200) {
if (response_code != http_error_codes::ok) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, response_code,
"failed to create file");
return api_error::comm_error;
@ -716,7 +712,7 @@ auto sia_provider::get_total_drive_space() const -> std::uint64_t {
json config_data{};
get.response_handler = [&config_data](const data_buffer &data,
long response_code) {
if (response_code == 200) {
if (response_code == http_error_codes::ok) {
config_data = nlohmann::json::parse(data.begin(), data.end());
}
};
@ -727,7 +723,7 @@ auto sia_provider::get_total_drive_space() const -> std::uint64_t {
return 0U;
}
if (response_code != 200) {
if (response_code != http_error_codes::ok) {
utils::error::raise_error(__FUNCTION__, response_code,
"failed to get total drive space");
return 0U;
@ -762,7 +758,7 @@ auto sia_provider::get_used_drive_space() const -> std::uint64_t {
json object_data{};
get.response_handler = [&object_data](const data_buffer &data,
long response_code) {
if (response_code == 200) {
if (response_code == http_error_codes::ok) {
object_data = nlohmann::json::parse(data.begin(), data.end());
}
};
@ -773,7 +769,7 @@ auto sia_provider::get_used_drive_space() const -> std::uint64_t {
return 0U;
}
if (response_code != 200) {
if (response_code != http_error_codes::ok) {
utils::error::raise_error(__FUNCTION__, response_code,
"failed to get used drive space");
return 0U;
@ -797,6 +793,8 @@ auto sia_provider::is_directory(const std::string &api_path, bool &exists) const
return api_error::success;
}
exists = false;
try {
json object_list{};
if (not get_object_list(utils::path::get_parent_api_path(api_path),
@ -821,24 +819,31 @@ auto sia_provider::is_directory(const std::string &api_path, bool &exists) const
auto sia_provider::is_file(const std::string &api_path, bool &exists) const
-> api_error {
exists = false;
if (api_path == "/") {
return api_error::success;
}
json file_data{};
auto res = get_object_info(api_path, file_data);
if (res == api_error::item_not_found) {
exists = false;
try {
json file_data{};
auto res = get_object_info(api_path, file_data);
if (res == api_error::item_not_found) {
return api_error::success;
}
if (res != api_error::success) {
return res;
}
exists = not file_data.contains("entries");
return api_error::success;
} catch (const std::exception &e) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, e,
"failed to determine path is directory");
}
if (res != api_error::success) {
return res;
}
exists = not file_data.contains("entries");
return api_error::success;
return api_error::error;
}
auto sia_provider::is_file_writeable(const std::string &api_path) const
@ -861,7 +866,7 @@ auto sia_provider::is_online() const -> bool {
json state_data{};
get.response_handler = [&state_data](const data_buffer &data,
long response_code) {
if (response_code == 200) {
if (response_code == http_error_codes::ok) {
state_data = nlohmann::json::parse(data.begin(), data.end());
}
};
@ -874,7 +879,7 @@ auto sia_provider::is_online() const -> bool {
return false;
}
if (response_code != 200) {
if (response_code != http_error_codes::ok) {
utils::error::raise_error(__FUNCTION__, response_code,
"failed to determine if provider is online");
return false;
@ -897,7 +902,10 @@ auto sia_provider::read_file_bytes(const std::string &api_path,
stop_type &stop_requested) -> api_error {
curl::requests::http_get get{};
get.path = "/api/worker/objects" + api_path;
get.range = {{offset, offset + size - 1U}};
get.range = {{
offset,
offset + size - 1U,
}};
get.response_handler = [&buffer](const data_buffer &data,
long /*response_code*/) { buffer = data; };
@ -928,7 +936,8 @@ auto sia_provider::read_file_bytes(const std::string &api_path,
continue;
}
if (response_code < 200 || response_code >= 300) {
if (response_code < http_error_codes::ok ||
response_code >= http_error_codes::multiple_choices) {
notify_retry();
continue;
}
@ -1064,7 +1073,7 @@ auto sia_provider::remove_directory(const std::string &api_path) -> api_error {
return notify_end(api_error::comm_error);
}
if (response_code != 200) {
if (response_code != http_error_codes::ok) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, response_code,
"failed to remove directory");
return notify_end(api_error::comm_error);
@ -1138,7 +1147,8 @@ auto sia_provider::remove_file(const std::string &api_path) -> api_error {
return notify_end(api_error::comm_error);
}
if (response_code != 200 && response_code != 404) {
if (response_code != http_error_codes::ok &&
response_code != http_error_codes::not_found) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, response_code,
"failed to remove file");
return notify_end(api_error::comm_error);
@ -1283,7 +1293,6 @@ void sia_provider::stop() {
auto sia_provider::upload_file(const std::string &api_path,
const std::string &source_path,
const std::string & /* encryption_token */,
stop_type &stop_requested) -> api_error {
event_system::instance().raise<provider_upload_begin>(api_path, source_path);
@ -1296,8 +1305,6 @@ auto sia_provider::upload_file(const std::string &api_path,
try {
curl::requests::http_put_file put_file{};
put_file.file_name =
*(utils::string::split(api_path, '/', false).end() - 1u);
put_file.path = "/api/worker/objects" + api_path;
put_file.source_path = source_path;
@ -1309,7 +1316,7 @@ auto sia_provider::upload_file(const std::string &api_path,
return notify_end(api_error::comm_error);
}
if (response_code != 200) {
if (response_code != http_error_codes::ok) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, source_path,
response_code,
"failed to upload file");

120
src/utils/encrypting_reader.cpp
View File

@ -31,6 +31,12 @@
namespace repertory::utils::encryption {
class encrypting_streambuf final : public encrypting_reader::streambuf {
public:
encrypting_streambuf(const encrypting_streambuf &) = default;
encrypting_streambuf(encrypting_streambuf &&) = delete;
auto operator=(const encrypting_streambuf &)
-> encrypting_streambuf & = delete;
auto operator=(encrypting_streambuf &&) -> encrypting_streambuf & = delete;
explicit encrypting_streambuf(const encrypting_reader &reader)
: reader_(reader) {
setg(reinterpret_cast<char *>(0), reinterpret_cast<char *>(0),
@ -102,7 +108,7 @@ protected:
static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(gptr())));
char c{};
const auto res = reader_.reader_function(&c, 1u, 1u, &reader_);
const auto res = encrypting_reader::reader_function(&c, 1U, 1U, &reader_);
if (res != 1) {
return traits_type::eof();
}
@ -118,7 +124,8 @@ protected:
reader_.set_read_position(
static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(gptr())));
const auto res = reader_.reader_function(ptr, 1u, count, &reader_);
const auto res = encrypting_reader::reader_function(
ptr, 1U, static_cast<std::size_t>(count), &reader_);
if ((res == reader_.get_error_return()) ||
(reader_.get_stop_requested() && (res == CURL_READFUNC_ABORT))) {
return traits_type::eof();
@ -126,12 +133,20 @@ protected:
setg(eback(), gptr() + res,
reinterpret_cast<char *>(reader_.get_total_size()));
return res;
return static_cast<std::streamsize>(res);
}
};
class encrypting_reader_iostream final : public encrypting_reader::iostream {
public:
encrypting_reader_iostream(const encrypting_reader_iostream &) = delete;
encrypting_reader_iostream(encrypting_reader_iostream &&) = delete;
auto operator=(const encrypting_reader_iostream &)
-> encrypting_reader_iostream & = delete;
auto operator=(encrypting_reader_iostream &&)
-> encrypting_reader_iostream & = delete;
explicit encrypting_reader_iostream(
std::unique_ptr<encrypting_streambuf> buffer)
: encrypting_reader::iostream(buffer.get()), buffer_(std::move(buffer)) {}
@ -146,14 +161,14 @@ const std::size_t encrypting_reader::header_size_ = ([]() {
return crypto_aead_xchacha20poly1305_IETF_NPUBBYTES +
crypto_aead_xchacha20poly1305_IETF_ABYTES;
})();
const std::size_t encrypting_reader::data_chunk_size_ = (8u * 1024u * 1024u);
const std::size_t encrypting_reader::data_chunk_size_ = (8UL * 1024UL * 1024UL);
const std::size_t encrypting_reader::encrypted_chunk_size_ =
data_chunk_size_ + header_size_;
encrypting_reader::encrypting_reader(
const std::string &file_name, const std::string &source_path,
stop_type &stop_requested, const std::string &token,
std::optional<std::string> relative_parent_path, const size_t error_return)
std::optional<std::string> relative_parent_path, std::size_t error_return)
: key_(utils::encryption::generate_key(token)),
stop_requested_(stop_requested),
error_return_(error_return) {
@ -191,24 +206,22 @@ encrypting_reader::encrypting_reader(
file_size, static_cast<std::uint64_t>(data_chunk_size_)));
total_size_ =
file_size + (total_chunks * encrypting_reader::get_header_size());
last_data_chunk_ = total_chunks - 1u;
last_data_chunk_ = total_chunks - 1U;
last_data_chunk_size_ = static_cast<std::size_t>(
(file_size <= data_chunk_size_) ? file_size
: (file_size % data_chunk_size_) ? file_size % data_chunk_size_
: data_chunk_size_);
(file_size <= data_chunk_size_) ? file_size
: (file_size % data_chunk_size_) == 0U ? data_chunk_size_
: file_size % data_chunk_size_);
iv_list_.resize(total_chunks);
for (auto &iv : iv_list_) {
randombytes_buf(iv.data(), iv.size());
}
}
encrypting_reader::encrypting_reader(
const std::string &encrypted_file_path, const std::string &source_path,
stop_type &stop_requested, const std::string &token,
std::vector<
std::array<unsigned char, crypto_aead_xchacha20poly1305_IETF_NPUBBYTES>>
iv_list,
const size_t error_return)
encrypting_reader::encrypting_reader(const std::string &encrypted_file_path,
const std::string &source_path,
stop_type &stop_requested,
const std::string &token,
std::size_t error_return)
: key_(utils::encryption::generate_key(token)),
stop_requested_(stop_requested),
error_return_(error_return) {
@ -232,27 +245,68 @@ encrypting_reader::encrypting_reader(
file_size, static_cast<std::uint64_t>(data_chunk_size_)));
total_size_ =
file_size + (total_chunks * encrypting_reader::get_header_size());
last_data_chunk_ = total_chunks - 1u;
last_data_chunk_ = total_chunks - 1U;
last_data_chunk_size_ = static_cast<std::size_t>(
(file_size <= data_chunk_size_) ? file_size
: (file_size % data_chunk_size_) ? file_size % data_chunk_size_
: data_chunk_size_);
(file_size <= data_chunk_size_) ? file_size
: (file_size % data_chunk_size_) == 0U ? data_chunk_size_
: file_size % data_chunk_size_);
iv_list_.resize(total_chunks);
for (auto &iv : iv_list_) {
randombytes_buf(iv.data(), iv.size());
}
}
encrypting_reader::encrypting_reader(
const std::string &encrypted_file_path, const std::string &source_path,
stop_type &stop_requested, const std::string &token,
std::vector<
std::array<unsigned char, crypto_aead_xchacha20poly1305_IETF_NPUBBYTES>>
iv_list,
std::size_t error_return)
: key_(utils::encryption::generate_key(token)),
stop_requested_(stop_requested),
error_return_(error_return) {
const auto res = native_file::open(source_path, false, source_file_);
if (res != api_error::success) {
throw std::runtime_error("file open failed|src|" + source_path + '|' +
api_error_to_string(res));
}
encrypted_file_path_ = encrypted_file_path;
encrypted_file_name_ =
std::filesystem::path(encrypted_file_path_).filename().string();
std::uint64_t file_size{};
if (not utils::file::get_file_size(source_path, file_size)) {
throw std::runtime_error("get file size failed|src|" + source_path + '|' +
std::to_string(utils::get_last_error_code()));
}
const auto total_chunks = static_cast<std::size_t>(utils::divide_with_ceiling(
file_size, static_cast<std::uint64_t>(data_chunk_size_)));
total_size_ =
file_size + (total_chunks * encrypting_reader::get_header_size());
last_data_chunk_ = total_chunks - 1U;
last_data_chunk_size_ = static_cast<std::size_t>(
(file_size <= data_chunk_size_) ? file_size
: (file_size % data_chunk_size_) == 0U ? data_chunk_size_
: file_size % data_chunk_size_);
iv_list_ = std::move(iv_list);
}
encrypting_reader::encrypting_reader(const encrypting_reader &r)
: key_(r.key_),
stop_requested_(r.stop_requested_),
error_return_(r.error_return_),
chunk_buffers_(r.chunk_buffers_),
encrypted_file_name_(r.encrypted_file_name_),
encrypted_file_path_(r.encrypted_file_path_),
iv_list_(r.iv_list_),
last_data_chunk_(r.last_data_chunk_),
last_data_chunk_size_(r.last_data_chunk_size_),
read_offset_(r.read_offset_),
source_file_(native_file::clone(r.source_file_)),
total_size_(r.total_size_) {}
encrypting_reader::encrypting_reader(const encrypting_reader &reader)
: key_(reader.key_),
stop_requested_(reader.stop_requested_),
error_return_(reader.error_return_),
chunk_buffers_(reader.chunk_buffers_),
encrypted_file_name_(reader.encrypted_file_name_),
encrypted_file_path_(reader.encrypted_file_path_),
iv_list_(reader.iv_list_),
last_data_chunk_(reader.last_data_chunk_),
last_data_chunk_size_(reader.last_data_chunk_size_),
read_offset_(reader.read_offset_),
source_file_(native_file::clone(reader.source_file_)),
total_size_(reader.total_size_) {}
encrypting_reader::~encrypting_reader() {
if (source_file_) {

6
src/utils/encryption.cpp
View File

@ -104,7 +104,7 @@ auto read_encrypted_range(
const auto start_chunk =
static_cast<std::size_t>(range.begin / data_chunk_size);
const auto end_chunk = static_cast<std::size_t>(range.end / data_chunk_size);
auto remain = range.end - range.begin + 1u;
auto remain = range.end - range.begin + 1U;
auto source_offset = static_cast<std::size_t>(range.begin % data_chunk_size);
for (std::size_t chunk = start_chunk; chunk <= end_chunk; chunk++) {
@ -112,8 +112,8 @@ auto read_encrypted_range(
const auto start_offset = chunk * encrypted_chunk_size;
const auto end_offset = std::min(
start_offset + (total_size - (chunk * data_chunk_size)) + header_size -
1u,
static_cast<std::uint64_t>(start_offset + encrypted_chunk_size - 1u));
1U,
static_cast<std::uint64_t>(start_offset + encrypted_chunk_size - 1U));
const auto result = reader(ct, start_offset, end_offset);
if (result != api_error::success) {

321
tests/directory_db_test.cpp
View File

@ -1,321 +0,0 @@
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "test_common.hpp"
#include "fixtures/directory_db_fixture.hpp"
namespace repertory {
static const auto dirs = {"/",
"/root",
"/root/sub1",
"/root/sub2",
"/root/sub2/sub2_sub1",
"/root/sub2/sub2_sub2",
"/root/sub2/sub2_sub2/sub2_sub2_sub1",
"/root/sub3"};
TEST_F(directory_db_test, is_directory) {
for (const auto &dir : dirs) {
EXPECT_EQ(api_error::success, db_->create_directory(dir));
}
for (const auto &dir : dirs) {
EXPECT_TRUE(db_->is_directory(dir));
}
}
TEST_F(directory_db_test, remove_directory) {
for (const auto &dir : dirs) {
EXPECT_EQ(api_error::success, db_->create_directory(dir));
}
EXPECT_EQ(api_error::success, db_->remove_directory("/root/sub2/sub2_sub1"));
EXPECT_FALSE(db_->is_directory("/root/sub2/sub2_sub1"));
EXPECT_EQ(1u, db_->get_sub_directory_count("/root/sub2"));
EXPECT_TRUE(db_->is_directory("/root/sub2/sub2_sub2"));
}
TEST_F(directory_db_test, get_sub_directory_count) {
for (const auto &dir : dirs) {
EXPECT_EQ(api_error::success, db_->create_directory(dir));
}
EXPECT_EQ(1u, db_->get_sub_directory_count("/"));
EXPECT_EQ(3u, db_->get_sub_directory_count("/root"));
EXPECT_EQ(0u, db_->get_sub_directory_count("/root/sub1"));
EXPECT_EQ(2u, db_->get_sub_directory_count("/root/sub2"));
EXPECT_EQ(0u, db_->get_sub_directory_count("/root/sub2/sub2_sub1"));
EXPECT_EQ(1u, db_->get_sub_directory_count("/root/sub2/sub2_sub2"));
EXPECT_EQ(0u, db_->get_sub_directory_count("/root/sub3"));
}
TEST_F(directory_db_test, populate_sub_directories) {
for (const auto &dir : dirs) {
EXPECT_EQ(api_error::success, db_->create_directory(dir));
}
directory_item_list list{};
const auto dump_directory_list = [&]() {
for (const auto &di : list) {
std::cout << di.to_json().dump(2) << std::endl;
}
list.clear();
};
std::cout << "/" << std::endl;
db_->populate_sub_directories(
"/", [](directory_item &) {}, list);
EXPECT_EQ(1u, list.size());
dump_directory_list();
std::cout << std::endl << "/root" << std::endl;
db_->populate_sub_directories(
"/root", [](directory_item &) {}, list);
EXPECT_EQ(3u, list.size());
dump_directory_list();
std::cout << std::endl << "/root/sub1" << std::endl;
db_->populate_sub_directories(
"/root/sub1", [](directory_item &) {}, list);
EXPECT_EQ(0u, list.size());
dump_directory_list();
std::cout << std::endl << "/root/sub2" << std::endl;
db_->populate_sub_directories(
"/root/sub2", [](directory_item &) {}, list);
EXPECT_EQ(2u, list.size());
dump_directory_list();
std::cout << std::endl << "/root/sub2/sub2_sub1" << std::endl;
db_->populate_sub_directories(
"/root/sub2/sub2_sub1", [](directory_item &) {}, list);
EXPECT_EQ(0u, list.size());
dump_directory_list();
std::cout << std::endl << "/root/sub2/sub2_sub2" << std::endl;
db_->populate_sub_directories(
"/root/sub2/sub2_sub2", [](directory_item &) {}, list);
EXPECT_EQ(1u, list.size());
dump_directory_list();
std::cout << std::endl << "/root/sub3" << std::endl;
db_->populate_sub_directories(
"/root/sub3", [](directory_item &) {}, list);
EXPECT_EQ(0u, list.size());
dump_directory_list();
}
TEST_F(directory_db_test, is_file) {
for (const auto &dir : dirs) {
EXPECT_EQ(api_error::success, db_->create_directory(dir));
}
EXPECT_EQ(api_error::success, db_->create_file("/cow.txt"));
EXPECT_TRUE(db_->is_file("/cow.txt"));
EXPECT_FALSE(db_->is_directory("/cow.txt"));
EXPECT_EQ(api_error::item_exists, db_->create_file("/cow.txt"));
EXPECT_EQ(api_error::item_exists, db_->create_directory("/cow.txt"));
}
TEST_F(directory_db_test, remove_file) {
for (const auto &dir : dirs) {
EXPECT_EQ(api_error::success, db_->create_directory(dir));
}
EXPECT_EQ(api_error::success, db_->create_file("/cow.txt"));
EXPECT_EQ(api_error::directory_not_found, db_->remove_directory("/cow.txt"));
EXPECT_TRUE(db_->remove_file("/cow.txt"));
EXPECT_FALSE(db_->is_file("/cow.txt"));
}
TEST_F(directory_db_test, get_directory_item_count) {
EXPECT_EQ(api_error::success, db_->create_directory("/"));
EXPECT_EQ(api_error::success, db_->create_file("/cow.txt"));
EXPECT_EQ(api_error::success, db_->create_file("/cow2.txt"));
EXPECT_EQ(api_error::success, db_->create_directory("/cow"));
EXPECT_EQ(3u, db_->get_directory_item_count("/"));
}
TEST_F(directory_db_test, get_file) {
EXPECT_EQ(api_error::success, db_->create_directory("/"));
EXPECT_EQ(api_error::success, db_->create_file("/cow.txt"));
api_file file{};
EXPECT_EQ(api_error::success,
db_->get_file("/cow.txt", file, [](api_file &cur_file) {
EXPECT_STREQ("/cow.txt", cur_file.api_path.c_str());
}));
EXPECT_STREQ("/cow.txt", file.api_path.c_str());
}
TEST_F(directory_db_test, get_file_list) {
EXPECT_EQ(api_error::success, db_->create_directory("/"));
EXPECT_EQ(api_error::success, db_->create_file("/cow.txt"));
EXPECT_EQ(api_error::success, db_->create_file("/cow2.txt"));
api_file_list list;
int i = 0;
EXPECT_EQ(api_error::success, db_->get_file_list(list, [&i](api_file &file) {
if (i++ == 0) {
EXPECT_STREQ("/cow.txt", file.api_path.c_str());
} else {
EXPECT_STREQ("/cow2.txt", file.api_path.c_str());
}
}));
EXPECT_EQ(std::size_t(2u), list.size());
EXPECT_STREQ("/cow.txt", list[0u].api_path.c_str());
EXPECT_STREQ("/cow2.txt", list[1u].api_path.c_str());
}
TEST_F(directory_db_test, get_total_item_count) {
EXPECT_EQ(api_error::success, db_->create_directory("/"));
EXPECT_EQ(api_error::success, db_->create_file("/cow.txt"));
EXPECT_EQ(api_error::success, db_->create_file("/cow2.txt"));
EXPECT_EQ(api_error::success, db_->create_directory("/cow"));
EXPECT_EQ(api_error::success, db_->create_directory("/cow/moose"));
EXPECT_EQ(std::uint64_t(5), db_->get_total_item_count());
}
TEST_F(directory_db_test, populate_directory_files) {
EXPECT_EQ(api_error::success, db_->create_directory("/"));
EXPECT_EQ(api_error::success, db_->create_file("/cow.txt"));
EXPECT_EQ(api_error::success, db_->create_file("/cow2.txt"));
directory_item_list list;
int i = 0;
db_->populate_directory_files(
"/",
[&i](directory_item &di) {
di.meta[META_SIZE] = std::to_string(i + 1);
EXPECT_FALSE(di.directory);
if (i++ == 0) {
EXPECT_STREQ("/cow.txt", &di.api_path[0]);
} else {
EXPECT_STREQ("/cow2.txt", &di.api_path[0]);
}
},
list);
EXPECT_EQ(std::size_t(2u), list.size());
EXPECT_EQ(1u, list[0].size);
EXPECT_STREQ("/cow.txt", &list[0].api_path[0]);
EXPECT_EQ(2u, list[1].size);
EXPECT_STREQ("/cow2.txt", &list[1].api_path[0]);
}
TEST_F(directory_db_test, create_directory_fails_if_directory_exists) {
for (const auto &dir : dirs) {
EXPECT_EQ(api_error::success, db_->create_directory(dir));
}
EXPECT_EQ(api_error::directory_exists, db_->create_file("/root/sub1"));
EXPECT_TRUE(db_->is_directory("/root/sub1"));
}
TEST_F(directory_db_test, create_file_fails_if_file_exists) {
for (const auto &dir : dirs) {
EXPECT_EQ(api_error::success, db_->create_directory(dir));
}
EXPECT_EQ(api_error::success, db_->create_file("/cow.txt"));
EXPECT_EQ(api_error::item_exists, db_->create_directory("/cow.txt"));
}
TEST_F(directory_db_test, create_file_fails_if_parent_does_not_exist) {
for (const auto &dir : dirs) {
EXPECT_EQ(api_error::success, db_->create_directory(dir));
}
EXPECT_EQ(api_error::directory_not_found, db_->create_file("/moose/cow.txt"));
}
TEST_F(directory_db_test, create_directory_fails_if_parent_does_not_exist) {
for (const auto &dir : dirs) {
EXPECT_EQ(api_error::success, db_->create_directory(dir));
}
EXPECT_EQ(api_error::directory_not_found, db_->create_file("/cow/moose"));
}
TEST_F(directory_db_test, remove_file_fails_if_directory_exists) {
for (const auto &dir : dirs) {
EXPECT_EQ(api_error::success, db_->create_directory(dir));
}
EXPECT_FALSE(db_->remove_file("/root/sub1"));
}
TEST_F(directory_db_test, remove_directory_fails_if_file_exists) {
for (const auto &dir : dirs) {
EXPECT_EQ(api_error::success, db_->create_directory(dir));
}
EXPECT_EQ(api_error::success, db_->create_file("/cow.txt"));
EXPECT_EQ(api_error::directory_not_found, db_->remove_directory("/cow.txt"));
}
TEST_F(directory_db_test, remove_directory_fails_if_sub_directories_exist) {
EXPECT_EQ(api_error::success, db_->create_directory("/"));
EXPECT_EQ(api_error::success, db_->create_directory("/sub"));
EXPECT_EQ(api_error::success, db_->create_directory("/sub/sub2"));
EXPECT_EQ(api_error::directory_not_empty, db_->remove_directory("/sub"));
EXPECT_TRUE(db_->is_directory("/sub"));
EXPECT_TRUE(db_->is_directory("/sub/sub2"));
}
TEST_F(directory_db_test, remove_directory_fails_if_files_exist) {
EXPECT_EQ(api_error::success, db_->create_directory("/"));
EXPECT_EQ(api_error::success, db_->create_directory("/sub"));
EXPECT_EQ(api_error::success, db_->create_file("/sub/test.txt"));
EXPECT_EQ(api_error::directory_not_empty, db_->remove_directory("/sub"));
EXPECT_TRUE(db_->is_directory("/sub"));
EXPECT_TRUE(db_->is_file("/sub/test.txt"));
}
TEST_F(directory_db_test,
remove_directory_fails_for_root_directory_by_default) {
EXPECT_EQ(api_error::success, db_->create_directory("/"));
EXPECT_EQ(api_error::access_denied, db_->remove_directory("/"));
EXPECT_TRUE(db_->is_directory("/"));
}
TEST_F(
directory_db_test,
remove_directory_succeeds_for_root_directory_if_allow_remove_root_is_true) {
EXPECT_EQ(api_error::success, db_->create_directory("/"));
EXPECT_EQ(api_error::success, db_->remove_directory("/", true));
EXPECT_FALSE(db_->is_directory("/"));
}
} // namespace repertory

2
tests/encrypt_provider_test.cpp
View File

@ -81,8 +81,6 @@ TEST(encrypt_provider, can_get_file_list) {
list2.at(idx).api_parent.c_str());
EXPECT_EQ(list.at(idx).accessed_date, list2.at(idx).accessed_date);
EXPECT_EQ(list.at(idx).changed_date, list2.at(idx).changed_date);
EXPECT_TRUE(list.at(idx).encryption_token.empty());
EXPECT_TRUE(list2.at(idx).encryption_token.empty());
EXPECT_EQ(list.at(idx).file_size, list2.at(idx).file_size);
EXPECT_TRUE(list.at(idx).key.empty());
EXPECT_TRUE(list2.at(idx).key.empty());

45
tests/file_manager_test.cpp
View File

@ -114,8 +114,8 @@ TEST(file_manager, can_create_and_close_file) {
const auto now = utils::get_file_time_now();
auto meta = create_meta_attributes(
now, FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_ARCHIVE, now + 1u,
now + 2u, false, "token", 1, "key", 2, now + 3u, 3u, 4u, 0u,
source_path, 10, now + 4u);
now + 2u, false, 1, "key", 2, now + 3u, 3u, 4u, 0u, source_path, 10,
now + 4u);
EXPECT_CALL(mp, create_file("/test_create.txt", meta))
.WillOnce(Return(api_error::success));
@ -127,7 +127,6 @@ TEST(file_manager, can_create_and_close_file) {
fsi.api_path = api_path;
fsi.api_parent = utils::path::get_parent_api_path(api_path);
fsi.directory = directory;
fsi.encryption_token = meta[META_ENCRYPTION_TOKEN];
fsi.size = utils::string::to_uint64(meta[META_SIZE]);
fsi.source_path = meta[META_SOURCE];
return api_error::success;
@ -223,8 +222,8 @@ TEST(file_manager, can_open_and_close_file) {
const auto now = utils::get_file_time_now();
auto meta = create_meta_attributes(
now, FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_ARCHIVE, now + 1u,
now + 2u, false, "token", 1, "key", 2, now + 3u, 3u, 4u, 0u,
source_path, 10, now + 4u);
now + 2u, false, 1, "key", 2, now + 3u, 3u, 4u, 0u, source_path, 10,
now + 4u);
EXPECT_CALL(mp, create_file).Times(0u);
@ -236,7 +235,6 @@ TEST(file_manager, can_open_and_close_file) {
fsi.api_path = api_path;
fsi.api_parent = utils::path::get_parent_api_path(api_path);
fsi.directory = directory;
fsi.encryption_token = meta[META_ENCRYPTION_TOKEN];
fsi.size = utils::string::to_uint64(meta[META_SIZE]);
fsi.source_path = meta[META_SOURCE];
return api_error::success;
@ -325,8 +323,8 @@ TEST(file_manager, can_open_and_close_multiple_handles_for_same_file) {
const auto now = utils::get_file_time_now();
auto meta = create_meta_attributes(
now, FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_ARCHIVE, now + 1u,
now + 2u, false, "token", 1, "key", 2, now + 3u, 3u, 4u, 0u,
source_path, 10, now + 4u);
now + 2u, false, 1, "key", 2, now + 3u, 3u, 4u, 0u, source_path, 10,
now + 4u);
EXPECT_CALL(mp, create_file).Times(0u);
@ -338,7 +336,6 @@ TEST(file_manager, can_open_and_close_multiple_handles_for_same_file) {
fsi.api_path = api_path;
fsi.api_parent = utils::path::get_parent_api_path(api_path);
fsi.directory = directory;
fsi.encryption_token = meta[META_ENCRYPTION_TOKEN];
fsi.size = utils::string::to_uint64(meta[META_SIZE]);
fsi.source_path = meta[META_SOURCE];
return api_error::success;
@ -404,7 +401,7 @@ TEST(file_manager, download_is_stored_after_write_if_partially_downloaded) {
const auto now = utils::get_file_time_now();
auto meta = create_meta_attributes(
now, FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_ARCHIVE, now + 1u, now + 2u,
false, "", 1, "key", 2, now + 3u, 3u, 4u,
false, 1, "key", 2, now + 3u, 3u, 4u,
utils::encryption::encrypting_reader::get_data_chunk_size() * 4u,
source_path, 10, now + 4u);
auto nf = create_random_file(generate_test_file_name(".", "test_src"),
@ -418,7 +415,6 @@ TEST(file_manager, download_is_stored_after_write_if_partially_downloaded) {
fsi.api_path = api_path;
fsi.api_parent = utils::path::get_parent_api_path(api_path);
fsi.directory = directory;
fsi.encryption_token = meta[META_ENCRYPTION_TOKEN];
fsi.size = utils::string::to_uint64(meta[META_SIZE]);
fsi.source_path = meta[META_SOURCE];
return api_error::success;
@ -566,7 +562,7 @@ TEST(file_manager, upload_occurs_after_write_if_fully_downloaded) {
const auto now = utils::get_file_time_now();
auto meta = create_meta_attributes(
now, FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_ARCHIVE, now + 1u, now + 2u,
false, "", 1, "key", 2, now + 3u, 3u, 4u,
false, 1, "key", 2, now + 3u, 3u, 4u,
utils::encryption::encrypting_reader::get_data_chunk_size() * 4u,
source_path, 10, now + 4u);
auto nf = create_random_file(generate_test_file_name(".", "test_src"),
@ -580,7 +576,6 @@ TEST(file_manager, upload_occurs_after_write_if_fully_downloaded) {
fsi.api_path = api_path;
fsi.api_parent = utils::path::get_parent_api_path(api_path);
fsi.directory = directory;
fsi.encryption_token = meta[META_ENCRYPTION_TOKEN];
fsi.size = utils::string::to_uint64(meta[META_SIZE]);
fsi.source_path = meta[META_SOURCE];
return api_error::success;
@ -625,8 +620,8 @@ TEST(file_manager, upload_occurs_after_write_if_fully_downloaded) {
ec.wait_for_empty();
EXPECT_CALL(
mp, upload_file("/test_write_full_download.txt", source_path, "", _))
EXPECT_CALL(mp,
upload_file("/test_write_full_download.txt", source_path, _))
.WillOnce(Return(api_error::success));
event_capture ec2({"file_upload_queued", "file_upload_completed"});
@ -634,8 +629,8 @@ TEST(file_manager, upload_occurs_after_write_if_fully_downloaded) {
ec2.wait_for_empty();
EXPECT_EQ(std::size_t(0u), fm.get_open_file_count());
EXPECT_EQ(std::size_t(0u), fm.get_open_handle_count());
EXPECT_EQ(std::size_t(0U), fm.get_open_file_count());
EXPECT_EQ(std::size_t(0U), fm.get_open_handle_count());
fm.stop();
@ -676,8 +671,7 @@ TEST(file_manager, can_evict_file) {
auto meta = create_meta_attributes(
now, FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_ARCHIVE, now + 1u, now + 2u,
false, "token", 1, "key", 2, now + 3u, 3u, 4u, 0u, source_path, 10,
now + 4u);
false, 1, "key", 2, now + 3u, 3u, 4u, 0u, source_path, 10, now + 4u);
std::uint64_t handle{};
{
std::shared_ptr<i_open_file> f;
@ -692,7 +686,6 @@ TEST(file_manager, can_evict_file) {
fsi.api_path = api_path;
fsi.api_parent = utils::path::get_parent_api_path(api_path);
fsi.directory = directory;
fsi.encryption_token = meta[META_ENCRYPTION_TOKEN];
fsi.size = utils::string::to_uint64(meta[META_SIZE]);
fsi.source_path = meta[META_SOURCE];
return api_error::success;
@ -708,7 +701,7 @@ TEST(file_manager, can_evict_file) {
EXPECT_CALL(mp, set_item_meta("/test_evict.txt", _))
.Times(2)
.WillRepeatedly(Return(api_error::success));
EXPECT_CALL(mp, upload_file(_, _, _, _))
EXPECT_CALL(mp, upload_file(_, _, _))
.WillOnce(Return(api_error::success));
data_buffer data{{0, 1, 1}};
@ -944,7 +937,7 @@ TEST(file_manager, evict_file_fails_if_file_is_uploading) {
auto meta = create_meta_attributes(
now, FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_ARCHIVE, now + 1u, now + 2u,
false, "", 1, "", 2, now + 3u, 3u, 4u, 0u, source_path, 10, now + 4u);
false, 1, "", 2, now + 3u, 3u, 4u, 0u, source_path, 10, now + 4u);
std::uint64_t handle{};
{
std::shared_ptr<i_open_file> f;
@ -959,7 +952,6 @@ TEST(file_manager, evict_file_fails_if_file_is_uploading) {
fsi.api_path = api_path;
fsi.api_parent = utils::path::get_parent_api_path(api_path);
fsi.directory = directory;
fsi.encryption_token = meta[META_ENCRYPTION_TOKEN];
fsi.size = utils::string::to_uint64(meta[META_SIZE]);
fsi.source_path = meta[META_SOURCE];
return api_error::success;
@ -978,11 +970,9 @@ TEST(file_manager, evict_file_fails_if_file_is_uploading) {
EXPECT_CALL(mp, upload_file)
.WillOnce([](const std::string &api_path,
const std::string &source_path2,
const std::string &encryption_token,
stop_type & /*stop_requested*/) -> api_error {
EXPECT_STREQ("/test_evict.txt", api_path.c_str());
EXPECT_FALSE(source_path2.empty());
EXPECT_TRUE(encryption_token.empty());
std::this_thread::sleep_for(3s);
return api_error::success;
});
@ -1178,7 +1168,7 @@ TEST(file_manager, file_is_not_opened_if_provider_create_file_fails) {
const auto now = utils::get_file_time_now();
auto meta = create_meta_attributes(
now, FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_ARCHIVE, now + 1u, now + 2u,
false, "", 1, "", 2, now + 3u, 3u, 4u, 0u, "/test_create.src", 10,
false, 1, "", 2, now + 3u, 3u, 4u, 0u, "/test_create.src", 10,
now + 4u);
file_manager fm(cfg, mp);
@ -1721,7 +1711,7 @@ TEST(file_manager, file_is_closed_after_download_timeout) {
const auto now = utils::get_file_time_now();
auto meta = create_meta_attributes(
now, FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_ARCHIVE, now + 1u, now + 2u,
false, "", 1, "key", 2, now + 3u, 3u, 4u,
false, 1, "key", 2, now + 3u, 3u, 4u,
utils::encryption::encrypting_reader::get_data_chunk_size() * 4u,
source_path, 10, now + 4u);
@ -1733,7 +1723,6 @@ TEST(file_manager, file_is_closed_after_download_timeout) {
fsi.api_path = api_path;
fsi.api_parent = utils::path::get_parent_api_path(api_path);
fsi.directory = directory;
fsi.encryption_token = meta[META_ENCRYPTION_TOKEN];
fsi.size = utils::string::to_uint64(meta[META_SIZE]);
fsi.source_path = meta[META_SOURCE];
return api_error::success;

12
tests/file_manager_upload_test.cpp
View File

@ -56,10 +56,8 @@ TEST(upload, can_upload_a_valid_file) {
EXPECT_STREQ("0", ee.get_cancelled().get<std::string>().c_str());
});
EXPECT_CALL(
mp, upload_file(fsi.api_path, fsi.source_path, fsi.encryption_token, _))
EXPECT_CALL(mp, upload_file(fsi.api_path, fsi.source_path, _))
.WillOnce([&fsi](const std::string &, const std::string &,
const std::string &,
stop_type &stop_requested) -> api_error {
EXPECT_FALSE(stop_requested);
return api_error::success;
@ -105,10 +103,8 @@ TEST(upload, can_cancel_upload) {
std::mutex mtx;
std::condition_variable cv;
EXPECT_CALL(
mp, upload_file(fsi.api_path, fsi.source_path, fsi.encryption_token, _))
EXPECT_CALL(mp, upload_file(fsi.api_path, fsi.source_path, _))
.WillOnce([&cv, &fsi, &mtx](const std::string &, const std::string &,
const std::string &,
stop_type &stop_requested) -> api_error {
EXPECT_FALSE(stop_requested);
@ -169,10 +165,8 @@ TEST(upload, can_stop_upload) {
EXPECT_STREQ("0", ee.get_cancelled().get<std::string>().c_str());
});
EXPECT_CALL(
mp, upload_file(fsi.api_path, fsi.source_path, fsi.encryption_token, _))
EXPECT_CALL(mp, upload_file(fsi.api_path, fsi.source_path, _))
.WillOnce([&fsi](const std::string &, const std::string &,
const std::string &,
stop_type &stop_requested) -> api_error {
std::this_thread::sleep_for(3s);
EXPECT_TRUE(stop_requested);

57
tests/fixtures/directory_db_fixture.hpp vendored
View File

@ -1,57 +0,0 @@
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#ifndef REPERTORY_DIRECTORY_DB_FIXTURE_H
#define REPERTORY_DIRECTORY_DB_FIXTURE_H
#include "test_common.hpp"
#include "app_config.hpp"
#include "db/directory_db.hpp"
#include "utils/path_utils.hpp"
namespace repertory {
class directory_db_test : public ::testing::Test {
private:
const std::string config_location_ = utils::path::absolute("./directorydb");
protected:
std::unique_ptr<app_config> config_;
std::unique_ptr<directory_db> db_;
public:
void SetUp() override {
ASSERT_TRUE(utils::file::delete_directory_recursively(config_location_));
config_ =
std::make_unique<app_config>(provider_type::encrypt, config_location_);
db_ = std::make_unique<directory_db>(*config_.get());
}
void TearDown() override {
db_.reset();
config_.reset();
EXPECT_TRUE(utils::file::delete_directory_recursively(config_location_));
}
};
} // namespace repertory
#endif // REPERTORY_DIRECTORY_DB_FIXTURE_H

57
tests/fixtures/meta_db_fixture.hpp vendored
View File

@ -1,57 +0,0 @@
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#ifndef REPERTORY_META_DB_FIXTURE_H
#define REPERTORY_META_DB_FIXTURE_H
#include "test_common.hpp"
#include "app_config.hpp"
#include "db/meta_db.hpp"
#include "utils/path_utils.hpp"
namespace repertory {
class meta_db_test : public ::testing::Test {
private:
const std::string config_location_ = utils::path::absolute("./metadb");
protected:
std::unique_ptr<app_config> config_;
std::unique_ptr<meta_db> db_;
public:
void SetUp() override {
ASSERT_TRUE(utils::file::delete_directory_recursively(config_location_));
config_ =
std::make_unique<app_config>(provider_type::sia, config_location_);
db_ = std::make_unique<meta_db>(*config_.get());
}
void TearDown() override {
db_.reset();
config_.reset();
EXPECT_TRUE(utils::file::delete_directory_recursively(config_location_));
}
};
} // namespace repertory
#endif // REPERTORY_META_DB_FIXTURE_H

88
tests/fixtures/s3_comm_fixture.hpp vendored
View File

@ -1,88 +0,0 @@
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#ifndef REPERTORY_S3_COMM_FIXTURE_H
#define REPERTORY_S3_COMM_FIXTURE_H
#if defined(REPERTORY_ENABLE_S3) && defined(REPERTORY_ENABLE_S3_TESTING)
#include "test_common.hpp"
#include "app_config.hpp"
#include "comm/s3/s3_comm.hpp"
#include "utils/path_utils.hpp"
namespace repertory {
class s3_comm_test : public ::testing::Test {
private:
console_consumer c_;
protected:
std::unique_ptr<app_config> config_;
std::unique_ptr<s3_comm> s3_comm_;
public:
void SetUp() override {
const auto path = utils::path::absolute("./test/");
ASSERT_TRUE(utils::file::delete_directory_recursively(path));
{
app_config config(provider_type::s3,
utils::path::combine(get_test_dir(), {"filebase"}));
config_ = std::make_unique<app_config>(provider_type::s3, "./test");
config_->set_event_level(event_level::verbose);
EXPECT_FALSE(config_
->set_value_by_name("S3Config.AccessKey",
config.get_s3_config().access_key)
.empty());
EXPECT_FALSE(config_
->set_value_by_name("S3Config.SecretKey",
config.get_s3_config().secret_key)
.empty());
EXPECT_FALSE(config_
->set_value_by_name("S3Config.Region",
config.get_s3_config().region)
.empty());
EXPECT_FALSE(
config_->set_value_by_name("S3Config.URL", config.get_s3_config().url)
.empty());
EXPECT_FALSE(
config_->set_value_by_name("S3Config.Bucket", "repertory").empty());
}
s3_comm_ = std::make_unique<s3_comm>(*config_);
event_system::instance().start();
}
void TearDown() override {
event_system::instance().stop();
s3_comm_.reset();
config_.reset();
const auto path = utils::path::absolute("./test/");
EXPECT_TRUE(utils::file::delete_directory_recursively(path));
}
};
} // namespace repertory
#endif // REPERTORY_ENABLE_S3_TESTING
#endif // REPERTORY_S3_COMM_FIXTURE_H

4
tests/fixtures/s3_provider_file_list_fixture.hpp vendored
View File

@ -23,7 +23,7 @@
#define REPERTORY_S3_PROVIDER_FILE_LIST_FIXTURE_H
#if defined(REPERTORY_ENABLE_S3) && defined(REPERTORY_ENABLE_S3_TESTING)
#if 0
#include "test_common.hpp"
#include "mocks/mock_s3_comm.hpp"
@ -76,7 +76,6 @@ public:
file.api_parent = utils::path::get_parent_api_path(file.api_path);
file.changed_date = times[idx] + 1u;
file.creation_date = times[idx] + 2u;
file.encryption_token = "";
file.file_size = 100u + idx;
file.modified_date = times[idx] + 3u;
this->list.emplace_back(std::move(file));
@ -145,5 +144,6 @@ public:
};
} // namespace repertory
#endif
#endif // REPERTORY_ENABLE_S3_TESTING
#endif // REPERTORY_S3_PROVIDER_FILE_LIST_FIXTURE_H

10
tests/fuse_drive_test.cpp
View File

@ -25,7 +25,6 @@
#include "app_config.hpp"
#include "comm/curl/curl_comm.hpp"
#include "comm/s3/s3_comm.hpp"
#include "drives/fuse/fuse_drive.hpp"
#include "platform/platform.hpp"
#include "providers/s3/s3_provider.hpp"
@ -54,10 +53,8 @@ static void execute_unmount(i_provider &provider,
EXPECT_STREQ("/", fsi.api_path.c_str());
EXPECT_TRUE(fsi.api_parent.empty());
EXPECT_TRUE(fsi.directory);
EXPECT_TRUE(fsi.encryption_token.empty());
EXPECT_EQ(std::uint64_t(0u), fsi.size);
EXPECT_TRUE(fsi.source_path.empty());
EXPECT_FALSE(fsi.is_encrypted());
api_meta_map meta{};
EXPECT_EQ(api_error::success, provider.get_item_meta("/", meta));
@ -511,9 +508,6 @@ TEST(fuse_drive, all_tests) {
EXPECT_TRUE(utils::file::create_full_directory_path(mount_location));
{
std::unique_ptr<app_config> config_ptr{};
#ifdef REPERTORY_ENABLE_S3
std::unique_ptr<s3_comm> s3_comm_ptr{};
#endif
std::unique_ptr<curl_comm> comm_ptr{};
std::unique_ptr<i_provider> provider_ptr{};
std::unique_ptr<lock_data> lock_data_ptr{};
@ -537,8 +531,8 @@ TEST(fuse_drive, all_tests) {
config_ptr->set_s3_config(src_cfg.get_s3_config());
}
s3_comm_ptr = std::make_unique<s3_comm>(*config_ptr);
provider_ptr = std::make_unique<s3_provider>(*config_ptr, *s3_comm_ptr);
comm_ptr = std::make_unique<curl_comm>(config_ptr->get_s3_config());
provider_ptr = std::make_unique<s3_provider>(*config_ptr, *comm_ptr);
lock_data_ptr = std::make_unique<lock_data>(
provider_type::s3, "unittests_" + std::to_string(idx));
#else

292
tests/meta_db_test.cpp
View File

@ -1,292 +0,0 @@
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "test_common.hpp"
#include "fixtures/meta_db_fixture.hpp"
namespace repertory {
TEST_F(meta_db_test, get_and_set_item_meta) {
api_meta_map meta{
{"test", "test_value"},
{"test2", "test_value2"},
{"test3", "test_value3"},
};
EXPECT_EQ(api_error::success, db_->set_item_meta("/test/item", meta));
api_meta_map meta2{};
EXPECT_EQ(api_error::success, db_->get_item_meta("/test/item", meta2));
EXPECT_EQ(meta, meta2);
}
TEST_F(meta_db_test, get_and_set_item_meta_single_key) {
EXPECT_EQ(api_error::success,
db_->set_item_meta("/test/item", "test", "moose"));
std::string value;
EXPECT_EQ(api_error::success,
db_->get_item_meta("/test/item", "test", value));
EXPECT_STREQ("moose", value.c_str());
}
TEST_F(meta_db_test,
get_item_meta_fails_with_not_found_for_items_that_dont_exist) {
api_meta_map meta{};
EXPECT_EQ(api_error::item_not_found, db_->get_item_meta("/test/item", meta));
std::string value;
EXPECT_EQ(api_error::item_not_found,
db_->get_item_meta("/test/item", "test", value));
EXPECT_TRUE(value.empty());
}
TEST_F(meta_db_test, get_item_meta_exists) {
EXPECT_EQ(api_error::success,
db_->set_item_meta("/test/item", "test", "value"));
EXPECT_TRUE(db_->get_item_meta_exists("/test/item"));
}
TEST_F(meta_db_test, get_item_meta_exists_is_false_if_not_found) {
EXPECT_FALSE(db_->get_item_meta_exists("/test/item"));
}
TEST_F(meta_db_test, remove_item_meta) {
api_meta_map meta{
{"test", "test_value"},
{"test2", "test_value2"},
{"test3", "test_value3"},
};
EXPECT_EQ(api_error::success, db_->set_item_meta("/test/item", meta));
EXPECT_EQ(api_error::success, db_->remove_item_meta("/test/item"));
api_meta_map meta2{};
EXPECT_EQ(api_error::item_not_found, db_->get_item_meta("/test/item", meta2));
EXPECT_TRUE(meta2.empty());
}
TEST_F(meta_db_test, remove_item_meta_single_key) {
api_meta_map meta{
{"test", "test_value"},
{"test2", "test_value2"},
{"test3", "test_value3"},
};
EXPECT_EQ(api_error::success, db_->set_item_meta("/test/item", meta));
EXPECT_EQ(api_error::success, db_->remove_item_meta("/test/item", "test"));
api_meta_map meta2{};
EXPECT_EQ(api_error::success, db_->get_item_meta("/test/item", meta2));
meta.erase("test");
EXPECT_EQ(meta, meta2);
}
TEST_F(meta_db_test, get_and_set_source_path) {
EXPECT_EQ(api_error::success,
db_->set_source_path("/test/item", "/test/path"));
std::string value;
EXPECT_EQ(api_error::success,
db_->get_item_meta("/test/item", META_SOURCE, value));
EXPECT_STREQ("/test/path", value.c_str());
}
// TEST_F(meta_db_test, set_source_path_fails_on_empty_path) {
// EXPECT_EQ(api_error::invalid_operation, db_->set_source_path("/test/item",
// ""));
//
// std::string value;
// EXPECT_EQ(api_error::item_not_found, db_->get_item_meta("/test/item",
// META_SOURCE, value)); EXPECT_TRUE(value.empty());
// }
TEST_F(meta_db_test, get_api_path_from_source) {
EXPECT_EQ(api_error::success,
db_->set_source_path("/test/item", "/test/path"));
std::string value;
EXPECT_EQ(api_error::success,
db_->get_api_path_from_source("/test/path", value));
EXPECT_STREQ("/test/item", value.c_str());
}
TEST_F(meta_db_test, get_api_path_from_source_succeeds_after_change) {
EXPECT_EQ(api_error::success,
db_->set_source_path("/test/item", "/test/path"));
EXPECT_EQ(api_error::success,
db_->set_source_path("/test/item", "/test/path2"));
std::string value;
EXPECT_EQ(api_error::success,
db_->get_api_path_from_source("/test/path2", value));
EXPECT_STREQ("/test/item", value.c_str());
value.clear();
EXPECT_EQ(api_error::item_not_found,
db_->get_api_path_from_source("/test/path", value));
EXPECT_TRUE(value.empty());
}
TEST_F(meta_db_test, get_api_path_from_source_fails_after_remove_all_meta) {
EXPECT_EQ(api_error::success,
db_->set_source_path("/test/item", "/test/path"));
EXPECT_EQ(api_error::success, db_->remove_item_meta("/test/item"));
std::string value;
EXPECT_EQ(api_error::item_not_found,
db_->get_api_path_from_source("/test/path", value));
EXPECT_TRUE(value.empty());
}
TEST_F(meta_db_test, get_api_path_from_source_fails_after_remove_source_key) {
EXPECT_EQ(api_error::success,
db_->set_source_path("/test/item", "/test/path"));
EXPECT_EQ(api_error::success,
db_->remove_item_meta("/test/item", META_SOURCE));
std::string value;
EXPECT_EQ(api_error::item_not_found,
db_->get_api_path_from_source("/test/path", value));
EXPECT_TRUE(value.empty());
}
TEST_F(meta_db_test, get_source_path_exists) {
EXPECT_EQ(api_error::success,
db_->set_source_path("/test/item", "/test/path"));
EXPECT_TRUE(db_->get_source_path_exists("/test/path"));
}
TEST_F(meta_db_test, get_source_path_exists_is_false_if_not_found) {
EXPECT_FALSE(db_->get_source_path_exists("/test/item"));
}
TEST_F(meta_db_test, get_api_path_from_key) {
EXPECT_EQ(api_error::success,
db_->set_item_meta("/test/item", META_KEY, "key"));
std::string value;
EXPECT_EQ(api_error::success, db_->get_api_path_from_key("key", value));
EXPECT_STREQ("/test/item", value.c_str());
}
TEST_F(meta_db_test, remove_item_meta_succeeds_for_items_that_dont_exist) {
EXPECT_EQ(api_error::success, db_->remove_item_meta("/test/item"));
EXPECT_EQ(api_error::success, db_->remove_item_meta("/test/item", "test"));
}
TEST_F(meta_db_test, remove_item_meta_removes_source_path) {
EXPECT_EQ(api_error::success,
db_->set_source_path("/test/item", "/source/path"));
EXPECT_EQ(api_error::success, db_->remove_item_meta("/test/item"));
std::string api_path;
EXPECT_EQ(api_error::item_not_found,
db_->get_api_path_from_source("/source/path", api_path));
EXPECT_TRUE(api_path.empty());
}
TEST_F(meta_db_test, remove_item_meta_by_key_removes_source_path) {
EXPECT_EQ(api_error::success,
db_->set_source_path("/test/item", "/source/path"));
EXPECT_EQ(api_error::success,
db_->remove_item_meta("/test/item", META_SOURCE));
std::string api_path;
EXPECT_EQ(api_error::item_not_found,
db_->get_api_path_from_source("/source/path", api_path));
EXPECT_TRUE(api_path.empty());
}
TEST_F(meta_db_test, remove_item_meta_removes_key) {
EXPECT_EQ(api_error::success,
db_->set_item_meta("/test/item", META_KEY, "key"));
EXPECT_EQ(api_error::success, db_->remove_item_meta("/test/item"));
std::string api_path;
EXPECT_EQ(api_error::item_not_found,
db_->get_api_path_from_key("key", api_path));
EXPECT_TRUE(api_path.empty());
}
TEST_F(meta_db_test, remove_item_meta_by_key_removes_key) {
EXPECT_EQ(api_error::success,
db_->set_item_meta("/test/item", META_KEY, "key"));
EXPECT_EQ(api_error::success, db_->remove_item_meta("/test/item", META_KEY));
std::string api_path;
EXPECT_EQ(api_error::item_not_found,
db_->get_api_path_from_key("/source/path", api_path));
EXPECT_TRUE(api_path.empty());
}
TEST_F(meta_db_test, rename_item_meta) {
api_meta_map meta{
{"test", "test_value"},
{"test2", "test_value2"},
{"test3", "test_value3"},
};
EXPECT_EQ(api_error::success, db_->set_item_meta("/test/item", meta));
EXPECT_EQ(api_error::success,
db_->rename_item_meta("", "/test/item", "/test/item2"));
api_meta_map meta2{};
EXPECT_EQ(api_error::item_not_found, db_->get_item_meta("/test/item", meta2));
EXPECT_EQ(api_error::success, db_->get_item_meta("/test/item2", meta2));
EXPECT_EQ(meta, meta2);
}
TEST_F(meta_db_test, rename_item_meta_with_key) {
api_meta_map meta{
{META_KEY, "test_key"},
{"test2", "test_value2"},
{"test3", "test_value3"},
};
EXPECT_EQ(api_error::success, db_->set_item_meta("/test/item", meta));
EXPECT_EQ(api_error::success,
db_->rename_item_meta("", "/test/item", "/test/item2"));
std::string api_path;
EXPECT_EQ(api_error::success,
db_->get_api_path_from_key("test_key", api_path));
EXPECT_STREQ("/test/item2", api_path.c_str());
}
TEST_F(meta_db_test, rename_item_meta_with_source_path) {
api_meta_map meta{
{META_SOURCE, "/test/source"},
{"test2", "test_value2"},
{"test3", "test_value3"},
};
EXPECT_EQ(api_error::success, db_->set_item_meta("/test/item", meta));
EXPECT_EQ(api_error::success,
db_->rename_item_meta("/test/source", "/test/item", "/test/item2"));
std::string api_path;
EXPECT_EQ(api_error::success,
db_->get_api_path_from_source("/test/source", api_path));
EXPECT_STREQ("/test/item2", api_path.c_str());
}
} // namespace repertory

2
tests/mocks/mock_provider.hpp
View File

@ -154,7 +154,7 @@ public:
MOCK_METHOD(api_error, upload_file,
(const std::string &api_path, const std::string &source_path,
const std::string &encryption_token, stop_type &stop_requested),
stop_type &stop_requested),
(override));
};
} // namespace repertory

186
tests/mocks/mock_s3_comm.hpp
View File

@ -1,186 +0,0 @@
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#ifndef TESTS_MOCKS_MOCK_S3_COMM_HPP_
#define TESTS_MOCKS_MOCK_S3_COMM_HPP_
#if defined(REPERTORY_ENABLE_S3) && defined(REPERTORY_ENABLE_S3_TESTING)
#include "test_common.hpp"
#include "comm/i_s3_comm.hpp"
#include "types/repertory.hpp"
namespace repertory {
class mock_s3_comm final : public i_s3_comm {
public:
mock_s3_comm(s3_config cfg) : s3_config_(cfg) {}
private:
s3_config s3_config_;
public:
// [[nodiscard]] virtual api_error create_directory(const std::string
// &api_path) = 0;
//
MOCK_METHOD(api_error, create_directory, (const std::string &api_path),
(override));
// [[nodiscard]] virtual api_error directory_exists(const std::string
// &api_path) const = 0;
//
MOCK_METHOD(api_error, directory_exists, (const std::string &api_path),
(const, override));
// [[nodiscard]] virtual api_error file_exists(const std::string &api_path,
// const get_key_callback
// &get_key) const = 0;
MOCK_METHOD(api_error, file_exists,
(const std::string &api_path, const get_key_callback &get_key),
(const, override));
// [[nodiscard]] virtual std::size_t
// get_directory_item_count(const std::string &api_path,
// meta_provider_callback meta_provider) const
// = 0;
//
MOCK_METHOD(std::size_t, get_directory_item_count,
(const std::string &api_path,
meta_provider_callback meta_provider),
(const, override));
// [[nodiscard]] virtual api_error get_directory_items(const std::string
// &api_path,
// const
// meta_provider_callback
// &meta_provider,
// directory_item_list
// &list) const = 0;
//
MOCK_METHOD(api_error, get_directory_items,
(const std::string &api_path,
meta_provider_callback meta_provider,
directory_item_list &list),
(const, override));
// [[nodiscard]] virtual api_error
// get_directory_list(api_file_list &list) const = 0;
//
MOCK_METHOD(api_error, get_directory_list, (api_file_list & list),
(const, override));
// [[nodiscard]] virtual api_error get_file(const std::string &api_path,
// const get_key_callback &get_key,
// const get_name_callback &get_name,
// const get_token_callback
// &get_token, api_file &file) const
// = 0;
//
MOCK_METHOD(api_error, get_file,
(const std::string &api_path, const get_key_callback &get_key,
const get_name_callback &get_name,
const get_token_callback &get_token, api_file &file),
(const, override));
// [[nodiscard]] virtual api_error
// get_file_list(const get_api_file_token_callback &get_api_file_token,
// const get_name_callback &get_name, api_file_list &list) const
// = 0;
//
MOCK_METHOD(api_error, get_file_list,
(const get_api_file_token_callback &get_api_file_token,
const get_name_callback &get_name, api_file_list &list),
(const, override));
// [[nodiscard]] virtual api_error get_object_list(std::vector<directory_item>
// &list) const = 0;
//
MOCK_METHOD(api_error, get_object_list, (std::vector<directory_item> & list),
(const, override));
// virtual std::string get_object_name(const std::string &api_path,
// const get_key_callback &get_key) const
// = 0;
//
MOCK_METHOD(std::string, get_object_name,
(const std::string &api_path, const get_key_callback &get_key),
(const, override));
[[nodiscard]] s3_config get_s3_config() override { return s3_config_; }
[[nodiscard]] s3_config get_s3_config() const override { return s3_config_; }
// [[nodiscard]] virtual bool is_online() const = 0;
//
MOCK_METHOD(bool, is_online, (), (const, override));
// [[nodiscard]] virtual api_error
// read_file_bytes(const std::string &api_path, std::size_t size, const
// std::uint64_t &offset,
// data_buffer &data, const get_key_callback &get_key,
// const get_size_callback &get_size, const get_token_callback
// &get_token, stop_type &stop_requested) const = 0;
//
MOCK_METHOD(api_error, read_file_bytes,
(const std::string &api_path, std::size_t size,
std::uint64_t offset, data_buffer &data,
const get_key_callback &get_key,
const get_size_callback &get_size,
const get_token_callback &get_token, stop_type &stop_requested),
(const, override));
// [[nodiscard]] virtual api_error remove_directory(const std::string
// &api_path) = 0;
//
MOCK_METHOD(api_error, remove_directory, (const std::string &api_path),
(override));
// [[nodiscard]] virtual api_error remove_file(const std::string &api_path,
// const get_key_callback
// &get_key) = 0;
//
MOCK_METHOD(api_error, remove_file,
(const std::string &api_path, const get_key_callback &get_key),
(override));
// [[nodiscard]] virtual api_error rename_file(const std::string &api_path,
// const std::string
// &new_api_path) = 0;
//
MOCK_METHOD(api_error, rename_file,
(const std::string &api_path, const std::string &new_api_path),
(override));
// [[nodiscard]] virtual api_error
// upload_file(const std::string &api_path, const std::string &source_path,
// const std::string &encryption_token, const get_key_callback
// &get_key, const set_key_callback &set_key, stop_type
// &stop_requested) = 0;
MOCK_METHOD(api_error, upload_file,
(const std::string &api_path, const std::string &source_path,
const std::string &encryption_token,
const get_key_callback &get_key, const set_key_callback &set_key,
stop_type &stop_requested),
(override));
};
} // namespace repertory
#endif // REPERTORY_ENABLE_S3_TESTING
#endif // TESTS_MOCKS_MOCK_S3_COMM_HPP_

12
tests/providers_test.cpp
View File

@ -22,7 +22,6 @@
#include "test_common.hpp"
#include "comm/curl/curl_comm.hpp"
#include "comm/s3/s3_comm.hpp"
#include "events/event_system.hpp"
#include "file_manager/file_manager.hpp"
#include "platform/platform.hpp"
@ -57,8 +56,8 @@ const auto create_directory = [](repertory::i_provider &provider,
const std::string &api_path) {
auto date = repertory::utils::get_file_time_now();
auto meta = repertory::create_meta_attributes(
date, 0U, date + 1U, date + 2U, true, "", getgid(), "", 0700, date + 3U,
1U, 2U, 0U, api_path + "_src", getuid(), date + 4U);
date, 0U, date + 1U, date + 2U, true, getgid(), "", 0700, date + 3U, 1U,
2U, 0U, api_path + "_src", getuid(), date + 4U);
EXPECT_EQ(repertory::api_error::success,
provider.create_directory(api_path, meta));
@ -74,8 +73,8 @@ const auto create_file = [](repertory::i_provider &provider,
auto date = repertory::utils::get_file_time_now();
auto meta = repertory::create_meta_attributes(
date, 0U, date + 1U, date + 2U, false, "", getgid(), "", 0700, date + 3U,
1U, 2U, 0U, source_path, getuid(), date + 4U);
date, 0U, date + 1U, date + 2U, false, getgid(), "", 0700, date + 3U, 1U,
2U, 0U, source_path, getuid(), date + 4U);
EXPECT_EQ(repertory::api_error::success,
provider.create_file(api_path, meta));
@ -438,7 +437,6 @@ static void get_file(const app_config &cfg, i_provider &provider) {
#else
EXPECT_EQ(std::size_t(46U), file.file_size);
#endif
EXPECT_TRUE(file.encryption_token.empty());
EXPECT_STREQ(source_path.c_str(), file.source_path.c_str());
}
}
@ -585,7 +583,7 @@ TEST(providers, s3_provider) {
cfg.set_s3_config(src_cfg.get_s3_config());
}
s3_comm comm{cfg};
curl_comm comm{cfg.get_s3_config()};
s3_provider provider{cfg, comm};
file_manager fm(cfg, provider);
fm.start();

205
tests/s3_comm_test.cpp
View File

@ -1,205 +0,0 @@
/*
Copyright <2018-2023> <scott.e.graves@protonmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#if defined(REPERTORY_ENABLE_S3) && defined(REPERTORY_ENABLE_S3_TESTING)
#include "test_common.hpp"
#include "comm/s3/s3_comm.hpp"
#include "fixtures/s3_comm_fixture.hpp"
#include "types/repertory.hpp"
namespace repertory {
TEST_F(s3_comm_test, create_and_remove_directory) {
auto ret = s3_comm_->create_directory("/dir");
EXPECT_EQ(api_error::success, ret);
EXPECT_EQ(api_error::directory_exists, s3_comm_->directory_exists("/dir"));
ret = s3_comm_->remove_directory("/dir");
EXPECT_EQ(api_error::success, ret);
EXPECT_EQ(api_error::directory_not_found, s3_comm_->directory_exists("/dir"));
ret = s3_comm_->remove_directory("/dir");
EXPECT_TRUE(ret == api_error::success ||
ret == api_error::directory_not_found);
}
TEST_F(s3_comm_test, upload_file) {
stop_type stop_requested = false;
auto ret = s3_comm_->upload_file(
"/test.txt", __FILE__, "", []() -> std::string { return ""; },
[](const std::string &) -> api_error { return api_error::success; },
stop_requested);
EXPECT_EQ(api_error::success, ret);
ret = s3_comm_->upload_file(
"/subdir/test2.txt", __FILE__, "", []() -> std::string { return ""; },
[](const std::string &) -> api_error { return api_error::success; },
stop_requested);
EXPECT_EQ(api_error::success, ret);
}
TEST_F(s3_comm_test, get_directory_items) {
directory_item_list list{};
auto ret = s3_comm_->get_directory_items(
"/subdir", [](directory_item &) {}, list);
EXPECT_EQ(api_error::success, ret);
}
TEST_F(s3_comm_test, list_directories) {
api_file_list list{};
auto ret = s3_comm_->get_directory_list(list);
EXPECT_EQ(api_error::success, ret);
}
TEST_F(s3_comm_test, list_files) {
api_file_list list{};
auto ret = s3_comm_->get_file_list(
[](const std::string &) -> std::string { return ""; },
[](const std::string &, const std::string &object_name) -> std::string {
return object_name;
},
list);
EXPECT_EQ(api_error::success, ret);
}
TEST_F(s3_comm_test, read_file_bytes) {
stop_type stop_requested = false;
data_buffer data;
auto ret = s3_comm_->read_file_bytes(
"/test.txt", 2, 0, data, []() -> std::string { return ""; },
[]() -> std::uint64_t { return 0ull; },
[]() -> std::string { return ""; }, stop_requested);
EXPECT_EQ(api_error::success, ret);
}
TEST_F(s3_comm_test, exists) {
EXPECT_EQ(
api_error::item_exists,
s3_comm_->file_exists("/test.txt", []() -> std::string { return ""; }));
EXPECT_EQ(api_error::item_not_found,
s3_comm_->file_exists("/subdir/test.txt",
[]() -> std::string { return ""; }));
}
TEST_F(s3_comm_test, get_file) {
api_file file{};
auto ret = s3_comm_->get_file(
"/test.txt", []() -> std::string { return ""; },
[](const std::string &, const std::string &object_name) -> std::string {
return object_name;
},
[]() -> std::string { return ""; }, file);
EXPECT_EQ(api_error::success, ret);
}
TEST_F(s3_comm_test, remove_file) {
auto ret =
s3_comm_->remove_file("/test.txt", []() -> std::string { return ""; });
EXPECT_EQ(api_error::success, ret);
ret = s3_comm_->remove_file("/subdir/test2.txt",
[]() -> std::string { return ""; });
EXPECT_EQ(api_error::success, ret);
}
TEST_F(s3_comm_test, rename_file) {
stop_type stop_requested = false;
auto ret =
s3_comm_->remove_file("/test_r2.txt", []() -> std::string { return ""; });
ret = s3_comm_->upload_file(
"/test_r1.txt", __FILE__, "", []() -> std::string { return ""; },
[](const std::string &) -> api_error { return api_error::success; },
stop_requested);
EXPECT_EQ(api_error::success, ret);
ret = s3_comm_->rename_file("/test_r1.txt", "/test_r2.txt");
EXPECT_EQ(api_error::not_implemented, ret);
EXPECT_EQ(api_error::item_exists,
s3_comm_->file_exists("/test_r1.txt",
[]() -> std::string { return ""; }));
EXPECT_EQ(api_error::item_not_found,
s3_comm_->file_exists("/test_r2.txt",
[]() -> std::string { return ""; }));
EXPECT_EQ(api_error::success,
s3_comm_->remove_file("/test_r1.txt",
[]() -> std::string { return ""; }));
}
TEST_F(s3_comm_test, upload_file_encrypted) {
const auto source_file_path = generate_test_file_name("./", "awscomm");
auto file_size =
2u * utils::encryption::encrypting_reader::get_data_chunk_size() + 3u;
auto source_file = create_random_file(source_file_path, file_size);
stop_type stop_requested = false;
std::string key;
auto ret = s3_comm_->upload_file(
"/test.txt", source_file_path, "test", []() -> std::string { return ""; },
[&key](const std::string &k) -> api_error {
key = k;
std::cout << "key:" << key << std::endl;
return api_error::success;
},
stop_requested);
EXPECT_EQ(api_error::success, ret);
std::uint64_t offset = 0u;
auto remain = file_size;
while ((ret == api_error::success) && remain) {
data_buffer data;
ret = s3_comm_->read_file_bytes(
"/test.txt",
std::min(remain,
utils::encryption::encrypting_reader::get_data_chunk_size()),
offset, data, [&key]() -> std::string { return key; },
[&file_size]() -> std::uint64_t { return file_size; },
[]() -> std::string { return "test"; }, stop_requested);
EXPECT_EQ(api_error::success, ret);
data_buffer data2(data.size());
std::size_t bytes_read{};
EXPECT_TRUE(
source_file->read_bytes(&data2[0u], data2.size(), offset, bytes_read));
EXPECT_EQ(0, std::memcmp(&data2[0u], &data[0u], data2.size()));
remain -= data.size();
offset += data.size();
}
source_file->close();
EXPECT_TRUE(utils::file::retry_delete_file(source_file_path));
EXPECT_EQ(api_error::success,
s3_comm_->remove_file("/test.txt",
[&key]() -> std::string { return key; }));
}
TEST_F(s3_comm_test, get_directory_item_count) {}
TEST_F(s3_comm_test, get_object_list) {}
TEST_F(s3_comm_test, get_object_name) {}
TEST_F(s3_comm_test, is_online) {}
} // namespace repertory
#endif // REPERTORY_ENABLE_S3_TESTING

170
tests/s3_provider_test.cpp
View File

@ -24,11 +24,10 @@
#include "test_common.hpp"
#include "app_config.hpp"
#include "comm/curl/curl_comm.hpp"
#include "events/consumers/console_consumer.hpp"
#include "events/event_system.hpp"
#include "file_manager/file_manager.hpp"
#include "fixtures/s3_provider_file_list_fixture.hpp"
#include "mocks/mock_s3_comm.hpp"
#include "platform/platform.hpp"
#include "providers/s3/s3_provider.hpp"
#include "types/s3.hpp"
@ -37,153 +36,44 @@
#include "utils/utils.hpp"
namespace repertory {
// TEST(s3_provider, can_construct_s3_provider) {
// {
// app_config cfg(provider_type::s3, "./s3_provider_test");
// EXPECT_FALSE(cfg.set_value_by_name("S3Config.Bucket", "bucket").empty());
// EXPECT_FALSE(
// cfg.set_value_by_name("S3Config.URL", "https://url.com").empty());
// mock_s3_comm comm(cfg.get_s3_config());
// s3_provider s3(cfg, comm);
// EXPECT_EQ(s3.get_total_item_count(), 0u);
// }
//
// EXPECT_TRUE(utils::file::delete_directory_recursively("./s3_provider_test"));
// }
//
TEST(s3_provider, start_fails_with_empty_bucket) {
TEST(s3_provider, get_file_list) {
{
app_config cfg(provider_type::s3, "./s3_provider_test");
EXPECT_TRUE(cfg.set_value_by_name("S3Config.Bucket", "").empty());
EXPECT_FALSE(
cfg.set_value_by_name("S3Config.URL", "https://url.com").empty());
mock_s3_comm comm(cfg.get_s3_config());
s3_provider s3(cfg, comm);
file_manager fm(cfg, s3);
const auto config_path = utils::path::absolute("./s3_provider_test");
ASSERT_TRUE(utils::file::delete_directory_recursively(config_path));
try {
auto res = s3.start(
[](bool, api_file & /* file */) -> api_error {
return api_error::success;
console_consumer con{};
event_system::instance().start();
{
app_config cfg(provider_type::s3, config_path);
{
app_config src_cfg(provider_type::s3,
utils::path::combine(get_test_dir(), {"storj"}));
cfg.set_s3_config(src_cfg.get_s3_config());
}
curl_comm comm{cfg.get_s3_config()};
s3_provider provider{cfg, comm};
file_manager mgr(cfg, provider);
mgr.start();
EXPECT_TRUE(provider.start(
[&provider](bool directory, api_file &file) -> api_error {
return provider_meta_handler(provider, directory, file);
},
&fm);
std::cerr << "unexpected return-should throw|err|" << res << std::endl;
} catch (const startup_exception &e) {
EXPECT_STREQ("s3 bucket name cannot be empty", e.what());
return;
&mgr));
api_file_list list{};
EXPECT_EQ(api_error::success, provider.get_file_list(list));
provider.stop();
mgr.stop();
}
event_system::instance().stop();
throw std::runtime_error("exception not thrown");
ASSERT_TRUE(utils::file::delete_directory_recursively(config_path));
}
EXPECT_TRUE(utils::file::delete_directory_recursively("./s3_provider_test"));
}
TEST(s3_provider, start_fails_when_provider_is_offline) {
{
app_config cfg(provider_type::s3, "./s3_provider_test");
EXPECT_FALSE(cfg.set_value_by_name("S3Config.Bucket", "bucket").empty());
EXPECT_FALSE(
cfg.set_value_by_name("S3Config.URL", "https://url.com").empty());
cfg.set_online_check_retry_secs(2u);
mock_s3_comm comm(cfg.get_s3_config());
s3_provider s3(cfg, comm);
file_manager fm(cfg, s3);
EXPECT_CALL(comm, is_online()).WillRepeatedly(Return(false));
EXPECT_FALSE(s3.start([](bool, api_file & /* file */)
-> api_error { return api_error::success; },
&fm));
}
EXPECT_TRUE(utils::file::delete_directory_recursively("./s3_provider_test"));
}
// TEST(s3_provider, get_empty_file_list) {
// {
// app_config cfg(provider_type::s3, "./s3_provider_test");
// EXPECT_FALSE(cfg.set_value_by_name("S3Config.Bucket", "bucket").empty());
// EXPECT_FALSE(
// cfg.set_value_by_name("S3Config.URL", "https://url.com").empty());
// cfg.set_online_check_retry_secs(2u);
// mock_s3_comm comm(cfg.get_s3_config());
// s3_provider s3(cfg, comm);
// file_manager fm(cfg, s3);
//
// api_file_list list{};
// EXPECT_CALL(comm, get_file_list)
// .WillOnce([](const get_api_file_token_callback &,
// const get_name_callback &,
// api_file_list &) { return api_error::success; });
//
// EXPECT_EQ(api_error::success, s3.get_file_list(list));
// }
//
// EXPECT_TRUE(utils::file::delete_directory_recursively("./s3_provider_test"));
// }
//
// TEST_F(s3_provider_file_list_test, can_add_new_files_and_directories) {
// provider->set_callback([this](bool directory, api_file &file) -> api_error
// {
// std::cout << "added|api_path|" << file.api_path << "|api_parent|"
// << file.api_parent << "|source|" << file.source_path
// << "|directory|" << directory << "|create_date|"
// << file.creation_date << "|access_date|" << file.accessed_date
// << "|modified_date|" << file.modified_date << "|changed_date|"
// << file.changed_date << std::endl;
// return provider_meta_handler(*provider, directory, file);
// });
//
// api_file_list l{};
// auto res = provider->get_file_list(l);
// EXPECT_EQ(api_error::success, res);
// EXPECT_EQ(list.size(), l.size());
// EXPECT_EQ(std::size_t(22u), provider->get_total_item_count());
//
// bool exists{};
// EXPECT_EQ(api_error::success, provider->is_directory("/", exists));
// EXPECT_TRUE(exists);
//
// EXPECT_EQ(api_error::success, provider->is_directory("/dir", exists));
// EXPECT_TRUE(exists);
//
// const auto check_file = [this, &l](std::size_t idx,
// bool check_sub_directory) {
// const auto &file = l.at(idx);
// const auto base_idx = idx - (check_sub_directory ? l.size() / 2 : 0u);
// EXPECT_EQ(this->times[base_idx], file.accessed_date);
// if (check_sub_directory) {
// EXPECT_EQ(utils::path::create_api_path("/dir/file_" +
// std::to_string(base_idx) +
// ".txt"),
// file.api_path);
// EXPECT_EQ(utils::path::get_parent_api_path(utils::path::create_api_path(
// "/dir/file_" + std::to_string(base_idx) + ".txt")),
// file.api_parent);
// } else {
// EXPECT_EQ(utils::path::create_api_path("/file_" +
// std::to_string(base_idx) +
// ".txt"),
// file.api_path);
// EXPECT_EQ(utils::path::get_parent_api_path(utils::path::create_api_path(
// "/file_" + std::to_string(base_idx) + ".txt")),
// file.api_parent);
// }
// EXPECT_EQ(this->times[base_idx] + 1u, file.changed_date);
// EXPECT_EQ(this->times[base_idx] + 2u, file.creation_date);
// EXPECT_TRUE(file.encryption_token.empty());
// EXPECT_EQ(100u + base_idx, file.file_size);
// EXPECT_EQ(this->times[base_idx] + 3u, file.modified_date);
// };
//
// for (std::size_t idx = 0u; idx < l.size() / 2u; idx++) {
// check_file(idx, false);
// }
//
// for (std::size_t idx = l.size() / 2u; idx < l.size(); idx++) {
// check_file(idx, true);
// }
// }
} // namespace repertory
#endif // REPERTORY_ENABLE_S3_TESTING