extract common behavior

This commit is contained in:
Scott E. Graves 2023-11-14 18:20:11 -06:00
parent 60052081b4
commit 19b1a33c48
7 changed files with 241 additions and 280 deletions

View File

@ -56,6 +56,16 @@ protected:
std::uint64_t size,
api_meta_map &meta) -> api_file;
[[nodiscard]] virtual auto create_directory_impl(const std::string &api_path,
api_meta_map &meta)
-> api_error = 0;
[[nodiscard]] virtual auto
create_file_extra(const std::string & /* api_path */,
api_meta_map & /* meta */) -> api_error {
return api_error::success;
}
[[nodiscard]] auto get_api_item_added() -> api_item_added_callback & {
return api_item_added_;
}
@ -87,6 +97,12 @@ public:
const std::string &api_path)
-> api_error override;
[[nodiscard]] auto create_directory(const std::string &api_path,
api_meta_map &meta) -> api_error override;
[[nodiscard]] auto create_file(const std::string &api_path,
api_meta_map &meta) -> api_error override;
[[nodiscard]] auto get_api_path_from_source(const std::string &source_path,
std::string &api_path) const
-> api_error override;

View File

@ -49,6 +49,14 @@ private:
const std::string &object_name) const
-> api_error;
[[nodiscard]] auto create_directory_impl(const std::string &api_path,
api_meta_map &meta)
-> api_error override;
[[nodiscard]] auto create_file_extra(const std::string &api_path,
api_meta_map &meta)
-> api_error override;
[[nodiscard]] auto create_path_directories(const std::string &api_path,
const std::string &key) const
-> api_error;
@ -62,12 +70,6 @@ private:
head_object_result &result) const -> api_error;
public:
[[nodiscard]] auto create_directory(const std::string &api_path,
api_meta_map &meta) -> api_error override;
[[nodiscard]] auto create_file(const std::string &api_path,
api_meta_map &meta) -> api_error override;
[[nodiscard]] auto get_directory_item_count(const std::string &api_path) const
-> std::uint64_t override;

View File

@ -49,16 +49,15 @@ private:
[[nodiscard]] auto get_object_list(const std::string &api_path,
nlohmann::json &object_list) const -> bool;
public:
[[nodiscard]] auto create_directory(const std::string &api_path,
api_meta_map &meta) -> api_error override;
protected:
[[nodiscard]] auto create_directory_impl(const std::string &api_path,
api_meta_map &meta)
-> api_error override;
public:
[[nodiscard]] auto get_directory_item_count(const std::string &api_path) const
-> std::uint64_t override;
[[nodiscard]] auto create_file(const std::string &api_path,
api_meta_map &meta) -> api_error override;
[[nodiscard]] auto get_directory_items(const std::string &api_path,
directory_item_list &list) const
-> api_error override;

View File

@ -114,6 +114,106 @@ auto base_provider::create_directory_clone_source_meta(
return create_directory(api_path, meta);
}
auto base_provider::create_directory(const std::string &api_path,
api_meta_map &meta) -> api_error {
bool exists{};
auto res = is_directory(api_path, exists);
if (res != api_error::success) {
return res;
}
if (exists) {
utils::error::raise_api_path_error(__FUNCTION__, api_path,
api_error::directory_exists,
"failed to create directory");
return api_error::directory_exists;
}
res = is_file(api_path, exists);
if (res != api_error::success) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, res,
"failed to create directory");
return res;
}
if (exists) {
utils::error::raise_api_path_error(__FUNCTION__, api_path,
api_error::item_exists,
"failed to create directory");
return api_error::item_exists;
}
try {
res = create_directory_impl(api_path, meta);
if (res != api_error::success) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, res,
"failed to create directory");
return res;
}
} catch (const std::exception &e) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, e,
"failed to create directory");
return api_error::error;
}
meta[META_DIRECTORY] = utils::string::from_bool(true);
return set_item_meta(api_path, meta);
}
auto base_provider::create_file(const std::string &api_path, api_meta_map &meta)
-> api_error {
bool exists{};
auto res = is_directory(api_path, exists);
if (res != api_error::success && res != api_error::item_not_found) {
return res;
}
if (exists) {
utils::error::raise_api_path_error(__FUNCTION__, api_path,
api_error::directory_exists,
"failed to create file");
return api_error::directory_exists;
}
res = is_file(api_path, exists);
if (res != api_error::success && res != api_error::item_not_found) {
return res;
}
if (exists) {
utils::error::raise_api_path_error(__FUNCTION__, api_path,
api_error::item_exists,
"failed to create file");
return api_error::item_exists;
}
try {
res = create_file_extra(api_path, meta);
if (res != api_error::success) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, res,
"failed to create file");
return res;
}
meta[META_DIRECTORY] = utils::string::from_bool(false);
res = set_item_meta(api_path, meta);
if (res != api_error::success) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, res,
"failed to create file");
return res;
}
stop_type stop_requested{false};
res = upload_file(api_path, meta[META_SOURCE], stop_requested);
if (res != api_error::success) {
get_db()->Delete(rocksdb::WriteOptions(), api_path);
}
return res;
} catch (const std::exception &e) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, e,
"failed to create file");
}
return api_error::error;
}
auto base_provider::get_api_path_from_source(const std::string &source_path,
std::string &api_path) const
-> api_error {

View File

@ -90,41 +90,16 @@ auto s3_provider::add_if_not_found(api_file &file,
return api_error::success;
}
auto s3_provider::create_directory(const std::string &api_path,
auto s3_provider::create_directory_impl(const std::string &api_path,
api_meta_map &meta) -> api_error {
bool exists{};
auto res = is_directory(api_path, exists);
if (res != api_error::success) {
return res;
}
if (exists) {
utils::error::raise_api_path_error(__FUNCTION__, api_path,
api_error::directory_exists,
"failed to create directory");
return api_error::directory_exists;
}
res = is_file(api_path, exists);
if (res != api_error::success) {
return res;
}
if (exists) {
utils::error::raise_api_path_error(__FUNCTION__, api_path,
api_error::item_exists,
"failed to create directory");
return api_error::item_exists;
}
try {
const auto cfg = get_config().get_s3_config();
const auto is_encrypted = not cfg.encryption_token.empty();
stop_type stop_requested{false};
if (is_encrypted) {
std::string encrypted_file_path;
res = get_item_meta(utils::path::get_parent_api_path(api_path), META_KEY,
encrypted_file_path);
auto res = get_item_meta(utils::path::get_parent_api_path(api_path),
META_KEY, encrypted_file_path);
if (res != api_error::success) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, res,
"failed to create file");
@ -136,8 +111,8 @@ auto s3_provider::create_directory(const std::string &api_path,
cfg.encryption_token,
*(utils::string::split(api_path, '/', false).end() - 1U), result);
meta[META_KEY] = utils::path::create_api_path(utils::path::combine(
utils::path::create_api_path(encrypted_file_path),
meta[META_KEY] = utils::path::create_api_path(
utils::path::combine(utils::path::create_api_path(encrypted_file_path),
{utils::to_hex_string(result)}));
}
@ -162,47 +137,16 @@ auto s3_provider::create_directory(const std::string &api_path,
"failed to create directory");
return api_error::comm_error;
}
} catch (const std::exception &e) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, e,
"failed to create directory");
return api_error::error;
return api_error::success;
}
meta[META_DIRECTORY] = utils::string::from_bool(true);
return set_item_meta(api_path, meta);
}
auto s3_provider::create_file(const std::string &api_path, api_meta_map &meta)
-> api_error {
bool exists{};
auto res = is_directory(api_path, exists);
if (res != api_error::success && res != api_error::item_not_found) {
return res;
}
if (exists) {
utils::error::raise_api_path_error(__FUNCTION__, api_path,
api_error::directory_exists,
"failed to create file");
return api_error::directory_exists;
}
res = is_file(api_path, exists);
if (res != api_error::success && res != api_error::item_not_found) {
return res;
}
if (exists) {
utils::error::raise_api_path_error(__FUNCTION__, api_path,
api_error::item_exists,
"failed to create file");
return api_error::item_exists;
}
stop_type stop_requested{false};
try {
auto s3_provider::create_file_extra(const std::string &api_path,
api_meta_map &meta) -> api_error {
if (not get_config().get_s3_config().encryption_token.empty()) {
std::string encrypted_file_path;
res = get_item_meta(utils::path::get_parent_api_path(api_path), META_KEY,
encrypted_file_path);
auto res = get_item_meta(utils::path::get_parent_api_path(api_path),
META_KEY, encrypted_file_path);
if (res != api_error::success) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, res,
"failed to create file");
@ -214,31 +158,12 @@ auto s3_provider::create_file(const std::string &api_path, api_meta_map &meta)
get_config().get_s3_config().encryption_token,
*(utils::string::split(api_path, '/', false).end() - 1U), result);
meta[META_KEY] = utils::path::create_api_path(utils::path::combine(
utils::path::create_api_path(encrypted_file_path),
meta[META_KEY] = utils::path::create_api_path(
utils::path::combine(utils::path::create_api_path(encrypted_file_path),
{utils::to_hex_string(result)}));
}
meta[META_DIRECTORY] = utils::string::from_bool(false);
res = set_item_meta(api_path, meta);
if (res != api_error::success) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, res,
"failed to create file");
return res;
}
res = upload_file(api_path, meta[META_SOURCE], stop_requested);
if (res != api_error::success) {
get_db()->Delete(rocksdb::WriteOptions(), api_path);
}
return res;
} catch (const std::exception &e) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, e,
"failed to create file");
}
return api_error::error;
return api_error::success;
}
auto s3_provider::create_path_directories(const std::string &api_path,

View File

@ -109,32 +109,9 @@ auto sia_provider::get_object_list(const std::string &api_path,
return true;
}
auto sia_provider::create_directory(const std::string &api_path,
api_meta_map &meta) -> api_error {
bool exists{};
auto res = is_directory(api_path, exists);
if (res != api_error::success) {
return res;
}
if (exists) {
utils::error::raise_api_path_error(__FUNCTION__, api_path,
api_error::directory_exists,
"failed to create directory");
return api_error::directory_exists;
}
res = is_file(api_path, exists);
if (res != api_error::success) {
return res;
}
if (exists) {
utils::error::raise_api_path_error(__FUNCTION__, api_path,
api_error::item_exists,
"failed to create directory");
return api_error::item_exists;
}
try {
auto sia_provider::create_directory_impl(const std::string &api_path,
api_meta_map & /* meta */)
-> api_error {
curl::requests::http_put_file put_file{};
put_file.path = "/api/worker/objects" + api_path + "/";
@ -152,67 +129,8 @@ auto sia_provider::create_directory(const std::string &api_path,
"failed to create directory");
return api_error::comm_error;
}
} catch (const std::exception &e) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, e,
"failed to create directory");
return api_error::error;
}
meta[META_DIRECTORY] = utils::string::from_bool(true);
return set_item_meta(api_path, meta);
}
auto sia_provider::create_file(const std::string &api_path, api_meta_map &meta)
-> api_error {
bool exists{};
auto res = is_directory(api_path, exists);
if (res != api_error::success && res != api_error::item_not_found) {
return res;
}
if (exists) {
utils::error::raise_api_path_error(__FUNCTION__, api_path,
api_error::directory_exists,
"failed to create file");
return api_error::directory_exists;
}
res = is_file(api_path, exists);
if (res != api_error::success && res != api_error::item_not_found) {
return res;
}
if (exists) {
utils::error::raise_api_path_error(__FUNCTION__, api_path,
api_error::item_exists,
"failed to create file");
return api_error::item_exists;
}
try {
curl::requests::http_put_file put_file{};
put_file.path = "/api/worker/objects" + api_path;
long response_code{};
stop_type stop_requested{};
if (not get_comm().make_request(put_file, response_code, stop_requested)) {
utils::error::raise_api_path_error(__FUNCTION__, api_path,
api_error::comm_error,
"failed to create file");
return api_error::comm_error;
}
if (response_code != http_error_codes::ok) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, response_code,
"failed to create file");
return api_error::comm_error;
}
} catch (const std::exception &e) {
utils::error::raise_api_path_error(__FUNCTION__, api_path, e,
"failed to create file");
return api_error::error;
}
meta[META_DIRECTORY] = utils::string::from_bool(false);
return set_item_meta(api_path, meta);
return api_error::success;
}
auto sia_provider::get_directory_item_count(const std::string &api_path) const

View File

@ -89,7 +89,7 @@ static auto create_file_and_test(const std::string &mount_location,
EXPECT_EQ(0u, file_size);
EXPECT_EQ(0, close(fd));
std::this_thread::sleep_for(1s);
std::this_thread::sleep_for(2s);
return file_path;
}
@ -102,7 +102,7 @@ static void rmdir_and_test(const std::string &directory_path) {
}
EXPECT_EQ(0, ret);
std::this_thread::sleep_for(1s);
std::this_thread::sleep_for(2s);
EXPECT_FALSE(utils::file::is_directory(directory_path));
EXPECT_FALSE(utils::file::is_file(directory_path));
@ -116,7 +116,7 @@ static void unlink_file_and_test(const std::string &file_path) {
EXPECT_EQ(0, ret);
std::this_thread::sleep_for(1s);
std::this_thread::sleep_for(2s);
EXPECT_FALSE(utils::file::is_directory(file_path));
EXPECT_FALSE(utils::file::is_file(file_path));
}
@ -182,19 +182,20 @@ static void test_write_and_read(const std::string &api_path,
std::string data = "TestData";
EXPECT_EQ(data.size(),
static_cast<std::size_t>(write(fd, &data[0], data.size())));
static_cast<std::size_t>(write(fd, data.data(), data.size())));
EXPECT_EQ(0, lseek(fd, 0, SEEK_SET));
fsync(fd);
data_buffer read_data;
read_data.resize(data.size());
EXPECT_EQ(data.size(), static_cast<std::size_t>(
read(fd, &read_data[0], read_data.size())));
read(fd, read_data.data(), read_data.size())));
EXPECT_EQ(0, memcmp(&data[0], &read_data[0], data.size()));
EXPECT_EQ(0, memcmp(data.data(), read_data.data(), data.size()));
EXPECT_EQ(0, close(fd));
std::this_thread::sleep_for(1s);
std::this_thread::sleep_for(2s);
std::uint64_t file_size{};
EXPECT_TRUE(utils::file::get_file_size(file_path, file_size));
@ -214,7 +215,7 @@ static void test_rename_file(const std::string &from_file_path,
EXPECT_LE(1, fd);
close(fd);
std::this_thread::sleep_for(1s);
std::this_thread::sleep_for(2s);
if (provider.is_rename_supported()) {
EXPECT_EQ(0, rename(from_file_path.c_str(), to_file_path.c_str()));
@ -232,7 +233,7 @@ static void test_rename_directory(const std::string &from_dir_path,
i_provider &provider) {
EXPECT_EQ(0, mkdir(from_dir_path.c_str(),
S_IRUSR | S_IWUSR | S_IXUSR | S_IRGRP | S_IXGRP));
std::this_thread::sleep_for(1s);
std::this_thread::sleep_for(2s);
EXPECT_TRUE(utils::file::is_directory(from_dir_path));
if (provider.is_rename_supported()) {
@ -334,7 +335,7 @@ test_write_operations_fail_if_read_only(const std::string &api_path,
EXPECT_LE(1, fd);
std::string data = "TestData";
EXPECT_EQ(-1, write(fd, &data[0], data.size()));
EXPECT_EQ(-1, write(fd, data.data(), data.size()));
EXPECT_EQ(-1, ftruncate(fd, 9u));
@ -344,7 +345,7 @@ test_write_operations_fail_if_read_only(const std::string &api_path,
EXPECT_EQ(0, close(fd));
std::this_thread::sleep_for(1s);
std::this_thread::sleep_for(2s);
std::uint64_t file_size{};
EXPECT_TRUE(utils::file::get_file_size(file_path, file_size));
@ -360,10 +361,10 @@ test_write_operations_fail_if_read_only(const std::string &api_path,
#if !__APPLE__ && HAS_SETXATTR
static void test_xattr_invalid_parameters(const std::string &file_path) {
std::string attr = "moose";
EXPECT_EQ(-1, setxattr(nullptr, "user.test_attr", &attr[0], attr.size(),
EXPECT_EQ(-1, setxattr(nullptr, "user.test_attr", attr.data(), attr.size(),
XATTR_CREATE));
EXPECT_EQ(errno, EFAULT);
EXPECT_EQ(-1, setxattr(file_path.c_str(), nullptr, &attr[0], attr.size(),
EXPECT_EQ(-1, setxattr(file_path.c_str(), nullptr, attr.data(), attr.size(),
XATTR_CREATE));
EXPECT_EQ(errno, EFAULT);
EXPECT_EQ(-1, setxattr(file_path.c_str(), "user.test_attr", nullptr,
@ -375,29 +376,29 @@ static void test_xattr_invalid_parameters(const std::string &file_path) {
static void test_xattr_create_and_get(const std::string &file_path) {
std::string attr = "moose";
EXPECT_EQ(0, setxattr(file_path.c_str(), "user.test_attr", &attr[0],
EXPECT_EQ(0, setxattr(file_path.c_str(), "user.test_attr", attr.data(),
attr.size(), XATTR_CREATE));
std::string val;
val.resize(attr.size());
EXPECT_EQ(attr.size(),
static_cast<std::size_t>(getxattr(
file_path.c_str(), "user.test_attr", &val[0], val.size())));
file_path.c_str(), "user.test_attr", val.data(), val.size())));
EXPECT_STREQ(attr.c_str(), val.c_str());
}
static void test_xattr_listxattr(const std::string &file_path) {
std::string attr = "moose";
EXPECT_EQ(0, setxattr(file_path.c_str(), "user.test_attr", &attr[0],
EXPECT_EQ(0, setxattr(file_path.c_str(), "user.test_attr", attr.data(),
attr.size(), XATTR_CREATE));
EXPECT_EQ(0, setxattr(file_path.c_str(), "user.test_attr2", &attr[0],
EXPECT_EQ(0, setxattr(file_path.c_str(), "user.test_attr2", attr.data(),
attr.size(), XATTR_CREATE));
std::string val;
val.resize(attr.size());
EXPECT_EQ(attr.size(),
static_cast<std::size_t>(getxattr(
file_path.c_str(), "user.test_attr", &val[0], val.size())));
file_path.c_str(), "user.test_attr", val.data(), val.size())));
EXPECT_STREQ(attr.c_str(), val.c_str());
std::string data;
@ -416,56 +417,56 @@ static void test_xattr_listxattr(const std::string &file_path) {
static void test_xattr_replace(const std::string &file_path) {
std::string attr = "moose";
EXPECT_EQ(0, setxattr(file_path.c_str(), "user.test_attr", &attr[0],
EXPECT_EQ(0, setxattr(file_path.c_str(), "user.test_attr", attr.data(),
attr.size(), XATTR_CREATE));
attr = "cow";
EXPECT_EQ(0, setxattr(file_path.c_str(), "user.test_attr", &attr[0],
EXPECT_EQ(0, setxattr(file_path.c_str(), "user.test_attr", attr.data(),
attr.size(), XATTR_REPLACE));
std::string val;
val.resize(attr.size());
EXPECT_EQ(attr.size(),
static_cast<std::size_t>(getxattr(
file_path.c_str(), "user.test_attr", &val[0], val.size())));
file_path.c_str(), "user.test_attr", val.data(), val.size())));
EXPECT_STREQ(attr.c_str(), val.c_str());
}
static void test_xattr_default_create(const std::string &file_path) {
std::string attr = "moose";
EXPECT_EQ(0, setxattr(file_path.c_str(), "user.test_attr", &attr[0],
EXPECT_EQ(0, setxattr(file_path.c_str(), "user.test_attr", attr.data(),
attr.size(), 0));
std::string val;
val.resize(attr.size());
EXPECT_EQ(attr.size(),
static_cast<std::size_t>(getxattr(
file_path.c_str(), "user.test_attr", &val[0], val.size())));
file_path.c_str(), "user.test_attr", val.data(), val.size())));
EXPECT_STREQ(attr.c_str(), val.c_str());
}
static void test_xattr_default_replace(const std::string &file_path) {
std::string attr = "moose";
EXPECT_EQ(0, setxattr(file_path.c_str(), "user.test_attr", &attr[0],
EXPECT_EQ(0, setxattr(file_path.c_str(), "user.test_attr", attr.data(),
attr.size(), 0));
attr = "cow";
EXPECT_EQ(0, setxattr(file_path.c_str(), "user.test_attr", &attr[0],
EXPECT_EQ(0, setxattr(file_path.c_str(), "user.test_attr", attr.data(),
attr.size(), 0));
std::string val;
val.resize(attr.size());
EXPECT_EQ(attr.size(),
static_cast<std::size_t>(getxattr(
file_path.c_str(), "user.test_attr", &val[0], val.size())));
file_path.c_str(), "user.test_attr", val.data(), val.size())));
EXPECT_STREQ(attr.c_str(), val.c_str());
}
static void test_xattr_create_fails_if_exists(const std::string &file_path) {
std::string attr = "moose";
EXPECT_EQ(0, setxattr(file_path.c_str(), "user.test_attr", &attr[0],
EXPECT_EQ(0, setxattr(file_path.c_str(), "user.test_attr", attr.data(),
attr.size(), 0));
EXPECT_EQ(-1, setxattr(file_path.c_str(), "user.test_attr", &attr[0],
EXPECT_EQ(-1, setxattr(file_path.c_str(), "user.test_attr", attr.data(),
attr.size(), XATTR_CREATE));
EXPECT_EQ(EEXIST, errno);
}
@ -473,22 +474,22 @@ static void test_xattr_create_fails_if_exists(const std::string &file_path) {
static void
test_xattr_create_fails_if_not_exists(const std::string &file_path) {
std::string attr = "moose";
EXPECT_EQ(-1, setxattr(file_path.c_str(), "user.test_attr", &attr[0],
EXPECT_EQ(-1, setxattr(file_path.c_str(), "user.test_attr", attr.data(),
attr.size(), XATTR_REPLACE));
EXPECT_EQ(ENODATA, errno);
}
static void test_xattr_removexattr(const std::string &file_path) {
std::string attr = "moose";
EXPECT_EQ(0, setxattr(file_path.c_str(), "user.test_attr", &attr[0],
EXPECT_EQ(0, setxattr(file_path.c_str(), "user.test_attr", attr.data(),
attr.size(), XATTR_CREATE));
EXPECT_EQ(0, removexattr(file_path.c_str(), "user.test_attr"));
std::string val;
val.resize(attr.size());
EXPECT_EQ(-1,
getxattr(file_path.c_str(), "user.test_attr", &val[0], val.size()));
EXPECT_EQ(-1, getxattr(file_path.c_str(), "user.test_attr", val.data(),
val.size()));
EXPECT_EQ(ENODATA, errno);
}
#endif