BlockStorage/repertory
1
0
Fork 0
Code Issues 7 Pull Requests Releases 8 Wiki Activity

refactor s3 provider

Browse Source
This commit is contained in:
Scott E. Graves
2023-11-11 11:32:14 -06:00
parent f2c1f64f02
commit 68476cbc00
55 changed files with 1504 additions and 5050 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
tests/file_manager_test.cpp
View File

@@ -114,8 +114,8 @@ TEST(file_manager, can_create_and_close_file) {
const auto now = utils::get_file_time_now();
auto meta = create_meta_attributes(
now, FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_ARCHIVE, now + 1u,
now + 2u, false, "token", 1, "key", 2, now + 3u, 3u, 4u, 0u,
source_path, 10, now + 4u);
now + 2u, false, 1, "key", 2, now + 3u, 3u, 4u, 0u, source_path, 10,
now + 4u);
EXPECT_CALL(mp, create_file("/test_create.txt", meta))
.WillOnce(Return(api_error::success));
@@ -127,7 +127,6 @@ TEST(file_manager, can_create_and_close_file) {
fsi.api_path = api_path;
fsi.api_parent = utils::path::get_parent_api_path(api_path);
fsi.directory = directory;
fsi.encryption_token = meta[META_ENCRYPTION_TOKEN];
fsi.size = utils::string::to_uint64(meta[META_SIZE]);
fsi.source_path = meta[META_SOURCE];
return api_error::success;
@@ -223,8 +222,8 @@ TEST(file_manager, can_open_and_close_file) {
const auto now = utils::get_file_time_now();
auto meta = create_meta_attributes(
now, FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_ARCHIVE, now + 1u,
now + 2u, false, "token", 1, "key", 2, now + 3u, 3u, 4u, 0u,
source_path, 10, now + 4u);
now + 2u, false, 1, "key", 2, now + 3u, 3u, 4u, 0u, source_path, 10,
now + 4u);
EXPECT_CALL(mp, create_file).Times(0u);
@@ -236,7 +235,6 @@ TEST(file_manager, can_open_and_close_file) {
fsi.api_path = api_path;
fsi.api_parent = utils::path::get_parent_api_path(api_path);
fsi.directory = directory;
fsi.encryption_token = meta[META_ENCRYPTION_TOKEN];
fsi.size = utils::string::to_uint64(meta[META_SIZE]);
fsi.source_path = meta[META_SOURCE];
return api_error::success;
@@ -325,8 +323,8 @@ TEST(file_manager, can_open_and_close_multiple_handles_for_same_file) {
const auto now = utils::get_file_time_now();
auto meta = create_meta_attributes(
now, FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_ARCHIVE, now + 1u,
now + 2u, false, "token", 1, "key", 2, now + 3u, 3u, 4u, 0u,
source_path, 10, now + 4u);
now + 2u, false, 1, "key", 2, now + 3u, 3u, 4u, 0u, source_path, 10,
now + 4u);
EXPECT_CALL(mp, create_file).Times(0u);
@@ -338,7 +336,6 @@ TEST(file_manager, can_open_and_close_multiple_handles_for_same_file) {
fsi.api_path = api_path;
fsi.api_parent = utils::path::get_parent_api_path(api_path);
fsi.directory = directory;
fsi.encryption_token = meta[META_ENCRYPTION_TOKEN];
fsi.size = utils::string::to_uint64(meta[META_SIZE]);
fsi.source_path = meta[META_SOURCE];
return api_error::success;
@@ -404,7 +401,7 @@ TEST(file_manager, download_is_stored_after_write_if_partially_downloaded) {
const auto now = utils::get_file_time_now();
auto meta = create_meta_attributes(
now, FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_ARCHIVE, now + 1u, now + 2u,
false, "", 1, "key", 2, now + 3u, 3u, 4u,
false, 1, "key", 2, now + 3u, 3u, 4u,
utils::encryption::encrypting_reader::get_data_chunk_size() * 4u,
source_path, 10, now + 4u);
auto nf = create_random_file(generate_test_file_name(".", "test_src"),
@@ -418,7 +415,6 @@ TEST(file_manager, download_is_stored_after_write_if_partially_downloaded) {
fsi.api_path = api_path;
fsi.api_parent = utils::path::get_parent_api_path(api_path);
fsi.directory = directory;
fsi.encryption_token = meta[META_ENCRYPTION_TOKEN];
fsi.size = utils::string::to_uint64(meta[META_SIZE]);
fsi.source_path = meta[META_SOURCE];
return api_error::success;
@@ -566,7 +562,7 @@ TEST(file_manager, upload_occurs_after_write_if_fully_downloaded) {
const auto now = utils::get_file_time_now();
auto meta = create_meta_attributes(
now, FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_ARCHIVE, now + 1u, now + 2u,
false, "", 1, "key", 2, now + 3u, 3u, 4u,
false, 1, "key", 2, now + 3u, 3u, 4u,
utils::encryption::encrypting_reader::get_data_chunk_size() * 4u,
source_path, 10, now + 4u);
auto nf = create_random_file(generate_test_file_name(".", "test_src"),
@@ -580,7 +576,6 @@ TEST(file_manager, upload_occurs_after_write_if_fully_downloaded) {
fsi.api_path = api_path;
fsi.api_parent = utils::path::get_parent_api_path(api_path);
fsi.directory = directory;
fsi.encryption_token = meta[META_ENCRYPTION_TOKEN];
fsi.size = utils::string::to_uint64(meta[META_SIZE]);
fsi.source_path = meta[META_SOURCE];
return api_error::success;
@@ -625,8 +620,8 @@ TEST(file_manager, upload_occurs_after_write_if_fully_downloaded) {
ec.wait_for_empty();
EXPECT_CALL(
mp, upload_file("/test_write_full_download.txt", source_path, "", _))
EXPECT_CALL(mp,
upload_file("/test_write_full_download.txt", source_path, _))
.WillOnce(Return(api_error::success));
event_capture ec2({"file_upload_queued", "file_upload_completed"});
@@ -634,8 +629,8 @@ TEST(file_manager, upload_occurs_after_write_if_fully_downloaded) {
ec2.wait_for_empty();
EXPECT_EQ(std::size_t(0u), fm.get_open_file_count());
EXPECT_EQ(std::size_t(0u), fm.get_open_handle_count());
EXPECT_EQ(std::size_t(0U), fm.get_open_file_count());
EXPECT_EQ(std::size_t(0U), fm.get_open_handle_count());
fm.stop();
@@ -676,8 +671,7 @@ TEST(file_manager, can_evict_file) {
auto meta = create_meta_attributes(
now, FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_ARCHIVE, now + 1u, now + 2u,
false, "token", 1, "key", 2, now + 3u, 3u, 4u, 0u, source_path, 10,
now + 4u);
false, 1, "key", 2, now + 3u, 3u, 4u, 0u, source_path, 10, now + 4u);
std::uint64_t handle{};
{
std::shared_ptr<i_open_file> f;
@@ -692,7 +686,6 @@ TEST(file_manager, can_evict_file) {
fsi.api_path = api_path;
fsi.api_parent = utils::path::get_parent_api_path(api_path);
fsi.directory = directory;
fsi.encryption_token = meta[META_ENCRYPTION_TOKEN];
fsi.size = utils::string::to_uint64(meta[META_SIZE]);
fsi.source_path = meta[META_SOURCE];
return api_error::success;
@@ -708,7 +701,7 @@ TEST(file_manager, can_evict_file) {
EXPECT_CALL(mp, set_item_meta("/test_evict.txt", _))
.Times(2)
.WillRepeatedly(Return(api_error::success));
EXPECT_CALL(mp, upload_file(_, _, _, _))
EXPECT_CALL(mp, upload_file(_, _, _))
.WillOnce(Return(api_error::success));
data_buffer data{{0, 1, 1}};
@@ -944,7 +937,7 @@ TEST(file_manager, evict_file_fails_if_file_is_uploading) {
auto meta = create_meta_attributes(
now, FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_ARCHIVE, now + 1u, now + 2u,
false, "", 1, "", 2, now + 3u, 3u, 4u, 0u, source_path, 10, now + 4u);
false, 1, "", 2, now + 3u, 3u, 4u, 0u, source_path, 10, now + 4u);
std::uint64_t handle{};
{
std::shared_ptr<i_open_file> f;
@@ -959,7 +952,6 @@ TEST(file_manager, evict_file_fails_if_file_is_uploading) {
fsi.api_path = api_path;
fsi.api_parent = utils::path::get_parent_api_path(api_path);
fsi.directory = directory;
fsi.encryption_token = meta[META_ENCRYPTION_TOKEN];
fsi.size = utils::string::to_uint64(meta[META_SIZE]);
fsi.source_path = meta[META_SOURCE];
return api_error::success;
@@ -978,11 +970,9 @@ TEST(file_manager, evict_file_fails_if_file_is_uploading) {
EXPECT_CALL(mp, upload_file)
.WillOnce([](const std::string &api_path,
const std::string &source_path2,
const std::string &encryption_token,
stop_type & /*stop_requested*/) -> api_error {
EXPECT_STREQ("/test_evict.txt", api_path.c_str());
EXPECT_FALSE(source_path2.empty());
EXPECT_TRUE(encryption_token.empty());
std::this_thread::sleep_for(3s);
return api_error::success;
});
@@ -1178,7 +1168,7 @@ TEST(file_manager, file_is_not_opened_if_provider_create_file_fails) {
const auto now = utils::get_file_time_now();
auto meta = create_meta_attributes(
now, FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_ARCHIVE, now + 1u, now + 2u,
false, "", 1, "", 2, now + 3u, 3u, 4u, 0u, "/test_create.src", 10,
false, 1, "", 2, now + 3u, 3u, 4u, 0u, "/test_create.src", 10,
now + 4u);
file_manager fm(cfg, mp);
@@ -1721,7 +1711,7 @@ TEST(file_manager, file_is_closed_after_download_timeout) {
const auto now = utils::get_file_time_now();
auto meta = create_meta_attributes(
now, FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_ARCHIVE, now + 1u, now + 2u,
false, "", 1, "key", 2, now + 3u, 3u, 4u,
false, 1, "key", 2, now + 3u, 3u, 4u,
utils::encryption::encrypting_reader::get_data_chunk_size() * 4u,
source_path, 10, now + 4u);
@@ -1733,7 +1723,6 @@ TEST(file_manager, file_is_closed_after_download_timeout) {
fsi.api_path = api_path;
fsi.api_parent = utils::path::get_parent_api_path(api_path);
fsi.directory = directory;
fsi.encryption_token = meta[META_ENCRYPTION_TOKEN];
fsi.size = utils::string::to_uint64(meta[META_SIZE]);
fsi.source_path = meta[META_SOURCE];
return api_error::success;