From efe61762b56582fe3b77f0397a54feb0d105e2c8 Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Sun, 22 Dec 2024 10:33:15 -0600 Subject: [PATCH] unit tests and fixes --- .../include/file_manager/direct_open_file.hpp | 8 +- .../src/file_manager/direct_open_file.cpp | 6 +- .../file_manager/ring_buffer_open_file.cpp | 2 +- .../src/direct_open_file_test.cpp | 273 ++++++++++++++++++ 4 files changed, 286 insertions(+), 3 deletions(-) create mode 100644 repertory/repertory_test/src/direct_open_file_test.cpp diff --git a/repertory/librepertory/include/file_manager/direct_open_file.hpp b/repertory/librepertory/include/file_manager/direct_open_file.hpp index 85137acb..b0649428 100644 --- a/repertory/librepertory/include/file_manager/direct_open_file.hpp +++ b/repertory/librepertory/include/file_manager/direct_open_file.hpp @@ -35,7 +35,7 @@ public: direct_open_file(std::uint64_t chunk_size, std::uint8_t chunk_timeout, filesystem_item fsi, i_provider &provider); - ~direct_open_file() override = default; + ~direct_open_file() override; public: direct_open_file() = delete; @@ -45,6 +45,9 @@ public: auto operator=(const direct_open_file &) noexcept -> direct_open_file & = delete; +private: + std::size_t total_chunks_; + protected: [[nodiscard]] auto is_download_complete() const -> bool override { return false; @@ -66,6 +69,9 @@ public: get_read_state(std::size_t /* chunk */) const -> bool override { return false; } + [[nodiscard]] auto get_total_chunks() const -> std::uint64_t { + return total_chunks_; + } [[nodiscard]] auto native_operation(native_operation_callback /* callback */) -> api_error override { diff --git a/repertory/librepertory/src/file_manager/direct_open_file.cpp b/repertory/librepertory/src/file_manager/direct_open_file.cpp index 49ca81c4..5ad3e568 100644 --- a/repertory/librepertory/src/file_manager/direct_open_file.cpp +++ b/repertory/librepertory/src/file_manager/direct_open_file.cpp @@ -30,7 +30,11 @@ namespace repertory { direct_open_file::direct_open_file(std::uint64_t chunk_size, std::uint8_t chunk_timeout, filesystem_item fsi, i_provider &provider) - : open_file_base(chunk_size, chunk_timeout, fsi, provider) {} + : open_file_base(chunk_size, chunk_timeout, fsi, provider), + total_chunks_(static_cast( + utils::divide_with_ceiling(fsi.size, chunk_size))) {} + +direct_open_file::~direct_open_file() { close(); } auto direct_open_file::read(std::size_t read_size, std::uint64_t read_offset, data_buffer &data) -> api_error { diff --git a/repertory/librepertory/src/file_manager/ring_buffer_open_file.cpp b/repertory/librepertory/src/file_manager/ring_buffer_open_file.cpp index 8cea1495..8e6f225b 100644 --- a/repertory/librepertory/src/file_manager/ring_buffer_open_file.cpp +++ b/repertory/librepertory/src/file_manager/ring_buffer_open_file.cpp @@ -52,7 +52,7 @@ ring_buffer_open_file::ring_buffer_open_file(std::string buffer_directory, : open_file_base(chunk_size, chunk_timeout, fsi, provider), ring_state_(ring_size), total_chunks_(static_cast( - utils::divide_with_ceiling(fsi.size, chunk_size_))) { + utils::divide_with_ceiling(fsi.size, chunk_size))) { if ((ring_size % 2U) != 0U) { throw std::runtime_error("ring size must be a multiple of 2"); } diff --git a/repertory/repertory_test/src/direct_open_file_test.cpp b/repertory/repertory_test/src/direct_open_file_test.cpp new file mode 100644 index 00000000..e565b8eb --- /dev/null +++ b/repertory/repertory_test/src/direct_open_file_test.cpp @@ -0,0 +1,273 @@ +/* + Copyright <2018-2024> + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ +#include "test_common.hpp" + +#include "file_manager/direct_open_file.hpp" +#include "mocks/mock_provider.hpp" + +namespace { +constexpr const std::size_t test_chunk_size{1024U}; +} // namespace + +namespace repertory { +TEST(direct_open_file, read_full_file) { + auto &source_file = test::create_random_file(test_chunk_size * 32U); + + auto dest_path = test::generate_test_file_name("direct_open_file"); + + mock_provider provider; + EXPECT_CALL(provider, is_read_only()).WillRepeatedly(Return(false)); + + filesystem_item fsi; + fsi.api_path = "/test.txt"; + fsi.directory = false; + fsi.size = test_chunk_size * 32U; + + EXPECT_CALL(provider, read_file_bytes) + .WillRepeatedly([&source_file](const std::string & /* api_path */, + std::size_t size, std::uint64_t offset, + data_buffer &data, + stop_type &stop_requested) -> api_error { + EXPECT_FALSE(stop_requested); + std::size_t bytes_read{}; + data.resize(size); + auto ret = source_file.read(data, offset, &bytes_read) + ? api_error::success + : api_error::os_error; + EXPECT_EQ(bytes_read, data.size()); + return ret; + }); + { + direct_open_file file(test_chunk_size, 30U, fsi, provider); + + auto dest_file = utils::file::file::open_or_create_file(dest_path); + EXPECT_TRUE(dest_file); + + auto to_read{fsi.size}; + std::size_t chunk{0U}; + while (to_read > 0U) { + data_buffer data{}; + EXPECT_EQ(api_error::success, + file.read(test_chunk_size, chunk * test_chunk_size, data)); + + std::size_t bytes_written{}; + EXPECT_TRUE( + dest_file->write(data, chunk * test_chunk_size, &bytes_written)); + ++chunk; + to_read -= data.size(); + } + dest_file->close(); + source_file.close(); + + auto hash1 = utils::file::file(source_file.get_path()).sha256(); + auto hash2 = utils::file::file(dest_path).sha256(); + + EXPECT_TRUE(hash1.has_value()); + EXPECT_TRUE(hash2.has_value()); + if (hash1.has_value() && hash2.has_value()) { + EXPECT_STREQ(hash1.value().c_str(), hash2.value().c_str()); + } + } +} + +TEST(direct_open_file, read_full_file_in_reverse) { + auto &source_file = test::create_random_file(test_chunk_size * 32U); + + auto dest_path = test::generate_test_file_name("direct_open_file"); + + mock_provider provider; + EXPECT_CALL(provider, is_read_only()).WillRepeatedly(Return(false)); + + filesystem_item fsi; + fsi.api_path = "/test.txt"; + fsi.directory = false; + fsi.size = test_chunk_size * 32U; + + EXPECT_CALL(provider, read_file_bytes) + .WillRepeatedly([&source_file](const std::string & /* api_path */, + std::size_t size, std::uint64_t offset, + data_buffer &data, + stop_type &stop_requested) -> api_error { + EXPECT_FALSE(stop_requested); + std::size_t bytes_read{}; + data.resize(size); + auto ret = source_file.read(data, offset, &bytes_read) + ? api_error::success + : api_error::os_error; + EXPECT_EQ(bytes_read, data.size()); + return ret; + }); + { + direct_open_file file(test_chunk_size, 30U, fsi, provider); + + auto dest_file = utils::file::file::open_or_create_file(dest_path); + EXPECT_TRUE(dest_file); + + auto to_read{fsi.size}; + std::size_t chunk{file.get_total_chunks() - 1U}; + while (to_read > 0U) { + data_buffer data{}; + EXPECT_EQ(api_error::success, + file.read(test_chunk_size, chunk * test_chunk_size, data)); + + std::size_t bytes_written{}; + EXPECT_TRUE( + dest_file->write(data, chunk * test_chunk_size, &bytes_written)); + --chunk; + to_read -= data.size(); + } + dest_file->close(); + source_file.close(); + + auto hash1 = utils::file::file(source_file.get_path()).sha256(); + auto hash2 = utils::file::file(dest_path).sha256(); + + EXPECT_TRUE(hash1.has_value()); + EXPECT_TRUE(hash2.has_value()); + if (hash1.has_value() && hash2.has_value()) { + EXPECT_STREQ(hash1.value().c_str(), hash2.value().c_str()); + } + } +} + +TEST(direct_open_file, read_full_file_in_partial_chunks) { + auto &source_file = test::create_random_file(test_chunk_size * 32U); + + auto dest_path = test::generate_test_file_name("test"); + + mock_provider provider; + EXPECT_CALL(provider, is_read_only()).WillRepeatedly(Return(false)); + + filesystem_item fsi; + fsi.directory = false; + fsi.api_path = "/test.txt"; + fsi.size = test_chunk_size * 32U; + + EXPECT_CALL(provider, read_file_bytes) + .WillRepeatedly([&source_file](const std::string & /* api_path */, + std::size_t size, std::uint64_t offset, + data_buffer &data, + stop_type &stop_requested) -> api_error { + EXPECT_FALSE(stop_requested); + std::size_t bytes_read{}; + data.resize(size); + auto ret = source_file.read(data, offset, &bytes_read) + ? api_error::success + : api_error::os_error; + EXPECT_EQ(bytes_read, data.size()); + return ret; + }); + { + direct_open_file file(test_chunk_size, 30U, fsi, provider); + + auto dest_file = utils::file::file::open_or_create_file(dest_path); + EXPECT_TRUE(dest_file); + + auto total_read{std::uint64_t(0U)}; + while (total_read < fsi.size) { + data_buffer data{}; + EXPECT_EQ(api_error::success, file.read(3U, total_read, data)); + + std::size_t bytes_written{}; + EXPECT_TRUE(dest_file->write(data.data(), data.size(), total_read, + &bytes_written)); + total_read += data.size(); + } + dest_file->close(); + source_file.close(); + + auto hash1 = utils::file::file(source_file.get_path()).sha256(); + auto hash2 = utils::file::file(dest_path).sha256(); + + EXPECT_TRUE(hash1.has_value()); + EXPECT_TRUE(hash2.has_value()); + if (hash1.has_value() && hash2.has_value()) { + EXPECT_STREQ(hash1.value().c_str(), hash2.value().c_str()); + } + } +} + +TEST(direct_open_file, read_full_file_in_partial_chunks_in_reverse) { + auto &source_file = test::create_random_file(test_chunk_size * 32U); + + auto dest_path = test::generate_test_file_name("direct_open_file"); + + mock_provider provider; + EXPECT_CALL(provider, is_read_only()).WillRepeatedly(Return(false)); + + filesystem_item fsi; + fsi.directory = false; + fsi.api_path = "/test.txt"; + fsi.size = test_chunk_size * 32U; + + EXPECT_CALL(provider, read_file_bytes) + .WillRepeatedly([&source_file](const std::string & /* api_path */, + std::size_t size, std::uint64_t offset, + data_buffer &data, + stop_type &stop_requested) -> api_error { + EXPECT_FALSE(stop_requested); + std::size_t bytes_read{}; + data.resize(size); + auto ret = source_file.read(data, offset, &bytes_read) + ? api_error::success + : api_error::os_error; + EXPECT_EQ(bytes_read, data.size()); + return ret; + }); + { + direct_open_file file(test_chunk_size, 30U, fsi, provider); + + auto dest_file = utils::file::file::open_or_create_file(dest_path); + EXPECT_TRUE(dest_file); + + std::uint64_t total_read{0U}; + auto read_size{3U}; + + while (total_read < fsi.size) { + auto offset = fsi.size - total_read - read_size; + auto remain = fsi.size - total_read; + + data_buffer data{}; + EXPECT_EQ(api_error::success, + file.read(static_cast( + std::min(remain, std::uint64_t(read_size))), + (remain >= read_size) ? offset : 0U, data)); + + std::size_t bytes_written{}; + EXPECT_TRUE(dest_file->write(data, (remain >= read_size) ? offset : 0U, + &bytes_written)); + total_read += data.size(); + } + dest_file->close(); + source_file.close(); + + auto hash1 = utils::file::file(source_file.get_path()).sha256(); + auto hash2 = utils::file::file(dest_path).sha256(); + + EXPECT_TRUE(hash1.has_value()); + EXPECT_TRUE(hash2.has_value()); + if (hash1.has_value() && hash2.has_value()) { + EXPECT_STREQ(hash1.value().c_str(), hash2.value().c_str()); + } + } +} +} // namespace repertory