From 5e8bce5d6357ccb4b0f0b1b84339830721464d30 Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Mon, 15 Sep 2025 12:19:08 -0500 Subject: [PATCH 01/26] partial remove support --- .../librepertory/include/types/repertory.hpp | 3 +- repertory/repertory/include/cli/remove.hpp | 66 +++++++++++++++++++ 2 files changed, 68 insertions(+), 1 deletion(-) create mode 100644 repertory/repertory/include/cli/remove.hpp diff --git a/repertory/librepertory/include/types/repertory.hpp b/repertory/librepertory/include/types/repertory.hpp index d2f37890..dfa2770b 100644 --- a/repertory/librepertory/include/types/repertory.hpp +++ b/repertory/librepertory/include/types/repertory.hpp @@ -204,7 +204,8 @@ enum class exit_code : std::int32_t { ui_mount_failed = -19, exception = -20, provider_offline = -21, - ui_failed = -22 + ui_failed = -22, + remove_failed = -23 }; enum http_error_codes : std::int32_t { diff --git a/repertory/repertory/include/cli/remove.hpp b/repertory/repertory/include/cli/remove.hpp new file mode 100644 index 00000000..5e824cdf --- /dev/null +++ b/repertory/repertory/include/cli/remove.hpp @@ -0,0 +1,66 @@ +/* + Copyright <2018-2025> + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ +#ifndef REPERTORY_INCLUDE_CLI_REMOVE_HPP_ +#define REPERTORY_INCLUDE_CLI_REMOVE_HPP_ + +#include "cli/common.hpp" + +#include "utils/string.hpp" +#include "utils/time.hpp" + +namespace repertory::cli::actions { +[[nodiscard]] inline auto +remove(std::vector /* args */, const std::string &data_directory, + provider_type prov, const std::string &unique_id, std::string /* user */, + std::string /* password */) -> exit_code { + lock_data lock(prov, unique_id); + auto res = lock.grab_lock(1); + if (res != lock_result::success) { + std::cout << static_cast(exit_code::lock_failed) << std::endl; + std::cerr << "FATAL: Unable to get provider lock" << std::endl; + return exit_code::lock_failed; + } + + auto trash_path = utils::path::combine( + app_config::get_root_data_directory(), + { + "trash", + app_config::get_provider_name(prov), + unique_id, + fmt::format("{}_{}", utils::time::get_current_time_utc(), + utils::collection::to_hex_string( + utils::generate_secure_random(4U))), + }); + if (utils::file::directory{data_directory}.move_to(trash_path)) { + fmt::println("0"); + fmt::println("successfully removed provider|type|{}|id|{}", + app_config::get_provider_name(prov), unique_id); + return exit_code::success; + } + + std::cout << static_cast(exit_code::remove_failed) << std::endl; + std::cerr << "FATAL: Failed to remove|" << data_directory << std::endl; + return exit_code::remove_failed; +} +} // namespace repertory::cli::actions + +#endif // REPERTORY_INCLUDE_CLI_REMOVE_HPP_ From 1f01a43dd506130ff90448d0fa72547354a00148 Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Mon, 15 Sep 2025 12:25:25 -0500 Subject: [PATCH 02/26] partial support for remove --- repertory/repertory/include/cli/remove.hpp | 1 - repertory/repertory/include/ui/ui_server.hpp | 3 +++ repertory/repertory/src/ui/ui_server.cpp | 19 +++++++++++++++++++ 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/repertory/repertory/include/cli/remove.hpp b/repertory/repertory/include/cli/remove.hpp index 5e824cdf..3564138a 100644 --- a/repertory/repertory/include/cli/remove.hpp +++ b/repertory/repertory/include/cli/remove.hpp @@ -24,7 +24,6 @@ #include "cli/common.hpp" -#include "utils/string.hpp" #include "utils/time.hpp" namespace repertory::cli::actions { diff --git a/repertory/repertory/include/ui/ui_server.hpp b/repertory/repertory/include/ui/ui_server.hpp index 0f11cd05..8da23394 100644 --- a/repertory/repertory/include/ui/ui_server.hpp +++ b/repertory/repertory/include/ui/ui_server.hpp @@ -83,6 +83,9 @@ private: static void handle_get_available_locations(httplib::Response &res); + void handle_del_remove_mount(const httplib::Request &req, + httplib::Response &res); + void handle_get_mount(const httplib::Request &req, httplib::Response &res) const; diff --git a/repertory/repertory/src/ui/ui_server.cpp b/repertory/repertory/src/ui/ui_server.cpp index 1ca6cf7f..2cffbaa5 100644 --- a/repertory/repertory/src/ui/ui_server.cpp +++ b/repertory/repertory/src/ui/ui_server.cpp @@ -207,6 +207,10 @@ ui_server::ui_server(mgmt_app_config *config) : http_error_codes::internal_error; }); + server_.Delete("/api/v1/remove_mount", [this](auto &&req, auto &&res) { + handle_del_remove_mount(req, res); + }); + server_.Get("/api/v1/locations", [](auto && /* req */, auto &&res) { handle_get_available_locations(res); }); @@ -339,6 +343,21 @@ void ui_server::generate_config(provider_type prov, std::string_view name, } } +void ui_server::handle_del_remove_mount(const httplib::Request &req, + httplib::Response &res) { + auto prov = provider_type_from_string(req.get_param_value("type")); + auto name = req.get_param_value("name"); + + if (not data_directory_exists(prov, name)) { + res.status = http_error_codes::not_found; + return; + } + + auto lines = launch_process(prov, name, {"-rp", "-f"}, false); + res.status = lines.at(0U) == "0" ? http_error_codes::ok + : http_error_codes::internal_error; +} + void ui_server::handle_put_mount_auto_start(const httplib::Request &req, httplib::Response &res) const { auto prov = provider_type_from_string(req.get_param_value("type")); From 826c3a9008c164813ff48617b1821569d5725ab8 Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Fri, 26 Sep 2025 14:12:14 -0500 Subject: [PATCH 03/26] added packet/packet_client tests --- .../repertory_test/src/packet_client_test.cpp | 104 ++++ repertory/repertory_test/src/packet_test.cpp | 467 +++++++++++++++++- 2 files changed, 550 insertions(+), 21 deletions(-) create mode 100644 repertory/repertory_test/src/packet_client_test.cpp diff --git a/repertory/repertory_test/src/packet_client_test.cpp b/repertory/repertory_test/src/packet_client_test.cpp new file mode 100644 index 00000000..d2969847 --- /dev/null +++ b/repertory/repertory_test/src/packet_client_test.cpp @@ -0,0 +1,104 @@ +/* + Copyright <2018-2025> + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to do so, subject to the + following conditions: The above copyright notice and this permission notice + shall be included in all copies or substantial portions of the Software. + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ +#include "test_common.hpp" + +#include "comm/packet/common.hpp" +#include "comm/packet/packet.hpp" +#include "comm/packet/packet_client.hpp" +#include "comm/packet/packet_server.hpp" +#include "types/remote.hpp" +#include "utils/common.hpp" +#include "utils/utils.hpp" +#include "version.hpp" + +using namespace repertory; +using namespace repertory::comm; + +namespace { +class test_packet_server final { +public: + test_packet_server(std::uint16_t port, std::string token, + std::uint8_t pool_size = 2) + : server_(std::make_unique( + port, std::move(token), pool_size, + [](const std::string & /*client_id*/) {}, + [](std::uint32_t /*service_flags_in*/, + const std::string & /*client_id*/, std::uint64_t /*thread_id*/, + const std::string &method, packet * /*request*/, + packet & /*response*/, + packet_server::message_complete_callback done) { + if (method == "ping") { + done(packet::error_type{0}); + } else { + done(packet::error_type{-1}); + } + })) {} + +private: + std::unique_ptr server_; +}; + +inline auto make_cfg(std::uint16_t port, const std::string &token) + -> remote::remote_config { + remote::remote_config cfg{}; + cfg.host_name_or_ip = "127.0.0.1"; + cfg.api_port = port; + cfg.max_connections = 2U; + cfg.conn_timeout_ms = 1500U; + cfg.recv_timeout_ms = 1500U; + cfg.send_timeout_ms = 1500U; + cfg.encryption_token = token; + return cfg; +} +} // namespace + +TEST(packet_client_test, can_check_version) { + std::string token = "cow_moose_doge_chicken"; + + std::uint16_t port{}; + ASSERT_TRUE(utils::get_next_available_port(50000U, port)); + + test_packet_server server(port, token); + + packet_client client(make_cfg(port, token)); + + std::uint32_t min_version{}; + auto api = client.check_version( + utils::get_version_number(project_get_version()), min_version); + + EXPECT_EQ(api, api_error::success); + EXPECT_NE(min_version, 0U); +} + +TEST(packet_client_test, can_send_request_and_receive_response) { + std::string token = "cow_moose_doge_chicken"; + + std::uint16_t port{}; + ASSERT_TRUE(utils::get_next_available_port(50000U, port)); + + test_packet_server server(port, token); + + packet_client client(make_cfg(port, token)); + + std::uint32_t service_flags{}; + packet request; + packet response; + auto ret = client.send("ping", request, response, service_flags); + + EXPECT_EQ(ret, 0); +} diff --git a/repertory/repertory_test/src/packet_test.cpp b/repertory/repertory_test/src/packet_test.cpp index 73ef16a8..4c8bbd70 100644 --- a/repertory/repertory_test/src/packet_test.cpp +++ b/repertory/repertory_test/src/packet_test.cpp @@ -1,27 +1,24 @@ -/* - Copyright <2018-2025> - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. +/* Copyright <2018-2025> + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. */ #include "test_common.hpp" #include "comm/packet/packet.hpp" +#include "types/remote.hpp" namespace repertory { TEST(packet_test, encrypt_and_decrypt) { @@ -35,7 +32,435 @@ TEST(packet_test, encrypt_and_decrypt) { std::string data; EXPECT_EQ(0, test_packet.decode(data)); - EXPECT_STREQ("test", data.c_str()); } + +TEST(packet_test, encode_decode_primitives_and_strings) { + packet pkt; + + const std::int8_t i8{-12}; + const std::uint8_t u8{250}; + const std::int16_t i16{-12345}; + const std::uint16_t u16{54321}; + const std::int32_t i32{-123456789}; + const std::uint32_t u32{3141592653U}; + const std::int64_t i64{-1234567890123456789LL}; + const std::uint64_t u64{12345678901234567890ULL}; + const std::string s{"hello world"}; + const std::wstring ws{L"wide 🌟"}; + + pkt.encode(i8); + pkt.encode(u8); + pkt.encode(i16); + pkt.encode(u16); + pkt.encode(i32); + pkt.encode(u32); + pkt.encode(i64); + pkt.encode(u64); + pkt.encode(s); + pkt.encode(ws); + + std::int8_t i8_r{}; + std::uint8_t u8_r{}; + std::int16_t i16_r{}; + std::uint16_t u16_r{}; + std::int32_t i32_r{}; + std::uint32_t u32_r{}; + std::int64_t i64_r{}; + std::uint64_t u64_r{}; + std::string s_r; + std::wstring ws_r; + + EXPECT_EQ(0, pkt.decode(i8_r)); + EXPECT_EQ(0, pkt.decode(u8_r)); + EXPECT_EQ(0, pkt.decode(i16_r)); + EXPECT_EQ(0, pkt.decode(u16_r)); + EXPECT_EQ(0, pkt.decode(i32_r)); + EXPECT_EQ(0, pkt.decode(u32_r)); + EXPECT_EQ(0, pkt.decode(i64_r)); + EXPECT_EQ(0, pkt.decode(u64_r)); + EXPECT_EQ(0, pkt.decode(s_r)); + EXPECT_EQ(0, pkt.decode(ws_r)); + + EXPECT_EQ(i8, i8_r); + EXPECT_EQ(u8, u8_r); + EXPECT_EQ(i16, i16_r); + EXPECT_EQ(u16, u16_r); + EXPECT_EQ(i32, i32_r); + EXPECT_EQ(u32, u32_r); + EXPECT_EQ(i64, i64_r); + EXPECT_EQ(u64, u64_r); + EXPECT_EQ(s, s_r); + EXPECT_EQ(ws, ws_r); +} + +TEST(packet_test, encode_decode_null_c_string_is_empty) { + packet pkt; + const char *p_null{nullptr}; + pkt.encode(p_null); + + std::string out; + EXPECT_EQ(0, pkt.decode(out)); + EXPECT_TRUE(out.empty()); +} + +TEST(packet_test, encode_decode_raw_buffer) { + packet pkt; + + std::array src{}; + for (std::size_t i = 0; i < src.size(); ++i) { + src[i] = static_cast(i); + } + + pkt.encode(src.data(), src.size()); + + std::array dst{}; + EXPECT_EQ(0, pkt.decode(dst.data(), dst.size())); + EXPECT_EQ(0, std::memcmp(src.data(), dst.data(), dst.size())); +} + +TEST(packet_test, encode_decode_pointer_round_trip) { + packet pkt; + + int val{42}; + void *in_ptr = &val; + + pkt.encode(in_ptr); + + void *out_ptr = nullptr; + EXPECT_EQ(0, pkt.decode(out_ptr)); + EXPECT_EQ(in_ptr, out_ptr); +} + +TEST(packet_test, encode_top_affects_decode_order) { + packet pkt; + + pkt.encode(static_cast(1)); + pkt.encode(static_cast(2)); + pkt.encode_top(static_cast(99)); + + std::uint32_t a{}, b{}, c{}; + EXPECT_EQ(0, pkt.decode(a)); + EXPECT_EQ(0, pkt.decode(b)); + EXPECT_EQ(0, pkt.decode(c)); + + EXPECT_EQ(99U, a); + EXPECT_EQ(1U, b); + EXPECT_EQ(2U, c); +} + +TEST(packet_test, copy_ctor_preserves_decode_offset) { + packet pkt; + pkt.encode(static_cast(10)); + pkt.encode(static_cast(20)); + pkt.encode(static_cast(30)); + + std::uint32_t first{}; + ASSERT_EQ(0, pkt.decode(first)); + ASSERT_EQ(10U, first); + + packet pkt_copy{pkt}; + + std::uint32_t from_copy{}; + EXPECT_EQ(0, pkt_copy.decode(from_copy)); + EXPECT_EQ(20U, from_copy); + + std::uint32_t from_src{}; + + EXPECT_EQ(0, pkt.decode(from_src)); + EXPECT_EQ(20U, from_src); +} + +TEST(packet_test, copy_assign_preserves_decode_offset) { + packet src; + src.encode(static_cast(1)); + src.encode(static_cast(2)); + src.encode(static_cast(3)); + + std::uint32_t first{}; + ASSERT_EQ(0, src.decode(first)); + ASSERT_EQ(1U, first); + + packet dst; + dst = src; + + std::uint32_t from_dst{}; + EXPECT_EQ(0, dst.decode(from_dst)); + EXPECT_EQ(2U, from_dst); + + std::uint32_t from_src{}; + EXPECT_EQ(0, src.decode(from_src)); + EXPECT_EQ(2U, from_src); +} + +TEST(packet_test, move_ctor_preserves_decode_offset) { + packet pkt; + pkt.encode(static_cast(100)); + pkt.encode(static_cast(200)); + + std::uint32_t x{}; + ASSERT_EQ(0, pkt.decode(x)); + ASSERT_EQ(100U, x); + + packet moved{std::move(pkt)}; + + std::uint32_t y{}; + EXPECT_EQ(0, moved.decode(y)); + EXPECT_EQ(200U, y); +} + +TEST(packet_test, move_assign_preserves_decode_offset) { + packet src; + src.encode(static_cast(7)); + src.encode(static_cast(8)); + src.encode(static_cast(9)); + + std::uint32_t first{}; + ASSERT_EQ(0, src.decode(first)); + ASSERT_EQ(7U, first); + + packet dst; + dst.encode( + static_cast(999)); // give dst some state to overwrite + dst = std::move(src); + + std::uint32_t next{}; + EXPECT_EQ(0, dst.decode(next)); + EXPECT_EQ(8U, next); +} + +TEST(packet_test, operator_index_read_write_byte) { + packet pkt; + pkt.encode(static_cast(7)); + + EXPECT_EQ(pkt[0], static_cast(7)); + + pkt[0] = static_cast(8); + + std::uint8_t out{}; + EXPECT_EQ(0, pkt.decode(out)); + EXPECT_EQ(8U, out); +} + +TEST(packet_test, current_pointer_tracks_decode_offset) { + packet pkt; + pkt.encode(static_cast(1)); + pkt.encode(static_cast(2)); + + std::uint8_t first{}; + EXPECT_EQ(0, pkt.decode(first)); + EXPECT_EQ(1U, first); + + auto *ptr = pkt.current_pointer(); + ASSERT_NE(ptr, nullptr); + EXPECT_EQ(*ptr, static_cast(2)); + + const auto &const_pkt = pkt; + const auto *cptr = const_pkt.current_pointer(); + ASSERT_NE(cptr, nullptr); + EXPECT_EQ(*cptr, static_cast(2)); +} + +TEST(packet_test, open_flags_round_trip) { + packet pkt; + + remote::open_flags flags = remote::open_flags::read_only | + remote::open_flags::create | + remote::open_flags::truncate; + + pkt.encode(flags); + + remote::open_flags decoded{}; + EXPECT_EQ(0, pkt.decode(decoded)); + + EXPECT_EQ(static_cast(flags), + static_cast(decoded)); +} + +TEST(packet_test, encrypt_and_decrypt_without_size_prefix) { + packet pkt; + pkt.encode("opaque"); + pkt.encrypt("moose", false); + + EXPECT_EQ(0, pkt.decrypt("moose")); + + std::string out; + EXPECT_EQ(0, pkt.decode(out)); + EXPECT_EQ("opaque", out); +} + +TEST(packet_test, decode_fails_when_empty) { + packet pkt; + std::uint32_t val{}; + EXPECT_NE(0, pkt.decode(val)); +} + +TEST(packet_test, decode_json_round_trip) { + packet pkt; + + const json src = {{"x", 1}, {"y", "z"}, {"ok", true}}; + pkt.encode(src.dump()); + + json got{}; + EXPECT_EQ(0, packet::decode_json(pkt, got)); + EXPECT_EQ(src, got); +} + +TEST(packet_test, remote_setattr_x_round_trip) { + packet pkt; + + remote::setattr_x sa{}; + sa.valid = 0x7; + sa.mode = static_cast(0644); + sa.uid = 1001U; + sa.gid = 1002U; + sa.size = 1234567ULL; + sa.acctime = 111ULL; + sa.modtime = 222ULL; + sa.crtime = 333ULL; + sa.chgtime = 444ULL; + sa.bkuptime = 555ULL; + sa.flags = 0xA5A5A5A5U; + + pkt.encode(sa); + + remote::setattr_x out{}; + EXPECT_EQ(0, pkt.decode(out)); + + EXPECT_EQ(sa.valid, out.valid); + EXPECT_EQ(sa.mode, out.mode); + EXPECT_EQ(sa.uid, out.uid); + EXPECT_EQ(sa.gid, out.gid); + EXPECT_EQ(sa.size, out.size); + EXPECT_EQ(sa.acctime, out.acctime); + EXPECT_EQ(sa.modtime, out.modtime); + EXPECT_EQ(sa.crtime, out.crtime); + EXPECT_EQ(sa.chgtime, out.chgtime); + EXPECT_EQ(sa.bkuptime, out.bkuptime); + EXPECT_EQ(sa.flags, out.flags); +} + +TEST(packet_test, remote_stat_round_trip) { + packet pkt; + + remote::stat st{}; + st.st_mode = static_cast(0755); + st.st_nlink = static_cast(2); + st.st_uid = 2001U; + st.st_gid = 2002U; + st.st_atimespec = 101ULL; + st.st_mtimespec = 202ULL; + st.st_ctimespec = 303ULL; + st.st_birthtimespec = 404ULL; + st.st_size = 987654321ULL; + st.st_blocks = 4096ULL; + st.st_blksize = 8192U; + st.st_flags = 0xDEADBEEFU; + + pkt.encode(st); + + remote::stat out{}; + EXPECT_EQ(0, pkt.decode(out)); + + EXPECT_EQ(st.st_mode, out.st_mode); + EXPECT_EQ(st.st_nlink, out.st_nlink); + EXPECT_EQ(st.st_uid, out.st_uid); + EXPECT_EQ(st.st_gid, out.st_gid); + EXPECT_EQ(st.st_atimespec, out.st_atimespec); + + EXPECT_EQ(st.st_mtimespec, out.st_mtimespec); + EXPECT_EQ(st.st_ctimespec, out.st_ctimespec); + EXPECT_EQ(st.st_birthtimespec, out.st_birthtimespec); + EXPECT_EQ(st.st_size, out.st_size); + EXPECT_EQ(st.st_blocks, out.st_blocks); + EXPECT_EQ(st.st_blksize, out.st_blksize); + EXPECT_EQ(st.st_flags, out.st_flags); +} + +TEST(packet_test, remote_statfs_round_trip) { + packet pkt; + + remote::statfs sfs{}; + sfs.f_bavail = 1'000'000ULL; + sfs.f_bfree = 2'000'000ULL; + sfs.f_blocks = 3'000'000ULL; + sfs.f_favail = 4'000'000ULL; + sfs.f_ffree = 5'000'000ULL; + sfs.f_files = 6'000'000ULL; + + pkt.encode(sfs); + + remote::statfs out{}; + EXPECT_EQ(0, pkt.decode(out)); + + EXPECT_EQ(sfs.f_bavail, out.f_bavail); + EXPECT_EQ(sfs.f_bfree, out.f_bfree); + EXPECT_EQ(sfs.f_blocks, out.f_blocks); + EXPECT_EQ(sfs.f_favail, out.f_favail); + EXPECT_EQ(sfs.f_ffree, out.f_ffree); + EXPECT_EQ(sfs.f_files, out.f_files); +} + +TEST(packet_test, remote_statfs_x_round_trip) { + packet pkt; + + remote::statfs_x sfsx{}; + sfsx.f_bavail = 7'000'000ULL; + sfsx.f_bfree = 8'000'000ULL; + sfsx.f_blocks = 9'000'000ULL; + sfsx.f_favail = 10'000'000ULL; + sfsx.f_ffree = 11'000'000ULL; + sfsx.f_files = 12'000'000ULL; + + const char *mnt = "test_mnt"; + std::memcpy(sfsx.f_mntfromname.data(), mnt, std::strlen(mnt)); + + pkt.encode(sfsx); + + remote::statfs_x out{}; + EXPECT_EQ(0, pkt.decode(out)); + + EXPECT_EQ(sfsx.f_bavail, out.f_bavail); + EXPECT_EQ(sfsx.f_bfree, out.f_bfree); + EXPECT_EQ(sfsx.f_blocks, out.f_blocks); + EXPECT_EQ(sfsx.f_favail, out.f_favail); + EXPECT_EQ(sfsx.f_ffree, out.f_ffree); + EXPECT_EQ(sfsx.f_files, out.f_files); + EXPECT_EQ(0, std::memcmp(sfsx.f_mntfromname.data(), out.f_mntfromname.data(), + sfsx.f_mntfromname.size())); +} + +TEST(packet_test, remote_file_info_round_trip) { + packet pkt; + + remote::file_info fi{}; + fi.FileAttributes = 0x1234U; + fi.ReparseTag = 0x5678U; + fi.AllocationSize = 1111ULL; + fi.FileSize = 2222ULL; + fi.CreationTime = 3333ULL; + fi.LastAccessTime = 4444ULL; + fi.LastWriteTime = 5555ULL; + fi.ChangeTime = 6666ULL; + fi.IndexNumber = 7777ULL; + fi.HardLinks = 3U; + fi.EaSize = 0U; + + pkt.encode(fi); + + remote::file_info out{}; + EXPECT_EQ(0, pkt.decode(out)); + + EXPECT_EQ(fi.FileAttributes, fi.FileAttributes); + EXPECT_EQ(fi.ReparseTag, out.ReparseTag); + EXPECT_EQ(fi.AllocationSize, out.AllocationSize); + EXPECT_EQ(fi.FileSize, out.FileSize); + EXPECT_EQ(fi.CreationTime, out.CreationTime); + EXPECT_EQ(fi.LastAccessTime, out.LastAccessTime); + EXPECT_EQ(fi.LastWriteTime, out.LastWriteTime); + EXPECT_EQ(fi.ChangeTime, out.ChangeTime); + EXPECT_EQ(fi.IndexNumber, out.IndexNumber); + EXPECT_EQ(fi.HardLinks, out.HardLinks); + EXPECT_EQ(fi.EaSize, out.EaSize); +} } // namespace repertory From 8a9394591b365e0ef53f5e6eb0cbe9b7d7d11f4b Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Fri, 26 Sep 2025 14:17:31 -0500 Subject: [PATCH 04/26] added additional test --- .../src/fuse_drive_unlink_test.cpp | 1 - .../repertory_test/src/packet_client_test.cpp | 18 ++++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/repertory/repertory_test/src/fuse_drive_unlink_test.cpp b/repertory/repertory_test/src/fuse_drive_unlink_test.cpp index 6818b543..6409f5be 100644 --- a/repertory/repertory_test/src/fuse_drive_unlink_test.cpp +++ b/repertory/repertory_test/src/fuse_drive_unlink_test.cpp @@ -54,7 +54,6 @@ TYPED_TEST(fuse_test, unlink_open_file_leaves_handle_intact) { ASSERT_EQ(0, ::unlink(path.c_str())); auto res = ::lseek(desc, 0, SEEK_END); - fmt::println("lseek|{}|{}", res, errno); ASSERT_NE(-1, res); this->write_all(desc, " WORLD"); diff --git a/repertory/repertory_test/src/packet_client_test.cpp b/repertory/repertory_test/src/packet_client_test.cpp index d2969847..5b2ea3bd 100644 --- a/repertory/repertory_test/src/packet_client_test.cpp +++ b/repertory/repertory_test/src/packet_client_test.cpp @@ -85,6 +85,24 @@ TEST(packet_client_test, can_check_version) { EXPECT_NE(min_version, 0U); } +TEST(packet_client_test, can_detect_incompatible_version) { + std::string token = "cow_moose_doge_chicken"; + + std::uint16_t port{}; + ASSERT_TRUE(utils::get_next_available_port(50000U, port)); + + test_packet_server server(port, token); + + packet_client client(make_cfg(port, token)); + + std::uint32_t min_version{}; + auto api = + client.check_version(utils::get_version_number("1.0.0-rc"), min_version); + + EXPECT_EQ(api, api_error::incompatible_version); + EXPECT_NE(min_version, 0U); +} + TEST(packet_client_test, can_send_request_and_receive_response) { std::string token = "cow_moose_doge_chicken"; From b3624427d5960d406745979a4064b87587ace97b Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Sat, 27 Sep 2025 06:14:52 -0500 Subject: [PATCH 05/26] [unit test] Complete FUSE unit tests #22 --- .../src/fuse_drive_directory_test.cpp | 16 +++ .../src/fuse_drive_fsync_test.cpp | 123 ++++++++++++++++++ .../src/fuse_drive_rdrw_test.cpp | 32 +++++ 3 files changed, 171 insertions(+) create mode 100644 repertory/repertory_test/src/fuse_drive_fsync_test.cpp diff --git a/repertory/repertory_test/src/fuse_drive_directory_test.cpp b/repertory/repertory_test/src/fuse_drive_directory_test.cpp index 88e2cdeb..22088085 100644 --- a/repertory/repertory_test/src/fuse_drive_directory_test.cpp +++ b/repertory/repertory_test/src/fuse_drive_directory_test.cpp @@ -113,6 +113,22 @@ TYPED_TEST(fuse_test, directory_can_opendir_after_closedir) { this->rmdir_and_test(dir); } + +TYPED_TEST(fuse_test, directory_rmdir_on_non_empty_directory_should_fail) { + std::string dir_name{"non_empty"}; + auto dir = this->create_directory_and_test(dir_name); + + std::string fname{dir_name + "/child"}; + auto fpath = this->create_file_and_test(fname, 0644); + this->overwrite_text(fpath, "X"); + + errno = 0; + EXPECT_EQ(-1, ::rmdir(dir.c_str())); + EXPECT_EQ(ENOTEMPTY, errno); + + this->unlink_file_and_test(fpath); + this->rmdir_and_test(dir); +} } // namespace repertory #endif // !defined(_WIN32) diff --git a/repertory/repertory_test/src/fuse_drive_fsync_test.cpp b/repertory/repertory_test/src/fuse_drive_fsync_test.cpp new file mode 100644 index 00000000..50a27b42 --- /dev/null +++ b/repertory/repertory_test/src/fuse_drive_fsync_test.cpp @@ -0,0 +1,123 @@ +/* + Copyright <2018-2025> + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ +#if !defined(_WIN32) + +#include "fixtures/drive_fixture.hpp" + +namespace repertory { +TYPED_TEST_SUITE(fuse_test, platform_provider_types); + +TYPED_TEST(fuse_test, fsync_basic_succeeds_on_dirty_desc) { + std::string name{"fsync_dirty"}; + auto path = this->create_file_and_test(name, 0644); + + auto desc = ::open(path.c_str(), O_RDWR); + ASSERT_NE(desc, -1); + + this->write_all(desc, "ABC"); + errno = 0; + ASSERT_EQ(0, ::fsync(desc)); + ::close(desc); + + EXPECT_EQ("ABC", this->slurp(path)); + this->unlink_file_and_test(path); +} + +TYPED_TEST(fuse_test, fsync_noop_on_clean_desc) { + std::string name{"fsync_clean"}; + auto path = this->create_file_and_test(name, 0644); + + auto desc = ::open(path.c_str(), O_RDONLY); + ASSERT_NE(desc, -1); + + errno = 0; + EXPECT_EQ(0, ::fsync(desc)); + ::close(desc); + + this->unlink_file_and_test(path); +} + +TYPED_TEST(fuse_test, fsync_on_unlinked_file) { + std::string name{"fsync_unlinked"}; + auto path = this->create_file_and_test(name, 0644); + + auto desc = ::open(path.c_str(), O_RDWR); + ASSERT_NE(desc, -1); + + ASSERT_EQ(0, ::unlink(path.c_str())); + + this->write_all(desc, "XYZ"); + errno = 0; + EXPECT_EQ(0, ::fsync(desc)); + ::close(desc); +} + +TYPED_TEST(fuse_test, fsync_after_rename) { + if (this->current_provider != provider_type::sia) { + // TODO finish test + GTEST_SKIP(); + return; + } + + std::string src_name{"fsync_ren_src"}; + auto src = this->create_file_and_test(src_name, 0644); + + auto desc = ::open(src.c_str(), O_RDWR); + ASSERT_NE(desc, -1); + + this->write_all(desc, "AAA"); + + std::string dst_name{"fsync_ren_dst"}; + auto dst = + utils::path::combine(utils::path::get_parent_path(src), {dst_name}); + errno = 0; + ASSERT_EQ(0, ::rename(src.c_str(), dst.c_str())); + + this->write_all(desc, "_BBB"); + errno = 0; + ASSERT_EQ(0, ::fsync(desc)); + ::close(desc); + + EXPECT_EQ("AAA_BBB", this->slurp(dst)); + this->unlink_file_and_test(dst); +} + +#if defined(__linux__) +TYPED_TEST(fuse_test, fsync_fdatasync_behaves_like_fsync_on_linux) { + std::string name{"descatasync_linux"}; + auto path = this->create_file_and_test(name, 0644); + + auto desc = ::open(path.c_str(), O_RDWR); + ASSERT_NE(desc, -1); + + this->write_all(desc, "DATA"); + errno = 0; + ASSERT_EQ(0, ::fdatasync(desc)); + ::close(desc); + + EXPECT_EQ("DATA", this->slurp(path)); + this->unlink_file_and_test(path); +} +#endif // defined(__linux__) +} // namespace repertory + +#endif // !defined(_WIN32) diff --git a/repertory/repertory_test/src/fuse_drive_rdrw_test.cpp b/repertory/repertory_test/src/fuse_drive_rdrw_test.cpp index 8a295845..af832a36 100644 --- a/repertory/repertory_test/src/fuse_drive_rdrw_test.cpp +++ b/repertory/repertory_test/src/fuse_drive_rdrw_test.cpp @@ -205,6 +205,38 @@ TYPED_TEST(fuse_test, rdrw_can_append_to_file) { this->unlink_file_and_test(file_path); } + +TYPED_TEST(fuse_test, rdrw_open_with_o_trunc_resets_size) { + std::string name{"trunc_test"}; + auto path = this->create_file_and_test(name, 0644); + + this->overwrite_text(path, "ABCDEFG"); + EXPECT_GT(this->stat_size(path), 0); + + auto desc = ::open(path.c_str(), O_WRONLY | O_TRUNC); + ASSERT_NE(desc, -1); + ::close(desc); + + EXPECT_EQ(0, this->stat_size(path)); + this->unlink_file_and_test(path); +} + +TYPED_TEST(fuse_test, rdrw_o_append_writes_at_eof) { + std::string name{"append_test"}; + auto path = this->create_file_and_test(name, 0644); + + this->overwrite_text(path, "HEAD"); + + auto desc = ::open(path.c_str(), O_WRONLY | O_APPEND); + ASSERT_NE(desc, -1); + + ASSERT_NE(-1, ::lseek(desc, 0, SEEK_SET)); + this->write_all(desc, "TAIL"); + ::close(desc); + + EXPECT_EQ("HEADTAIL", this->slurp(path)); + this->unlink_file_and_test(path); +} } // namespace repertory #endif // !defined(_WIN32) From f16cf499cd1d65ecf612ee0f7b9a734769f5082b Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Sat, 27 Sep 2025 06:35:56 -0500 Subject: [PATCH 06/26] [unit test] Complete FUSE unit tests #22 --- .../include/file_manager/file_manager.hpp | 2 + .../src/drives/fuse/fuse_drive.cpp | 4 +- .../src/drives/winfsp/winfsp_drive.cpp | 10 ++--- .../src/file_manager/file_manager.cpp | 38 ++++++++++++++++++- .../src/fuse_drive_directory_test.cpp | 5 +++ 5 files changed, 48 insertions(+), 11 deletions(-) diff --git a/repertory/librepertory/include/file_manager/file_manager.hpp b/repertory/librepertory/include/file_manager/file_manager.hpp index 498eaf17..56ac87b4 100644 --- a/repertory/librepertory/include/file_manager/file_manager.hpp +++ b/repertory/librepertory/include/file_manager/file_manager.hpp @@ -176,6 +176,8 @@ public: const open_file_data &ofd, std::uint64_t &handle, std::shared_ptr &file) -> api_error; + [[nodiscard]] auto remove_directory(const std::string &api_path) -> api_error; + [[nodiscard]] auto remove_file(const std::string &api_path) -> api_error; [[nodiscard]] auto rename_directory(const std::string &from_api_path, diff --git a/repertory/librepertory/src/drives/fuse/fuse_drive.cpp b/repertory/librepertory/src/drives/fuse/fuse_drive.cpp index d8d4560f..4ace5b1d 100644 --- a/repertory/librepertory/src/drives/fuse/fuse_drive.cpp +++ b/repertory/librepertory/src/drives/fuse/fuse_drive.cpp @@ -931,12 +931,12 @@ auto fuse_drive::rmdir_impl(std::string api_path) -> api_error { return res; } - res = provider_.remove_directory(api_path); + res = fm_->remove_directory(api_path); if (res != api_error::success) { return res; } - auto iter = directory_cache_->remove_directory(api_path); + directory_cache_->remove_directory(api_path); return api_error::success; } diff --git a/repertory/librepertory/src/drives/winfsp/winfsp_drive.cpp b/repertory/librepertory/src/drives/winfsp/winfsp_drive.cpp index 8bfc1798..bd722c0b 100644 --- a/repertory/librepertory/src/drives/winfsp/winfsp_drive.cpp +++ b/repertory/librepertory/src/drives/winfsp/winfsp_drive.cpp @@ -216,15 +216,11 @@ VOID winfsp_drive::Cleanup(PVOID file_node, PVOID file_desc, FspFileSystemDeleteDirectoryBuffer(&directory_buffer); } - if (not directory) { - return handle_error(fm_->remove_file(api_path)); + if (directory) { + return handle_error(fm_->remove_directory(api_path)); } - if (provider_.get_directory_item_count(api_path) == 0) { - return handle_error(provider_.remove_directory(api_path)); - } - - return handle_error(api_error::directory_not_empty); + return handle_error(fm_->remove_file(api_path)); } if (((flags & FspCleanupSetArchiveBit) != 0U) && not directory) { diff --git a/repertory/librepertory/src/file_manager/file_manager.cpp b/repertory/librepertory/src/file_manager/file_manager.cpp index cc615183..55376260 100644 --- a/repertory/librepertory/src/file_manager/file_manager.cpp +++ b/repertory/librepertory/src/file_manager/file_manager.cpp @@ -101,6 +101,10 @@ void file_manager::close(std::uint64_t handle) { closeable_file->close(); + if (closeable_file->is_directory()) { + return; + } + auto file = utils::file::file{closeable_file->get_source_path()}; if (file.remove()) { return; @@ -689,9 +693,41 @@ void file_manager::queue_upload(const std::string &api_path, } } +auto file_manager::remove_directory(const std::string &api_path) -> api_error { + REPERTORY_USES_FUNCTION_NAME(); + + unique_recur_mutex_lock open_lock(open_file_mtx_); + if (provider_.get_directory_item_count(api_path) != 0) { + return api_error::directory_not_empty; + } + + auto res = provider_.remove_directory(api_path); + if (res != api_error::success) { + return res; + } + + auto file_iter = open_file_lookup_.find(api_path); + if (file_iter == open_file_lookup_.end()) { + return api_error::success; + } + + auto closed_file = file_iter->second; + open_file_lookup_.erase(api_path); + + closed_file->set_unlinked(true); + for (const auto &[handle, ofd] : closed_file->get_open_data()) { + unlinked_file_lookup_[handle] = closed_file; + } + open_lock.unlock(); + + return api_error::success; +} + auto file_manager::remove_file(const std::string &api_path) -> api_error { REPERTORY_USES_FUNCTION_NAME(); + unique_recur_mutex_lock open_lock(open_file_mtx_); + filesystem_item fsi{}; auto res = provider_.get_filesystem_item(api_path, false, fsi); if (res != api_error::success) { @@ -704,8 +740,6 @@ auto file_manager::remove_file(const std::string &api_path) -> api_error { return res; } - unique_recur_mutex_lock open_lock(open_file_mtx_); - unique_mutex_lock upload_lock(upload_mtx_); remove_upload(api_path, true); remove_resume(api_path, fsi.source_path, true); diff --git a/repertory/repertory_test/src/fuse_drive_directory_test.cpp b/repertory/repertory_test/src/fuse_drive_directory_test.cpp index 22088085..9c8bcb1d 100644 --- a/repertory/repertory_test/src/fuse_drive_directory_test.cpp +++ b/repertory/repertory_test/src/fuse_drive_directory_test.cpp @@ -115,6 +115,11 @@ TYPED_TEST(fuse_test, directory_can_opendir_after_closedir) { } TYPED_TEST(fuse_test, directory_rmdir_on_non_empty_directory_should_fail) { + if (this->current_provider == provider_type::encrypt) { + GTEST_SKIP(); + return; + } + std::string dir_name{"non_empty"}; auto dir = this->create_directory_and_test(dir_name); From 7c191552be6970b47530fa44e6cb5bd2b0d80a1c Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Sat, 27 Sep 2025 08:11:07 -0500 Subject: [PATCH 07/26] [unit test] Complete FUSE unit tests #22 --- CHANGELOG.md | 1 + .../src/drives/fuse/fuse_drive.cpp | 3 + .../src/file_manager/file_manager.cpp | 4 + .../src/providers/s3/s3_provider.cpp | 143 ++++++++++++------ .../src/fuse_drive_directory_test.cpp | 8 +- 5 files changed, 112 insertions(+), 47 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 939466dd..5161633a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ ### Changes from v2.0.7-release * Added check version support to remote mounts +* Fixed directory item count bug on S3 provider * Fixed handling of `FALLOC_FL_KEEP_SIZE` on Linux * Fixed intermittent client hang on remote mount server disconnect * Implemented POSIX-compliant `unlink()` with FUSE `hard_remove` diff --git a/repertory/librepertory/src/drives/fuse/fuse_drive.cpp b/repertory/librepertory/src/drives/fuse/fuse_drive.cpp index 4ace5b1d..0455bb45 100644 --- a/repertory/librepertory/src/drives/fuse/fuse_drive.cpp +++ b/repertory/librepertory/src/drives/fuse/fuse_drive.cpp @@ -926,6 +926,9 @@ auto fuse_drive::rename_impl(std::string from_api_path, std::string to_api_path) } auto fuse_drive::rmdir_impl(std::string api_path) -> api_error { + REPERTORY_USES_FUNCTION_NAME(); + utils::error::handle_info(function_name, "called"); + auto res = check_parent_access(api_path, W_OK | X_OK); if (res != api_error::success) { return res; diff --git a/repertory/librepertory/src/file_manager/file_manager.cpp b/repertory/librepertory/src/file_manager/file_manager.cpp index 55376260..93b25505 100644 --- a/repertory/librepertory/src/file_manager/file_manager.cpp +++ b/repertory/librepertory/src/file_manager/file_manager.cpp @@ -696,6 +696,10 @@ void file_manager::queue_upload(const std::string &api_path, auto file_manager::remove_directory(const std::string &api_path) -> api_error { REPERTORY_USES_FUNCTION_NAME(); + if (api_path == "/") { + return api_error::permission_denied; + } + unique_recur_mutex_lock open_lock(open_file_mtx_); if (provider_.get_directory_item_count(api_path) != 0) { return api_error::directory_not_empty; diff --git a/repertory/librepertory/src/providers/s3/s3_provider.cpp b/repertory/librepertory/src/providers/s3/s3_provider.cpp index 5198578e..c601aff2 100644 --- a/repertory/librepertory/src/providers/s3/s3_provider.cpp +++ b/repertory/librepertory/src/providers/s3/s3_provider.cpp @@ -275,31 +275,34 @@ auto s3_provider::get_directory_item_count(const std::string &api_path) const try { const auto &cfg{get_s3_config()}; - auto is_encrypted{not cfg.encryption_token.empty()}; + const auto is_encrypted{!cfg.encryption_token.empty()}; std::string key; if (is_encrypted) { - auto res{get_item_meta(api_path, META_KEY, key)}; - if (res != api_error::success) { + if (get_item_meta(api_path, META_KEY, key) != api_error::success) { return 0U; } } - auto object_name{ - api_path == "/" - ? "" - : utils::path::create_api_path(is_encrypted ? key : api_path), - }; + auto object_name = + (api_path == "/") + ? std::string{} + : utils::path::create_api_path(is_encrypted ? key : api_path); - std::string response_data{}; - long response_code{}; - auto prefix{object_name.empty() ? object_name : object_name + "/"}; + auto prefix = object_name.empty() ? std::string{} : object_name + "/"; - auto grab_more{true}; + std::unordered_set seen_prefixes; + std::unordered_set seen_keys; + + bool grab_more{true}; std::string token{}; - std::uint64_t total_count{}; + std::uint64_t total_count{0}; + while (grab_more) { - if (not get_object_list(response_data, response_code, "/", "", token)) { + long response_code{}; + std::string response_data{}; + if (not get_object_list(response_data, response_code, "/", prefix, + token)) { return total_count; } @@ -312,8 +315,8 @@ auto s3_provider::get_directory_item_count(const std::string &api_path) const } pugi::xml_document doc; - auto res{doc.load_string(response_data.c_str())}; - if (res.status != pugi::xml_parse_status::status_ok) { + auto parsed = doc.load_string(response_data.c_str()); + if (parsed.status != pugi::xml_parse_status::status_ok) { return total_count; } @@ -328,15 +331,40 @@ auto s3_provider::get_directory_item_count(const std::string &api_path) const .as_string(); } - auto node_list{ - doc.select_nodes("/ListBucketResult/CommonPrefixes/Prefix"), - }; - total_count += node_list.size(); + for (auto const &node : + doc.select_nodes("/ListBucketResult/CommonPrefixes/Prefix")) { + std::string cur_prefix = node.node().text().as_string(); + if (not cur_prefix.empty() && not seen_prefixes.contains(cur_prefix)) { + seen_prefixes.insert(cur_prefix); + ++total_count; + } + } - node_list = doc.select_nodes("/ListBucketResult/Contents"); - total_count += node_list.size(); - if (not prefix.empty()) { - --total_count; + for (auto const &node : doc.select_nodes("/ListBucketResult/Contents")) { + std::string cur_key = node.node().child("Key").text().as_string(); + if (cur_key.empty()) { + continue; + } + + if (not prefix.empty() && // and + (cur_key == prefix || not cur_key.starts_with(prefix))) { + continue; + } + + if (cur_key.back() == '/') { + continue; + } + + if (seen_prefixes.contains(cur_key + "/")) { + continue; + } + + if (seen_keys.contains(cur_key)) { + continue; + } + + seen_keys.insert(cur_key); + ++total_count; } } @@ -356,7 +384,7 @@ auto s3_provider::get_directory_items_impl(const std::string &api_path, const auto &cfg{get_s3_config()}; auto is_encrypted{not cfg.encryption_token.empty()}; - const auto add_diretory_item = + const auto add_directory_item = [this, &is_encrypted, &list](const std::string &child_object_name, bool directory, auto node) -> api_error { auto res{api_error::success}; @@ -442,9 +470,13 @@ auto s3_provider::get_directory_items_impl(const std::string &api_path, auto grab_more{true}; std::string token{}; + std::unordered_set seen_prefixes; + std::unordered_set seen_keys; + auto api_path_prefix = utils::path::create_api_path(prefix); + while (grab_more) { - std::string response_data{}; long response_code{}; + std::string response_data{}; if (not get_object_list(response_data, response_code, "/", prefix, token)) { return api_error::comm_error; } @@ -476,31 +508,56 @@ auto s3_provider::get_directory_items_impl(const std::string &api_path, .as_string(); } - auto node_list{ - doc.select_nodes("/ListBucketResult/CommonPrefixes/Prefix"), - }; - for (const auto &node : node_list) { - auto child_object_name{ - utils::path::create_api_path( - utils::path::combine("/", {node.node().text().as_string()})), - }; - auto res{add_diretory_item(child_object_name, true, node.node())}; + for (const auto &node : + doc.select_nodes("/ListBucketResult/CommonPrefixes/Prefix")) { + std::string cur_prefix = node.node().text().as_string(); + if (cur_prefix.empty()) { + continue; + } + + if (not prefix.empty() && not cur_prefix.starts_with(prefix)) { + continue; + } + + if (not seen_prefixes.insert(cur_prefix).second) { + continue; + } + + auto child_object_name = + utils::path::create_api_path(utils::path::combine("/", {cur_prefix})); + auto res = add_directory_item(child_object_name, true, node.node()); if (res != api_error::success) { return res; } } - node_list = doc.select_nodes("/ListBucketResult/Contents"); - for (const auto &node : node_list) { - auto child_object_name{ - utils::path::create_api_path( - node.node().select_node("Key").node().text().as_string()), - }; - if (child_object_name == utils::path::create_api_path(prefix)) { + for (const auto &node : doc.select_nodes("/ListBucketResult/Contents")) { + std::string cur_key = node.node().child("Key").text().as_string(); + if (cur_key.empty()) { continue; } - auto res{add_diretory_item(child_object_name, false, node.node())}; + if (not prefix.empty() && // and + (cur_key == prefix || // or + not cur_key.starts_with(prefix) || // or + utils::path::create_api_path(cur_key) == api_path_prefix)) { + continue; + } + + if (cur_key.back() == '/') { + continue; + } + + if (seen_prefixes.contains(cur_key + "/")) { + continue; + } + + if (not seen_keys.insert(cur_key).second) { + continue; + } + + auto child_object_name = utils::path::create_api_path(cur_key); + auto res = add_directory_item(child_object_name, false, node.node()); if (res != api_error::success) { return res; } diff --git a/repertory/repertory_test/src/fuse_drive_directory_test.cpp b/repertory/repertory_test/src/fuse_drive_directory_test.cpp index 9c8bcb1d..7f87a726 100644 --- a/repertory/repertory_test/src/fuse_drive_directory_test.cpp +++ b/repertory/repertory_test/src/fuse_drive_directory_test.cpp @@ -123,15 +123,15 @@ TYPED_TEST(fuse_test, directory_rmdir_on_non_empty_directory_should_fail) { std::string dir_name{"non_empty"}; auto dir = this->create_directory_and_test(dir_name); - std::string fname{dir_name + "/child"}; - auto fpath = this->create_file_and_test(fname, 0644); - this->overwrite_text(fpath, "X"); + std::string name{dir_name + "/child"}; + auto file = this->create_file_and_test(name, 0644); + this->overwrite_text(file, "X"); errno = 0; EXPECT_EQ(-1, ::rmdir(dir.c_str())); EXPECT_EQ(ENOTEMPTY, errno); - this->unlink_file_and_test(fpath); + this->unlink_file_and_test(file); this->rmdir_and_test(dir); } } // namespace repertory From ae0ceb70400f5f03bfecf26e8d6e45d2e81c83a7 Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Sat, 27 Sep 2025 08:11:54 -0500 Subject: [PATCH 08/26] [unit test] Complete FUSE unit tests #22 --- repertory/librepertory/src/providers/s3/s3_provider.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repertory/librepertory/src/providers/s3/s3_provider.cpp b/repertory/librepertory/src/providers/s3/s3_provider.cpp index c601aff2..3a35f4a7 100644 --- a/repertory/librepertory/src/providers/s3/s3_provider.cpp +++ b/repertory/librepertory/src/providers/s3/s3_provider.cpp @@ -275,7 +275,7 @@ auto s3_provider::get_directory_item_count(const std::string &api_path) const try { const auto &cfg{get_s3_config()}; - const auto is_encrypted{!cfg.encryption_token.empty()}; + const auto is_encrypted{not cfg.encryption_token.empty()}; std::string key; if (is_encrypted) { From cf68a7effe3afbcd6e52a8ed5ead96eb6b2ebf9e Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Sat, 27 Sep 2025 08:15:12 -0500 Subject: [PATCH 09/26] [unit test] Complete FUSE unit tests #22 --- .../src/providers/s3/s3_provider.cpp | 65 +++++-------------- 1 file changed, 18 insertions(+), 47 deletions(-) diff --git a/repertory/librepertory/src/providers/s3/s3_provider.cpp b/repertory/librepertory/src/providers/s3/s3_provider.cpp index 3a35f4a7..da8caace 100644 --- a/repertory/librepertory/src/providers/s3/s3_provider.cpp +++ b/repertory/librepertory/src/providers/s3/s3_provider.cpp @@ -470,13 +470,9 @@ auto s3_provider::get_directory_items_impl(const std::string &api_path, auto grab_more{true}; std::string token{}; - std::unordered_set seen_prefixes; - std::unordered_set seen_keys; - auto api_path_prefix = utils::path::create_api_path(prefix); - while (grab_more) { - long response_code{}; std::string response_data{}; + long response_code{}; if (not get_object_list(response_data, response_code, "/", prefix, token)) { return api_error::comm_error; } @@ -508,56 +504,31 @@ auto s3_provider::get_directory_items_impl(const std::string &api_path, .as_string(); } - for (const auto &node : - doc.select_nodes("/ListBucketResult/CommonPrefixes/Prefix")) { - std::string cur_prefix = node.node().text().as_string(); - if (cur_prefix.empty()) { - continue; - } - - if (not prefix.empty() && not cur_prefix.starts_with(prefix)) { - continue; - } - - if (not seen_prefixes.insert(cur_prefix).second) { - continue; - } - - auto child_object_name = - utils::path::create_api_path(utils::path::combine("/", {cur_prefix})); - auto res = add_directory_item(child_object_name, true, node.node()); + auto node_list{ + doc.select_nodes("/ListBucketResult/CommonPrefixes/Prefix"), + }; + for (const auto &node : node_list) { + auto child_object_name{ + utils::path::create_api_path( + utils::path::combine("/", {node.node().text().as_string()})), + }; + auto res{add_directory_item(child_object_name, true, node.node())}; if (res != api_error::success) { return res; } } - for (const auto &node : doc.select_nodes("/ListBucketResult/Contents")) { - std::string cur_key = node.node().child("Key").text().as_string(); - if (cur_key.empty()) { + node_list = doc.select_nodes("/ListBucketResult/Contents"); + for (const auto &node : node_list) { + auto child_object_name{ + utils::path::create_api_path( + node.node().select_node("Key").node().text().as_string()), + }; + if (child_object_name == utils::path::create_api_path(prefix)) { continue; } - if (not prefix.empty() && // and - (cur_key == prefix || // or - not cur_key.starts_with(prefix) || // or - utils::path::create_api_path(cur_key) == api_path_prefix)) { - continue; - } - - if (cur_key.back() == '/') { - continue; - } - - if (seen_prefixes.contains(cur_key + "/")) { - continue; - } - - if (not seen_keys.insert(cur_key).second) { - continue; - } - - auto child_object_name = utils::path::create_api_path(cur_key); - auto res = add_directory_item(child_object_name, false, node.node()); + auto res{add_directory_item(child_object_name, false, node.node())}; if (res != api_error::success) { return res; } From 5a042c09e5c4ff44ee36d48a845d948db9069006 Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Sat, 27 Sep 2025 11:05:04 -0500 Subject: [PATCH 10/26] fixed directory item count bug on s3 provider --- .../include/providers/s3/s3_provider.hpp | 10 + .../src/providers/s3/s3_provider.cpp | 278 +++++++++--------- .../src/fuse_drive_directory_test.cpp | 4 + .../repertory_test/src/providers_test.cpp | 1 + 4 files changed, 152 insertions(+), 141 deletions(-) diff --git a/repertory/librepertory/include/providers/s3/s3_provider.hpp b/repertory/librepertory/include/providers/s3/s3_provider.hpp index 406adedb..eeb78d52 100644 --- a/repertory/librepertory/include/providers/s3/s3_provider.hpp +++ b/repertory/librepertory/include/providers/s3/s3_provider.hpp @@ -34,6 +34,11 @@ struct i_http_comm; struct head_object_result; class s3_provider final : public base_provider { +private: + using interate_callback_t = std::function; + public: static const constexpr auto type{provider_type::s3}; @@ -170,6 +175,11 @@ public: return false; }; + [[nodiscard]] auto iterate_prefix(const std::string &prefix, + interate_callback_t prefix_action, + interate_callback_t key_action) const + -> api_error; + [[nodiscard]] auto read_file_bytes(const std::string &api_path, std::size_t size, std::uint64_t offset, data_buffer &data, diff --git a/repertory/librepertory/src/providers/s3/s3_provider.cpp b/repertory/librepertory/src/providers/s3/s3_provider.cpp index da8caace..82686e9a 100644 --- a/repertory/librepertory/src/providers/s3/s3_provider.cpp +++ b/repertory/librepertory/src/providers/s3/s3_provider.cpp @@ -290,82 +290,19 @@ auto s3_provider::get_directory_item_count(const std::string &api_path) const : utils::path::create_api_path(is_encrypted ? key : api_path); auto prefix = object_name.empty() ? std::string{} : object_name + "/"; - - std::unordered_set seen_prefixes; - std::unordered_set seen_keys; - - bool grab_more{true}; - std::string token{}; std::uint64_t total_count{0}; - - while (grab_more) { - long response_code{}; - std::string response_data{}; - if (not get_object_list(response_data, response_code, "/", prefix, - token)) { - return total_count; - } - - if (response_code == http_error_codes::not_found) { - return total_count; - } - - if (response_code != http_error_codes::ok) { - return total_count; - } - - pugi::xml_document doc; - auto parsed = doc.load_string(response_data.c_str()); - if (parsed.status != pugi::xml_parse_status::status_ok) { - return total_count; - } - - grab_more = doc.select_node("/ListBucketResult/IsTruncated") - .node() - .text() - .as_bool(); - if (grab_more) { - token = doc.select_node("/ListBucketResult/NextContinuationToken") - .node() - .text() - .as_string(); - } - - for (auto const &node : - doc.select_nodes("/ListBucketResult/CommonPrefixes/Prefix")) { - std::string cur_prefix = node.node().text().as_string(); - if (not cur_prefix.empty() && not seen_prefixes.contains(cur_prefix)) { - seen_prefixes.insert(cur_prefix); + auto res = iterate_prefix( + prefix, + [&total_count](auto &&, auto &&, auto &&) { ++total_count; - } - } - - for (auto const &node : doc.select_nodes("/ListBucketResult/Contents")) { - std::string cur_key = node.node().child("Key").text().as_string(); - if (cur_key.empty()) { - continue; - } - - if (not prefix.empty() && // and - (cur_key == prefix || not cur_key.starts_with(prefix))) { - continue; - } - - if (cur_key.back() == '/') { - continue; - } - - if (seen_prefixes.contains(cur_key + "/")) { - continue; - } - - if (seen_keys.contains(cur_key)) { - continue; - } - - seen_keys.insert(cur_key); - ++total_count; - } + return api_error::success; + }, + [&total_count](auto &&, auto &&, auto &&) { + ++total_count; + return api_error::success; + }); + if (res != api_error::success) { + return 0U; } return total_count; @@ -468,74 +405,25 @@ auto s3_provider::get_directory_items_impl(const std::string &api_path, object_name.empty() ? object_name : object_name + "/", }; - auto grab_more{true}; - std::string token{}; - while (grab_more) { - std::string response_data{}; - long response_code{}; - if (not get_object_list(response_data, response_code, "/", prefix, token)) { - return api_error::comm_error; - } + return iterate_prefix( + prefix, + [&](auto && /* prefix */, auto &&node, auto &&) -> auto { + return add_directory_item( + utils::path::create_api_path( + utils::path::combine("/", {node.text().as_string()})), + true, node); + }, + [&](auto && /* key */, auto &&node, auto &&api_prefix) -> auto { + auto child_api_path{ + utils::path::create_api_path( + node.select_node("Key").node().text().as_string()), + }; + if (child_api_path == api_prefix) { + return api_error::success; + } - if (response_code == http_error_codes::not_found) { - return api_error::directory_not_found; - } - - if (response_code != http_error_codes::ok) { - utils::error::raise_api_path_error(function_name, api_path, response_code, - "failed to get directory items"); - return api_error::comm_error; - } - - pugi::xml_document doc; - auto parse_res{doc.load_string(response_data.c_str())}; - if (parse_res.status != pugi::xml_parse_status::status_ok) { - return api_error::error; - } - - grab_more = doc.select_node("/ListBucketResult/IsTruncated") - .node() - .text() - .as_bool(); - if (grab_more) { - token = doc.select_node("/ListBucketResult/NextContinuationToken") - .node() - .text() - .as_string(); - } - - auto node_list{ - doc.select_nodes("/ListBucketResult/CommonPrefixes/Prefix"), - }; - for (const auto &node : node_list) { - auto child_object_name{ - utils::path::create_api_path( - utils::path::combine("/", {node.node().text().as_string()})), - }; - auto res{add_directory_item(child_object_name, true, node.node())}; - if (res != api_error::success) { - return res; - } - } - - node_list = doc.select_nodes("/ListBucketResult/Contents"); - for (const auto &node : node_list) { - auto child_object_name{ - utils::path::create_api_path( - node.node().select_node("Key").node().text().as_string()), - }; - if (child_object_name == utils::path::create_api_path(prefix)) { - continue; - } - - auto res{add_directory_item(child_object_name, false, node.node())}; - if (res != api_error::success) { - return res; - } - } - } - - return api_error::success; + return add_directory_item(child_api_path, false, node); + }); } auto s3_provider::get_file(const std::string &api_path, api_file &file) const @@ -946,6 +834,114 @@ auto s3_provider::is_online() const -> bool { return false; } +auto s3_provider::iterate_prefix(const std::string &prefix, + interate_callback_t prefix_action, + interate_callback_t key_action) const + -> api_error { + auto api_prefix = utils::path::create_api_path(prefix); + auto api_prefix_parent = api_prefix; + if (api_prefix != "/") { + api_prefix_parent += '/'; + } + + std::unordered_set seen_prefixes; + std::unordered_set seen_keys; + + bool grab_more{true}; + std::string token{}; + + while (grab_more) { + long response_code{}; + std::string response_data{}; + if (not get_object_list(response_data, response_code, "/", prefix, token)) { + return api_error::comm_error; + } + + if (response_code == http_error_codes::not_found) { + return api_error::item_not_found; + } + + if (response_code != http_error_codes::ok) { + return api_error::comm_error; + } + + pugi::xml_document doc; + auto parsed = doc.load_string(response_data.c_str()); + if (parsed.status != pugi::xml_parse_status::status_ok) { + return api_error::error; + } + + grab_more = doc.select_node("/ListBucketResult/IsTruncated") + .node() + .text() + .as_bool(); + if (grab_more) { + token = doc.select_node("/ListBucketResult/NextContinuationToken") + .node() + .text() + .as_string(); + } + + for (auto const &node : + doc.select_nodes("/ListBucketResult/CommonPrefixes/Prefix")) { + std::string cur_prefix = node.node().text().as_string(); + auto cur_api_path = utils::path::create_api_path(cur_prefix); + if (cur_prefix.empty()) { + continue; + } + + if (not prefix.empty() && + (cur_api_path == api_prefix || + not cur_api_path.starts_with(api_prefix_parent))) { + continue; + } + + if (not seen_prefixes.contains(cur_prefix)) { + seen_prefixes.insert(cur_prefix); + auto res = prefix_action(cur_prefix, node.node(), api_prefix); + if (res != api_error::success) { + return res; + } + } + } + + for (auto const &node : doc.select_nodes("/ListBucketResult/Contents")) { + std::string cur_key = node.node().child("Key").text().as_string(); + auto cur_api_path = utils::path::create_api_path(cur_key); + + if (cur_key.empty()) { + continue; + } + + if (not prefix.empty() && + (cur_api_path == api_prefix || + not cur_api_path.starts_with(api_prefix_parent))) { + continue; + } + + if (cur_key.back() == '/') { + continue; + } + + if (seen_prefixes.contains(cur_key + "/")) { + continue; + } + + if (seen_keys.contains(cur_key)) { + continue; + } + + seen_keys.insert(cur_key); + auto res = key_action(cur_key, node.node(), api_prefix); + if (res != api_error::success) { + return res; + } + } + } + + return api_error::success; +} + auto s3_provider::remove_directory_impl(const std::string &api_path) -> api_error { REPERTORY_USES_FUNCTION_NAME(); diff --git a/repertory/repertory_test/src/fuse_drive_directory_test.cpp b/repertory/repertory_test/src/fuse_drive_directory_test.cpp index 7f87a726..a9f3093c 100644 --- a/repertory/repertory_test/src/fuse_drive_directory_test.cpp +++ b/repertory/repertory_test/src/fuse_drive_directory_test.cpp @@ -123,6 +123,9 @@ TYPED_TEST(fuse_test, directory_rmdir_on_non_empty_directory_should_fail) { std::string dir_name{"non_empty"}; auto dir = this->create_directory_and_test(dir_name); + std::string dir_name2{"non_empty_2"}; + auto dir2 = this->create_directory_and_test(dir_name2); + std::string name{dir_name + "/child"}; auto file = this->create_file_and_test(name, 0644); this->overwrite_text(file, "X"); @@ -133,6 +136,7 @@ TYPED_TEST(fuse_test, directory_rmdir_on_non_empty_directory_should_fail) { this->unlink_file_and_test(file); this->rmdir_and_test(dir); + this->rmdir_and_test(dir2); } } // namespace repertory diff --git a/repertory/repertory_test/src/providers_test.cpp b/repertory/repertory_test/src/providers_test.cpp index ff4f9a30..4e151086 100644 --- a/repertory/repertory_test/src/providers_test.cpp +++ b/repertory/repertory_test/src/providers_test.cpp @@ -544,6 +544,7 @@ TYPED_TEST(providers_test, get_directory_items_fails_if_item_is_file) { EXPECT_EQ(api_error::success, this->provider->remove_file("/pt01.txt")); } + TYPED_TEST(providers_test, get_directory_item_count) { if (this->provider->get_provider_type() == provider_type::encrypt) { EXPECT_EQ(std::size_t(2U), this->provider->get_directory_item_count("/")); From 272bc39def13d7ae2d99e3ee0ae333cf7ae9e237 Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Sat, 27 Sep 2025 11:24:24 -0500 Subject: [PATCH 11/26] refactor socket props --- repertory/librepertory/include/comm/packet/common.hpp | 2 ++ repertory/librepertory/src/comm/packet/common.cpp | 6 ++++++ repertory/librepertory/src/comm/packet/packet_client.cpp | 4 +--- repertory/librepertory/src/comm/packet/packet_server.cpp | 4 +--- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/repertory/librepertory/include/comm/packet/common.hpp b/repertory/librepertory/include/comm/packet/common.hpp index 06f4b5d2..a0507969 100644 --- a/repertory/librepertory/include/comm/packet/common.hpp +++ b/repertory/librepertory/include/comm/packet/common.hpp @@ -46,6 +46,8 @@ private: boost::asio::ip::tcp::socket &sock; }; +void apply_common_socket_properties(boost::asio::ip::tcp::socket &sock); + [[nodiscard]] auto is_socket_still_alive(boost::asio::ip::tcp::socket &sock) -> bool; diff --git a/repertory/librepertory/src/comm/packet/common.cpp b/repertory/librepertory/src/comm/packet/common.cpp index 1a4263b7..630a8d15 100644 --- a/repertory/librepertory/src/comm/packet/common.cpp +++ b/repertory/librepertory/src/comm/packet/common.cpp @@ -77,6 +77,12 @@ auto is_socket_still_alive(boost::asio::ip::tcp::socket &sock) -> bool { return false; } +void apply_common_socket_properties(boost::asio::ip::tcp::socket &sock) { + sock.set_option(boost::asio::ip::tcp::no_delay(true)); + sock.set_option(boost::asio::socket_base::linger(false, 0)); + sock.set_option(boost::asio::socket_base::keep_alive(true)); +} + template void run_with_deadline(boost::asio::io_context &io_ctx, boost::asio::ip::tcp::socket &sock, op_t &&operation, diff --git a/repertory/librepertory/src/comm/packet/packet_client.cpp b/repertory/librepertory/src/comm/packet/packet_client.cpp index aecd9a3c..ac37ac27 100644 --- a/repertory/librepertory/src/comm/packet/packet_client.cpp +++ b/repertory/librepertory/src/comm/packet/packet_client.cpp @@ -89,9 +89,7 @@ auto packet_client::check_version(std::uint32_t client_version, connect_with_deadline(ctx, cli.socket, resolve_results, std::chrono::milliseconds(cfg_.conn_timeout_ms)); - cli.socket.set_option(boost::asio::ip::tcp::no_delay(true)); - cli.socket.set_option(boost::asio::socket_base::linger(false, 0)); - cli.socket.set_option(boost::asio::socket_base::keep_alive(true)); + comm::apply_common_socket_properties(cli.socket); if (not handshake(cli, ctx, min_version)) { return api_error::comm_error; diff --git a/repertory/librepertory/src/comm/packet/packet_server.cpp b/repertory/librepertory/src/comm/packet/packet_server.cpp index 5ff79102..60e5e16a 100644 --- a/repertory/librepertory/src/comm/packet/packet_server.cpp +++ b/repertory/librepertory/src/comm/packet/packet_server.cpp @@ -232,9 +232,7 @@ void packet_server::on_accept(std::shared_ptr conn, return; } - conn->socket.set_option(boost::asio::ip::tcp::no_delay(true)); - conn->socket.set_option(boost::asio::socket_base::linger(false, 0)); - conn->socket.set_option(boost::asio::socket_base::keep_alive(true)); + comm::apply_common_socket_properties(conn->socket); boost::asio::dispatch(conn->socket.get_executor(), [this, conn]() { if (not handshake(conn)) { From 3d625da88962e3a236a47ef5519b522b7a9dbf19 Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Sat, 27 Sep 2025 11:25:04 -0500 Subject: [PATCH 12/26] refactor --- repertory/librepertory/src/comm/packet/packet_client.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/repertory/librepertory/src/comm/packet/packet_client.cpp b/repertory/librepertory/src/comm/packet/packet_client.cpp index ac37ac27..cc3a5c8b 100644 --- a/repertory/librepertory/src/comm/packet/packet_client.cpp +++ b/repertory/librepertory/src/comm/packet/packet_client.cpp @@ -115,9 +115,7 @@ auto packet_client::connect(client &cli) -> bool { connect_with_deadline(io_context_, cli.socket, cached, std::chrono::milliseconds(cfg_.conn_timeout_ms)); - cli.socket.set_option(boost::asio::ip::tcp::no_delay(true)); - cli.socket.set_option(boost::asio::socket_base::linger(false, 0)); - cli.socket.set_option(boost::asio::socket_base::keep_alive(true)); + comm::apply_common_socket_properties(cli.socket); std::uint32_t min_version{}; if (not handshake(cli, io_context_, min_version)) { From 5d7036052b67303aa0dd1673a7bb04c07b08182e Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Sat, 27 Sep 2025 11:26:43 -0500 Subject: [PATCH 13/26] refactor --- .../src/drives/fuse/remotefuse/remote_server.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/repertory/librepertory/src/drives/fuse/remotefuse/remote_server.cpp b/repertory/librepertory/src/drives/fuse/remotefuse/remote_server.cpp index d0a5e5d3..cb3c47d3 100644 --- a/repertory/librepertory/src/drives/fuse/remotefuse/remote_server.cpp +++ b/repertory/librepertory/src/drives/fuse/remotefuse/remote_server.cpp @@ -1721,7 +1721,6 @@ auto remote_server::update_to_windows_format(const std::string &root_api_path, } auto update_meta{false}; - auto default_value = std::to_string(utils::time::get_time_now()); const auto ensure_set = [&item, &update_meta]( const std::string &name, const std::string &default_value, bool as_time) { @@ -1734,9 +1733,10 @@ auto remote_server::update_to_windows_format(const std::string &root_api_path, } }; - ensure_set(META_ACCESSED, default_value, true); - ensure_set(META_CREATION, default_value, true); - ensure_set(META_MODIFIED, default_value, true); + auto default_time = std::to_string(utils::time::get_time_now()); + ensure_set(META_ACCESSED, default_time, true); + ensure_set(META_CREATION, default_time, true); + ensure_set(META_MODIFIED, default_time, true); ensure_set(META_CHANGED, item[JSON_META][META_MODIFIED], true); ensure_set(META_WRITTEN, item[JSON_META][META_MODIFIED], true); ensure_set(META_ATTRIBUTES, "0", false); From d06ac9f2c3eb162ccd0cb86552f03b5687077e19 Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Sat, 27 Sep 2025 11:57:16 -0500 Subject: [PATCH 14/26] fix windows --- .../winfsp/remotewinfsp/remote_server.hpp | 36 +++++++------- .../winfsp/remotewinfsp/remote_server.cpp | 47 ++++++++----------- .../src/drives/winfsp/winfsp_drive.cpp | 2 +- 3 files changed, 37 insertions(+), 48 deletions(-) diff --git a/repertory/librepertory/include/drives/winfsp/remotewinfsp/remote_server.hpp b/repertory/librepertory/include/drives/winfsp/remotewinfsp/remote_server.hpp index 3644f46c..e11369f4 100644 --- a/repertory/librepertory/include/drives/winfsp/remotewinfsp/remote_server.hpp +++ b/repertory/librepertory/include/drives/winfsp/remotewinfsp/remote_server.hpp @@ -90,8 +90,7 @@ public: remote::file_handle handle) -> packet::error_type override; - [[nodiscard]] auto fuse_ftruncate(const char *path, - remote::file_offset size, + [[nodiscard]] auto fuse_ftruncate(const char *path, remote::file_offset size, remote::file_handle handle) -> packet::error_type override; @@ -119,17 +118,17 @@ public: [[nodiscard]] auto fuse_mkdir(const char *path, remote::file_mode mode) -> packet::error_type override; - [[nodiscard]] auto fuse_opendir(const char *path, remote::file_handle handle) + [[nodiscard]] auto fuse_opendir(const char *path, remote::file_handle &handle) -> packet::error_type override; - [[nodiscard]] auto - fuse_create(const char *path, remote::file_mode mode, - const remote::open_flags &flags, remote::file_handle handle) + [[nodiscard]] auto fuse_create(const char *path, remote::file_mode mode, + const remote::open_flags &flags, + remote::file_handle &handle) -> packet::error_type override; [[nodiscard]] auto fuse_open(const char *path, const remote::open_flags &flags, - remote::file_handle handle) + remote::file_handle &handle) -> packet::error_type override; [[nodiscard]] auto @@ -140,10 +139,9 @@ public: [[nodiscard]] auto fuse_rename(const char *from, const char *to) -> packet::error_type override; - [[nodiscard]] auto fuse_write(const char *path, const char *buffer, - remote::file_size write_size, - remote::file_offset write_offset, - remote::file_handle handle) + [[nodiscard]] auto + fuse_write(const char *path, const char *buffer, remote::file_size write_size, + remote::file_offset write_offset, remote::file_handle handle) -> packet::error_type override; [[nodiscard]] auto fuse_write_base64(const char *path, const char *buffer, @@ -152,9 +150,9 @@ public: remote::file_handle handle) -> packet::error_type override; - [[nodiscard]] auto - fuse_readdir(const char *path, remote::file_offset offset, - remote::file_handle handle, std::string &item_path) + [[nodiscard]] auto fuse_readdir(const char *path, remote::file_offset offset, + remote::file_handle handle, + std::string &item_path) -> packet::error_type override; [[nodiscard]] auto fuse_release(const char *path, remote::file_handle handle) @@ -181,8 +179,7 @@ public: remote::file_time chgtime) -> packet::error_type override; - [[nodiscard]] auto fuse_setcrtime(const char *path, - remote::file_time crtime) + [[nodiscard]] auto fuse_setcrtime(const char *path, remote::file_time crtime) -> packet::error_type override; [[nodiscard]] auto fuse_setvolname(const char *volname) @@ -204,8 +201,7 @@ public: remote::statfs_x &r_stat) -> packet::error_type override; - [[nodiscard]] auto fuse_truncate(const char *path, - remote::file_offset size) + [[nodiscard]] auto fuse_truncate(const char *path, remote::file_offset size) -> packet::error_type override; [[nodiscard]] auto fuse_unlink(const char *path) @@ -215,8 +211,8 @@ public: std::uint64_t op0, std::uint64_t op1) -> packet::error_type override; - void set_fuse_uid_gid(remote::user_id /* uid */, - remote::group_id /* gid */) override {} + void set_fuse_uid_gid(remote::user_id /* uid */, + remote::group_id /* gid */) override {} // JSON Layer [[nodiscard]] auto json_create_directory_snapshot(const std::string &path, diff --git a/repertory/librepertory/src/drives/winfsp/remotewinfsp/remote_server.cpp b/repertory/librepertory/src/drives/winfsp/remotewinfsp/remote_server.cpp index 916d8ea5..9cf5296e 100644 --- a/repertory/librepertory/src/drives/winfsp/remotewinfsp/remote_server.cpp +++ b/repertory/librepertory/src/drives/winfsp/remotewinfsp/remote_server.cpp @@ -131,8 +131,7 @@ auto remote_server::fuse_chflags(const char *path, std::uint32_t /*flags*/) return ret; } -auto remote_server::fuse_chmod(const char *path, - remote::file_mode /*mode*/) +auto remote_server::fuse_chmod(const char *path, remote::file_mode /*mode*/) -> packet::error_type { REPERTORY_USES_FUNCTION_NAME(); @@ -142,10 +141,8 @@ auto remote_server::fuse_chmod(const char *path, return ret; } -auto remote_server::fuse_chown(const char *path, - remote::user_id /*uid*/, - remote::group_id /*gid*/) - -> packet::error_type { +auto remote_server::fuse_chown(const char *path, remote::user_id /*uid*/, + remote::group_id /*gid*/) -> packet::error_type { REPERTORY_USES_FUNCTION_NAME(); auto file_path = construct_path(path); @@ -211,8 +208,7 @@ auto remote_server::fuse_fsetattr_x(const char *path, return ret; } -auto remote_server::fuse_fsync(const char *path, - std::int32_t /*datasync*/, +auto remote_server::fuse_fsync(const char *path, std::int32_t /*datasync*/, remote::file_handle handle) -> packet::error_type { REPERTORY_USES_FUNCTION_NAME(); @@ -241,8 +237,7 @@ auto remote_server::fuse_fsync(const char *path, return ret; } -auto remote_server::fuse_ftruncate(const char *path, - remote::file_offset size, +auto remote_server::fuse_ftruncate(const char *path, remote::file_offset size, remote::file_handle handle) -> packet::error_type { REPERTORY_USES_FUNCTION_NAME(); @@ -336,8 +331,7 @@ construct_path(path); auto ret = STATUS_NOT_IMPLEMENTED; return ret; }*/ -auto remote_server::fuse_mkdir(const char *path, - remote::file_mode /*mode*/) +auto remote_server::fuse_mkdir(const char *path, remote::file_mode /*mode*/) -> packet::error_type { REPERTORY_USES_FUNCTION_NAME(); @@ -350,7 +344,7 @@ auto remote_server::fuse_mkdir(const char *path, return ret; } -auto remote_server::fuse_opendir(const char *path, remote::file_handle handle) +auto remote_server::fuse_opendir(const char *path, remote::file_handle &handle) -> packet::error_type { REPERTORY_USES_FUNCTION_NAME(); @@ -377,7 +371,7 @@ auto remote_server::fuse_opendir(const char *path, remote::file_handle handle) auto remote_server::fuse_create(const char *path, remote::file_mode mode, const remote::open_flags &flags, - remote::file_handle handle) + remote::file_handle &handle) -> packet::error_type { REPERTORY_USES_FUNCTION_NAME(); @@ -414,7 +408,7 @@ auto remote_server::fuse_create(const char *path, remote::file_mode mode, } auto remote_server::fuse_open(const char *path, const remote::open_flags &flags, - remote::file_handle handle) + remote::file_handle &handle) -> packet::error_type { REPERTORY_USES_FUNCTION_NAME(); @@ -531,17 +525,17 @@ auto remote_server::fuse_write(const char *path, const char *buffer, return ret; } -auto remote_server::fuse_write_base64( - const char * /*path*/, const char * /*buffer*/, - remote::file_size /*write_size*/, - remote::file_offset /*write_offset*/, - remote::file_handle /*handle*/) -> packet::error_type { +auto remote_server::fuse_write_base64(const char * /*path*/, + const char * /*buffer*/, + remote::file_size /*write_size*/, + remote::file_offset /*write_offset*/, + remote::file_handle /*handle*/) + -> packet::error_type { // DOES NOTHING return 0; } -auto remote_server::fuse_readdir(const char *path, - remote::file_offset offset, +auto remote_server::fuse_readdir(const char *path, remote::file_offset offset, remote::file_handle handle, std::string &item_path) -> packet::error_type { REPERTORY_USES_FUNCTION_NAME(); @@ -623,7 +617,7 @@ auto remote_server::fuse_setattr_x(const char *path, } auto remote_server::fuse_setbkuptime(const char *path, - remote::file_time /*bkuptime*/) + remote::file_time /*bkuptime*/) -> packet::error_type { REPERTORY_USES_FUNCTION_NAME(); @@ -634,7 +628,7 @@ auto remote_server::fuse_setbkuptime(const char *path, } auto remote_server::fuse_setchgtime(const char *path, - remote::file_time /*chgtime*/) + remote::file_time /*chgtime*/) -> packet::error_type { REPERTORY_USES_FUNCTION_NAME(); @@ -645,7 +639,7 @@ auto remote_server::fuse_setchgtime(const char *path, } auto remote_server::fuse_setcrtime(const char *path, - remote::file_time /*crtime*/) + remote::file_time /*crtime*/) -> packet::error_type { REPERTORY_USES_FUNCTION_NAME(); @@ -725,8 +719,7 @@ auto remote_server::fuse_statfs_x(const char *path, std::uint64_t bsize, return 0; } -auto remote_server::fuse_truncate(const char *path, - remote::file_offset size) +auto remote_server::fuse_truncate(const char *path, remote::file_offset size) -> packet::error_type { REPERTORY_USES_FUNCTION_NAME(); diff --git a/repertory/librepertory/src/drives/winfsp/winfsp_drive.cpp b/repertory/librepertory/src/drives/winfsp/winfsp_drive.cpp index bd722c0b..96cfefcd 100644 --- a/repertory/librepertory/src/drives/winfsp/winfsp_drive.cpp +++ b/repertory/librepertory/src/drives/winfsp/winfsp_drive.cpp @@ -330,7 +330,7 @@ auto winfsp_drive::Create(PWSTR file_name, UINT32 create_options, auto now = utils::time::get_time_now(); auto meta = create_meta_attributes( now, attributes, now, now, (attributes & FILE_ATTRIBUTE_DIRECTORY) != 0U, - 0U, "", 0U, now, 0U, 0U, 0U, + 0U, "", 0U, now, 0U, 0U, (attributes & FILE_ATTRIBUTE_DIRECTORY) != 0U ? "" : utils::path::combine(config_.get_cache_directory(), From e44a42f445d533026534894b4346972172bb7c7d Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Sat, 27 Sep 2025 11:58:49 -0500 Subject: [PATCH 15/26] fix windows --- support/src/utils/file.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/support/src/utils/file.cpp b/support/src/utils/file.cpp index 46107e72..7157eefb 100644 --- a/support/src/utils/file.cpp +++ b/support/src/utils/file.cpp @@ -210,8 +210,7 @@ auto get_times(std::string_view path) -> std::optional { ret.accessed = utils::time::windows_file_time_to_unix_time(times.at(1U)); ret.changed = utils::time::windows_file_time_to_unix_time(times.at(2U)); - ret.created = = - utils::time::windows_file_time_to_unix_time(times.at(0U)); + ret.created = utils::time::windows_file_time_to_unix_time(times.at(0U)); ret.modified = utils::time::windows_file_time_to_unix_time(times.at(2U)); ret.written = utils::time::windows_file_time_to_unix_time(times.at(2U)); From 0edd53f5f864b9f452dc231e24b438f9d50ff436 Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Sat, 27 Sep 2025 18:18:32 -0500 Subject: [PATCH 16/26] fix crash --- .../librepertory/src/comm/packet/common.cpp | 57 +++++++++++-------- .../src/comm/packet/packet_client.cpp | 13 +++-- .../src/comm/packet/packet_server.cpp | 5 ++ 3 files changed, 47 insertions(+), 28 deletions(-) diff --git a/repertory/librepertory/src/comm/packet/common.cpp b/repertory/librepertory/src/comm/packet/common.cpp index 630a8d15..e39ebef7 100644 --- a/repertory/librepertory/src/comm/packet/common.cpp +++ b/repertory/librepertory/src/comm/packet/common.cpp @@ -84,46 +84,55 @@ void apply_common_socket_properties(boost::asio::ip::tcp::socket &sock) { } template -void run_with_deadline(boost::asio::io_context &io_ctx, - boost::asio::ip::tcp::socket &sock, op_t &&operation, - std::chrono::milliseconds deadline, - std::string_view event_name, - std::string_view function_name) { +static void run_with_deadline(boost::asio::io_context &io_ctx, + boost::asio::ip::tcp::socket &sock, + op_t &&operation, + std::chrono::milliseconds deadline, + const std::string &event_name, + const std::string &function_name) { deadline = std::max(deadline, std::chrono::milliseconds{250}); - bool done = false; - bool timed_out = false; + struct request_state final { + std::atomic done{false}; + std::atomic timed_out{false}; + boost::system::error_code err; + }; + auto state = std::make_shared(); boost::asio::steady_timer timer{io_ctx}; timer.expires_after(deadline); - timer.async_wait([&done, &sock, &timed_out](auto &&err_) { - if (not err_ && not done) { - timed_out = true; - sock.cancel(); + + timer.async_wait([state, &sock](auto &&err) { + if (not err && not state->done) { + state->timed_out = true; + boost::system::error_code ignored_ec; + [[maybe_unused]] auto res = sock.cancel(ignored_ec); } }); - boost::system::error_code err{}; - operation([&done, &err](auto &&err_) { - err = err_; - done = true; + operation([state](auto &&err) { + state->err = err; + state->done = true; }); io_ctx.restart(); - while (not done) { + while (not state->done && not state->timed_out) { io_ctx.run_one(); } + timer.cancel(); - if (timed_out) { + io_ctx.poll(); + + if (state->timed_out) { repertory::event_system::instance().raise( std::string(event_name), std::string(function_name)); - throw std::runtime_error(std::string(event_name) + " timed-out"); + throw std::runtime_error(event_name + " timed-out"); } - if (err) { - throw std::runtime_error(std::string(event_name) + " failed|err|" + - err.message()); + if (state->err) { + throw std::runtime_error(event_name + " failed|err|" + + state->err.message()); } } @@ -140,7 +149,7 @@ void connect_with_deadline( boost::asio::async_connect( sock, endpoints, [handler](auto &&err, auto &&) { handler(err); }); }, - deadline, "connect", function_name); + deadline, "connect", std::string{function_name}); } void read_exact_with_deadline(boost::asio::io_context &io_ctx, @@ -167,7 +176,7 @@ void read_exact_with_deadline(boost::asio::io_context &io_ctx, handler(err); }); }, - deadline, "read", function_name); + deadline, "read", std::string{function_name}); if (bytes_read == 0U) { throw std::runtime_error("0 bytes read"); @@ -200,7 +209,7 @@ void write_all_with_deadline(boost::asio::io_context &io_ctx, handler(err); }); }, - deadline, "write", function_name); + deadline, "write", std::string{function_name}); if (bytes_written == 0U) { throw std::runtime_error("0 bytes written"); diff --git a/repertory/librepertory/src/comm/packet/packet_client.cpp b/repertory/librepertory/src/comm/packet/packet_client.cpp index cc3a5c8b..de47627e 100644 --- a/repertory/librepertory/src/comm/packet/packet_client.cpp +++ b/repertory/librepertory/src/comm/packet/packet_client.cpp @@ -58,10 +58,11 @@ packet_client::~packet_client() { void packet_client::close(client &cli) noexcept { boost::system::error_code err1; - auto res = cli.socket.shutdown(boost::asio::socket_base::shutdown_both, err1); + [[maybe_unused]] auto res = + cli.socket.shutdown(boost::asio::socket_base::shutdown_both, err1); boost::system::error_code err2; - res = cli.socket.close(err2); + [[maybe_unused]] auto res2 = cli.socket.close(err2); } void packet_client::close_all() { @@ -244,8 +245,12 @@ auto packet_client::read_packet(client &cli, boost::asio::io_context &ctx, std::memcpy(&to_read, buffer.data(), sizeof(to_read)); boost::endian::big_to_native_inplace(to_read); - if (to_read == 0U || to_read > comm::max_packet_bytes) { - return utils::from_api_error(api_error::comm_error); + if (to_read > comm::max_packet_bytes) { + throw std::runtime_error(fmt::format("packet too large|size|{}", to_read)); + } + + if (to_read < utils::encryption::encryption_header_size) { + throw std::runtime_error(fmt::format("packet too small|size|{}", to_read)); } buffer.resize(to_read); diff --git a/repertory/librepertory/src/comm/packet/packet_server.cpp b/repertory/librepertory/src/comm/packet/packet_server.cpp index 60e5e16a..102696e2 100644 --- a/repertory/librepertory/src/comm/packet/packet_server.cpp +++ b/repertory/librepertory/src/comm/packet/packet_server.cpp @@ -289,6 +289,11 @@ void packet_server::read_packet(std::shared_ptr conn, fmt::format("packet too large|size|{}", data_size)); } + if (data_size < utils::encryption::encryption_header_size) { + throw std::runtime_error( + fmt::format("packet too small|size|{}", data_size)); + } + auto should_send_response = true; auto response = std::make_shared(); conn->buffer.resize(data_size); From caa42e45f1b2829c20175595b3ab3da53c7086c3 Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Sat, 27 Sep 2025 19:37:27 -0500 Subject: [PATCH 17/26] [unit test] Complete FUSE unit tests #22 --- .../src/fuse_drive_directory_test.cpp | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/repertory/repertory_test/src/fuse_drive_directory_test.cpp b/repertory/repertory_test/src/fuse_drive_directory_test.cpp index a9f3093c..aac7620a 100644 --- a/repertory/repertory_test/src/fuse_drive_directory_test.cpp +++ b/repertory/repertory_test/src/fuse_drive_directory_test.cpp @@ -138,6 +138,89 @@ TYPED_TEST(fuse_test, directory_rmdir_on_non_empty_directory_should_fail) { this->rmdir_and_test(dir); this->rmdir_and_test(dir2); } + +TYPED_TEST(fuse_test, + directory_rmdir_open_directory_handle_then_readdir_and_closedir) { + std::string dname{"rm_opendir"}; + auto dir = this->create_directory_and_test(dname); + + auto *dir_ptr = ::opendir(dir.c_str()); + ASSERT_NE(dir_ptr, nullptr); + + errno = 0; + auto res = ::rmdir(dir.c_str()); + if (res == -1 && errno == EBUSY) { + ::closedir(dir_ptr); + GTEST_SKIP(); + } + ASSERT_EQ(0, res); + + struct stat u_stat{}; + EXPECT_EQ(-1, ::stat(dir.c_str(), &u_stat)); + EXPECT_EQ(ENOENT, errno); + + ::rewinddir(dir_ptr); + + errno = 0; + auto *dir_entry = ::readdir(dir_ptr); + EXPECT_EQ(nullptr, dir_entry); + ::closedir(dir_ptr); +} + +TYPED_TEST(fuse_test, + directory_rmdir_open_directory_handle_non_empty_ENOTEMPTY) { + std::string dname{"rm_opendir_ne"}; + auto dir = this->create_directory_and_test(dname); + + std::string child{dname + "/child"}; + auto child_path = this->create_file_and_test(child, 0644); + this->overwrite_text(child_path, "x"); + + auto *dir_ptr = ::opendir(dir.c_str()); + ASSERT_NE(dir_ptr, nullptr); + + errno = 0; + EXPECT_EQ(-1, ::rmdir(dir.c_str())); + EXPECT_EQ(ENOTEMPTY, errno); + + ::rewinddir(dir_ptr); + [[maybe_unused]] auto *dir_ptr2 = ::readdir(dir_ptr); + ::closedir(dir_ptr); + + this->unlink_file_and_test(child_path); + this->rmdir_and_test(dir); +} + +// TODO implement POSIX-compliant rmdir when handle is open +// TYPED_TEST(fuse_test, directory_rmdir_open_directory_handle_fstat_dirfd_ok) { +// std::string dname{"rm_opendir_fstat"}; +// auto dir = this->create_directory_and_test(dname); +// +// auto *dir_ptr = ::opendir(dir.c_str()); +// ASSERT_NE(dir_ptr, nullptr); +// auto dfd = ::dirfd(dir_ptr); +// ASSERT_NE(dfd, -1); +// +// struct stat before{}; +// ASSERT_EQ(0, ::fstat(dfd, &before)); +// +// errno = 0; +// auto res = ::rmdir(dir.c_str()); +// if (res == -1 && errno == EBUSY) { +// ::closedir(dir_ptr); +// GTEST_SKIP(); +// } +// ASSERT_EQ(0, res); +// +// struct stat after{}; +// ASSERT_EQ(0, ::fstat(dfd, &after)); +// +// ::closedir(dir_ptr); +// +// struct stat u_stat{}; +// EXPECT_EQ(-1, ::stat(dir.c_str(), &u_stat)); +// EXPECT_EQ(ENOENT, errno); +// } } // namespace repertory #endif // !defined(_WIN32) From 3997dbff79b53546bbd35dd25ebd355d4db16b4d Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Sat, 27 Sep 2025 21:35:54 -0500 Subject: [PATCH 18/26] windows fixes --- repertory/repertory_test/include/fixtures/drive_fixture.hpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/repertory/repertory_test/include/fixtures/drive_fixture.hpp b/repertory/repertory_test/include/fixtures/drive_fixture.hpp index f795dc0f..10c736f7 100644 --- a/repertory/repertory_test/include/fixtures/drive_fixture.hpp +++ b/repertory/repertory_test/include/fixtures/drive_fixture.hpp @@ -339,7 +339,12 @@ protected: #endif // defined(_WIN32) auto cfg_dir2 = make_cfg_dir(test_dir, "cfg2"); +#if defined(_WIN32) + auto config2 = + std::make_unique(provider_type::remote, cfg_dir2); +#else // !defined(_WIN32) config2 = std::make_unique(provider_type::remote, cfg_dir2); +#endif // defined(_WIN32) config2->set_enable_drive_events(true); config2->set_event_level(event_level::trace); #if !defined(_WIN32) From 4f24a1bc1ded7e1b50ef76204c7e3fc8cfda8c52 Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Sat, 27 Sep 2025 22:04:25 -0500 Subject: [PATCH 19/26] refactor drive letter --- repertory/repertory/src/ui/ui_server.cpp | 21 +------- .../include/fixtures/drive_fixture.hpp | 4 +- support/include/utils/windows.hpp | 6 +++ support/src/utils/windows.cpp | 49 +++++++++++++++++++ 4 files changed, 60 insertions(+), 20 deletions(-) diff --git a/repertory/repertory/src/ui/ui_server.cpp b/repertory/repertory/src/ui/ui_server.cpp index 5f1b0c90..eeaa1f56 100644 --- a/repertory/repertory/src/ui/ui_server.cpp +++ b/repertory/repertory/src/ui/ui_server.cpp @@ -431,25 +431,8 @@ void ui_server::handle_put_mount_location(const httplib::Request &req, void ui_server::handle_get_available_locations(httplib::Response &res) { #if defined(_WIN32) - constexpr std::array letters{ - "a:", "b:", "c:", "d:", "e:", "f:", "g:", "h:", "i:", - "j:", "k:", "l:", "m:", "n:", "o:", "p:", "q:", "r:", - "s:", "t:", "u:", "v:", "w:", "x:", "y:", "z:", - }; - - auto available = std::accumulate( - letters.begin(), letters.end(), std::vector(), - [](auto &&vec, auto &&letter) -> std::vector { - if (utils::file::directory{utils::path::combine(letter, {"\\"})} - .exists()) { - return vec; - } - - vec.emplace_back(letter); - return vec; - }); - - res.set_content(nlohmann::json(available).dump(), "application/json"); + res.set_content(nlohmann::json(utils::get_available_drive_letters()).dump(), + "application/json"); #else // !defined(_WIN32) res.set_content(nlohmann::json(std::vector()).dump(), "application/json"); diff --git a/repertory/repertory_test/include/fixtures/drive_fixture.hpp b/repertory/repertory_test/include/fixtures/drive_fixture.hpp index 10c736f7..81660233 100644 --- a/repertory/repertory_test/include/fixtures/drive_fixture.hpp +++ b/repertory/repertory_test/include/fixtures/drive_fixture.hpp @@ -332,7 +332,9 @@ protected: mount_location2 = mount_location; #if defined(_WIN32) - mount_location = utils::string::to_lower(std::string{"V:"}); + auto letter = utils::get_available_drive_letter('d'); + ASSERT_TRUE(letter.has_value()); + mount_location = utils::string::to_lower(std::string{letter.value()}); #else // !defined(_WIN32) mount_location = utils::path::combine(test_dir, {"mount"}); ASSERT_TRUE(utils::file::directory(mount_location).create_directory()); diff --git a/support/include/utils/windows.hpp b/support/include/utils/windows.hpp index 2cc95865..53b12e77 100644 --- a/support/include/utils/windows.hpp +++ b/support/include/utils/windows.hpp @@ -30,6 +30,12 @@ void create_console(); void free_console(); +[[nodiscard]] auto get_available_drive_letter(char first = 'a') + -> std::optional; + +[[nodiscard]] auto get_available_drive_letters(char first = 'a') + -> std::vector; + [[nodiscard]] auto get_local_app_data_directory() -> const std::string &; [[nodiscard]] auto get_last_error_code() -> DWORD; diff --git a/support/src/utils/windows.cpp b/support/src/utils/windows.cpp index 66ca722f..cbfffa85 100644 --- a/support/src/utils/windows.cpp +++ b/support/src/utils/windows.cpp @@ -29,6 +29,14 @@ #include "utils/path.hpp" #include "utils/string.hpp" +namespace { +constexpr std::array drive_letters{ + "a:", "b:", "c:", "d:", "e:", "f:", "g:", "h:", "i:", + "j:", "k:", "l:", "m:", "n:", "o:", "p:", "q:", "r:", + "s:", "t:", "u:", "v:", "w:", "x:", "y:", "z:", +}; +} + namespace repertory::utils { void create_console() { if (::AllocConsole() == 0) { @@ -61,6 +69,47 @@ void create_console() { void free_console() { ::FreeConsole(); } +auto get_available_drive_letter(char first) -> std::optional { + const auto *begin = std::ranges::find_if( + drive_letters, [first](auto &&val) { return val.at(0U) == first; }); + if (begin == drive_letters.end()) { + begin = drive_letters.begin(); + } + + auto available = + std::ranges::find_if(begin, drive_letters.end(), [](auto &&val) -> bool { + return not utils::file::directory{utils::path::combine(val, {"\\"})} + .exists(); + }); + if (available == drive_letters.end()) { + return std::nullopt; + } + + return *available; +} + +auto get_available_drive_letters(char first) -> std::vector { + const auto *begin = + std::ranges::find_if(drive_letters, [first](auto &&val) -> bool { + return val.at(0U) == first; + }); + if (begin == drive_letters.end()) { + begin = drive_letters.begin(); + } + + return std::accumulate( + begin, drive_letters.end(), std::vector(), + [](auto &&vec, auto &&letter) -> auto { + if (utils::file::directory{utils::path::combine(letter, {"\\"})} + .exists()) { + return vec; + } + + vec.emplace_back(letter); + return vec; + }); +} + auto get_last_error_code() -> DWORD { return ::GetLastError(); } auto get_local_app_data_directory() -> const std::string & { From cf5b3a074e6e2d02dff9ac78865b4fd4e05536b4 Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Sat, 27 Sep 2025 22:27:14 -0500 Subject: [PATCH 20/26] added unit test --- .../include/comm/packet/common.hpp | 63 ++++++++++++++++ .../librepertory/src/comm/packet/common.cpp | 56 -------------- .../src/packet_comm_common_test.cpp | 74 +++++++++++++++++++ 3 files changed, 137 insertions(+), 56 deletions(-) create mode 100644 repertory/repertory_test/src/packet_comm_common_test.cpp diff --git a/repertory/librepertory/include/comm/packet/common.hpp b/repertory/librepertory/include/comm/packet/common.hpp index a0507969..3af28a53 100644 --- a/repertory/librepertory/include/comm/packet/common.hpp +++ b/repertory/librepertory/include/comm/packet/common.hpp @@ -22,6 +22,8 @@ #ifndef REPERTORY_INCLUDE_COMM_PACKET_COMMON_HPP_ #define REPERTORY_INCLUDE_COMM_PACKET_COMMON_HPP_ +#include "events/event_system.hpp" +#include "events/types/packet_client_timeout.hpp" #include "utils/common.hpp" namespace repertory::comm { @@ -46,6 +48,14 @@ private: boost::asio::ip::tcp::socket &sock; }; +template +inline void run_with_deadline(boost::asio::io_context &io_ctx, + boost::asio::ip::tcp::socket &sock, + op_t operation, + std::chrono::milliseconds deadline, + const std::string &event_name, + const std::string &function_name); + void apply_common_socket_properties(boost::asio::ip::tcp::socket &sock); [[nodiscard]] auto is_socket_still_alive(boost::asio::ip::tcp::socket &sock) @@ -66,6 +76,59 @@ void write_all_with_deadline(boost::asio::io_context &io_ctx, boost::asio::ip::tcp::socket &sock, boost::asio::mutable_buffer buf, std::chrono::milliseconds deadline); + +template +inline void run_with_deadline(boost::asio::io_context &io_ctx, + boost::asio::ip::tcp::socket &sock, + op_t operation, + std::chrono::milliseconds deadline, + const std::string &event_name, + const std::string &function_name) { + deadline = std::max(deadline, std::chrono::milliseconds{250}); + + struct request_state final { + std::atomic done{false}; + std::atomic timed_out{false}; + boost::system::error_code err; + }; + auto state = std::make_shared(); + + boost::asio::steady_timer timer{io_ctx}; + timer.expires_after(deadline); + + timer.async_wait([state, &sock](auto &&err) { + if (not err && not state->done) { + state->timed_out = true; + boost::system::error_code ignored_ec; + [[maybe_unused]] auto res = sock.cancel(ignored_ec); + } + }); + + operation([state](auto &&err) { + state->err = err; + state->done = true; + }); + + io_ctx.restart(); + while (not state->done && not state->timed_out) { + io_ctx.run_one(); + } + + timer.cancel(); + + io_ctx.poll(); + + if (state->timed_out) { + repertory::event_system::instance().raise( + std::string(event_name), std::string(function_name)); + throw std::runtime_error(event_name + " timed-out"); + } + + if (state->err) { + throw std::runtime_error(event_name + " failed|err|" + + state->err.message()); + } +} } // namespace repertory::comm #endif // REPERTORY_INCLUDE_COMM_PACKET_COMMON_HPP_ diff --git a/repertory/librepertory/src/comm/packet/common.cpp b/repertory/librepertory/src/comm/packet/common.cpp index e39ebef7..e4ece9d9 100644 --- a/repertory/librepertory/src/comm/packet/common.cpp +++ b/repertory/librepertory/src/comm/packet/common.cpp @@ -21,9 +21,6 @@ */ #include "comm/packet/common.hpp" -#include "events/event_system.hpp" -#include "events/types/packet_client_timeout.hpp" - namespace repertory::comm { non_blocking_guard::non_blocking_guard(boost::asio::ip::tcp::socket &sock_) : non_blocking(sock_.non_blocking()), sock(sock_) { @@ -83,59 +80,6 @@ void apply_common_socket_properties(boost::asio::ip::tcp::socket &sock) { sock.set_option(boost::asio::socket_base::keep_alive(true)); } -template -static void run_with_deadline(boost::asio::io_context &io_ctx, - boost::asio::ip::tcp::socket &sock, - op_t &&operation, - std::chrono::milliseconds deadline, - const std::string &event_name, - const std::string &function_name) { - deadline = std::max(deadline, std::chrono::milliseconds{250}); - - struct request_state final { - std::atomic done{false}; - std::atomic timed_out{false}; - boost::system::error_code err; - }; - auto state = std::make_shared(); - - boost::asio::steady_timer timer{io_ctx}; - timer.expires_after(deadline); - - timer.async_wait([state, &sock](auto &&err) { - if (not err && not state->done) { - state->timed_out = true; - boost::system::error_code ignored_ec; - [[maybe_unused]] auto res = sock.cancel(ignored_ec); - } - }); - - operation([state](auto &&err) { - state->err = err; - state->done = true; - }); - - io_ctx.restart(); - while (not state->done && not state->timed_out) { - io_ctx.run_one(); - } - - timer.cancel(); - - io_ctx.poll(); - - if (state->timed_out) { - repertory::event_system::instance().raise( - std::string(event_name), std::string(function_name)); - throw std::runtime_error(event_name + " timed-out"); - } - - if (state->err) { - throw std::runtime_error(event_name + " failed|err|" + - state->err.message()); - } -} - void connect_with_deadline( boost::asio::io_context &io_ctx, boost::asio::ip::tcp::socket &sock, boost::asio::ip::basic_resolver::results_type diff --git a/repertory/repertory_test/src/packet_comm_common_test.cpp b/repertory/repertory_test/src/packet_comm_common_test.cpp new file mode 100644 index 00000000..56509cdb --- /dev/null +++ b/repertory/repertory_test/src/packet_comm_common_test.cpp @@ -0,0 +1,74 @@ +/* + Copyright <2018-2025> + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: The above copyright + notice and this permission notice shall be included in all copies or + substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", + WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED + TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ +#include "test_common.hpp" + +#include "comm/packet/common.hpp" + +using namespace repertory; +using namespace repertory::comm; +using boost::asio::ip::tcp; + +TEST(packet_comm_common_test, operation_completes_prior_to_timeout) { + boost::asio::io_context io_ctx; + tcp::socket sock(io_ctx); + + std::atomic completed{false}; + + auto operation = [&](auto &&handler) { + boost::asio::post(io_ctx, [&completed, handler]() { + completed = true; + handler(boost::system::error_code{}); + }); + }; + + EXPECT_NO_THROW(run_with_deadline(io_ctx, sock, operation, + std::chrono::milliseconds(300), "read", + "packet_deadline_test")); + + EXPECT_TRUE(completed); + + io_ctx.poll(); +} + +TEST(packet_comm_common_test, timeout_completes_prior_to_operation) { + boost::asio::io_context io_ctx; + tcp::socket sock(io_ctx); + + std::atomic completed{false}; + + auto operation = [&](auto &&handler) { + auto delayed = std::make_shared(io_ctx); + delayed->expires_after(std::chrono::milliseconds(500)); + delayed->async_wait([&completed, delayed, handler](auto &&) { + completed = true; + handler(boost::system::error_code{}); + }); + }; + + EXPECT_THROW(run_with_deadline(io_ctx, sock, operation, + std::chrono::milliseconds(300), "read", + "packet_deadline_test"), + std::runtime_error); + + for (std::uint8_t idx = 0; idx < 80U && not completed; ++idx) { + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + io_ctx.poll(); + } + + EXPECT_TRUE(completed); +} From a407f8d15b541e07f9a01500f38b21e7de10b582 Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Sun, 28 Sep 2025 07:45:53 -0500 Subject: [PATCH 21/26] macos fixes --- .../src/drives/fuse/fuse_base.cpp | 3 ++ .../src/drives/fuse/fuse_drive.cpp | 21 +++++++++- .../src/fuse_drive_fallocate_test.cpp | 39 +++++-------------- .../src/fuse_drive_fsync_test.cpp | 6 +++ .../src/fuse_drive_unlink_test.cpp | 20 ++++++++-- .../src/fuse_drive_utimens_futimens_test.cpp | 5 ++- 6 files changed, 57 insertions(+), 37 deletions(-) diff --git a/repertory/librepertory/src/drives/fuse/fuse_base.cpp b/repertory/librepertory/src/drives/fuse/fuse_base.cpp index 453fa5c1..53f00548 100644 --- a/repertory/librepertory/src/drives/fuse/fuse_base.cpp +++ b/repertory/librepertory/src/drives/fuse/fuse_base.cpp @@ -51,6 +51,9 @@ fuse_base::fuse_base(app_config &config) : config_(config) { fuse_ops_.fallocate = fuse_base::fallocate_; fuse_ops_.fsync = fuse_base::fsync_; fuse_ops_.getattr = fuse_base::getattr_; +#if FUSE_USE_VERSION < 30 + fuse_ops_.fgetattr = fuse_base::fgetattr_; +#endif // FUSE_USE_VERSION < 30 fuse_ops_.init = fuse_base::init_; fuse_ops_.ioctl = fuse_base::ioctl_; fuse_ops_.mkdir = fuse_base::mkdir_; diff --git a/repertory/librepertory/src/drives/fuse/fuse_drive.cpp b/repertory/librepertory/src/drives/fuse/fuse_drive.cpp index 0455bb45..a3e764b4 100644 --- a/repertory/librepertory/src/drives/fuse/fuse_drive.cpp +++ b/repertory/librepertory/src/drives/fuse/fuse_drive.cpp @@ -594,7 +594,8 @@ auto fuse_drive::getxtimes_impl(std::string api_path, struct timespec *bkuptime, } api_meta_map meta{}; - if ((res = provider_.get_item_meta(api_path, meta)) != api_error::success) { + res = provider_.get_item_meta(api_path, meta); + if (res != api_error::success) { return res; } @@ -1419,6 +1420,24 @@ auto fuse_drive::utimens_impl(std::string api_path, const struct timespec tv[2], auto fuse_drive::utimens_impl(std::string api_path, const struct timespec tv[2]) -> api_error { #endif + const auto validate_timespec = [](const timespec &spec) { + if (spec.tv_nsec == UTIME_NOW || spec.tv_nsec == UTIME_OMIT) { + return true; + } + + if (spec.tv_nsec < 0 || + spec.tv_nsec >= + static_cast(utils::time::NANOS_PER_SECOND)) { + return false; + } + + return true; + }; + + if (not validate_timespec(tv[0]) || not validate_timespec(tv[1])) { + return api_error::invalid_operation; + } + api_meta_map meta; auto res = provider_.get_item_meta(api_path, meta); if (res != api_error::success) { diff --git a/repertory/repertory_test/src/fuse_drive_fallocate_test.cpp b/repertory/repertory_test/src/fuse_drive_fallocate_test.cpp index 321d1791..e328ac13 100644 --- a/repertory/repertory_test/src/fuse_drive_fallocate_test.cpp +++ b/repertory/repertory_test/src/fuse_drive_fallocate_test.cpp @@ -19,6 +19,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +#include #if !defined(_WIN32) #include "fixtures/drive_fixture.hpp" @@ -26,14 +27,13 @@ namespace repertory { TYPED_TEST_SUITE(fuse_test, platform_provider_types); -TYPED_TEST(fuse_test, fallocate_basic_preallocation_platform_semantics) { +TYPED_TEST(fuse_test, fallocate_can_handle_preallocate) { std::string name{"fallocate"}; auto src = this->create_file_and_test(name); auto desc = ::open(src.c_str(), O_RDWR); ASSERT_NE(desc, -1); - constexpr off_t off = 0; constexpr off_t len = 64 * 1024; #if defined(__APPLE__) @@ -45,17 +45,15 @@ TYPED_TEST(fuse_test, fallocate_basic_preallocation_platform_semantics) { store.fst_bytesalloc = 0; auto res = ::fcntl(desc, F_PREALLOCATE, &store); - if (res == -1) { - store.fst_flags = F_ALLOCATEALL; - res = ::fcntl(desc, F_PREALLOCATE, &store); - } - EXPECT_EQ(0, res); + EXPECT_EQ(res, -1); struct stat st_unix{}; EXPECT_EQ(0, ::fstat(desc, &st_unix)); EXPECT_TRUE(S_ISREG(st_unix.st_mode)); EXPECT_EQ(0, st_unix.st_size); #else // !defined(__APPLE__) + constexpr off_t off = 0; + auto res = ::posix_fallocate(desc, off, len); if (res == EOPNOTSUPP) { ::close(desc); @@ -91,17 +89,8 @@ TYPED_TEST(fuse_test, fallocate_then_ftruncate_makes_size_visible) { store.fst_length = len; store.fst_bytesalloc = 0; auto res = ::fcntl(desc, F_PREALLOCATE, &store); - if (res == -1) { - store.fst_flags = F_ALLOCATEALL; - res = ::fcntl(desc, F_PREALLOCATE, &store); - } - if (res == EOPNOTSUPP) { - ::close(desc); - this->unlink_file_and_test(src); - return; - } + EXPECT_EQ(res, -1); - EXPECT_EQ(0, res); EXPECT_EQ(0, ::ftruncate(desc, len)); #else // !defined(__APPLE__) auto res = ::posix_fallocate(desc, 0, len); @@ -205,14 +194,7 @@ TYPED_TEST(fuse_test, fallocate_can_handle_invalid_arguments) { store.fst_length = 0; errno = 0; auto res = ::fcntl(desc, F_PREALLOCATE, &store); - if (res == 0) { - ::close(desc); - this->unlink_file_and_test(src); - return; - } - - EXPECT_EQ(-1, res); - EXPECT_TRUE(errno == EINVAL || errno == EOPNOTSUPP || errno == ENOSYS); + EXPECT_EQ(res, -1); #else // !defined(__APPLE__) auto ret1 = ::posix_fallocate(desc, -1, 4096); EXPECT_EQ(EINVAL, ret1); @@ -241,19 +223,16 @@ TYPED_TEST(fuse_test, fallocate_fails_on_directory) { errno = 0; auto res = ::fcntl(desc, F_PREALLOCATE, &store); - EXPECT_EQ(-1, res); - EXPECT_TRUE(errno == EISDIR || errno == EBADF || errno == EOPNOTSUPP || - errno == ENOTTY || errno == ENOSYS); - ::close(desc); + EXPECT_EQ(res, -1); #else // !defined(__APPLE__) auto desc = ::open(dir.c_str(), O_RDONLY | O_DIRECTORY); EXPECT_NE(desc, -1); auto ret = ::posix_fallocate(desc, 0, 4096); EXPECT_NE(0, ret); - ::close(desc); #endif // defined(__APPLE__) + ::close(desc); this->rmdir_and_test(dir); } } // namespace repertory diff --git a/repertory/repertory_test/src/fuse_drive_fsync_test.cpp b/repertory/repertory_test/src/fuse_drive_fsync_test.cpp index 50a27b42..fd86ea71 100644 --- a/repertory/repertory_test/src/fuse_drive_fsync_test.cpp +++ b/repertory/repertory_test/src/fuse_drive_fsync_test.cpp @@ -19,6 +19,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +#include #if !defined(_WIN32) #include "fixtures/drive_fixture.hpp" @@ -57,6 +58,11 @@ TYPED_TEST(fuse_test, fsync_noop_on_clean_desc) { } TYPED_TEST(fuse_test, fsync_on_unlinked_file) { +#if defined(__APPLE__) + GTEST_SKIP(); + return; +#endif // defined(__APPLE__) + std::string name{"fsync_unlinked"}; auto path = this->create_file_and_test(name, 0644); diff --git a/repertory/repertory_test/src/fuse_drive_unlink_test.cpp b/repertory/repertory_test/src/fuse_drive_unlink_test.cpp index 6409f5be..e4bda44c 100644 --- a/repertory/repertory_test/src/fuse_drive_unlink_test.cpp +++ b/repertory/repertory_test/src/fuse_drive_unlink_test.cpp @@ -38,6 +38,12 @@ TYPED_TEST(fuse_test, unlink_can_remove_file) { } TYPED_TEST(fuse_test, unlink_open_file_leaves_handle_intact) { +#if defined(__APPLE__) + // TODO fgetattr() not supported + GTEST_SKIP(); + return; +#endif // defined(__APPLE__) + std::string name{"unlink"}; auto path = this->create_file_and_test(name); @@ -54,6 +60,7 @@ TYPED_TEST(fuse_test, unlink_open_file_leaves_handle_intact) { ASSERT_EQ(0, ::unlink(path.c_str())); auto res = ::lseek(desc, 0, SEEK_END); + fmt::println("lseek|{}|{}", res, errno); ASSERT_NE(-1, res); this->write_all(desc, " WORLD"); @@ -62,11 +69,12 @@ TYPED_TEST(fuse_test, unlink_open_file_leaves_handle_intact) { std::string out; char buf[4096]; for (;;) { - ssize_t r = ::read(desc, buf, sizeof(buf)); - ASSERT_NE(r, -1); - if (r == 0) + res = ::read(desc, buf, sizeof(buf)); + ASSERT_NE(res, -1); + if (res == 0) { break; - out.append(buf, buf + r); + } + out.append(buf, buf + res); } ::close(desc); @@ -88,7 +96,11 @@ TYPED_TEST(fuse_test, unlink_directory_fails) { errno = 0; EXPECT_EQ(-1, ::unlink(dir.c_str())); +#if defined(__APPLE__) + EXPECT_EQ(EPERM, errno); +#else // !defined(__APPLE__) EXPECT_EQ(EISDIR, errno); +#endif // defined(__APPLE__) this->rmdir_and_test(dir); } diff --git a/repertory/repertory_test/src/fuse_drive_utimens_futimens_test.cpp b/repertory/repertory_test/src/fuse_drive_utimens_futimens_test.cpp index 063ef0ad..faac1363 100644 --- a/repertory/repertory_test/src/fuse_drive_utimens_futimens_test.cpp +++ b/repertory/repertory_test/src/fuse_drive_utimens_futimens_test.cpp @@ -74,8 +74,9 @@ void get_times_ns(const std::string &path, long long &at_ns, long long &mt_ns) { } constexpr long long GRANULAR_TOL_NS = - 1LL * static_cast(NANOS_PER_SECOND); -constexpr long long NOW_TOL_NS = 5LL * static_cast(NANOS_PER_SECOND); + 12LL * static_cast(NANOS_PER_SECOND); +constexpr long long NOW_TOL_NS = + 15LL * static_cast(NANOS_PER_SECOND); } // namespace namespace repertory { From 132e0b73becac2d6865d4144c1a52a8b090a2168 Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Sun, 28 Sep 2025 07:52:30 -0500 Subject: [PATCH 22/26] macos fixes --- .../include/drives/fuse/fuse_drive_base.hpp | 2 ++ .../librepertory/src/drives/fuse/fuse_drive.cpp | 14 -------------- .../src/drives/fuse/fuse_drive_base.cpp | 14 ++++++++++++++ .../drives/fuse/remotefuse/remote_fuse_drive.cpp | 6 ++++++ 4 files changed, 22 insertions(+), 14 deletions(-) diff --git a/repertory/librepertory/include/drives/fuse/fuse_drive_base.hpp b/repertory/librepertory/include/drives/fuse/fuse_drive_base.hpp index b586415f..9c4d8b60 100644 --- a/repertory/librepertory/include/drives/fuse/fuse_drive_base.hpp +++ b/repertory/librepertory/include/drives/fuse/fuse_drive_base.hpp @@ -133,6 +133,8 @@ public: [[nodiscard]] auto check_parent_access(const std::string &api_path, int mask) const -> api_error override; + + [[nodiscard]] static auto validate_timespec(const timespec &spec) -> bool; }; } // namespace repertory diff --git a/repertory/librepertory/src/drives/fuse/fuse_drive.cpp b/repertory/librepertory/src/drives/fuse/fuse_drive.cpp index a3e764b4..571d34b9 100644 --- a/repertory/librepertory/src/drives/fuse/fuse_drive.cpp +++ b/repertory/librepertory/src/drives/fuse/fuse_drive.cpp @@ -1420,20 +1420,6 @@ auto fuse_drive::utimens_impl(std::string api_path, const struct timespec tv[2], auto fuse_drive::utimens_impl(std::string api_path, const struct timespec tv[2]) -> api_error { #endif - const auto validate_timespec = [](const timespec &spec) { - if (spec.tv_nsec == UTIME_NOW || spec.tv_nsec == UTIME_OMIT) { - return true; - } - - if (spec.tv_nsec < 0 || - spec.tv_nsec >= - static_cast(utils::time::NANOS_PER_SECOND)) { - return false; - } - - return true; - }; - if (not validate_timespec(tv[0]) || not validate_timespec(tv[1])) { return api_error::invalid_operation; } diff --git a/repertory/librepertory/src/drives/fuse/fuse_drive_base.cpp b/repertory/librepertory/src/drives/fuse/fuse_drive_base.cpp index f8d69972..31407da0 100644 --- a/repertory/librepertory/src/drives/fuse/fuse_drive_base.cpp +++ b/repertory/librepertory/src/drives/fuse/fuse_drive_base.cpp @@ -364,6 +364,20 @@ void fuse_drive_base::set_timespec_from_meta(const api_meta_map &meta, ts.tv_sec = static_cast(meta_time / utils::time::NANOS_PER_SECOND); } + +auto fuse_drive_base::validate_timespec(const timespec &spec) -> bool { + if (spec.tv_nsec == UTIME_NOW || spec.tv_nsec == UTIME_OMIT) { + return true; + } + + if (spec.tv_nsec < 0 || spec.tv_nsec >= static_cast( + utils::time::NANOS_PER_SECOND)) { + return false; + } + + return true; +}; + } // namespace repertory #endif // !defined(_WIN32) diff --git a/repertory/librepertory/src/drives/fuse/remotefuse/remote_fuse_drive.cpp b/repertory/librepertory/src/drives/fuse/remotefuse/remote_fuse_drive.cpp index e783d229..71c79d54 100644 --- a/repertory/librepertory/src/drives/fuse/remotefuse/remote_fuse_drive.cpp +++ b/repertory/librepertory/src/drives/fuse/remotefuse/remote_fuse_drive.cpp @@ -24,6 +24,7 @@ #include "drives/fuse/remotefuse/remote_fuse_drive.hpp" #include "app_config.hpp" +#include "drives/fuse/fuse_drive_base.hpp" #include "events/consumers/console_consumer.hpp" #include "events/consumers/logging_consumer.hpp" #include "events/event_system.hpp" @@ -623,6 +624,11 @@ auto remote_fuse_drive::utimens_impl(std::string api_path, auto remote_fuse_drive::utimens_impl(std::string api_path, const struct timespec tv[2]) -> api_error { #endif // FUSE_USE_VERSION >= 30 + if (not fuse_drive_base::validate_timespec(tv[0]) || + not fuse_drive_base::validate_timespec(tv[1])) { + return api_error::invalid_operation; + } + remote::file_time rtv[2U] = {0}; if (tv != nullptr) { const auto update_timespec = [](auto &dst, const auto &src) { From 691a9e3d0a32843a14eab01fdfb5c8d6b0fff221 Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Sun, 28 Sep 2025 08:28:55 -0500 Subject: [PATCH 23/26] macos fixes --- .../src/drives/fuse/fuse_drive.cpp | 10 ++- .../fuse/remotefuse/remote_fuse_drive.cpp | 64 ++++++++++++++----- support/3rd_party/icu_configure.sh | 0 3 files changed, 57 insertions(+), 17 deletions(-) mode change 100644 => 100755 support/3rd_party/icu_configure.sh diff --git a/repertory/librepertory/src/drives/fuse/fuse_drive.cpp b/repertory/librepertory/src/drives/fuse/fuse_drive.cpp index 571d34b9..85d0bc29 100644 --- a/repertory/librepertory/src/drives/fuse/fuse_drive.cpp +++ b/repertory/librepertory/src/drives/fuse/fuse_drive.cpp @@ -1222,11 +1222,19 @@ auto fuse_drive::setattr_x_impl(std::string api_path, struct setattr_x *attr) if (SETATTR_WANTS_MODTIME(attr)) { struct timespec ts[2]; if (SETATTR_WANTS_ACCTIME(attr)) { + if (not fuse_drive_base::validate_timespec(attr->acctime)) { + return api_error::invalid_operation; + } + ts[0].tv_sec = attr->acctime.tv_sec; ts[0].tv_nsec = attr->acctime.tv_nsec; } else { + if (not fuse_drive_base::validate_timespec(attr->modtime)) { + return api_error::invalid_operation; + } + struct timeval tv{}; - gettimeofday(&tv, NULL); + gettimeofday(&tv, nullptr); ts[0].tv_sec = tv.tv_sec; ts[0].tv_nsec = tv.tv_usec * 1000; } diff --git a/repertory/librepertory/src/drives/fuse/remotefuse/remote_fuse_drive.cpp b/repertory/librepertory/src/drives/fuse/remotefuse/remote_fuse_drive.cpp index 71c79d54..7476dcb8 100644 --- a/repertory/librepertory/src/drives/fuse/remotefuse/remote_fuse_drive.cpp +++ b/repertory/librepertory/src/drives/fuse/remotefuse/remote_fuse_drive.cpp @@ -51,8 +51,8 @@ auto remote_fuse_drive::access_impl(std::string api_path, int mask) } #if defined(__APPLE__) -api_error remote_fuse_drive::chflags_impl(std::string api_path, - uint32_t flags) { +auto remote_fuse_drive::chflags_impl(std::string api_path, uint32_t flags) + -> api_error { return utils::to_api_error( remote_instance_->fuse_chflags(api_path.c_str(), flags)); } @@ -142,6 +142,18 @@ auto remote_fuse_drive::fsetattr_x_impl(std::string api_path, struct setattr_x *attr, struct fuse_file_info *f_info) -> api_error { + if (SETATTR_WANTS_MODTIME(attr)) { + if (not fuse_drive_base::validate_timespec(attr->modtime)) { + return api_error::invalid_operation; + } + } + + if (SETATTR_WANTS_ACCTIME(attr)) { + if (not fuse_drive_base::validate_timespec(attr->acctime)) { + return api_error::invalid_operation; + } + } + remote::setattr_x attributes{}; attributes.valid = attr->valid; attributes.mode = attr->mode; @@ -210,9 +222,9 @@ auto remote_fuse_drive::getattr_impl(std::string api_path, struct stat *u_stat) } #if defined(__APPLE__) -api_error remote_fuse_drive::getxtimes_impl(std::string api_path, - struct timespec *bkuptime, - struct timespec *crtime) { +auto remote_fuse_drive::getxtimes_impl(std::string api_path, + struct timespec *bkuptime, + struct timespec *crtime) -> api_error { if (not(bkuptime && crtime)) { return utils::to_api_error(-EFAULT); } @@ -489,8 +501,20 @@ auto remote_fuse_drive::rmdir_impl(std::string api_path) -> api_error { } #if defined(__APPLE__) -api_error remote_fuse_drive::setattr_x_impl(std::string api_path, - struct setattr_x *attr) { +auto remote_fuse_drive::setattr_x_impl(std::string api_path, + struct setattr_x *attr) -> api_error { + if (SETATTR_WANTS_MODTIME(attr)) { + if (not fuse_drive_base::validate_timespec(attr->modtime)) { + return api_error::invalid_operation; + } + } + + if (SETATTR_WANTS_ACCTIME(attr)) { + if (not fuse_drive_base::validate_timespec(attr->acctime)) { + return api_error::invalid_operation; + } + } + remote::setattr_x attributes{}; attributes.valid = attr->valid; attributes.mode = attr->mode; @@ -518,8 +542,9 @@ api_error remote_fuse_drive::setattr_x_impl(std::string api_path, remote_instance_->fuse_setattr_x(api_path.c_str(), attributes)); } -api_error remote_fuse_drive::setbkuptime_impl(std::string api_path, - const struct timespec *bkuptime) { +auto remote_fuse_drive::setbkuptime_impl(std::string api_path, + const struct timespec *bkuptime) + -> api_error { remote::file_time repertory_bkuptime = ((static_cast(bkuptime->tv_sec) * utils::time::NANOS_PER_SECOND) + @@ -528,8 +553,9 @@ api_error remote_fuse_drive::setbkuptime_impl(std::string api_path, remote_instance_->fuse_setbkuptime(api_path.c_str(), repertory_bkuptime)); } -api_error remote_fuse_drive::setchgtime_impl(std::string api_path, - const struct timespec *chgtime) { +auto remote_fuse_drive::setchgtime_impl(std::string api_path, + const struct timespec *chgtime) + -> api_error { remote::file_time repertory_chgtime = ((static_cast(chgtime->tv_sec) * utils::time::NANOS_PER_SECOND) + @@ -538,8 +564,9 @@ api_error remote_fuse_drive::setchgtime_impl(std::string api_path, remote_instance_->fuse_setchgtime(api_path.c_str(), repertory_chgtime)); } -api_error remote_fuse_drive::setcrtime_impl(std::string api_path, - const struct timespec *crtime) { +auto remote_fuse_drive::setcrtime_impl(std::string api_path, + const struct timespec *crtime) + -> api_error { remote::file_time repertory_crtime = ((static_cast(crtime->tv_sec) * utils::time::NANOS_PER_SECOND) + @@ -548,12 +575,12 @@ api_error remote_fuse_drive::setcrtime_impl(std::string api_path, remote_instance_->fuse_setcrtime(api_path.c_str(), repertory_crtime)); } -api_error remote_fuse_drive::setvolname_impl(const char *volname) { +auto remote_fuse_drive::setvolname_impl(const char *volname) -> api_error { return utils::to_api_error(remote_instance_->fuse_setvolname(volname)); } -api_error remote_fuse_drive::statfs_x_impl(std::string api_path, - struct statfs *stbuf) { +auto remote_fuse_drive::statfs_x_impl(std::string api_path, + struct statfs *stbuf) -> api_error { auto res = statfs(config_.get_data_directory().c_str(), stbuf); if (res == 0) { remote::statfs_x r_stat{}; @@ -624,8 +651,13 @@ auto remote_fuse_drive::utimens_impl(std::string api_path, auto remote_fuse_drive::utimens_impl(std::string api_path, const struct timespec tv[2]) -> api_error { #endif // FUSE_USE_VERSION >= 30 + REPERTORY_USES_FUNCTION_NAME(); + if (not fuse_drive_base::validate_timespec(tv[0]) || not fuse_drive_base::validate_timespec(tv[1])) { + utils::error::handle_info( + function_name, + fmt::format("failed|{}|{}", tv[0].tv_nsec, tv[1].tv_nsec)); return api_error::invalid_operation; } diff --git a/support/3rd_party/icu_configure.sh b/support/3rd_party/icu_configure.sh old mode 100644 new mode 100755 From ef4afe5ca0af89b904697dee3d5bc4ff38b245d9 Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Sun, 28 Sep 2025 16:22:30 -0500 Subject: [PATCH 24/26] keep ring buffer read pos centered for event seek ahead/behind --- .../src/file_manager/ring_buffer_base.cpp | 31 +++++++++++++++-- .../src/ring_buffer_open_file_test.cpp | 33 +++++++++++++------ 2 files changed, 51 insertions(+), 13 deletions(-) diff --git a/repertory/librepertory/src/file_manager/ring_buffer_base.cpp b/repertory/librepertory/src/file_manager/ring_buffer_base.cpp index d1bebd8b..491dbd9e 100644 --- a/repertory/librepertory/src/file_manager/ring_buffer_base.cpp +++ b/repertory/librepertory/src/file_manager/ring_buffer_base.cpp @@ -350,7 +350,33 @@ void ring_buffer_base::set_api_path(const std::string &api_path) { } void ring_buffer_base::update_position(std::size_t count, bool is_forward) { + if (count == 0U) { + return; + } + mutex_lock chunk_lock(chunk_mtx_); + const auto center_ring = [this]() -> void { + auto ring_size = static_cast(read_state_.size()); + auto half_ring_size = ring_size / 2U; + + auto offset = ring_pos_ - ring_begin_; + if (offset > half_ring_size) { + auto steps = offset - half_ring_size; + + auto max_steps = (total_chunks_ - 1U) - ring_end_; + if (steps > max_steps) { + steps = max_steps; + } + + while (steps-- != 0U) { + ++ring_begin_; + ++ring_end_; + read_state_[ring_end_ % ring_size] = false; + } + } + + chunk_notify_.notify_all(); + }; if (is_forward) { if ((ring_pos_ + count) > (total_chunks_ - 1U)) { @@ -363,7 +389,7 @@ void ring_buffer_base::update_position(std::size_t count, bool is_forward) { if (is_forward ? (ring_pos_ + count) <= ring_end_ : (ring_pos_ - count) >= ring_begin_) { ring_pos_ += is_forward ? count : -count; - chunk_notify_.notify_all(); + center_ring(); return; } @@ -387,7 +413,6 @@ void ring_buffer_base::update_position(std::size_t count, bool is_forward) { ring_end_ = std::min(total_chunks_ - 1U, ring_begin_ + read_state_.size() - 1U); - - chunk_notify_.notify_all(); + center_ring(); } } // namespace repertory diff --git a/repertory/repertory_test/src/ring_buffer_open_file_test.cpp b/repertory/repertory_test/src/ring_buffer_open_file_test.cpp index 1f0ca6c9..97df5f79 100644 --- a/repertory/repertory_test/src/ring_buffer_open_file_test.cpp +++ b/repertory/repertory_test/src/ring_buffer_open_file_test.cpp @@ -64,15 +64,21 @@ TEST_F(ring_buffer_open_file_test, can_forward_to_last_chunk) { { ring_buffer_open_file file(ring_buffer_dir, test_chunk_size, 30U, fsi, provider, 8U); + file.set(0U, 3U); file.forward(4U); EXPECT_EQ(std::size_t(7U), file.get_current_chunk()); - EXPECT_EQ(std::size_t(0U), file.get_first_chunk()); - EXPECT_EQ(std::size_t(7U), file.get_last_chunk()); - for (std::size_t chunk = 0U; chunk < 8U; chunk++) { + EXPECT_EQ(std::size_t(3U), file.get_first_chunk()); + EXPECT_EQ(std::size_t(10U), file.get_last_chunk()); + + for (std::size_t chunk = 3U; chunk <= 7U; ++chunk) { EXPECT_TRUE(file.get_read_state(chunk)); } + + for (std::size_t chunk = 8U; chunk <= 10U; ++chunk) { + EXPECT_FALSE(file.get_read_state(chunk)); + } } } @@ -117,16 +123,22 @@ TEST_F(ring_buffer_open_file_test, can_forward_after_last_chunk) { { ring_buffer_open_file file(ring_buffer_dir, test_chunk_size, 30U, fsi, provider, 8U); + file.set(0U, 3U); file.forward(5U); EXPECT_EQ(std::size_t(8U), file.get_current_chunk()); - EXPECT_EQ(std::size_t(1U), file.get_first_chunk()); - EXPECT_EQ(std::size_t(8U), file.get_last_chunk()); + EXPECT_EQ(std::size_t(4U), file.get_first_chunk()); + EXPECT_EQ(std::size_t(11U), file.get_last_chunk()); + + EXPECT_TRUE(file.get_read_state(4U)); + EXPECT_TRUE(file.get_read_state(5U)); + EXPECT_TRUE(file.get_read_state(6U)); + EXPECT_TRUE(file.get_read_state(7U)); EXPECT_FALSE(file.get_read_state(8U)); - for (std::size_t chunk = 1U; chunk < 8U; chunk++) { - EXPECT_TRUE(file.get_read_state(chunk)); - } + EXPECT_FALSE(file.get_read_state(9U)); + EXPECT_FALSE(file.get_read_state(10U)); + EXPECT_FALSE(file.get_read_state(11U)); } } @@ -144,12 +156,13 @@ TEST_F(ring_buffer_open_file_test, can_forward_and_rollover_after_last_chunk) { { ring_buffer_open_file file(ring_buffer_dir, test_chunk_size, 30U, fsi, provider, 8U); + file.set(16U, 20U); file.forward(8U); EXPECT_EQ(std::size_t(28U), file.get_current_chunk()); - EXPECT_EQ(std::size_t(21U), file.get_first_chunk()); - EXPECT_EQ(std::size_t(28U), file.get_last_chunk()); + EXPECT_EQ(std::size_t(24U), file.get_first_chunk()); + EXPECT_EQ(std::size_t(31U), file.get_last_chunk()); } } From 0f6ca91faa6896b93382a50248cdb6aa9d4afe45 Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Sun, 28 Sep 2025 16:28:57 -0500 Subject: [PATCH 25/26] added tests --- .../src/ring_buffer_open_file_test.cpp | 158 ++++++++++++++++++ 1 file changed, 158 insertions(+) diff --git a/repertory/repertory_test/src/ring_buffer_open_file_test.cpp b/repertory/repertory_test/src/ring_buffer_open_file_test.cpp index 97df5f79..fe953519 100644 --- a/repertory/repertory_test/src/ring_buffer_open_file_test.cpp +++ b/repertory/repertory_test/src/ring_buffer_open_file_test.cpp @@ -576,4 +576,162 @@ TEST_F(ring_buffer_open_file_test, } } } + +TEST_F(ring_buffer_open_file_test, forward_center_noop_when_within_half) { + auto source_path = test::generate_test_file_name("ring_buffer_open_file"); + EXPECT_CALL(provider, is_read_only()).WillRepeatedly(Return(false)); + + filesystem_item fsi; + fsi.directory = false; + fsi.api_path = "/test.txt"; + fsi.size = test_chunk_size * 16U; + fsi.source_path = source_path; + + { + ring_buffer_open_file file(ring_buffer_dir, test_chunk_size, 30U, fsi, + provider, 8U); + + file.set(0U, 3U); + file.forward(1U); + + EXPECT_EQ(std::size_t(4U), file.get_current_chunk()); + EXPECT_EQ(std::size_t(0U), file.get_first_chunk()); + EXPECT_EQ(std::size_t(7U), file.get_last_chunk()); + + for (std::size_t chunk = 0U; chunk <= 7U; ++chunk) { + EXPECT_TRUE(file.get_read_state(chunk)) << "chunk " << chunk; + } + } +} + +TEST_F(ring_buffer_open_file_test, forward_center_caps_at_file_end) { + auto source_path = test::generate_test_file_name("ring_buffer_open_file"); + EXPECT_CALL(provider, is_read_only()).WillRepeatedly(Return(false)); + + filesystem_item fsi; + fsi.directory = false; + fsi.api_path = "/test.txt"; + fsi.size = test_chunk_size * 16U; + fsi.source_path = source_path; + + { + ring_buffer_open_file file(ring_buffer_dir, test_chunk_size, 30U, fsi, + provider, 8U); + + file.set(6U, 9U); + file.forward(100U); + + EXPECT_EQ(std::size_t(15U), file.get_current_chunk()); + EXPECT_EQ(std::size_t(8U), file.get_first_chunk()); + EXPECT_EQ(std::size_t(15U), file.get_last_chunk()); + } +} + +TEST_F(ring_buffer_open_file_test, + forward_delta_ge_ring_triggers_full_invalidation_path) { + auto source_path = test::generate_test_file_name("ring_buffer_open_file"); + EXPECT_CALL(provider, is_read_only()).WillRepeatedly(Return(false)); + + filesystem_item fsi; + fsi.directory = false; + fsi.api_path = "/test.txt"; + fsi.size = test_chunk_size * 16U; + fsi.source_path = source_path; + + { + ring_buffer_open_file file(ring_buffer_dir, test_chunk_size, 30U, fsi, + provider, 8U); + + file.set(0U, 0U); + file.forward(100U); + + EXPECT_EQ(std::size_t(15U), file.get_current_chunk()); + EXPECT_EQ(std::size_t(8U), file.get_first_chunk()); + EXPECT_EQ(std::size_t(15U), file.get_last_chunk()); + + for (std::size_t chunk = 8U; chunk <= 15U; ++chunk) { + EXPECT_FALSE(file.get_read_state(chunk)) << "chunk " << chunk; + } + } +} + +TEST_F(ring_buffer_open_file_test, + forward_center_marks_only_tail_entrants_unread) { + auto source_path = test::generate_test_file_name("ring_buffer_open_file"); + EXPECT_CALL(provider, is_read_only()).WillRepeatedly(Return(false)); + + filesystem_item fsi; + fsi.directory = false; + fsi.api_path = "/test.txt"; + fsi.size = test_chunk_size * 16U; + fsi.source_path = source_path; + + { + ring_buffer_open_file file(ring_buffer_dir, test_chunk_size, 30U, fsi, + provider, 8U); + + file.set(0U, 3U); + file.forward(2U); + + EXPECT_EQ(std::size_t(5U), file.get_current_chunk()); + EXPECT_EQ(std::size_t(1U), file.get_first_chunk()); + EXPECT_EQ(std::size_t(8U), file.get_last_chunk()); + + for (std::size_t chunk = 1U; chunk <= 7U; ++chunk) { + EXPECT_TRUE(file.get_read_state(chunk)) << "chunk " << chunk; + } + EXPECT_FALSE(file.get_read_state(8U)); + } +} + +TEST_F(ring_buffer_open_file_test, reverse_does_not_trigger_forward_centering) { + auto source_path = test::generate_test_file_name("ring_buffer_open_file"); + EXPECT_CALL(provider, is_read_only()).WillRepeatedly(Return(false)); + + filesystem_item fsi; + fsi.directory = false; + fsi.api_path = "/test.txt"; + fsi.size = test_chunk_size * 16U; + fsi.source_path = source_path; + + { + ring_buffer_open_file file(ring_buffer_dir, test_chunk_size, 30U, fsi, + provider, 8U); + + file.set(8U, 12U); + file.reverse(1U); + + EXPECT_EQ(std::size_t(11U), file.get_current_chunk()); + EXPECT_EQ(std::size_t(8U), file.get_first_chunk()); + EXPECT_EQ(std::size_t(15U), file.get_last_chunk()); + } +} + +TEST_F(ring_buffer_open_file_test, + forward_minimal_slide_then_center_multi_step) { + auto source_path = test::generate_test_file_name("ring_buffer_open_file"); + EXPECT_CALL(provider, is_read_only()).WillRepeatedly(Return(false)); + + filesystem_item fsi; + fsi.directory = false; + fsi.api_path = "/test.txt"; + fsi.size = test_chunk_size * 32U; + fsi.source_path = source_path; + + { + ring_buffer_open_file file(ring_buffer_dir, test_chunk_size, 30U, fsi, + provider, 8U); + + file.set(0U, 3U); + file.forward(7U); + + EXPECT_EQ(std::size_t(10U), file.get_current_chunk()); + EXPECT_EQ(std::size_t(6U), file.get_first_chunk()); + EXPECT_EQ(std::size_t(13U), file.get_last_chunk()); + + EXPECT_FALSE(file.get_read_state(11U)); + EXPECT_FALSE(file.get_read_state(12U)); + EXPECT_FALSE(file.get_read_state(13U)); + } +} } // namespace repertory From e67ac1dc13f9d4a9bcc1fc51ae944bd961985d93 Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Sun, 28 Sep 2025 17:57:37 -0500 Subject: [PATCH 26/26] added tests --- .../src/direct_open_file_test.cpp | 463 ++++++++++++++---- 1 file changed, 375 insertions(+), 88 deletions(-) diff --git a/repertory/repertory_test/src/direct_open_file_test.cpp b/repertory/repertory_test/src/direct_open_file_test.cpp index ba0751fb..e2fa7d62 100644 --- a/repertory/repertory_test/src/direct_open_file_test.cpp +++ b/repertory/repertory_test/src/direct_open_file_test.cpp @@ -1,23 +1,6 @@ /* Copyright <2018-2025> - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. + ... */ #include "test_common.hpp" @@ -36,13 +19,20 @@ public: protected: void SetUp() override { event_system::instance().start(); } - void TearDown() override { event_system::instance().stop(); } }; +namespace { +[[nodiscard]] auto build_test_buffer(repertory::utils::file::i_file &file) + -> repertory::data_buffer { + repertory::data_buffer buf; + EXPECT_TRUE(file.read_all(buf, 0U)); + return buf; +} +} // namespace + TEST_F(direct_open_file_test, read_full_file) { auto &source_file = test::create_random_file(test_chunk_size * 32U); - auto dest_path = test::generate_test_file_name("direct_open_file"); EXPECT_CALL(provider, is_read_only()).WillRepeatedly(Return(false)); @@ -52,23 +42,29 @@ TEST_F(direct_open_file_test, read_full_file) { fsi.directory = false; fsi.size = test_chunk_size * 32U; - std::mutex read_mtx; - EXPECT_CALL(provider, read_file_bytes) - .WillRepeatedly([&read_mtx, &source_file]( - const std::string & /* api_path */, std::size_t size, - std::uint64_t offset, data_buffer &data, - stop_type &stop_requested) -> api_error { - mutex_lock lock(read_mtx); + auto test_buffer = build_test_buffer(source_file); - EXPECT_FALSE(stop_requested); - std::size_t bytes_read{}; - data.resize(size); - auto ret = source_file.read(data, offset, &bytes_read) - ? api_error::success - : api_error::os_error; - EXPECT_EQ(bytes_read, data.size()); - return ret; + EXPECT_CALL(provider, read_file_bytes) + .WillRepeatedly([test_buffer](const std::string &, std::size_t size, + std::uint64_t offset, data_buffer &data, + stop_type &stop_requested) -> api_error { + if (stop_requested) { + return api_error::download_stopped; + } + std::size_t avail = + (offset < test_buffer.size()) + ? (test_buffer.size() - static_cast(offset)) + : 0U; + std::size_t to_copy = std::min(size, avail); + data.resize(to_copy); + if (to_copy != 0U) { + std::memcpy(data.data(), + test_buffer.data() + static_cast(offset), + to_copy); + } + return api_error::success; }); + { direct_open_file file(test_chunk_size, 30U, fsi, provider); @@ -88,23 +84,23 @@ TEST_F(direct_open_file_test, read_full_file) { ++chunk; to_read -= data.size(); } + dest_file->close(); - source_file.close(); auto hash1 = utils::file::file(source_file.get_path()).sha256(); auto hash2 = utils::file::file(dest_path).sha256(); - EXPECT_TRUE(hash1.has_value()); EXPECT_TRUE(hash2.has_value()); if (hash1.has_value() && hash2.has_value()) { EXPECT_STREQ(hash1.value().c_str(), hash2.value().c_str()); } } + + source_file.close(); } TEST_F(direct_open_file_test, read_full_file_in_reverse) { auto &source_file = test::create_random_file(test_chunk_size * 32U); - auto dest_path = test::generate_test_file_name("direct_open_file"); EXPECT_CALL(provider, is_read_only()).WillRepeatedly(Return(false)); @@ -114,23 +110,29 @@ TEST_F(direct_open_file_test, read_full_file_in_reverse) { fsi.directory = false; fsi.size = test_chunk_size * 32U; - std::mutex read_mtx; - EXPECT_CALL(provider, read_file_bytes) - .WillRepeatedly([&read_mtx, &source_file]( - const std::string & /* api_path */, std::size_t size, - std::uint64_t offset, data_buffer &data, - stop_type &stop_requested) -> api_error { - mutex_lock lock(read_mtx); + auto test_buffer = build_test_buffer(source_file); - EXPECT_FALSE(stop_requested); - std::size_t bytes_read{}; - data.resize(size); - auto ret = source_file.read(data, offset, &bytes_read) - ? api_error::success - : api_error::os_error; - EXPECT_EQ(bytes_read, data.size()); - return ret; + EXPECT_CALL(provider, read_file_bytes) + .WillRepeatedly([test_buffer](const std::string &, std::size_t size, + std::uint64_t offset, data_buffer &data, + stop_type &stop_requested) -> api_error { + if (stop_requested) { + return api_error::download_stopped; + } + std::size_t avail = + (offset < test_buffer.size()) + ? (test_buffer.size() - static_cast(offset)) + : 0U; + std::size_t to_copy = std::min(size, avail); + data.resize(to_copy); + if (to_copy != 0U) { + std::memcpy(data.data(), + test_buffer.data() + static_cast(offset), + to_copy); + } + return api_error::success; }); + { direct_open_file file(test_chunk_size, 30U, fsi, provider); @@ -155,7 +157,6 @@ TEST_F(direct_open_file_test, read_full_file_in_reverse) { auto hash1 = utils::file::file(source_file.get_path()).sha256(); auto hash2 = utils::file::file(dest_path).sha256(); - EXPECT_TRUE(hash1.has_value()); EXPECT_TRUE(hash2.has_value()); if (hash1.has_value() && hash2.has_value()) { @@ -166,7 +167,6 @@ TEST_F(direct_open_file_test, read_full_file_in_reverse) { TEST_F(direct_open_file_test, read_full_file_in_partial_chunks) { auto &source_file = test::create_random_file(test_chunk_size * 32U); - auto dest_path = test::generate_test_file_name("test"); EXPECT_CALL(provider, is_read_only()).WillRepeatedly(Return(false)); @@ -176,23 +176,29 @@ TEST_F(direct_open_file_test, read_full_file_in_partial_chunks) { fsi.api_path = "/test.txt"; fsi.size = test_chunk_size * 32U; - std::mutex read_mtx; - EXPECT_CALL(provider, read_file_bytes) - .WillRepeatedly([&read_mtx, &source_file]( - const std::string & /* api_path */, std::size_t size, - std::uint64_t offset, data_buffer &data, - stop_type &stop_requested) -> api_error { - mutex_lock lock(read_mtx); + auto test_buffer = build_test_buffer(source_file); - EXPECT_FALSE(stop_requested); - std::size_t bytes_read{}; - data.resize(size); - auto ret = source_file.read(data, offset, &bytes_read) - ? api_error::success - : api_error::os_error; - EXPECT_EQ(bytes_read, data.size()); - return ret; + EXPECT_CALL(provider, read_file_bytes) + .WillRepeatedly([test_buffer](const std::string &, std::size_t size, + std::uint64_t offset, data_buffer &data, + stop_type &stop_requested) -> api_error { + if (stop_requested) { + return api_error::download_stopped; + } + std::size_t avail = + (offset < test_buffer.size()) + ? (test_buffer.size() - static_cast(offset)) + : 0U; + std::size_t to_copy = std::min(size, avail); + data.resize(to_copy); + if (to_copy != 0U) { + std::memcpy(data.data(), + test_buffer.data() + static_cast(offset), + to_copy); + } + return api_error::success; }); + { direct_open_file file(test_chunk_size, 30U, fsi, provider); @@ -214,7 +220,6 @@ TEST_F(direct_open_file_test, read_full_file_in_partial_chunks) { auto hash1 = utils::file::file(source_file.get_path()).sha256(); auto hash2 = utils::file::file(dest_path).sha256(); - EXPECT_TRUE(hash1.has_value()); EXPECT_TRUE(hash2.has_value()); if (hash1.has_value() && hash2.has_value()) { @@ -225,7 +230,6 @@ TEST_F(direct_open_file_test, read_full_file_in_partial_chunks) { TEST_F(direct_open_file_test, read_full_file_in_partial_chunks_in_reverse) { auto &source_file = test::create_random_file(test_chunk_size * 32U); - auto dest_path = test::generate_test_file_name("direct_open_file"); EXPECT_CALL(provider, is_read_only()).WillRepeatedly(Return(false)); @@ -235,23 +239,29 @@ TEST_F(direct_open_file_test, read_full_file_in_partial_chunks_in_reverse) { fsi.api_path = "/test.txt"; fsi.size = test_chunk_size * 32U; - std::mutex read_mtx; - EXPECT_CALL(provider, read_file_bytes) - .WillRepeatedly([&read_mtx, &source_file]( - const std::string & /* api_path */, std::size_t size, - std::uint64_t offset, data_buffer &data, - stop_type &stop_requested) -> api_error { - mutex_lock lock(read_mtx); + auto test_buffer = build_test_buffer(source_file); - EXPECT_FALSE(stop_requested); - std::size_t bytes_read{}; - data.resize(size); - auto ret = source_file.read(data, offset, &bytes_read) - ? api_error::success - : api_error::os_error; - EXPECT_EQ(bytes_read, data.size()); - return ret; + EXPECT_CALL(provider, read_file_bytes) + .WillRepeatedly([test_buffer](const std::string &, std::size_t size, + std::uint64_t offset, data_buffer &data, + stop_type &stop_requested) -> api_error { + if (stop_requested) { + return api_error::download_stopped; + } + std::size_t avail = + (offset < test_buffer.size()) + ? (test_buffer.size() - static_cast(offset)) + : 0U; + std::size_t to_copy = std::min(size, avail); + data.resize(to_copy); + if (to_copy != 0U) { + std::memcpy(data.data(), + test_buffer.data() + static_cast(offset), + to_copy); + } + return api_error::success; }); + { direct_open_file file(test_chunk_size, 30U, fsi, provider); @@ -281,7 +291,6 @@ TEST_F(direct_open_file_test, read_full_file_in_partial_chunks_in_reverse) { auto hash1 = utils::file::file(source_file.get_path()).sha256(); auto hash2 = utils::file::file(dest_path).sha256(); - EXPECT_TRUE(hash1.has_value()); EXPECT_TRUE(hash2.has_value()); if (hash1.has_value() && hash2.has_value()) { @@ -289,4 +298,282 @@ TEST_F(direct_open_file_test, read_full_file_in_partial_chunks_in_reverse) { } } } + +TEST_F(direct_open_file_test, clamp_read_past_eof_returns_remaining_only) { + auto &source_file = test::create_random_file(test_chunk_size * 32U + 11U); + auto dest_path = test::generate_test_file_name("direct_open_file"); + + EXPECT_CALL(provider, is_read_only()).WillRepeatedly(Return(false)); + + filesystem_item fsi; + fsi.api_path = "/test.txt"; + fsi.directory = false; + fsi.size = test_chunk_size * 32U + 11U; + + auto test_buffer = build_test_buffer(source_file); + + EXPECT_CALL(provider, read_file_bytes) + .WillRepeatedly([test_buffer](const std::string &, std::size_t size, + std::uint64_t offset, data_buffer &data, + stop_type &stop_requested) -> api_error { + if (stop_requested) { + return api_error::download_stopped; + } + std::size_t avail = + (offset < test_buffer.size()) + ? (test_buffer.size() - static_cast(offset)) + : 0U; + std::size_t to_copy = std::min(size, avail); + data.resize(to_copy); + if (to_copy != 0U) { + std::memcpy(data.data(), + test_buffer.data() + static_cast(offset), + to_copy); + } + return api_error::success; + }); + + { + direct_open_file file(test_chunk_size, 30U, fsi, provider); + + auto out = utils::file::file::open_or_create_file(dest_path); + EXPECT_TRUE(out); + + std::uint64_t total_read{0U}; + for (std::size_t i = 0U; i < 32U; ++i) { + data_buffer data{}; + EXPECT_EQ(api_error::success, + file.read(test_chunk_size, total_read, data)); + std::size_t bytes_written{}; + EXPECT_TRUE(out->write(data, total_read, &bytes_written)); + total_read += data.size(); + } + + { + data_buffer data{}; + EXPECT_EQ(api_error::success, + file.read(test_chunk_size, total_read, data)); + EXPECT_EQ(std::size_t(11U), data.size()); + std::size_t bytes_written{}; + EXPECT_TRUE(out->write(data, total_read, &bytes_written)); + total_read += data.size(); + } + + out->close(); + source_file.close(); + + auto h1 = utils::file::file(source_file.get_path()).sha256(); + auto h2 = utils::file::file(dest_path).sha256(); + ASSERT_TRUE(h1.has_value()); + ASSERT_TRUE(h2.has_value()); + EXPECT_STREQ(h1->c_str(), h2->c_str()); + } +} + +TEST_F(direct_open_file_test, cross_boundary_small_read_is_correct) { + auto &source_file = test::create_random_file(test_chunk_size * 4U); + EXPECT_CALL(provider, is_read_only()).WillRepeatedly(Return(false)); + + filesystem_item fsi; + fsi.api_path = "/test.txt"; + fsi.directory = false; + fsi.size = test_chunk_size * 4U; + + auto test_buffer = build_test_buffer(source_file); + + EXPECT_CALL(provider, read_file_bytes) + .WillRepeatedly([test_buffer](const std::string &, std::size_t size, + std::uint64_t offset, data_buffer &data, + stop_type &stop_requested) -> api_error { + if (stop_requested) { + return api_error::download_stopped; + } + std::size_t avail = + (offset < test_buffer.size()) + ? (test_buffer.size() - static_cast(offset)) + : 0U; + std::size_t to_copy = std::min(size, avail); + data.resize(to_copy); + if (to_copy != 0U) { + std::memcpy(data.data(), + test_buffer.data() + static_cast(offset), + to_copy); + } + return api_error::success; + }); + + { + direct_open_file file(test_chunk_size, 30U, fsi, provider); + + std::size_t read_size{7U}; + std::uint64_t offset{test_chunk_size - 3U}; + data_buffer data{}; + EXPECT_EQ(api_error::success, file.read(read_size, offset, data)); + EXPECT_EQ(read_size, data.size()); + + EXPECT_EQ(0, + std::memcmp(test_buffer.data() + offset, data.data(), read_size)); + } + + source_file.close(); +} + +TEST_F(direct_open_file_test, random_seek_pattern_reads_match_source) { + auto &source_file = test::create_random_file(test_chunk_size * 16U); + EXPECT_CALL(provider, is_read_only()).WillRepeatedly(Return(false)); + + filesystem_item fsi; + fsi.api_path = "/test.txt"; + fsi.directory = false; + fsi.size = test_chunk_size * 16U; + + auto test_buffer = build_test_buffer(source_file); + + EXPECT_CALL(provider, read_file_bytes) + .WillRepeatedly([test_buffer](const std::string &, std::size_t size, + std::uint64_t offset, data_buffer &data, + stop_type &stop_requested) -> api_error { + if (stop_requested) { + return api_error::download_stopped; + } + std::size_t avail = + (offset < test_buffer.size()) + ? (test_buffer.size() - static_cast(offset)) + : 0U; + std::size_t to_copy = std::min(size, avail); + data.resize(to_copy); + if (to_copy != 0U) { + std::memcpy(data.data(), + test_buffer.data() + static_cast(offset), + to_copy); + } + return api_error::success; + }); + + { + direct_open_file file(test_chunk_size, 30U, fsi, provider); + + struct req { + std::size_t size; + std::uint64_t off; + }; + std::vector pattern_list{ + {3U, 0U}, + {64U, test_chunk_size - 1U}, + {11U, test_chunk_size * 2U + 5U}, + {512U, test_chunk_size * 7U + 13U}, + {5U, test_chunk_size * 16U - 5U}, + }; + + for (const auto &pattern : pattern_list) { + data_buffer data{}; + EXPECT_EQ(api_error::success, file.read(pattern.size, pattern.off, data)); + std::size_t avail = + (pattern.off < test_buffer.size()) + ? (test_buffer.size() - static_cast(pattern.off)) + : 0U; + std::size_t expected_size = std::min(pattern.size, avail); + ASSERT_EQ(expected_size, data.size()); + EXPECT_EQ(0, std::memcmp(test_buffer.data() + pattern.off, data.data(), + expected_size)); + } + } + + source_file.close(); +} + +TEST_F(direct_open_file_test, provider_error_is_propagated) { + auto &source_file = test::create_random_file(test_chunk_size * 4U); + EXPECT_CALL(provider, is_read_only()).WillRepeatedly(Return(false)); + + filesystem_item fsi; + fsi.api_path = "/test.txt"; + fsi.directory = false; + fsi.size = test_chunk_size * 4U; + + auto test_buffer = build_test_buffer(source_file); + + bool fail_once{true}; + EXPECT_CALL(provider, read_file_bytes) + .WillRepeatedly( + [test_buffer, &fail_once](const std::string &, std::size_t size, + std::uint64_t offset, data_buffer &data, + stop_type &stop_requested) -> api_error { + if (stop_requested) { + return api_error::download_stopped; + } + if (fail_once) { + fail_once = false; + return api_error::os_error; + } + std::size_t avail = + (offset < test_buffer.size()) + ? (test_buffer.size() - static_cast(offset)) + : 0U; + std::size_t to_copy = std::min(size, avail); + data.resize(to_copy); + if (to_copy != 0U) { + std::memcpy(data.data(), + test_buffer.data() + static_cast(offset), + to_copy); + } + return api_error::success; + }); + + { + direct_open_file file(test_chunk_size, 30U, fsi, provider); + + data_buffer data{}; + EXPECT_EQ(api_error::os_error, file.read(17U, 0U, data)); + + data_buffer data2{}; + EXPECT_EQ(api_error::success, file.read(17U, 0U, data2)); + EXPECT_EQ(std::size_t(17U), data2.size()); + } + + source_file.close(); +} + +TEST_F(direct_open_file_test, tiny_file_smaller_than_chunk) { + auto &source_file = test::create_random_file(17U); + EXPECT_CALL(provider, is_read_only()).WillRepeatedly(Return(false)); + + filesystem_item fsi; + fsi.api_path = "/test.txt"; + fsi.directory = false; + fsi.size = 17U; + + auto test_buffer = build_test_buffer(source_file); + + EXPECT_CALL(provider, read_file_bytes) + .WillRepeatedly([test_buffer](const std::string &, std::size_t size, + std::uint64_t offset, data_buffer &data, + stop_type &stop_requested) -> api_error { + if (stop_requested) { + return api_error::download_stopped; + } + std::size_t avail = + (offset < test_buffer.size()) + ? (test_buffer.size() - static_cast(offset)) + : 0U; + std::size_t to_copy = std::min(size, avail); + data.resize(to_copy); + if (to_copy != 0U) { + std::memcpy(data.data(), + test_buffer.data() + static_cast(offset), + to_copy); + } + return api_error::success; + }); + + { + direct_open_file file(test_chunk_size, 30U, fsi, provider); + + data_buffer data{}; + EXPECT_EQ(api_error::success, file.read(test_chunk_size, 0U, data)); + EXPECT_EQ(std::size_t(17U), data.size()); + } + + source_file.close(); +} } // namespace repertory