From 9c1ec6758cad25418ae80ae8583478b76a78bfbd Mon Sep 17 00:00:00 2001 From: TapTap Date: Sun, 6 Sep 2026 21:04:23 +0200 Subject: [PATCH] test: --fuzzy coverage (CLI, config wire, integration) Unit: parse -y/--fuzzy and --no-fuzzy negation (order-independent), -W and --no-delta leave fuzzy inert, --fuzzy implies --incremental/--delta, and validate_config rejects fuzzy with -s (chunk serialization) and -f (sendfile). Config: fuzzy survives the socketpair wire round-trip. Integration (TestFuzzy, byte counts via a new CountingProxy helper): the rename case transfers a 2 MiB file in a few percent of its size with byte- exact output under --fuzzy, while the no-fuzzy, no-candidate, dissimilar- sibling and --whole-file runs send the whole file; -y alias; -m parity; dest-holds-unsuitable-file falls back to the sibling; --delay-updates and --remove-source-files keep their semantics on fuzzy transfers. --- tests/integration/common.py | 71 +++++++++ tests/integration/test_features.py | 244 ++++++++++++++++++++++++++++- tests/test_client_cli.c | 119 ++++++++++++++ tests/test_config.c | 3 + 4 files changed, 436 insertions(+), 1 deletion(-) diff --git a/tests/integration/common.py b/tests/integration/common.py index 2e9653a..09b737a 100644 --- a/tests/integration/common.py +++ b/tests/integration/common.py @@ -6,6 +6,7 @@ import socket import subprocess import sys import tempfile +import threading import time PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) @@ -53,6 +54,76 @@ class ServerManager: self.stop() +class CountingProxy: + """One-shot TCP forwarder that counts the bytes flowing in each direction + between one client and the real server. + + Client output and --stats report SOURCE lengths, so a delta/fuzzy transfer + that moves only a few percent of the file is invisible in normal output. + Routing the client through this proxy makes the actual wire usage + observable: client_to_server counts every byte the client sent (config, + paths, and file/delta payloads), server_to_client counts the reply bytes + (including the receiver's delta signatures). + """ + + def __init__(self, target_port): + self.target_port = target_port + self._listener = socket.socket() + self._listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self._listener.bind(("127.0.0.1", 0)) + self._listener.listen(1) + self._listener.settimeout(30) + self.port = self._listener.getsockname()[1] + self.client_to_server = 0 + self.server_to_client = 0 + + @staticmethod + def _pump(src, dst, counter): + while True: + try: + data = src.recv(65536) + except OSError: + return + if not data: + try: + dst.shutdown(socket.SHUT_WR) + except OSError: + pass + return + try: + dst.sendall(data) + except OSError: + return + counter[0] += len(data) + + def run(self, cmd): + """Forward one client run (the full command list) to the real server and + return the CompletedProcess after the counts have settled.""" + + def serve(): + try: + client_sock, _ = self._listener.accept() + server_sock = socket.create_connection(("127.0.0.1", self.target_port), + timeout=10) + except OSError: + return + c2s, s2c = [0], [0] + a = threading.Thread(target=self._pump, args=(client_sock, server_sock, c2s)) + b = threading.Thread(target=self._pump, args=(server_sock, client_sock, s2c)) + a.start() + b.start() + a.join() + b.join() + self.client_to_server = c2s[0] + self.server_to_client = s2c[0] + + thread = threading.Thread(target=serve) + thread.start() + result = subprocess.run(cmd, capture_output=True, text=True, timeout=180) + thread.join(20) + return result + + def run_client(source_dir, dest_dir, flags=None, port=None, extra_args=None): """Run the client and return (result, duration).""" cmd = CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir, "--save-to-disk"] diff --git a/tests/integration/test_features.py b/tests/integration/test_features.py index a42bf29..8425b4a 100644 --- a/tests/integration/test_features.py +++ b/tests/integration/test_features.py @@ -1,6 +1,7 @@ """Feature tests: incremental sync, bandwidth limiting, dry run, metadata, filters.""" import filecmp import os +import random import shutil import subprocess import sys @@ -10,7 +11,7 @@ import pytest sys.path.insert(0, os.path.dirname(__file__)) from common import ( PROJECT_ROOT, BUILD_DIR, TEST_DATA_DIR, - run_client, + run_client, CountingProxy, generate_test_files, verify_transfer, clean_dir, make_result, get_dest_received_dir, CLIENT_CMD, ServerManager, ) @@ -2479,3 +2480,244 @@ class TestBasisDestDirs: received = get_dest_received_dir(dest, source) assert not os.path.exists(received), \ "over-limit basis run transferred files before failing" + + +def _random_payloads(size=2 * 1024 * 1024, changed=64 * 1024, seed=1234): + """Return (old, new) byte strings of equal length where `new` differs from + `old` only in one contiguous `changed`-byte region. Incompressible (random) + data keeps the whole-file wire cost near the file size, so a delta transfer + is distinguishable from a whole-file one by its wire bytes.""" + r = random.Random(seed) + data = bytearray(r.randbytes(size)) + old = bytes(data) + off = size // 3 + for i in range(off, off + changed): + data[i] = r.randrange(256) + return old, bytes(data) + + +class TestFuzzy: + """-y/--fuzzy similar-file delta basis. + + Scenario modelled on rsync's --fuzzy: a file is recreated under a similar + NEW basename in the same directory. The destination still holds the + old-named file (nothing deleted it), but the new path has no content of its + own at the destination, so without --fuzzy the receiver has no delta basis + and the whole file is sent. With --fuzzy the receiver searches the + destination directory, picks the similar-named sibling as the delta basis, + sends its block signature, and the sender transmits only the differences. + The reconstructed file must be byte-identical to the source in every mode; + only the wire usage changes (observed through CountingProxy, because client + --stats report source lengths, not wire bytes). + """ + + OLD_NAME = "report-2025.dat" + NEW_NAME = "report-2026.dat" + + def _client_via_proxy(self, source, dest, flags, proxy): + cmd = (CLIENT_CMD + ["--source-dir", source, "--dest-dir", dest, + "--save-to-disk", "--server-port", str(proxy.port)] + flags) + return proxy.run(cmd) + + def _seed_dest(self, source, dest, files, port): + """Write `files` {rel: bytes} into source and mirror them to dest.""" + for rel, content in files.items(): + full = os.path.join(source, rel) + os.makedirs(os.path.dirname(full), exist_ok=True) + with open(full, "wb") as fh: + fh.write(content) + clean_dir(dest) + result, _ = run_client(source, dest, port=port) + assert result.returncode == 0, f"seed failed: {(result.stderr or result.stdout)[:300]}" + + def _prepare(self, tag): + source = os.path.join(TEST_DATA_DIR, f"fuzzy_{tag}_src") + dest = os.path.join(TEST_DATA_DIR, f"fuzzy_{tag}_dst") + clean_dir(source) + return source, dest + + def _run_measured(self, source, dest, flags, port): + """Run a transfer through a byte-counting proxy. Returns (result, proxy).""" + proxy = CountingProxy(port) + result = self._client_via_proxy(source, dest, flags, proxy) + return result, proxy + + def test_fuzzy_uses_similar_sibling_as_delta_basis(self, shared_server): + source, dest = self._prepare("basis") + old_bytes, new_bytes = _random_payloads() + self._seed_dest(source, dest, {self.OLD_NAME: old_bytes}, shared_server.port) + # Recreate the file under a similar new name; the old sibling stays on + # the destination (nothing deletes it). + os.unlink(os.path.join(source, self.OLD_NAME)) + with open(os.path.join(source, self.NEW_NAME), "wb") as fh: + fh.write(new_bytes) + + result, proxy = self._run_measured(source, dest, ["--fuzzy"], shared_server.port) + assert result.returncode == 0, \ + f"--fuzzy rename transfer failed: {(result.stderr or result.stdout)[:300]}" + received = get_dest_received_dir(dest, source) + assert _read_file(os.path.join(received, self.NEW_NAME)) == new_bytes, \ + "fuzzy reconstruction is not byte-exact" + # The wire carried the delta, not the 2 MiB whole file. + assert proxy.client_to_server < len(new_bytes) // 4, \ + f"fuzzy transfer sent {proxy.client_to_server} bytes; expected a delta" + + def test_without_fuzzy_sends_the_whole_file(self, shared_server): + source, dest = self._prepare("whole") + old_bytes, new_bytes = _random_payloads() + self._seed_dest(source, dest, {self.OLD_NAME: old_bytes}, shared_server.port) + os.unlink(os.path.join(source, self.OLD_NAME)) + with open(os.path.join(source, self.NEW_NAME), "wb") as fh: + fh.write(new_bytes) + + result, proxy = self._run_measured(source, dest, ["--incremental", "--delta"], shared_server.port) + assert result.returncode == 0, f"no-fuzzy rename failed: {result.stderr[:300]}" + received = get_dest_received_dir(dest, source) + assert _read_file(os.path.join(received, self.NEW_NAME)) == new_bytes + # No similar basis: the whole file goes over the wire. + assert proxy.client_to_server > len(new_bytes) // 2, \ + f"expected a whole-file transfer, got {proxy.client_to_server} bytes" + + @pytest.mark.parametrize("mt", [False, True]) + def test_fuzzy_byte_exact_single_and_multithreaded(self, shared_server, mt): + source, dest = self._prepare(f"mt{'1' if mt else '0'}") + old_bytes, new_bytes = _random_payloads() + self._seed_dest(source, dest, {self.OLD_NAME: old_bytes}, shared_server.port) + os.unlink(os.path.join(source, self.OLD_NAME)) + with open(os.path.join(source, self.NEW_NAME), "wb") as fh: + fh.write(new_bytes) + + flags = ["--fuzzy"] + (["-m"] if mt else []) + result, proxy = self._run_measured(source, dest, flags, shared_server.port) + assert result.returncode == 0, \ + f"--fuzzy {'-m ' if mt else ''}rename failed: {(result.stderr or result.stdout)[:300]}" + received = get_dest_received_dir(dest, source) + assert _read_file(os.path.join(received, self.NEW_NAME)) == new_bytes, \ + f"fuzzy {'-m ' if mt else ''}reconstruction is not byte-exact" + assert proxy.client_to_server < len(new_bytes) // 4 + + def test_no_candidate_falls_back_to_whole_file(self, shared_server): + # A brand-new destination directory holds no sibling at all, so --fuzzy + # finds nothing and the file is transferred whole (and correctly). + source, dest = self._prepare("nocand") + clean_dir(dest) + os.makedirs(source, exist_ok=True) + _, new_bytes = _random_payloads() + with open(os.path.join(source, self.NEW_NAME), "wb") as fh: + fh.write(new_bytes) + result, proxy = self._run_measured(source, dest, ["--fuzzy"], shared_server.port) + assert result.returncode == 0, \ + f"--fuzzy no-candidate fallback failed: {(result.stderr or result.stdout)[:300]}" + received = get_dest_received_dir(dest, source) + assert _read_file(os.path.join(received, self.NEW_NAME)) == new_bytes + assert proxy.client_to_server > len(new_bytes) // 2, \ + "no-candidate fuzzy run should have sent the whole file" + + def test_dissimilar_sibling_is_not_used(self, shared_server): + # The destination holds a large sibling whose basename is too different + # from the incoming name; the name gate must reject it and fall back to + # a whole-file transfer. + source, dest = self._prepare("dissim") + old_bytes, new_bytes = _random_payloads() + self._seed_dest(source, dest, {"totally-unrelated-notes.bin": old_bytes}, + shared_server.port) + with open(os.path.join(source, self.NEW_NAME), "wb") as fh: + fh.write(new_bytes) + result, proxy = self._run_measured(source, dest, ["--fuzzy"], shared_server.port) + assert result.returncode == 0, \ + f"--fuzzy dissimilar-sibling run failed: {(result.stderr or result.stdout)[:300]}" + received = get_dest_received_dir(dest, source) + assert _read_file(os.path.join(received, self.NEW_NAME)) == new_bytes + assert proxy.client_to_server > len(new_bytes) // 2, \ + "a dissimilar-named sibling must not be used as a fuzzy basis" + + def test_fuzzy_helps_when_dest_holds_an_unsuitable_file(self, shared_server): + # The destination DOES hold the exact new name, but it is a tiny stale + # file (below the delta engine's minimum, ratio far outside its window), + # so it cannot serve as the basis. --fuzzy then falls back to the + # similar-named sibling. + source, dest = self._prepare("unsuitable") + old_bytes, new_bytes = _random_payloads() + self._seed_dest(source, dest, + {self.OLD_NAME: old_bytes, self.NEW_NAME: b"stale small file\n"}, + shared_server.port) + with open(os.path.join(source, self.NEW_NAME), "wb") as fh: + fh.write(new_bytes) + result, proxy = self._run_measured(source, dest, ["--fuzzy"], shared_server.port) + assert result.returncode == 0, \ + f"--fuzzy unsuitable-dest run failed: {(result.stderr or result.stdout)[:300]}" + received = get_dest_received_dir(dest, source) + assert _read_file(os.path.join(received, self.NEW_NAME)) == new_bytes, \ + "byte-exactness broken when the destination file was unsuitable" + assert proxy.client_to_server < len(new_bytes) // 4, \ + "fuzzy should have reused the similar sibling as the basis" + + def test_whole_file_makes_fuzzy_inert(self, shared_server): + # -W/--whole-file switches the delta machinery off, so --fuzzy has + # nothing to attach to and the file is transferred whole (rsync parity). + source, dest = self._prepare("wholefile") + old_bytes, new_bytes = _random_payloads() + self._seed_dest(source, dest, {self.OLD_NAME: old_bytes}, shared_server.port) + os.unlink(os.path.join(source, self.OLD_NAME)) + with open(os.path.join(source, self.NEW_NAME), "wb") as fh: + fh.write(new_bytes) + result, proxy = self._run_measured(source, dest, ["--fuzzy", "-W"], shared_server.port) + assert result.returncode == 0, \ + f"--fuzzy -W run failed: {(result.stderr or result.stdout)[:300]}" + received = get_dest_received_dir(dest, source) + assert _read_file(os.path.join(received, self.NEW_NAME)) == new_bytes + assert proxy.client_to_server > len(new_bytes) // 2, \ + "--whole-file must disable the fuzzy delta basis" + + def test_fuzzy_via_short_y_alias(self, shared_server): + source, dest = self._prepare("shorty") + old_bytes, new_bytes = _random_payloads() + self._seed_dest(source, dest, {self.OLD_NAME: old_bytes}, shared_server.port) + os.unlink(os.path.join(source, self.OLD_NAME)) + with open(os.path.join(source, self.NEW_NAME), "wb") as fh: + fh.write(new_bytes) + result, proxy = self._run_measured(source, dest, ["-y"], shared_server.port) + assert result.returncode == 0, f"-y rename failed: {(result.stderr or result.stdout)[:300]}" + received = get_dest_received_dir(dest, source) + assert _read_file(os.path.join(received, self.NEW_NAME)) == new_bytes + assert proxy.client_to_server < len(new_bytes) // 4, "-y did not enable fuzzy" + + def test_fuzzy_with_delay_updates_publishes_cleanly(self, shared_server): + # A fuzzy-reconstructed file goes through the normal store engine, so + # --delay-updates must stage and publish it with no staging leftovers. + source, dest = self._prepare("delay") + old_bytes, new_bytes = _random_payloads() + self._seed_dest(source, dest, {self.OLD_NAME: old_bytes}, shared_server.port) + os.unlink(os.path.join(source, self.OLD_NAME)) + with open(os.path.join(source, self.NEW_NAME), "wb") as fh: + fh.write(new_bytes) + result, _ = run_client(source, dest, + flags=["--fuzzy", "--delay-updates"], + port=shared_server.port) + assert result.returncode == 0, \ + f"--fuzzy --delay-updates failed: {(result.stderr or result.stdout)[:300]}" + received = get_dest_received_dir(dest, source) + assert _read_file(os.path.join(received, self.NEW_NAME)) == new_bytes + assert not os.path.isdir(os.path.join(dest, ".fastsync-stage")), \ + "delay-updates staging tree was not cleaned up" + + def test_fuzzy_source_removed_after_transfer(self, shared_server): + # A fuzzy transfer is a real transfer (not a skip), so + # --remove-source-files must remove the renamed source file. + source, dest = self._prepare("rm") + old_bytes, new_bytes = _random_payloads() + self._seed_dest(source, dest, {self.OLD_NAME: old_bytes}, shared_server.port) + os.unlink(os.path.join(source, self.OLD_NAME)) + with open(os.path.join(source, self.NEW_NAME), "wb") as fh: + fh.write(new_bytes) + result, _ = run_client(source, dest, + flags=["--fuzzy", "--remove-source-files"], + port=shared_server.port) + assert result.returncode == 0, \ + f"--fuzzy --remove-source-files failed: {(result.stderr or result.stdout)[:300]}" + assert not os.path.exists(os.path.join(source, self.NEW_NAME)), \ + "a fuzzy-transferred source should have been removed" + received = get_dest_received_dir(dest, source) + assert _read_file(os.path.join(received, self.NEW_NAME)) == new_bytes + assert _read_file(os.path.join(received, self.OLD_NAME)) == old_bytes + diff --git a/tests/test_client_cli.c b/tests/test_client_cli.c index 3d21fc6..45170e8 100644 --- a/tests/test_client_cli.c +++ b/tests/test_client_cli.c @@ -1169,6 +1169,120 @@ static void test_parse_args_whole_file() { config_delete(cfg); } +/* -y/--fuzzy reuses a similar destination file as a delta basis, so it implies + * the receiver-driven delta path (--incremental + --delta): FastSync's delta + * machinery is OFF by default, so without the implication a bare --fuzzy would + * be a silent no-op. Both spellings behave identically. */ +static void test_parse_args_fuzzy_implies_delta() { + static const char* const spellings[] = {"--fuzzy", "-y"}; + for (size_t i = 0; i < sizeof(spellings) / sizeof(spellings[0]); i++) { + Config* cfg = config_create(); + char* argv[] = {"fastsync", (char*)spellings[i], "/src", "/dst"}; + int positional_args[2]; + int positional_count = 0; + EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0); + EXPECT_TRUE(cfg->fuzzy); + EXPECT_TRUE(cfg->use_incremental); + EXPECT_TRUE(cfg->use_delta); + EXPECT_TRUE(cfg->use_metadata); + config_delete(cfg); + } +} + +/* --no-fuzzy turns the flag back off; the incremental/delta implication must + * only fire when the FINAL value of the flag is true (order-independent). */ +static void test_parse_args_fuzzy_negation() { + Config* cfg = config_create(); + char* argv[] = {"fastsync", "--fuzzy", "--no-fuzzy", "/src", "/dst"}; + int positional_args[2]; + int positional_count = 0; + EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0); + EXPECT_FALSE(cfg->fuzzy); + EXPECT_FALSE(cfg->use_delta); + EXPECT_FALSE(cfg->use_incremental); + config_delete(cfg); + + cfg = config_create(); + char* reordered[] = {"fastsync", "--no-fuzzy", "--fuzzy", "/src", "/dst"}; + positional_count = 0; + EXPECT_EQ_INT(parse_args(cfg, 5, reordered, positional_args, &positional_count), 0); + EXPECT_TRUE(cfg->fuzzy); + EXPECT_TRUE(cfg->use_incremental); + EXPECT_TRUE(cfg->use_delta); + config_delete(cfg); +} + +/* -W/--whole-file switches the delta machinery off, so --fuzzy is inert (the + * per-file quick check still needs --incremental, which stays implied). */ +static void test_parse_args_fuzzy_with_whole_file() { + Config* cfg = config_create(); + char* argv[] = {"fastsync", "--fuzzy", "-W", "/src", "/dst"}; + int positional_args[2]; + int positional_count = 0; + EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0); + EXPECT_TRUE(cfg->fuzzy); + EXPECT_TRUE(cfg->whole_file); + EXPECT_TRUE(cfg->use_incremental); + EXPECT_FALSE(cfg->use_delta); + config_delete(cfg); +} + +/* An explicit --no-delta is respected by the --fuzzy implication in either + * argument order (a user who switched delta off does not want it forced on). */ +static void test_parse_args_fuzzy_respects_no_delta() { + static const char* const combos[][2] = { + {"--fuzzy", "--no-delta"}, + {"--no-delta", "--fuzzy"}, + }; + for (size_t i = 0; i < sizeof(combos) / sizeof(combos[0]); i++) { + Config* cfg = config_create(); + char* argv[] = {"fastsync", (char*)combos[i][0], (char*)combos[i][1], "/src", "/dst"}; + int positional_args[2]; + int positional_count = 0; + EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0); + EXPECT_TRUE(cfg->fuzzy); + EXPECT_FALSE(cfg->use_delta); + config_delete(cfg); + } +} + +/* --fuzzy requires the delta machinery, which the chunk-serialization (-s) and + * sendfile (-f) modes reject -- mirroring the --delta constraint checks. */ +static void test_validate_config_fuzzy_incompatible_modes() { + Config* cfg = config_create(); + char* argv[] = {"fastsync", "--fuzzy", "-s", "/src", "/dst"}; + int positional_args[2]; + int positional_count = 0; + EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0); + cfg->send_directory = str_dup("/src"); + cfg->receive_root_directory = str_dup("/dst"); + EXPECT_TRUE(cfg->use_incremental); + EXPECT_FALSE(validate_config(cfg)); + config_delete(cfg); + + cfg = config_create(); + char* sendfile_argv[] = {"fastsync", "--fuzzy", "-f", "/src", "/dst"}; + positional_count = 0; + EXPECT_EQ_INT(parse_args(cfg, 5, sendfile_argv, positional_args, &positional_count), 0); + cfg->send_directory = str_dup("/src"); + cfg->receive_root_directory = str_dup("/dst"); + EXPECT_TRUE(cfg->use_delta); + EXPECT_FALSE(validate_config(cfg)); + config_delete(cfg); + + /* A plain --fuzzy run is a valid configuration. */ + cfg = config_create(); + char* ok_argv[] = {"fastsync", "--fuzzy", "/src", "/dst"}; + positional_count = 0; + EXPECT_EQ_INT(parse_args(cfg, 4, ok_argv, positional_args, &positional_count), 0); + cfg->send_directory = str_dup("/src"); + cfg->receive_root_directory = str_dup("/dst"); + EXPECT_TRUE(cfg->use_delta); + EXPECT_TRUE(cfg->use_incremental); + EXPECT_TRUE(validate_config(cfg)); + config_delete(cfg); +} + /* -x and --one-file-system enable client-side single-filesystem scanning. */ static void test_parse_args_one_file_system() { Config* cfg = config_create(); @@ -1713,6 +1827,11 @@ void test_client_cli() { test_parse_args_secluded_args(); test_parse_args_short_s_remains_chunk_serialization(); test_parse_args_whole_file(); + test_parse_args_fuzzy_implies_delta(); + test_parse_args_fuzzy_negation(); + test_parse_args_fuzzy_with_whole_file(); + test_parse_args_fuzzy_respects_no_delta(); + test_validate_config_fuzzy_incompatible_modes(); test_parse_args_one_file_system(); test_parse_args_compression_aliases(); test_parse_args_compression_equals_and_none(); diff --git a/tests/test_config.c b/tests/test_config.c index b0c8ceb..2cbacbd 100644 --- a/tests/test_config.c +++ b/tests/test_config.c @@ -128,6 +128,7 @@ static void test_config_send_receive() { send_cfg->use_executability = true; send_cfg->use_delta = true; send_cfg->whole_file = true; + send_cfg->fuzzy = true; send_cfg->ignore_times = true; send_cfg->size_only = true; send_cfg->compression_level = 5; @@ -188,6 +189,8 @@ static void test_config_send_receive() { ok = false; if (recv_cfg->use_delta) ok = false; + if (!recv_cfg->fuzzy) + ok = false; if (recv_cfg->modify_window != 4) ok = false; if (!recv_cfg->existing)