"""Feature tests: incremental sync, bandwidth limiting, dry run, metadata, filters.""" import filecmp import os import random import shutil import subprocess import sys import time import pytest sys.path.insert(0, os.path.dirname(__file__)) from common import ( PROJECT_ROOT, BUILD_DIR, TEST_DATA_DIR, run_client, CountingProxy, generate_test_files, verify_transfer, clean_dir, make_result, get_dest_received_dir, CLIENT_CMD, ServerManager, ) SOURCE_DIR = os.path.join(TEST_DATA_DIR, "feature_source") DEST_DIR = os.path.join(TEST_DATA_DIR, "feature_dest") @pytest.fixture(scope="module", autouse=True) def setup_test_data(): generate_test_files(SOURCE_DIR, full=False) clean_dir(DEST_DIR) yield shutil.rmtree(TEST_DATA_DIR, ignore_errors=True) class TestDryRun: def test_human_readable_dry_run(self): result, dur = run_client(SOURCE_DIR, DEST_DIR, flags=["-h", "--dry-run"]) assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:100]}" assert "Total:" in result.stdout assert "KB" in result.stdout def test_dry_run(self): clean_dir(DEST_DIR) result, dur = run_client( SOURCE_DIR, DEST_DIR, flags=["-n"], ) assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:100]}" assert "Dry run:" in result.stdout, f"No dry run output: {result.stdout[:200]}" def test_quiet_suppresses_dry_run_output(self): result, dur = run_client( SOURCE_DIR, DEST_DIR, flags=["-q", "-n", "--progress", "--stats"], ) assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:100]}" assert result.stdout == "" assert result.stderr == "" def test_quiet_preserves_errors(self): result, dur = run_client( SOURCE_DIR, DEST_DIR, flags=["--quiet", "--server-port", "1"], ) assert result.returncode != 0 assert result.stderr != "" @pytest.mark.parametrize("flags", [["-q", "-v"], ["-v", "-q"]]) def test_quiet_successful_transfer_and_verbose_order(self, shared_server, flags): clean_dir(DEST_DIR) result, dur = run_client( SOURCE_DIR, DEST_DIR, flags=flags, port=shared_server.port, ) assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:100]}" assert result.stdout == "" assert result.stderr == "" received = get_dest_received_dir(DEST_DIR, SOURCE_DIR) mismatches, missing = verify_transfer(SOURCE_DIR, received) assert not missing, f"Missing: {missing}" assert not mismatches, f"Mismatch: {mismatches}" class TestRemoveSourceFiles: def test_removes_only_transferred_regular_files(self, shared_server): source = os.path.join(TEST_DATA_DIR, "remove_source") dest = os.path.join(TEST_DATA_DIR, "remove_dest") clean_dir(source) clean_dir(dest) with open(os.path.join(source, "one.txt"), "wb") as f: f.write(b"one") with open(os.path.join(source, "two.txt"), "wb") as f: f.write(b"two") os.makedirs(os.path.join(source, "directory")) os.symlink("one.txt", os.path.join(source, "link.txt")) result, _ = run_client(source, dest, flags=["--remove-source-files", "-m"], port=shared_server.port) assert result.returncode == 0, f"Remove-source sync failed: {result.stderr[:200]}" assert not os.path.exists(os.path.join(source, "one.txt")) assert not os.path.exists(os.path.join(source, "two.txt")) assert os.path.isdir(os.path.join(source, "directory")) assert os.path.islink(os.path.join(source, "link.txt")) def test_single_threaded_removes_transferred_file(self, shared_server): source = os.path.join(TEST_DATA_DIR, "remove_single_source") dest = os.path.join(TEST_DATA_DIR, "remove_single_dest") clean_dir(source) clean_dir(dest) source_file = os.path.join(source, "file.txt") with open(source_file, "wb") as f: f.write(b"single threaded") result, _ = run_client(source, dest, flags=["--remove-source-files"], port=shared_server.port) assert result.returncode == 0 assert not os.path.exists(source_file) def test_dry_run_preserves_source_files(self): source = os.path.join(TEST_DATA_DIR, "remove_dry_source") dest = os.path.join(TEST_DATA_DIR, "remove_dry_dest") clean_dir(source) clean_dir(dest) source_file = os.path.join(source, "file.txt") with open(source_file, "wb") as f: f.write(b"keep") result, _ = run_client(source, dest, flags=["--remove-source-files", "--dry-run"]) assert result.returncode == 0 assert os.path.isfile(source_file) def test_failed_connection_preserves_source_files(self): source = os.path.join(TEST_DATA_DIR, "remove_failed_source") dest = os.path.join(TEST_DATA_DIR, "remove_failed_dest") clean_dir(source) clean_dir(dest) source_file = os.path.join(source, "file.txt") with open(source_file, "wb") as f: f.write(b"keep after failure") result, _ = run_client(source, dest, flags=["--remove-source-files"], port=1) assert result.returncode != 0 assert os.path.isfile(source_file) def test_incremental_skip_preserves_source_file(self, shared_server): source = os.path.join(TEST_DATA_DIR, "remove_skipped_source") dest = os.path.join(TEST_DATA_DIR, "remove_skipped_dest") clean_dir(source) clean_dir(dest) source_file = os.path.join(source, "file.txt") with open(source_file, "wb") as f: f.write(b"keep after skip") # The seed run preserves timestamps (-M) so the destination copy has the # source's exact mtime; otherwise the incremental skip would depend on # both writes landing in the same whole second (a race). result, _ = run_client(source, dest, flags=["-M"], port=shared_server.port) assert result.returncode == 0 result, _ = run_client(source, dest, flags=["--remove-source-files", "--incremental"], port=shared_server.port) assert result.returncode == 0 assert os.path.isfile(source_file) class TestArchiveMode: def test_archive_mode(self, shared_server): clean_dir(DEST_DIR) result, dur = run_client( SOURCE_DIR, DEST_DIR, flags=["-a"], port=shared_server.port, ) if result.returncode != 0: pytest.fail(f"Exit {result.returncode}: {(result.stderr or result.stdout)[:200]}") received = get_dest_received_dir(DEST_DIR, SOURCE_DIR) mismatches, missing = verify_transfer(SOURCE_DIR, received) assert not missing, f"Missing: {missing}" assert not mismatches, f"Mismatch: {mismatches}" def test_archive_implied_options_can_be_negated(self, shared_server): clean_dir(DEST_DIR) result, dur = run_client( SOURCE_DIR, DEST_DIR, flags=["--archive", "--no-compress", "--no-m", "--no-preserve"], port=shared_server.port, ) if result.returncode != 0: pytest.fail(f"Exit {result.returncode}: {(result.stderr or result.stdout)[:200]}") received = get_dest_received_dir(DEST_DIR, SOURCE_DIR) mismatches, missing = verify_transfer(SOURCE_DIR, received) assert not missing, f"Missing: {missing}" assert not mismatches, f"Mismatch: {mismatches}" class TestExecutability: def test_preserves_only_executable_bits(self, shared_server): source = os.path.join(TEST_DATA_DIR, "executability_source") dest = os.path.join(TEST_DATA_DIR, "executability_dest") clean_dir(source) clean_dir(dest) source_file = os.path.join(source, "tool.sh") with open(source_file, "w") as f: f.write("#!/bin/sh\necho test\n") os.chmod(source_file, 0o751) result, _ = run_client(source, dest, flags=["-E"], port=shared_server.port) assert result.returncode == 0, f"Executability sync failed: {result.stderr[:200]}" received_file = os.path.join(get_dest_received_dir(dest, source), "tool.sh") received_mode = os.stat(received_file).st_mode assert received_mode & 0o111 == 0o111 assert received_mode & 0o600 == 0o600 assert received_mode & 0o077 == 0o011 class TestChmod: def test_chmod_applies_to_transferred_files(self, shared_server): clean_dir(DEST_DIR) source_file = os.path.join(SOURCE_DIR, "small.txt") os.chmod(source_file, 0o777) result, dur = run_client( SOURCE_DIR, DEST_DIR, flags=["--chmod=u=rw,go=r"], port=shared_server.port, ) if result.returncode != 0: pytest.fail(f"Exit {result.returncode}: {(result.stderr or result.stdout)[:200]}") received = get_dest_received_dir(DEST_DIR, SOURCE_DIR) assert (os.stat(os.path.join(received, "small.txt")).st_mode & 0o777) == 0o644 class TestCompressionChoice: def test_zstd_choice_compresses(self, shared_server): clean_dir(DEST_DIR) result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["--zc", "zstd"], port=shared_server.port) assert result.returncode == 0, f"zstd sync failed: {(result.stderr or result.stdout)[:200]}" received = get_dest_received_dir(DEST_DIR, SOURCE_DIR) mismatches, missing = verify_transfer(SOURCE_DIR, received) assert not missing, f"Missing: {missing}" assert not mismatches, f"Mismatch: {mismatches}" def test_none_choice_disables_compression(self, shared_server): clean_dir(DEST_DIR) result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-z", "--compress-choice", "none"], port=shared_server.port) assert result.returncode == 0, f"none sync failed: {(result.stderr or result.stdout)[:200]}" received = get_dest_received_dir(DEST_DIR, SOURCE_DIR) mismatches, missing = verify_transfer(SOURCE_DIR, received) assert not missing, f"Missing: {missing}" assert not mismatches, f"Mismatch: {mismatches}" class TestSkipCompress: def test_skip_compress_case_insensitive(self, shared_server): clean_dir(DEST_DIR) with open(os.path.join(SOURCE_DIR, "skip-case.TXT"), "wb") as f: f.write((b"skip compression case test\n" * 100)) result, _ = run_client( SOURCE_DIR, DEST_DIR, flags=["-c", "--skip-compress=.txt"], port=shared_server.port, ) assert result.returncode == 0, f"Skip-compress sync failed: {(result.stderr or result.stdout)[:200]}" received = get_dest_received_dir(DEST_DIR, SOURCE_DIR) with open(os.path.join(received, "skip-case.TXT"), "rb") as f: assert f.read() == b"skip compression case test\n" * 100 def test_skip_compress_empty_list(self, shared_server): clean_dir(DEST_DIR) result, _ = run_client( SOURCE_DIR, DEST_DIR, flags=["-c", "--skip-compress="], port=shared_server.port, ) assert result.returncode == 0, f"Empty skip-compress sync failed: {(result.stderr or result.stdout)[:200]}" received = get_dest_received_dir(DEST_DIR, SOURCE_DIR) mismatches, missing = verify_transfer(SOURCE_DIR, received) assert not missing, f"Missing: {missing}" assert not mismatches, f"Mismatch: {mismatches}" def test_skip_compress_incremental_full_fallback(self, shared_server): clean_dir(DEST_DIR) path = os.path.join(SOURCE_DIR, "incremental-skip.TXT") with open(path, "wb") as f: f.write(b"original skipped content\n") flags = ["-c", "-M", "--skip-compress=.txt"] result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=flags, port=shared_server.port) assert result.returncode == 0, f"Initial sync failed: {(result.stderr or result.stdout)[:200]}" with open(path, "wb") as f: f.write(b"updated skipped content\n") result, _ = run_client( SOURCE_DIR, DEST_DIR, flags=flags + ["--incremental"], port=shared_server.port, ) assert result.returncode == 0, f"Incremental sync failed: {(result.stderr or result.stdout)[:200]}" received = get_dest_received_dir(DEST_DIR, SOURCE_DIR) with open(os.path.join(received, "incremental-skip.TXT"), "rb") as f: assert f.read() == b"updated skipped content\n" def test_skip_compress_rejects_chunk_serialization(self, shared_server): result, _ = run_client( SOURCE_DIR, DEST_DIR, flags=["-c", "-s", "--skip-compress=.txt"], port=shared_server.port, ) assert result.returncode != 0 assert "cannot be combined" in (result.stderr or result.stdout) class TestExclude: def test_exclude_single(self, shared_server): clean_dir(DEST_DIR) result, dur = run_client( SOURCE_DIR, DEST_DIR, flags=["--exclude", "small.txt"], port=shared_server.port, ) if result.returncode != 0: pytest.fail(f"Exit {result.returncode}: {(result.stderr or result.stdout)[:200]}") received = get_dest_received_dir(DEST_DIR, SOURCE_DIR) mismatches, missing = verify_transfer(SOURCE_DIR, received) assert "small.txt" in missing, "small.txt should be excluded but was transferred" other_missing = [m for m in missing if m != "small.txt"] assert not other_missing, f"Other files missing: {other_missing}" assert not mismatches, f"Mismatch: {mismatches}" def test_exclude_glob(self, shared_server): clean_dir(DEST_DIR) result, dur = run_client( SOURCE_DIR, DEST_DIR, flags=["--exclude", "*.txt"], port=shared_server.port, ) if result.returncode != 0: pytest.fail(f"Exit {result.returncode}: {(result.stderr or result.stdout)[:200]}") received = get_dest_received_dir(DEST_DIR, SOURCE_DIR) assert not os.path.exists(os.path.join(received, "small.txt")), "small.txt should be excluded" assert os.path.exists(os.path.join(received, "binary.bin")), "binary.bin should be present" class TestInclude: def test_include_single(self, shared_server): clean_dir(DEST_DIR) result, dur = run_client( SOURCE_DIR, DEST_DIR, flags=["--include", "binary.bin"], port=shared_server.port, ) if result.returncode != 0: pytest.fail(f"Exit {result.returncode}: {(result.stderr or result.stdout)[:200]}") received = get_dest_received_dir(DEST_DIR, SOURCE_DIR) assert os.path.exists(os.path.join(received, "binary.bin")), "binary.bin should be included" assert not os.path.exists(os.path.join(received, "small.txt")), "small.txt should not be included" def test_include_glob(self, shared_server): clean_dir(DEST_DIR) result, dur = run_client( SOURCE_DIR, DEST_DIR, flags=["--include", "*.bin"], port=shared_server.port, ) if result.returncode != 0: pytest.fail(f"Exit {result.returncode}: {(result.stderr or result.stdout)[:200]}") received = get_dest_received_dir(DEST_DIR, SOURCE_DIR) assert os.path.exists(os.path.join(received, "binary.bin")), "binary.bin should be included" class TestSizeFilters: def test_max_size(self, shared_server): clean_dir(DEST_DIR) result, dur = run_client( SOURCE_DIR, DEST_DIR, flags=["--max-size", "100"], port=shared_server.port, ) if result.returncode != 0: pytest.fail(f"Exit {result.returncode}: {(result.stderr or result.stdout)[:200]}") received = get_dest_received_dir(DEST_DIR, SOURCE_DIR) assert os.path.exists(os.path.join(received, "small.txt")), "small.txt should be present" assert not os.path.exists(os.path.join(received, "medium.txt")), "medium.txt should be skipped" def test_min_size(self, shared_server): clean_dir(DEST_DIR) result, dur = run_client( SOURCE_DIR, DEST_DIR, flags=["--min-size", "1000"], port=shared_server.port, ) if result.returncode != 0: pytest.fail(f"Exit {result.returncode}: {(result.stderr or result.stdout)[:200]}") received = get_dest_received_dir(DEST_DIR, SOURCE_DIR) assert not os.path.exists(os.path.join(received, "small.txt")), "small.txt should be skipped" assert os.path.exists(os.path.join(received, "medium.txt")), "medium.txt should be present" class TestIncremental: def test_incremental_skips_unchanged(self, shared_server): clean_dir(DEST_DIR) result, dur = run_client( SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port, ) assert result.returncode == 0, f"First sync failed: {result.stderr[:100]}" start = time.monotonic() result, dur = run_client( SOURCE_DIR, DEST_DIR, flags=["-M", "--incremental"], port=shared_server.port, ) incremental_time = time.monotonic() - start assert result.returncode == 0, f"Incremental sync failed: {(result.stderr or result.stdout)[:200]}" received = get_dest_received_dir(DEST_DIR, SOURCE_DIR) mismatches, missing = verify_transfer(SOURCE_DIR, received) assert not missing, f"Missing: {missing}" assert not mismatches, f"Mismatch: {mismatches}" def test_incremental_detects_changes(self, shared_server): clean_dir(DEST_DIR) result, _ = run_client( SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port, ) assert result.returncode == 0 modified_file = os.path.join(SOURCE_DIR, "small.txt") with open(modified_file, "wb") as f: f.write(b"modified content for incremental test\n") result, dur = run_client( SOURCE_DIR, DEST_DIR, flags=["-M", "--incremental"], port=shared_server.port, ) assert result.returncode == 0 with open(modified_file, "wb") as f: f.write(b"hello world\n") received = get_dest_received_dir(DEST_DIR, SOURCE_DIR) received_file = os.path.join(received, "small.txt") assert os.path.exists(received_file), "Modified file should be present" with open(received_file, "rb") as f: content = f.read() assert b"modified content" in content, f"Modified content not transferred: {content[:50]}" def test_checksum_detects_same_size_and_mtime_change(self, shared_server): clean_dir(DEST_DIR) result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port) assert result.returncode == 0 received = get_dest_received_dir(DEST_DIR, SOURCE_DIR) source_file = os.path.join(SOURCE_DIR, "small.txt") received_file = os.path.join(received, "small.txt") source_stat = os.stat(source_file) with open(received_file, "wb") as f: f.write(b"different!\n") os.utime(received_file, (source_stat.st_atime, source_stat.st_mtime)) result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M", "--incremental", "--checksum"], port=shared_server.port) assert result.returncode == 0, f"Checksum sync failed: {result.stderr[:200]}" with open(received_file, "rb") as f: assert f.read() == b"hello world\n" def test_size_only_skips_same_size_with_different_mtime(self, shared_server): clean_dir(DEST_DIR) result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port) assert result.returncode == 0 received = get_dest_received_dir(DEST_DIR, SOURCE_DIR) received_file = os.path.join(received, "small.txt") with open(received_file, "wb") as f: f.write(b"different!!\n") os.utime(received_file, (time.time() - 3600, time.time() - 3600)) result, _ = run_client( SOURCE_DIR, DEST_DIR, flags=["-M", "--incremental", "--size-only"], port=shared_server.port, ) assert result.returncode == 0, f"Size-only sync failed: {result.stderr[:200]}" with open(received_file, "rb") as f: assert f.read() == b"different!!\n" def test_ignore_times_transfers_same_size_and_mtime(self, shared_server): clean_dir(DEST_DIR) result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port) assert result.returncode == 0 received = get_dest_received_dir(DEST_DIR, SOURCE_DIR) source_file = os.path.join(SOURCE_DIR, "small.txt") received_file = os.path.join(received, "small.txt") source_stat = os.stat(source_file) with open(received_file, "wb") as f: f.write(b"stale data!\n") os.utime(received_file, (source_stat.st_atime, source_stat.st_mtime)) result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M", "--incremental", "--ignore-times"], port=shared_server.port) assert result.returncode == 0, f"Ignore-times sync failed: {result.stderr[:200]}" with open(received_file, "rb") as f: assert f.read() == b"hello world\n" def test_modify_window_allows_subsecond_mtime_difference(self, shared_server): clean_dir(DEST_DIR) result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port) assert result.returncode == 0 received = get_dest_received_dir(DEST_DIR, SOURCE_DIR) source_file = os.path.join(SOURCE_DIR, "small.txt") received_file = os.path.join(received, "small.txt") source_stat = os.stat(source_file) with open(received_file, "wb") as f: f.write(b"modified!!!\n") os.utime(received_file, ns=(source_stat.st_atime_ns, source_stat.st_mtime_ns - 1500000000)) result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M", "--incremental", "--modify-window=2"], port=shared_server.port) assert result.returncode == 0, f"Modify-window sync failed: {result.stderr[:200]}" with open(received_file, "rb") as f: assert f.read() == b"modified!!!\n" def test_whole_file_disables_delta_and_keeps_compression(self, shared_server): clean_dir(DEST_DIR) result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port) assert result.returncode == 0 source_file = os.path.join(SOURCE_DIR, "medium.txt") with open(source_file, "wb") as f: f.write(b"whole-file replacement\n" * 5000) result, _ = run_client( SOURCE_DIR, DEST_DIR, flags=["-M", "--incremental", "--delta", "-W", "-c"], port=shared_server.port, ) assert result.returncode == 0, f"Whole-file sync failed: {(result.stderr or result.stdout)[:200]}" received = get_dest_received_dir(DEST_DIR, SOURCE_DIR) mismatches, missing = verify_transfer(SOURCE_DIR, received) assert not missing, f"Missing: {missing}" assert not mismatches, f"Mismatch: {mismatches}" class TestChecksumChoice: """--checksum-choice/--cc and --checksum-seed: the whole-file digest used by the --incremental/--checksum handshake is selectable and seedable. The receiver hashes the on-disk old file with the SAME algorithm+seed, so an unchanged file is skipped and a changed file (even with identical size and mtime) is transferred -- and the transfer always lands byte-exact. FastSync accepts xxh64 (default, seed-aware) and md5; names it does not implement are rejected, never silently ignored.""" def test_unsupported_algorithm_is_rejected(self, shared_server): result, _ = run_client( SOURCE_DIR, DEST_DIR, flags=["--checksum", "--checksum-choice=sha256"], port=shared_server.port, ) assert result.returncode != 0, "sha256 must be rejected, not silently ignored" @pytest.mark.parametrize("algo", ["xxh64", "md5"]) @pytest.mark.parametrize("mt", [False, True]) def test_unchanged_skipped_and_bytes_preserved(self, shared_server, algo, mt): clean_dir(DEST_DIR) result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port) assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}" flags = (["-M", "--incremental", "--checksum", f"--checksum-choice={algo}"] + (["-m"] if mt else [])) result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=flags, port=shared_server.port) assert result.returncode == 0, f"checksum {algo} run failed: {result.stderr[:200]}" received = get_dest_received_dir(DEST_DIR, SOURCE_DIR) mismatches, missing = verify_transfer(SOURCE_DIR, received) assert not missing, f"Missing: {missing}" assert not mismatches, f"Mismatch: {mismatches}" # A changed source file with the SAME size and mtime must still be # detected (and re-transferred byte-exactly) because the whole-file digest # differs -- the explicit reason --checksum exists. This exercises the # sender/receiver digest agreement for a non-default algorithm. @pytest.mark.parametrize("algo", ["xxh64", "md5"]) @pytest.mark.parametrize("mt", [False, True]) def test_changed_same_size_mtime_redetected(self, shared_server, algo, mt): clean_dir(DEST_DIR) result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port) assert result.returncode == 0 received = get_dest_received_dir(DEST_DIR, SOURCE_DIR) source_file = os.path.join(SOURCE_DIR, "small.txt") # "hello world\n" (12 bytes) received_file = os.path.join(received, "small.txt") source_stat = os.stat(source_file) with open(received_file, "wb") as f: f.write(b"DDDDDDDDDDDD") # same size, different content os.utime(received_file, (source_stat.st_atime, source_stat.st_mtime)) flags = (["-M", "--incremental", "--checksum", f"--checksum-choice={algo}"] + (["-m"] if mt else [])) result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=flags, port=shared_server.port) assert result.returncode == 0, f"checksum {algo} redetect failed: {result.stderr[:200]}" with open(received_file, "rb") as f: assert f.read() == b"hello world\n" @pytest.mark.parametrize("algo", ["xxh64", "md5"]) def test_unchanged_run_transfers_almost_no_data(self, shared_server, algo): # A fully-unchanged --checksum run skips every file: only the config + a # small handshake travels, not the payloads. Proxy byte counts are not # available for -m (multithreaded connections), so single-thread only. clean_dir(DEST_DIR) result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port) assert result.returncode == 0 flags = ["-M", "--incremental", "--checksum", f"--checksum-choice={algo}"] proxy = CountingProxy(shared_server.port) cmd = (CLIENT_CMD + ["--source-dir", SOURCE_DIR, "--dest-dir", DEST_DIR, "--save-to-disk", "--server-port", str(proxy.port)] + flags) result = proxy.run(cmd) assert result.returncode == 0, f"checksum {algo} skip run failed: {result.stderr[:200]}" assert proxy.client_to_server < 100000, \ f"unchanged --checksum run sent {proxy.client_to_server} bytes; expected a skip" @pytest.mark.parametrize("mt", [False, True]) def test_seed_is_deterministic_and_preserves_content(self, shared_server, mt): clean_dir(DEST_DIR) flags = ["-M", "--incremental", "--checksum", "--checksum-choice=xxh64", "--checksum-seed=987654"] + (["-m"] if mt else []) first, _ = run_client(SOURCE_DIR, DEST_DIR, flags=flags, port=shared_server.port) assert first.returncode == 0, f"seeded run failed: {first.stderr[:200]}" # A second run with the SAME seed and unchanged content skips everything # deterministically (same digests both sides). second, _ = run_client(SOURCE_DIR, DEST_DIR, flags=flags, port=shared_server.port) assert second.returncode == 0, f"deterministic rerun failed: {second.stderr[:200]}" received = get_dest_received_dir(DEST_DIR, SOURCE_DIR) mismatches, missing = verify_transfer(SOURCE_DIR, received) assert not missing and not mismatches, f"missing={missing} mismatches={mismatches}" # A changed file with the same size and mtime is still caught and fixed # (a non-zero seed does not weaken the comparison). source_file = os.path.join(SOURCE_DIR, "medium.txt") received_file = os.path.join(received, "medium.txt") source_stat = os.stat(source_file) with open(received_file, "wb") as f: f.write(b"z" * os.path.getsize(source_file)) os.utime(received_file, (source_stat.st_atime, source_stat.st_mtime)) third, _ = run_client(SOURCE_DIR, DEST_DIR, flags=flags, port=shared_server.port) assert third.returncode == 0, f"seeded redetect failed: {third.stderr[:200]}" with open(received_file, "rb") as f: assert f.read() == open(source_file, "rb").read() # --checksum-seed also feeds the delta path's per-block strong checksum on # both ends (receiver signature and sender window hash use the same seed), # so a seeded delta transfer still lands byte-exact. @pytest.mark.parametrize("mt", [False, True]) def test_seed_delta_block_hash_transfers_byte_exact(self, shared_server, mt): source = os.path.join(TEST_DATA_DIR, f"ccseed_{'m' if mt else 's'}_src") dest = os.path.join(TEST_DATA_DIR, f"ccseed_{'m' if mt else 's'}_dst") clean_dir(source) clean_dir(dest) big = os.path.join(source, "big.bin") with open(big, "wb") as f: f.write(bytes(range(256)) * 200) # 51200 bytes > delta 16K floor result, _ = run_client(source, dest, flags=["-M"], port=shared_server.port) assert result.returncode == 0, f"seed delta seed failed: {result.stderr[:200]}" # Edit a region so the receiver must match a changed block with the seed. with open(big, "r+b") as f: f.seek(1000) f.write(b"\x00" * 64) # Force an mtime mismatch: the incremental quick-check skips files whose # stored mtime second equals the source's, which can collide when the # edit and the prior sync share a second. Setting an old dest mtime # guarantees the delta path is exercised deterministically. os.utime(os.path.join(get_dest_received_dir(dest, source), "big.bin"), (0, 0)) flags = (["-M", "--incremental", "--delta", "--checksum-seed=314159"] + (["-m"] if mt else [])) result, _ = run_client(source, dest, flags=flags, port=shared_server.port) assert result.returncode == 0, f"seed delta run failed: {result.stderr[:200]}" received = get_dest_received_dir(dest, source) assert _read_file(os.path.join(received, "big.bin")) == _read_file(big), \ "seeded delta transfer is not byte-exact" class TestUpdate: def test_update_skips_older_destination_and_allows_equal_or_newer_source(self, shared_server): clean_dir(DEST_DIR) result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-u"], port=shared_server.port) assert result.returncode == 0 received = get_dest_received_dir(DEST_DIR, SOURCE_DIR) source_file = os.path.join(SOURCE_DIR, "small.txt") received_file = os.path.join(received, "small.txt") source_stat = os.stat(source_file) with open(received_file, "wb") as f: f.write(b"newer destination\n") os.utime(received_file, ns=(source_stat.st_atime_ns, source_stat.st_mtime_ns + 10_000_000_000)) result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-u"], port=shared_server.port) assert result.returncode == 0 with open(received_file, "rb") as f: assert f.read() == b"newer destination\n" os.utime(received_file, ns=(source_stat.st_atime_ns, source_stat.st_mtime_ns)) result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-u"], port=shared_server.port) assert result.returncode == 0 with open(received_file, "rb") as f: assert f.read() == b"hello world\n" def test_update_skips_unreadable_newer_destination(self, shared_server): clean_dir(DEST_DIR) result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-u"], port=shared_server.port) assert result.returncode == 0 received_file = os.path.join(get_dest_received_dir(DEST_DIR, SOURCE_DIR), "small.txt") source_stat = os.stat(os.path.join(SOURCE_DIR, "small.txt")) with open(received_file, "wb") as f: f.write(b"protected destination\n") os.utime(received_file, ns=(source_stat.st_atime_ns, source_stat.st_mtime_ns + 10_000_000_000)) original_mode = os.stat(received_file).st_mode try: os.chmod(received_file, 0) result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-u"], port=shared_server.port) assert result.returncode == 0 os.chmod(received_file, original_mode) with open(received_file, "rb") as f: assert f.read() == b"protected destination\n" finally: os.chmod(received_file, original_mode) with open(received_file, "wb") as f: f.write(b"older destination\n") os.utime(received_file, ns=(source_stat.st_atime_ns, source_stat.st_mtime_ns - 10_000_000_000)) result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-u"], port=shared_server.port) assert result.returncode == 0 with open(received_file, "rb") as f: assert f.read() == b"hello world\n" class TestExisting: def test_existing_updates_existing_and_skips_new(self, shared_server): clean_dir(DEST_DIR) result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port) assert result.returncode == 0, f"Initial sync failed: {(result.stderr or result.stdout)[:200]}" received = get_dest_received_dir(DEST_DIR, SOURCE_DIR) source_file = os.path.join(SOURCE_DIR, "small.txt") new_source_file = os.path.join(SOURCE_DIR, "new-existing-test.txt") with open(source_file, "wb") as f: f.write(b"updated existing content\n") with open(new_source_file, "wb") as f: f.write(b"this file must not be created\n") try: result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M", "--existing"], port=shared_server.port) assert result.returncode == 0, f"--existing sync failed: {(result.stderr or result.stdout)[:200]}" with open(os.path.join(received, "small.txt"), "rb") as f: assert f.read() == b"updated existing content\n" assert not os.path.exists(os.path.join(received, "new-existing-test.txt")) finally: os.unlink(new_source_file) with open(source_file, "wb") as f: f.write(b"hello world\n") class TestIgnoreExisting: def test_ignore_existing_preserves_existing_and_transfers_new(self, shared_server): clean_dir(DEST_DIR) result, _ = run_client(SOURCE_DIR, DEST_DIR, port=shared_server.port) assert result.returncode == 0 received = get_dest_received_dir(DEST_DIR, SOURCE_DIR) existing_file = os.path.join(received, "small.txt") with open(existing_file, "wb") as f: f.write(b"destination content\n") new_source = os.path.join(SOURCE_DIR, "new.txt") try: with open(new_source, "wb") as f: f.write(b"new file\n") result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["--ignore-existing"], port=shared_server.port) assert result.returncode == 0, f"Sync failed: {(result.stderr or result.stdout)[:200]}" with open(existing_file, "rb") as f: assert f.read() == b"destination content\n" with open(os.path.join(received, "new.txt"), "rb") as f: assert f.read() == b"new file\n" finally: if os.path.lexists(new_source): os.unlink(new_source) class TestIgnoreExisting: def test_ignore_existing_preserves_existing_and_transfers_new(self, shared_server): clean_dir(DEST_DIR) result, _ = run_client(SOURCE_DIR, DEST_DIR, port=shared_server.port) assert result.returncode == 0 received = get_dest_received_dir(DEST_DIR, SOURCE_DIR) existing_file = os.path.join(received, "small.txt") with open(existing_file, "wb") as f: f.write(b"destination content\n") new_source = os.path.join(SOURCE_DIR, "new.txt") try: with open(new_source, "wb") as f: f.write(b"new file\n") result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["--ignore-existing"], port=shared_server.port) assert result.returncode == 0, f"Sync failed: {(result.stderr or result.stdout)[:200]}" with open(existing_file, "rb") as f: assert f.read() == b"destination content\n" with open(os.path.join(received, "new.txt"), "rb") as f: assert f.read() == b"new file\n" finally: if os.path.lexists(new_source): os.unlink(new_source) class TestDelete: def test_delete_removes_extra_files(self, shared_server): clean_dir(DEST_DIR) result, _ = run_client( SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port, ) assert result.returncode == 0 received = get_dest_received_dir(DEST_DIR, SOURCE_DIR) extra_file = os.path.join(received, "extra_file.txt") extra_dir = os.path.join(received, "extra_dir") with open(extra_file, "w") as f: f.write("should be deleted") os.makedirs(extra_dir, exist_ok=True) with open(os.path.join(extra_dir, "nested.txt"), "w") as f: f.write("nested extra") result, dur = run_client( SOURCE_DIR, DEST_DIR, flags=["-M", "--delete"], port=shared_server.port, ) assert result.returncode == 0, f"Delete sync failed: {(result.stderr or result.stdout)[:200]}" # The default server policy intentionally refuses client-requested # deletion unless it is started with --allow-delete. assert os.path.exists(extra_file), "unauthorized delete removed an extra file" assert os.path.exists(extra_dir), "unauthorized delete removed an extra directory" mismatches, missing = verify_transfer(SOURCE_DIR, received) assert not missing, f"Missing: {missing}" assert not mismatches, f"Mismatch: {mismatches}" class TestProgress: def test_progress_output(self, shared_server): clean_dir(DEST_DIR) result, dur = run_client( SOURCE_DIR, DEST_DIR, flags=["--progress"], port=shared_server.port, ) assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:100]}" output = result.stdout + result.stderr assert "Sent " in output and "MB" in output, "--progress produced no stable byte marker" assert "Done." in output, "--progress did not report completion" def test_human_readable_stats(self, shared_server): clean_dir(DEST_DIR) result, dur = run_client( SOURCE_DIR, DEST_DIR, flags=["-h", "--stats"], port=shared_server.port, ) assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:100]}" assert "Stats:" in result.stderr assert "KB" in result.stderr def test_human_readable_progress_multithreaded(self, shared_server): clean_dir(DEST_DIR) result, dur = run_client( SOURCE_DIR, DEST_DIR, flags=["-m", "-h", "--progress"], port=shared_server.port, ) assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:100]}" output = result.stdout + result.stderr assert "Sent " in output assert "KB" in output assert "Done." in output class TestInfo: def test_info_copy_reports_transfers(self, shared_server): clean_dir(DEST_DIR) result, _ = run_client( SOURCE_DIR, DEST_DIR, flags=["--info=copy"], port=shared_server.port, ) assert result.returncode == 0, f"Info sync failed: {(result.stderr or result.stdout)[:200]}" output = result.stdout + result.stderr assert "[INFO]" in output and "Transferring" in output def test_info_stats_reports_multithreaded_transfer(self, shared_server): clean_dir(DEST_DIR) result, _ = run_client( SOURCE_DIR, DEST_DIR, flags=["-m", "--info=stats"], port=shared_server.port, ) assert result.returncode == 0, f"Info stats sync failed: {(result.stderr or result.stdout)[:200]}" output = result.stdout + result.stderr assert "[INFO]" in output and "Transfer summary:" in output def test_info_rejects_unknown_flag(self): result, _ = run_client( SOURCE_DIR, DEST_DIR, flags=["--info=unknown"], ) assert result.returncode != 0 assert "unsupported --info flag" in result.stderr @pytest.mark.parametrize("flags", [ ["--info=none", "--verbose"], ["--verbose", "--info=none"], ]) def test_info_none_suppresses_verbose_info(self, shared_server, flags): clean_dir(DEST_DIR) result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=flags, port=shared_server.port) assert result.returncode == 0, f"Info sync failed: {(result.stderr or result.stdout)[:200]}" output = result.stdout + result.stderr assert "[INFO]" not in output assert "Transferring" not in output assert "Transfer summary:" not in output class TestBandwidthLimit: def test_bwlimit_runs(self, shared_server): clean_dir(DEST_DIR) result, dur = run_client( SOURCE_DIR, DEST_DIR, flags=["--bwlimit", "10240"], port=shared_server.port, ) assert result.returncode == 0, f"Exit {result.returncode}: {(result.stderr or result.stdout)[:200]}" received = get_dest_received_dir(DEST_DIR, SOURCE_DIR) mismatches, missing = verify_transfer(SOURCE_DIR, received) assert not missing, f"Missing: {missing}" assert not mismatches, f"Mismatch: {mismatches}" def _read_file(path): with open(path, "rb") as fh: return fh.read() class TestRemoveSourceFilesSkips: """--remove-source-files must not delete sources the receiver skipped (rsync reference behavior).""" def test_existing_first_sync_keeps_new_source(self, shared_server): source = os.path.join(TEST_DATA_DIR, "remove_rsf_existing_src") dest = os.path.join(TEST_DATA_DIR, "remove_rsf_existing_dst") clean_dir(source) clean_dir(dest) with open(os.path.join(source, "only.txt"), "wb") as f: f.write(b"keep me") result, _ = run_client(source, dest, flags=["--remove-source-files", "--existing"], port=shared_server.port) assert result.returncode == 0, f"Sync failed: {result.stderr[:200]}" # The file exists only on the source side, so --existing makes the # receiver skip it; the source must therefore not be removed. assert os.path.isfile(os.path.join(source, "only.txt")) received = get_dest_received_dir(dest, source) assert not os.path.exists(os.path.join(received, "only.txt")) def test_ignore_existing_keeps_skipped_source(self, shared_server): source = os.path.join(TEST_DATA_DIR, "remove_rsf_ignore_src") dest = os.path.join(TEST_DATA_DIR, "remove_rsf_ignore_dst") clean_dir(source) clean_dir(dest) source_file = os.path.join(source, "file.txt") with open(source_file, "wb") as f: f.write(b"payload") result, _ = run_client(source, dest, port=shared_server.port) assert result.returncode == 0 result, _ = run_client(source, dest, flags=["--remove-source-files", "--ignore-existing"], port=shared_server.port) assert result.returncode == 0, f"Sync failed: {result.stderr[:200]}" # Destination already has the file, so the second run is a receiver # skip; the source file must survive. assert os.path.isfile(source_file) received = get_dest_received_dir(dest, source) assert _read_file(os.path.join(received, "file.txt")) == b"payload" def test_update_newer_destination_keeps_source(self, shared_server): source = os.path.join(TEST_DATA_DIR, "remove_rsf_update_src") dest = os.path.join(TEST_DATA_DIR, "remove_rsf_update_dst") clean_dir(source) clean_dir(dest) source_file = os.path.join(source, "file.txt") with open(source_file, "wb") as f: f.write(b"source payload") result, _ = run_client(source, dest, port=shared_server.port) assert result.returncode == 0 received = get_dest_received_dir(dest, source) received_file = os.path.join(received, "file.txt") with open(received_file, "wb") as f: f.write(b"newer destination payload") os.utime(received_file, ns=(time.time_ns() + 10**9, time.time_ns() + 10**9)) result, _ = run_client(source, dest, flags=["--remove-source-files", "--update"], port=shared_server.port) assert result.returncode == 0, f"Sync failed: {result.stderr[:200]}" # --update skips a destination that is newer than the source, so the # source must not be removed. assert os.path.isfile(source_file) assert _read_file(received_file) == b"newer destination payload" def test_multithreaded_ignore_existing_keeps_skipped_source(self, shared_server): """The multithreaded writer path must also report per-file outcomes so a --remove-source-files sender does not delete skipped sources.""" source = os.path.join(TEST_DATA_DIR, "remove_rsf_mt_ignore_src") dest = os.path.join(TEST_DATA_DIR, "remove_rsf_mt_ignore_dst") clean_dir(source) clean_dir(dest) source_file = os.path.join(source, "file.txt") with open(source_file, "wb") as f: f.write(b"payload") result, _ = run_client(source, dest, port=shared_server.port) assert result.returncode == 0 result, _ = run_client(source, dest, flags=["--remove-source-files", "--ignore-existing", "-m"], port=shared_server.port) assert result.returncode == 0, f"Sync failed: {result.stderr[:200]}" # Destination already has the file, so the receiver (writer thread) # skips it; the source must survive. assert os.path.isfile(source_file) received = get_dest_received_dir(dest, source) assert _read_file(os.path.join(received, "file.txt")) == b"payload" class TestBackup: def _sync(self, source, dest, flags, port): return run_client(source, dest, flags=flags, port=port) def test_plain_backup_keeps_previous_version(self, shared_server): source = os.path.join(TEST_DATA_DIR, "backup_src") dest = os.path.join(TEST_DATA_DIR, "backup_dst") clean_dir(source) clean_dir(dest) source_file = os.path.join(source, "f.txt") with open(source_file, "wb") as f: f.write(b"AAAA") result, _ = self._sync(source, dest, ["--backup"], shared_server.port) assert result.returncode == 0, f"Backup sync failed: {result.stderr[:200]}" with open(source_file, "wb") as f: f.write(b"BBBB") result, _ = self._sync(source, dest, ["--backup"], shared_server.port) assert result.returncode == 0, f"Backup sync failed: {result.stderr[:200]}" received = get_dest_received_dir(dest, source) assert _read_file(os.path.join(received, "f.txt")) == b"BBBB" # rsync default suffix "~" keeps the overwritten version. assert _read_file(os.path.join(received, "f.txt~")) == b"AAAA" def test_backup_custom_suffix(self, shared_server): source = os.path.join(TEST_DATA_DIR, "backup_suffix_src") dest = os.path.join(TEST_DATA_DIR, "backup_suffix_dst") clean_dir(source) clean_dir(dest) source_file = os.path.join(source, "f.txt") with open(source_file, "wb") as f: f.write(b"AAAA") flags = ["--backup", "--suffix", ".bak"] result, _ = self._sync(source, dest, flags, shared_server.port) assert result.returncode == 0, f"Backup sync failed: {result.stderr[:200]}" with open(source_file, "wb") as f: f.write(b"BBBB") result, _ = self._sync(source, dest, flags, shared_server.port) assert result.returncode == 0, f"Backup sync failed: {result.stderr[:200]}" received = get_dest_received_dir(dest, source) assert _read_file(os.path.join(received, "f.txt")) == b"BBBB" assert _read_file(os.path.join(received, "f.txt.bak")) == b"AAAA" def test_backup_dir_stores_backups_separately(self, shared_server): source = os.path.join(TEST_DATA_DIR, "backup_dir_src") dest = os.path.join(TEST_DATA_DIR, "backup_dir_dst") clean_dir(source) clean_dir(dest) source_file = os.path.join(source, "f.txt") with open(source_file, "wb") as f: f.write(b"AAAA") flags = ["--backup", "--backup-dir", "backups"] result, _ = self._sync(source, dest, flags, shared_server.port) assert result.returncode == 0, f"Backup sync failed: {result.stderr[:200]}" with open(source_file, "wb") as f: f.write(b"BBBB") result, _ = self._sync(source, dest, flags, shared_server.port) assert result.returncode == 0, f"Backup sync failed: {result.stderr[:200]}" received = get_dest_received_dir(dest, source) assert _read_file(os.path.join(received, "f.txt")) == b"BBBB" backup = os.path.join(dest, "backups", os.path.relpath(source_file, os.path.sep)) assert _read_file(backup) == b"AAAA" class TestPartialDir: def test_completed_transfer_installed_in_destination(self, shared_server): source = os.path.join(TEST_DATA_DIR, "partial_src") dest = os.path.join(TEST_DATA_DIR, "partial_dst") clean_dir(source) clean_dir(dest) source_file = os.path.join(source, "f.txt") with open(source_file, "wb") as f: f.write(b"partial payload") result, _ = run_client(source, dest, flags=["--partial", "--partial-dir", ".partial"], port=shared_server.port) assert result.returncode == 0, f"Partial sync failed: {result.stderr[:200]}" received = get_dest_received_dir(dest, source) assert _read_file(os.path.join(received, "f.txt")) == b"partial payload" # A completed transfer must not remain under the partial directory. partial = os.path.join(dest, ".partial", os.path.relpath(source_file, os.path.sep)) assert not os.path.exists(partial) class TestLargeFile: def test_transfer_100mb_file(self, shared_server): source = os.path.join(TEST_DATA_DIR, "large_src") dest = os.path.join(TEST_DATA_DIR, "large_dst") clean_dir(source) clean_dir(dest) source_file = os.path.join(source, "big.bin") chunk = os.urandom(1024 * 1024) with open(source_file, "wb") as f: for _ in range(100): f.write(chunk) result, _ = run_client(source, dest, port=shared_server.port) assert result.returncode == 0, f"Large-file sync failed: {result.stderr[:200]}" received = get_dest_received_dir(dest, source) assert filecmp.cmp(source_file, os.path.join(received, "big.bin"), shallow=False) class TestOneFileSystem: def _make_tree(self, source): clean_dir(source) os.makedirs(os.path.join(source, "nested", "deeper")) with open(os.path.join(source, "root.txt"), "wb") as f: f.write(b"root") with open(os.path.join(source, "nested", "inner.txt"), "wb") as f: f.write(b"inner") with open(os.path.join(source, "nested", "deeper", "deep.txt"), "wb") as f: f.write(b"deep") def _assert_full_tree_transferred(self, source, dest, port, flags): clean_dir(dest) result, _ = run_client(source, dest, flags=flags, port=port) assert result.returncode == 0, f"Sync failed: {result.stderr[:200]}" received = get_dest_received_dir(dest, source) mismatches, missing = verify_transfer(source, received) assert not missing, f"Missing: {missing}" assert not mismatches, f"Mismatch: {mismatches}" def test_x_transfer_matches_plain_over_single_filesystem(self, shared_server): source = os.path.join(TEST_DATA_DIR, "ofs_src") self._make_tree(source) self._assert_full_tree_transferred(source, os.path.join(TEST_DATA_DIR, "ofs_dst"), shared_server.port, ["-x"]) def test_x_multithreaded_transfer_matches_plain_over_single_filesystem(self, shared_server): source = os.path.join(TEST_DATA_DIR, "ofs_m_src") self._make_tree(source) self._assert_full_tree_transferred(source, os.path.join(TEST_DATA_DIR, "ofs_m_dst"), shared_server.port, ["-m", "--one-file-system"]) def test_x_skips_other_device_mountpoint(self, shared_server): if os.geteuid() != 0 or shutil.which("mount") is None or shutil.which("umount") is None: pytest.skip("cross-device test requires root and mount(8)") source = os.path.join(TEST_DATA_DIR, "ofs_mnt_src") dest = os.path.join(TEST_DATA_DIR, "ofs_mnt_dst") dest_plain = os.path.join(TEST_DATA_DIR, "ofs_mnt_plain_dst") mountpoint = os.path.join(source, "external") clean_dir(source) clean_dir(dest) clean_dir(dest_plain) os.makedirs(mountpoint) os.makedirs(os.path.join(source, "nested")) with open(os.path.join(source, "root.txt"), "wb") as f: f.write(b"root") with open(os.path.join(source, "nested", "inner.txt"), "wb") as f: f.write(b"inner") mounted = False unmount_error = "" try: mount = subprocess.run(["mount", "-t", "tmpfs", "tmpfs", mountpoint], capture_output=True, text=True) if mount.returncode != 0: pytest.skip(f"cannot mount tmpfs: {mount.stderr.strip()}") mounted = True with open(os.path.join(mountpoint, "away.txt"), "wb") as f: f.write(b"cross device") result, _ = run_client(source, dest, flags=["-x"], port=shared_server.port) assert result.returncode == 0, f"-x sync failed: {result.stderr[:200]}" received = get_dest_received_dir(dest, source) assert os.path.isfile(os.path.join(received, "root.txt")) assert os.path.isfile(os.path.join(received, "nested", "inner.txt")) assert not os.path.exists(os.path.join(received, "external", "away.txt")), \ "-x must not cross into the mounted filesystem" result, _ = run_client(source, dest_plain, port=shared_server.port) assert result.returncode == 0, f"plain sync failed: {result.stderr[:200]}" received_plain = get_dest_received_dir(dest_plain, source) assert os.path.isfile(os.path.join(received_plain, "external", "away.txt")), \ "without -x the mounted subtree must be transferred" finally: if mounted: umount = subprocess.run(["umount", mountpoint], capture_output=True, text=True) if umount.returncode != 0: unmount_error = umount.stderr.strip() if unmount_error: pytest.fail(f"test mountpoint {mountpoint} still mounted after umount: {unmount_error}") def _walk_tmp_files(root): """Recursively list *.tmp* leftovers under root (empty if root missing).""" leftovers = [] if not os.path.isdir(root): return leftovers for base, _, files in os.walk(root): for name in files: if ".tmp." in name: leftovers.append(os.path.join(base, name)) return leftovers class TestTempDir: """--temp-dir=DIR puts the receiver's temporary working copies in a scratch directory below the destination root and atomically renames each completed file into its final destination. Files sharing a basename across directories exercise the flat scratch namespace.""" def _make_source(self, name): source = os.path.join(TEST_DATA_DIR, name) clean_dir(source) entries = { "top.txt": b"top level\n", "sub/file.txt": b"nested file\n" * 20, "other/file.txt": b"other nested file\n", "sub/deep.bin": bytes(range(256)) * 8, } for rel, content in entries.items(): full = os.path.join(source, rel) os.makedirs(os.path.dirname(full), exist_ok=True) with open(full, "wb") as fh: fh.write(content) return source def _assert_clean_scratch(self, scratch): assert os.path.isdir(scratch), f"scratch dir {scratch} was not created" leftovers = _walk_tmp_files(scratch) assert leftovers == [], f"leftover temp files in scratch dir: {leftovers}" @pytest.mark.parametrize("mt", [False, True]) def test_temp_dir_scratch(self, shared_server, mt): source = self._make_source("tempdir_src") dest = os.path.join(TEST_DATA_DIR, "tempdir_dst") clean_dir(dest) flags = ["--temp-dir=scratch"] + (["-m"] if mt else []) result, _ = run_client(source, dest, flags=flags, port=shared_server.port) assert result.returncode == 0, f"temp-dir sync failed: {result.stderr[:200]}" received = get_dest_received_dir(dest, source) mismatches, missing = verify_transfer(source, received) assert not missing, f"Missing: {missing}" assert not mismatches, f"Mismatch: {mismatches}" self._assert_clean_scratch(os.path.join(dest, "scratch")) def test_default_behavior_has_no_scratch_dir(self, shared_server): source = self._make_source("tempdir_default_src") dest = os.path.join(TEST_DATA_DIR, "tempdir_default_dst") clean_dir(dest) result, _ = run_client(source, dest, port=shared_server.port) assert result.returncode == 0, f"Default sync failed: {result.stderr[:200]}" received = get_dest_received_dir(dest, source) mismatches, missing = verify_transfer(source, received) assert not missing, f"Missing: {missing}" assert not mismatches, f"Mismatch: {mismatches}" assert not os.path.exists(os.path.join(dest, "scratch")) def test_temp_dir_ignored_with_inplace(self, shared_server): """--inplace writes directly into the destination; --temp-dir must not redirect those writes into a scratch dir.""" source = self._make_source("tempdir_inplace_src") dest = os.path.join(TEST_DATA_DIR, "tempdir_inplace_dst") clean_dir(dest) result, _ = run_client(source, dest, flags=["--inplace", "--temp-dir=scratch"], port=shared_server.port) assert result.returncode == 0, f"inplace+temp-dir sync failed: {result.stderr[:200]}" received = get_dest_received_dir(dest, source) mismatches, missing = verify_transfer(source, received) assert not missing, f"Missing: {missing}" assert not mismatches, f"Mismatch: {mismatches}" assert not os.path.exists(os.path.join(dest, "scratch")), \ "--inplace wrote through the scratch dir" def test_temp_dir_ignored_with_partial_dir(self, shared_server): """--partial --partial-dir already stages in a separate directory; --temp-dir must not be used on top of it.""" source = self._make_source("tempdir_partial_src") dest = os.path.join(TEST_DATA_DIR, "tempdir_partial_dst") clean_dir(dest) result, _ = run_client(source, dest, flags=["--partial", "--partial-dir", ".partial", "--temp-dir=scratch"], port=shared_server.port) assert result.returncode == 0, f"partial+temp-dir sync failed: {result.stderr[:200]}" received = get_dest_received_dir(dest, source) mismatches, missing = verify_transfer(source, received) assert not missing, f"Missing: {missing}" assert not mismatches, f"Mismatch: {mismatches}" partial = os.path.join(dest, ".partial", os.path.relpath(os.path.join(source, "top.txt"), os.path.sep)) assert not os.path.exists(partial), "completed file remained under the partial dir" assert not os.path.exists(os.path.join(dest, "scratch")), \ "--partial-dir wrote through the scratch dir" def test_temp_dir_escape_rejected(self, shared_server): source = self._make_source("tempdir_escape_src") dest = os.path.join(TEST_DATA_DIR, "tempdir_escape_dst") clean_dir(dest) # "../escape" would resolve one level above the destination root. outside = os.path.join(TEST_DATA_DIR, "escape") assert not os.path.lexists(outside) result, _ = run_client(source, dest, flags=["--temp-dir=../escape"], port=shared_server.port) assert result.returncode != 0, "relative escaping --temp-dir was not rejected" assert not os.path.lexists(outside), "file created outside the destination root" clean_dir(dest) abs_escape = os.path.join(TEST_DATA_DIR, "abs_escape_probe") assert not os.path.lexists(abs_escape) result, _ = run_client(source, dest, flags=["--temp-dir", abs_escape], port=shared_server.port) assert result.returncode != 0, "absolute --temp-dir was not rejected" assert not os.path.lexists(abs_escape), "file created outside the destination root" def _source_files(): """All source paths (absolute) that a transfer would send right now.""" return [ os.path.join(root, name) for root, _dirs, names in os.walk(SOURCE_DIR) for name in names ] class TestListOnly: """--list-only prints every transfer candidate and changes nothing.""" def test_list_only_prints_each_file_and_does_not_transfer(self): clean_dir(DEST_DIR) result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["--list-only"]) assert result.returncode == 0, f"list-only failed: {result.stderr[:200]}" for full_path in _source_files(): assert full_path in result.stdout, f"list-only omitted {full_path}" received = get_dest_received_dir(DEST_DIR, SOURCE_DIR) assert not os.path.exists(received), "list-only wrote to the destination" def test_list_only_with_dry_run_does_not_error(self): clean_dir(DEST_DIR) result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["--list-only", "--dry-run"]) assert result.returncode == 0, f"list-only -n failed: {result.stderr[:200]}" received = get_dest_received_dir(DEST_DIR, SOURCE_DIR) assert not os.path.exists(received) def test_list_only_multithreaded(self): clean_dir(DEST_DIR) result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["--list-only", "-m"]) assert result.returncode == 0, f"list-only -m failed: {result.stderr[:200]}" for full_path in _source_files(): assert full_path in result.stdout, f"list-only -m omitted {full_path}" received = get_dest_received_dir(DEST_DIR, SOURCE_DIR) assert not os.path.exists(received), "list-only -m wrote to the destination" class TestItemizeChanges: """-i/--itemize-changes prints rsync-style lines only for files sent.""" def test_first_run_prints_sent_lines(self, shared_server): clean_dir(DEST_DIR) result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M", "-i"], port=shared_server.port) assert result.returncode == 0, f"itemize sync failed: {result.stderr[:200]}" sent_lines = {">f+++++++++ " + p for p in _source_files()} assert sent_lines <= set(result.stdout.splitlines()), ( f"missing itemize lines; got {result.stdout[:500]}" ) def test_incremental_second_run_prints_no_line_for_unchanged(self, shared_server): clean_dir(DEST_DIR) result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port) assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}" result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M", "-i", "--incremental"], port=shared_server.port) assert result.returncode == 0, f"incremental itemize failed: {result.stderr[:200]}" itemized = [line for line in result.stdout.splitlines() if line and line[0] in ">.f+++++++++ " + p for p in _source_files()} assert sent_lines <= set(result.stdout.splitlines()), ( f"missing itemize lines in -m mode; got {result.stdout[:500]}" ) def test_dry_run_with_itemize_does_not_error(self): clean_dir(DEST_DIR) result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-i", "--dry-run"]) assert result.returncode == 0, f"dry-run -i failed: {result.stderr[:200]}" def test_changed_file_on_second_incremental_run_prints_exactly_one_line(self, shared_server): """A changed file itemizes exactly once on an incremental rerun while unchanged files print nothing (no double emission).""" source = os.path.join(TEST_DATA_DIR, "itemize_change_src") dest = os.path.join(TEST_DATA_DIR, "itemize_change_dst") clean_dir(source) clean_dir(dest) changed = os.path.join(source, "changed.txt") untouched = os.path.join(source, "untouched.txt") with open(changed, "wb") as fh: fh.write(b"original\n") with open(untouched, "wb") as fh: fh.write(b"stable\n") result, _ = run_client(source, dest, flags=["-M"], port=shared_server.port) assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}" with open(changed, "wb") as fh: fh.write(b"edited payload\n") result, _ = run_client(source, dest, flags=["-M", "-i", "--incremental"], port=shared_server.port) assert result.returncode == 0, f"incremental itemize failed: {result.stderr[:200]}" itemized = [line for line in result.stdout.splitlines() if line.startswith(">f")] assert itemized == [">f+++++++++ " + changed], ( f"expected exactly one itemize line for {changed}, got {itemized}" ) received = get_dest_received_dir(dest, source) assert _read_file(os.path.join(received, "changed.txt")) == b"edited payload\n" assert _read_file(os.path.join(received, "untouched.txt")) == b"stable\n" class TestOutFormat: """--out-format prints a line per transferred file using the template.""" def test_out_format_path_and_size(self, shared_server): clean_dir(DEST_DIR) result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["--out-format=%f %l"], port=shared_server.port) assert result.returncode == 0, f"out-format sync failed: {result.stderr[:200]}" expected = {f"{p} {os.path.getsize(p)}" for p in _source_files()} got = set(result.stdout.splitlines()) assert expected <= got, f"out-format lines missing: expected {len(expected)} got {len(got)}" def test_out_format_multithreaded_matches_single(self, shared_server): clean_dir(DEST_DIR) result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["--out-format=%f %l", "-m"], port=shared_server.port) assert result.returncode == 0, f"out-format -m sync failed: {result.stderr[:200]}" expected = {f"{p} {os.path.getsize(p)}" for p in _source_files()} got = set(result.stdout.splitlines()) assert expected <= got, f"out-format -m lines missing: {result.stdout[:500]}" class TestLogFileFormat: """--log-file plus --log-file-format writes per-file lines to the log.""" def test_log_file_format_writes_transferred_files(self, shared_server): clean_dir(DEST_DIR) log_path = os.path.join(TEST_DATA_DIR, "itemize_transfer.log") if os.path.exists(log_path): os.unlink(log_path) result, _ = run_client( SOURCE_DIR, DEST_DIR, flags=["--log-file", log_path, "--log-file-format=%f %l"], port=shared_server.port, ) assert result.returncode == 0, f"log-file sync failed: {result.stderr[:200]}" assert os.path.exists(log_path), "--log-file created no log" with open(log_path, encoding="utf-8", errors="replace") as fh: content = fh.read() expected = {f"{p} {os.path.getsize(p)}" for p in _source_files()} for line in expected: assert line in content, f"log file missing {line!r}" def test_log_file_format_multithreaded_writes_transferred_files(self, shared_server): source = os.path.join(TEST_DATA_DIR, "itemize_log_mt_src") dest = os.path.join(TEST_DATA_DIR, "itemize_log_mt_dst") clean_dir(source) clean_dir(dest) files = {"a.txt": b"alpha\n", "b.txt": b"beta\n"} for rel, data in files.items(): with open(os.path.join(source, rel), "wb") as fh: fh.write(data) log_path = os.path.join(TEST_DATA_DIR, "itemize_mt.log") if os.path.exists(log_path): os.unlink(log_path) result, _ = run_client( source, dest, flags=["--log-file", log_path, "--log-file-format=%f %l", "-m"], port=shared_server.port, ) assert result.returncode == 0, f"log-file -m sync failed: {result.stderr[:200]}" assert os.path.exists(log_path), "--log-file created no log" with open(log_path, encoding="utf-8", errors="replace") as fh: content = fh.read() expected = {f"{os.path.join(source, rel)} {len(data)}" for rel, data in files.items()} for line in expected: assert line in content, f"log file (-m) missing {line!r}" class TestDelayUpdates: """--delay-updates stages every updated file under a private 0700 staging directory inside the receive root and atomically publishes all of them only after the whole transfer succeeds.""" STAGING = ".fastsync-stage" def _make_source(self, name): source = os.path.join(TEST_DATA_DIR, name) clean_dir(source) entries = { "top.txt": b"top level\n", "sub/deep.txt": b"deeply nested file\n", "sub/another.txt": b"another nested file\n" * 20, "binary.bin": bytes(range(256)) * 4, } for rel, content in entries.items(): full = os.path.join(source, rel) os.makedirs(os.path.dirname(full), exist_ok=True) with open(full, "wb") as fh: fh.write(content) return source @pytest.mark.parametrize("mt", [False, True]) def test_delay_updates_matches_plain_transfer(self, shared_server, mt): source = self._make_source("delay_match_src") plain_dest = os.path.join(TEST_DATA_DIR, "delay_match_plain_dst") delay_dest = os.path.join(TEST_DATA_DIR, "delay_match_delay_dst") clean_dir(plain_dest) clean_dir(delay_dest) result, _ = run_client(source, plain_dest, port=shared_server.port) assert result.returncode == 0, f"plain sync failed: {result.stderr[:200]}" flags = ["--delay-updates"] + (["-m"] if mt else []) result, _ = run_client(source, delay_dest, flags=flags, port=shared_server.port) assert result.returncode == 0, f"delay-updates sync failed: {result.stderr[:200]}" plain_received = get_dest_received_dir(plain_dest, source) delay_received = get_dest_received_dir(delay_dest, source) mismatches, missing = verify_transfer(source, delay_received) assert not missing, f"Missing: {missing}" assert not mismatches, f"Mismatch: {mismatches}" for root, _dirs, files in os.walk(delay_received): for name in files: rel = os.path.relpath(os.path.join(root, name), delay_received) assert filecmp.cmp(os.path.join(plain_received, rel), os.path.join(delay_received, rel), shallow=False), rel assert not os.path.isdir(os.path.join(delay_dest, self.STAGING)), \ "staging directory left behind after a successful delayed transfer" @pytest.mark.parametrize("mt", [False, True]) def test_delay_updates_incremental_rerun_no_leftovers(self, shared_server, mt): source = self._make_source("delay_rerun_src") dest = os.path.join(TEST_DATA_DIR, "delay_rerun_dst") clean_dir(dest) flags = ["--delay-updates", "-M", "--incremental"] + (["-m"] if mt else []) result, _ = run_client(source, dest, flags=flags, port=shared_server.port) assert result.returncode == 0, f"first delayed sync failed: {result.stderr[:200]}" received = get_dest_received_dir(dest, source) mismatches, missing = verify_transfer(source, received) assert not missing and not mismatches assert not os.path.isdir(os.path.join(dest, self.STAGING)) result, _ = run_client(source, dest, flags=flags, port=shared_server.port) assert result.returncode == 0, f"second delayed sync failed: {result.stderr[:200]}" assert not os.path.isdir(os.path.join(dest, self.STAGING)), \ "fully-skipped delayed run left a staging directory" @pytest.mark.parametrize("mt", [False, True]) def test_remove_source_files_with_delay_updates(self, shared_server, mt): source = self._make_source("delay_rsf_src") dest = os.path.join(TEST_DATA_DIR, "delay_rsf_dst") clean_dir(dest) flags = ["--remove-source-files", "--delay-updates"] + (["-m"] if mt else []) result, _ = run_client(source, dest, flags=flags, port=shared_server.port) assert result.returncode == 0, f"delayed remove-source sync failed: {result.stderr[:200]}" # Sources are removed only after the receiver published every file. for root, _dirs, files in os.walk(source): assert files == [], f"source files survived delayed remove-source-files: {files}" received = get_dest_received_dir(dest, source) assert os.path.isfile(os.path.join(received, "top.txt")) assert os.path.isfile(os.path.join(received, "sub", "deep.txt")) assert not os.path.isdir(os.path.join(dest, self.STAGING)) @pytest.mark.parametrize("mt", [False, True]) def test_delete_with_delay_updates(self, mt): """--delete runs before publication, so the delete walker must not treat the staging directory as a set of extras: a changed file must still be published after genuine extras are removed. Uses its own server started with --allow-delete (the shared session server refuses deletion).""" source = os.path.join(TEST_DATA_DIR, "delay_delete_src") dest = os.path.join(TEST_DATA_DIR, "delay_delete_dst") clean_dir(source) clean_dir(dest) with open(os.path.join(source, "f.txt"), "wb") as fh: fh.write(b"AAAA") with open(os.path.join(source, "extra.txt"), "wb") as fh: fh.write(b"seed extra") with ServerManager() as server: server.start(extra_args=["--allow-delete"]) result, _ = run_client(source, dest, port=server.port) assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}" received = get_dest_received_dir(dest, source) assert _read_file(os.path.join(received, "extra.txt")) == b"seed extra" # Second source state: f.txt changed, extra.txt removed from source. with open(os.path.join(source, "f.txt"), "wb") as fh: fh.write(b"BBBB") os.remove(os.path.join(source, "extra.txt")) flags = ["--delete", "--delay-updates"] + (["-m"] if mt else []) result, _ = run_client(source, dest, flags=flags, port=server.port) assert result.returncode == 0, \ f"delete+delay-updates sync failed: {result.stderr[:200]}" assert _read_file(os.path.join(received, "f.txt")) == b"BBBB", \ "changed file was not published after deletion" assert not os.path.exists(os.path.join(received, "extra.txt")), \ "genuine extra file was not deleted" assert not os.path.isdir(os.path.join(dest, self.STAGING)) def test_delay_updates_rejects_reserved_backup_dir(self): """--backup-dir equal to the internal staging name must be rejected so an old backup can never be silently installed as the "new" file.""" source = self._make_source("delay_reserved_bak_src") for variant, suffix in (("bare", ""), ("slash", "/")): dest = os.path.join(TEST_DATA_DIR, f"delay_reserved_bak_{variant}_dst") clean_dir(dest) flags = ["--delay-updates", "--backup", "--backup-dir", ".fastsync-stage" + suffix] result, _ = run_client(source, dest, flags=flags, port=None) assert result.returncode != 0, \ f"reserved --backup-dir '{suffix}' was accepted" assert not os.path.isdir(os.path.join(dest, self.STAGING)), \ "staging directory created by a rejected run" @pytest.mark.parametrize("remove_source_files", [False, True]) @pytest.mark.parametrize("mt", [False, True]) def test_mid_publish_failure_keeps_published_no_rollback(self, shared_server, mt, remove_source_files): """A stage->publish rename failing part way through publication must fail the whole transfer, keep the already-published top-level file (no rollback), leave the not-yet-published nested file absent, and clean up the staging area. A regular file is planted where the final "sub" directory must be created, so the nested rename fails (mkdir over a file is impossible even for root) while the top-level file, which is always staged first, publishes. With --remove-source-files the sender must keep every source because no success/outcome frame is ever sent.""" source = os.path.join(TEST_DATA_DIR, "delay_mid_src") dest = os.path.join(TEST_DATA_DIR, "delay_mid_dst") clean_dir(source) clean_dir(dest) top_path = os.path.join(source, "top.txt") deep_path = os.path.join(source, "sub", "deep.txt") with open(top_path, "wb") as fh: fh.write(b"top payload\n") os.makedirs(os.path.dirname(deep_path)) with open(deep_path, "wb") as fh: fh.write(b"deep payload\n") received = get_dest_received_dir(dest, source) os.makedirs(received) with open(os.path.join(received, "sub"), "wb") as fh: fh.write(b"blocks the nested destination directory") flags = ["--delay-updates"] + (["-m"] if mt else []) if remove_source_files: flags += ["--remove-source-files"] result, _ = run_client(source, dest, flags=flags, port=shared_server.port) assert result.returncode != 0, "blocked nested publish did not fail" # The top-level file was published before the nested rename failed and # is intentionally NOT rolled back. assert _read_file(os.path.join(received, "top.txt")) == b"top payload\n" # The nested file was never published. assert not os.path.lexists(os.path.join(received, "sub", "deep.txt")), \ "nested file appeared despite a failed publish" assert not os.path.isdir(os.path.join(dest, self.STAGING)), \ "staging leftovers after a failed mid-publish" # Sources survive: no success frame was sent, so a remove-source-files # sender must not delete anything. assert os.path.isfile(top_path) assert os.path.isfile(deep_path) @pytest.mark.parametrize("mt", [False, True]) def test_remove_source_files_keeps_receiver_skipped_source(self, shared_server, mt): """With --delay-updates + --ignore-existing a receiver-skipped source must survive (its outcome is sent only after publication) while a freshly delivered file is published and its source removed.""" source = os.path.join(TEST_DATA_DIR, "delay_rsf_skip_src") dest = os.path.join(TEST_DATA_DIR, "delay_rsf_skip_dst") clean_dir(source) clean_dir(dest) with open(os.path.join(source, "keep.txt"), "wb") as fh: fh.write(b"existing on dest") result, _ = run_client(source, dest, port=shared_server.port) assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}" with open(os.path.join(source, "keep.txt"), "wb") as fh: fh.write(b"changed on source") with open(os.path.join(source, "deliver.txt"), "wb") as fh: fh.write(b"new file") flags = ["--remove-source-files", "--ignore-existing", "--delay-updates"] + (["-m"] if mt else []) result, _ = run_client(source, dest, flags=flags, port=shared_server.port) assert result.returncode == 0, f"delayed skip sync failed: {result.stderr[:200]}" # keep.txt already existed at the destination: receiver skip -> source stays. assert os.path.isfile(os.path.join(source, "keep.txt")), \ "receiver-skipped source was removed despite --ignore-existing" # deliver.txt was new: staged, published, and its source removed. assert not os.path.isfile(os.path.join(source, "deliver.txt")), \ "published source was not removed" received = get_dest_received_dir(dest, source) assert not os.path.isdir(os.path.join(dest, self.STAGING)) def test_delay_updates_rejects_inplace(self): source = self._make_source("delay_inplace_src") dest = os.path.join(TEST_DATA_DIR, "delay_inplace_dst") clean_dir(dest) result, _ = run_client(source, dest, flags=["--delay-updates", "--inplace"]) assert result.returncode != 0, "--inplace with --delay-updates was accepted" assert not os.path.isdir(os.path.join(dest, self.STAGING)) class TestFilesFrom: """--files-from transfers exactly the listed files; a listed directory transfers its whole subtree. The manifest (and thus --delete) derives from what was actually sent.""" def _make_relative_source(name): """A small tree used by the -R/--dirs/--no-implied-dirs tests.""" source = os.path.join(TEST_DATA_DIR, name) clean_dir(source) entries = { "top.txt": b"top\n", "a/b.txt": b"nested\n", "sub/x.txt": b"x\n", "sub/y.txt": b"y\n", "dir1/keep.txt": b"dir content\n", } for rel, content in entries.items(): full = os.path.join(source, rel) os.makedirs(os.path.dirname(full), exist_ok=True) with open(full, "wb") as fh: fh.write(content) return source def _write_rel_list(rel_text): path = os.path.join(TEST_DATA_DIR, "rel_list.txt") with open(path, "wb") as fh: fh.write(rel_text) return path class TestRelativeFilesFrom: """-R/--relative with --files-from keeps each listed entry's bare relative destination path below the destination root instead of mirroring the full source path. Without -R the layout is unchanged (full source mirror).""" @pytest.mark.parametrize("mt", [False, True]) def test_relative_files_from_keeps_relative_layout(self, shared_server, mt): source = _make_relative_source("rel_src") dest = os.path.join(TEST_DATA_DIR, "rel_dst") clean_dir(dest) lst = _write_rel_list(b"top.txt\nsub/x.txt\n") flags = ["--files-from", lst, "-R"] + (["-m"] if mt else []) result, _ = run_client(source, dest, flags=flags, port=shared_server.port) assert result.returncode == 0, f"-R files-from sync failed: {result.stderr[:200]}" assert _read_file(os.path.join(dest, "sub", "x.txt")) == b"x\n", \ "listed file must land at /sub/x.txt" assert _read_file(os.path.join(dest, "top.txt")) == b"top\n", \ "top-level listed file must land at /top.txt" assert not os.path.exists(os.path.join(dest, "sub", "y.txt")) # The source-root mirror must not be reproduced under -R. assert not os.path.exists(get_dest_received_dir(dest, source)), \ "-R must not mirror the full source path" @pytest.mark.parametrize("mt", [False, True]) def test_relative_without_files_from_has_no_effect(self, shared_server, mt): """-R alone (no --files-from) must leave the normal full-source mirror layout untouched.""" source = _make_relative_source("rel_only_src") dest = os.path.join(TEST_DATA_DIR, "rel_only_dst") clean_dir(dest) flags = ["-R"] + (["-m"] if mt else []) result, _ = run_client(source, dest, flags=flags, port=shared_server.port) assert result.returncode == 0, f"-R alone sync failed: {result.stderr[:200]}" received = get_dest_received_dir(dest, source) mismatches, missing = verify_transfer(source, received) assert not missing and not mismatches @pytest.mark.parametrize("mt", [False, True]) def test_without_relative_layout_unchanged(self, shared_server, mt): source = _make_relative_source("rel_noR_src") dest = os.path.join(TEST_DATA_DIR, "rel_noR_dst") clean_dir(dest) lst = _write_rel_list(b"sub/x.txt\n") flags = ["--files-from", lst] + (["-m"] if mt else []) result, _ = run_client(source, dest, flags=flags, port=shared_server.port) assert result.returncode == 0, f"files-from sync failed: {result.stderr[:200]}" received = get_dest_received_dir(dest, source) assert _read_file(os.path.join(received, "sub", "x.txt")) == b"x\n", \ "without -R the full source mirror layout is preserved" assert not os.path.exists(os.path.join(dest, "sub")), \ "bare relative layout must not appear without -R" def test_relative_delete_manifest_stays_consistent(self): """--delete derives from the sent (-R) relative paths, so a later subset run removes unlisted relative entries but keeps listed ones.""" source = _make_relative_source("rel_del_src") dest = os.path.join(TEST_DATA_DIR, "rel_del_dst") clean_dir(dest) with ServerManager() as server: server.start(extra_args=["--allow-delete"]) lst = _write_rel_list(b"sub/x.txt\nsub/y.txt\n") result, _ = run_client(source, dest, flags=["--files-from", lst, "-R"], port=server.port) assert result.returncode == 0, f"seed -R sync failed: {result.stderr[:200]}" assert os.path.isfile(os.path.join(dest, "sub", "y.txt")) subset = _write_rel_list(b"sub/x.txt\n") result, _ = run_client(source, dest, flags=["--files-from", subset, "-R", "--delete"], port=server.port) assert result.returncode == 0, f"-R delete sync failed: {result.stderr[:200]}" assert os.path.isfile(os.path.join(dest, "sub", "x.txt")), "listed file was deleted" assert not os.path.exists(os.path.join(dest, "sub", "y.txt")), \ "unlisted relative file was not deleted" class TestMissingArgs: """--ignore-missing-args / --delete-missing-args: a --files-from entry that does not exist under the source is skipped instead of failing the run, and (delete-missing) its destination mirror is removed receiver-side. Following rsync, --delete-missing-args implies --ignore-missing-args but is independent of --delete: unrelated extras stay unless --delete is also given, and the missing-args deletion (an explicit user request) is never blocked by filter-exclusion protection.""" def _make_source(self, name): source = os.path.join(TEST_DATA_DIR, name) clean_dir(source) for rel, content in { "a.txt": b"a\n", "sub/b.txt": b"b\n", "keep.txt": b"keep\n", "prot/kept.txt": b"kept\n", }.items(): full = os.path.join(source, rel) os.makedirs(os.path.dirname(full), exist_ok=True) with open(full, "wb") as fh: fh.write(content) return source @pytest.mark.parametrize("mt", [False, True]) def test_missing_entry_is_hard_error_before_transfer(self, shared_server, mt): source = self._make_source("mg_default_src") dest = os.path.join(TEST_DATA_DIR, "mg_default_dst") clean_dir(dest) lst = _write_rel_list(b"a.txt\ngone.txt\nsub/b.txt\n") flags = ["--files-from", lst] + (["-m"] if mt else []) result, _ = run_client(source, dest, flags=flags, port=shared_server.port) assert result.returncode != 0, "a listed-but-missing entry did not fail the run" assert "gone.txt" in (result.stderr or result.stdout) received = get_dest_received_dir(dest, source) assert not os.path.isfile(os.path.join(received, "a.txt")), \ "the transfer started despite the missing-entry hard error" @pytest.mark.parametrize("mt", [False, True]) def test_ignore_missing_args_transfers_the_rest(self, shared_server, mt): source = self._make_source("mg_ignore_src") dest = os.path.join(TEST_DATA_DIR, "mg_ignore_dst") clean_dir(dest) lst = _write_rel_list(b"a.txt\ngone.txt\nsub/b.txt\n") flags = ["--files-from", lst, "--ignore-missing-args"] + (["-m"] if mt else []) result, _ = run_client(source, dest, flags=flags, port=shared_server.port) assert result.returncode == 0, f"ignore-missing-args sync failed: {result.stderr[:300]}" received = get_dest_received_dir(dest, source) assert _read_file(os.path.join(received, "a.txt")) == b"a\n" assert _read_file(os.path.join(received, "sub", "b.txt")) == b"b\n" assert not os.path.exists(os.path.join(received, "gone.txt")), \ "nothing was transferred for the missing entry" assert "--ignore-missing-args" in (result.stderr or result.stdout), \ "the skipped entry must be observable (not a silent no-op)" @pytest.mark.parametrize("mt", [False, True]) def test_all_missing_entries_succeed_transferring_nothing(self, shared_server, mt): source = self._make_source("mg_all_missing_src") dest = os.path.join(TEST_DATA_DIR, "mg_all_missing_dst") clean_dir(dest) lst = _write_rel_list(b"gone1.txt\ngone2.txt\n") flags = ["--files-from", lst, "--ignore-missing-args"] + (["-m"] if mt else []) result, _ = run_client(source, dest, flags=flags, port=shared_server.port) assert result.returncode == 0, \ f"all-missing run should succeed (rsync parity): {result.stderr[:300]}" received = get_dest_received_dir(dest, source) assert not os.path.exists(os.path.join(received, "gone1.txt")) @pytest.mark.parametrize("mt", [False, True]) def test_empty_list_stays_a_hard_error(self, shared_server, mt): source = self._make_source("mg_empty_src") dest = os.path.join(TEST_DATA_DIR, "mg_empty_dst") clean_dir(dest) lst = _write_rel_list(b"") flags = ["--files-from", lst, "--ignore-missing-args"] + (["-m"] if mt else []) result, _ = run_client(source, dest, flags=flags, port=shared_server.port) assert result.returncode != 0, "an empty --files-from list must stay a hard error" assert "contains no entries" in (result.stderr or result.stdout) @pytest.mark.parametrize("mt", [False, True]) def test_delete_missing_removes_mirror_not_unrelated(self, mt): """-R layout: --delete-missing-args deletes exactly the missing entry's destination mirror (bare relative path) and leaves unrelated extras untouched; with --delete also present the unrelated extras go too.""" source = self._make_source("mg_del_src") dest = os.path.join(TEST_DATA_DIR, "mg_del_dst") clean_dir(dest) with ServerManager() as server: server.start(extra_args=["--allow-delete"]) seed = _write_rel_list(b"a.txt\nsub/b.txt\n") result, _ = run_client(source, dest, flags=["--files-from", seed, "-R"] + (["-m"] if mt else []), port=server.port) assert result.returncode == 0, f"seed -R sync failed: {result.stderr[:200]}" assert os.path.isfile(os.path.join(dest, "a.txt")) assert os.path.isfile(os.path.join(dest, "sub", "b.txt")) # Plant the missing entry's destination mirror and an unrelated extra. with open(os.path.join(dest, "gone.txt"), "w") as fh: fh.write("stale mirror") with open(os.path.join(dest, "unrelated.txt"), "w") as fh: fh.write("unrelated") lst = _write_rel_list(b"a.txt\ngone.txt\nsub/b.txt\n") flags = ["--files-from", lst, "-R", "--delete-missing-args"] + (["-m"] if mt else []) result, _ = run_client(source, dest, flags=flags, port=server.port) assert result.returncode == 0, f"delete-missing sync failed: {result.stderr[:300]}" assert not os.path.exists(os.path.join(dest, "gone.txt")), \ "the missing entry's destination mirror was not deleted" assert os.path.isfile(os.path.join(dest, "unrelated.txt")), \ "--delete-missing-args removed an unrelated extra (only --delete may)" assert os.path.isfile(os.path.join(dest, "a.txt")) assert os.path.isfile(os.path.join(dest, "sub", "b.txt")) # Now with --delete the unrelated extra is an ordinary extra and must go. lst2 = _write_rel_list(b"a.txt\ngone.txt\nsub/b.txt\n") flags2 = ["--files-from", lst2, "-R", "--delete-missing-args", "--delete"] + \ (["-m"] if mt else []) result, _ = run_client(source, dest, flags=flags2, port=server.port) assert result.returncode == 0, f"delete-missing + delete sync failed: {result.stderr[:300]}" assert not os.path.exists(os.path.join(dest, "unrelated.txt")), \ "--delete did not remove the unrelated extra" assert not os.path.exists(os.path.join(dest, "gone.txt")) assert os.path.isfile(os.path.join(dest, "a.txt")) @pytest.mark.parametrize("mt", [False, True]) def test_delete_missing_mirror_outside_relative_layout(self, mt): """Without -R the missing entry's mirror mirrors the full source path below the destination root, exactly like a present sibling's.""" source = self._make_source("mg_del_nor_src") dest = os.path.join(TEST_DATA_DIR, "mg_del_nor_dst") clean_dir(dest) with ServerManager() as server: server.start(extra_args=["--allow-delete"]) # Full-tree seed places every current source file in the mirrored layout. result, _ = run_client(source, dest, flags=["--delete"], port=server.port) assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}" received = get_dest_received_dir(dest, source) assert os.path.isfile(os.path.join(received, "a.txt")) # Plant a stale mirror for an entry not (yet) on the source. with open(os.path.join(received, "gone.txt"), "w") as fh: fh.write("stale") lst = _write_rel_list(b"a.txt\ngone.txt\n") result, _ = run_client(source, dest, flags=["--files-from", lst, "--delete-missing-args"], port=server.port) assert result.returncode == 0, f"delete-missing no-R sync failed: {result.stderr[:300]}" assert not os.path.exists(os.path.join(received, "gone.txt")), \ "the full-source-mirror path of the missing entry was not deleted" assert os.path.isfile(os.path.join(received, "a.txt")) @pytest.mark.parametrize("mt", [False, True]) def test_delete_missing_args_not_blocked_by_exclude_protection(self, mt): """A missing-arg mirror that sits under a filter-excluded directory is an explicit user request, so --delete-missing-args removes it even though an ordinary --delete honours the exclusion protection (rsync parity). Uses the non-relative layout: exclusion protection is only recorded there.""" source = self._make_source("mg_excl_src") dest = os.path.join(TEST_DATA_DIR, "mg_excl_dst") clean_dir(dest) with ServerManager() as server: server.start(extra_args=["--allow-delete"]) # Full-tree seed mirrors the whole source below the destination root. result, _ = run_client(source, dest, port=server.port) assert result.returncode == 0, f"seed failed: {result.stderr[:200]}" received = get_dest_received_dir(dest, source) assert os.path.isfile(os.path.join(received, "prot", "kept.txt")) # A stale mirror under the (now excluded) prot/ directory, plus an extra. with open(os.path.join(received, "prot", "gone.txt"), "w") as fh: fh.write("stale") with open(os.path.join(received, "extra.txt"), "w") as fh: fh.write("extra") lst = _write_rel_list(b"a.txt\nprot/gone.txt\n") flags = ["--files-from", lst, "--filter=- prot/", "--delete-missing-args", "--delete"] + (["-m"] if mt else []) result, _ = run_client(source, dest, flags=flags, port=server.port) assert result.returncode == 0, f"delete-missing exclude sync failed: {result.stderr[:300]}" assert not os.path.exists(os.path.join(received, "prot", "gone.txt")), \ "the explicit missing-arg deletion was blocked by exclusion protection" assert os.path.isfile(os.path.join(received, "prot", "kept.txt")), \ "the excluded-but-present destination file must stay (default protection)" assert not os.path.exists(os.path.join(received, "extra.txt")), \ "--delete did not remove the unrelated extra" assert os.path.isfile(os.path.join(received, "a.txt")) @pytest.mark.parametrize("mt", [False, True]) def test_delete_missing_args_with_delete_before(self, mt): """--delete-before (early delete timing) composes with --delete-missing-args: the exact-path deletions commit with the early manifest, before data, and --delete-before implies --delete (so unrelated extras go too).""" source = self._make_source("mg_early_src") dest = os.path.join(TEST_DATA_DIR, "mg_early_dst") clean_dir(dest) with open(os.path.join(dest, "gone.txt"), "w") as fh: fh.write("stale") with open(os.path.join(dest, "extra.txt"), "w") as fh: fh.write("extra") with ServerManager() as server: server.start(extra_args=["--allow-delete"]) lst = _write_rel_list(b"a.txt\ngone.txt\ngone2.txt\n") flags = ["--files-from", lst, "-R", "--delete-missing-args", "--delete-before"] + \ (["-m"] if mt else []) result, _ = run_client(source, dest, flags=flags, port=server.port) assert result.returncode == 0, f"early delete-missing sync failed: {result.stderr[:300]}" assert not os.path.exists(os.path.join(dest, "gone.txt")), \ "early timing did not remove the missing-arg mirror" assert os.path.isfile(os.path.join(dest, "a.txt")), "a.txt was not transferred" assert not os.path.exists(os.path.join(dest, "extra.txt")), \ "--delete-before implies --delete: unrelated extras must go" @pytest.mark.parametrize("mt", [False, True]) @pytest.mark.parametrize("relative", [False, True]) def test_delete_missing_deep_entry_with_absent_parent(self, mt, relative): """A missing entry whose destination mirror's parent directory does not exist is a no-op (nothing to delete), never a run failure: the exact-path deletions must not abort the --delete extras walk. Covers the -R bare-relative layout and the full source-mirror layout.""" source = self._make_source("mg_deep_src") dest = os.path.join(TEST_DATA_DIR, "mg_deep_dst") clean_dir(dest) rel_flags = ["-R"] if relative else [] with ServerManager() as server: server.start(extra_args=["--allow-delete"]) if relative: target_root = dest else: # Non-relative layout: seed a.txt so the receive-root mirror # tree exists (its sub/ sibling deliberately does not). seed = _write_rel_list(b"a.txt\n") result, _ = run_client(source, dest, flags=["--files-from", seed] + rel_flags, port=server.port) assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}" target_root = get_dest_received_dir(dest, source) assert os.path.isfile(os.path.join(target_root, "a.txt")) with open(os.path.join(target_root, "extra.txt"), "w") as fh: fh.write("extra") lst = _write_rel_list(b"a.txt\nsub/gone.txt\n") flags = ["--files-from", lst, "--delete-missing-args", "--delete"] + rel_flags + \ (["-m"] if mt else []) result, _ = run_client(source, dest, flags=flags, port=server.port) assert result.returncode == 0, \ f"deep missing-entry sync failed: {result.stderr[:300]}" assert _read_file(os.path.join(target_root, "a.txt")) == b"a\n" assert not os.path.exists(os.path.join(target_root, "extra.txt")), \ "--delete extras walk was aborted by the absent-parent missing entry" assert not os.path.exists(os.path.join(target_root, "sub")), \ "the absent parent directory of the missing entry was created" @pytest.mark.parametrize("mt", [False, True]) def test_dirs_missing_entry_skipped_in_scanner(self, shared_server, mt): """--dirs + --files-from: a listed-but-missing entry is skipped in the --dirs generator (which would otherwise hard-fail), transferring the rest of the list.""" source = self._make_source("mg_dirs_src") dest = os.path.join(TEST_DATA_DIR, "mg_dirs_dst") clean_dir(dest) lst = _write_rel_list(b"a.txt\ngone.txt\n") flags = ["--files-from", lst, "--dirs", "-R", "--ignore-missing-args"] + \ (["-m"] if mt else []) result, _ = run_client(source, dest, flags=flags, port=shared_server.port) assert result.returncode == 0, f"--dirs ignore-missing sync failed: {result.stderr[:300]}" assert _read_file(os.path.join(dest, "a.txt")) == b"a\n", \ "the listed present file was not transferred" assert not os.path.exists(os.path.join(dest, "gone.txt")), \ "a directory/file was created for the missing --dirs entry" class TestNoImpliedDirs: """--no-implied-dirs (only meaningful with -R + --files-from) refuses to place a listed file whose parent directory is not itself listed.""" def _make(self): return _make_relative_source("noimplied_src") @pytest.mark.parametrize("mt", [False, True]) def test_implied_dir_only_fails_entry(self, shared_server, mt): source = self._make() dest = os.path.join(TEST_DATA_DIR, "noimplied_dst") clean_dir(dest) lst = _write_rel_list(b"a/b.txt\n") # "a" itself is not listed flags = ["--files-from", lst, "-R", "--no-implied-dirs"] + (["-m"] if mt else []) result, _ = run_client(source, dest, flags=flags, port=shared_server.port) assert result.returncode != 0, "implied parent directory was not rejected" assert "--no-implied-dirs" in (result.stderr or result.stdout) assert not os.path.exists(os.path.join(dest, "a", "b.txt")) @pytest.mark.parametrize("mt", [False, True]) def test_listed_dir_allows_file(self, shared_server, mt): source = self._make() dest = os.path.join(TEST_DATA_DIR, "noimplied_ok_dst") clean_dir(dest) lst = _write_rel_list(b"a\na/b.txt\n") flags = ["--files-from", lst, "-R", "--no-implied-dirs"] + (["-m"] if mt else []) result, _ = run_client(source, dest, flags=flags, port=shared_server.port) assert result.returncode == 0, f"listed dir + file sync failed: {result.stderr[:200]}" assert _read_file(os.path.join(dest, "a", "b.txt")) == b"nested\n" @pytest.mark.parametrize("mt", [False, True]) def test_no_implied_dirs_without_relative_changes_nothing(self, shared_server, mt): source = self._make() dest = os.path.join(TEST_DATA_DIR, "noimplied_noR_dst") clean_dir(dest) lst = _write_rel_list(b"a/b.txt\n") flags = ["--files-from", lst, "--no-implied-dirs"] + (["-m"] if mt else []) result, _ = run_client(source, dest, flags=flags, port=shared_server.port) assert result.returncode == 0, "--no-implied-dirs without -R changed behavior" received = get_dest_received_dir(dest, source) assert _read_file(os.path.join(received, "a", "b.txt")) == b"nested\n" class TestDirs: """-d/--dirs (and the --old-dirs/--old-d aliases) transfer directory entries without recursing into their contents.""" def _make(self): return _make_relative_source("dirs_src") def _assert_only_empty_mirror(self, dest, source): mirror = get_dest_received_dir(dest, source) assert os.path.isdir(mirror), "source-root mirror directory was not created" files = [] for root, _dirs, names in os.walk(mirror): files.extend(os.path.relpath(os.path.join(root, n), mirror) for n in names) assert files == [], f"--dirs descended into contents: {files}" @pytest.mark.parametrize("flag", ["--dirs", "-d", "--old-dirs", "--old-d"]) @pytest.mark.parametrize("mt", [False, True]) def test_dirs_transfers_empty_dir_only(self, shared_server, flag, mt): source = self._make() dest = os.path.join(TEST_DATA_DIR, "dirs_dst") clean_dir(dest) flags = [flag] + (["-m"] if mt else []) result, _ = run_client(source, dest, flags=flags, port=shared_server.port) assert result.returncode == 0, f"{flag} sync failed: {result.stderr[:200]}" self._assert_only_empty_mirror(dest, source) @pytest.mark.parametrize("mt", [False, True]) def test_dirs_with_files_from(self, shared_server, mt): source = self._make() dest = os.path.join(TEST_DATA_DIR, "dirs_ff_dst") clean_dir(dest) # A listed directory is created empty; a listed file is transferred. lst = _write_rel_list(b"dir1\nsub/x.txt\n") flags = ["--files-from", lst, "--dirs", "-R"] + (["-m"] if mt else []) result, _ = run_client(source, dest, flags=flags, port=shared_server.port) assert result.returncode == 0, f"dirs files-from sync failed: {result.stderr[:200]}" assert os.path.isdir(os.path.join(dest, "dir1")), "listed dir was not created" assert not os.path.exists(os.path.join(dest, "dir1", "keep.txt")), \ "--dirs must not descend into a listed directory" assert _read_file(os.path.join(dest, "sub", "x.txt")) == b"x\n", \ "listed file content was not transferred" assert not os.path.exists(os.path.join(dest, "sub", "y.txt")), \ "unlisted file appeared" @pytest.mark.parametrize("mt", [False, True]) def test_dirs_with_files_from_mirror_layout(self, shared_server, mt): """Without -R the dirs+files-from entries still mirror the source path.""" source = self._make() dest = os.path.join(TEST_DATA_DIR, "dirs_ff_noR_dst") clean_dir(dest) lst = _write_rel_list(b"dir1\n") flags = ["--files-from", lst, "--dirs"] + (["-m"] if mt else []) result, _ = run_client(source, dest, flags=flags, port=shared_server.port) assert result.returncode == 0, f"dirs files-from no-R sync failed: {result.stderr[:200]}" received = get_dest_received_dir(dest, source) assert os.path.isdir(os.path.join(received, "dir1")), "mirrored dir entry not created" assert not os.path.exists(os.path.join(received, "dir1", "keep.txt")), \ "--dirs must not descend into a listed directory" assert not os.path.exists(os.path.join(received, "sub")), \ "unlisted subtree appeared" @pytest.mark.parametrize("mt", [False, True]) def test_dirs_chunk_serialization(self, shared_server, mt): """--dirs entries survive the chunk-serialization wire path (type marker round-trips); a listed dir lands empty and a listed file lands with content, with no protocol desync under -s -m.""" source = self._make() dest = os.path.join(TEST_DATA_DIR, "dirs_s_dst") clean_dir(dest) lst = _write_rel_list(b"dir1\nsub/x.txt\n") flags = ["--files-from", lst, "--dirs", "-R", "-s"] + (["-m"] if mt else []) result, _ = run_client(source, dest, flags=flags, port=shared_server.port) assert result.returncode == 0, f"dirs -s sync failed: {result.stderr[:200]}" assert os.path.isdir(os.path.join(dest, "dir1")), "listed dir was not created" assert not os.path.exists(os.path.join(dest, "dir1", "keep.txt")), \ "--dirs must not descend into a listed directory" assert _read_file(os.path.join(dest, "sub", "x.txt")) == b"x\n", \ "listed file content was not transferred" def test_dirs_delete_keeps_transferred_empty_dir(self): """Directory entries appear in the delete manifest, so the empty dir a --dirs run just created is not pruned as an extra by --delete.""" source = self._make() dest = os.path.join(TEST_DATA_DIR, "dirs_del_dst") clean_dir(dest) extra = os.path.join(dest, "extra.txt") with open(extra, "wb") as fh: fh.write(b"delete me") with ServerManager() as server: server.start(extra_args=["--allow-delete"]) result, _ = run_client(source, dest, flags=["--dirs", "--delete"], port=server.port) assert result.returncode == 0, f"--dirs --delete sync failed: {result.stderr[:200]}" assert not os.path.exists(extra), "--delete did not remove the extra file" mirror = get_dest_received_dir(dest, source) assert os.path.isdir(mirror), "transferred empty dir was pruned as an extra" files = [] for root, _dirs, names in os.walk(mirror): files.extend(os.path.relpath(os.path.join(root, n), mirror) for n in names) assert files == [], f"--dirs descended into contents: {files}" def test_dirs_listed_dir_colliding_with_file_fails(self, shared_server): """A listed directory that already exists as a regular file at the destination fails the transfer cleanly instead of clobbering the file.""" source = self._make() dest = os.path.join(TEST_DATA_DIR, "dirs_coll_dst") clean_dir(dest) blocker = os.path.join(dest, "dir1") with open(blocker, "wb") as fh: fh.write(b"blocking file") lst = _write_rel_list(b"dir1\n") result, _ = run_client(source, dest, flags=["--files-from", lst, "--dirs", "-R"], port=shared_server.port) assert result.returncode != 0, "dir entry over an existing file did not fail" assert os.path.isfile(blocker), "blocking regular file was clobbered" class TestMkpath: """--mkpath tells the server to create the destination root directory (and missing leading components) when it does not exist yet; without it a missing destination root fails the transfer.""" @pytest.mark.parametrize("mt", [False, True]) def test_missing_root_fails_without_mkpath(self, mt): source = _make_relative_source("mkpath_fail_src") dest = os.path.join(TEST_DATA_DIR, "mkpath_missing_dst") shutil.rmtree(dest, ignore_errors=True) with ServerManager() as server: server.start() flags = ["-m"] if mt else [] result, _ = run_client(source, dest, flags=flags, port=server.port) assert result.returncode != 0, "missing destination root did not fail without --mkpath" assert not os.path.exists(dest), "missing root was created without --mkpath" @pytest.mark.parametrize("mt", [False, True]) def test_mkpath_creates_missing_root(self, mt): source = _make_relative_source("mkpath_ok_src") dest = os.path.join(TEST_DATA_DIR, "deep", "mkpath_dst") shutil.rmtree(os.path.join(TEST_DATA_DIR, "deep"), ignore_errors=True) with ServerManager() as server: server.start() flags = ["--mkpath"] + (["-m"] if mt else []) result, _ = run_client(source, dest, flags=flags, port=server.port) assert result.returncode == 0, f"--mkpath sync failed: {result.stderr[:200]}" received = get_dest_received_dir(dest, source) assert _read_file(os.path.join(received, "sub", "x.txt")) == b"x\n", \ "file not transferred into the --mkpath-created root" @pytest.mark.parametrize("mkpath", [False, True]) def test_existing_dest_with_trailing_slash(self, shared_server, mkpath): """A destination root written with a trailing slash must keep working: an existing root is accepted both with and without --mkpath.""" source = _make_relative_source("mkpath_trail_src") dest = os.path.join(TEST_DATA_DIR, "mkpath_trail_dst") clean_dir(dest) dest_slash = dest + "/" flags = ["--mkpath"] if mkpath else [] result, _ = run_client(source, dest_slash, flags=flags, port=shared_server.port) assert result.returncode == 0, \ f"trailing-slash dest sync (mkpath={mkpath}) failed: {result.stderr[:200]}" received = get_dest_received_dir(dest, source) assert _read_file(os.path.join(received, "sub", "x.txt")) == b"x\n", \ "file not transferred into the trailing-slash destination root" @pytest.mark.parametrize("mkpath", [False, True]) def test_dest_equal_authorized_root(self, mkpath): """A destination that is exactly the server's authorized root works without --mkpath, and with --mkpath creates no stray / nested directory.""" root = os.path.join(TEST_DATA_DIR, "mkpath_eq_root") clean_dir(root) source = _make_relative_source("mkpath_eq_src") with ServerManager() as server: server.start(extra_args=["--destination-root", root]) flags = ["--mkpath"] if mkpath else [] result, _ = run_client(source, root, flags=flags, port=server.port) assert result.returncode == 0, \ f"dest==authorized-root sync (mkpath={mkpath}) failed: {result.stderr[:200]}" received = get_dest_received_dir(root, source) assert _read_file(os.path.join(received, "sub", "x.txt")) == b"x\n", \ "file not transferred when the dest equals the authorized root" basename = os.path.basename(root.rstrip(os.sep)) assert not os.path.exists(os.path.join(root, basename)), \ "--mkpath created a spurious nested / directory" class TestFilters: """--filter/-C/-F rule layer: excludes prune, ordering is first-match-wins, the default with no matching rule is include, and legacy --exclude remains an independent layer.""" class TestDeleteTiming: """rsync deletion-timing family. --delete-before/--delete-during transmit the keep-set manifest BEFORE any file data (the receiver deletes extras and acks first); --delete/--delete-after/--delete-delay commit deletions only after the whole transfer succeeded. Every timing flag implies --delete.""" def _seed(self, tag): source = os.path.join(TEST_DATA_DIR, f"deltiming_{tag}_src") clean_dir(source) entries = { "top.txt": b"top level\n", "sub/deep.txt": b"deeply nested file\n", } for rel, content in entries.items(): full = os.path.join(source, rel) os.makedirs(os.path.dirname(full), exist_ok=True) with open(full, "wb") as fh: fh.write(content) return source @pytest.mark.parametrize("flag", ["--delete-before", "--delete-during", "--del", "--delete-after", "--delete-delay"]) @pytest.mark.parametrize("mt", [False, True]) def test_flag_removes_extras_on_success(self, flag, mt): """Every timing flag is accepted, implies --delete, and on a successful transfer removes the destination extras exactly like plain --delete.""" source = self._seed("ok") dest = os.path.join(TEST_DATA_DIR, "deltiming_ok_dst") clean_dir(dest) with ServerManager() as server: server.start(extra_args=["--allow-delete"]) result, _ = run_client(source, dest, port=server.port) assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}" received = get_dest_received_dir(dest, source) extra = os.path.join(received, "extra.txt") with open(extra, "wb") as fh: fh.write(b"should be deleted") flags = [flag] + (["-m"] if mt else []) result, _ = run_client(source, dest, flags=flags, port=server.port) assert result.returncode == 0, \ f"{flag} sync failed: {(result.stderr or result.stdout)[:300]}" assert not os.path.exists(extra), f"{flag} did not remove the extra file" mismatches, missing = verify_transfer(source, received) assert not missing, f"{flag} missing files: {missing}" assert not mismatches, f"{flag} mismatched files: {mismatches}" @pytest.mark.parametrize("flag", ["--delete-before", "--delete-during", "--del"]) @pytest.mark.parametrize("mt", [False, True]) def test_early_flags_delete_before_data(self, flag, mt): """--delete-before/--delete-during remove extras (and a file blocking a destination directory) BEFORE data is applied, so a nested write that would fail while the blocker still exists succeeds.""" source = self._seed("early") dest = os.path.join(TEST_DATA_DIR, "deltiming_early_dst") clean_dir(dest) with ServerManager() as server: server.start(extra_args=["--allow-delete"]) result, _ = run_client(source, dest, port=server.port) assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}" received = get_dest_received_dir(dest, source) extra = os.path.join(received, "extra.txt") with open(extra, "wb") as fh: fh.write(b"extra file") blocker = os.path.join(received, "sub") shutil.rmtree(blocker) with open(blocker, "wb") as fh: fh.write(b"blocks the nested destination directory") flags = [flag] + (["-m"] if mt else []) result, _ = run_client(source, dest, flags=flags, port=server.port) assert result.returncode == 0, \ f"{flag} (early delete) did not remove the blocker in time: " \ f"{(result.stderr or result.stdout)[:300]}" assert not os.path.exists(extra), f"{flag} did not delete the extra before data" assert _read_file(os.path.join(received, "sub", "deep.txt")) == b"deeply nested file\n", \ f"{flag}: nested file was not written after the early deletion" @pytest.mark.parametrize("flag", ["--delete", "--delete-after", "--delete-delay"]) @pytest.mark.parametrize("mt", [False, True]) def test_late_flags_commit_only_after_success(self, flag, mt): """Plain --delete/--delete-after/--delete-delay defer deletion until the whole transfer succeeds: a mid-transfer write failure must leave every extra in place (commit-style safety). The -m receiver must also keep the extras: the deferred keep-set is committed by the server only after the disk-writer thread has finished, and a failing writer means the manifest is freed, never applied.""" source = self._seed("late") dest = os.path.join(TEST_DATA_DIR, "deltiming_late_dst") clean_dir(dest) with ServerManager() as server: server.start(extra_args=["--allow-delete"]) result, _ = run_client(source, dest, port=server.port) assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}" received = get_dest_received_dir(dest, source) extra = os.path.join(received, "extra.txt") with open(extra, "wb") as fh: fh.write(b"extra file") blocker = os.path.join(received, "sub") shutil.rmtree(blocker) with open(blocker, "wb") as fh: fh.write(b"blocks the nested destination directory") flags = [flag] + (["-m"] if mt else []) result, _ = run_client(source, dest, flags=flags, port=server.port) assert result.returncode != 0, \ f"{flag} (mt={mt}) unexpectedly succeeded (deletion must be deferred)" assert os.path.exists(extra), \ f"{flag} (mt={mt}) removed an extra although the transfer failed" assert os.path.isfile(blocker), \ f"{flag} (mt={mt}) deleted the blocker although the transfer failed" def test_early_flag_respected_when_server_refuses_delete(self, shared_server): """With an --allow-delete-less server the client's early timing still completes (no deadlock on the pre-delete ack) and simply never deletes, exactly like the plain server policy.""" source = self._seed("refused") dest = os.path.join(TEST_DATA_DIR, "deltiming_refused_dst") clean_dir(dest) result, _ = run_client(source, dest, port=shared_server.port) assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}" received = get_dest_received_dir(dest, source) extra = os.path.join(received, "extra.txt") with open(extra, "wb") as fh: fh.write(b"extra file") result, _ = run_client(source, dest, flags=["--delete-before"], port=shared_server.port) assert result.returncode == 0, \ f"--delete-before against a refuse-delete server failed: {result.stderr[:300]}" assert os.path.exists(extra), "unauthorized delete removed an extra file" def _seed_delete_tree(tag, entries, dest): """Create a source tree and seed a full mirror at `dest`, returning (source, received_mirror).""" source = os.path.join(TEST_DATA_DIR, f"delpol_{tag}_src") clean_dir(source) for rel, content in entries.items(): full = os.path.join(source, rel) os.makedirs(os.path.dirname(full), exist_ok=True) with open(full, "wb") as fh: fh.write(content) clean_dir(dest) with ServerManager() as server: server.start(extra_args=["--allow-delete"]) result, _ = run_client(source, dest, port=server.port) assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}" received = get_dest_received_dir(dest, source) return source, received class TestDeletePolicy: """Deletion-policy family: --delete-excluded, --max-delete, --force, --ignore-errors and --prune-empty-dirs.""" def _write(self, path, content): os.makedirs(os.path.dirname(path), exist_ok=True) with open(path, "wb") as fh: fh.write(content) @pytest.mark.parametrize("mt", [False, True]) @pytest.mark.parametrize("timing", ["--delete", "--delete-before", "--delete-after", "--delete-delay"]) def test_delete_protects_excluded_by_default_and_delete_excluded_removes(self, mt, timing): """rsync parity: with a --delete timing the destination mirror path whose source was excluded survives (protected by default); --delete-excluded opts back into deleting it. Verified single-threaded and -m across every timing (commit and early).""" source = os.path.join(TEST_DATA_DIR, f"delexcl_{timing.strip('-')}_{mt}_src") clean_dir(source) entries = { "keep.txt": b"kept\n", "secret.log": b"secret\n", "sub/nested.log": b"nested secret\n", } for rel, content in entries.items(): self._write(os.path.join(source, rel), content) dest = os.path.join(TEST_DATA_DIR, f"delexcl_{timing.strip('-')}_{mt}_dst") clean_dir(dest) with ServerManager() as server: server.start(extra_args=["--allow-delete"]) result, _ = run_client(source, dest, port=server.port) assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}" received = get_dest_received_dir(dest, source) self._write(os.path.join(received, "extra.txt"), b"extra\n") # Default: the excluded mirrors survive --delete, genuine extras die. flags = ["--exclude", "*.log", timing] + (["-m"] if mt else []) result, _ = run_client(source, dest, flags=flags, port=server.port) assert result.returncode == 0, \ f"default delete sync failed: {(result.stderr or result.stdout)[:300]}" assert os.path.exists(os.path.join(received, "secret.log")), \ "excluded dest file was deleted under plain --delete (rsync protects it)" assert os.path.exists(os.path.join(received, "sub", "nested.log")), \ "nested excluded dest file was deleted under plain --delete" assert not os.path.exists(os.path.join(received, "extra.txt")), \ "genuine extra was not deleted" # --delete-excluded: excluded mirrors are extras again and die. self._write(os.path.join(received, "extra.txt"), b"extra\n") flags = ["--exclude", "*.log", timing, "--delete-excluded"] + (["-m"] if mt else []) result, _ = run_client(source, dest, flags=flags, port=server.port) assert result.returncode == 0, \ f"--delete-excluded sync failed: {(result.stderr or result.stdout)[:300]}" assert not os.path.exists(os.path.join(received, "secret.log")), \ "--delete-excluded did not remove the excluded dest file" assert not os.path.exists(os.path.join(received, "sub", "nested.log")), \ "--delete-excluded did not remove the nested excluded dest file" assert not os.path.exists(os.path.join(received, "extra.txt")), \ "genuine extra survived --delete-excluded" assert _read_file(os.path.join(received, "keep.txt")) == b"kept\n" @pytest.mark.parametrize("mt", [False, True]) def test_delete_excluded_excluded_directory_subtree(self, mt): """A whole source directory excluded by a filter rule protects its whole destination mirror by default; --delete-excluded removes the subtree.""" source = os.path.join(TEST_DATA_DIR, f"delexcldir_{mt}_src") clean_dir(source) self._write(os.path.join(source, "keep.txt"), b"kept\n") self._write(os.path.join(source, "skipdir", "a.log"), b"a\n") self._write(os.path.join(source, "skipdir", "deep", "b.log"), b"b\n") dest = os.path.join(TEST_DATA_DIR, f"delexcldir_{mt}_dst") clean_dir(dest) with ServerManager() as server: server.start(extra_args=["--allow-delete"]) result, _ = run_client(source, dest, port=server.port) assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}" received = get_dest_received_dir(dest, source) flags = ["--filter=- skipdir/", "--delete"] + (["-m"] if mt else []) result, _ = run_client(source, dest, flags=flags, port=server.port) assert result.returncode == 0, \ f"default delete sync failed: {(result.stderr or result.stdout)[:300]}" assert os.path.exists(os.path.join(received, "skipdir", "a.log")), \ "excluded dir subtree was deleted under plain --delete" assert os.path.exists(os.path.join(received, "skipdir", "deep", "b.log")), \ "nested excluded dir content was deleted under plain --delete" flags = ["--filter=- skipdir/", "--delete", "--delete-excluded"] + (["-m"] if mt else []) result, _ = run_client(source, dest, flags=flags, port=server.port) assert result.returncode == 0, \ f"--delete-excluded sync failed: {(result.stderr or result.stdout)[:300]}" assert not os.path.exists(os.path.join(received, "skipdir")), \ "--delete-excluded did not remove the excluded dir subtree" @pytest.mark.parametrize("mt", [False, True]) @pytest.mark.parametrize("timing", ["--delete", "--delete-before"]) def test_max_delete_exceeded_fails_without_deleting(self, mt, timing): """A run that would exceed --max-delete deletes nothing and fails.""" source = os.path.join(TEST_DATA_DIR, f"maxdel_{timing.strip('-')}_{mt}_src") clean_dir(source) self._write(os.path.join(source, "keep.txt"), b"kept\n") dest = os.path.join(TEST_DATA_DIR, f"maxdel_{timing.strip('-')}_{mt}_dst") clean_dir(dest) with ServerManager() as server: server.start(extra_args=["--allow-delete"]) result, _ = run_client(source, dest, port=server.port) assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}" received = get_dest_received_dir(dest, source) extras = [] for i in range(4): name = f"e{i}.txt" self._write(os.path.join(received, name), b"extra\n") extras.append(os.path.join(received, name)) flags = ["--max-delete=2", timing] + (["-m"] if mt else []) result, _ = run_client(source, dest, flags=flags, port=server.port) assert result.returncode != 0, \ f"--max-delete=2 with 4 extras unexpectedly succeeded: {result.stderr[:300]}" for path in extras: assert os.path.exists(path), \ "--max-delete overrun deleted files (must be all-or-nothing)" @pytest.mark.parametrize("mt", [False, True]) def test_max_delete_not_exceeded_deletes_exactly(self, mt): """When the extras are at or below --max-delete the run succeeds and removes exactly the extras.""" source = os.path.join(TEST_DATA_DIR, f"maxdelok_{mt}_src") clean_dir(source) self._write(os.path.join(source, "keep.txt"), b"kept\n") dest = os.path.join(TEST_DATA_DIR, f"maxdelok_{mt}_dst") clean_dir(dest) with ServerManager() as server: server.start(extra_args=["--allow-delete"]) result, _ = run_client(source, dest, port=server.port) assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}" received = get_dest_received_dir(dest, source) for i in range(3): self._write(os.path.join(received, f"e{i}.txt"), b"extra\n") flags = ["--max-delete=3", "--delete"] + (["-m"] if mt else []) result, _ = run_client(source, dest, flags=flags, port=server.port) assert result.returncode == 0, \ f"--max-delete=3 with 3 extras failed: {(result.stderr or result.stdout)[:300]}" for i in range(3): assert not os.path.exists(os.path.join(received, f"e{i}.txt")), \ f"extra e{i}.txt not deleted under --max-delete=3" @pytest.mark.parametrize("mt", [False, True]) def test_force_replaces_nonempty_dir_with_file(self, mt): """--force lets an incoming regular file replace a non-empty destination directory; without it the write (and the run) fails.""" source = os.path.join(TEST_DATA_DIR, f"force_{mt}_src") clean_dir(source) self._write(os.path.join(source, "sub", "old.txt"), b"old\n") self._write(os.path.join(source, "keep.txt"), b"kept\n") dest = os.path.join(TEST_DATA_DIR, f"force_{mt}_dst") clean_dir(dest) with ServerManager() as server: server.start(extra_args=["--allow-delete"]) result, _ = run_client(source, dest, port=server.port) assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}" received = get_dest_received_dir(dest, source) # The source path `sub` becomes a regular file (the dir is gone). os.unlink(os.path.join(source, "sub", "old.txt")) os.rmdir(os.path.join(source, "sub")) self._write(os.path.join(source, "sub"), b"now a file\n") result, _ = run_client(source, dest, port=server.port) assert result.returncode != 0, \ "a file over a non-empty directory must fail without --force" assert os.path.isdir(os.path.join(received, "sub")), \ "directory was destroyed although the run failed without --force" assert os.path.exists(os.path.join(received, "sub", "old.txt")), \ "non-empty dir content was lost although the run failed without --force" flags = ["--force"] + (["-m"] if mt else []) result, _ = run_client(source, dest, flags=flags, port=server.port) assert result.returncode == 0, \ f"--force run failed: {(result.stderr or result.stdout)[:300]}" assert os.path.isfile(os.path.join(received, "sub")), \ "--force did not replace the directory with the file" assert _read_file(os.path.join(received, "sub")) == b"now a file\n" def test_force_inert_under_delay_updates(self): """Documented divergence: --force acts on the immediate-install path; a --delay-updates run stages into its own tree and its publication renames over regular files only, so a blocking directory is not cleared and the run fails.""" source = os.path.join(TEST_DATA_DIR, "force_delay_src") clean_dir(source) self._write(os.path.join(source, "sub", "old.txt"), b"old\n") self._write(os.path.join(source, "keep.txt"), b"kept\n") dest = os.path.join(TEST_DATA_DIR, "force_delay_dst") clean_dir(dest) with ServerManager() as server: server.start(extra_args=["--allow-delete"]) result, _ = run_client(source, dest, port=server.port) assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}" received = get_dest_received_dir(dest, source) os.unlink(os.path.join(source, "sub", "old.txt")) os.rmdir(os.path.join(source, "sub")) self._write(os.path.join(source, "sub"), b"now a file\n") result, _ = run_client(source, dest, flags=["--force", "--delay-updates"], port=server.port) assert result.returncode != 0, \ "--force --delay-updates unexpectedly replaced the blocking directory" assert os.path.isdir(os.path.join(received, "sub")), \ "blocking directory was cleared although --delay-updates should keep --force inert" assert os.path.exists(os.path.join(received, "sub", "old.txt")), \ "blocking directory content was lost" @pytest.mark.parametrize("mt", [False, True]) def test_prune_empty_dirs_dirs_mode(self, mt): """--prune-empty-dirs omits an empty source directory's explicit entry in --dirs mode (nothing is created, and an existing empty mirror is removed by --delete). Recursive transfers never emit empty dirs, so the flag is a no-op there (documented rsync -m parity).""" source = os.path.join(TEST_DATA_DIR, f"prune_{mt}_src") clean_dir(source) os.makedirs(source, exist_ok=True) # physically empty source dir dest = os.path.join(TEST_DATA_DIR, f"prune_{mt}_dst") clean_dir(dest) with ServerManager() as server: server.start(extra_args=["--allow-delete"]) result, _ = run_client(source, dest, flags=["--dirs"], port=server.port) assert result.returncode == 0, f"-d seed failed: {result.stderr[:200]}" received = get_dest_received_dir(dest, source) assert os.path.isdir(received), "-d should create the empty mirror dir" assert os.listdir(received) == [] # prune-empty-dirs: the empty mirror is pruned by --delete. flags = ["--dirs", "--prune-empty-dirs", "--delete"] + (["-m"] if mt else []) result, _ = run_client(source, dest, flags=flags, port=server.port) assert result.returncode == 0, \ f"--dirs --prune-empty-dirs --delete failed: {(result.stderr or result.stdout)[:300]}" assert not os.path.exists(received), \ "--prune-empty-dirs did not prune the empty dir (--delete left it)" # A fresh destination: prune-empty-dirs means the empty dir is never sent. dest2 = os.path.join(TEST_DATA_DIR, f"prune2_{mt}_dst") clean_dir(dest2) with ServerManager() as server: server.start(extra_args=["--allow-delete"]) flags = ["--dirs", "--prune-empty-dirs", "-i"] + (["-m"] if mt else []) result, _ = run_client(source, dest2, flags=flags, port=server.port) assert result.returncode == 0, \ f"--dirs --prune-empty-dirs failed: {(result.stderr or result.stdout)[:300]}" received2 = get_dest_received_dir(dest2, source) assert not os.path.exists(received2), \ "--prune-empty-dirs transferred the empty directory" assert result.stdout == "", \ f"--prune-empty-dirs leaked an itemize line: {result.stdout[:200]}" @pytest.mark.parametrize("mt", [False, True]) def test_prune_empty_dirs_recursion_inherent(self, mt): """In recursive mode FastSync never transfers empty directories (rsync -m parity): a truly-empty destination directory chain is removed by --delete whether or not --prune-empty-dirs is given (the flag has no additional effect there), while directories holding kept files survive. A filter-excluded file's mirror is protected, so a directory that still holds one is left intact (rsync default delete-excluded semantics).""" source = os.path.join(TEST_DATA_DIR, f"prunerec_{mt}_src") clean_dir(source) self._write(os.path.join(source, "keep.txt"), b"kept\n") self._write(os.path.join(source, "a", "keep.log"), b"a log\n") self._write(os.path.join(source, "b", "deep", "kept.txt"), b"deep kept\n") dest = os.path.join(TEST_DATA_DIR, f"prunerec_{mt}_dst") clean_dir(dest) with ServerManager() as server: server.start(extra_args=["--allow-delete"]) result, _ = run_client(source, dest, port=server.port) assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}" received = get_dest_received_dir(dest, source) # A stray empty chain (FastSync recursion never creates such dirs, so # this models one left by an external tool / an earlier --dirs run). os.makedirs(os.path.join(received, "empty", "chain")) for prune in ([], ["--prune-empty-dirs"]): flags = prune + ["--delete"] + (["-m"] if mt else []) result, _ = run_client(source, dest, flags=flags, port=server.port) assert result.returncode == 0, \ f"prune recursive sync failed: {(result.stderr or result.stdout)[:300]}" assert not os.path.exists(os.path.join(received, "empty")), \ "truly-empty dir chain was not removed by --delete" assert os.path.exists(os.path.join(received, "b", "deep", "kept.txt")), \ "non-empty dir subtree was wrongly removed" assert _read_file(os.path.join(received, "keep.txt")) == b"kept\n" # An excluded file's mirror is protected: the dir that holds it stays. flags = ["--exclude", "*.log", "--delete", "--prune-empty-dirs"] + (["-m"] if mt else []) result, _ = run_client(source, dest, flags=flags, port=server.port) assert result.returncode == 0, \ f"prune recursive sync failed: {(result.stderr or result.stdout)[:300]}" assert os.path.exists(os.path.join(received, "a", "keep.log")), \ "excluded file mirror was deleted under --delete (rsync protects it)" def _run_client_as_nobody(self, source, dest, port, flags): cmd = CLIENT_CMD + ["--source-dir", source, "--dest-dir", dest, "--save-to-disk", "--server-port", str(port)] + flags return subprocess.run(["setpriv", "--reuid=65534", "--regid=65534", "--clear-groups"] + cmd, text=True, capture_output=True) @pytest.mark.parametrize("mt", [False, True]) def test_ignore_errors_keeps_deletion_active_on_scan_error(self, mt): """A source I/O error (unreadable subdirectory) aborts the run so no deletion happens by default; --ignore-errors continues, still transfers the readable tree and still deletes, single-threaded and under -m. Run as an unprivileged user so the mode-000 directory is genuinely unreadable.""" if os.geteuid() != 0 or shutil.which("setpriv") is None: pytest.skip("requires root + setpriv to drop privileges for the client") tag = f"ioerr_{os.getpid()}_{mt}" source = os.path.join(TEST_DATA_DIR, f"{tag}_src") clean_dir(source) self._write(os.path.join(source, "top.txt"), b"top\n") self._write(os.path.join(source, "ok", "inside.txt"), b"inside\n") self._write(os.path.join(source, "locked", "blocked.txt"), b"blocked\n") dest = os.path.join(TEST_DATA_DIR, f"{tag}_dst") clean_dir(dest) with ServerManager() as server: server.start(extra_args=["--allow-delete"]) # Seed as root (server is root too). result, _ = run_client(source, dest, port=server.port) assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}" received = get_dest_received_dir(dest, source) try: os.chmod(os.path.join(source, "locked"), 0) # Default: scan error aborts the run; nothing is deleted. self._write(os.path.join(received, "extra.txt"), b"extra\n") flags = ["--delete"] + (["-m"] if mt else []) result = self._run_client_as_nobody(source, dest, server.port, flags) assert result.returncode != 0, "unreadable source dir did not fail the run" assert os.path.exists(os.path.join(received, "extra.txt")), \ "default run deleted although the scan hit an I/O error" # --ignore-errors: the readable tree transfers, deletion still runs. self._write(os.path.join(received, "extra.txt"), b"extra\n") flags = ["--delete", "--ignore-errors"] + (["-m"] if mt else []) result = self._run_client_as_nobody(source, dest, server.port, flags) assert not os.path.exists(os.path.join(received, "extra.txt")), \ f"--ignore-errors did not keep deletion active: {result.stderr[:300]}" assert not os.path.exists(os.path.join(received, "locked")), \ "mirror of the unreadable dir was left behind (should be an extra)" finally: os.chmod(os.path.join(source, "locked"), 0o755) @pytest.mark.parametrize("mt", [False, True]) @pytest.mark.parametrize("timing", ["--delete", "--delete-before"]) def test_ignore_errors_unreadable_root_never_deletes(self, mt, timing): """An unreadable SOURCE ROOT must never be treated as a skippable scan error: with --ignore-errors the sequential scanner treats the root as fatal (matching the -m path, which cannot even create its scanner), so no empty keep-set manifest is sent and the destination is never wiped. Run as an unprivileged user so the mode-000 root is genuinely unreadable.""" if os.geteuid() != 0 or shutil.which("setpriv") is None: pytest.skip("requires root + setpriv to drop privileges for the client") tag = f"rootio_{os.getpid()}_{mt}_{timing.strip('-')}" source = os.path.join(TEST_DATA_DIR, f"{tag}_src") clean_dir(source) self._write(os.path.join(source, "file.txt"), b"content\n") dest = os.path.join(TEST_DATA_DIR, f"{tag}_dst") clean_dir(dest) with ServerManager() as server: server.start(extra_args=["--allow-delete"]) result, _ = run_client(source, dest, port=server.port) assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}" received = get_dest_received_dir(dest, source) try: os.chmod(source, 0) self._write(os.path.join(received, "extra.txt"), b"extra\n") flags = [timing, "--ignore-errors"] + (["-m"] if mt else []) result = self._run_client_as_nobody(source, dest, server.port, flags) assert result.returncode != 0, \ f"unreadable source root with {timing} (mt={mt}) unexpectedly succeeded" assert os.path.exists(os.path.join(received, "file.txt")), \ f"{timing} (mt={mt}) wiped a kept destination file" assert os.path.exists(os.path.join(received, "extra.txt")), \ f"{timing} (mt={mt}) deleted the extra although the scan could not read the root" finally: os.chmod(source, 0o755) def test_delete_excluded_protection_is_sender_derived(self): """Plain --delete protects destination mirrors of files the SOURCE scan excluded, but a destination-only file that merely matches an exclude rule is still an extra and is removed (protection never re-applies rules to the destination).""" source = os.path.join(TEST_DATA_DIR, "senderderived_src") clean_dir(source) self._write(os.path.join(source, "keep.txt"), b"kept\n") self._write(os.path.join(source, "secret.log"), b"secret\n") dest = os.path.join(TEST_DATA_DIR, "senderderived_dst") clean_dir(dest) with ServerManager() as server: server.start(extra_args=["--allow-delete"]) result, _ = run_client(source, dest, port=server.port) assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}" received = get_dest_received_dir(dest, source) # A destination-only file that happens to match the exclude rule. self._write(os.path.join(received, "stray.log"), b"never on the source\n") result, _ = run_client(source, dest, flags=["--exclude", "*.log", "--delete"], port=server.port) assert result.returncode == 0, \ f"delete sync failed: {(result.stderr or result.stdout)[:300]}" assert os.path.exists(os.path.join(received, "secret.log")), \ "source-excluded mirror was deleted under plain --delete" assert not os.path.exists(os.path.join(received, "stray.log")), \ "destination-only file matching the exclude rule was left (should be deleted)" def _pin_mtime(path, ts): os.utime(path, (ts, ts)) class TestBasisDestDirs: """--compare-dest / --copy-dest / --link-dest alternate basis directories. FastSync's basis directories are relative to the destination root and are confined below it. The "unchanged" decision is receiver-side and requires the per-file --incremental handshake (implied by these flags), so the basis snapshot must reproduce the exact destination-relative mirror path of the incoming files. """ STAGING = ".fastsync-stage" TS = 1577836800 # 2020-01-01 00:00:00 UTC, used to pin matching mtimes # fixture files: source and basis share the mtime pin, so a basis "match" # is decided purely by content (xxHash). unchanged.txt is byte-identical; # changed.txt is byte-DIFFERENT but has the SAME SIZE as the source (and # the same pinned mtime), which is what forces the content-hash gate; # added.txt does not exist in the basis at all. UNCHANGED = "unchanged.txt" CHANGED = "changed.txt" ADDED = "added.txt" def _make_source(self, name, source_files): src = os.path.join(TEST_DATA_DIR, name) clean_dir(src) for rel, content in source_files.items(): full = os.path.join(src, rel) os.makedirs(os.path.dirname(full), exist_ok=True) with open(full, "wb") as fh: fh.write(content) _pin_mtime(full, self.TS) return src def _seed_basis_file(self, dest, source, basis_dir, rel, content, ts=None): base = os.path.join(dest, basis_dir, os.path.relpath( get_dest_received_dir(dest, source), dest)) full = os.path.join(base, rel) os.makedirs(os.path.dirname(full), exist_ok=True) with open(full, "wb") as fh: fh.write(content) _pin_mtime(full, self.TS if ts is None else ts) return full def _seed_basis(self, dest, source, basis_dir, basis_files): for rel, content in basis_files.items(): self._seed_basis_file(dest, source, basis_dir, rel, content) return os.path.join(dest, basis_dir, os.path.relpath( get_dest_received_dir(dest, source), dest)) def _source_tree(self, prefix): return { self.UNCHANGED: b"stable content v1\n", self.CHANGED: b"changed content now\n", self.ADDED: b"brand new content\n", } def _basis_tree(self, prefix): # unchanged.txt is identical to the source; changed.txt has the SAME # byte size and pinned mtime but a different body (equal size forces # the xxHash gate); added.txt is missing from the basis. return { self.UNCHANGED: b"stable content v1\n", self.CHANGED: b"CHANGED CONTENT NOW\n", } def test_same_size_different_content_is_not_a_basis_match(self, shared_server): # Core safety property: equal size + pinned mtime but different content # must NEVER be hard-linked or copied from the basis -- the xxHash gate # rejects it and the sender's data is transferred instead. for flag, basis_dir in (("--link-dest", "szlb"), ("--copy-dest", "szcp"), ("--compare-dest", "szcmp")): source = self._make_source("basis_same_size_src", {self.UNCHANGED: b"same length body\n"}) dest = os.path.join(TEST_DATA_DIR, f"basis_same_size_dst_{basis_dir}") clean_dir(dest) basis_file = self._seed_basis_file(dest, source, basis_dir, self.UNCHANGED, b"SAME LENGTH BODY!") result, _ = run_client(source, dest, flags=[f"{flag}={basis_dir}"], port=shared_server.port) assert result.returncode == 0, \ f"{flag} same-size mismatch failed: {result.stderr[:300]}" received = get_dest_received_dir(dest, source) dest_file = os.path.join(received, self.UNCHANGED) assert _read_file(dest_file) == b"same length body\n", \ f"{flag}: basis content leaked into the destination on a hash mismatch" if flag != "--compare-dest": assert os.stat(dest_file).st_ino != os.stat(basis_file).st_ino, \ f"{flag}: linked/copied from a content-mismatched basis file" def test_compare_dest_skips_matching_and_transfers_missing(self, shared_server): source = self._make_source("basis_compare_src", self._source_tree("c")) dest = os.path.join(TEST_DATA_DIR, "basis_compare_dst") clean_dir(dest) self._seed_basis(dest, source, "cbasis", self._basis_tree("c")) result, _ = run_client(source, dest, flags=["--compare-dest=cbasis"], port=shared_server.port) assert result.returncode == 0, f"compare-dest failed: {result.stderr[:300]}" received = get_dest_received_dir(dest, source) # compare-dest never copies: an exact basis match is skipped, leaving a # sparse destination (rsync parity). assert not os.path.exists(os.path.join(received, self.UNCHANGED)), \ "compare-dest materialized the unchanged file" # Files the destination lacks AND the basis cannot satisfy are still # transferred normally. assert _read_file(os.path.join(received, self.CHANGED)) == \ self._source_tree("c")[self.CHANGED], "changed file not transferred" assert _read_file(os.path.join(received, self.ADDED)) == \ self._source_tree("c")[self.ADDED], "added file not transferred" def test_compare_dest_content_mismatch_forces_transfer(self, shared_server): # The basis holds a file with a DIFFERENT body: even though it shares # the mtime pin, the xxHash check fails and the data must be sent. source = self._make_source("basis_compare_mismatch_src", {self.UNCHANGED: b"real data\n"}) dest = os.path.join(TEST_DATA_DIR, "basis_compare_mismatch_dst") clean_dir(dest) basis = self._seed_basis(dest, source, "cbasis", {self.UNCHANGED: b"stale data!!\n"}) result, _ = run_client(source, dest, flags=["--compare-dest=cbasis"], port=shared_server.port) assert result.returncode == 0, f"compare-dest mismatch failed: {result.stderr[:300]}" received = get_dest_received_dir(dest, source) assert _read_file(os.path.join(received, self.UNCHANGED)) == b"real data\n", \ "content mismatch did not fall back to a normal transfer" assert os.stat(os.path.join(received, self.UNCHANGED)).st_ino != \ os.stat(os.path.join(basis, self.UNCHANGED)).st_ino def test_copy_dest_copies_unchanged_and_transfers_changed(self, shared_server): source = self._make_source("basis_copy_src", self._source_tree("cp")) dest = os.path.join(TEST_DATA_DIR, "basis_copy_dst") clean_dir(dest) basis = self._seed_basis(dest, source, "cpbasis", self._basis_tree("cp")) result, _ = run_client(source, dest, flags=["--copy-dest=cpbasis"], port=shared_server.port) assert result.returncode == 0, f"copy-dest failed: {result.stderr[:300]}" received = get_dest_received_dir(dest, source) unchanged = os.path.join(received, self.UNCHANGED) assert _read_file(unchanged) == b"stable content v1\n", "unchanged file not materialized" # A real local copy, NOT a hard link to the basis file. assert os.stat(unchanged).st_ino != os.stat(os.path.join(basis, self.UNCHANGED)).st_ino # Equal-size/different-content basis file falls back to the sender data. assert _read_file(os.path.join(received, self.CHANGED)) == \ self._source_tree("cp")[self.CHANGED] assert _read_file(os.path.join(received, self.ADDED)) == \ self._source_tree("cp")[self.ADDED] def test_link_dest_hardlinks_and_falls_back(self, shared_server): source = self._make_source("basis_link_src", self._source_tree("ln")) dest = os.path.join(TEST_DATA_DIR, "basis_link_dst") clean_dir(dest) basis = self._seed_basis(dest, source, "lnbasis", self._basis_tree("ln")) result, _ = run_client(source, dest, flags=["--link-dest=lnbasis"], port=shared_server.port) assert result.returncode == 0, f"link-dest failed: {result.stderr[:300]}" received = get_dest_received_dir(dest, source) unchanged = os.path.join(received, self.UNCHANGED) basis_file = os.path.join(basis, self.UNCHANGED) # Real hard link: same inode as the DIR file, nlink >= 2, no data copy. assert os.path.exists(unchanged) assert os.stat(unchanged).st_ino == os.stat(basis_file).st_ino, \ "link-dest did not produce a hard link" assert os.stat(unchanged).st_nlink >= 2 # Equal-size/different-content basis file must fall back to a plain # transfer (not a link). changed = os.path.join(received, self.CHANGED) assert _read_file(changed) == self._source_tree("ln")[self.CHANGED] assert os.stat(changed).st_ino != os.stat(os.path.join(basis, self.CHANGED)).st_ino @pytest.mark.parametrize("flag", ["--compare-dest", "--copy-dest", "--link-dest"]) def test_basis_dir_missing_is_a_clean_noop(self, shared_server, flag): # A basis directory that does not exist must simply transfer everything. source = self._make_source("basis_missing_src", {self.UNCHANGED: b"content\n"}) dest = os.path.join(TEST_DATA_DIR, "basis_missing_dst") clean_dir(dest) result, _ = run_client(source, dest, flags=[f"{flag}=nope"], port=shared_server.port) assert result.returncode == 0, f"{flag} with missing dir failed: {result.stderr[:300]}" received = get_dest_received_dir(dest, source) assert _read_file(os.path.join(received, self.UNCHANGED)) == b"content\n" def test_link_dest_multithreaded(self, shared_server): source = self._make_source("basis_link_mt_src", self._source_tree("mt")) dest = os.path.join(TEST_DATA_DIR, "basis_link_mt_dst") clean_dir(dest) basis = self._seed_basis(dest, source, "mtbasis", self._basis_tree("mt")) result, _ = run_client(source, dest, flags=["--link-dest=mtbasis", "-m"], port=shared_server.port) assert result.returncode == 0, f"-m link-dest failed: {result.stderr[:300]}" received = get_dest_received_dir(dest, source) assert os.stat(os.path.join(received, self.UNCHANGED)).st_ino == \ os.stat(os.path.join(basis, self.UNCHANGED)).st_ino assert _read_file(os.path.join(received, self.ADDED)) == \ self._source_tree("mt")[self.ADDED] def test_link_dest_with_delay_updates_stages_and_publishes_link(self, shared_server): source = self._make_source("basis_link_delay_src", {self.UNCHANGED: b"v1\n"}) dest = os.path.join(TEST_DATA_DIR, "basis_link_delay_dst") clean_dir(dest) basis = self._seed_basis(dest, source, "delaybasis", {self.UNCHANGED: b"v1\n"}) result, _ = run_client(source, dest, flags=["--link-dest=delaybasis", "--delay-updates"], port=shared_server.port) assert result.returncode == 0, f"delay-updates link-dest failed: {result.stderr[:300]}" received = get_dest_received_dir(dest, source) unchanged = os.path.join(received, self.UNCHANGED) assert os.stat(unchanged).st_ino == \ os.stat(os.path.join(basis, self.UNCHANGED)).st_ino assert not os.path.isdir(os.path.join(dest, self.STAGING)), \ "delay-updates staging tree was not cleaned up" def test_delete_does_not_touch_basis_dir(self): """--delete removes genuine extras but must never treat a basis-dir snapshot (which a --link-dest run just linked from) as destination content.""" source = self._make_source("basis_delete_src", {self.UNCHANGED: b"v1\n"}) dest = os.path.join(TEST_DATA_DIR, "basis_delete_dst") clean_dir(dest) basis = self._seed_basis(dest, source, "delbasis", {self.UNCHANGED: b"v1\n"}) received = get_dest_received_dir(dest, source) os.makedirs(received, exist_ok=True) extra = os.path.join(received, "extra.txt") with open(extra, "wb") as fh: fh.write(b"extra") with ServerManager() as server: server.start(extra_args=["--allow-delete"]) result, _ = run_client(source, dest, flags=["--link-dest=delbasis", "--delete"], port=server.port) assert result.returncode == 0, \ f"delete+link-dest failed: {result.stderr[:300]}" assert not os.path.exists(extra), "genuine extra file was not deleted" assert _read_file(os.path.join(received, self.UNCHANGED)) == b"v1\n" assert os.path.exists(os.path.join(basis, self.UNCHANGED)), \ "basis directory was deleted by --delete" assert os.stat(os.path.join(received, self.UNCHANGED)).st_ino == \ os.stat(os.path.join(basis, self.UNCHANGED)).st_ino def test_delay_delete_keeps_nested_staging_named_dir_as_content(self): # The real --delay-updates staging directory is protected from --delete # only as a DIRECT child of the receive root. A nested destination # directory that merely shares the staging name is ordinary content, so # its extras must still be deleted (regression guard for the walker). source = self._make_source("basis_nested_stage_src", {"top.txt": b"top\n", "sub/real.txt": b"real\n"}) dest = os.path.join(TEST_DATA_DIR, "basis_nested_stage_dst") clean_dir(dest) self._seed_basis(dest, source, "nstbasis", {"top.txt": b"top\n", "sub/real.txt": b"real\n"}) received = get_dest_received_dir(dest, source) nested = os.path.join(received, "sub", self.STAGING) os.makedirs(nested, exist_ok=True) extra = os.path.join(nested, "extra.txt") with open(extra, "wb") as fh: fh.write(b"nested extra") with ServerManager() as server: server.start(extra_args=["--allow-delete"]) result, _ = run_client(source, dest, flags=["--link-dest=nstbasis", "--delete", "--delay-updates"], port=server.port) assert result.returncode == 0, \ f"delay-delete nested staging failed: {result.stderr[:300]}" assert not os.path.exists(extra), \ "extra inside a nested .fastsync-stage dir was not deleted" assert not os.path.isdir(nested), \ "nested .fastsync-stage dir should have been removed after its extra" assert _read_file(os.path.join(received, "sub", "real.txt")) == b"real\n" assert not os.path.isdir(os.path.join(dest, self.STAGING)), \ "real delay-updates staging tree was not cleaned up" def test_basis_priority_first_match_wins(self, shared_server): # Two link-dest dirs both hold the exact file: the FIRST (command-line # order) basis directory must win and supply the hard link. source = self._make_source("basis_prio_src", {"f.txt": b"content\n"}) dest = os.path.join(TEST_DATA_DIR, "basis_prio_dst") clean_dir(dest) first = self._seed_basis_file(dest, source, "b1", "f.txt", b"content\n") self._seed_basis_file(dest, source, "b2", "f.txt", b"content\n") result, _ = run_client(source, dest, flags=["--link-dest=b1", "--link-dest=b2"], port=shared_server.port) assert result.returncode == 0, f"link-dest priority failed: {result.stderr[:300]}" received = get_dest_received_dir(dest, source) assert os.stat(os.path.join(received, "f.txt")).st_ino == os.stat(first).st_ino, \ "first basis dir did not win over the second" def test_basis_priority_across_compare_and_link(self, shared_server): # A compare-dest entry listed BEFORE a link-dest entry shadows it (the # exact match is found first and nothing is materialized); reversing the # order lets the link-dest entry win and materialize a hard link. source = self._make_source("basis_prio_mixed_src", {"f.txt": b"content\n"}) dest = os.path.join(TEST_DATA_DIR, "basis_prio_mixed_dst") clean_dir(dest) self._seed_basis_file(dest, source, "cmpb", "f.txt", b"content\n") self._seed_basis_file(dest, source, "lnb", "f.txt", b"content\n") result, _ = run_client(source, dest, flags=["--compare-dest=cmpb", "--link-dest=lnb"], port=shared_server.port) assert result.returncode == 0, \ f"mixed priority (compare first) failed: {result.stderr[:300]}" received = get_dest_received_dir(dest, source) assert not os.path.exists(os.path.join(received, "f.txt")), \ "compare-dest matched first, so the file must stay sparse (no link-dest materialize)" dest = os.path.join(TEST_DATA_DIR, "basis_prio_mixed_dst2") clean_dir(dest) self._seed_basis_file(dest, source, "cmpb", "f.txt", b"content\n") linkb2 = self._seed_basis_file(dest, source, "lnb", "f.txt", b"content\n") result, _ = run_client(source, dest, flags=["--link-dest=lnb", "--compare-dest=cmpb"], port=shared_server.port) assert result.returncode == 0, \ f"mixed priority (link first) failed: {result.stderr[:300]}" received = get_dest_received_dir(dest, source) assert os.stat(os.path.join(received, "f.txt")).st_ino == os.stat(linkb2).st_ino, \ "link-dest did not materialize when listed before compare-dest" def test_link_dest_size_only_ignores_mtime(self, shared_server): # --size-only drops the mtime leg of the quick check: a basis file with # the SAME content but a DIFFERENT mtime is still an exact match. source = self._make_source("basis_sizeonly_src", {"f.txt": b"content\n"}) dest = os.path.join(TEST_DATA_DIR, "basis_sizeonly_dst") clean_dir(dest) basis_file = self._seed_basis_file(dest, source, "sob", "f.txt", b"content\n", ts=self.TS + 500) result, _ = run_client(source, dest, flags=["--link-dest=sob", "--size-only"], port=shared_server.port) assert result.returncode == 0, f"size-only link-dest failed: {result.stderr[:300]}" received = get_dest_received_dir(dest, source) assert os.stat(os.path.join(received, "f.txt")).st_ino == os.stat(basis_file).st_ino, \ "--size-only should link a basis file whose mtime differs" def test_link_dest_ignore_times_never_links(self, shared_server): # -I/--ignore-times forces every file to be updated, so a basis dir is # never used to hard-link (rsync parity). The file is transferred and # stored as a fresh inode even though it matches the basis exactly. source = self._make_source("basis_igntimes_src", {"f.txt": b"content\n"}) dest = os.path.join(TEST_DATA_DIR, "basis_igntimes_dst") clean_dir(dest) basis_file = self._seed_basis_file(dest, source, "itb", "f.txt", b"content\n") result, _ = run_client(source, dest, flags=["--link-dest=itb", "--ignore-times"], port=shared_server.port) assert result.returncode == 0, f"ignore-times link-dest failed: {result.stderr[:300]}" received = get_dest_received_dir(dest, source) dest_file = os.path.join(received, "f.txt") assert _read_file(dest_file) == b"content\n" assert os.stat(dest_file).st_ino != os.stat(basis_file).st_ino, \ "--ignore-times must not hard-link to a basis file" def test_basis_refuses_file_above_whole_file_limit(self, shared_server): # Every whole-file payload path in FastSync (basis dirs included) is # bounded by MAX_RECEIVE_WHOLE_FILE_SIZE. rsync supports basis dirs for # arbitrary sizes; FastSync refuses such a run up front with a clear # diagnostic instead of letting the receiver abort the whole transfer # mid-stream with no client-side explanation. source = self._make_source("basis_oversize_src", {"small.txt": b"ok\n"}) big = os.path.join(source, "huge.bin") with open(big, "wb") as fh: os.ftruncate(fh.fileno(), 256 * 1024 * 1024 + 4096) dest = os.path.join(TEST_DATA_DIR, "basis_oversize_dst") clean_dir(dest) result, _ = run_client(source, dest, flags=["--link-dest=nope"], port=shared_server.port) assert result.returncode != 0, \ "basis run with an over-limit file unexpectedly succeeded" assert "larger than" in result.stderr, \ f"no clear over-limit diagnostic: {result.stderr[:300]}" received = get_dest_received_dir(dest, source) assert not os.path.exists(received), \ "over-limit basis run transferred files before failing" def _random_payloads(size=2 * 1024 * 1024, changed=64 * 1024, seed=1234): """Return (old, new) byte strings of equal length where `new` differs from `old` only in one contiguous `changed`-byte region. Incompressible (random) data keeps the whole-file wire cost near the file size, so a delta transfer is distinguishable from a whole-file one by its wire bytes.""" r = random.Random(seed) data = bytearray(r.randbytes(size)) old = bytes(data) off = size // 3 for i in range(off, off + changed): data[i] = r.randrange(256) return old, bytes(data) class TestFuzzy: """-y/--fuzzy similar-file delta basis. Scenario modelled on rsync's --fuzzy: a file is recreated under a similar NEW basename in the same directory. The destination still holds the old-named file (nothing deleted it), but the new path has no content of its own at the destination, so without --fuzzy the receiver has no delta basis and the whole file is sent. With --fuzzy the receiver searches the destination directory, picks the similar-named sibling as the delta basis, sends its block signature, and the sender transmits only the differences. The reconstructed file must be byte-identical to the source in every mode; only the wire usage changes (observed through CountingProxy, because client --stats report source lengths, not wire bytes). """ OLD_NAME = "report-2025.dat" NEW_NAME = "report-2026.dat" TS = 1577836800 # 2020-01-01, used to pin stale destination mtimes SIZE = 2 * 1024 * 1024 def _client_via_proxy(self, source, dest, flags, proxy): cmd = (CLIENT_CMD + ["--source-dir", source, "--dest-dir", dest, "--save-to-disk", "--server-port", str(proxy.port)] + flags) return proxy.run(cmd) def _seed_dest(self, source, dest, files, port): """Write `files` {rel: bytes} into source and mirror them to dest.""" for rel, content in files.items(): full = os.path.join(source, rel) os.makedirs(os.path.dirname(full), exist_ok=True) with open(full, "wb") as fh: fh.write(content) clean_dir(dest) result, _ = run_client(source, dest, port=port) assert result.returncode == 0, f"seed failed: {(result.stderr or result.stdout)[:300]}" def _prepare(self, tag): source = os.path.join(TEST_DATA_DIR, f"fuzzy_{tag}_src") dest = os.path.join(TEST_DATA_DIR, f"fuzzy_{tag}_dst") clean_dir(source) return source, dest def _run_measured(self, source, dest, flags, port): """Run a transfer through a byte-counting proxy. Returns (result, proxy).""" proxy = CountingProxy(port) result = self._client_via_proxy(source, dest, flags, proxy) return result, proxy def test_fuzzy_uses_similar_sibling_as_delta_basis(self, shared_server): source, dest = self._prepare("basis") old_bytes, new_bytes = _random_payloads() self._seed_dest(source, dest, {self.OLD_NAME: old_bytes}, shared_server.port) # Recreate the file under a similar new name; the old sibling stays on # the destination (nothing deletes it). os.unlink(os.path.join(source, self.OLD_NAME)) with open(os.path.join(source, self.NEW_NAME), "wb") as fh: fh.write(new_bytes) result, proxy = self._run_measured(source, dest, ["--fuzzy"], shared_server.port) assert result.returncode == 0, \ f"--fuzzy rename transfer failed: {(result.stderr or result.stdout)[:300]}" received = get_dest_received_dir(dest, source) assert _read_file(os.path.join(received, self.NEW_NAME)) == new_bytes, \ "fuzzy reconstruction is not byte-exact" # The wire carried the delta, not the 2 MiB whole file. assert proxy.client_to_server < len(new_bytes) // 4, \ f"fuzzy transfer sent {proxy.client_to_server} bytes; expected a delta" def test_without_fuzzy_sends_the_whole_file(self, shared_server): source, dest = self._prepare("whole") old_bytes, new_bytes = _random_payloads() self._seed_dest(source, dest, {self.OLD_NAME: old_bytes}, shared_server.port) os.unlink(os.path.join(source, self.OLD_NAME)) with open(os.path.join(source, self.NEW_NAME), "wb") as fh: fh.write(new_bytes) result, proxy = self._run_measured(source, dest, ["--incremental", "--delta"], shared_server.port) assert result.returncode == 0, f"no-fuzzy rename failed: {result.stderr[:300]}" received = get_dest_received_dir(dest, source) assert _read_file(os.path.join(received, self.NEW_NAME)) == new_bytes # No similar basis: the whole file goes over the wire. assert proxy.client_to_server > len(new_bytes) // 2, \ f"expected a whole-file transfer, got {proxy.client_to_server} bytes" @pytest.mark.parametrize("mt", [False, True]) def test_fuzzy_byte_exact_single_and_multithreaded(self, shared_server, mt): source, dest = self._prepare(f"mt{'1' if mt else '0'}") old_bytes, new_bytes = _random_payloads() self._seed_dest(source, dest, {self.OLD_NAME: old_bytes}, shared_server.port) os.unlink(os.path.join(source, self.OLD_NAME)) with open(os.path.join(source, self.NEW_NAME), "wb") as fh: fh.write(new_bytes) flags = ["--fuzzy"] + (["-m"] if mt else []) result, proxy = self._run_measured(source, dest, flags, shared_server.port) assert result.returncode == 0, \ f"--fuzzy {'-m ' if mt else ''}rename failed: {(result.stderr or result.stdout)[:300]}" received = get_dest_received_dir(dest, source) assert _read_file(os.path.join(received, self.NEW_NAME)) == new_bytes, \ f"fuzzy {'-m ' if mt else ''}reconstruction is not byte-exact" assert proxy.client_to_server < len(new_bytes) // 4 def test_no_candidate_falls_back_to_whole_file(self, shared_server): # A brand-new destination directory holds no sibling at all, so --fuzzy # finds nothing and the file is transferred whole (and correctly). source, dest = self._prepare("nocand") clean_dir(dest) os.makedirs(source, exist_ok=True) _, new_bytes = _random_payloads() with open(os.path.join(source, self.NEW_NAME), "wb") as fh: fh.write(new_bytes) result, proxy = self._run_measured(source, dest, ["--fuzzy"], shared_server.port) assert result.returncode == 0, \ f"--fuzzy no-candidate fallback failed: {(result.stderr or result.stdout)[:300]}" received = get_dest_received_dir(dest, source) assert _read_file(os.path.join(received, self.NEW_NAME)) == new_bytes assert proxy.client_to_server > len(new_bytes) // 2, \ "no-candidate fuzzy run should have sent the whole file" def test_dissimilar_sibling_is_not_used(self, shared_server): # The destination holds a large sibling whose basename is too different # from the incoming name; the name gate must reject it and fall back to # a whole-file transfer. source, dest = self._prepare("dissim") old_bytes, new_bytes = _random_payloads() self._seed_dest(source, dest, {"totally-unrelated-notes.bin": old_bytes}, shared_server.port) with open(os.path.join(source, self.NEW_NAME), "wb") as fh: fh.write(new_bytes) result, proxy = self._run_measured(source, dest, ["--fuzzy"], shared_server.port) assert result.returncode == 0, \ f"--fuzzy dissimilar-sibling run failed: {(result.stderr or result.stdout)[:300]}" received = get_dest_received_dir(dest, source) assert _read_file(os.path.join(received, self.NEW_NAME)) == new_bytes assert proxy.client_to_server > len(new_bytes) // 2, \ "a dissimilar-named sibling must not be used as a fuzzy basis" def test_fuzzy_helps_when_dest_holds_an_unsuitable_file(self, shared_server): # The destination DOES hold the exact new name, but it is a tiny stale # file (below the delta engine's minimum, ratio far outside its window), # so it cannot serve as the basis. --fuzzy then falls back to the # similar-named sibling. source, dest = self._prepare("unsuitable") old_bytes, new_bytes = _random_payloads() self._seed_dest(source, dest, {self.OLD_NAME: old_bytes, self.NEW_NAME: b"stale small file\n"}, shared_server.port) with open(os.path.join(source, self.NEW_NAME), "wb") as fh: fh.write(new_bytes) result, proxy = self._run_measured(source, dest, ["--fuzzy"], shared_server.port) assert result.returncode == 0, \ f"--fuzzy unsuitable-dest run failed: {(result.stderr or result.stdout)[:300]}" received = get_dest_received_dir(dest, source) assert _read_file(os.path.join(received, self.NEW_NAME)) == new_bytes, \ "byte-exactness broken when the destination file was unsuitable" assert proxy.client_to_server < len(new_bytes) // 4, \ "fuzzy should have reused the similar sibling as the basis" def test_whole_file_makes_fuzzy_inert(self, shared_server): # -W/--whole-file switches the delta machinery off, so --fuzzy has # nothing to attach to and the file is transferred whole (rsync parity). source, dest = self._prepare("wholefile") old_bytes, new_bytes = _random_payloads() self._seed_dest(source, dest, {self.OLD_NAME: old_bytes}, shared_server.port) os.unlink(os.path.join(source, self.OLD_NAME)) with open(os.path.join(source, self.NEW_NAME), "wb") as fh: fh.write(new_bytes) result, proxy = self._run_measured(source, dest, ["--fuzzy", "-W"], shared_server.port) assert result.returncode == 0, \ f"--fuzzy -W run failed: {(result.stderr or result.stdout)[:300]}" received = get_dest_received_dir(dest, source) assert _read_file(os.path.join(received, self.NEW_NAME)) == new_bytes assert proxy.client_to_server > len(new_bytes) // 2, \ "--whole-file must disable the fuzzy delta basis" def test_fuzzy_via_short_y_alias(self, shared_server): source, dest = self._prepare("shorty") old_bytes, new_bytes = _random_payloads() self._seed_dest(source, dest, {self.OLD_NAME: old_bytes}, shared_server.port) os.unlink(os.path.join(source, self.OLD_NAME)) with open(os.path.join(source, self.NEW_NAME), "wb") as fh: fh.write(new_bytes) result, proxy = self._run_measured(source, dest, ["-y"], shared_server.port) assert result.returncode == 0, f"-y rename failed: {(result.stderr or result.stdout)[:300]}" received = get_dest_received_dir(dest, source) assert _read_file(os.path.join(received, self.NEW_NAME)) == new_bytes assert proxy.client_to_server < len(new_bytes) // 4, "-y did not enable fuzzy" def test_fuzzy_with_delay_updates_publishes_cleanly(self, shared_server): # A fuzzy-reconstructed file goes through the normal store engine, so # --delay-updates must stage and publish it with no staging leftovers. source, dest = self._prepare("delay") old_bytes, new_bytes = _random_payloads() self._seed_dest(source, dest, {self.OLD_NAME: old_bytes}, shared_server.port) os.unlink(os.path.join(source, self.OLD_NAME)) with open(os.path.join(source, self.NEW_NAME), "wb") as fh: fh.write(new_bytes) result, _ = run_client(source, dest, flags=["--fuzzy", "--delay-updates"], port=shared_server.port) assert result.returncode == 0, \ f"--fuzzy --delay-updates failed: {(result.stderr or result.stdout)[:300]}" received = get_dest_received_dir(dest, source) assert _read_file(os.path.join(received, self.NEW_NAME)) == new_bytes assert not os.path.isdir(os.path.join(dest, ".fastsync-stage")), \ "delay-updates staging tree was not cleaned up" def test_fuzzy_source_removed_after_transfer(self, shared_server): # A fuzzy transfer is a real transfer (not a skip), so # --remove-source-files must remove the renamed source file. source, dest = self._prepare("rm") old_bytes, new_bytes = _random_payloads() self._seed_dest(source, dest, {self.OLD_NAME: old_bytes}, shared_server.port) os.unlink(os.path.join(source, self.OLD_NAME)) with open(os.path.join(source, self.NEW_NAME), "wb") as fh: fh.write(new_bytes) result, _ = run_client(source, dest, flags=["--fuzzy", "--remove-source-files"], port=shared_server.port) assert result.returncode == 0, \ f"--fuzzy --remove-source-files failed: {(result.stderr or result.stdout)[:300]}" assert not os.path.exists(os.path.join(source, self.NEW_NAME)), \ "a fuzzy-transferred source should have been removed" received = get_dest_received_dir(dest, source) assert _read_file(os.path.join(received, self.NEW_NAME)) == new_bytes assert _read_file(os.path.join(received, self.OLD_NAME)) == old_bytes @staticmethod def _rand_bytes(size, seed): return random.Random(seed).randbytes(size) def _replace_source_file(self, source, old_name, new_name, new_bytes): """Remove old_name from source and add new_name with new_bytes.""" os.unlink(os.path.join(source, old_name)) with open(os.path.join(source, new_name), "wb") as fh: fh.write(new_bytes) def test_worthless_fuzzy_basis_falls_back_inside_handshake(self, shared_server): # The sibling passes the name AND size gates but shares no blocks with # the incoming file, so the sender's delta is not worthwhile: it replies # STATUS_NEXT and the receiver consumes the WHOLE file inside the delta # handshake. This proves a bad fuzzy basis cannot desync the protocol # or corrupt the result. source, dest = self._prepare("worthless") basis = self._rand_bytes(self.SIZE, 424242) target = self._rand_bytes(self.SIZE, 777777) self._seed_dest(source, dest, {self.OLD_NAME: basis}, shared_server.port) self._replace_source_file(source, self.OLD_NAME, self.NEW_NAME, target) result, proxy = self._run_measured(source, dest, ["--fuzzy"], shared_server.port) assert result.returncode == 0, \ f"--fuzzy worthless-basis run failed: {(result.stderr or result.stdout)[:300]}" received = get_dest_received_dir(dest, source) assert _read_file(os.path.join(received, self.NEW_NAME)) == target, \ "whole-file fallback after a worthless fuzzy basis is not byte-exact" assert proxy.client_to_server > self.SIZE // 2, \ "a worthless basis should have made the sender fall back to the whole file" def test_existing_dest_file_preferred_over_fuzzy_sibling(self, shared_server): # Non-displacement: the destination holds a file at the exact path that # is inside the delta size bounds (same size, different content, older # mtime). FastSync must delta against THAT file -- even though it # shares nothing with the source -- and must NOT reuse a similar-named # sibling that is byte-identical to the source. source, dest = self._prepare("nondisp") sibling = self._rand_bytes(self.SIZE, 111) # will equal the incoming file stale = self._rand_bytes(self.SIZE, 333) # worthless exact-path file self._seed_dest(source, dest, {self.OLD_NAME: sibling, self.NEW_NAME: stale}, shared_server.port) # Force the exact-path destination file's mtime into the past so the # quick check deterministically decides to transfer it. os.utime(os.path.join(get_dest_received_dir(dest, source), self.NEW_NAME), (self.TS, self.TS)) os.unlink(os.path.join(source, self.OLD_NAME)) with open(os.path.join(source, self.NEW_NAME), "wb") as fh: fh.write(sibling) result, proxy = self._run_measured(source, dest, ["--fuzzy"], shared_server.port) assert result.returncode == 0, \ f"--fuzzy non-displacement run failed: {(result.stderr or result.stdout)[:300]}" received = get_dest_received_dir(dest, source) assert _read_file(os.path.join(received, self.NEW_NAME)) == sibling assert proxy.client_to_server > self.SIZE // 2, \ "the exact-path destination file must be the delta basis, not the fuzzy sibling" def test_fuzzy_basis_larger_than_source(self, shared_server): # The similar sibling is LARGER than the incoming file (within the delta # engine's 10x ratio); the new file is an exact prefix of the basis, so # every block matches and only a tiny delta travels. source, dest = self._prepare("largerbasis") big = self._rand_bytes(1536 * 1024, 1) prefix = big[:1024 * 1024] self._seed_dest(source, dest, {self.OLD_NAME: big}, shared_server.port) self._replace_source_file(source, self.OLD_NAME, self.NEW_NAME, prefix) result, proxy = self._run_measured(source, dest, ["--fuzzy"], shared_server.port) assert result.returncode == 0, \ f"--fuzzy larger-basis run failed: {(result.stderr or result.stdout)[:300]}" received = get_dest_received_dir(dest, source) assert _read_file(os.path.join(received, self.NEW_NAME)) == prefix, \ "shrunken file reconstructed from a larger fuzzy basis is not byte-exact" assert proxy.client_to_server < len(prefix) // 4, \ "larger fuzzy basis should have carried most of the file as block matches" def test_fuzzy_basis_smaller_than_source(self, shared_server): # The similar sibling is SMALLER than the incoming file; the new file # appends data past the basis, so the appended tail travels as literals # while the shared prefix is block-matched. source, dest = self._prepare("smallerbasis") base = self._rand_bytes(self.SIZE, 2) tail = self._rand_bytes(64 * 1024, 3) new_bytes = base + tail self._seed_dest(source, dest, {self.OLD_NAME: base}, shared_server.port) self._replace_source_file(source, self.OLD_NAME, self.NEW_NAME, new_bytes) result, proxy = self._run_measured(source, dest, ["--fuzzy"], shared_server.port) assert result.returncode == 0, \ f"--fuzzy smaller-basis run failed: {(result.stderr or result.stdout)[:300]}" received = get_dest_received_dir(dest, source) assert _read_file(os.path.join(received, self.NEW_NAME)) == new_bytes, \ "grown file reconstructed from a smaller fuzzy basis is not byte-exact" assert proxy.client_to_server < len(new_bytes) // 4, \ "smaller fuzzy basis should have block-matched the shared prefix" def test_no_fuzzy_end_to_end_equals_no_flag(self, shared_server): # --no-fuzzy must not enable anything: a run with it behaves exactly # like a run without it (whole-file transfer, byte-exact output). source, dest = self._prepare("nofuzzye2e") old_bytes, new_bytes = _random_payloads() self._seed_dest(source, dest, {self.OLD_NAME: old_bytes}, shared_server.port) self._replace_source_file(source, self.OLD_NAME, self.NEW_NAME, new_bytes) result, proxy = self._run_measured(source, dest, ["--no-fuzzy"], shared_server.port) assert result.returncode == 0, \ f"--no-fuzzy run failed: {(result.stderr or result.stdout)[:300]}" received = get_dest_received_dir(dest, source) assert _read_file(os.path.join(received, self.NEW_NAME)) == new_bytes assert proxy.client_to_server > len(new_bytes) // 2, \ "--no-fuzzy should leave the default whole-file behavior intact"