TapTap/FastSync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity
Files
329fc60b142699099760c8d28a3048f4629c53b6
FastSync/tests/integration/test_features.py
T
TapTap 329fc60b14
CI / lint (pull_request) Successful in 32s
Details
CI / sanitizers (address) (pull_request) Successful in 42s
Details
CI / sanitizers (undefined) (pull_request) Successful in 40s
Details
CI / fuzz-build (pull_request) Successful in 17s
Details
CI / coverage (pull_request) Successful in 35s
Details
CI / valgrind (pull_request) Successful in 36s
Details
CI / build-and-test (pull_request) Successful in 3m29s
Details
test: make remove-source incremental-skip test deterministic 
The seed run did not preserve timestamps, so the destination copy's mtime was
the write time; the incremental --remove-source-files rerun only skipped the
file when both writes happened to land in the same whole second, making the
test flaky (observed intermittently in local full-suite runs and on CI).  Seed
with -M so the destination stores the source's exact mtime.
2026-09-06 19:53:22 +02:00

2351 lines
114 KiB
Python
Raw Blame History

"""Feature tests: incremental sync, bandwidth limiting, dry run, metadata, filters."""
import filecmp
import os
import shutil
import subprocess
import sys
import time
import pytest
sys.path.insert(0, os.path.dirname(__file__))
from common import (
PROJECT_ROOT, BUILD_DIR, TEST_DATA_DIR,
run_client,
generate_test_files, verify_transfer, clean_dir, make_result,
get_dest_received_dir, CLIENT_CMD, ServerManager,
)
SOURCE_DIR = os.path.join(TEST_DATA_DIR, "feature_source")
DEST_DIR = os.path.join(TEST_DATA_DIR, "feature_dest")
@pytest.fixture(scope="module", autouse=True)
def setup_test_data():
generate_test_files(SOURCE_DIR, full=False)
clean_dir(DEST_DIR)
yield
shutil.rmtree(TEST_DATA_DIR, ignore_errors=True)
class TestDryRun:
def test_human_readable_dry_run(self):
result, dur = run_client(SOURCE_DIR, DEST_DIR, flags=["-h", "--dry-run"])
assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:100]}"
assert "Total:" in result.stdout
assert "KB" in result.stdout
def test_dry_run(self):
clean_dir(DEST_DIR)
result, dur = run_client(
SOURCE_DIR, DEST_DIR,
flags=["-n"],
)
assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:100]}"
assert "Dry run:" in result.stdout, f"No dry run output: {result.stdout[:200]}"
def test_quiet_suppresses_dry_run_output(self):
result, dur = run_client(
SOURCE_DIR, DEST_DIR,
flags=["-q", "-n", "--progress", "--stats"],
)
assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:100]}"
assert result.stdout == ""
assert result.stderr == ""
def test_quiet_preserves_errors(self):
result, dur = run_client(
SOURCE_DIR, DEST_DIR,
flags=["--quiet", "--server-port", "1"],
)
assert result.returncode != 0
assert result.stderr != ""
@pytest.mark.parametrize("flags", [["-q", "-v"], ["-v", "-q"]])
def test_quiet_successful_transfer_and_verbose_order(self, shared_server, flags):
clean_dir(DEST_DIR)
result, dur = run_client(
SOURCE_DIR, DEST_DIR,
flags=flags,
port=shared_server.port,
)
assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:100]}"
assert result.stdout == ""
assert result.stderr == ""
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
mismatches, missing = verify_transfer(SOURCE_DIR, received)
assert not missing, f"Missing: {missing}"
assert not mismatches, f"Mismatch: {mismatches}"
class TestRemoveSourceFiles:
def test_removes_only_transferred_regular_files(self, shared_server):
source = os.path.join(TEST_DATA_DIR, "remove_source")
dest = os.path.join(TEST_DATA_DIR, "remove_dest")
clean_dir(source)
clean_dir(dest)
with open(os.path.join(source, "one.txt"), "wb") as f:
f.write(b"one")
with open(os.path.join(source, "two.txt"), "wb") as f:
f.write(b"two")
os.makedirs(os.path.join(source, "directory"))
os.symlink("one.txt", os.path.join(source, "link.txt"))
result, _ = run_client(source, dest, flags=["--remove-source-files", "-m"],
port=shared_server.port)
assert result.returncode == 0, f"Remove-source sync failed: {result.stderr[:200]}"
assert not os.path.exists(os.path.join(source, "one.txt"))
assert not os.path.exists(os.path.join(source, "two.txt"))
assert os.path.isdir(os.path.join(source, "directory"))
assert os.path.islink(os.path.join(source, "link.txt"))
def test_single_threaded_removes_transferred_file(self, shared_server):
source = os.path.join(TEST_DATA_DIR, "remove_single_source")
dest = os.path.join(TEST_DATA_DIR, "remove_single_dest")
clean_dir(source)
clean_dir(dest)
source_file = os.path.join(source, "file.txt")
with open(source_file, "wb") as f:
f.write(b"single threaded")
result, _ = run_client(source, dest, flags=["--remove-source-files"],
port=shared_server.port)
assert result.returncode == 0
assert not os.path.exists(source_file)
def test_dry_run_preserves_source_files(self):
source = os.path.join(TEST_DATA_DIR, "remove_dry_source")
dest = os.path.join(TEST_DATA_DIR, "remove_dry_dest")
clean_dir(source)
clean_dir(dest)
source_file = os.path.join(source, "file.txt")
with open(source_file, "wb") as f:
f.write(b"keep")
result, _ = run_client(source, dest, flags=["--remove-source-files", "--dry-run"])
assert result.returncode == 0
assert os.path.isfile(source_file)
def test_failed_connection_preserves_source_files(self):
source = os.path.join(TEST_DATA_DIR, "remove_failed_source")
dest = os.path.join(TEST_DATA_DIR, "remove_failed_dest")
clean_dir(source)
clean_dir(dest)
source_file = os.path.join(source, "file.txt")
with open(source_file, "wb") as f:
f.write(b"keep after failure")
result, _ = run_client(source, dest, flags=["--remove-source-files"], port=1)
assert result.returncode != 0
assert os.path.isfile(source_file)
def test_incremental_skip_preserves_source_file(self, shared_server):
source = os.path.join(TEST_DATA_DIR, "remove_skipped_source")
dest = os.path.join(TEST_DATA_DIR, "remove_skipped_dest")
clean_dir(source)
clean_dir(dest)
source_file = os.path.join(source, "file.txt")
with open(source_file, "wb") as f:
f.write(b"keep after skip")
# The seed run preserves timestamps (-M) so the destination copy has the
# source's exact mtime; otherwise the incremental skip would depend on
# both writes landing in the same whole second (a race).
result, _ = run_client(source, dest, flags=["-M"], port=shared_server.port)
assert result.returncode == 0
result, _ = run_client(source, dest,
flags=["--remove-source-files", "--incremental"],
port=shared_server.port)
assert result.returncode == 0
assert os.path.isfile(source_file)
class TestArchiveMode:
def test_archive_mode(self, shared_server):
clean_dir(DEST_DIR)
result, dur = run_client(
SOURCE_DIR, DEST_DIR,
flags=["-a"],
port=shared_server.port,
)
if result.returncode != 0:
pytest.fail(f"Exit {result.returncode}: {(result.stderr or result.stdout)[:200]}")
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
mismatches, missing = verify_transfer(SOURCE_DIR, received)
assert not missing, f"Missing: {missing}"
assert not mismatches, f"Mismatch: {mismatches}"
def test_archive_implied_options_can_be_negated(self, shared_server):
clean_dir(DEST_DIR)
result, dur = run_client(
SOURCE_DIR, DEST_DIR,
flags=["--archive", "--no-compress", "--no-m", "--no-preserve"],
port=shared_server.port,
)
if result.returncode != 0:
pytest.fail(f"Exit {result.returncode}: {(result.stderr or result.stdout)[:200]}")
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
mismatches, missing = verify_transfer(SOURCE_DIR, received)
assert not missing, f"Missing: {missing}"
assert not mismatches, f"Mismatch: {mismatches}"
class TestExecutability:
def test_preserves_only_executable_bits(self, shared_server):
source = os.path.join(TEST_DATA_DIR, "executability_source")
dest = os.path.join(TEST_DATA_DIR, "executability_dest")
clean_dir(source)
clean_dir(dest)
source_file = os.path.join(source, "tool.sh")
with open(source_file, "w") as f:
f.write("#!/bin/sh\necho test\n")
os.chmod(source_file, 0o751)
result, _ = run_client(source, dest, flags=["-E"], port=shared_server.port)
assert result.returncode == 0, f"Executability sync failed: {result.stderr[:200]}"
received_file = os.path.join(get_dest_received_dir(dest, source), "tool.sh")
received_mode = os.stat(received_file).st_mode
assert received_mode & 0o111 == 0o111
assert received_mode & 0o600 == 0o600
assert received_mode & 0o077 == 0o011
class TestChmod:
def test_chmod_applies_to_transferred_files(self, shared_server):
clean_dir(DEST_DIR)
source_file = os.path.join(SOURCE_DIR, "small.txt")
os.chmod(source_file, 0o777)
result, dur = run_client(
SOURCE_DIR, DEST_DIR,
flags=["--chmod=u=rw,go=r"],
port=shared_server.port,
)
if result.returncode != 0:
pytest.fail(f"Exit {result.returncode}: {(result.stderr or result.stdout)[:200]}")
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
assert (os.stat(os.path.join(received, "small.txt")).st_mode & 0o777) == 0o644
class TestCompressionChoice:
def test_zstd_choice_compresses(self, shared_server):
clean_dir(DEST_DIR)
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["--zc", "zstd"],
port=shared_server.port)
assert result.returncode == 0, f"zstd sync failed: {(result.stderr or result.stdout)[:200]}"
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
mismatches, missing = verify_transfer(SOURCE_DIR, received)
assert not missing, f"Missing: {missing}"
assert not mismatches, f"Mismatch: {mismatches}"
def test_none_choice_disables_compression(self, shared_server):
clean_dir(DEST_DIR)
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-z", "--compress-choice", "none"],
port=shared_server.port)
assert result.returncode == 0, f"none sync failed: {(result.stderr or result.stdout)[:200]}"
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
mismatches, missing = verify_transfer(SOURCE_DIR, received)
assert not missing, f"Missing: {missing}"
assert not mismatches, f"Mismatch: {mismatches}"
class TestSkipCompress:
def test_skip_compress_case_insensitive(self, shared_server):
clean_dir(DEST_DIR)
with open(os.path.join(SOURCE_DIR, "skip-case.TXT"), "wb") as f:
f.write((b"skip compression case test\n" * 100))
result, _ = run_client(
SOURCE_DIR, DEST_DIR,
flags=["-c", "--skip-compress=.txt"],
port=shared_server.port,
)
assert result.returncode == 0, f"Skip-compress sync failed: {(result.stderr or result.stdout)[:200]}"
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
with open(os.path.join(received, "skip-case.TXT"), "rb") as f:
assert f.read() == b"skip compression case test\n" * 100
def test_skip_compress_empty_list(self, shared_server):
clean_dir(DEST_DIR)
result, _ = run_client(
SOURCE_DIR, DEST_DIR,
flags=["-c", "--skip-compress="],
port=shared_server.port,
)
assert result.returncode == 0, f"Empty skip-compress sync failed: {(result.stderr or result.stdout)[:200]}"
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
mismatches, missing = verify_transfer(SOURCE_DIR, received)
assert not missing, f"Missing: {missing}"
assert not mismatches, f"Mismatch: {mismatches}"
def test_skip_compress_incremental_full_fallback(self, shared_server):
clean_dir(DEST_DIR)
path = os.path.join(SOURCE_DIR, "incremental-skip.TXT")
with open(path, "wb") as f:
f.write(b"original skipped content\n")
flags = ["-c", "-M", "--skip-compress=.txt"]
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=flags, port=shared_server.port)
assert result.returncode == 0, f"Initial sync failed: {(result.stderr or result.stdout)[:200]}"
with open(path, "wb") as f:
f.write(b"updated skipped content\n")
result, _ = run_client(
SOURCE_DIR, DEST_DIR,
flags=flags + ["--incremental"],
port=shared_server.port,
)
assert result.returncode == 0, f"Incremental sync failed: {(result.stderr or result.stdout)[:200]}"
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
with open(os.path.join(received, "incremental-skip.TXT"), "rb") as f:
assert f.read() == b"updated skipped content\n"
def test_skip_compress_rejects_chunk_serialization(self, shared_server):
result, _ = run_client(
SOURCE_DIR, DEST_DIR,
flags=["-c", "-s", "--skip-compress=.txt"],
port=shared_server.port,
)
assert result.returncode != 0
assert "cannot be combined" in (result.stderr or result.stdout)
class TestExclude:
def test_exclude_single(self, shared_server):
clean_dir(DEST_DIR)
result, dur = run_client(
SOURCE_DIR, DEST_DIR,
flags=["--exclude", "small.txt"],
port=shared_server.port,
)
if result.returncode != 0:
pytest.fail(f"Exit {result.returncode}: {(result.stderr or result.stdout)[:200]}")
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
mismatches, missing = verify_transfer(SOURCE_DIR, received)
assert "small.txt" in missing, "small.txt should be excluded but was transferred"
other_missing = [m for m in missing if m != "small.txt"]
assert not other_missing, f"Other files missing: {other_missing}"
assert not mismatches, f"Mismatch: {mismatches}"
def test_exclude_glob(self, shared_server):
clean_dir(DEST_DIR)
result, dur = run_client(
SOURCE_DIR, DEST_DIR,
flags=["--exclude", "*.txt"],
port=shared_server.port,
)
if result.returncode != 0:
pytest.fail(f"Exit {result.returncode}: {(result.stderr or result.stdout)[:200]}")
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
assert not os.path.exists(os.path.join(received, "small.txt")), "small.txt should be excluded"
assert os.path.exists(os.path.join(received, "binary.bin")), "binary.bin should be present"
class TestInclude:
def test_include_single(self, shared_server):
clean_dir(DEST_DIR)
result, dur = run_client(
SOURCE_DIR, DEST_DIR,
flags=["--include", "binary.bin"],
port=shared_server.port,
)
if result.returncode != 0:
pytest.fail(f"Exit {result.returncode}: {(result.stderr or result.stdout)[:200]}")
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
assert os.path.exists(os.path.join(received, "binary.bin")), "binary.bin should be included"
assert not os.path.exists(os.path.join(received, "small.txt")), "small.txt should not be included"
def test_include_glob(self, shared_server):
clean_dir(DEST_DIR)
result, dur = run_client(
SOURCE_DIR, DEST_DIR,
flags=["--include", "*.bin"],
port=shared_server.port,
)
if result.returncode != 0:
pytest.fail(f"Exit {result.returncode}: {(result.stderr or result.stdout)[:200]}")
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
assert os.path.exists(os.path.join(received, "binary.bin")), "binary.bin should be included"
class TestSizeFilters:
def test_max_size(self, shared_server):
clean_dir(DEST_DIR)
result, dur = run_client(
SOURCE_DIR, DEST_DIR,
flags=["--max-size", "100"],
port=shared_server.port,
)
if result.returncode != 0:
pytest.fail(f"Exit {result.returncode}: {(result.stderr or result.stdout)[:200]}")
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
assert os.path.exists(os.path.join(received, "small.txt")), "small.txt should be present"
assert not os.path.exists(os.path.join(received, "medium.txt")), "medium.txt should be skipped"
def test_min_size(self, shared_server):
clean_dir(DEST_DIR)
result, dur = run_client(
SOURCE_DIR, DEST_DIR,
flags=["--min-size", "1000"],
port=shared_server.port,
)
if result.returncode != 0:
pytest.fail(f"Exit {result.returncode}: {(result.stderr or result.stdout)[:200]}")
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
assert not os.path.exists(os.path.join(received, "small.txt")), "small.txt should be skipped"
assert os.path.exists(os.path.join(received, "medium.txt")), "medium.txt should be present"
class TestIncremental:
def test_incremental_skips_unchanged(self, shared_server):
clean_dir(DEST_DIR)
result, dur = run_client(
SOURCE_DIR, DEST_DIR,
flags=["-M"],
port=shared_server.port,
)
assert result.returncode == 0, f"First sync failed: {result.stderr[:100]}"
start = time.monotonic()
result, dur = run_client(
SOURCE_DIR, DEST_DIR,
flags=["-M", "--incremental"],
port=shared_server.port,
)
incremental_time = time.monotonic() - start
assert result.returncode == 0, f"Incremental sync failed: {(result.stderr or result.stdout)[:200]}"
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
mismatches, missing = verify_transfer(SOURCE_DIR, received)
assert not missing, f"Missing: {missing}"
assert not mismatches, f"Mismatch: {mismatches}"
def test_incremental_detects_changes(self, shared_server):
clean_dir(DEST_DIR)
result, _ = run_client(
SOURCE_DIR, DEST_DIR,
flags=["-M"],
port=shared_server.port,
)
assert result.returncode == 0
modified_file = os.path.join(SOURCE_DIR, "small.txt")
with open(modified_file, "wb") as f:
f.write(b"modified content for incremental test\n")
result, dur = run_client(
SOURCE_DIR, DEST_DIR,
flags=["-M", "--incremental"],
port=shared_server.port,
)
assert result.returncode == 0
with open(modified_file, "wb") as f:
f.write(b"hello world\n")
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
received_file = os.path.join(received, "small.txt")
assert os.path.exists(received_file), "Modified file should be present"
with open(received_file, "rb") as f:
content = f.read()
assert b"modified content" in content, f"Modified content not transferred: {content[:50]}"
def test_checksum_detects_same_size_and_mtime_change(self, shared_server):
clean_dir(DEST_DIR)
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port)
assert result.returncode == 0
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
source_file = os.path.join(SOURCE_DIR, "small.txt")
received_file = os.path.join(received, "small.txt")
source_stat = os.stat(source_file)
with open(received_file, "wb") as f:
f.write(b"different!\n")
os.utime(received_file, (source_stat.st_atime, source_stat.st_mtime))
result, _ = run_client(SOURCE_DIR, DEST_DIR,
flags=["-M", "--incremental", "--checksum"],
port=shared_server.port)
assert result.returncode == 0, f"Checksum sync failed: {result.stderr[:200]}"
with open(received_file, "rb") as f:
assert f.read() == b"hello world\n"
def test_size_only_skips_same_size_with_different_mtime(self, shared_server):
clean_dir(DEST_DIR)
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port)
assert result.returncode == 0
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
received_file = os.path.join(received, "small.txt")
with open(received_file, "wb") as f:
f.write(b"different!!\n")
os.utime(received_file, (time.time() - 3600, time.time() - 3600))
result, _ = run_client(
SOURCE_DIR, DEST_DIR,
flags=["-M", "--incremental", "--size-only"],
port=shared_server.port,
)
assert result.returncode == 0, f"Size-only sync failed: {result.stderr[:200]}"
with open(received_file, "rb") as f:
assert f.read() == b"different!!\n"
def test_ignore_times_transfers_same_size_and_mtime(self, shared_server):
clean_dir(DEST_DIR)
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port)
assert result.returncode == 0
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
source_file = os.path.join(SOURCE_DIR, "small.txt")
received_file = os.path.join(received, "small.txt")
source_stat = os.stat(source_file)
with open(received_file, "wb") as f:
f.write(b"stale data!\n")
os.utime(received_file, (source_stat.st_atime, source_stat.st_mtime))
result, _ = run_client(SOURCE_DIR, DEST_DIR,
flags=["-M", "--incremental", "--ignore-times"],
port=shared_server.port)
assert result.returncode == 0, f"Ignore-times sync failed: {result.stderr[:200]}"
with open(received_file, "rb") as f:
assert f.read() == b"hello world\n"
def test_modify_window_allows_subsecond_mtime_difference(self, shared_server):
clean_dir(DEST_DIR)
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port)
assert result.returncode == 0
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
source_file = os.path.join(SOURCE_DIR, "small.txt")
received_file = os.path.join(received, "small.txt")
source_stat = os.stat(source_file)
with open(received_file, "wb") as f:
f.write(b"modified!!!\n")
os.utime(received_file, ns=(source_stat.st_atime_ns,
source_stat.st_mtime_ns - 1500000000))
result, _ = run_client(SOURCE_DIR, DEST_DIR,
flags=["-M", "--incremental", "--modify-window=2"],
port=shared_server.port)
assert result.returncode == 0, f"Modify-window sync failed: {result.stderr[:200]}"
with open(received_file, "rb") as f:
assert f.read() == b"modified!!!\n"
def test_whole_file_disables_delta_and_keeps_compression(self, shared_server):
clean_dir(DEST_DIR)
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port)
assert result.returncode == 0
source_file = os.path.join(SOURCE_DIR, "medium.txt")
with open(source_file, "wb") as f:
f.write(b"whole-file replacement\n" * 5000)
result, _ = run_client(
SOURCE_DIR,
DEST_DIR,
flags=["-M", "--incremental", "--delta", "-W", "-c"],
port=shared_server.port,
)
assert result.returncode == 0, f"Whole-file sync failed: {(result.stderr or result.stdout)[:200]}"
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
mismatches, missing = verify_transfer(SOURCE_DIR, received)
assert not missing, f"Missing: {missing}"
assert not mismatches, f"Mismatch: {mismatches}"
class TestUpdate:
def test_update_skips_older_destination_and_allows_equal_or_newer_source(self, shared_server):
clean_dir(DEST_DIR)
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-u"], port=shared_server.port)
assert result.returncode == 0
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
source_file = os.path.join(SOURCE_DIR, "small.txt")
received_file = os.path.join(received, "small.txt")
source_stat = os.stat(source_file)
with open(received_file, "wb") as f:
f.write(b"newer destination\n")
os.utime(received_file, ns=(source_stat.st_atime_ns, source_stat.st_mtime_ns + 10_000_000_000))
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-u"], port=shared_server.port)
assert result.returncode == 0
with open(received_file, "rb") as f:
assert f.read() == b"newer destination\n"
os.utime(received_file, ns=(source_stat.st_atime_ns, source_stat.st_mtime_ns))
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-u"], port=shared_server.port)
assert result.returncode == 0
with open(received_file, "rb") as f:
assert f.read() == b"hello world\n"
def test_update_skips_unreadable_newer_destination(self, shared_server):
clean_dir(DEST_DIR)
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-u"], port=shared_server.port)
assert result.returncode == 0
received_file = os.path.join(get_dest_received_dir(DEST_DIR, SOURCE_DIR), "small.txt")
source_stat = os.stat(os.path.join(SOURCE_DIR, "small.txt"))
with open(received_file, "wb") as f:
f.write(b"protected destination\n")
os.utime(received_file, ns=(source_stat.st_atime_ns, source_stat.st_mtime_ns + 10_000_000_000))
original_mode = os.stat(received_file).st_mode
try:
os.chmod(received_file, 0)
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-u"], port=shared_server.port)
assert result.returncode == 0
os.chmod(received_file, original_mode)
with open(received_file, "rb") as f:
assert f.read() == b"protected destination\n"
finally:
os.chmod(received_file, original_mode)
with open(received_file, "wb") as f:
f.write(b"older destination\n")
os.utime(received_file, ns=(source_stat.st_atime_ns, source_stat.st_mtime_ns - 10_000_000_000))
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-u"], port=shared_server.port)
assert result.returncode == 0
with open(received_file, "rb") as f:
assert f.read() == b"hello world\n"
class TestExisting:
def test_existing_updates_existing_and_skips_new(self, shared_server):
clean_dir(DEST_DIR)
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port)
assert result.returncode == 0, f"Initial sync failed: {(result.stderr or result.stdout)[:200]}"
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
source_file = os.path.join(SOURCE_DIR, "small.txt")
new_source_file = os.path.join(SOURCE_DIR, "new-existing-test.txt")
with open(source_file, "wb") as f:
f.write(b"updated existing content\n")
with open(new_source_file, "wb") as f:
f.write(b"this file must not be created\n")
try:
result, _ = run_client(SOURCE_DIR, DEST_DIR,
flags=["-M", "--existing"], port=shared_server.port)
assert result.returncode == 0, f"--existing sync failed: {(result.stderr or result.stdout)[:200]}"
with open(os.path.join(received, "small.txt"), "rb") as f:
assert f.read() == b"updated existing content\n"
assert not os.path.exists(os.path.join(received, "new-existing-test.txt"))
finally:
os.unlink(new_source_file)
with open(source_file, "wb") as f:
f.write(b"hello world\n")
class TestIgnoreExisting:
def test_ignore_existing_preserves_existing_and_transfers_new(self, shared_server):
clean_dir(DEST_DIR)
result, _ = run_client(SOURCE_DIR, DEST_DIR, port=shared_server.port)
assert result.returncode == 0
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
existing_file = os.path.join(received, "small.txt")
with open(existing_file, "wb") as f:
f.write(b"destination content\n")
new_source = os.path.join(SOURCE_DIR, "new.txt")
try:
with open(new_source, "wb") as f:
f.write(b"new file\n")
result, _ = run_client(SOURCE_DIR, DEST_DIR,
flags=["--ignore-existing"], port=shared_server.port)
assert result.returncode == 0, f"Sync failed: {(result.stderr or result.stdout)[:200]}"
with open(existing_file, "rb") as f:
assert f.read() == b"destination content\n"
with open(os.path.join(received, "new.txt"), "rb") as f:
assert f.read() == b"new file\n"
finally:
if os.path.lexists(new_source):
os.unlink(new_source)
class TestIgnoreExisting:
def test_ignore_existing_preserves_existing_and_transfers_new(self, shared_server):
clean_dir(DEST_DIR)
result, _ = run_client(SOURCE_DIR, DEST_DIR, port=shared_server.port)
assert result.returncode == 0
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
existing_file = os.path.join(received, "small.txt")
with open(existing_file, "wb") as f:
f.write(b"destination content\n")
new_source = os.path.join(SOURCE_DIR, "new.txt")
try:
with open(new_source, "wb") as f:
f.write(b"new file\n")
result, _ = run_client(SOURCE_DIR, DEST_DIR,
flags=["--ignore-existing"], port=shared_server.port)
assert result.returncode == 0, f"Sync failed: {(result.stderr or result.stdout)[:200]}"
with open(existing_file, "rb") as f:
assert f.read() == b"destination content\n"
with open(os.path.join(received, "new.txt"), "rb") as f:
assert f.read() == b"new file\n"
finally:
if os.path.lexists(new_source):
os.unlink(new_source)
class TestDelete:
def test_delete_removes_extra_files(self, shared_server):
clean_dir(DEST_DIR)
result, _ = run_client(
SOURCE_DIR, DEST_DIR,
flags=["-M"],
port=shared_server.port,
)
assert result.returncode == 0
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
extra_file = os.path.join(received, "extra_file.txt")
extra_dir = os.path.join(received, "extra_dir")
with open(extra_file, "w") as f:
f.write("should be deleted")
os.makedirs(extra_dir, exist_ok=True)
with open(os.path.join(extra_dir, "nested.txt"), "w") as f:
f.write("nested extra")
result, dur = run_client(
SOURCE_DIR, DEST_DIR,
flags=["-M", "--delete"],
port=shared_server.port,
)
assert result.returncode == 0, f"Delete sync failed: {(result.stderr or result.stdout)[:200]}"
# The default server policy intentionally refuses client-requested
# deletion unless it is started with --allow-delete.
assert os.path.exists(extra_file), "unauthorized delete removed an extra file"
assert os.path.exists(extra_dir), "unauthorized delete removed an extra directory"
mismatches, missing = verify_transfer(SOURCE_DIR, received)
assert not missing, f"Missing: {missing}"
assert not mismatches, f"Mismatch: {mismatches}"
class TestProgress:
def test_progress_output(self, shared_server):
clean_dir(DEST_DIR)
result, dur = run_client(
SOURCE_DIR, DEST_DIR,
flags=["--progress"],
port=shared_server.port,
)
assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:100]}"
output = result.stdout + result.stderr
assert "Sent " in output and "MB" in output, "--progress produced no stable byte marker"
assert "Done." in output, "--progress did not report completion"
def test_human_readable_stats(self, shared_server):
clean_dir(DEST_DIR)
result, dur = run_client(
SOURCE_DIR, DEST_DIR,
flags=["-h", "--stats"],
port=shared_server.port,
)
assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:100]}"
assert "Stats:" in result.stderr
assert "KB" in result.stderr
def test_human_readable_progress_multithreaded(self, shared_server):
clean_dir(DEST_DIR)
result, dur = run_client(
SOURCE_DIR, DEST_DIR,
flags=["-m", "-h", "--progress"],
port=shared_server.port,
)
assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:100]}"
output = result.stdout + result.stderr
assert "Sent " in output
assert "KB" in output
assert "Done." in output
class TestInfo:
def test_info_copy_reports_transfers(self, shared_server):
clean_dir(DEST_DIR)
result, _ = run_client(
SOURCE_DIR, DEST_DIR,
flags=["--info=copy"],
port=shared_server.port,
)
assert result.returncode == 0, f"Info sync failed: {(result.stderr or result.stdout)[:200]}"
output = result.stdout + result.stderr
assert "[INFO]" in output and "Transferring" in output
def test_info_stats_reports_multithreaded_transfer(self, shared_server):
clean_dir(DEST_DIR)
result, _ = run_client(
SOURCE_DIR, DEST_DIR,
flags=["-m", "--info=stats"],
port=shared_server.port,
)
assert result.returncode == 0, f"Info stats sync failed: {(result.stderr or result.stdout)[:200]}"
output = result.stdout + result.stderr
assert "[INFO]" in output and "Transfer summary:" in output
def test_info_rejects_unknown_flag(self):
result, _ = run_client(
SOURCE_DIR, DEST_DIR,
flags=["--info=unknown"],
)
assert result.returncode != 0
assert "unsupported --info flag" in result.stderr
@pytest.mark.parametrize("flags", [
["--info=none", "--verbose"],
["--verbose", "--info=none"],
])
def test_info_none_suppresses_verbose_info(self, shared_server, flags):
clean_dir(DEST_DIR)
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=flags, port=shared_server.port)
assert result.returncode == 0, f"Info sync failed: {(result.stderr or result.stdout)[:200]}"
output = result.stdout + result.stderr
assert "[INFO]" not in output
assert "Transferring" not in output
assert "Transfer summary:" not in output
class TestBandwidthLimit:
def test_bwlimit_runs(self, shared_server):
clean_dir(DEST_DIR)
result, dur = run_client(
SOURCE_DIR, DEST_DIR,
flags=["--bwlimit", "10240"],
port=shared_server.port,
)
assert result.returncode == 0, f"Exit {result.returncode}: {(result.stderr or result.stdout)[:200]}"
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
mismatches, missing = verify_transfer(SOURCE_DIR, received)
assert not missing, f"Missing: {missing}"
assert not mismatches, f"Mismatch: {mismatches}"
def _read_file(path):
with open(path, "rb") as fh:
return fh.read()
class TestRemoveSourceFilesSkips:
"""--remove-source-files must not delete sources the receiver skipped
(rsync reference behavior)."""
def test_existing_first_sync_keeps_new_source(self, shared_server):
source = os.path.join(TEST_DATA_DIR, "remove_rsf_existing_src")
dest = os.path.join(TEST_DATA_DIR, "remove_rsf_existing_dst")
clean_dir(source)
clean_dir(dest)
with open(os.path.join(source, "only.txt"), "wb") as f:
f.write(b"keep me")
result, _ = run_client(source, dest, flags=["--remove-source-files", "--existing"],
port=shared_server.port)
assert result.returncode == 0, f"Sync failed: {result.stderr[:200]}"
# The file exists only on the source side, so --existing makes the
# receiver skip it; the source must therefore not be removed.
assert os.path.isfile(os.path.join(source, "only.txt"))
received = get_dest_received_dir(dest, source)
assert not os.path.exists(os.path.join(received, "only.txt"))
def test_ignore_existing_keeps_skipped_source(self, shared_server):
source = os.path.join(TEST_DATA_DIR, "remove_rsf_ignore_src")
dest = os.path.join(TEST_DATA_DIR, "remove_rsf_ignore_dst")
clean_dir(source)
clean_dir(dest)
source_file = os.path.join(source, "file.txt")
with open(source_file, "wb") as f:
f.write(b"payload")
result, _ = run_client(source, dest, port=shared_server.port)
assert result.returncode == 0
result, _ = run_client(source, dest, flags=["--remove-source-files", "--ignore-existing"],
port=shared_server.port)
assert result.returncode == 0, f"Sync failed: {result.stderr[:200]}"
# Destination already has the file, so the second run is a receiver
# skip; the source file must survive.
assert os.path.isfile(source_file)
received = get_dest_received_dir(dest, source)
assert _read_file(os.path.join(received, "file.txt")) == b"payload"
def test_update_newer_destination_keeps_source(self, shared_server):
source = os.path.join(TEST_DATA_DIR, "remove_rsf_update_src")
dest = os.path.join(TEST_DATA_DIR, "remove_rsf_update_dst")
clean_dir(source)
clean_dir(dest)
source_file = os.path.join(source, "file.txt")
with open(source_file, "wb") as f:
f.write(b"source payload")
result, _ = run_client(source, dest, port=shared_server.port)
assert result.returncode == 0
received = get_dest_received_dir(dest, source)
received_file = os.path.join(received, "file.txt")
with open(received_file, "wb") as f:
f.write(b"newer destination payload")
os.utime(received_file, ns=(time.time_ns() + 10**9, time.time_ns() + 10**9))
result, _ = run_client(source, dest, flags=["--remove-source-files", "--update"],
port=shared_server.port)
assert result.returncode == 0, f"Sync failed: {result.stderr[:200]}"
# --update skips a destination that is newer than the source, so the
# source must not be removed.
assert os.path.isfile(source_file)
assert _read_file(received_file) == b"newer destination payload"
def test_multithreaded_ignore_existing_keeps_skipped_source(self, shared_server):
"""The multithreaded writer path must also report per-file outcomes so a
--remove-source-files sender does not delete skipped sources."""
source = os.path.join(TEST_DATA_DIR, "remove_rsf_mt_ignore_src")
dest = os.path.join(TEST_DATA_DIR, "remove_rsf_mt_ignore_dst")
clean_dir(source)
clean_dir(dest)
source_file = os.path.join(source, "file.txt")
with open(source_file, "wb") as f:
f.write(b"payload")
result, _ = run_client(source, dest, port=shared_server.port)
assert result.returncode == 0
result, _ = run_client(source, dest,
flags=["--remove-source-files", "--ignore-existing", "-m"],
port=shared_server.port)
assert result.returncode == 0, f"Sync failed: {result.stderr[:200]}"
# Destination already has the file, so the receiver (writer thread)
# skips it; the source must survive.
assert os.path.isfile(source_file)
received = get_dest_received_dir(dest, source)
assert _read_file(os.path.join(received, "file.txt")) == b"payload"
class TestBackup:
def _sync(self, source, dest, flags, port):
return run_client(source, dest, flags=flags, port=port)
def test_plain_backup_keeps_previous_version(self, shared_server):
source = os.path.join(TEST_DATA_DIR, "backup_src")
dest = os.path.join(TEST_DATA_DIR, "backup_dst")
clean_dir(source)
clean_dir(dest)
source_file = os.path.join(source, "f.txt")
with open(source_file, "wb") as f:
f.write(b"AAAA")
result, _ = self._sync(source, dest, ["--backup"], shared_server.port)
assert result.returncode == 0, f"Backup sync failed: {result.stderr[:200]}"
with open(source_file, "wb") as f:
f.write(b"BBBB")
result, _ = self._sync(source, dest, ["--backup"], shared_server.port)
assert result.returncode == 0, f"Backup sync failed: {result.stderr[:200]}"
received = get_dest_received_dir(dest, source)
assert _read_file(os.path.join(received, "f.txt")) == b"BBBB"
# rsync default suffix "~" keeps the overwritten version.
assert _read_file(os.path.join(received, "f.txt~")) == b"AAAA"
def test_backup_custom_suffix(self, shared_server):
source = os.path.join(TEST_DATA_DIR, "backup_suffix_src")
dest = os.path.join(TEST_DATA_DIR, "backup_suffix_dst")
clean_dir(source)
clean_dir(dest)
source_file = os.path.join(source, "f.txt")
with open(source_file, "wb") as f:
f.write(b"AAAA")
flags = ["--backup", "--suffix", ".bak"]
result, _ = self._sync(source, dest, flags, shared_server.port)
assert result.returncode == 0, f"Backup sync failed: {result.stderr[:200]}"
with open(source_file, "wb") as f:
f.write(b"BBBB")
result, _ = self._sync(source, dest, flags, shared_server.port)
assert result.returncode == 0, f"Backup sync failed: {result.stderr[:200]}"
received = get_dest_received_dir(dest, source)
assert _read_file(os.path.join(received, "f.txt")) == b"BBBB"
assert _read_file(os.path.join(received, "f.txt.bak")) == b"AAAA"
def test_backup_dir_stores_backups_separately(self, shared_server):
source = os.path.join(TEST_DATA_DIR, "backup_dir_src")
dest = os.path.join(TEST_DATA_DIR, "backup_dir_dst")
clean_dir(source)
clean_dir(dest)
source_file = os.path.join(source, "f.txt")
with open(source_file, "wb") as f:
f.write(b"AAAA")
flags = ["--backup", "--backup-dir", "backups"]
result, _ = self._sync(source, dest, flags, shared_server.port)
assert result.returncode == 0, f"Backup sync failed: {result.stderr[:200]}"
with open(source_file, "wb") as f:
f.write(b"BBBB")
result, _ = self._sync(source, dest, flags, shared_server.port)
assert result.returncode == 0, f"Backup sync failed: {result.stderr[:200]}"
received = get_dest_received_dir(dest, source)
assert _read_file(os.path.join(received, "f.txt")) == b"BBBB"
backup = os.path.join(dest, "backups", os.path.relpath(source_file, os.path.sep))
assert _read_file(backup) == b"AAAA"
class TestPartialDir:
def test_completed_transfer_installed_in_destination(self, shared_server):
source = os.path.join(TEST_DATA_DIR, "partial_src")
dest = os.path.join(TEST_DATA_DIR, "partial_dst")
clean_dir(source)
clean_dir(dest)
source_file = os.path.join(source, "f.txt")
with open(source_file, "wb") as f:
f.write(b"partial payload")
result, _ = run_client(source, dest, flags=["--partial", "--partial-dir", ".partial"],
port=shared_server.port)
assert result.returncode == 0, f"Partial sync failed: {result.stderr[:200]}"
received = get_dest_received_dir(dest, source)
assert _read_file(os.path.join(received, "f.txt")) == b"partial payload"
# A completed transfer must not remain under the partial directory.
partial = os.path.join(dest, ".partial", os.path.relpath(source_file, os.path.sep))
assert not os.path.exists(partial)
class TestLargeFile:
def test_transfer_100mb_file(self, shared_server):
source = os.path.join(TEST_DATA_DIR, "large_src")
dest = os.path.join(TEST_DATA_DIR, "large_dst")
clean_dir(source)
clean_dir(dest)
source_file = os.path.join(source, "big.bin")
chunk = os.urandom(1024 * 1024)
with open(source_file, "wb") as f:
for _ in range(100):
f.write(chunk)
result, _ = run_client(source, dest, port=shared_server.port)
assert result.returncode == 0, f"Large-file sync failed: {result.stderr[:200]}"
received = get_dest_received_dir(dest, source)
assert filecmp.cmp(source_file, os.path.join(received, "big.bin"), shallow=False)
class TestOneFileSystem:
def _make_tree(self, source):
clean_dir(source)
os.makedirs(os.path.join(source, "nested", "deeper"))
with open(os.path.join(source, "root.txt"), "wb") as f:
f.write(b"root")
with open(os.path.join(source, "nested", "inner.txt"), "wb") as f:
f.write(b"inner")
with open(os.path.join(source, "nested", "deeper", "deep.txt"), "wb") as f:
f.write(b"deep")
def _assert_full_tree_transferred(self, source, dest, port, flags):
clean_dir(dest)
result, _ = run_client(source, dest, flags=flags, port=port)
assert result.returncode == 0, f"Sync failed: {result.stderr[:200]}"
received = get_dest_received_dir(dest, source)
mismatches, missing = verify_transfer(source, received)
assert not missing, f"Missing: {missing}"
assert not mismatches, f"Mismatch: {mismatches}"
def test_x_transfer_matches_plain_over_single_filesystem(self, shared_server):
source = os.path.join(TEST_DATA_DIR, "ofs_src")
self._make_tree(source)
self._assert_full_tree_transferred(source, os.path.join(TEST_DATA_DIR, "ofs_dst"),
shared_server.port, ["-x"])
def test_x_multithreaded_transfer_matches_plain_over_single_filesystem(self, shared_server):
source = os.path.join(TEST_DATA_DIR, "ofs_m_src")
self._make_tree(source)
self._assert_full_tree_transferred(source, os.path.join(TEST_DATA_DIR, "ofs_m_dst"),
shared_server.port, ["-m", "--one-file-system"])
def test_x_skips_other_device_mountpoint(self, shared_server):
if os.geteuid() != 0 or shutil.which("mount") is None or shutil.which("umount") is None:
pytest.skip("cross-device test requires root and mount(8)")
source = os.path.join(TEST_DATA_DIR, "ofs_mnt_src")
dest = os.path.join(TEST_DATA_DIR, "ofs_mnt_dst")
dest_plain = os.path.join(TEST_DATA_DIR, "ofs_mnt_plain_dst")
mountpoint = os.path.join(source, "external")
clean_dir(source)
clean_dir(dest)
clean_dir(dest_plain)
os.makedirs(mountpoint)
os.makedirs(os.path.join(source, "nested"))
with open(os.path.join(source, "root.txt"), "wb") as f:
f.write(b"root")
with open(os.path.join(source, "nested", "inner.txt"), "wb") as f:
f.write(b"inner")
mounted = False
unmount_error = ""
try:
mount = subprocess.run(["mount", "-t", "tmpfs", "tmpfs", mountpoint],
capture_output=True, text=True)
if mount.returncode != 0:
pytest.skip(f"cannot mount tmpfs: {mount.stderr.strip()}")
mounted = True
with open(os.path.join(mountpoint, "away.txt"), "wb") as f:
f.write(b"cross device")
result, _ = run_client(source, dest, flags=["-x"], port=shared_server.port)
assert result.returncode == 0, f"-x sync failed: {result.stderr[:200]}"
received = get_dest_received_dir(dest, source)
assert os.path.isfile(os.path.join(received, "root.txt"))
assert os.path.isfile(os.path.join(received, "nested", "inner.txt"))
assert not os.path.exists(os.path.join(received, "external", "away.txt")), \
"-x must not cross into the mounted filesystem"
result, _ = run_client(source, dest_plain, port=shared_server.port)
assert result.returncode == 0, f"plain sync failed: {result.stderr[:200]}"
received_plain = get_dest_received_dir(dest_plain, source)
assert os.path.isfile(os.path.join(received_plain, "external", "away.txt")), \
"without -x the mounted subtree must be transferred"
finally:
if mounted:
umount = subprocess.run(["umount", mountpoint], capture_output=True, text=True)
if umount.returncode != 0:
unmount_error = umount.stderr.strip()
if unmount_error:
pytest.fail(f"test mountpoint {mountpoint} still mounted after umount: {unmount_error}")
def _walk_tmp_files(root):
"""Recursively list *.tmp* leftovers under root (empty if root missing)."""
leftovers = []
if not os.path.isdir(root):
return leftovers
for base, _, files in os.walk(root):
for name in files:
if ".tmp." in name:
leftovers.append(os.path.join(base, name))
return leftovers
class TestTempDir:
"""--temp-dir=DIR puts the receiver's temporary working copies in a scratch
directory below the destination root and atomically renames each completed
file into its final destination. Files sharing a basename across
directories exercise the flat scratch namespace."""
def _make_source(self, name):
source = os.path.join(TEST_DATA_DIR, name)
clean_dir(source)
entries = {
"top.txt": b"top level\n",
"sub/file.txt": b"nested file\n" * 20,
"other/file.txt": b"other nested file\n",
"sub/deep.bin": bytes(range(256)) * 8,
}
for rel, content in entries.items():
full = os.path.join(source, rel)
os.makedirs(os.path.dirname(full), exist_ok=True)
with open(full, "wb") as fh:
fh.write(content)
return source
def _assert_clean_scratch(self, scratch):
assert os.path.isdir(scratch), f"scratch dir {scratch} was not created"
leftovers = _walk_tmp_files(scratch)
assert leftovers == [], f"leftover temp files in scratch dir: {leftovers}"
@pytest.mark.parametrize("mt", [False, True])
def test_temp_dir_scratch(self, shared_server, mt):
source = self._make_source("tempdir_src")
dest = os.path.join(TEST_DATA_DIR, "tempdir_dst")
clean_dir(dest)
flags = ["--temp-dir=scratch"] + (["-m"] if mt else [])
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
assert result.returncode == 0, f"temp-dir sync failed: {result.stderr[:200]}"
received = get_dest_received_dir(dest, source)
mismatches, missing = verify_transfer(source, received)
assert not missing, f"Missing: {missing}"
assert not mismatches, f"Mismatch: {mismatches}"
self._assert_clean_scratch(os.path.join(dest, "scratch"))
def test_default_behavior_has_no_scratch_dir(self, shared_server):
source = self._make_source("tempdir_default_src")
dest = os.path.join(TEST_DATA_DIR, "tempdir_default_dst")
clean_dir(dest)
result, _ = run_client(source, dest, port=shared_server.port)
assert result.returncode == 0, f"Default sync failed: {result.stderr[:200]}"
received = get_dest_received_dir(dest, source)
mismatches, missing = verify_transfer(source, received)
assert not missing, f"Missing: {missing}"
assert not mismatches, f"Mismatch: {mismatches}"
assert not os.path.exists(os.path.join(dest, "scratch"))
def test_temp_dir_ignored_with_inplace(self, shared_server):
"""--inplace writes directly into the destination; --temp-dir must not
redirect those writes into a scratch dir."""
source = self._make_source("tempdir_inplace_src")
dest = os.path.join(TEST_DATA_DIR, "tempdir_inplace_dst")
clean_dir(dest)
result, _ = run_client(source, dest,
flags=["--inplace", "--temp-dir=scratch"],
port=shared_server.port)
assert result.returncode == 0, f"inplace+temp-dir sync failed: {result.stderr[:200]}"
received = get_dest_received_dir(dest, source)
mismatches, missing = verify_transfer(source, received)
assert not missing, f"Missing: {missing}"
assert not mismatches, f"Mismatch: {mismatches}"
assert not os.path.exists(os.path.join(dest, "scratch")), \
"--inplace wrote through the scratch dir"
def test_temp_dir_ignored_with_partial_dir(self, shared_server):
"""--partial --partial-dir already stages in a separate directory;
--temp-dir must not be used on top of it."""
source = self._make_source("tempdir_partial_src")
dest = os.path.join(TEST_DATA_DIR, "tempdir_partial_dst")
clean_dir(dest)
result, _ = run_client(source, dest,
flags=["--partial", "--partial-dir", ".partial",
"--temp-dir=scratch"],
port=shared_server.port)
assert result.returncode == 0, f"partial+temp-dir sync failed: {result.stderr[:200]}"
received = get_dest_received_dir(dest, source)
mismatches, missing = verify_transfer(source, received)
assert not missing, f"Missing: {missing}"
assert not mismatches, f"Mismatch: {mismatches}"
partial = os.path.join(dest, ".partial",
os.path.relpath(os.path.join(source, "top.txt"), os.path.sep))
assert not os.path.exists(partial), "completed file remained under the partial dir"
assert not os.path.exists(os.path.join(dest, "scratch")), \
"--partial-dir wrote through the scratch dir"
def test_temp_dir_escape_rejected(self, shared_server):
source = self._make_source("tempdir_escape_src")
dest = os.path.join(TEST_DATA_DIR, "tempdir_escape_dst")
clean_dir(dest)
# "../escape" would resolve one level above the destination root.
outside = os.path.join(TEST_DATA_DIR, "escape")
assert not os.path.lexists(outside)
result, _ = run_client(source, dest, flags=["--temp-dir=../escape"],
port=shared_server.port)
assert result.returncode != 0, "relative escaping --temp-dir was not rejected"
assert not os.path.lexists(outside), "file created outside the destination root"
clean_dir(dest)
abs_escape = os.path.join(TEST_DATA_DIR, "abs_escape_probe")
assert not os.path.lexists(abs_escape)
result, _ = run_client(source, dest, flags=["--temp-dir", abs_escape],
port=shared_server.port)
assert result.returncode != 0, "absolute --temp-dir was not rejected"
assert not os.path.lexists(abs_escape), "file created outside the destination root"
def _source_files():
"""All source paths (absolute) that a transfer would send right now."""
return [
os.path.join(root, name)
for root, _dirs, names in os.walk(SOURCE_DIR)
for name in names
]
class TestListOnly:
"""--list-only prints every transfer candidate and changes nothing."""
def test_list_only_prints_each_file_and_does_not_transfer(self):
clean_dir(DEST_DIR)
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["--list-only"])
assert result.returncode == 0, f"list-only failed: {result.stderr[:200]}"
for full_path in _source_files():
assert full_path in result.stdout, f"list-only omitted {full_path}"
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
assert not os.path.exists(received), "list-only wrote to the destination"
def test_list_only_with_dry_run_does_not_error(self):
clean_dir(DEST_DIR)
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["--list-only", "--dry-run"])
assert result.returncode == 0, f"list-only -n failed: {result.stderr[:200]}"
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
assert not os.path.exists(received)
def test_list_only_multithreaded(self):
clean_dir(DEST_DIR)
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["--list-only", "-m"])
assert result.returncode == 0, f"list-only -m failed: {result.stderr[:200]}"
for full_path in _source_files():
assert full_path in result.stdout, f"list-only -m omitted {full_path}"
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
assert not os.path.exists(received), "list-only -m wrote to the destination"
class TestItemizeChanges:
"""-i/--itemize-changes prints rsync-style lines only for files sent."""
def test_first_run_prints_sent_lines(self, shared_server):
clean_dir(DEST_DIR)
result, _ = run_client(SOURCE_DIR, DEST_DIR,
flags=["-M", "-i"], port=shared_server.port)
assert result.returncode == 0, f"itemize sync failed: {result.stderr[:200]}"
sent_lines = {">f+++++++++ " + p for p in _source_files()}
assert sent_lines <= set(result.stdout.splitlines()), (
f"missing itemize lines; got {result.stdout[:500]}"
)
def test_incremental_second_run_prints_no_line_for_unchanged(self, shared_server):
clean_dir(DEST_DIR)
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port)
assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}"
result, _ = run_client(SOURCE_DIR, DEST_DIR,
flags=["-M", "-i", "--incremental"],
port=shared_server.port)
assert result.returncode == 0, f"incremental itemize failed: {result.stderr[:200]}"
itemized = [line for line in result.stdout.splitlines() if line and line[0] in ">.<c"]
assert itemized == [], f"unchanged files were itemized: {itemized[:5]}"
def test_multithreaded_emits_same_itemize_lines(self, shared_server):
clean_dir(DEST_DIR)
result, _ = run_client(SOURCE_DIR, DEST_DIR,
flags=["-M", "-i", "-m"], port=shared_server.port)
assert result.returncode == 0, f"itemize -m sync failed: {result.stderr[:200]}"
sent_lines = {">f+++++++++ " + p for p in _source_files()}
assert sent_lines <= set(result.stdout.splitlines()), (
f"missing itemize lines in -m mode; got {result.stdout[:500]}"
)
def test_dry_run_with_itemize_does_not_error(self):
clean_dir(DEST_DIR)
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-i", "--dry-run"])
assert result.returncode == 0, f"dry-run -i failed: {result.stderr[:200]}"
def test_changed_file_on_second_incremental_run_prints_exactly_one_line(self, shared_server):
"""A changed file itemizes exactly once on an incremental rerun while
unchanged files print nothing (no double emission)."""
source = os.path.join(TEST_DATA_DIR, "itemize_change_src")
dest = os.path.join(TEST_DATA_DIR, "itemize_change_dst")
clean_dir(source)
clean_dir(dest)
changed = os.path.join(source, "changed.txt")
untouched = os.path.join(source, "untouched.txt")
with open(changed, "wb") as fh:
fh.write(b"original\n")
with open(untouched, "wb") as fh:
fh.write(b"stable\n")
result, _ = run_client(source, dest, flags=["-M"], port=shared_server.port)
assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}"
with open(changed, "wb") as fh:
fh.write(b"edited payload\n")
result, _ = run_client(source, dest,
flags=["-M", "-i", "--incremental"],
port=shared_server.port)
assert result.returncode == 0, f"incremental itemize failed: {result.stderr[:200]}"
itemized = [line for line in result.stdout.splitlines() if line.startswith(">f")]
assert itemized == [">f+++++++++ " + changed], (
f"expected exactly one itemize line for {changed}, got {itemized}"
)
received = get_dest_received_dir(dest, source)
assert _read_file(os.path.join(received, "changed.txt")) == b"edited payload\n"
assert _read_file(os.path.join(received, "untouched.txt")) == b"stable\n"
class TestOutFormat:
"""--out-format prints a line per transferred file using the template."""
def test_out_format_path_and_size(self, shared_server):
clean_dir(DEST_DIR)
result, _ = run_client(SOURCE_DIR, DEST_DIR,
flags=["--out-format=%f %l"], port=shared_server.port)
assert result.returncode == 0, f"out-format sync failed: {result.stderr[:200]}"
expected = {f"{p} {os.path.getsize(p)}" for p in _source_files()}
got = set(result.stdout.splitlines())
assert expected <= got, f"out-format lines missing: expected {len(expected)} got {len(got)}"
def test_out_format_multithreaded_matches_single(self, shared_server):
clean_dir(DEST_DIR)
result, _ = run_client(SOURCE_DIR, DEST_DIR,
flags=["--out-format=%f %l", "-m"], port=shared_server.port)
assert result.returncode == 0, f"out-format -m sync failed: {result.stderr[:200]}"
expected = {f"{p} {os.path.getsize(p)}" for p in _source_files()}
got = set(result.stdout.splitlines())
assert expected <= got, f"out-format -m lines missing: {result.stdout[:500]}"
class TestLogFileFormat:
"""--log-file plus --log-file-format writes per-file lines to the log."""
def test_log_file_format_writes_transferred_files(self, shared_server):
clean_dir(DEST_DIR)
log_path = os.path.join(TEST_DATA_DIR, "itemize_transfer.log")
if os.path.exists(log_path):
os.unlink(log_path)
result, _ = run_client(
SOURCE_DIR, DEST_DIR,
flags=["--log-file", log_path, "--log-file-format=%f %l"],
port=shared_server.port,
)
assert result.returncode == 0, f"log-file sync failed: {result.stderr[:200]}"
assert os.path.exists(log_path), "--log-file created no log"
with open(log_path, encoding="utf-8", errors="replace") as fh:
content = fh.read()
expected = {f"{p} {os.path.getsize(p)}" for p in _source_files()}
for line in expected:
assert line in content, f"log file missing {line!r}"
def test_log_file_format_multithreaded_writes_transferred_files(self, shared_server):
source = os.path.join(TEST_DATA_DIR, "itemize_log_mt_src")
dest = os.path.join(TEST_DATA_DIR, "itemize_log_mt_dst")
clean_dir(source)
clean_dir(dest)
files = {"a.txt": b"alpha\n", "b.txt": b"beta\n"}
for rel, data in files.items():
with open(os.path.join(source, rel), "wb") as fh:
fh.write(data)
log_path = os.path.join(TEST_DATA_DIR, "itemize_mt.log")
if os.path.exists(log_path):
os.unlink(log_path)
result, _ = run_client(
source,
dest,
flags=["--log-file", log_path, "--log-file-format=%f %l", "-m"],
port=shared_server.port,
)
assert result.returncode == 0, f"log-file -m sync failed: {result.stderr[:200]}"
assert os.path.exists(log_path), "--log-file created no log"
with open(log_path, encoding="utf-8", errors="replace") as fh:
content = fh.read()
expected = {f"{os.path.join(source, rel)} {len(data)}" for rel, data in files.items()}
for line in expected:
assert line in content, f"log file (-m) missing {line!r}"
class TestDelayUpdates:
"""--delay-updates stages every updated file under a private 0700 staging
directory inside the receive root and atomically publishes all of them only
after the whole transfer succeeds."""
STAGING = ".fastsync-stage"
def _make_source(self, name):
source = os.path.join(TEST_DATA_DIR, name)
clean_dir(source)
entries = {
"top.txt": b"top level\n",
"sub/deep.txt": b"deeply nested file\n",
"sub/another.txt": b"another nested file\n" * 20,
"binary.bin": bytes(range(256)) * 4,
}
for rel, content in entries.items():
full = os.path.join(source, rel)
os.makedirs(os.path.dirname(full), exist_ok=True)
with open(full, "wb") as fh:
fh.write(content)
return source
@pytest.mark.parametrize("mt", [False, True])
def test_delay_updates_matches_plain_transfer(self, shared_server, mt):
source = self._make_source("delay_match_src")
plain_dest = os.path.join(TEST_DATA_DIR, "delay_match_plain_dst")
delay_dest = os.path.join(TEST_DATA_DIR, "delay_match_delay_dst")
clean_dir(plain_dest)
clean_dir(delay_dest)
result, _ = run_client(source, plain_dest, port=shared_server.port)
assert result.returncode == 0, f"plain sync failed: {result.stderr[:200]}"
flags = ["--delay-updates"] + (["-m"] if mt else [])
result, _ = run_client(source, delay_dest, flags=flags, port=shared_server.port)
assert result.returncode == 0, f"delay-updates sync failed: {result.stderr[:200]}"
plain_received = get_dest_received_dir(plain_dest, source)
delay_received = get_dest_received_dir(delay_dest, source)
mismatches, missing = verify_transfer(source, delay_received)
assert not missing, f"Missing: {missing}"
assert not mismatches, f"Mismatch: {mismatches}"
for root, _dirs, files in os.walk(delay_received):
for name in files:
rel = os.path.relpath(os.path.join(root, name), delay_received)
assert filecmp.cmp(os.path.join(plain_received, rel),
os.path.join(delay_received, rel), shallow=False), rel
assert not os.path.isdir(os.path.join(delay_dest, self.STAGING)), \
"staging directory left behind after a successful delayed transfer"
@pytest.mark.parametrize("mt", [False, True])
def test_delay_updates_incremental_rerun_no_leftovers(self, shared_server, mt):
source = self._make_source("delay_rerun_src")
dest = os.path.join(TEST_DATA_DIR, "delay_rerun_dst")
clean_dir(dest)
flags = ["--delay-updates", "-M", "--incremental"] + (["-m"] if mt else [])
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
assert result.returncode == 0, f"first delayed sync failed: {result.stderr[:200]}"
received = get_dest_received_dir(dest, source)
mismatches, missing = verify_transfer(source, received)
assert not missing and not mismatches
assert not os.path.isdir(os.path.join(dest, self.STAGING))
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
assert result.returncode == 0, f"second delayed sync failed: {result.stderr[:200]}"
assert not os.path.isdir(os.path.join(dest, self.STAGING)), \
"fully-skipped delayed run left a staging directory"
@pytest.mark.parametrize("mt", [False, True])
def test_remove_source_files_with_delay_updates(self, shared_server, mt):
source = self._make_source("delay_rsf_src")
dest = os.path.join(TEST_DATA_DIR, "delay_rsf_dst")
clean_dir(dest)
flags = ["--remove-source-files", "--delay-updates"] + (["-m"] if mt else [])
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
assert result.returncode == 0, f"delayed remove-source sync failed: {result.stderr[:200]}"
# Sources are removed only after the receiver published every file.
for root, _dirs, files in os.walk(source):
assert files == [], f"source files survived delayed remove-source-files: {files}"
received = get_dest_received_dir(dest, source)
assert os.path.isfile(os.path.join(received, "top.txt"))
assert os.path.isfile(os.path.join(received, "sub", "deep.txt"))
assert not os.path.isdir(os.path.join(dest, self.STAGING))
@pytest.mark.parametrize("mt", [False, True])
def test_delete_with_delay_updates(self, mt):
"""--delete runs before publication, so the delete walker must not treat
the staging directory as a set of extras: a changed file must still be
published after genuine extras are removed. Uses its own server started
with --allow-delete (the shared session server refuses deletion)."""
source = os.path.join(TEST_DATA_DIR, "delay_delete_src")
dest = os.path.join(TEST_DATA_DIR, "delay_delete_dst")
clean_dir(source)
clean_dir(dest)
with open(os.path.join(source, "f.txt"), "wb") as fh:
fh.write(b"AAAA")
with open(os.path.join(source, "extra.txt"), "wb") as fh:
fh.write(b"seed extra")
with ServerManager() as server:
server.start(extra_args=["--allow-delete"])
result, _ = run_client(source, dest, port=server.port)
assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}"
received = get_dest_received_dir(dest, source)
assert _read_file(os.path.join(received, "extra.txt")) == b"seed extra"
# Second source state: f.txt changed, extra.txt removed from source.
with open(os.path.join(source, "f.txt"), "wb") as fh:
fh.write(b"BBBB")
os.remove(os.path.join(source, "extra.txt"))
flags = ["--delete", "--delay-updates"] + (["-m"] if mt else [])
result, _ = run_client(source, dest, flags=flags, port=server.port)
assert result.returncode == 0, \
f"delete+delay-updates sync failed: {result.stderr[:200]}"
assert _read_file(os.path.join(received, "f.txt")) == b"BBBB", \
"changed file was not published after deletion"
assert not os.path.exists(os.path.join(received, "extra.txt")), \
"genuine extra file was not deleted"
assert not os.path.isdir(os.path.join(dest, self.STAGING))
def test_delay_updates_rejects_reserved_backup_dir(self):
"""--backup-dir equal to the internal staging name must be rejected so
an old backup can never be silently installed as the "new" file."""
source = self._make_source("delay_reserved_bak_src")
for variant, suffix in (("bare", ""), ("slash", "/")):
dest = os.path.join(TEST_DATA_DIR, f"delay_reserved_bak_{variant}_dst")
clean_dir(dest)
flags = ["--delay-updates", "--backup", "--backup-dir",
".fastsync-stage" + suffix]
result, _ = run_client(source, dest, flags=flags, port=None)
assert result.returncode != 0, \
f"reserved --backup-dir '{suffix}' was accepted"
assert not os.path.isdir(os.path.join(dest, self.STAGING)), \
"staging directory created by a rejected run"
@pytest.mark.parametrize("remove_source_files", [False, True])
@pytest.mark.parametrize("mt", [False, True])
def test_mid_publish_failure_keeps_published_no_rollback(self, shared_server, mt,
remove_source_files):
"""A stage->publish rename failing part way through publication must
fail the whole transfer, keep the already-published top-level file (no
rollback), leave the not-yet-published nested file absent, and clean up
the staging area. A regular file is planted where the final "sub"
directory must be created, so the nested rename fails (mkdir over a
file is impossible even for root) while the top-level file, which is
always staged first, publishes. With --remove-source-files the sender
must keep every source because no success/outcome frame is ever sent."""
source = os.path.join(TEST_DATA_DIR, "delay_mid_src")
dest = os.path.join(TEST_DATA_DIR, "delay_mid_dst")
clean_dir(source)
clean_dir(dest)
top_path = os.path.join(source, "top.txt")
deep_path = os.path.join(source, "sub", "deep.txt")
with open(top_path, "wb") as fh:
fh.write(b"top payload\n")
os.makedirs(os.path.dirname(deep_path))
with open(deep_path, "wb") as fh:
fh.write(b"deep payload\n")
received = get_dest_received_dir(dest, source)
os.makedirs(received)
with open(os.path.join(received, "sub"), "wb") as fh:
fh.write(b"blocks the nested destination directory")
flags = ["--delay-updates"] + (["-m"] if mt else [])
if remove_source_files:
flags += ["--remove-source-files"]
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
assert result.returncode != 0, "blocked nested publish did not fail"
# The top-level file was published before the nested rename failed and
# is intentionally NOT rolled back.
assert _read_file(os.path.join(received, "top.txt")) == b"top payload\n"
# The nested file was never published.
assert not os.path.lexists(os.path.join(received, "sub", "deep.txt")), \
"nested file appeared despite a failed publish"
assert not os.path.isdir(os.path.join(dest, self.STAGING)), \
"staging leftovers after a failed mid-publish"
# Sources survive: no success frame was sent, so a remove-source-files
# sender must not delete anything.
assert os.path.isfile(top_path)
assert os.path.isfile(deep_path)
@pytest.mark.parametrize("mt", [False, True])
def test_remove_source_files_keeps_receiver_skipped_source(self, shared_server, mt):
"""With --delay-updates + --ignore-existing a receiver-skipped source
must survive (its outcome is sent only after publication) while a
freshly delivered file is published and its source removed."""
source = os.path.join(TEST_DATA_DIR, "delay_rsf_skip_src")
dest = os.path.join(TEST_DATA_DIR, "delay_rsf_skip_dst")
clean_dir(source)
clean_dir(dest)
with open(os.path.join(source, "keep.txt"), "wb") as fh:
fh.write(b"existing on dest")
result, _ = run_client(source, dest, port=shared_server.port)
assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}"
with open(os.path.join(source, "keep.txt"), "wb") as fh:
fh.write(b"changed on source")
with open(os.path.join(source, "deliver.txt"), "wb") as fh:
fh.write(b"new file")
flags = ["--remove-source-files", "--ignore-existing", "--delay-updates"] + (["-m"] if mt else [])
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
assert result.returncode == 0, f"delayed skip sync failed: {result.stderr[:200]}"
# keep.txt already existed at the destination: receiver skip -> source stays.
assert os.path.isfile(os.path.join(source, "keep.txt")), \
"receiver-skipped source was removed despite --ignore-existing"
# deliver.txt was new: staged, published, and its source removed.
assert not os.path.isfile(os.path.join(source, "deliver.txt")), \
"published source was not removed"
received = get_dest_received_dir(dest, source)
assert not os.path.isdir(os.path.join(dest, self.STAGING))
def test_delay_updates_rejects_inplace(self):
source = self._make_source("delay_inplace_src")
dest = os.path.join(TEST_DATA_DIR, "delay_inplace_dst")
clean_dir(dest)
result, _ = run_client(source, dest, flags=["--delay-updates", "--inplace"])
assert result.returncode != 0, "--inplace with --delay-updates was accepted"
assert not os.path.isdir(os.path.join(dest, self.STAGING))
class TestFilesFrom:
"""--files-from transfers exactly the listed files; a listed directory
transfers its whole subtree. The manifest (and thus --delete) derives from
what was actually sent."""
def _make_relative_source(name):
"""A small tree used by the -R/--dirs/--no-implied-dirs tests."""
source = os.path.join(TEST_DATA_DIR, name)
clean_dir(source)
entries = {
"top.txt": b"top\n",
"a/b.txt": b"nested\n",
"sub/x.txt": b"x\n",
"sub/y.txt": b"y\n",
"dir1/keep.txt": b"dir content\n",
}
for rel, content in entries.items():
full = os.path.join(source, rel)
os.makedirs(os.path.dirname(full), exist_ok=True)
with open(full, "wb") as fh:
fh.write(content)
return source
def _write_rel_list(rel_text):
path = os.path.join(TEST_DATA_DIR, "rel_list.txt")
with open(path, "wb") as fh:
fh.write(rel_text)
return path
class TestRelativeFilesFrom:
"""-R/--relative with --files-from keeps each listed entry's bare relative
destination path below the destination root instead of mirroring the full
source path. Without -R the layout is unchanged (full source mirror)."""
@pytest.mark.parametrize("mt", [False, True])
def test_relative_files_from_keeps_relative_layout(self, shared_server, mt):
source = _make_relative_source("rel_src")
dest = os.path.join(TEST_DATA_DIR, "rel_dst")
clean_dir(dest)
lst = _write_rel_list(b"top.txt\nsub/x.txt\n")
flags = ["--files-from", lst, "-R"] + (["-m"] if mt else [])
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
assert result.returncode == 0, f"-R files-from sync failed: {result.stderr[:200]}"
assert _read_file(os.path.join(dest, "sub", "x.txt")) == b"x\n", \
"listed file must land at <dest>/sub/x.txt"
assert _read_file(os.path.join(dest, "top.txt")) == b"top\n", \
"top-level listed file must land at <dest>/top.txt"
assert not os.path.exists(os.path.join(dest, "sub", "y.txt"))
# The source-root mirror must not be reproduced under -R.
assert not os.path.exists(get_dest_received_dir(dest, source)), \
"-R must not mirror the full source path"
@pytest.mark.parametrize("mt", [False, True])
def test_relative_without_files_from_has_no_effect(self, shared_server, mt):
"""-R alone (no --files-from) must leave the normal full-source mirror
layout untouched."""
source = _make_relative_source("rel_only_src")
dest = os.path.join(TEST_DATA_DIR, "rel_only_dst")
clean_dir(dest)
flags = ["-R"] + (["-m"] if mt else [])
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
assert result.returncode == 0, f"-R alone sync failed: {result.stderr[:200]}"
received = get_dest_received_dir(dest, source)
mismatches, missing = verify_transfer(source, received)
assert not missing and not mismatches
@pytest.mark.parametrize("mt", [False, True])
def test_without_relative_layout_unchanged(self, shared_server, mt):
source = _make_relative_source("rel_noR_src")
dest = os.path.join(TEST_DATA_DIR, "rel_noR_dst")
clean_dir(dest)
lst = _write_rel_list(b"sub/x.txt\n")
flags = ["--files-from", lst] + (["-m"] if mt else [])
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
assert result.returncode == 0, f"files-from sync failed: {result.stderr[:200]}"
received = get_dest_received_dir(dest, source)
assert _read_file(os.path.join(received, "sub", "x.txt")) == b"x\n", \
"without -R the full source mirror layout is preserved"
assert not os.path.exists(os.path.join(dest, "sub")), \
"bare relative layout must not appear without -R"
def test_relative_delete_manifest_stays_consistent(self):
"""--delete derives from the sent (-R) relative paths, so a later
subset run removes unlisted relative entries but keeps listed ones."""
source = _make_relative_source("rel_del_src")
dest = os.path.join(TEST_DATA_DIR, "rel_del_dst")
clean_dir(dest)
with ServerManager() as server:
server.start(extra_args=["--allow-delete"])
lst = _write_rel_list(b"sub/x.txt\nsub/y.txt\n")
result, _ = run_client(source, dest, flags=["--files-from", lst, "-R"],
port=server.port)
assert result.returncode == 0, f"seed -R sync failed: {result.stderr[:200]}"
assert os.path.isfile(os.path.join(dest, "sub", "y.txt"))
subset = _write_rel_list(b"sub/x.txt\n")
result, _ = run_client(source, dest,
flags=["--files-from", subset, "-R", "--delete"],
port=server.port)
assert result.returncode == 0, f"-R delete sync failed: {result.stderr[:200]}"
assert os.path.isfile(os.path.join(dest, "sub", "x.txt")), "listed file was deleted"
assert not os.path.exists(os.path.join(dest, "sub", "y.txt")), \
"unlisted relative file was not deleted"
class TestNoImpliedDirs:
"""--no-implied-dirs (only meaningful with -R + --files-from) refuses to
place a listed file whose parent directory is not itself listed."""
def _make(self):
return _make_relative_source("noimplied_src")
@pytest.mark.parametrize("mt", [False, True])
def test_implied_dir_only_fails_entry(self, shared_server, mt):
source = self._make()
dest = os.path.join(TEST_DATA_DIR, "noimplied_dst")
clean_dir(dest)
lst = _write_rel_list(b"a/b.txt\n") # "a" itself is not listed
flags = ["--files-from", lst, "-R", "--no-implied-dirs"] + (["-m"] if mt else [])
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
assert result.returncode != 0, "implied parent directory was not rejected"
assert "--no-implied-dirs" in (result.stderr or result.stdout)
assert not os.path.exists(os.path.join(dest, "a", "b.txt"))
@pytest.mark.parametrize("mt", [False, True])
def test_listed_dir_allows_file(self, shared_server, mt):
source = self._make()
dest = os.path.join(TEST_DATA_DIR, "noimplied_ok_dst")
clean_dir(dest)
lst = _write_rel_list(b"a\na/b.txt\n")
flags = ["--files-from", lst, "-R", "--no-implied-dirs"] + (["-m"] if mt else [])
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
assert result.returncode == 0, f"listed dir + file sync failed: {result.stderr[:200]}"
assert _read_file(os.path.join(dest, "a", "b.txt")) == b"nested\n"
@pytest.mark.parametrize("mt", [False, True])
def test_no_implied_dirs_without_relative_changes_nothing(self, shared_server, mt):
source = self._make()
dest = os.path.join(TEST_DATA_DIR, "noimplied_noR_dst")
clean_dir(dest)
lst = _write_rel_list(b"a/b.txt\n")
flags = ["--files-from", lst, "--no-implied-dirs"] + (["-m"] if mt else [])
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
assert result.returncode == 0, "--no-implied-dirs without -R changed behavior"
received = get_dest_received_dir(dest, source)
assert _read_file(os.path.join(received, "a", "b.txt")) == b"nested\n"
class TestDirs:
"""-d/--dirs (and the --old-dirs/--old-d aliases) transfer directory entries
without recursing into their contents."""
def _make(self):
return _make_relative_source("dirs_src")
def _assert_only_empty_mirror(self, dest, source):
mirror = get_dest_received_dir(dest, source)
assert os.path.isdir(mirror), "source-root mirror directory was not created"
files = []
for root, _dirs, names in os.walk(mirror):
files.extend(os.path.relpath(os.path.join(root, n), mirror) for n in names)
assert files == [], f"--dirs descended into contents: {files}"
@pytest.mark.parametrize("flag", ["--dirs", "-d", "--old-dirs", "--old-d"])
@pytest.mark.parametrize("mt", [False, True])
def test_dirs_transfers_empty_dir_only(self, shared_server, flag, mt):
source = self._make()
dest = os.path.join(TEST_DATA_DIR, "dirs_dst")
clean_dir(dest)
flags = [flag] + (["-m"] if mt else [])
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
assert result.returncode == 0, f"{flag} sync failed: {result.stderr[:200]}"
self._assert_only_empty_mirror(dest, source)
@pytest.mark.parametrize("mt", [False, True])
def test_dirs_with_files_from(self, shared_server, mt):
source = self._make()
dest = os.path.join(TEST_DATA_DIR, "dirs_ff_dst")
clean_dir(dest)
# A listed directory is created empty; a listed file is transferred.
lst = _write_rel_list(b"dir1\nsub/x.txt\n")
flags = ["--files-from", lst, "--dirs", "-R"] + (["-m"] if mt else [])
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
assert result.returncode == 0, f"dirs files-from sync failed: {result.stderr[:200]}"
assert os.path.isdir(os.path.join(dest, "dir1")), "listed dir was not created"
assert not os.path.exists(os.path.join(dest, "dir1", "keep.txt")), \
"--dirs must not descend into a listed directory"
assert _read_file(os.path.join(dest, "sub", "x.txt")) == b"x\n", \
"listed file content was not transferred"
assert not os.path.exists(os.path.join(dest, "sub", "y.txt")), \
"unlisted file appeared"
@pytest.mark.parametrize("mt", [False, True])
def test_dirs_with_files_from_mirror_layout(self, shared_server, mt):
"""Without -R the dirs+files-from entries still mirror the source path."""
source = self._make()
dest = os.path.join(TEST_DATA_DIR, "dirs_ff_noR_dst")
clean_dir(dest)
lst = _write_rel_list(b"dir1\n")
flags = ["--files-from", lst, "--dirs"] + (["-m"] if mt else [])
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
assert result.returncode == 0, f"dirs files-from no-R sync failed: {result.stderr[:200]}"
received = get_dest_received_dir(dest, source)
assert os.path.isdir(os.path.join(received, "dir1")), "mirrored dir entry not created"
assert not os.path.exists(os.path.join(received, "dir1", "keep.txt")), \
"--dirs must not descend into a listed directory"
assert not os.path.exists(os.path.join(received, "sub")), \
"unlisted subtree appeared"
@pytest.mark.parametrize("mt", [False, True])
def test_dirs_chunk_serialization(self, shared_server, mt):
"""--dirs entries survive the chunk-serialization wire path (type
marker round-trips); a listed dir lands empty and a listed file lands
with content, with no protocol desync under -s -m."""
source = self._make()
dest = os.path.join(TEST_DATA_DIR, "dirs_s_dst")
clean_dir(dest)
lst = _write_rel_list(b"dir1\nsub/x.txt\n")
flags = ["--files-from", lst, "--dirs", "-R", "-s"] + (["-m"] if mt else [])
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
assert result.returncode == 0, f"dirs -s sync failed: {result.stderr[:200]}"
assert os.path.isdir(os.path.join(dest, "dir1")), "listed dir was not created"
assert not os.path.exists(os.path.join(dest, "dir1", "keep.txt")), \
"--dirs must not descend into a listed directory"
assert _read_file(os.path.join(dest, "sub", "x.txt")) == b"x\n", \
"listed file content was not transferred"
def test_dirs_delete_keeps_transferred_empty_dir(self):
"""Directory entries appear in the delete manifest, so the empty dir a
--dirs run just created is not pruned as an extra by --delete."""
source = self._make()
dest = os.path.join(TEST_DATA_DIR, "dirs_del_dst")
clean_dir(dest)
extra = os.path.join(dest, "extra.txt")
with open(extra, "wb") as fh:
fh.write(b"delete me")
with ServerManager() as server:
server.start(extra_args=["--allow-delete"])
result, _ = run_client(source, dest, flags=["--dirs", "--delete"],
port=server.port)
assert result.returncode == 0, f"--dirs --delete sync failed: {result.stderr[:200]}"
assert not os.path.exists(extra), "--delete did not remove the extra file"
mirror = get_dest_received_dir(dest, source)
assert os.path.isdir(mirror), "transferred empty dir was pruned as an extra"
files = []
for root, _dirs, names in os.walk(mirror):
files.extend(os.path.relpath(os.path.join(root, n), mirror) for n in names)
assert files == [], f"--dirs descended into contents: {files}"
def test_dirs_listed_dir_colliding_with_file_fails(self, shared_server):
"""A listed directory that already exists as a regular file at the
destination fails the transfer cleanly instead of clobbering the file."""
source = self._make()
dest = os.path.join(TEST_DATA_DIR, "dirs_coll_dst")
clean_dir(dest)
blocker = os.path.join(dest, "dir1")
with open(blocker, "wb") as fh:
fh.write(b"blocking file")
lst = _write_rel_list(b"dir1\n")
result, _ = run_client(source, dest, flags=["--files-from", lst, "--dirs", "-R"],
port=shared_server.port)
assert result.returncode != 0, "dir entry over an existing file did not fail"
assert os.path.isfile(blocker), "blocking regular file was clobbered"
class TestMkpath:
"""--mkpath tells the server to create the destination root directory (and
missing leading components) when it does not exist yet; without it a missing
destination root fails the transfer."""
@pytest.mark.parametrize("mt", [False, True])
def test_missing_root_fails_without_mkpath(self, mt):
source = _make_relative_source("mkpath_fail_src")
dest = os.path.join(TEST_DATA_DIR, "mkpath_missing_dst")
shutil.rmtree(dest, ignore_errors=True)
with ServerManager() as server:
server.start()
flags = ["-m"] if mt else []
result, _ = run_client(source, dest, flags=flags, port=server.port)
assert result.returncode != 0, "missing destination root did not fail without --mkpath"
assert not os.path.exists(dest), "missing root was created without --mkpath"
@pytest.mark.parametrize("mt", [False, True])
def test_mkpath_creates_missing_root(self, mt):
source = _make_relative_source("mkpath_ok_src")
dest = os.path.join(TEST_DATA_DIR, "deep", "mkpath_dst")
shutil.rmtree(os.path.join(TEST_DATA_DIR, "deep"), ignore_errors=True)
with ServerManager() as server:
server.start()
flags = ["--mkpath"] + (["-m"] if mt else [])
result, _ = run_client(source, dest, flags=flags, port=server.port)
assert result.returncode == 0, f"--mkpath sync failed: {result.stderr[:200]}"
received = get_dest_received_dir(dest, source)
assert _read_file(os.path.join(received, "sub", "x.txt")) == b"x\n", \
"file not transferred into the --mkpath-created root"
@pytest.mark.parametrize("mkpath", [False, True])
def test_existing_dest_with_trailing_slash(self, shared_server, mkpath):
"""A destination root written with a trailing slash must keep working:
an existing root is accepted both with and without --mkpath."""
source = _make_relative_source("mkpath_trail_src")
dest = os.path.join(TEST_DATA_DIR, "mkpath_trail_dst")
clean_dir(dest)
dest_slash = dest + "/"
flags = ["--mkpath"] if mkpath else []
result, _ = run_client(source, dest_slash, flags=flags, port=shared_server.port)
assert result.returncode == 0, \
f"trailing-slash dest sync (mkpath={mkpath}) failed: {result.stderr[:200]}"
received = get_dest_received_dir(dest, source)
assert _read_file(os.path.join(received, "sub", "x.txt")) == b"x\n", \
"file not transferred into the trailing-slash destination root"
@pytest.mark.parametrize("mkpath", [False, True])
def test_dest_equal_authorized_root(self, mkpath):
"""A destination that is exactly the server's authorized root works
without --mkpath, and with --mkpath creates no stray <root>/<basename>
nested directory."""
root = os.path.join(TEST_DATA_DIR, "mkpath_eq_root")
clean_dir(root)
source = _make_relative_source("mkpath_eq_src")
with ServerManager() as server:
server.start(extra_args=["--destination-root", root])
flags = ["--mkpath"] if mkpath else []
result, _ = run_client(source, root, flags=flags, port=server.port)
assert result.returncode == 0, \
f"dest==authorized-root sync (mkpath={mkpath}) failed: {result.stderr[:200]}"
received = get_dest_received_dir(root, source)
assert _read_file(os.path.join(received, "sub", "x.txt")) == b"x\n", \
"file not transferred when the dest equals the authorized root"
basename = os.path.basename(root.rstrip(os.sep))
assert not os.path.exists(os.path.join(root, basename)), \
"--mkpath created a spurious nested <root>/<basename> directory"
class TestFilters:
"""--filter/-C/-F rule layer: excludes prune, ordering is first-match-wins,
the default with no matching rule is include, and legacy --exclude remains
an independent layer."""
def _pin_mtime(path, ts):
os.utime(path, (ts, ts))
class TestBasisDestDirs:
"""--compare-dest / --copy-dest / --link-dest alternate basis directories.
FastSync's basis directories are relative to the destination root and are
confined below it. The "unchanged" decision is receiver-side and requires
the per-file --incremental handshake (implied by these flags), so the basis
snapshot must reproduce the exact destination-relative mirror path of the
incoming files.
"""
STAGING = ".fastsync-stage"
TS = 1577836800 # 2020-01-01 00:00:00 UTC, used to pin matching mtimes
# fixture files: source and basis share the mtime pin, so a basis "match"
# is decided purely by content (xxHash). unchanged.txt is byte-identical;
# changed.txt is byte-DIFFERENT but has the SAME SIZE as the source (and
# the same pinned mtime), which is what forces the content-hash gate;
# added.txt does not exist in the basis at all.
UNCHANGED = "unchanged.txt"
CHANGED = "changed.txt"
ADDED = "added.txt"
def _make_source(self, name, source_files):
src = os.path.join(TEST_DATA_DIR, name)
clean_dir(src)
for rel, content in source_files.items():
full = os.path.join(src, rel)
os.makedirs(os.path.dirname(full), exist_ok=True)
with open(full, "wb") as fh:
fh.write(content)
_pin_mtime(full, self.TS)
return src
def _seed_basis_file(self, dest, source, basis_dir, rel, content, ts=None):
base = os.path.join(dest, basis_dir, os.path.relpath(
get_dest_received_dir(dest, source), dest))
full = os.path.join(base, rel)
os.makedirs(os.path.dirname(full), exist_ok=True)
with open(full, "wb") as fh:
fh.write(content)
_pin_mtime(full, self.TS if ts is None else ts)
return full
def _seed_basis(self, dest, source, basis_dir, basis_files):
for rel, content in basis_files.items():
self._seed_basis_file(dest, source, basis_dir, rel, content)
return os.path.join(dest, basis_dir, os.path.relpath(
get_dest_received_dir(dest, source), dest))
def _source_tree(self, prefix):
return {
self.UNCHANGED: b"stable content v1\n",
self.CHANGED: b"changed content now\n",
self.ADDED: b"brand new content\n",
}
def _basis_tree(self, prefix):
# unchanged.txt is identical to the source; changed.txt has the SAME
# byte size and pinned mtime but a different body (equal size forces
# the xxHash gate); added.txt is missing from the basis.
return {
self.UNCHANGED: b"stable content v1\n",
self.CHANGED: b"CHANGED CONTENT NOW\n",
}
def test_same_size_different_content_is_not_a_basis_match(self, shared_server):
# Core safety property: equal size + pinned mtime but different content
# must NEVER be hard-linked or copied from the basis -- the xxHash gate
# rejects it and the sender's data is transferred instead.
for flag, basis_dir in (("--link-dest", "szlb"), ("--copy-dest", "szcp"),
("--compare-dest", "szcmp")):
source = self._make_source("basis_same_size_src",
{self.UNCHANGED: b"same length body\n"})
dest = os.path.join(TEST_DATA_DIR, f"basis_same_size_dst_{basis_dir}")
clean_dir(dest)
basis_file = self._seed_basis_file(dest, source, basis_dir, self.UNCHANGED,
b"SAME LENGTH BODY!")
result, _ = run_client(source, dest, flags=[f"{flag}={basis_dir}"],
port=shared_server.port)
assert result.returncode == 0, \
f"{flag} same-size mismatch failed: {result.stderr[:300]}"
received = get_dest_received_dir(dest, source)
dest_file = os.path.join(received, self.UNCHANGED)
assert _read_file(dest_file) == b"same length body\n", \
f"{flag}: basis content leaked into the destination on a hash mismatch"
if flag != "--compare-dest":
assert os.stat(dest_file).st_ino != os.stat(basis_file).st_ino, \
f"{flag}: linked/copied from a content-mismatched basis file"
def test_compare_dest_skips_matching_and_transfers_missing(self, shared_server):
source = self._make_source("basis_compare_src", self._source_tree("c"))
dest = os.path.join(TEST_DATA_DIR, "basis_compare_dst")
clean_dir(dest)
self._seed_basis(dest, source, "cbasis", self._basis_tree("c"))
result, _ = run_client(source, dest,
flags=["--compare-dest=cbasis"],
port=shared_server.port)
assert result.returncode == 0, f"compare-dest failed: {result.stderr[:300]}"
received = get_dest_received_dir(dest, source)
# compare-dest never copies: an exact basis match is skipped, leaving a
# sparse destination (rsync parity).
assert not os.path.exists(os.path.join(received, self.UNCHANGED)), \
"compare-dest materialized the unchanged file"
# Files the destination lacks AND the basis cannot satisfy are still
# transferred normally.
assert _read_file(os.path.join(received, self.CHANGED)) == \
self._source_tree("c")[self.CHANGED], "changed file not transferred"
assert _read_file(os.path.join(received, self.ADDED)) == \
self._source_tree("c")[self.ADDED], "added file not transferred"
def test_compare_dest_content_mismatch_forces_transfer(self, shared_server):
# The basis holds a file with a DIFFERENT body: even though it shares
# the mtime pin, the xxHash check fails and the data must be sent.
source = self._make_source("basis_compare_mismatch_src", {self.UNCHANGED: b"real data\n"})
dest = os.path.join(TEST_DATA_DIR, "basis_compare_mismatch_dst")
clean_dir(dest)
basis = self._seed_basis(dest, source, "cbasis", {self.UNCHANGED: b"stale data!!\n"})
result, _ = run_client(source, dest, flags=["--compare-dest=cbasis"],
port=shared_server.port)
assert result.returncode == 0, f"compare-dest mismatch failed: {result.stderr[:300]}"
received = get_dest_received_dir(dest, source)
assert _read_file(os.path.join(received, self.UNCHANGED)) == b"real data\n", \
"content mismatch did not fall back to a normal transfer"
assert os.stat(os.path.join(received, self.UNCHANGED)).st_ino != \
os.stat(os.path.join(basis, self.UNCHANGED)).st_ino
def test_copy_dest_copies_unchanged_and_transfers_changed(self, shared_server):
source = self._make_source("basis_copy_src", self._source_tree("cp"))
dest = os.path.join(TEST_DATA_DIR, "basis_copy_dst")
clean_dir(dest)
basis = self._seed_basis(dest, source, "cpbasis", self._basis_tree("cp"))
result, _ = run_client(source, dest, flags=["--copy-dest=cpbasis"],
port=shared_server.port)
assert result.returncode == 0, f"copy-dest failed: {result.stderr[:300]}"
received = get_dest_received_dir(dest, source)
unchanged = os.path.join(received, self.UNCHANGED)
assert _read_file(unchanged) == b"stable content v1\n", "unchanged file not materialized"
# A real local copy, NOT a hard link to the basis file.
assert os.stat(unchanged).st_ino != os.stat(os.path.join(basis, self.UNCHANGED)).st_ino
# Equal-size/different-content basis file falls back to the sender data.
assert _read_file(os.path.join(received, self.CHANGED)) == \
self._source_tree("cp")[self.CHANGED]
assert _read_file(os.path.join(received, self.ADDED)) == \
self._source_tree("cp")[self.ADDED]
def test_link_dest_hardlinks_and_falls_back(self, shared_server):
source = self._make_source("basis_link_src", self._source_tree("ln"))
dest = os.path.join(TEST_DATA_DIR, "basis_link_dst")
clean_dir(dest)
basis = self._seed_basis(dest, source, "lnbasis", self._basis_tree("ln"))
result, _ = run_client(source, dest, flags=["--link-dest=lnbasis"],
port=shared_server.port)
assert result.returncode == 0, f"link-dest failed: {result.stderr[:300]}"
received = get_dest_received_dir(dest, source)
unchanged = os.path.join(received, self.UNCHANGED)
basis_file = os.path.join(basis, self.UNCHANGED)
# Real hard link: same inode as the DIR file, nlink >= 2, no data copy.
assert os.path.exists(unchanged)
assert os.stat(unchanged).st_ino == os.stat(basis_file).st_ino, \
"link-dest did not produce a hard link"
assert os.stat(unchanged).st_nlink >= 2
# Equal-size/different-content basis file must fall back to a plain
# transfer (not a link).
changed = os.path.join(received, self.CHANGED)
assert _read_file(changed) == self._source_tree("ln")[self.CHANGED]
assert os.stat(changed).st_ino != os.stat(os.path.join(basis, self.CHANGED)).st_ino
@pytest.mark.parametrize("flag", ["--compare-dest", "--copy-dest", "--link-dest"])
def test_basis_dir_missing_is_a_clean_noop(self, shared_server, flag):
# A basis directory that does not exist must simply transfer everything.
source = self._make_source("basis_missing_src", {self.UNCHANGED: b"content\n"})
dest = os.path.join(TEST_DATA_DIR, "basis_missing_dst")
clean_dir(dest)
result, _ = run_client(source, dest, flags=[f"{flag}=nope"],
port=shared_server.port)
assert result.returncode == 0, f"{flag} with missing dir failed: {result.stderr[:300]}"
received = get_dest_received_dir(dest, source)
assert _read_file(os.path.join(received, self.UNCHANGED)) == b"content\n"
def test_link_dest_multithreaded(self, shared_server):
source = self._make_source("basis_link_mt_src", self._source_tree("mt"))
dest = os.path.join(TEST_DATA_DIR, "basis_link_mt_dst")
clean_dir(dest)
basis = self._seed_basis(dest, source, "mtbasis", self._basis_tree("mt"))
result, _ = run_client(source, dest, flags=["--link-dest=mtbasis", "-m"],
port=shared_server.port)
assert result.returncode == 0, f"-m link-dest failed: {result.stderr[:300]}"
received = get_dest_received_dir(dest, source)
assert os.stat(os.path.join(received, self.UNCHANGED)).st_ino == \
os.stat(os.path.join(basis, self.UNCHANGED)).st_ino
assert _read_file(os.path.join(received, self.ADDED)) == \
self._source_tree("mt")[self.ADDED]
def test_link_dest_with_delay_updates_stages_and_publishes_link(self, shared_server):
source = self._make_source("basis_link_delay_src", {self.UNCHANGED: b"v1\n"})
dest = os.path.join(TEST_DATA_DIR, "basis_link_delay_dst")
clean_dir(dest)
basis = self._seed_basis(dest, source, "delaybasis", {self.UNCHANGED: b"v1\n"})
result, _ = run_client(source, dest,
flags=["--link-dest=delaybasis", "--delay-updates"],
port=shared_server.port)
assert result.returncode == 0, f"delay-updates link-dest failed: {result.stderr[:300]}"
received = get_dest_received_dir(dest, source)
unchanged = os.path.join(received, self.UNCHANGED)
assert os.stat(unchanged).st_ino == \
os.stat(os.path.join(basis, self.UNCHANGED)).st_ino
assert not os.path.isdir(os.path.join(dest, self.STAGING)), \
"delay-updates staging tree was not cleaned up"
def test_delete_does_not_touch_basis_dir(self):
"""--delete removes genuine extras but must never treat a basis-dir
snapshot (which a --link-dest run just linked from) as destination
content."""
source = self._make_source("basis_delete_src", {self.UNCHANGED: b"v1\n"})
dest = os.path.join(TEST_DATA_DIR, "basis_delete_dst")
clean_dir(dest)
basis = self._seed_basis(dest, source, "delbasis", {self.UNCHANGED: b"v1\n"})
received = get_dest_received_dir(dest, source)
os.makedirs(received, exist_ok=True)
extra = os.path.join(received, "extra.txt")
with open(extra, "wb") as fh:
fh.write(b"extra")
with ServerManager() as server:
server.start(extra_args=["--allow-delete"])
result, _ = run_client(source, dest, flags=["--link-dest=delbasis", "--delete"],
port=server.port)
assert result.returncode == 0, \
f"delete+link-dest failed: {result.stderr[:300]}"
assert not os.path.exists(extra), "genuine extra file was not deleted"
assert _read_file(os.path.join(received, self.UNCHANGED)) == b"v1\n"
assert os.path.exists(os.path.join(basis, self.UNCHANGED)), \
"basis directory was deleted by --delete"
assert os.stat(os.path.join(received, self.UNCHANGED)).st_ino == \
os.stat(os.path.join(basis, self.UNCHANGED)).st_ino
def test_delay_delete_keeps_nested_staging_named_dir_as_content(self):
# The real --delay-updates staging directory is protected from --delete
# only as a DIRECT child of the receive root. A nested destination
# directory that merely shares the staging name is ordinary content, so
# its extras must still be deleted (regression guard for the walker).
source = self._make_source("basis_nested_stage_src",
{"top.txt": b"top\n", "sub/real.txt": b"real\n"})
dest = os.path.join(TEST_DATA_DIR, "basis_nested_stage_dst")
clean_dir(dest)
self._seed_basis(dest, source, "nstbasis",
{"top.txt": b"top\n", "sub/real.txt": b"real\n"})
received = get_dest_received_dir(dest, source)
nested = os.path.join(received, "sub", self.STAGING)
os.makedirs(nested, exist_ok=True)
extra = os.path.join(nested, "extra.txt")
with open(extra, "wb") as fh:
fh.write(b"nested extra")
with ServerManager() as server:
server.start(extra_args=["--allow-delete"])
result, _ = run_client(source, dest,
flags=["--link-dest=nstbasis", "--delete",
"--delay-updates"],
port=server.port)
assert result.returncode == 0, \
f"delay-delete nested staging failed: {result.stderr[:300]}"
assert not os.path.exists(extra), \
"extra inside a nested .fastsync-stage dir was not deleted"
assert not os.path.isdir(nested), \
"nested .fastsync-stage dir should have been removed after its extra"
assert _read_file(os.path.join(received, "sub", "real.txt")) == b"real\n"
assert not os.path.isdir(os.path.join(dest, self.STAGING)), \
"real delay-updates staging tree was not cleaned up"
def test_basis_priority_first_match_wins(self, shared_server):
# Two link-dest dirs both hold the exact file: the FIRST (command-line
# order) basis directory must win and supply the hard link.
source = self._make_source("basis_prio_src", {"f.txt": b"content\n"})
dest = os.path.join(TEST_DATA_DIR, "basis_prio_dst")
clean_dir(dest)
first = self._seed_basis_file(dest, source, "b1", "f.txt", b"content\n")
self._seed_basis_file(dest, source, "b2", "f.txt", b"content\n")
result, _ = run_client(source, dest, flags=["--link-dest=b1", "--link-dest=b2"],
port=shared_server.port)
assert result.returncode == 0, f"link-dest priority failed: {result.stderr[:300]}"
received = get_dest_received_dir(dest, source)
assert os.stat(os.path.join(received, "f.txt")).st_ino == os.stat(first).st_ino, \
"first basis dir did not win over the second"
def test_basis_priority_across_compare_and_link(self, shared_server):
# A compare-dest entry listed BEFORE a link-dest entry shadows it (the
# exact match is found first and nothing is materialized); reversing the
# order lets the link-dest entry win and materialize a hard link.
source = self._make_source("basis_prio_mixed_src", {"f.txt": b"content\n"})
dest = os.path.join(TEST_DATA_DIR, "basis_prio_mixed_dst")
clean_dir(dest)
self._seed_basis_file(dest, source, "cmpb", "f.txt", b"content\n")
self._seed_basis_file(dest, source, "lnb", "f.txt", b"content\n")
result, _ = run_client(source, dest,
flags=["--compare-dest=cmpb", "--link-dest=lnb"],
port=shared_server.port)
assert result.returncode == 0, \
f"mixed priority (compare first) failed: {result.stderr[:300]}"
received = get_dest_received_dir(dest, source)
assert not os.path.exists(os.path.join(received, "f.txt")), \
"compare-dest matched first, so the file must stay sparse (no link-dest materialize)"
dest = os.path.join(TEST_DATA_DIR, "basis_prio_mixed_dst2")
clean_dir(dest)
self._seed_basis_file(dest, source, "cmpb", "f.txt", b"content\n")
linkb2 = self._seed_basis_file(dest, source, "lnb", "f.txt", b"content\n")
result, _ = run_client(source, dest,
flags=["--link-dest=lnb", "--compare-dest=cmpb"],
port=shared_server.port)
assert result.returncode == 0, \
f"mixed priority (link first) failed: {result.stderr[:300]}"
received = get_dest_received_dir(dest, source)
assert os.stat(os.path.join(received, "f.txt")).st_ino == os.stat(linkb2).st_ino, \
"link-dest did not materialize when listed before compare-dest"
def test_link_dest_size_only_ignores_mtime(self, shared_server):
# --size-only drops the mtime leg of the quick check: a basis file with
# the SAME content but a DIFFERENT mtime is still an exact match.
source = self._make_source("basis_sizeonly_src", {"f.txt": b"content\n"})
dest = os.path.join(TEST_DATA_DIR, "basis_sizeonly_dst")
clean_dir(dest)
basis_file = self._seed_basis_file(dest, source, "sob", "f.txt", b"content\n",
ts=self.TS + 500)
result, _ = run_client(source, dest, flags=["--link-dest=sob", "--size-only"],
port=shared_server.port)
assert result.returncode == 0, f"size-only link-dest failed: {result.stderr[:300]}"
received = get_dest_received_dir(dest, source)
assert os.stat(os.path.join(received, "f.txt")).st_ino == os.stat(basis_file).st_ino, \
"--size-only should link a basis file whose mtime differs"
def test_link_dest_ignore_times_never_links(self, shared_server):
# -I/--ignore-times forces every file to be updated, so a basis dir is
# never used to hard-link (rsync parity). The file is transferred and
# stored as a fresh inode even though it matches the basis exactly.
source = self._make_source("basis_igntimes_src", {"f.txt": b"content\n"})
dest = os.path.join(TEST_DATA_DIR, "basis_igntimes_dst")
clean_dir(dest)
basis_file = self._seed_basis_file(dest, source, "itb", "f.txt", b"content\n")
result, _ = run_client(source, dest, flags=["--link-dest=itb", "--ignore-times"],
port=shared_server.port)
assert result.returncode == 0, f"ignore-times link-dest failed: {result.stderr[:300]}"
received = get_dest_received_dir(dest, source)
dest_file = os.path.join(received, "f.txt")
assert _read_file(dest_file) == b"content\n"
assert os.stat(dest_file).st_ino != os.stat(basis_file).st_ino, \
"--ignore-times must not hard-link to a basis file"
def test_basis_refuses_file_above_whole_file_limit(self, shared_server):
# Every whole-file payload path in FastSync (basis dirs included) is
# bounded by MAX_RECEIVE_WHOLE_FILE_SIZE. rsync supports basis dirs for
# arbitrary sizes; FastSync refuses such a run up front with a clear
# diagnostic instead of letting the receiver abort the whole transfer
# mid-stream with no client-side explanation.
source = self._make_source("basis_oversize_src", {"small.txt": b"ok\n"})
big = os.path.join(source, "huge.bin")
with open(big, "wb") as fh:
os.ftruncate(fh.fileno(), 256 * 1024 * 1024 + 4096)
dest = os.path.join(TEST_DATA_DIR, "basis_oversize_dst")
clean_dir(dest)
result, _ = run_client(source, dest, flags=["--link-dest=nope"],
port=shared_server.port)
assert result.returncode != 0, \
"basis run with an over-limit file unexpectedly succeeded"
assert "larger than" in result.stderr, \
f"no clear over-limit diagnostic: {result.stderr[:300]}"
received = get_dest_received_dir(dest, source)
assert not os.path.exists(received), \
"over-limit basis run transferred files before failing"
Reference in New Issue View Git Blame Copy Permalink