3784 lines
195 KiB
Python
3784 lines
195 KiB
Python
"""Feature tests: incremental sync, bandwidth limiting, dry run, metadata, filters."""
|
|
import filecmp
|
|
import os
|
|
import random
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import pytest
|
|
|
|
sys.path.insert(0, os.path.dirname(__file__))
|
|
from common import (
|
|
PROJECT_ROOT, BUILD_DIR, TEST_DATA_DIR,
|
|
run_client, CountingProxy,
|
|
generate_test_files, verify_transfer, clean_dir, make_result,
|
|
get_dest_received_dir, CLIENT_CMD, ServerManager,
|
|
)
|
|
|
|
SOURCE_DIR = os.path.join(TEST_DATA_DIR, "feature_source")
|
|
DEST_DIR = os.path.join(TEST_DATA_DIR, "feature_dest")
|
|
|
|
|
|
@pytest.fixture(scope="module", autouse=True)
|
|
def setup_test_data():
|
|
generate_test_files(SOURCE_DIR, full=False)
|
|
clean_dir(DEST_DIR)
|
|
yield
|
|
# Remove only this module's own dirs. Under pytest-xdist the whole
|
|
# (worker-keyed) TEST_DATA_DIR is shared with concurrently-interleaved
|
|
# modules, so never rmtree it here.
|
|
shutil.rmtree(SOURCE_DIR, ignore_errors=True)
|
|
shutil.rmtree(DEST_DIR, ignore_errors=True)
|
|
|
|
|
|
class TestDryRun:
|
|
def test_human_readable_dry_run(self):
|
|
result, dur = run_client(SOURCE_DIR, DEST_DIR, flags=["-h", "--dry-run"])
|
|
assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:100]}"
|
|
assert "Total:" in result.stdout
|
|
assert "KB" in result.stdout
|
|
|
|
def test_dry_run(self):
|
|
clean_dir(DEST_DIR)
|
|
result, dur = run_client(
|
|
SOURCE_DIR, DEST_DIR,
|
|
flags=["-n"],
|
|
)
|
|
assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:100]}"
|
|
assert "Dry run:" in result.stdout, f"No dry run output: {result.stdout[:200]}"
|
|
|
|
def test_quiet_suppresses_dry_run_output(self):
|
|
result, dur = run_client(
|
|
SOURCE_DIR, DEST_DIR,
|
|
flags=["-q", "-n", "--progress", "--stats"],
|
|
)
|
|
assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:100]}"
|
|
assert result.stdout == ""
|
|
assert result.stderr == ""
|
|
|
|
def test_quiet_preserves_errors(self):
|
|
result, dur = run_client(
|
|
SOURCE_DIR, DEST_DIR,
|
|
flags=["--quiet", "--server-port", "1"],
|
|
)
|
|
assert result.returncode != 0
|
|
assert result.stderr != ""
|
|
|
|
@pytest.mark.parametrize("flags", [["-q", "-v"], ["-v", "-q"]])
|
|
def test_quiet_successful_transfer_and_verbose_order(self, shared_server, flags):
|
|
clean_dir(DEST_DIR)
|
|
result, dur = run_client(
|
|
SOURCE_DIR, DEST_DIR,
|
|
flags=flags,
|
|
port=shared_server.port,
|
|
)
|
|
assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:100]}"
|
|
assert result.stdout == ""
|
|
assert result.stderr == ""
|
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
|
mismatches, missing = verify_transfer(SOURCE_DIR, received)
|
|
assert not missing, f"Missing: {missing}"
|
|
assert not mismatches, f"Mismatch: {mismatches}"
|
|
|
|
|
|
class TestRemoveSourceFiles:
|
|
def test_removes_only_transferred_regular_files(self, shared_server):
|
|
source = os.path.join(TEST_DATA_DIR, "remove_source")
|
|
dest = os.path.join(TEST_DATA_DIR, "remove_dest")
|
|
clean_dir(source)
|
|
clean_dir(dest)
|
|
with open(os.path.join(source, "one.txt"), "wb") as f:
|
|
f.write(b"one")
|
|
with open(os.path.join(source, "two.txt"), "wb") as f:
|
|
f.write(b"two")
|
|
os.makedirs(os.path.join(source, "directory"))
|
|
os.symlink("one.txt", os.path.join(source, "link.txt"))
|
|
|
|
result, _ = run_client(source, dest, flags=["--remove-source-files", "-m"],
|
|
port=shared_server.port)
|
|
assert result.returncode == 0, f"Remove-source sync failed: {result.stderr[:200]}"
|
|
assert not os.path.exists(os.path.join(source, "one.txt"))
|
|
assert not os.path.exists(os.path.join(source, "two.txt"))
|
|
assert os.path.isdir(os.path.join(source, "directory"))
|
|
assert os.path.islink(os.path.join(source, "link.txt"))
|
|
|
|
def test_single_threaded_removes_transferred_file(self, shared_server):
|
|
source = os.path.join(TEST_DATA_DIR, "remove_single_source")
|
|
dest = os.path.join(TEST_DATA_DIR, "remove_single_dest")
|
|
clean_dir(source)
|
|
clean_dir(dest)
|
|
source_file = os.path.join(source, "file.txt")
|
|
with open(source_file, "wb") as f:
|
|
f.write(b"single threaded")
|
|
|
|
result, _ = run_client(source, dest, flags=["--remove-source-files"],
|
|
port=shared_server.port)
|
|
assert result.returncode == 0
|
|
assert not os.path.exists(source_file)
|
|
|
|
def test_dry_run_preserves_source_files(self):
|
|
source = os.path.join(TEST_DATA_DIR, "remove_dry_source")
|
|
dest = os.path.join(TEST_DATA_DIR, "remove_dry_dest")
|
|
clean_dir(source)
|
|
clean_dir(dest)
|
|
source_file = os.path.join(source, "file.txt")
|
|
with open(source_file, "wb") as f:
|
|
f.write(b"keep")
|
|
|
|
result, _ = run_client(source, dest, flags=["--remove-source-files", "--dry-run"])
|
|
assert result.returncode == 0
|
|
assert os.path.isfile(source_file)
|
|
|
|
def test_failed_connection_preserves_source_files(self):
|
|
source = os.path.join(TEST_DATA_DIR, "remove_failed_source")
|
|
dest = os.path.join(TEST_DATA_DIR, "remove_failed_dest")
|
|
clean_dir(source)
|
|
clean_dir(dest)
|
|
source_file = os.path.join(source, "file.txt")
|
|
with open(source_file, "wb") as f:
|
|
f.write(b"keep after failure")
|
|
|
|
result, _ = run_client(source, dest, flags=["--remove-source-files"], port=1)
|
|
assert result.returncode != 0
|
|
assert os.path.isfile(source_file)
|
|
|
|
def test_incremental_skip_preserves_source_file(self, shared_server):
|
|
source = os.path.join(TEST_DATA_DIR, "remove_skipped_source")
|
|
dest = os.path.join(TEST_DATA_DIR, "remove_skipped_dest")
|
|
clean_dir(source)
|
|
clean_dir(dest)
|
|
source_file = os.path.join(source, "file.txt")
|
|
with open(source_file, "wb") as f:
|
|
f.write(b"keep after skip")
|
|
|
|
# The seed run preserves timestamps (-M) so the destination copy has the
|
|
# source's exact mtime; otherwise the incremental skip would depend on
|
|
# both writes landing in the same whole second (a race).
|
|
result, _ = run_client(source, dest, flags=["-M"], port=shared_server.port)
|
|
assert result.returncode == 0
|
|
result, _ = run_client(source, dest,
|
|
flags=["--remove-source-files", "--incremental"],
|
|
port=shared_server.port)
|
|
assert result.returncode == 0
|
|
assert os.path.isfile(source_file)
|
|
|
|
|
|
class TestArchiveMode:
|
|
@pytest.mark.ci
|
|
def test_archive_mode(self, shared_server):
|
|
clean_dir(DEST_DIR)
|
|
result, dur = run_client(
|
|
SOURCE_DIR, DEST_DIR,
|
|
flags=["-a"],
|
|
port=shared_server.port,
|
|
)
|
|
if result.returncode != 0:
|
|
pytest.fail(f"Exit {result.returncode}: {(result.stderr or result.stdout)[:200]}")
|
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
|
mismatches, missing = verify_transfer(SOURCE_DIR, received)
|
|
assert not missing, f"Missing: {missing}"
|
|
assert not mismatches, f"Mismatch: {mismatches}"
|
|
|
|
def test_archive_implied_options_can_be_negated(self, shared_server):
|
|
clean_dir(DEST_DIR)
|
|
result, dur = run_client(
|
|
SOURCE_DIR, DEST_DIR,
|
|
flags=["--archive", "--no-compress", "--no-m", "--no-preserve"],
|
|
port=shared_server.port,
|
|
)
|
|
if result.returncode != 0:
|
|
pytest.fail(f"Exit {result.returncode}: {(result.stderr or result.stdout)[:200]}")
|
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
|
mismatches, missing = verify_transfer(SOURCE_DIR, received)
|
|
assert not missing, f"Missing: {missing}"
|
|
assert not mismatches, f"Mismatch: {mismatches}"
|
|
|
|
|
|
class TestExecutability:
|
|
@pytest.mark.ci
|
|
def test_preserves_only_executable_bits(self, shared_server):
|
|
source = os.path.join(TEST_DATA_DIR, "executability_source")
|
|
dest = os.path.join(TEST_DATA_DIR, "executability_dest")
|
|
clean_dir(source)
|
|
clean_dir(dest)
|
|
source_file = os.path.join(source, "tool.sh")
|
|
with open(source_file, "w") as f:
|
|
f.write("#!/bin/sh\necho test\n")
|
|
os.chmod(source_file, 0o751)
|
|
|
|
result, _ = run_client(source, dest, flags=["-E"], port=shared_server.port)
|
|
assert result.returncode == 0, f"Executability sync failed: {result.stderr[:200]}"
|
|
received_file = os.path.join(get_dest_received_dir(dest, source), "tool.sh")
|
|
received_mode = os.stat(received_file).st_mode
|
|
assert received_mode & 0o111 == 0o111
|
|
assert received_mode & 0o600 == 0o600
|
|
assert received_mode & 0o077 == 0o011
|
|
|
|
|
|
class TestChmod:
|
|
@pytest.mark.ci
|
|
def test_chmod_applies_to_transferred_files(self, shared_server):
|
|
clean_dir(DEST_DIR)
|
|
source_file = os.path.join(SOURCE_DIR, "small.txt")
|
|
os.chmod(source_file, 0o777)
|
|
result, dur = run_client(
|
|
SOURCE_DIR, DEST_DIR,
|
|
flags=["--chmod=u=rw,go=r"],
|
|
port=shared_server.port,
|
|
)
|
|
if result.returncode != 0:
|
|
pytest.fail(f"Exit {result.returncode}: {(result.stderr or result.stdout)[:200]}")
|
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
|
assert (os.stat(os.path.join(received, "small.txt")).st_mode & 0o777) == 0o644
|
|
|
|
|
|
class TestCompressionChoice:
|
|
@pytest.mark.ci
|
|
def test_zstd_choice_compresses(self, shared_server):
|
|
clean_dir(DEST_DIR)
|
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["--zc", "zstd"],
|
|
port=shared_server.port)
|
|
assert result.returncode == 0, f"zstd sync failed: {(result.stderr or result.stdout)[:200]}"
|
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
|
mismatches, missing = verify_transfer(SOURCE_DIR, received)
|
|
assert not missing, f"Missing: {missing}"
|
|
assert not mismatches, f"Mismatch: {mismatches}"
|
|
|
|
def test_none_choice_disables_compression(self, shared_server):
|
|
clean_dir(DEST_DIR)
|
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-z", "--compress-choice", "none"],
|
|
port=shared_server.port)
|
|
assert result.returncode == 0, f"none sync failed: {(result.stderr or result.stdout)[:200]}"
|
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
|
mismatches, missing = verify_transfer(SOURCE_DIR, received)
|
|
assert not missing, f"Missing: {missing}"
|
|
assert not mismatches, f"Mismatch: {mismatches}"
|
|
|
|
|
|
class TestSkipCompress:
|
|
@pytest.mark.ci
|
|
def test_skip_compress_case_insensitive(self, shared_server):
|
|
clean_dir(DEST_DIR)
|
|
with open(os.path.join(SOURCE_DIR, "skip-case.TXT"), "wb") as f:
|
|
f.write((b"skip compression case test\n" * 100))
|
|
result, _ = run_client(
|
|
SOURCE_DIR, DEST_DIR,
|
|
flags=["-c", "--skip-compress=.txt"],
|
|
port=shared_server.port,
|
|
)
|
|
assert result.returncode == 0, f"Skip-compress sync failed: {(result.stderr or result.stdout)[:200]}"
|
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
|
with open(os.path.join(received, "skip-case.TXT"), "rb") as f:
|
|
assert f.read() == b"skip compression case test\n" * 100
|
|
|
|
def test_skip_compress_empty_list(self, shared_server):
|
|
clean_dir(DEST_DIR)
|
|
result, _ = run_client(
|
|
SOURCE_DIR, DEST_DIR,
|
|
flags=["-c", "--skip-compress="],
|
|
port=shared_server.port,
|
|
)
|
|
assert result.returncode == 0, f"Empty skip-compress sync failed: {(result.stderr or result.stdout)[:200]}"
|
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
|
mismatches, missing = verify_transfer(SOURCE_DIR, received)
|
|
assert not missing, f"Missing: {missing}"
|
|
assert not mismatches, f"Mismatch: {mismatches}"
|
|
|
|
def test_skip_compress_incremental_full_fallback(self, shared_server):
|
|
clean_dir(DEST_DIR)
|
|
path = os.path.join(SOURCE_DIR, "incremental-skip.TXT")
|
|
with open(path, "wb") as f:
|
|
f.write(b"original skipped content\n")
|
|
flags = ["-c", "-M", "--skip-compress=.txt"]
|
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=flags, port=shared_server.port)
|
|
assert result.returncode == 0, f"Initial sync failed: {(result.stderr or result.stdout)[:200]}"
|
|
with open(path, "wb") as f:
|
|
f.write(b"updated skipped content\n")
|
|
result, _ = run_client(
|
|
SOURCE_DIR, DEST_DIR,
|
|
flags=flags + ["--incremental"],
|
|
port=shared_server.port,
|
|
)
|
|
assert result.returncode == 0, f"Incremental sync failed: {(result.stderr or result.stdout)[:200]}"
|
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
|
with open(os.path.join(received, "incremental-skip.TXT"), "rb") as f:
|
|
assert f.read() == b"updated skipped content\n"
|
|
|
|
def test_skip_compress_rejects_chunk_serialization(self, shared_server):
|
|
result, _ = run_client(
|
|
SOURCE_DIR, DEST_DIR,
|
|
flags=["-c", "-s", "--skip-compress=.txt"],
|
|
port=shared_server.port,
|
|
)
|
|
assert result.returncode != 0
|
|
assert "cannot be combined" in (result.stderr or result.stdout)
|
|
|
|
|
|
class TestExclude:
|
|
def test_exclude_single(self, shared_server):
|
|
clean_dir(DEST_DIR)
|
|
result, dur = run_client(
|
|
SOURCE_DIR, DEST_DIR,
|
|
flags=["--exclude", "small.txt"],
|
|
port=shared_server.port,
|
|
)
|
|
if result.returncode != 0:
|
|
pytest.fail(f"Exit {result.returncode}: {(result.stderr or result.stdout)[:200]}")
|
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
|
mismatches, missing = verify_transfer(SOURCE_DIR, received)
|
|
assert "small.txt" in missing, "small.txt should be excluded but was transferred"
|
|
other_missing = [m for m in missing if m != "small.txt"]
|
|
assert not other_missing, f"Other files missing: {other_missing}"
|
|
assert not mismatches, f"Mismatch: {mismatches}"
|
|
|
|
def test_exclude_glob(self, shared_server):
|
|
clean_dir(DEST_DIR)
|
|
result, dur = run_client(
|
|
SOURCE_DIR, DEST_DIR,
|
|
flags=["--exclude", "*.txt"],
|
|
port=shared_server.port,
|
|
)
|
|
if result.returncode != 0:
|
|
pytest.fail(f"Exit {result.returncode}: {(result.stderr or result.stdout)[:200]}")
|
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
|
assert not os.path.exists(os.path.join(received, "small.txt")), "small.txt should be excluded"
|
|
assert os.path.exists(os.path.join(received, "binary.bin")), "binary.bin should be present"
|
|
|
|
|
|
class TestInclude:
|
|
def test_include_single(self, shared_server):
|
|
clean_dir(DEST_DIR)
|
|
result, dur = run_client(
|
|
SOURCE_DIR, DEST_DIR,
|
|
flags=["--include", "binary.bin"],
|
|
port=shared_server.port,
|
|
)
|
|
if result.returncode != 0:
|
|
pytest.fail(f"Exit {result.returncode}: {(result.stderr or result.stdout)[:200]}")
|
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
|
assert os.path.exists(os.path.join(received, "binary.bin")), "binary.bin should be included"
|
|
assert not os.path.exists(os.path.join(received, "small.txt")), "small.txt should not be included"
|
|
|
|
def test_include_glob(self, shared_server):
|
|
clean_dir(DEST_DIR)
|
|
result, dur = run_client(
|
|
SOURCE_DIR, DEST_DIR,
|
|
flags=["--include", "*.bin"],
|
|
port=shared_server.port,
|
|
)
|
|
if result.returncode != 0:
|
|
pytest.fail(f"Exit {result.returncode}: {(result.stderr or result.stdout)[:200]}")
|
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
|
assert os.path.exists(os.path.join(received, "binary.bin")), "binary.bin should be included"
|
|
|
|
|
|
class TestSizeFilters:
|
|
def test_max_size(self, shared_server):
|
|
clean_dir(DEST_DIR)
|
|
result, dur = run_client(
|
|
SOURCE_DIR, DEST_DIR,
|
|
flags=["--max-size", "100"],
|
|
port=shared_server.port,
|
|
)
|
|
if result.returncode != 0:
|
|
pytest.fail(f"Exit {result.returncode}: {(result.stderr or result.stdout)[:200]}")
|
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
|
assert os.path.exists(os.path.join(received, "small.txt")), "small.txt should be present"
|
|
assert not os.path.exists(os.path.join(received, "medium.txt")), "medium.txt should be skipped"
|
|
|
|
def test_min_size(self, shared_server):
|
|
clean_dir(DEST_DIR)
|
|
result, dur = run_client(
|
|
SOURCE_DIR, DEST_DIR,
|
|
flags=["--min-size", "1000"],
|
|
port=shared_server.port,
|
|
)
|
|
if result.returncode != 0:
|
|
pytest.fail(f"Exit {result.returncode}: {(result.stderr or result.stdout)[:200]}")
|
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
|
assert not os.path.exists(os.path.join(received, "small.txt")), "small.txt should be skipped"
|
|
assert os.path.exists(os.path.join(received, "medium.txt")), "medium.txt should be present"
|
|
|
|
|
|
class TestIncremental:
|
|
@pytest.mark.ci
|
|
def test_incremental_skips_unchanged(self, shared_server):
|
|
clean_dir(DEST_DIR)
|
|
result, dur = run_client(
|
|
SOURCE_DIR, DEST_DIR,
|
|
flags=["-M"],
|
|
port=shared_server.port,
|
|
)
|
|
assert result.returncode == 0, f"First sync failed: {result.stderr[:100]}"
|
|
|
|
start = time.monotonic()
|
|
result, dur = run_client(
|
|
SOURCE_DIR, DEST_DIR,
|
|
flags=["-M", "--incremental"],
|
|
port=shared_server.port,
|
|
)
|
|
incremental_time = time.monotonic() - start
|
|
|
|
assert result.returncode == 0, f"Incremental sync failed: {(result.stderr or result.stdout)[:200]}"
|
|
|
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
|
mismatches, missing = verify_transfer(SOURCE_DIR, received)
|
|
assert not missing, f"Missing: {missing}"
|
|
assert not mismatches, f"Mismatch: {mismatches}"
|
|
|
|
@pytest.mark.ci
|
|
def test_incremental_detects_changes(self, shared_server):
|
|
clean_dir(DEST_DIR)
|
|
result, _ = run_client(
|
|
SOURCE_DIR, DEST_DIR,
|
|
flags=["-M"],
|
|
port=shared_server.port,
|
|
)
|
|
assert result.returncode == 0
|
|
|
|
modified_file = os.path.join(SOURCE_DIR, "small.txt")
|
|
with open(modified_file, "wb") as f:
|
|
f.write(b"modified content for incremental test\n")
|
|
|
|
result, dur = run_client(
|
|
SOURCE_DIR, DEST_DIR,
|
|
flags=["-M", "--incremental"],
|
|
port=shared_server.port,
|
|
)
|
|
assert result.returncode == 0
|
|
|
|
with open(modified_file, "wb") as f:
|
|
f.write(b"hello world\n")
|
|
|
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
|
received_file = os.path.join(received, "small.txt")
|
|
assert os.path.exists(received_file), "Modified file should be present"
|
|
with open(received_file, "rb") as f:
|
|
content = f.read()
|
|
assert b"modified content" in content, f"Modified content not transferred: {content[:50]}"
|
|
|
|
def test_checksum_detects_same_size_and_mtime_change(self, shared_server):
|
|
clean_dir(DEST_DIR)
|
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port)
|
|
assert result.returncode == 0
|
|
|
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
|
source_file = os.path.join(SOURCE_DIR, "small.txt")
|
|
received_file = os.path.join(received, "small.txt")
|
|
source_stat = os.stat(source_file)
|
|
with open(received_file, "wb") as f:
|
|
f.write(b"different!\n")
|
|
os.utime(received_file, (source_stat.st_atime, source_stat.st_mtime))
|
|
|
|
result, _ = run_client(SOURCE_DIR, DEST_DIR,
|
|
flags=["-M", "--incremental", "--checksum"],
|
|
port=shared_server.port)
|
|
assert result.returncode == 0, f"Checksum sync failed: {result.stderr[:200]}"
|
|
with open(received_file, "rb") as f:
|
|
assert f.read() == b"hello world\n"
|
|
|
|
def test_size_only_skips_same_size_with_different_mtime(self, shared_server):
|
|
clean_dir(DEST_DIR)
|
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port)
|
|
assert result.returncode == 0
|
|
|
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
|
received_file = os.path.join(received, "small.txt")
|
|
with open(received_file, "wb") as f:
|
|
f.write(b"different!!\n")
|
|
os.utime(received_file, (time.time() - 3600, time.time() - 3600))
|
|
|
|
result, _ = run_client(
|
|
SOURCE_DIR, DEST_DIR,
|
|
flags=["-M", "--incremental", "--size-only"],
|
|
port=shared_server.port,
|
|
)
|
|
assert result.returncode == 0, f"Size-only sync failed: {result.stderr[:200]}"
|
|
with open(received_file, "rb") as f:
|
|
assert f.read() == b"different!!\n"
|
|
|
|
def test_ignore_times_transfers_same_size_and_mtime(self, shared_server):
|
|
clean_dir(DEST_DIR)
|
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port)
|
|
assert result.returncode == 0
|
|
|
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
|
source_file = os.path.join(SOURCE_DIR, "small.txt")
|
|
received_file = os.path.join(received, "small.txt")
|
|
source_stat = os.stat(source_file)
|
|
with open(received_file, "wb") as f:
|
|
f.write(b"stale data!\n")
|
|
os.utime(received_file, (source_stat.st_atime, source_stat.st_mtime))
|
|
|
|
result, _ = run_client(SOURCE_DIR, DEST_DIR,
|
|
flags=["-M", "--incremental", "--ignore-times"],
|
|
port=shared_server.port)
|
|
assert result.returncode == 0, f"Ignore-times sync failed: {result.stderr[:200]}"
|
|
with open(received_file, "rb") as f:
|
|
assert f.read() == b"hello world\n"
|
|
|
|
def test_modify_window_allows_subsecond_mtime_difference(self, shared_server):
|
|
clean_dir(DEST_DIR)
|
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port)
|
|
assert result.returncode == 0
|
|
|
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
|
source_file = os.path.join(SOURCE_DIR, "small.txt")
|
|
received_file = os.path.join(received, "small.txt")
|
|
source_stat = os.stat(source_file)
|
|
with open(received_file, "wb") as f:
|
|
f.write(b"modified!!!\n")
|
|
os.utime(received_file, ns=(source_stat.st_atime_ns,
|
|
source_stat.st_mtime_ns - 1500000000))
|
|
|
|
result, _ = run_client(SOURCE_DIR, DEST_DIR,
|
|
flags=["-M", "--incremental", "--modify-window=2"],
|
|
port=shared_server.port)
|
|
assert result.returncode == 0, f"Modify-window sync failed: {result.stderr[:200]}"
|
|
with open(received_file, "rb") as f:
|
|
assert f.read() == b"modified!!!\n"
|
|
|
|
def test_whole_file_disables_delta_and_keeps_compression(self, shared_server):
|
|
clean_dir(DEST_DIR)
|
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port)
|
|
assert result.returncode == 0
|
|
|
|
source_file = os.path.join(SOURCE_DIR, "medium.txt")
|
|
with open(source_file, "wb") as f:
|
|
f.write(b"whole-file replacement\n" * 5000)
|
|
|
|
result, _ = run_client(
|
|
SOURCE_DIR,
|
|
DEST_DIR,
|
|
flags=["-M", "--incremental", "--delta", "-W", "-c"],
|
|
port=shared_server.port,
|
|
)
|
|
assert result.returncode == 0, f"Whole-file sync failed: {(result.stderr or result.stdout)[:200]}"
|
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
|
mismatches, missing = verify_transfer(SOURCE_DIR, received)
|
|
assert not missing, f"Missing: {missing}"
|
|
assert not mismatches, f"Mismatch: {mismatches}"
|
|
|
|
|
|
class TestChecksumChoice:
|
|
"""--checksum-choice/--cc and --checksum-seed: the whole-file digest used by
|
|
the --incremental/--checksum handshake is selectable and seedable. The
|
|
receiver hashes the on-disk old file with the SAME algorithm+seed, so an
|
|
unchanged file is skipped and a changed file (even with identical size and
|
|
mtime) is transferred -- and the transfer always lands byte-exact.
|
|
FastSync accepts xxh64 (default, seed-aware) and md5; names it does not
|
|
implement are rejected, never silently ignored."""
|
|
|
|
def test_unsupported_algorithm_is_rejected(self, shared_server):
|
|
result, _ = run_client(
|
|
SOURCE_DIR, DEST_DIR,
|
|
flags=["--checksum", "--checksum-choice=sha256"],
|
|
port=shared_server.port,
|
|
)
|
|
assert result.returncode != 0, "sha256 must be rejected, not silently ignored"
|
|
|
|
@pytest.mark.parametrize("algo", ["xxh64", "md5"])
|
|
@pytest.mark.parametrize("mt", [False, True])
|
|
def test_unchanged_skipped_and_bytes_preserved(self, shared_server, algo, mt):
|
|
clean_dir(DEST_DIR)
|
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port)
|
|
assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}"
|
|
|
|
flags = (["-M", "--incremental", "--checksum", f"--checksum-choice={algo}"] +
|
|
(["-m"] if mt else []))
|
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=flags, port=shared_server.port)
|
|
assert result.returncode == 0, f"checksum {algo} run failed: {result.stderr[:200]}"
|
|
|
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
|
mismatches, missing = verify_transfer(SOURCE_DIR, received)
|
|
assert not missing, f"Missing: {missing}"
|
|
assert not mismatches, f"Mismatch: {mismatches}"
|
|
|
|
# A changed source file with the SAME size and mtime must still be
|
|
# detected (and re-transferred byte-exactly) because the whole-file digest
|
|
# differs -- the explicit reason --checksum exists. This exercises the
|
|
# sender/receiver digest agreement for a non-default algorithm.
|
|
@pytest.mark.parametrize("algo", ["xxh64", "md5"])
|
|
@pytest.mark.parametrize("mt", [False, True])
|
|
def test_changed_same_size_mtime_redetected(self, shared_server, algo, mt):
|
|
clean_dir(DEST_DIR)
|
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port)
|
|
assert result.returncode == 0
|
|
|
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
|
source_file = os.path.join(SOURCE_DIR, "small.txt") # "hello world\n" (12 bytes)
|
|
received_file = os.path.join(received, "small.txt")
|
|
source_stat = os.stat(source_file)
|
|
with open(received_file, "wb") as f:
|
|
f.write(b"DDDDDDDDDDDD") # same size, different content
|
|
os.utime(received_file, (source_stat.st_atime, source_stat.st_mtime))
|
|
|
|
flags = (["-M", "--incremental", "--checksum", f"--checksum-choice={algo}"] +
|
|
(["-m"] if mt else []))
|
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=flags, port=shared_server.port)
|
|
assert result.returncode == 0, f"checksum {algo} redetect failed: {result.stderr[:200]}"
|
|
with open(received_file, "rb") as f:
|
|
assert f.read() == b"hello world\n"
|
|
|
|
@pytest.mark.parametrize("algo", ["xxh64", "md5"])
|
|
def test_unchanged_run_transfers_almost_no_data(self, shared_server, algo):
|
|
# A fully-unchanged --checksum run skips every file: only the config + a
|
|
# small handshake travels, not the payloads. Proxy byte counts are not
|
|
# available for -m (multithreaded connections), so single-thread only.
|
|
clean_dir(DEST_DIR)
|
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port)
|
|
assert result.returncode == 0
|
|
|
|
flags = ["-M", "--incremental", "--checksum", f"--checksum-choice={algo}"]
|
|
proxy = CountingProxy(shared_server.port)
|
|
cmd = (CLIENT_CMD + ["--source-dir", SOURCE_DIR, "--dest-dir", DEST_DIR,
|
|
"--save-to-disk", "--server-port", str(proxy.port)] + flags)
|
|
result = proxy.run(cmd)
|
|
assert result.returncode == 0, f"checksum {algo} skip run failed: {result.stderr[:200]}"
|
|
assert proxy.client_to_server < 100000, \
|
|
f"unchanged --checksum run sent {proxy.client_to_server} bytes; expected a skip"
|
|
|
|
@pytest.mark.parametrize("mt", [False, True])
|
|
def test_seed_is_deterministic_and_preserves_content(self, shared_server, mt):
|
|
clean_dir(DEST_DIR)
|
|
flags = ["-M", "--incremental", "--checksum",
|
|
"--checksum-choice=xxh64", "--checksum-seed=987654"] + (["-m"] if mt else [])
|
|
first, _ = run_client(SOURCE_DIR, DEST_DIR, flags=flags, port=shared_server.port)
|
|
assert first.returncode == 0, f"seeded run failed: {first.stderr[:200]}"
|
|
|
|
# A second run with the SAME seed and unchanged content skips everything
|
|
# deterministically (same digests both sides).
|
|
second, _ = run_client(SOURCE_DIR, DEST_DIR, flags=flags, port=shared_server.port)
|
|
assert second.returncode == 0, f"deterministic rerun failed: {second.stderr[:200]}"
|
|
|
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
|
mismatches, missing = verify_transfer(SOURCE_DIR, received)
|
|
assert not missing and not mismatches, f"missing={missing} mismatches={mismatches}"
|
|
|
|
# A changed file with the same size and mtime is still caught and fixed
|
|
# (a non-zero seed does not weaken the comparison).
|
|
source_file = os.path.join(SOURCE_DIR, "medium.txt")
|
|
received_file = os.path.join(received, "medium.txt")
|
|
source_stat = os.stat(source_file)
|
|
with open(received_file, "wb") as f:
|
|
f.write(b"z" * os.path.getsize(source_file))
|
|
os.utime(received_file, (source_stat.st_atime, source_stat.st_mtime))
|
|
third, _ = run_client(SOURCE_DIR, DEST_DIR, flags=flags, port=shared_server.port)
|
|
assert third.returncode == 0, f"seeded redetect failed: {third.stderr[:200]}"
|
|
with open(received_file, "rb") as f:
|
|
assert f.read() == open(source_file, "rb").read()
|
|
|
|
# --checksum-seed also feeds the delta path's per-block strong checksum on
|
|
# both ends (receiver signature and sender window hash use the same seed),
|
|
# so a seeded delta transfer still lands byte-exact.
|
|
@pytest.mark.parametrize("mt", [False, True])
|
|
def test_seed_delta_block_hash_transfers_byte_exact(self, shared_server, mt):
|
|
source = os.path.join(TEST_DATA_DIR, f"ccseed_{'m' if mt else 's'}_src")
|
|
dest = os.path.join(TEST_DATA_DIR, f"ccseed_{'m' if mt else 's'}_dst")
|
|
clean_dir(source)
|
|
clean_dir(dest)
|
|
big = os.path.join(source, "big.bin")
|
|
with open(big, "wb") as f:
|
|
f.write(bytes(range(256)) * 200) # 51200 bytes > delta 16K floor
|
|
result, _ = run_client(source, dest, flags=["-M"], port=shared_server.port)
|
|
assert result.returncode == 0, f"seed delta seed failed: {result.stderr[:200]}"
|
|
|
|
# Edit a region so the receiver must match a changed block with the seed.
|
|
with open(big, "r+b") as f:
|
|
f.seek(1000)
|
|
f.write(b"\x00" * 64)
|
|
# Force an mtime mismatch: the incremental quick-check skips files whose
|
|
# stored mtime second equals the source's, which can collide when the
|
|
# edit and the prior sync share a second. Setting an old dest mtime
|
|
# guarantees the delta path is exercised deterministically.
|
|
os.utime(os.path.join(get_dest_received_dir(dest, source), "big.bin"), (0, 0))
|
|
flags = (["-M", "--incremental", "--delta", "--checksum-seed=314159"] +
|
|
(["-m"] if mt else []))
|
|
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
|
|
assert result.returncode == 0, f"seed delta run failed: {result.stderr[:200]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
assert _read_file(os.path.join(received, "big.bin")) == _read_file(big), \
|
|
"seeded delta transfer is not byte-exact"
|
|
|
|
|
|
class TestUpdate:
|
|
@pytest.mark.ci
|
|
def test_update_skips_older_destination_and_allows_equal_or_newer_source(self, shared_server):
|
|
clean_dir(DEST_DIR)
|
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-u"], port=shared_server.port)
|
|
assert result.returncode == 0
|
|
|
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
|
source_file = os.path.join(SOURCE_DIR, "small.txt")
|
|
received_file = os.path.join(received, "small.txt")
|
|
source_stat = os.stat(source_file)
|
|
|
|
with open(received_file, "wb") as f:
|
|
f.write(b"newer destination\n")
|
|
os.utime(received_file, ns=(source_stat.st_atime_ns, source_stat.st_mtime_ns + 10_000_000_000))
|
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-u"], port=shared_server.port)
|
|
assert result.returncode == 0
|
|
with open(received_file, "rb") as f:
|
|
assert f.read() == b"newer destination\n"
|
|
|
|
os.utime(received_file, ns=(source_stat.st_atime_ns, source_stat.st_mtime_ns))
|
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-u"], port=shared_server.port)
|
|
assert result.returncode == 0
|
|
with open(received_file, "rb") as f:
|
|
assert f.read() == b"hello world\n"
|
|
|
|
def test_update_skips_unreadable_newer_destination(self, shared_server):
|
|
clean_dir(DEST_DIR)
|
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-u"], port=shared_server.port)
|
|
assert result.returncode == 0
|
|
|
|
received_file = os.path.join(get_dest_received_dir(DEST_DIR, SOURCE_DIR), "small.txt")
|
|
source_stat = os.stat(os.path.join(SOURCE_DIR, "small.txt"))
|
|
with open(received_file, "wb") as f:
|
|
f.write(b"protected destination\n")
|
|
os.utime(received_file, ns=(source_stat.st_atime_ns, source_stat.st_mtime_ns + 10_000_000_000))
|
|
original_mode = os.stat(received_file).st_mode
|
|
try:
|
|
os.chmod(received_file, 0)
|
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-u"], port=shared_server.port)
|
|
assert result.returncode == 0
|
|
os.chmod(received_file, original_mode)
|
|
with open(received_file, "rb") as f:
|
|
assert f.read() == b"protected destination\n"
|
|
finally:
|
|
os.chmod(received_file, original_mode)
|
|
|
|
with open(received_file, "wb") as f:
|
|
f.write(b"older destination\n")
|
|
os.utime(received_file, ns=(source_stat.st_atime_ns, source_stat.st_mtime_ns - 10_000_000_000))
|
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-u"], port=shared_server.port)
|
|
assert result.returncode == 0
|
|
with open(received_file, "rb") as f:
|
|
assert f.read() == b"hello world\n"
|
|
|
|
|
|
class TestExisting:
|
|
@pytest.mark.ci
|
|
def test_existing_updates_existing_and_skips_new(self, shared_server):
|
|
clean_dir(DEST_DIR)
|
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port)
|
|
assert result.returncode == 0, f"Initial sync failed: {(result.stderr or result.stdout)[:200]}"
|
|
|
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
|
source_file = os.path.join(SOURCE_DIR, "small.txt")
|
|
new_source_file = os.path.join(SOURCE_DIR, "new-existing-test.txt")
|
|
with open(source_file, "wb") as f:
|
|
f.write(b"updated existing content\n")
|
|
with open(new_source_file, "wb") as f:
|
|
f.write(b"this file must not be created\n")
|
|
|
|
try:
|
|
result, _ = run_client(SOURCE_DIR, DEST_DIR,
|
|
flags=["-M", "--existing"], port=shared_server.port)
|
|
assert result.returncode == 0, f"--existing sync failed: {(result.stderr or result.stdout)[:200]}"
|
|
|
|
with open(os.path.join(received, "small.txt"), "rb") as f:
|
|
assert f.read() == b"updated existing content\n"
|
|
assert not os.path.exists(os.path.join(received, "new-existing-test.txt"))
|
|
finally:
|
|
os.unlink(new_source_file)
|
|
with open(source_file, "wb") as f:
|
|
f.write(b"hello world\n")
|
|
|
|
|
|
class TestIgnoreExisting:
|
|
def test_ignore_existing_preserves_existing_and_transfers_new(self, shared_server):
|
|
clean_dir(DEST_DIR)
|
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, port=shared_server.port)
|
|
assert result.returncode == 0
|
|
|
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
|
existing_file = os.path.join(received, "small.txt")
|
|
with open(existing_file, "wb") as f:
|
|
f.write(b"destination content\n")
|
|
new_source = os.path.join(SOURCE_DIR, "new.txt")
|
|
try:
|
|
with open(new_source, "wb") as f:
|
|
f.write(b"new file\n")
|
|
|
|
result, _ = run_client(SOURCE_DIR, DEST_DIR,
|
|
flags=["--ignore-existing"], port=shared_server.port)
|
|
assert result.returncode == 0, f"Sync failed: {(result.stderr or result.stdout)[:200]}"
|
|
with open(existing_file, "rb") as f:
|
|
assert f.read() == b"destination content\n"
|
|
with open(os.path.join(received, "new.txt"), "rb") as f:
|
|
assert f.read() == b"new file\n"
|
|
finally:
|
|
if os.path.lexists(new_source):
|
|
os.unlink(new_source)
|
|
|
|
|
|
class TestIgnoreExisting:
|
|
def test_ignore_existing_preserves_existing_and_transfers_new(self, shared_server):
|
|
clean_dir(DEST_DIR)
|
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, port=shared_server.port)
|
|
assert result.returncode == 0
|
|
|
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
|
existing_file = os.path.join(received, "small.txt")
|
|
with open(existing_file, "wb") as f:
|
|
f.write(b"destination content\n")
|
|
new_source = os.path.join(SOURCE_DIR, "new.txt")
|
|
try:
|
|
with open(new_source, "wb") as f:
|
|
f.write(b"new file\n")
|
|
|
|
result, _ = run_client(SOURCE_DIR, DEST_DIR,
|
|
flags=["--ignore-existing"], port=shared_server.port)
|
|
assert result.returncode == 0, f"Sync failed: {(result.stderr or result.stdout)[:200]}"
|
|
with open(existing_file, "rb") as f:
|
|
assert f.read() == b"destination content\n"
|
|
with open(os.path.join(received, "new.txt"), "rb") as f:
|
|
assert f.read() == b"new file\n"
|
|
finally:
|
|
if os.path.lexists(new_source):
|
|
os.unlink(new_source)
|
|
|
|
|
|
class TestDelete:
|
|
@pytest.mark.ci
|
|
def test_delete_removes_extra_files(self, shared_server):
|
|
clean_dir(DEST_DIR)
|
|
result, _ = run_client(
|
|
SOURCE_DIR, DEST_DIR,
|
|
flags=["-M"],
|
|
port=shared_server.port,
|
|
)
|
|
assert result.returncode == 0
|
|
|
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
|
extra_file = os.path.join(received, "extra_file.txt")
|
|
extra_dir = os.path.join(received, "extra_dir")
|
|
with open(extra_file, "w") as f:
|
|
f.write("should be deleted")
|
|
os.makedirs(extra_dir, exist_ok=True)
|
|
with open(os.path.join(extra_dir, "nested.txt"), "w") as f:
|
|
f.write("nested extra")
|
|
|
|
result, dur = run_client(
|
|
SOURCE_DIR, DEST_DIR,
|
|
flags=["-M", "--delete"],
|
|
port=shared_server.port,
|
|
)
|
|
|
|
assert result.returncode == 0, f"Delete sync failed: {(result.stderr or result.stdout)[:200]}"
|
|
# The default server policy intentionally refuses client-requested
|
|
# deletion unless it is started with --allow-delete.
|
|
assert os.path.exists(extra_file), "unauthorized delete removed an extra file"
|
|
assert os.path.exists(extra_dir), "unauthorized delete removed an extra directory"
|
|
|
|
mismatches, missing = verify_transfer(SOURCE_DIR, received)
|
|
assert not missing, f"Missing: {missing}"
|
|
assert not mismatches, f"Mismatch: {mismatches}"
|
|
|
|
|
|
class TestProgress:
|
|
def test_progress_output(self, shared_server):
|
|
clean_dir(DEST_DIR)
|
|
result, dur = run_client(
|
|
SOURCE_DIR, DEST_DIR,
|
|
flags=["--progress"],
|
|
port=shared_server.port,
|
|
)
|
|
assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:100]}"
|
|
output = result.stdout + result.stderr
|
|
assert "Sent " in output and "MB" in output, "--progress produced no stable byte marker"
|
|
assert "Done." in output, "--progress did not report completion"
|
|
|
|
def test_human_readable_stats(self, shared_server):
|
|
clean_dir(DEST_DIR)
|
|
result, dur = run_client(
|
|
SOURCE_DIR, DEST_DIR,
|
|
flags=["-h", "--stats"],
|
|
port=shared_server.port,
|
|
)
|
|
assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:100]}"
|
|
assert "Stats:" in result.stderr
|
|
assert "KB" in result.stderr
|
|
|
|
def test_human_readable_progress_multithreaded(self, shared_server):
|
|
clean_dir(DEST_DIR)
|
|
result, dur = run_client(
|
|
SOURCE_DIR, DEST_DIR,
|
|
flags=["-m", "-h", "--progress"],
|
|
port=shared_server.port,
|
|
)
|
|
assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:100]}"
|
|
output = result.stdout + result.stderr
|
|
assert "Sent " in output
|
|
assert "KB" in output
|
|
assert "Done." in output
|
|
|
|
|
|
class TestInfo:
|
|
def test_info_copy_reports_transfers(self, shared_server):
|
|
clean_dir(DEST_DIR)
|
|
result, _ = run_client(
|
|
SOURCE_DIR, DEST_DIR,
|
|
flags=["--info=copy"],
|
|
port=shared_server.port,
|
|
)
|
|
assert result.returncode == 0, f"Info sync failed: {(result.stderr or result.stdout)[:200]}"
|
|
output = result.stdout + result.stderr
|
|
assert "[INFO]" in output and "Transferring" in output
|
|
|
|
def test_info_stats_reports_multithreaded_transfer(self, shared_server):
|
|
clean_dir(DEST_DIR)
|
|
result, _ = run_client(
|
|
SOURCE_DIR, DEST_DIR,
|
|
flags=["-m", "--info=stats"],
|
|
port=shared_server.port,
|
|
)
|
|
assert result.returncode == 0, f"Info stats sync failed: {(result.stderr or result.stdout)[:200]}"
|
|
output = result.stdout + result.stderr
|
|
assert "[INFO]" in output and "Transfer summary:" in output
|
|
|
|
def test_info_rejects_unknown_flag(self):
|
|
result, _ = run_client(
|
|
SOURCE_DIR, DEST_DIR,
|
|
flags=["--info=unknown"],
|
|
)
|
|
assert result.returncode != 0
|
|
assert "unsupported --info flag" in result.stderr
|
|
|
|
@pytest.mark.parametrize("flags", [
|
|
["--info=none", "--verbose"],
|
|
["--verbose", "--info=none"],
|
|
])
|
|
def test_info_none_suppresses_verbose_info(self, shared_server, flags):
|
|
clean_dir(DEST_DIR)
|
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=flags, port=shared_server.port)
|
|
assert result.returncode == 0, f"Info sync failed: {(result.stderr or result.stdout)[:200]}"
|
|
output = result.stdout + result.stderr
|
|
assert "[INFO]" not in output
|
|
assert "Transferring" not in output
|
|
assert "Transfer summary:" not in output
|
|
|
|
|
|
class TestBandwidthLimit:
|
|
def test_bwlimit_runs(self, shared_server):
|
|
clean_dir(DEST_DIR)
|
|
result, dur = run_client(
|
|
SOURCE_DIR, DEST_DIR,
|
|
flags=["--bwlimit", "10240"],
|
|
port=shared_server.port,
|
|
)
|
|
assert result.returncode == 0, f"Exit {result.returncode}: {(result.stderr or result.stdout)[:200]}"
|
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
|
mismatches, missing = verify_transfer(SOURCE_DIR, received)
|
|
assert not missing, f"Missing: {missing}"
|
|
assert not mismatches, f"Mismatch: {mismatches}"
|
|
|
|
|
|
def _read_file(path):
|
|
with open(path, "rb") as fh:
|
|
return fh.read()
|
|
|
|
|
|
class TestRemoveSourceFilesSkips:
|
|
"""--remove-source-files must not delete sources the receiver skipped
|
|
(rsync reference behavior)."""
|
|
|
|
def test_existing_first_sync_keeps_new_source(self, shared_server):
|
|
source = os.path.join(TEST_DATA_DIR, "remove_rsf_existing_src")
|
|
dest = os.path.join(TEST_DATA_DIR, "remove_rsf_existing_dst")
|
|
clean_dir(source)
|
|
clean_dir(dest)
|
|
with open(os.path.join(source, "only.txt"), "wb") as f:
|
|
f.write(b"keep me")
|
|
|
|
result, _ = run_client(source, dest, flags=["--remove-source-files", "--existing"],
|
|
port=shared_server.port)
|
|
assert result.returncode == 0, f"Sync failed: {result.stderr[:200]}"
|
|
# The file exists only on the source side, so --existing makes the
|
|
# receiver skip it; the source must therefore not be removed.
|
|
assert os.path.isfile(os.path.join(source, "only.txt"))
|
|
received = get_dest_received_dir(dest, source)
|
|
assert not os.path.exists(os.path.join(received, "only.txt"))
|
|
|
|
def test_ignore_existing_keeps_skipped_source(self, shared_server):
|
|
source = os.path.join(TEST_DATA_DIR, "remove_rsf_ignore_src")
|
|
dest = os.path.join(TEST_DATA_DIR, "remove_rsf_ignore_dst")
|
|
clean_dir(source)
|
|
clean_dir(dest)
|
|
source_file = os.path.join(source, "file.txt")
|
|
with open(source_file, "wb") as f:
|
|
f.write(b"payload")
|
|
|
|
result, _ = run_client(source, dest, port=shared_server.port)
|
|
assert result.returncode == 0
|
|
|
|
result, _ = run_client(source, dest, flags=["--remove-source-files", "--ignore-existing"],
|
|
port=shared_server.port)
|
|
assert result.returncode == 0, f"Sync failed: {result.stderr[:200]}"
|
|
# Destination already has the file, so the second run is a receiver
|
|
# skip; the source file must survive.
|
|
assert os.path.isfile(source_file)
|
|
received = get_dest_received_dir(dest, source)
|
|
assert _read_file(os.path.join(received, "file.txt")) == b"payload"
|
|
|
|
def test_update_newer_destination_keeps_source(self, shared_server):
|
|
source = os.path.join(TEST_DATA_DIR, "remove_rsf_update_src")
|
|
dest = os.path.join(TEST_DATA_DIR, "remove_rsf_update_dst")
|
|
clean_dir(source)
|
|
clean_dir(dest)
|
|
source_file = os.path.join(source, "file.txt")
|
|
with open(source_file, "wb") as f:
|
|
f.write(b"source payload")
|
|
|
|
result, _ = run_client(source, dest, port=shared_server.port)
|
|
assert result.returncode == 0
|
|
|
|
received = get_dest_received_dir(dest, source)
|
|
received_file = os.path.join(received, "file.txt")
|
|
with open(received_file, "wb") as f:
|
|
f.write(b"newer destination payload")
|
|
os.utime(received_file, ns=(time.time_ns() + 10**9, time.time_ns() + 10**9))
|
|
|
|
result, _ = run_client(source, dest, flags=["--remove-source-files", "--update"],
|
|
port=shared_server.port)
|
|
assert result.returncode == 0, f"Sync failed: {result.stderr[:200]}"
|
|
# --update skips a destination that is newer than the source, so the
|
|
# source must not be removed.
|
|
assert os.path.isfile(source_file)
|
|
assert _read_file(received_file) == b"newer destination payload"
|
|
|
|
def test_multithreaded_ignore_existing_keeps_skipped_source(self, shared_server):
|
|
"""The multithreaded writer path must also report per-file outcomes so a
|
|
--remove-source-files sender does not delete skipped sources."""
|
|
source = os.path.join(TEST_DATA_DIR, "remove_rsf_mt_ignore_src")
|
|
dest = os.path.join(TEST_DATA_DIR, "remove_rsf_mt_ignore_dst")
|
|
clean_dir(source)
|
|
clean_dir(dest)
|
|
source_file = os.path.join(source, "file.txt")
|
|
with open(source_file, "wb") as f:
|
|
f.write(b"payload")
|
|
|
|
result, _ = run_client(source, dest, port=shared_server.port)
|
|
assert result.returncode == 0
|
|
|
|
result, _ = run_client(source, dest,
|
|
flags=["--remove-source-files", "--ignore-existing", "-m"],
|
|
port=shared_server.port)
|
|
assert result.returncode == 0, f"Sync failed: {result.stderr[:200]}"
|
|
# Destination already has the file, so the receiver (writer thread)
|
|
# skips it; the source must survive.
|
|
assert os.path.isfile(source_file)
|
|
received = get_dest_received_dir(dest, source)
|
|
assert _read_file(os.path.join(received, "file.txt")) == b"payload"
|
|
|
|
|
|
class TestBackup:
|
|
def _sync(self, source, dest, flags, port):
|
|
return run_client(source, dest, flags=flags, port=port)
|
|
|
|
def test_plain_backup_keeps_previous_version(self, shared_server):
|
|
source = os.path.join(TEST_DATA_DIR, "backup_src")
|
|
dest = os.path.join(TEST_DATA_DIR, "backup_dst")
|
|
clean_dir(source)
|
|
clean_dir(dest)
|
|
source_file = os.path.join(source, "f.txt")
|
|
with open(source_file, "wb") as f:
|
|
f.write(b"AAAA")
|
|
|
|
result, _ = self._sync(source, dest, ["--backup"], shared_server.port)
|
|
assert result.returncode == 0, f"Backup sync failed: {result.stderr[:200]}"
|
|
|
|
with open(source_file, "wb") as f:
|
|
f.write(b"BBBB")
|
|
result, _ = self._sync(source, dest, ["--backup"], shared_server.port)
|
|
assert result.returncode == 0, f"Backup sync failed: {result.stderr[:200]}"
|
|
|
|
received = get_dest_received_dir(dest, source)
|
|
assert _read_file(os.path.join(received, "f.txt")) == b"BBBB"
|
|
# rsync default suffix "~" keeps the overwritten version.
|
|
assert _read_file(os.path.join(received, "f.txt~")) == b"AAAA"
|
|
|
|
def test_backup_custom_suffix(self, shared_server):
|
|
source = os.path.join(TEST_DATA_DIR, "backup_suffix_src")
|
|
dest = os.path.join(TEST_DATA_DIR, "backup_suffix_dst")
|
|
clean_dir(source)
|
|
clean_dir(dest)
|
|
source_file = os.path.join(source, "f.txt")
|
|
with open(source_file, "wb") as f:
|
|
f.write(b"AAAA")
|
|
|
|
flags = ["--backup", "--suffix", ".bak"]
|
|
result, _ = self._sync(source, dest, flags, shared_server.port)
|
|
assert result.returncode == 0, f"Backup sync failed: {result.stderr[:200]}"
|
|
with open(source_file, "wb") as f:
|
|
f.write(b"BBBB")
|
|
result, _ = self._sync(source, dest, flags, shared_server.port)
|
|
assert result.returncode == 0, f"Backup sync failed: {result.stderr[:200]}"
|
|
|
|
received = get_dest_received_dir(dest, source)
|
|
assert _read_file(os.path.join(received, "f.txt")) == b"BBBB"
|
|
assert _read_file(os.path.join(received, "f.txt.bak")) == b"AAAA"
|
|
|
|
def test_backup_dir_stores_backups_separately(self, shared_server):
|
|
source = os.path.join(TEST_DATA_DIR, "backup_dir_src")
|
|
dest = os.path.join(TEST_DATA_DIR, "backup_dir_dst")
|
|
clean_dir(source)
|
|
clean_dir(dest)
|
|
source_file = os.path.join(source, "f.txt")
|
|
with open(source_file, "wb") as f:
|
|
f.write(b"AAAA")
|
|
|
|
flags = ["--backup", "--backup-dir", "backups"]
|
|
result, _ = self._sync(source, dest, flags, shared_server.port)
|
|
assert result.returncode == 0, f"Backup sync failed: {result.stderr[:200]}"
|
|
with open(source_file, "wb") as f:
|
|
f.write(b"BBBB")
|
|
result, _ = self._sync(source, dest, flags, shared_server.port)
|
|
assert result.returncode == 0, f"Backup sync failed: {result.stderr[:200]}"
|
|
|
|
received = get_dest_received_dir(dest, source)
|
|
assert _read_file(os.path.join(received, "f.txt")) == b"BBBB"
|
|
backup = os.path.join(dest, "backups", os.path.relpath(source_file, os.path.sep))
|
|
assert _read_file(backup) == b"AAAA"
|
|
|
|
|
|
class TestPartialDir:
|
|
def test_completed_transfer_installed_in_destination(self, shared_server):
|
|
source = os.path.join(TEST_DATA_DIR, "partial_src")
|
|
dest = os.path.join(TEST_DATA_DIR, "partial_dst")
|
|
clean_dir(source)
|
|
clean_dir(dest)
|
|
source_file = os.path.join(source, "f.txt")
|
|
with open(source_file, "wb") as f:
|
|
f.write(b"partial payload")
|
|
|
|
result, _ = run_client(source, dest, flags=["--partial", "--partial-dir", ".partial"],
|
|
port=shared_server.port)
|
|
assert result.returncode == 0, f"Partial sync failed: {result.stderr[:200]}"
|
|
|
|
received = get_dest_received_dir(dest, source)
|
|
assert _read_file(os.path.join(received, "f.txt")) == b"partial payload"
|
|
# A completed transfer must not remain under the partial directory.
|
|
partial = os.path.join(dest, ".partial", os.path.relpath(source_file, os.path.sep))
|
|
assert not os.path.exists(partial)
|
|
|
|
|
|
class TestLargeFile:
|
|
def test_transfer_100mb_file(self, shared_server):
|
|
source = os.path.join(TEST_DATA_DIR, "large_src")
|
|
dest = os.path.join(TEST_DATA_DIR, "large_dst")
|
|
clean_dir(source)
|
|
clean_dir(dest)
|
|
source_file = os.path.join(source, "big.bin")
|
|
chunk = os.urandom(1024 * 1024)
|
|
with open(source_file, "wb") as f:
|
|
for _ in range(100):
|
|
f.write(chunk)
|
|
|
|
result, _ = run_client(source, dest, port=shared_server.port)
|
|
assert result.returncode == 0, f"Large-file sync failed: {result.stderr[:200]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
assert filecmp.cmp(source_file, os.path.join(received, "big.bin"), shallow=False)
|
|
|
|
class TestOneFileSystem:
|
|
def _make_tree(self, source):
|
|
clean_dir(source)
|
|
os.makedirs(os.path.join(source, "nested", "deeper"))
|
|
with open(os.path.join(source, "root.txt"), "wb") as f:
|
|
f.write(b"root")
|
|
with open(os.path.join(source, "nested", "inner.txt"), "wb") as f:
|
|
f.write(b"inner")
|
|
with open(os.path.join(source, "nested", "deeper", "deep.txt"), "wb") as f:
|
|
f.write(b"deep")
|
|
|
|
def _assert_full_tree_transferred(self, source, dest, port, flags):
|
|
clean_dir(dest)
|
|
result, _ = run_client(source, dest, flags=flags, port=port)
|
|
assert result.returncode == 0, f"Sync failed: {result.stderr[:200]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
mismatches, missing = verify_transfer(source, received)
|
|
assert not missing, f"Missing: {missing}"
|
|
assert not mismatches, f"Mismatch: {mismatches}"
|
|
|
|
def test_x_transfer_matches_plain_over_single_filesystem(self, shared_server):
|
|
source = os.path.join(TEST_DATA_DIR, "ofs_src")
|
|
self._make_tree(source)
|
|
self._assert_full_tree_transferred(source, os.path.join(TEST_DATA_DIR, "ofs_dst"),
|
|
shared_server.port, ["-x"])
|
|
|
|
def test_x_multithreaded_transfer_matches_plain_over_single_filesystem(self, shared_server):
|
|
source = os.path.join(TEST_DATA_DIR, "ofs_m_src")
|
|
self._make_tree(source)
|
|
self._assert_full_tree_transferred(source, os.path.join(TEST_DATA_DIR, "ofs_m_dst"),
|
|
shared_server.port, ["-m", "--one-file-system"])
|
|
|
|
def test_x_skips_other_device_mountpoint(self, shared_server):
|
|
if os.geteuid() != 0 or shutil.which("mount") is None or shutil.which("umount") is None:
|
|
pytest.skip("cross-device test requires root and mount(8)")
|
|
source = os.path.join(TEST_DATA_DIR, "ofs_mnt_src")
|
|
dest = os.path.join(TEST_DATA_DIR, "ofs_mnt_dst")
|
|
dest_plain = os.path.join(TEST_DATA_DIR, "ofs_mnt_plain_dst")
|
|
mountpoint = os.path.join(source, "external")
|
|
clean_dir(source)
|
|
clean_dir(dest)
|
|
clean_dir(dest_plain)
|
|
os.makedirs(mountpoint)
|
|
os.makedirs(os.path.join(source, "nested"))
|
|
with open(os.path.join(source, "root.txt"), "wb") as f:
|
|
f.write(b"root")
|
|
with open(os.path.join(source, "nested", "inner.txt"), "wb") as f:
|
|
f.write(b"inner")
|
|
mounted = False
|
|
unmount_error = ""
|
|
try:
|
|
mount = subprocess.run(["mount", "-t", "tmpfs", "tmpfs", mountpoint],
|
|
capture_output=True, text=True)
|
|
if mount.returncode != 0:
|
|
pytest.skip(f"cannot mount tmpfs: {mount.stderr.strip()}")
|
|
mounted = True
|
|
with open(os.path.join(mountpoint, "away.txt"), "wb") as f:
|
|
f.write(b"cross device")
|
|
result, _ = run_client(source, dest, flags=["-x"], port=shared_server.port)
|
|
assert result.returncode == 0, f"-x sync failed: {result.stderr[:200]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
assert os.path.isfile(os.path.join(received, "root.txt"))
|
|
assert os.path.isfile(os.path.join(received, "nested", "inner.txt"))
|
|
assert not os.path.exists(os.path.join(received, "external", "away.txt")), \
|
|
"-x must not cross into the mounted filesystem"
|
|
result, _ = run_client(source, dest_plain, port=shared_server.port)
|
|
assert result.returncode == 0, f"plain sync failed: {result.stderr[:200]}"
|
|
received_plain = get_dest_received_dir(dest_plain, source)
|
|
assert os.path.isfile(os.path.join(received_plain, "external", "away.txt")), \
|
|
"without -x the mounted subtree must be transferred"
|
|
finally:
|
|
if mounted:
|
|
umount = subprocess.run(["umount", mountpoint], capture_output=True, text=True)
|
|
if umount.returncode != 0:
|
|
unmount_error = umount.stderr.strip()
|
|
if unmount_error:
|
|
pytest.fail(f"test mountpoint {mountpoint} still mounted after umount: {unmount_error}")
|
|
|
|
|
|
def _walk_tmp_files(root):
|
|
"""Recursively list *.tmp* leftovers under root (empty if root missing)."""
|
|
leftovers = []
|
|
if not os.path.isdir(root):
|
|
return leftovers
|
|
for base, _, files in os.walk(root):
|
|
for name in files:
|
|
if ".tmp." in name:
|
|
leftovers.append(os.path.join(base, name))
|
|
return leftovers
|
|
|
|
|
|
class TestTempDir:
|
|
"""--temp-dir=DIR puts the receiver's temporary working copies in a scratch
|
|
directory below the destination root and atomically renames each completed
|
|
file into its final destination. Files sharing a basename across
|
|
directories exercise the flat scratch namespace."""
|
|
|
|
def _make_source(self, name):
|
|
source = os.path.join(TEST_DATA_DIR, name)
|
|
clean_dir(source)
|
|
entries = {
|
|
"top.txt": b"top level\n",
|
|
"sub/file.txt": b"nested file\n" * 20,
|
|
"other/file.txt": b"other nested file\n",
|
|
"sub/deep.bin": bytes(range(256)) * 8,
|
|
}
|
|
for rel, content in entries.items():
|
|
full = os.path.join(source, rel)
|
|
os.makedirs(os.path.dirname(full), exist_ok=True)
|
|
with open(full, "wb") as fh:
|
|
fh.write(content)
|
|
return source
|
|
|
|
def _assert_clean_scratch(self, scratch):
|
|
assert os.path.isdir(scratch), f"scratch dir {scratch} was not created"
|
|
leftovers = _walk_tmp_files(scratch)
|
|
assert leftovers == [], f"leftover temp files in scratch dir: {leftovers}"
|
|
|
|
@pytest.mark.parametrize("mt", [False, True])
|
|
def test_temp_dir_scratch(self, shared_server, mt):
|
|
source = self._make_source("tempdir_src")
|
|
dest = os.path.join(TEST_DATA_DIR, "tempdir_dst")
|
|
clean_dir(dest)
|
|
flags = ["--temp-dir=scratch"] + (["-m"] if mt else [])
|
|
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
|
|
assert result.returncode == 0, f"temp-dir sync failed: {result.stderr[:200]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
mismatches, missing = verify_transfer(source, received)
|
|
assert not missing, f"Missing: {missing}"
|
|
assert not mismatches, f"Mismatch: {mismatches}"
|
|
self._assert_clean_scratch(os.path.join(dest, "scratch"))
|
|
|
|
def test_default_behavior_has_no_scratch_dir(self, shared_server):
|
|
source = self._make_source("tempdir_default_src")
|
|
dest = os.path.join(TEST_DATA_DIR, "tempdir_default_dst")
|
|
clean_dir(dest)
|
|
result, _ = run_client(source, dest, port=shared_server.port)
|
|
assert result.returncode == 0, f"Default sync failed: {result.stderr[:200]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
mismatches, missing = verify_transfer(source, received)
|
|
assert not missing, f"Missing: {missing}"
|
|
assert not mismatches, f"Mismatch: {mismatches}"
|
|
assert not os.path.exists(os.path.join(dest, "scratch"))
|
|
|
|
def test_temp_dir_ignored_with_inplace(self, shared_server):
|
|
"""--inplace writes directly into the destination; --temp-dir must not
|
|
redirect those writes into a scratch dir."""
|
|
source = self._make_source("tempdir_inplace_src")
|
|
dest = os.path.join(TEST_DATA_DIR, "tempdir_inplace_dst")
|
|
clean_dir(dest)
|
|
result, _ = run_client(source, dest,
|
|
flags=["--inplace", "--temp-dir=scratch"],
|
|
port=shared_server.port)
|
|
assert result.returncode == 0, f"inplace+temp-dir sync failed: {result.stderr[:200]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
mismatches, missing = verify_transfer(source, received)
|
|
assert not missing, f"Missing: {missing}"
|
|
assert not mismatches, f"Mismatch: {mismatches}"
|
|
assert not os.path.exists(os.path.join(dest, "scratch")), \
|
|
"--inplace wrote through the scratch dir"
|
|
|
|
def test_temp_dir_ignored_with_partial_dir(self, shared_server):
|
|
"""--partial --partial-dir already stages in a separate directory;
|
|
--temp-dir must not be used on top of it."""
|
|
source = self._make_source("tempdir_partial_src")
|
|
dest = os.path.join(TEST_DATA_DIR, "tempdir_partial_dst")
|
|
clean_dir(dest)
|
|
result, _ = run_client(source, dest,
|
|
flags=["--partial", "--partial-dir", ".partial",
|
|
"--temp-dir=scratch"],
|
|
port=shared_server.port)
|
|
assert result.returncode == 0, f"partial+temp-dir sync failed: {result.stderr[:200]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
mismatches, missing = verify_transfer(source, received)
|
|
assert not missing, f"Missing: {missing}"
|
|
assert not mismatches, f"Mismatch: {mismatches}"
|
|
partial = os.path.join(dest, ".partial",
|
|
os.path.relpath(os.path.join(source, "top.txt"), os.path.sep))
|
|
assert not os.path.exists(partial), "completed file remained under the partial dir"
|
|
assert not os.path.exists(os.path.join(dest, "scratch")), \
|
|
"--partial-dir wrote through the scratch dir"
|
|
|
|
def test_temp_dir_escape_rejected(self, shared_server):
|
|
source = self._make_source("tempdir_escape_src")
|
|
dest = os.path.join(TEST_DATA_DIR, "tempdir_escape_dst")
|
|
clean_dir(dest)
|
|
# "../escape" would resolve one level above the destination root.
|
|
outside = os.path.join(TEST_DATA_DIR, "escape")
|
|
assert not os.path.lexists(outside)
|
|
|
|
result, _ = run_client(source, dest, flags=["--temp-dir=../escape"],
|
|
port=shared_server.port)
|
|
assert result.returncode != 0, "relative escaping --temp-dir was not rejected"
|
|
assert not os.path.lexists(outside), "file created outside the destination root"
|
|
|
|
clean_dir(dest)
|
|
abs_escape = os.path.join(TEST_DATA_DIR, "abs_escape_probe")
|
|
assert not os.path.lexists(abs_escape)
|
|
result, _ = run_client(source, dest, flags=["--temp-dir", abs_escape],
|
|
port=shared_server.port)
|
|
assert result.returncode != 0, "absolute --temp-dir was not rejected"
|
|
assert not os.path.lexists(abs_escape), "file created outside the destination root"
|
|
|
|
|
|
def _source_files():
|
|
"""All source paths (absolute) that a transfer would send right now."""
|
|
return [
|
|
os.path.join(root, name)
|
|
for root, _dirs, names in os.walk(SOURCE_DIR)
|
|
for name in names
|
|
]
|
|
|
|
|
|
class TestListOnly:
|
|
"""--list-only prints every transfer candidate and changes nothing."""
|
|
|
|
def test_list_only_prints_each_file_and_does_not_transfer(self):
|
|
clean_dir(DEST_DIR)
|
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["--list-only"])
|
|
assert result.returncode == 0, f"list-only failed: {result.stderr[:200]}"
|
|
for full_path in _source_files():
|
|
assert full_path in result.stdout, f"list-only omitted {full_path}"
|
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
|
assert not os.path.exists(received), "list-only wrote to the destination"
|
|
|
|
def test_list_only_with_dry_run_does_not_error(self):
|
|
clean_dir(DEST_DIR)
|
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["--list-only", "--dry-run"])
|
|
assert result.returncode == 0, f"list-only -n failed: {result.stderr[:200]}"
|
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
|
assert not os.path.exists(received)
|
|
|
|
def test_list_only_multithreaded(self):
|
|
clean_dir(DEST_DIR)
|
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["--list-only", "-m"])
|
|
assert result.returncode == 0, f"list-only -m failed: {result.stderr[:200]}"
|
|
for full_path in _source_files():
|
|
assert full_path in result.stdout, f"list-only -m omitted {full_path}"
|
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
|
assert not os.path.exists(received), "list-only -m wrote to the destination"
|
|
|
|
|
|
class TestItemizeChanges:
|
|
"""-i/--itemize-changes prints rsync-style lines only for files sent."""
|
|
|
|
def test_first_run_prints_sent_lines(self, shared_server):
|
|
clean_dir(DEST_DIR)
|
|
result, _ = run_client(SOURCE_DIR, DEST_DIR,
|
|
flags=["-M", "-i"], port=shared_server.port)
|
|
assert result.returncode == 0, f"itemize sync failed: {result.stderr[:200]}"
|
|
sent_lines = {">f+++++++++ " + p for p in _source_files()}
|
|
assert sent_lines <= set(result.stdout.splitlines()), (
|
|
f"missing itemize lines; got {result.stdout[:500]}"
|
|
)
|
|
|
|
def test_incremental_second_run_prints_no_line_for_unchanged(self, shared_server):
|
|
clean_dir(DEST_DIR)
|
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port)
|
|
assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}"
|
|
result, _ = run_client(SOURCE_DIR, DEST_DIR,
|
|
flags=["-M", "-i", "--incremental"],
|
|
port=shared_server.port)
|
|
assert result.returncode == 0, f"incremental itemize failed: {result.stderr[:200]}"
|
|
itemized = [line for line in result.stdout.splitlines() if line and line[0] in ">.<c"]
|
|
assert itemized == [], f"unchanged files were itemized: {itemized[:5]}"
|
|
|
|
def test_multithreaded_emits_same_itemize_lines(self, shared_server):
|
|
clean_dir(DEST_DIR)
|
|
result, _ = run_client(SOURCE_DIR, DEST_DIR,
|
|
flags=["-M", "-i", "-m"], port=shared_server.port)
|
|
assert result.returncode == 0, f"itemize -m sync failed: {result.stderr[:200]}"
|
|
sent_lines = {">f+++++++++ " + p for p in _source_files()}
|
|
assert sent_lines <= set(result.stdout.splitlines()), (
|
|
f"missing itemize lines in -m mode; got {result.stdout[:500]}"
|
|
)
|
|
|
|
def test_dry_run_with_itemize_does_not_error(self):
|
|
clean_dir(DEST_DIR)
|
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-i", "--dry-run"])
|
|
assert result.returncode == 0, f"dry-run -i failed: {result.stderr[:200]}"
|
|
|
|
def test_changed_file_on_second_incremental_run_prints_exactly_one_line(self, shared_server):
|
|
"""A changed file itemizes exactly once on an incremental rerun while
|
|
unchanged files print nothing (no double emission)."""
|
|
source = os.path.join(TEST_DATA_DIR, "itemize_change_src")
|
|
dest = os.path.join(TEST_DATA_DIR, "itemize_change_dst")
|
|
clean_dir(source)
|
|
clean_dir(dest)
|
|
changed = os.path.join(source, "changed.txt")
|
|
untouched = os.path.join(source, "untouched.txt")
|
|
with open(changed, "wb") as fh:
|
|
fh.write(b"original\n")
|
|
with open(untouched, "wb") as fh:
|
|
fh.write(b"stable\n")
|
|
|
|
result, _ = run_client(source, dest, flags=["-M"], port=shared_server.port)
|
|
assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}"
|
|
|
|
with open(changed, "wb") as fh:
|
|
fh.write(b"edited payload\n")
|
|
|
|
result, _ = run_client(source, dest,
|
|
flags=["-M", "-i", "--incremental"],
|
|
port=shared_server.port)
|
|
assert result.returncode == 0, f"incremental itemize failed: {result.stderr[:200]}"
|
|
itemized = [line for line in result.stdout.splitlines() if line.startswith(">f")]
|
|
assert itemized == [">f+++++++++ " + changed], (
|
|
f"expected exactly one itemize line for {changed}, got {itemized}"
|
|
)
|
|
received = get_dest_received_dir(dest, source)
|
|
assert _read_file(os.path.join(received, "changed.txt")) == b"edited payload\n"
|
|
assert _read_file(os.path.join(received, "untouched.txt")) == b"stable\n"
|
|
|
|
|
|
class TestOutFormat:
|
|
"""--out-format prints a line per transferred file using the template."""
|
|
|
|
def test_out_format_path_and_size(self, shared_server):
|
|
clean_dir(DEST_DIR)
|
|
result, _ = run_client(SOURCE_DIR, DEST_DIR,
|
|
flags=["--out-format=%f %l"], port=shared_server.port)
|
|
assert result.returncode == 0, f"out-format sync failed: {result.stderr[:200]}"
|
|
expected = {f"{p} {os.path.getsize(p)}" for p in _source_files()}
|
|
got = set(result.stdout.splitlines())
|
|
assert expected <= got, f"out-format lines missing: expected {len(expected)} got {len(got)}"
|
|
|
|
def test_out_format_multithreaded_matches_single(self, shared_server):
|
|
clean_dir(DEST_DIR)
|
|
result, _ = run_client(SOURCE_DIR, DEST_DIR,
|
|
flags=["--out-format=%f %l", "-m"], port=shared_server.port)
|
|
assert result.returncode == 0, f"out-format -m sync failed: {result.stderr[:200]}"
|
|
expected = {f"{p} {os.path.getsize(p)}" for p in _source_files()}
|
|
got = set(result.stdout.splitlines())
|
|
assert expected <= got, f"out-format -m lines missing: {result.stdout[:500]}"
|
|
|
|
|
|
class TestLogFileFormat:
|
|
"""--log-file plus --log-file-format writes per-file lines to the log."""
|
|
|
|
def test_log_file_format_writes_transferred_files(self, shared_server):
|
|
clean_dir(DEST_DIR)
|
|
log_path = os.path.join(TEST_DATA_DIR, "itemize_transfer.log")
|
|
if os.path.exists(log_path):
|
|
os.unlink(log_path)
|
|
result, _ = run_client(
|
|
SOURCE_DIR, DEST_DIR,
|
|
flags=["--log-file", log_path, "--log-file-format=%f %l"],
|
|
port=shared_server.port,
|
|
)
|
|
assert result.returncode == 0, f"log-file sync failed: {result.stderr[:200]}"
|
|
assert os.path.exists(log_path), "--log-file created no log"
|
|
with open(log_path, encoding="utf-8", errors="replace") as fh:
|
|
content = fh.read()
|
|
expected = {f"{p} {os.path.getsize(p)}" for p in _source_files()}
|
|
for line in expected:
|
|
assert line in content, f"log file missing {line!r}"
|
|
|
|
def test_log_file_format_multithreaded_writes_transferred_files(self, shared_server):
|
|
source = os.path.join(TEST_DATA_DIR, "itemize_log_mt_src")
|
|
dest = os.path.join(TEST_DATA_DIR, "itemize_log_mt_dst")
|
|
clean_dir(source)
|
|
clean_dir(dest)
|
|
files = {"a.txt": b"alpha\n", "b.txt": b"beta\n"}
|
|
for rel, data in files.items():
|
|
with open(os.path.join(source, rel), "wb") as fh:
|
|
fh.write(data)
|
|
log_path = os.path.join(TEST_DATA_DIR, "itemize_mt.log")
|
|
if os.path.exists(log_path):
|
|
os.unlink(log_path)
|
|
result, _ = run_client(
|
|
source,
|
|
dest,
|
|
flags=["--log-file", log_path, "--log-file-format=%f %l", "-m"],
|
|
port=shared_server.port,
|
|
)
|
|
assert result.returncode == 0, f"log-file -m sync failed: {result.stderr[:200]}"
|
|
assert os.path.exists(log_path), "--log-file created no log"
|
|
with open(log_path, encoding="utf-8", errors="replace") as fh:
|
|
content = fh.read()
|
|
expected = {f"{os.path.join(source, rel)} {len(data)}" for rel, data in files.items()}
|
|
for line in expected:
|
|
assert line in content, f"log file (-m) missing {line!r}"
|
|
|
|
|
|
class TestDelayUpdates:
|
|
"""--delay-updates stages every updated file under a private 0700 staging
|
|
directory inside the receive root and atomically publishes all of them only
|
|
after the whole transfer succeeds."""
|
|
|
|
STAGING = ".fastsync-stage"
|
|
|
|
def _make_source(self, name):
|
|
source = os.path.join(TEST_DATA_DIR, name)
|
|
clean_dir(source)
|
|
entries = {
|
|
"top.txt": b"top level\n",
|
|
"sub/deep.txt": b"deeply nested file\n",
|
|
"sub/another.txt": b"another nested file\n" * 20,
|
|
"binary.bin": bytes(range(256)) * 4,
|
|
}
|
|
for rel, content in entries.items():
|
|
full = os.path.join(source, rel)
|
|
os.makedirs(os.path.dirname(full), exist_ok=True)
|
|
with open(full, "wb") as fh:
|
|
fh.write(content)
|
|
return source
|
|
|
|
@pytest.mark.parametrize("mt", [False, True])
|
|
def test_delay_updates_matches_plain_transfer(self, shared_server, mt):
|
|
source = self._make_source("delay_match_src")
|
|
plain_dest = os.path.join(TEST_DATA_DIR, "delay_match_plain_dst")
|
|
delay_dest = os.path.join(TEST_DATA_DIR, "delay_match_delay_dst")
|
|
clean_dir(plain_dest)
|
|
clean_dir(delay_dest)
|
|
|
|
result, _ = run_client(source, plain_dest, port=shared_server.port)
|
|
assert result.returncode == 0, f"plain sync failed: {result.stderr[:200]}"
|
|
flags = ["--delay-updates"] + (["-m"] if mt else [])
|
|
result, _ = run_client(source, delay_dest, flags=flags, port=shared_server.port)
|
|
assert result.returncode == 0, f"delay-updates sync failed: {result.stderr[:200]}"
|
|
|
|
plain_received = get_dest_received_dir(plain_dest, source)
|
|
delay_received = get_dest_received_dir(delay_dest, source)
|
|
mismatches, missing = verify_transfer(source, delay_received)
|
|
assert not missing, f"Missing: {missing}"
|
|
assert not mismatches, f"Mismatch: {mismatches}"
|
|
for root, _dirs, files in os.walk(delay_received):
|
|
for name in files:
|
|
rel = os.path.relpath(os.path.join(root, name), delay_received)
|
|
assert filecmp.cmp(os.path.join(plain_received, rel),
|
|
os.path.join(delay_received, rel), shallow=False), rel
|
|
assert not os.path.isdir(os.path.join(delay_dest, self.STAGING)), \
|
|
"staging directory left behind after a successful delayed transfer"
|
|
|
|
@pytest.mark.parametrize("mt", [False, True])
|
|
def test_delay_updates_incremental_rerun_no_leftovers(self, shared_server, mt):
|
|
source = self._make_source("delay_rerun_src")
|
|
dest = os.path.join(TEST_DATA_DIR, "delay_rerun_dst")
|
|
clean_dir(dest)
|
|
flags = ["--delay-updates", "-M", "--incremental"] + (["-m"] if mt else [])
|
|
|
|
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
|
|
assert result.returncode == 0, f"first delayed sync failed: {result.stderr[:200]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
mismatches, missing = verify_transfer(source, received)
|
|
assert not missing and not mismatches
|
|
assert not os.path.isdir(os.path.join(dest, self.STAGING))
|
|
|
|
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
|
|
assert result.returncode == 0, f"second delayed sync failed: {result.stderr[:200]}"
|
|
assert not os.path.isdir(os.path.join(dest, self.STAGING)), \
|
|
"fully-skipped delayed run left a staging directory"
|
|
|
|
@pytest.mark.parametrize("mt", [False, True])
|
|
def test_remove_source_files_with_delay_updates(self, shared_server, mt):
|
|
source = self._make_source("delay_rsf_src")
|
|
dest = os.path.join(TEST_DATA_DIR, "delay_rsf_dst")
|
|
clean_dir(dest)
|
|
flags = ["--remove-source-files", "--delay-updates"] + (["-m"] if mt else [])
|
|
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
|
|
assert result.returncode == 0, f"delayed remove-source sync failed: {result.stderr[:200]}"
|
|
|
|
# Sources are removed only after the receiver published every file.
|
|
for root, _dirs, files in os.walk(source):
|
|
assert files == [], f"source files survived delayed remove-source-files: {files}"
|
|
received = get_dest_received_dir(dest, source)
|
|
assert os.path.isfile(os.path.join(received, "top.txt"))
|
|
assert os.path.isfile(os.path.join(received, "sub", "deep.txt"))
|
|
assert not os.path.isdir(os.path.join(dest, self.STAGING))
|
|
|
|
@pytest.mark.parametrize("mt", [False, True])
|
|
def test_delete_with_delay_updates(self, mt):
|
|
"""--delete runs before publication, so the delete walker must not treat
|
|
the staging directory as a set of extras: a changed file must still be
|
|
published after genuine extras are removed. Uses its own server started
|
|
with --allow-delete (the shared session server refuses deletion)."""
|
|
source = os.path.join(TEST_DATA_DIR, "delay_delete_src")
|
|
dest = os.path.join(TEST_DATA_DIR, "delay_delete_dst")
|
|
clean_dir(source)
|
|
clean_dir(dest)
|
|
with open(os.path.join(source, "f.txt"), "wb") as fh:
|
|
fh.write(b"AAAA")
|
|
with open(os.path.join(source, "extra.txt"), "wb") as fh:
|
|
fh.write(b"seed extra")
|
|
with ServerManager() as server:
|
|
server.start(extra_args=["--allow-delete"])
|
|
result, _ = run_client(source, dest, port=server.port)
|
|
assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
assert _read_file(os.path.join(received, "extra.txt")) == b"seed extra"
|
|
|
|
# Second source state: f.txt changed, extra.txt removed from source.
|
|
with open(os.path.join(source, "f.txt"), "wb") as fh:
|
|
fh.write(b"BBBB")
|
|
os.remove(os.path.join(source, "extra.txt"))
|
|
|
|
flags = ["--delete", "--delay-updates"] + (["-m"] if mt else [])
|
|
result, _ = run_client(source, dest, flags=flags, port=server.port)
|
|
assert result.returncode == 0, \
|
|
f"delete+delay-updates sync failed: {result.stderr[:200]}"
|
|
assert _read_file(os.path.join(received, "f.txt")) == b"BBBB", \
|
|
"changed file was not published after deletion"
|
|
assert not os.path.exists(os.path.join(received, "extra.txt")), \
|
|
"genuine extra file was not deleted"
|
|
assert not os.path.isdir(os.path.join(dest, self.STAGING))
|
|
|
|
def test_delay_updates_rejects_reserved_backup_dir(self):
|
|
"""--backup-dir equal to the internal staging name must be rejected so
|
|
an old backup can never be silently installed as the "new" file."""
|
|
source = self._make_source("delay_reserved_bak_src")
|
|
for variant, suffix in (("bare", ""), ("slash", "/")):
|
|
dest = os.path.join(TEST_DATA_DIR, f"delay_reserved_bak_{variant}_dst")
|
|
clean_dir(dest)
|
|
flags = ["--delay-updates", "--backup", "--backup-dir",
|
|
".fastsync-stage" + suffix]
|
|
result, _ = run_client(source, dest, flags=flags, port=None)
|
|
assert result.returncode != 0, \
|
|
f"reserved --backup-dir '{suffix}' was accepted"
|
|
assert not os.path.isdir(os.path.join(dest, self.STAGING)), \
|
|
"staging directory created by a rejected run"
|
|
|
|
@pytest.mark.parametrize("remove_source_files", [False, True])
|
|
@pytest.mark.parametrize("mt", [False, True])
|
|
def test_mid_publish_failure_keeps_published_no_rollback(self, shared_server, mt,
|
|
remove_source_files):
|
|
"""A stage->publish rename failing part way through publication must
|
|
fail the whole transfer, keep the already-published top-level file (no
|
|
rollback), leave the not-yet-published nested file absent, and clean up
|
|
the staging area. A regular file is planted where the final "sub"
|
|
directory must be created, so the nested rename fails (mkdir over a
|
|
file is impossible even for root) while the top-level file, which is
|
|
always staged first, publishes. With --remove-source-files the sender
|
|
must keep every source because no success/outcome frame is ever sent."""
|
|
source = os.path.join(TEST_DATA_DIR, "delay_mid_src")
|
|
dest = os.path.join(TEST_DATA_DIR, "delay_mid_dst")
|
|
clean_dir(source)
|
|
clean_dir(dest)
|
|
top_path = os.path.join(source, "top.txt")
|
|
deep_path = os.path.join(source, "sub", "deep.txt")
|
|
with open(top_path, "wb") as fh:
|
|
fh.write(b"top payload\n")
|
|
os.makedirs(os.path.dirname(deep_path))
|
|
with open(deep_path, "wb") as fh:
|
|
fh.write(b"deep payload\n")
|
|
|
|
received = get_dest_received_dir(dest, source)
|
|
os.makedirs(received)
|
|
with open(os.path.join(received, "sub"), "wb") as fh:
|
|
fh.write(b"blocks the nested destination directory")
|
|
|
|
flags = ["--delay-updates"] + (["-m"] if mt else [])
|
|
if remove_source_files:
|
|
flags += ["--remove-source-files"]
|
|
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
|
|
assert result.returncode != 0, "blocked nested publish did not fail"
|
|
|
|
# The top-level file was published before the nested rename failed and
|
|
# is intentionally NOT rolled back.
|
|
assert _read_file(os.path.join(received, "top.txt")) == b"top payload\n"
|
|
# The nested file was never published.
|
|
assert not os.path.lexists(os.path.join(received, "sub", "deep.txt")), \
|
|
"nested file appeared despite a failed publish"
|
|
assert not os.path.isdir(os.path.join(dest, self.STAGING)), \
|
|
"staging leftovers after a failed mid-publish"
|
|
# Sources survive: no success frame was sent, so a remove-source-files
|
|
# sender must not delete anything.
|
|
assert os.path.isfile(top_path)
|
|
assert os.path.isfile(deep_path)
|
|
|
|
@pytest.mark.parametrize("mt", [False, True])
|
|
def test_remove_source_files_keeps_receiver_skipped_source(self, shared_server, mt):
|
|
"""With --delay-updates + --ignore-existing a receiver-skipped source
|
|
must survive (its outcome is sent only after publication) while a
|
|
freshly delivered file is published and its source removed."""
|
|
source = os.path.join(TEST_DATA_DIR, "delay_rsf_skip_src")
|
|
dest = os.path.join(TEST_DATA_DIR, "delay_rsf_skip_dst")
|
|
clean_dir(source)
|
|
clean_dir(dest)
|
|
with open(os.path.join(source, "keep.txt"), "wb") as fh:
|
|
fh.write(b"existing on dest")
|
|
result, _ = run_client(source, dest, port=shared_server.port)
|
|
assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}"
|
|
|
|
with open(os.path.join(source, "keep.txt"), "wb") as fh:
|
|
fh.write(b"changed on source")
|
|
with open(os.path.join(source, "deliver.txt"), "wb") as fh:
|
|
fh.write(b"new file")
|
|
flags = ["--remove-source-files", "--ignore-existing", "--delay-updates"] + (["-m"] if mt else [])
|
|
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
|
|
assert result.returncode == 0, f"delayed skip sync failed: {result.stderr[:200]}"
|
|
# keep.txt already existed at the destination: receiver skip -> source stays.
|
|
assert os.path.isfile(os.path.join(source, "keep.txt")), \
|
|
"receiver-skipped source was removed despite --ignore-existing"
|
|
# deliver.txt was new: staged, published, and its source removed.
|
|
assert not os.path.isfile(os.path.join(source, "deliver.txt")), \
|
|
"published source was not removed"
|
|
received = get_dest_received_dir(dest, source)
|
|
assert not os.path.isdir(os.path.join(dest, self.STAGING))
|
|
|
|
def test_delay_updates_rejects_inplace(self):
|
|
source = self._make_source("delay_inplace_src")
|
|
dest = os.path.join(TEST_DATA_DIR, "delay_inplace_dst")
|
|
clean_dir(dest)
|
|
result, _ = run_client(source, dest, flags=["--delay-updates", "--inplace"])
|
|
assert result.returncode != 0, "--inplace with --delay-updates was accepted"
|
|
assert not os.path.isdir(os.path.join(dest, self.STAGING))
|
|
|
|
class TestFilesFrom:
|
|
"""--files-from transfers exactly the listed files; a listed directory
|
|
transfers its whole subtree. The manifest (and thus --delete) derives from
|
|
what was actually sent."""
|
|
|
|
|
|
def _make_relative_source(name):
|
|
"""A small tree used by the -R/--dirs/--no-implied-dirs tests."""
|
|
source = os.path.join(TEST_DATA_DIR, name)
|
|
clean_dir(source)
|
|
entries = {
|
|
"top.txt": b"top\n",
|
|
"a/b.txt": b"nested\n",
|
|
"sub/x.txt": b"x\n",
|
|
"sub/y.txt": b"y\n",
|
|
"dir1/keep.txt": b"dir content\n",
|
|
}
|
|
for rel, content in entries.items():
|
|
full = os.path.join(source, rel)
|
|
os.makedirs(os.path.dirname(full), exist_ok=True)
|
|
with open(full, "wb") as fh:
|
|
fh.write(content)
|
|
return source
|
|
|
|
|
|
def _write_rel_list(rel_text):
|
|
path = os.path.join(TEST_DATA_DIR, "rel_list.txt")
|
|
with open(path, "wb") as fh:
|
|
fh.write(rel_text)
|
|
return path
|
|
|
|
|
|
class TestRelativeFilesFrom:
|
|
"""-R/--relative with --files-from keeps each listed entry's bare relative
|
|
destination path below the destination root instead of mirroring the full
|
|
source path. Without -R the layout is unchanged (full source mirror)."""
|
|
|
|
@pytest.mark.parametrize("mt", [False, True])
|
|
def test_relative_files_from_keeps_relative_layout(self, shared_server, mt):
|
|
source = _make_relative_source("rel_src")
|
|
dest = os.path.join(TEST_DATA_DIR, "rel_dst")
|
|
clean_dir(dest)
|
|
lst = _write_rel_list(b"top.txt\nsub/x.txt\n")
|
|
flags = ["--files-from", lst, "-R"] + (["-m"] if mt else [])
|
|
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
|
|
assert result.returncode == 0, f"-R files-from sync failed: {result.stderr[:200]}"
|
|
assert _read_file(os.path.join(dest, "sub", "x.txt")) == b"x\n", \
|
|
"listed file must land at <dest>/sub/x.txt"
|
|
assert _read_file(os.path.join(dest, "top.txt")) == b"top\n", \
|
|
"top-level listed file must land at <dest>/top.txt"
|
|
assert not os.path.exists(os.path.join(dest, "sub", "y.txt"))
|
|
# The source-root mirror must not be reproduced under -R.
|
|
assert not os.path.exists(get_dest_received_dir(dest, source)), \
|
|
"-R must not mirror the full source path"
|
|
|
|
@pytest.mark.parametrize("mt", [False, True])
|
|
def test_relative_without_files_from_has_no_effect(self, shared_server, mt):
|
|
"""-R alone (no --files-from) must leave the normal full-source mirror
|
|
layout untouched."""
|
|
source = _make_relative_source("rel_only_src")
|
|
dest = os.path.join(TEST_DATA_DIR, "rel_only_dst")
|
|
clean_dir(dest)
|
|
flags = ["-R"] + (["-m"] if mt else [])
|
|
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
|
|
assert result.returncode == 0, f"-R alone sync failed: {result.stderr[:200]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
mismatches, missing = verify_transfer(source, received)
|
|
assert not missing and not mismatches
|
|
|
|
@pytest.mark.parametrize("mt", [False, True])
|
|
def test_without_relative_layout_unchanged(self, shared_server, mt):
|
|
source = _make_relative_source("rel_noR_src")
|
|
dest = os.path.join(TEST_DATA_DIR, "rel_noR_dst")
|
|
clean_dir(dest)
|
|
lst = _write_rel_list(b"sub/x.txt\n")
|
|
flags = ["--files-from", lst] + (["-m"] if mt else [])
|
|
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
|
|
assert result.returncode == 0, f"files-from sync failed: {result.stderr[:200]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
assert _read_file(os.path.join(received, "sub", "x.txt")) == b"x\n", \
|
|
"without -R the full source mirror layout is preserved"
|
|
assert not os.path.exists(os.path.join(dest, "sub")), \
|
|
"bare relative layout must not appear without -R"
|
|
|
|
def test_relative_delete_manifest_stays_consistent(self):
|
|
"""--delete derives from the sent (-R) relative paths, so a later
|
|
subset run removes unlisted relative entries but keeps listed ones."""
|
|
source = _make_relative_source("rel_del_src")
|
|
dest = os.path.join(TEST_DATA_DIR, "rel_del_dst")
|
|
clean_dir(dest)
|
|
with ServerManager() as server:
|
|
server.start(extra_args=["--allow-delete"])
|
|
lst = _write_rel_list(b"sub/x.txt\nsub/y.txt\n")
|
|
result, _ = run_client(source, dest, flags=["--files-from", lst, "-R"],
|
|
port=server.port)
|
|
assert result.returncode == 0, f"seed -R sync failed: {result.stderr[:200]}"
|
|
assert os.path.isfile(os.path.join(dest, "sub", "y.txt"))
|
|
|
|
subset = _write_rel_list(b"sub/x.txt\n")
|
|
result, _ = run_client(source, dest,
|
|
flags=["--files-from", subset, "-R", "--delete"],
|
|
port=server.port)
|
|
assert result.returncode == 0, f"-R delete sync failed: {result.stderr[:200]}"
|
|
assert os.path.isfile(os.path.join(dest, "sub", "x.txt")), "listed file was deleted"
|
|
assert not os.path.exists(os.path.join(dest, "sub", "y.txt")), \
|
|
"unlisted relative file was not deleted"
|
|
|
|
|
|
class TestMissingArgs:
|
|
"""--ignore-missing-args / --delete-missing-args: a --files-from entry that
|
|
does not exist under the source is skipped instead of failing the run, and
|
|
(delete-missing) its destination mirror is removed receiver-side. Following
|
|
rsync, --delete-missing-args implies --ignore-missing-args but is
|
|
independent of --delete: unrelated extras stay unless --delete is also
|
|
given, and the missing-args deletion (an explicit user request) is never
|
|
blocked by filter-exclusion protection."""
|
|
|
|
def _make_source(self, name):
|
|
source = os.path.join(TEST_DATA_DIR, name)
|
|
clean_dir(source)
|
|
for rel, content in {
|
|
"a.txt": b"a\n",
|
|
"sub/b.txt": b"b\n",
|
|
"keep.txt": b"keep\n",
|
|
"prot/kept.txt": b"kept\n",
|
|
}.items():
|
|
full = os.path.join(source, rel)
|
|
os.makedirs(os.path.dirname(full), exist_ok=True)
|
|
with open(full, "wb") as fh:
|
|
fh.write(content)
|
|
return source
|
|
|
|
@pytest.mark.parametrize("mt", [False, True])
|
|
def test_missing_entry_is_hard_error_before_transfer(self, shared_server, mt):
|
|
source = self._make_source("mg_default_src")
|
|
dest = os.path.join(TEST_DATA_DIR, "mg_default_dst")
|
|
clean_dir(dest)
|
|
lst = _write_rel_list(b"a.txt\ngone.txt\nsub/b.txt\n")
|
|
flags = ["--files-from", lst] + (["-m"] if mt else [])
|
|
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
|
|
assert result.returncode != 0, "a listed-but-missing entry did not fail the run"
|
|
assert "gone.txt" in (result.stderr or result.stdout)
|
|
received = get_dest_received_dir(dest, source)
|
|
assert not os.path.isfile(os.path.join(received, "a.txt")), \
|
|
"the transfer started despite the missing-entry hard error"
|
|
|
|
@pytest.mark.parametrize("mt", [False, True])
|
|
def test_ignore_missing_args_transfers_the_rest(self, shared_server, mt):
|
|
source = self._make_source("mg_ignore_src")
|
|
dest = os.path.join(TEST_DATA_DIR, "mg_ignore_dst")
|
|
clean_dir(dest)
|
|
lst = _write_rel_list(b"a.txt\ngone.txt\nsub/b.txt\n")
|
|
flags = ["--files-from", lst, "--ignore-missing-args"] + (["-m"] if mt else [])
|
|
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
|
|
assert result.returncode == 0, f"ignore-missing-args sync failed: {result.stderr[:300]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
assert _read_file(os.path.join(received, "a.txt")) == b"a\n"
|
|
assert _read_file(os.path.join(received, "sub", "b.txt")) == b"b\n"
|
|
assert not os.path.exists(os.path.join(received, "gone.txt")), \
|
|
"nothing was transferred for the missing entry"
|
|
assert "--ignore-missing-args" in (result.stderr or result.stdout), \
|
|
"the skipped entry must be observable (not a silent no-op)"
|
|
|
|
@pytest.mark.parametrize("mt", [False, True])
|
|
def test_all_missing_entries_succeed_transferring_nothing(self, shared_server, mt):
|
|
source = self._make_source("mg_all_missing_src")
|
|
dest = os.path.join(TEST_DATA_DIR, "mg_all_missing_dst")
|
|
clean_dir(dest)
|
|
lst = _write_rel_list(b"gone1.txt\ngone2.txt\n")
|
|
flags = ["--files-from", lst, "--ignore-missing-args"] + (["-m"] if mt else [])
|
|
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
|
|
assert result.returncode == 0, \
|
|
f"all-missing run should succeed (rsync parity): {result.stderr[:300]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
assert not os.path.exists(os.path.join(received, "gone1.txt"))
|
|
|
|
@pytest.mark.parametrize("mt", [False, True])
|
|
def test_empty_list_stays_a_hard_error(self, shared_server, mt):
|
|
source = self._make_source("mg_empty_src")
|
|
dest = os.path.join(TEST_DATA_DIR, "mg_empty_dst")
|
|
clean_dir(dest)
|
|
lst = _write_rel_list(b"")
|
|
flags = ["--files-from", lst, "--ignore-missing-args"] + (["-m"] if mt else [])
|
|
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
|
|
assert result.returncode != 0, "an empty --files-from list must stay a hard error"
|
|
assert "contains no entries" in (result.stderr or result.stdout)
|
|
|
|
@pytest.mark.parametrize("mt", [False, True])
|
|
def test_delete_missing_removes_mirror_not_unrelated(self, mt):
|
|
"""-R layout: --delete-missing-args deletes exactly the missing entry's
|
|
destination mirror (bare relative path) and leaves unrelated extras
|
|
untouched; with --delete also present the unrelated extras go too."""
|
|
source = self._make_source("mg_del_src")
|
|
dest = os.path.join(TEST_DATA_DIR, "mg_del_dst")
|
|
clean_dir(dest)
|
|
with ServerManager() as server:
|
|
server.start(extra_args=["--allow-delete"])
|
|
seed = _write_rel_list(b"a.txt\nsub/b.txt\n")
|
|
result, _ = run_client(source, dest,
|
|
flags=["--files-from", seed, "-R"] + (["-m"] if mt else []),
|
|
port=server.port)
|
|
assert result.returncode == 0, f"seed -R sync failed: {result.stderr[:200]}"
|
|
assert os.path.isfile(os.path.join(dest, "a.txt"))
|
|
assert os.path.isfile(os.path.join(dest, "sub", "b.txt"))
|
|
|
|
# Plant the missing entry's destination mirror and an unrelated extra.
|
|
with open(os.path.join(dest, "gone.txt"), "w") as fh:
|
|
fh.write("stale mirror")
|
|
with open(os.path.join(dest, "unrelated.txt"), "w") as fh:
|
|
fh.write("unrelated")
|
|
|
|
lst = _write_rel_list(b"a.txt\ngone.txt\nsub/b.txt\n")
|
|
flags = ["--files-from", lst, "-R", "--delete-missing-args"] + (["-m"] if mt else [])
|
|
result, _ = run_client(source, dest, flags=flags, port=server.port)
|
|
assert result.returncode == 0, f"delete-missing sync failed: {result.stderr[:300]}"
|
|
assert not os.path.exists(os.path.join(dest, "gone.txt")), \
|
|
"the missing entry's destination mirror was not deleted"
|
|
assert os.path.isfile(os.path.join(dest, "unrelated.txt")), \
|
|
"--delete-missing-args removed an unrelated extra (only --delete may)"
|
|
assert os.path.isfile(os.path.join(dest, "a.txt"))
|
|
assert os.path.isfile(os.path.join(dest, "sub", "b.txt"))
|
|
|
|
# Now with --delete the unrelated extra is an ordinary extra and must go.
|
|
lst2 = _write_rel_list(b"a.txt\ngone.txt\nsub/b.txt\n")
|
|
flags2 = ["--files-from", lst2, "-R", "--delete-missing-args", "--delete"] + \
|
|
(["-m"] if mt else [])
|
|
result, _ = run_client(source, dest, flags=flags2, port=server.port)
|
|
assert result.returncode == 0, f"delete-missing + delete sync failed: {result.stderr[:300]}"
|
|
assert not os.path.exists(os.path.join(dest, "unrelated.txt")), \
|
|
"--delete did not remove the unrelated extra"
|
|
assert not os.path.exists(os.path.join(dest, "gone.txt"))
|
|
assert os.path.isfile(os.path.join(dest, "a.txt"))
|
|
|
|
@pytest.mark.parametrize("mt", [False, True])
|
|
def test_delete_missing_mirror_outside_relative_layout(self, mt):
|
|
"""Without -R the missing entry's mirror mirrors the full source path
|
|
below the destination root, exactly like a present sibling's."""
|
|
source = self._make_source("mg_del_nor_src")
|
|
dest = os.path.join(TEST_DATA_DIR, "mg_del_nor_dst")
|
|
clean_dir(dest)
|
|
with ServerManager() as server:
|
|
server.start(extra_args=["--allow-delete"])
|
|
# Full-tree seed places every current source file in the mirrored layout.
|
|
result, _ = run_client(source, dest, flags=["--delete"], port=server.port)
|
|
assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
assert os.path.isfile(os.path.join(received, "a.txt"))
|
|
|
|
# Plant a stale mirror for an entry not (yet) on the source.
|
|
with open(os.path.join(received, "gone.txt"), "w") as fh:
|
|
fh.write("stale")
|
|
lst = _write_rel_list(b"a.txt\ngone.txt\n")
|
|
result, _ = run_client(source, dest,
|
|
flags=["--files-from", lst, "--delete-missing-args"],
|
|
port=server.port)
|
|
assert result.returncode == 0, f"delete-missing no-R sync failed: {result.stderr[:300]}"
|
|
assert not os.path.exists(os.path.join(received, "gone.txt")), \
|
|
"the full-source-mirror path of the missing entry was not deleted"
|
|
assert os.path.isfile(os.path.join(received, "a.txt"))
|
|
|
|
@pytest.mark.parametrize("mt", [False, True])
|
|
def test_delete_missing_args_not_blocked_by_exclude_protection(self, mt):
|
|
"""A missing-arg mirror that sits under a filter-excluded directory is an
|
|
explicit user request, so --delete-missing-args removes it even though an
|
|
ordinary --delete honours the exclusion protection (rsync parity). Uses
|
|
the non-relative layout: exclusion protection is only recorded there."""
|
|
source = self._make_source("mg_excl_src")
|
|
dest = os.path.join(TEST_DATA_DIR, "mg_excl_dst")
|
|
clean_dir(dest)
|
|
with ServerManager() as server:
|
|
server.start(extra_args=["--allow-delete"])
|
|
# Full-tree seed mirrors the whole source below the destination root.
|
|
result, _ = run_client(source, dest, port=server.port)
|
|
assert result.returncode == 0, f"seed failed: {result.stderr[:200]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
assert os.path.isfile(os.path.join(received, "prot", "kept.txt"))
|
|
|
|
# A stale mirror under the (now excluded) prot/ directory, plus an extra.
|
|
with open(os.path.join(received, "prot", "gone.txt"), "w") as fh:
|
|
fh.write("stale")
|
|
with open(os.path.join(received, "extra.txt"), "w") as fh:
|
|
fh.write("extra")
|
|
|
|
lst = _write_rel_list(b"a.txt\nprot/gone.txt\n")
|
|
flags = ["--files-from", lst, "--filter=- prot/", "--delete-missing-args",
|
|
"--delete"] + (["-m"] if mt else [])
|
|
result, _ = run_client(source, dest, flags=flags, port=server.port)
|
|
assert result.returncode == 0, f"delete-missing exclude sync failed: {result.stderr[:300]}"
|
|
assert not os.path.exists(os.path.join(received, "prot", "gone.txt")), \
|
|
"the explicit missing-arg deletion was blocked by exclusion protection"
|
|
assert os.path.isfile(os.path.join(received, "prot", "kept.txt")), \
|
|
"the excluded-but-present destination file must stay (default protection)"
|
|
assert not os.path.exists(os.path.join(received, "extra.txt")), \
|
|
"--delete did not remove the unrelated extra"
|
|
assert os.path.isfile(os.path.join(received, "a.txt"))
|
|
|
|
@pytest.mark.parametrize("mt", [False, True])
|
|
def test_delete_missing_args_with_delete_before(self, mt):
|
|
"""--delete-before (early delete timing) composes with --delete-missing-args:
|
|
the exact-path deletions commit with the early manifest, before data, and
|
|
--delete-before implies --delete (so unrelated extras go too)."""
|
|
source = self._make_source("mg_early_src")
|
|
dest = os.path.join(TEST_DATA_DIR, "mg_early_dst")
|
|
clean_dir(dest)
|
|
with open(os.path.join(dest, "gone.txt"), "w") as fh:
|
|
fh.write("stale")
|
|
with open(os.path.join(dest, "extra.txt"), "w") as fh:
|
|
fh.write("extra")
|
|
with ServerManager() as server:
|
|
server.start(extra_args=["--allow-delete"])
|
|
lst = _write_rel_list(b"a.txt\ngone.txt\ngone2.txt\n")
|
|
flags = ["--files-from", lst, "-R", "--delete-missing-args", "--delete-before"] + \
|
|
(["-m"] if mt else [])
|
|
result, _ = run_client(source, dest, flags=flags, port=server.port)
|
|
assert result.returncode == 0, f"early delete-missing sync failed: {result.stderr[:300]}"
|
|
assert not os.path.exists(os.path.join(dest, "gone.txt")), \
|
|
"early timing did not remove the missing-arg mirror"
|
|
assert os.path.isfile(os.path.join(dest, "a.txt")), "a.txt was not transferred"
|
|
assert not os.path.exists(os.path.join(dest, "extra.txt")), \
|
|
"--delete-before implies --delete: unrelated extras must go"
|
|
|
|
@pytest.mark.parametrize("mt", [False, True])
|
|
@pytest.mark.parametrize("relative", [False, True])
|
|
def test_delete_missing_deep_entry_with_absent_parent(self, mt, relative):
|
|
"""A missing entry whose destination mirror's parent directory does not
|
|
exist is a no-op (nothing to delete), never a run failure: the
|
|
exact-path deletions must not abort the --delete extras walk. Covers
|
|
the -R bare-relative layout and the full source-mirror layout."""
|
|
source = self._make_source("mg_deep_src")
|
|
dest = os.path.join(TEST_DATA_DIR, "mg_deep_dst")
|
|
clean_dir(dest)
|
|
rel_flags = ["-R"] if relative else []
|
|
with ServerManager() as server:
|
|
server.start(extra_args=["--allow-delete"])
|
|
if relative:
|
|
target_root = dest
|
|
else:
|
|
# Non-relative layout: seed a.txt so the receive-root mirror
|
|
# tree exists (its sub/ sibling deliberately does not).
|
|
seed = _write_rel_list(b"a.txt\n")
|
|
result, _ = run_client(source, dest,
|
|
flags=["--files-from", seed] + rel_flags,
|
|
port=server.port)
|
|
assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}"
|
|
target_root = get_dest_received_dir(dest, source)
|
|
assert os.path.isfile(os.path.join(target_root, "a.txt"))
|
|
with open(os.path.join(target_root, "extra.txt"), "w") as fh:
|
|
fh.write("extra")
|
|
|
|
lst = _write_rel_list(b"a.txt\nsub/gone.txt\n")
|
|
flags = ["--files-from", lst, "--delete-missing-args", "--delete"] + rel_flags + \
|
|
(["-m"] if mt else [])
|
|
result, _ = run_client(source, dest, flags=flags, port=server.port)
|
|
assert result.returncode == 0, \
|
|
f"deep missing-entry sync failed: {result.stderr[:300]}"
|
|
assert _read_file(os.path.join(target_root, "a.txt")) == b"a\n"
|
|
assert not os.path.exists(os.path.join(target_root, "extra.txt")), \
|
|
"--delete extras walk was aborted by the absent-parent missing entry"
|
|
assert not os.path.exists(os.path.join(target_root, "sub")), \
|
|
"the absent parent directory of the missing entry was created"
|
|
|
|
@pytest.mark.parametrize("mt", [False, True])
|
|
def test_dirs_missing_entry_skipped_in_scanner(self, shared_server, mt):
|
|
"""--dirs + --files-from: a listed-but-missing entry is skipped in the
|
|
--dirs generator (which would otherwise hard-fail), transferring the
|
|
rest of the list."""
|
|
source = self._make_source("mg_dirs_src")
|
|
dest = os.path.join(TEST_DATA_DIR, "mg_dirs_dst")
|
|
clean_dir(dest)
|
|
lst = _write_rel_list(b"a.txt\ngone.txt\n")
|
|
flags = ["--files-from", lst, "--dirs", "-R", "--ignore-missing-args"] + \
|
|
(["-m"] if mt else [])
|
|
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
|
|
assert result.returncode == 0, f"--dirs ignore-missing sync failed: {result.stderr[:300]}"
|
|
assert _read_file(os.path.join(dest, "a.txt")) == b"a\n", \
|
|
"the listed present file was not transferred"
|
|
assert not os.path.exists(os.path.join(dest, "gone.txt")), \
|
|
"a directory/file was created for the missing --dirs entry"
|
|
|
|
|
|
class TestNoImpliedDirs:
|
|
"""--no-implied-dirs (only meaningful with -R + --files-from) refuses to
|
|
place a listed file whose parent directory is not itself listed."""
|
|
|
|
def _make(self):
|
|
return _make_relative_source("noimplied_src")
|
|
|
|
@pytest.mark.parametrize("mt", [False, True])
|
|
def test_implied_dir_only_fails_entry(self, shared_server, mt):
|
|
source = self._make()
|
|
dest = os.path.join(TEST_DATA_DIR, "noimplied_dst")
|
|
clean_dir(dest)
|
|
lst = _write_rel_list(b"a/b.txt\n") # "a" itself is not listed
|
|
flags = ["--files-from", lst, "-R", "--no-implied-dirs"] + (["-m"] if mt else [])
|
|
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
|
|
assert result.returncode != 0, "implied parent directory was not rejected"
|
|
assert "--no-implied-dirs" in (result.stderr or result.stdout)
|
|
assert not os.path.exists(os.path.join(dest, "a", "b.txt"))
|
|
|
|
@pytest.mark.parametrize("mt", [False, True])
|
|
def test_listed_dir_allows_file(self, shared_server, mt):
|
|
source = self._make()
|
|
dest = os.path.join(TEST_DATA_DIR, "noimplied_ok_dst")
|
|
clean_dir(dest)
|
|
lst = _write_rel_list(b"a\na/b.txt\n")
|
|
flags = ["--files-from", lst, "-R", "--no-implied-dirs"] + (["-m"] if mt else [])
|
|
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
|
|
assert result.returncode == 0, f"listed dir + file sync failed: {result.stderr[:200]}"
|
|
assert _read_file(os.path.join(dest, "a", "b.txt")) == b"nested\n"
|
|
|
|
@pytest.mark.parametrize("mt", [False, True])
|
|
def test_no_implied_dirs_without_relative_changes_nothing(self, shared_server, mt):
|
|
source = self._make()
|
|
dest = os.path.join(TEST_DATA_DIR, "noimplied_noR_dst")
|
|
clean_dir(dest)
|
|
lst = _write_rel_list(b"a/b.txt\n")
|
|
flags = ["--files-from", lst, "--no-implied-dirs"] + (["-m"] if mt else [])
|
|
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
|
|
assert result.returncode == 0, "--no-implied-dirs without -R changed behavior"
|
|
received = get_dest_received_dir(dest, source)
|
|
assert _read_file(os.path.join(received, "a", "b.txt")) == b"nested\n"
|
|
|
|
|
|
class TestDirs:
|
|
"""-d/--dirs (and the --old-dirs/--old-d aliases) transfer directory entries
|
|
without recursing into their contents."""
|
|
|
|
def _make(self):
|
|
return _make_relative_source("dirs_src")
|
|
|
|
def _assert_only_empty_mirror(self, dest, source):
|
|
mirror = get_dest_received_dir(dest, source)
|
|
assert os.path.isdir(mirror), "source-root mirror directory was not created"
|
|
files = []
|
|
for root, _dirs, names in os.walk(mirror):
|
|
files.extend(os.path.relpath(os.path.join(root, n), mirror) for n in names)
|
|
assert files == [], f"--dirs descended into contents: {files}"
|
|
|
|
@pytest.mark.parametrize("flag", ["--dirs", "-d", "--old-dirs", "--old-d"])
|
|
@pytest.mark.parametrize("mt", [False, True])
|
|
def test_dirs_transfers_empty_dir_only(self, shared_server, flag, mt):
|
|
source = self._make()
|
|
dest = os.path.join(TEST_DATA_DIR, "dirs_dst")
|
|
clean_dir(dest)
|
|
flags = [flag] + (["-m"] if mt else [])
|
|
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
|
|
assert result.returncode == 0, f"{flag} sync failed: {result.stderr[:200]}"
|
|
self._assert_only_empty_mirror(dest, source)
|
|
|
|
@pytest.mark.parametrize("mt", [False, True])
|
|
def test_dirs_with_files_from(self, shared_server, mt):
|
|
source = self._make()
|
|
dest = os.path.join(TEST_DATA_DIR, "dirs_ff_dst")
|
|
clean_dir(dest)
|
|
# A listed directory is created empty; a listed file is transferred.
|
|
lst = _write_rel_list(b"dir1\nsub/x.txt\n")
|
|
flags = ["--files-from", lst, "--dirs", "-R"] + (["-m"] if mt else [])
|
|
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
|
|
assert result.returncode == 0, f"dirs files-from sync failed: {result.stderr[:200]}"
|
|
assert os.path.isdir(os.path.join(dest, "dir1")), "listed dir was not created"
|
|
assert not os.path.exists(os.path.join(dest, "dir1", "keep.txt")), \
|
|
"--dirs must not descend into a listed directory"
|
|
assert _read_file(os.path.join(dest, "sub", "x.txt")) == b"x\n", \
|
|
"listed file content was not transferred"
|
|
assert not os.path.exists(os.path.join(dest, "sub", "y.txt")), \
|
|
"unlisted file appeared"
|
|
|
|
@pytest.mark.parametrize("mt", [False, True])
|
|
def test_dirs_with_files_from_mirror_layout(self, shared_server, mt):
|
|
"""Without -R the dirs+files-from entries still mirror the source path."""
|
|
source = self._make()
|
|
dest = os.path.join(TEST_DATA_DIR, "dirs_ff_noR_dst")
|
|
clean_dir(dest)
|
|
lst = _write_rel_list(b"dir1\n")
|
|
flags = ["--files-from", lst, "--dirs"] + (["-m"] if mt else [])
|
|
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
|
|
assert result.returncode == 0, f"dirs files-from no-R sync failed: {result.stderr[:200]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
assert os.path.isdir(os.path.join(received, "dir1")), "mirrored dir entry not created"
|
|
assert not os.path.exists(os.path.join(received, "dir1", "keep.txt")), \
|
|
"--dirs must not descend into a listed directory"
|
|
assert not os.path.exists(os.path.join(received, "sub")), \
|
|
"unlisted subtree appeared"
|
|
|
|
@pytest.mark.parametrize("mt", [False, True])
|
|
def test_dirs_chunk_serialization(self, shared_server, mt):
|
|
"""--dirs entries survive the chunk-serialization wire path (type
|
|
marker round-trips); a listed dir lands empty and a listed file lands
|
|
with content, with no protocol desync under -s -m."""
|
|
source = self._make()
|
|
dest = os.path.join(TEST_DATA_DIR, "dirs_s_dst")
|
|
clean_dir(dest)
|
|
lst = _write_rel_list(b"dir1\nsub/x.txt\n")
|
|
flags = ["--files-from", lst, "--dirs", "-R", "-s"] + (["-m"] if mt else [])
|
|
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
|
|
assert result.returncode == 0, f"dirs -s sync failed: {result.stderr[:200]}"
|
|
assert os.path.isdir(os.path.join(dest, "dir1")), "listed dir was not created"
|
|
assert not os.path.exists(os.path.join(dest, "dir1", "keep.txt")), \
|
|
"--dirs must not descend into a listed directory"
|
|
assert _read_file(os.path.join(dest, "sub", "x.txt")) == b"x\n", \
|
|
"listed file content was not transferred"
|
|
|
|
def test_dirs_delete_keeps_transferred_empty_dir(self):
|
|
"""Directory entries appear in the delete manifest, so the empty dir a
|
|
--dirs run just created is not pruned as an extra by --delete."""
|
|
source = self._make()
|
|
dest = os.path.join(TEST_DATA_DIR, "dirs_del_dst")
|
|
clean_dir(dest)
|
|
extra = os.path.join(dest, "extra.txt")
|
|
with open(extra, "wb") as fh:
|
|
fh.write(b"delete me")
|
|
with ServerManager() as server:
|
|
server.start(extra_args=["--allow-delete"])
|
|
result, _ = run_client(source, dest, flags=["--dirs", "--delete"],
|
|
port=server.port)
|
|
assert result.returncode == 0, f"--dirs --delete sync failed: {result.stderr[:200]}"
|
|
assert not os.path.exists(extra), "--delete did not remove the extra file"
|
|
mirror = get_dest_received_dir(dest, source)
|
|
assert os.path.isdir(mirror), "transferred empty dir was pruned as an extra"
|
|
files = []
|
|
for root, _dirs, names in os.walk(mirror):
|
|
files.extend(os.path.relpath(os.path.join(root, n), mirror) for n in names)
|
|
assert files == [], f"--dirs descended into contents: {files}"
|
|
|
|
def test_dirs_listed_dir_colliding_with_file_fails(self, shared_server):
|
|
"""A listed directory that already exists as a regular file at the
|
|
destination fails the transfer cleanly instead of clobbering the file."""
|
|
source = self._make()
|
|
dest = os.path.join(TEST_DATA_DIR, "dirs_coll_dst")
|
|
clean_dir(dest)
|
|
blocker = os.path.join(dest, "dir1")
|
|
with open(blocker, "wb") as fh:
|
|
fh.write(b"blocking file")
|
|
lst = _write_rel_list(b"dir1\n")
|
|
result, _ = run_client(source, dest, flags=["--files-from", lst, "--dirs", "-R"],
|
|
port=shared_server.port)
|
|
assert result.returncode != 0, "dir entry over an existing file did not fail"
|
|
assert os.path.isfile(blocker), "blocking regular file was clobbered"
|
|
|
|
|
|
class TestMkpath:
|
|
"""--mkpath tells the server to create the destination root directory (and
|
|
missing leading components) when it does not exist yet; without it a missing
|
|
destination root fails the transfer."""
|
|
|
|
@pytest.mark.parametrize("mt", [False, True])
|
|
def test_missing_root_fails_without_mkpath(self, mt):
|
|
source = _make_relative_source("mkpath_fail_src")
|
|
dest = os.path.join(TEST_DATA_DIR, "mkpath_missing_dst")
|
|
shutil.rmtree(dest, ignore_errors=True)
|
|
with ServerManager() as server:
|
|
server.start()
|
|
flags = ["-m"] if mt else []
|
|
result, _ = run_client(source, dest, flags=flags, port=server.port)
|
|
assert result.returncode != 0, "missing destination root did not fail without --mkpath"
|
|
assert not os.path.exists(dest), "missing root was created without --mkpath"
|
|
|
|
@pytest.mark.parametrize("mt", [False, True])
|
|
def test_mkpath_creates_missing_root(self, mt):
|
|
source = _make_relative_source("mkpath_ok_src")
|
|
dest = os.path.join(TEST_DATA_DIR, "deep", "mkpath_dst")
|
|
shutil.rmtree(os.path.join(TEST_DATA_DIR, "deep"), ignore_errors=True)
|
|
with ServerManager() as server:
|
|
server.start()
|
|
flags = ["--mkpath"] + (["-m"] if mt else [])
|
|
result, _ = run_client(source, dest, flags=flags, port=server.port)
|
|
assert result.returncode == 0, f"--mkpath sync failed: {result.stderr[:200]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
assert _read_file(os.path.join(received, "sub", "x.txt")) == b"x\n", \
|
|
"file not transferred into the --mkpath-created root"
|
|
|
|
@pytest.mark.parametrize("mkpath", [False, True])
|
|
def test_existing_dest_with_trailing_slash(self, shared_server, mkpath):
|
|
"""A destination root written with a trailing slash must keep working:
|
|
an existing root is accepted both with and without --mkpath."""
|
|
source = _make_relative_source("mkpath_trail_src")
|
|
dest = os.path.join(TEST_DATA_DIR, "mkpath_trail_dst")
|
|
clean_dir(dest)
|
|
dest_slash = dest + "/"
|
|
flags = ["--mkpath"] if mkpath else []
|
|
result, _ = run_client(source, dest_slash, flags=flags, port=shared_server.port)
|
|
assert result.returncode == 0, \
|
|
f"trailing-slash dest sync (mkpath={mkpath}) failed: {result.stderr[:200]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
assert _read_file(os.path.join(received, "sub", "x.txt")) == b"x\n", \
|
|
"file not transferred into the trailing-slash destination root"
|
|
|
|
@pytest.mark.parametrize("mkpath", [False, True])
|
|
def test_dest_equal_authorized_root(self, mkpath):
|
|
"""A destination that is exactly the server's authorized root works
|
|
without --mkpath, and with --mkpath creates no stray <root>/<basename>
|
|
nested directory."""
|
|
root = os.path.join(TEST_DATA_DIR, "mkpath_eq_root")
|
|
clean_dir(root)
|
|
source = _make_relative_source("mkpath_eq_src")
|
|
with ServerManager() as server:
|
|
server.start(extra_args=["--destination-root", root])
|
|
flags = ["--mkpath"] if mkpath else []
|
|
result, _ = run_client(source, root, flags=flags, port=server.port)
|
|
assert result.returncode == 0, \
|
|
f"dest==authorized-root sync (mkpath={mkpath}) failed: {result.stderr[:200]}"
|
|
received = get_dest_received_dir(root, source)
|
|
assert _read_file(os.path.join(received, "sub", "x.txt")) == b"x\n", \
|
|
"file not transferred when the dest equals the authorized root"
|
|
basename = os.path.basename(root.rstrip(os.sep))
|
|
assert not os.path.exists(os.path.join(root, basename)), \
|
|
"--mkpath created a spurious nested <root>/<basename> directory"
|
|
|
|
|
|
class TestFilters:
|
|
"""--filter/-C/-F rule layer: excludes prune, ordering is first-match-wins,
|
|
the default with no matching rule is include, and legacy --exclude remains
|
|
an independent layer."""
|
|
|
|
|
|
class TestDeleteTiming:
|
|
"""rsync deletion-timing family. --delete-before/--delete-during transmit
|
|
the keep-set manifest BEFORE any file data (the receiver deletes extras and
|
|
acks first); --delete/--delete-after/--delete-delay commit deletions only
|
|
after the whole transfer succeeded. Every timing flag implies --delete."""
|
|
|
|
def _seed(self, tag):
|
|
source = os.path.join(TEST_DATA_DIR, f"deltiming_{tag}_src")
|
|
clean_dir(source)
|
|
entries = {
|
|
"top.txt": b"top level\n",
|
|
"sub/deep.txt": b"deeply nested file\n",
|
|
}
|
|
for rel, content in entries.items():
|
|
full = os.path.join(source, rel)
|
|
os.makedirs(os.path.dirname(full), exist_ok=True)
|
|
with open(full, "wb") as fh:
|
|
fh.write(content)
|
|
return source
|
|
|
|
@pytest.mark.parametrize("flag", ["--delete-before", "--delete-during", "--del",
|
|
"--delete-after", "--delete-delay"])
|
|
@pytest.mark.parametrize("mt", [False, True])
|
|
@pytest.mark.ci
|
|
def test_flag_removes_extras_on_success(self, flag, mt):
|
|
"""Every timing flag is accepted, implies --delete, and on a successful
|
|
transfer removes the destination extras exactly like plain --delete."""
|
|
source = self._seed("ok")
|
|
dest = os.path.join(TEST_DATA_DIR, "deltiming_ok_dst")
|
|
clean_dir(dest)
|
|
with ServerManager() as server:
|
|
server.start(extra_args=["--allow-delete"])
|
|
result, _ = run_client(source, dest, port=server.port)
|
|
assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
extra = os.path.join(received, "extra.txt")
|
|
with open(extra, "wb") as fh:
|
|
fh.write(b"should be deleted")
|
|
|
|
flags = [flag] + (["-m"] if mt else [])
|
|
result, _ = run_client(source, dest, flags=flags, port=server.port)
|
|
assert result.returncode == 0, \
|
|
f"{flag} sync failed: {(result.stderr or result.stdout)[:300]}"
|
|
assert not os.path.exists(extra), f"{flag} did not remove the extra file"
|
|
mismatches, missing = verify_transfer(source, received)
|
|
assert not missing, f"{flag} missing files: {missing}"
|
|
assert not mismatches, f"{flag} mismatched files: {mismatches}"
|
|
|
|
@pytest.mark.parametrize("flag", ["--delete-before", "--delete-during", "--del"])
|
|
@pytest.mark.parametrize("mt", [False, True])
|
|
def test_early_flags_delete_before_data(self, flag, mt):
|
|
"""--delete-before/--delete-during remove extras (and a file blocking a
|
|
destination directory) BEFORE data is applied, so a nested write that
|
|
would fail while the blocker still exists succeeds."""
|
|
source = self._seed("early")
|
|
dest = os.path.join(TEST_DATA_DIR, "deltiming_early_dst")
|
|
clean_dir(dest)
|
|
with ServerManager() as server:
|
|
server.start(extra_args=["--allow-delete"])
|
|
result, _ = run_client(source, dest, port=server.port)
|
|
assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
extra = os.path.join(received, "extra.txt")
|
|
with open(extra, "wb") as fh:
|
|
fh.write(b"extra file")
|
|
blocker = os.path.join(received, "sub")
|
|
shutil.rmtree(blocker)
|
|
with open(blocker, "wb") as fh:
|
|
fh.write(b"blocks the nested destination directory")
|
|
|
|
flags = [flag] + (["-m"] if mt else [])
|
|
result, _ = run_client(source, dest, flags=flags, port=server.port)
|
|
assert result.returncode == 0, \
|
|
f"{flag} (early delete) did not remove the blocker in time: " \
|
|
f"{(result.stderr or result.stdout)[:300]}"
|
|
assert not os.path.exists(extra), f"{flag} did not delete the extra before data"
|
|
assert _read_file(os.path.join(received, "sub", "deep.txt")) == b"deeply nested file\n", \
|
|
f"{flag}: nested file was not written after the early deletion"
|
|
|
|
@pytest.mark.parametrize("flag", ["--delete", "--delete-after", "--delete-delay"])
|
|
@pytest.mark.parametrize("mt", [False, True])
|
|
def test_late_flags_commit_only_after_success(self, flag, mt):
|
|
"""Plain --delete/--delete-after/--delete-delay defer deletion until the
|
|
whole transfer succeeds: a mid-transfer write failure must leave every
|
|
extra in place (commit-style safety). The -m receiver must also keep
|
|
the extras: the deferred keep-set is committed by the server only after
|
|
the disk-writer thread has finished, and a failing writer means the
|
|
manifest is freed, never applied."""
|
|
source = self._seed("late")
|
|
dest = os.path.join(TEST_DATA_DIR, "deltiming_late_dst")
|
|
clean_dir(dest)
|
|
with ServerManager() as server:
|
|
server.start(extra_args=["--allow-delete"])
|
|
result, _ = run_client(source, dest, port=server.port)
|
|
assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
extra = os.path.join(received, "extra.txt")
|
|
with open(extra, "wb") as fh:
|
|
fh.write(b"extra file")
|
|
blocker = os.path.join(received, "sub")
|
|
shutil.rmtree(blocker)
|
|
with open(blocker, "wb") as fh:
|
|
fh.write(b"blocks the nested destination directory")
|
|
|
|
flags = [flag] + (["-m"] if mt else [])
|
|
result, _ = run_client(source, dest, flags=flags, port=server.port)
|
|
assert result.returncode != 0, \
|
|
f"{flag} (mt={mt}) unexpectedly succeeded (deletion must be deferred)"
|
|
assert os.path.exists(extra), \
|
|
f"{flag} (mt={mt}) removed an extra although the transfer failed"
|
|
assert os.path.isfile(blocker), \
|
|
f"{flag} (mt={mt}) deleted the blocker although the transfer failed"
|
|
|
|
def test_early_flag_respected_when_server_refuses_delete(self, shared_server):
|
|
"""With an --allow-delete-less server the client's early timing still
|
|
completes (no deadlock on the pre-delete ack) and simply never deletes,
|
|
exactly like the plain server policy."""
|
|
source = self._seed("refused")
|
|
dest = os.path.join(TEST_DATA_DIR, "deltiming_refused_dst")
|
|
clean_dir(dest)
|
|
result, _ = run_client(source, dest, port=shared_server.port)
|
|
assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
extra = os.path.join(received, "extra.txt")
|
|
with open(extra, "wb") as fh:
|
|
fh.write(b"extra file")
|
|
result, _ = run_client(source, dest, flags=["--delete-before"], port=shared_server.port)
|
|
assert result.returncode == 0, \
|
|
f"--delete-before against a refuse-delete server failed: {result.stderr[:300]}"
|
|
assert os.path.exists(extra), "unauthorized delete removed an extra file"
|
|
|
|
|
|
def _seed_delete_tree(tag, entries, dest):
|
|
"""Create a source tree and seed a full mirror at `dest`, returning
|
|
(source, received_mirror)."""
|
|
source = os.path.join(TEST_DATA_DIR, f"delpol_{tag}_src")
|
|
clean_dir(source)
|
|
for rel, content in entries.items():
|
|
full = os.path.join(source, rel)
|
|
os.makedirs(os.path.dirname(full), exist_ok=True)
|
|
with open(full, "wb") as fh:
|
|
fh.write(content)
|
|
clean_dir(dest)
|
|
with ServerManager() as server:
|
|
server.start(extra_args=["--allow-delete"])
|
|
result, _ = run_client(source, dest, port=server.port)
|
|
assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
return source, received
|
|
|
|
|
|
class TestDeletePolicy:
|
|
"""Deletion-policy family: --delete-excluded, --max-delete, --force,
|
|
--ignore-errors and --prune-empty-dirs."""
|
|
|
|
def _write(self, path, content):
|
|
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
with open(path, "wb") as fh:
|
|
fh.write(content)
|
|
|
|
@pytest.mark.parametrize("mt", [False, True])
|
|
@pytest.mark.parametrize("timing",
|
|
["--delete", "--delete-before", "--delete-after", "--delete-delay"])
|
|
def test_delete_protects_excluded_by_default_and_delete_excluded_removes(self, mt, timing):
|
|
"""rsync parity: with a --delete timing the destination mirror path whose
|
|
source was excluded survives (protected by default); --delete-excluded
|
|
opts back into deleting it. Verified single-threaded and -m across every
|
|
timing (commit and early)."""
|
|
source = os.path.join(TEST_DATA_DIR, f"delexcl_{timing.strip('-')}_{mt}_src")
|
|
clean_dir(source)
|
|
entries = {
|
|
"keep.txt": b"kept\n",
|
|
"secret.log": b"secret\n",
|
|
"sub/nested.log": b"nested secret\n",
|
|
}
|
|
for rel, content in entries.items():
|
|
self._write(os.path.join(source, rel), content)
|
|
dest = os.path.join(TEST_DATA_DIR, f"delexcl_{timing.strip('-')}_{mt}_dst")
|
|
clean_dir(dest)
|
|
with ServerManager() as server:
|
|
server.start(extra_args=["--allow-delete"])
|
|
result, _ = run_client(source, dest, port=server.port)
|
|
assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
self._write(os.path.join(received, "extra.txt"), b"extra\n")
|
|
|
|
# Default: the excluded mirrors survive --delete, genuine extras die.
|
|
flags = ["--exclude", "*.log", timing] + (["-m"] if mt else [])
|
|
result, _ = run_client(source, dest, flags=flags, port=server.port)
|
|
assert result.returncode == 0, \
|
|
f"default delete sync failed: {(result.stderr or result.stdout)[:300]}"
|
|
assert os.path.exists(os.path.join(received, "secret.log")), \
|
|
"excluded dest file was deleted under plain --delete (rsync protects it)"
|
|
assert os.path.exists(os.path.join(received, "sub", "nested.log")), \
|
|
"nested excluded dest file was deleted under plain --delete"
|
|
assert not os.path.exists(os.path.join(received, "extra.txt")), \
|
|
"genuine extra was not deleted"
|
|
|
|
# --delete-excluded: excluded mirrors are extras again and die.
|
|
self._write(os.path.join(received, "extra.txt"), b"extra\n")
|
|
flags = ["--exclude", "*.log", timing, "--delete-excluded"] + (["-m"] if mt else [])
|
|
result, _ = run_client(source, dest, flags=flags, port=server.port)
|
|
assert result.returncode == 0, \
|
|
f"--delete-excluded sync failed: {(result.stderr or result.stdout)[:300]}"
|
|
assert not os.path.exists(os.path.join(received, "secret.log")), \
|
|
"--delete-excluded did not remove the excluded dest file"
|
|
assert not os.path.exists(os.path.join(received, "sub", "nested.log")), \
|
|
"--delete-excluded did not remove the nested excluded dest file"
|
|
assert not os.path.exists(os.path.join(received, "extra.txt")), \
|
|
"genuine extra survived --delete-excluded"
|
|
assert _read_file(os.path.join(received, "keep.txt")) == b"kept\n"
|
|
|
|
@pytest.mark.parametrize("mt", [False, True])
|
|
def test_delete_excluded_excluded_directory_subtree(self, mt):
|
|
"""A whole source directory excluded by a filter rule protects its whole
|
|
destination mirror by default; --delete-excluded removes the subtree."""
|
|
source = os.path.join(TEST_DATA_DIR, f"delexcldir_{mt}_src")
|
|
clean_dir(source)
|
|
self._write(os.path.join(source, "keep.txt"), b"kept\n")
|
|
self._write(os.path.join(source, "skipdir", "a.log"), b"a\n")
|
|
self._write(os.path.join(source, "skipdir", "deep", "b.log"), b"b\n")
|
|
dest = os.path.join(TEST_DATA_DIR, f"delexcldir_{mt}_dst")
|
|
clean_dir(dest)
|
|
with ServerManager() as server:
|
|
server.start(extra_args=["--allow-delete"])
|
|
result, _ = run_client(source, dest, port=server.port)
|
|
assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
|
|
flags = ["--filter=- skipdir/", "--delete"] + (["-m"] if mt else [])
|
|
result, _ = run_client(source, dest, flags=flags, port=server.port)
|
|
assert result.returncode == 0, \
|
|
f"default delete sync failed: {(result.stderr or result.stdout)[:300]}"
|
|
assert os.path.exists(os.path.join(received, "skipdir", "a.log")), \
|
|
"excluded dir subtree was deleted under plain --delete"
|
|
assert os.path.exists(os.path.join(received, "skipdir", "deep", "b.log")), \
|
|
"nested excluded dir content was deleted under plain --delete"
|
|
|
|
flags = ["--filter=- skipdir/", "--delete", "--delete-excluded"] + (["-m"] if mt else [])
|
|
result, _ = run_client(source, dest, flags=flags, port=server.port)
|
|
assert result.returncode == 0, \
|
|
f"--delete-excluded sync failed: {(result.stderr or result.stdout)[:300]}"
|
|
assert not os.path.exists(os.path.join(received, "skipdir")), \
|
|
"--delete-excluded did not remove the excluded dir subtree"
|
|
|
|
@pytest.mark.parametrize("mt", [False, True])
|
|
@pytest.mark.parametrize("timing", ["--delete", "--delete-before"])
|
|
def test_max_delete_exceeded_fails_without_deleting(self, mt, timing):
|
|
"""A run that would exceed --max-delete deletes nothing and fails."""
|
|
source = os.path.join(TEST_DATA_DIR, f"maxdel_{timing.strip('-')}_{mt}_src")
|
|
clean_dir(source)
|
|
self._write(os.path.join(source, "keep.txt"), b"kept\n")
|
|
dest = os.path.join(TEST_DATA_DIR, f"maxdel_{timing.strip('-')}_{mt}_dst")
|
|
clean_dir(dest)
|
|
with ServerManager() as server:
|
|
server.start(extra_args=["--allow-delete"])
|
|
result, _ = run_client(source, dest, port=server.port)
|
|
assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
extras = []
|
|
for i in range(4):
|
|
name = f"e{i}.txt"
|
|
self._write(os.path.join(received, name), b"extra\n")
|
|
extras.append(os.path.join(received, name))
|
|
|
|
flags = ["--max-delete=2", timing] + (["-m"] if mt else [])
|
|
result, _ = run_client(source, dest, flags=flags, port=server.port)
|
|
assert result.returncode != 0, \
|
|
f"--max-delete=2 with 4 extras unexpectedly succeeded: {result.stderr[:300]}"
|
|
for path in extras:
|
|
assert os.path.exists(path), \
|
|
"--max-delete overrun deleted files (must be all-or-nothing)"
|
|
|
|
@pytest.mark.parametrize("mt", [False, True])
|
|
def test_max_delete_not_exceeded_deletes_exactly(self, mt):
|
|
"""When the extras are at or below --max-delete the run succeeds and
|
|
removes exactly the extras."""
|
|
source = os.path.join(TEST_DATA_DIR, f"maxdelok_{mt}_src")
|
|
clean_dir(source)
|
|
self._write(os.path.join(source, "keep.txt"), b"kept\n")
|
|
dest = os.path.join(TEST_DATA_DIR, f"maxdelok_{mt}_dst")
|
|
clean_dir(dest)
|
|
with ServerManager() as server:
|
|
server.start(extra_args=["--allow-delete"])
|
|
result, _ = run_client(source, dest, port=server.port)
|
|
assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
for i in range(3):
|
|
self._write(os.path.join(received, f"e{i}.txt"), b"extra\n")
|
|
flags = ["--max-delete=3", "--delete"] + (["-m"] if mt else [])
|
|
result, _ = run_client(source, dest, flags=flags, port=server.port)
|
|
assert result.returncode == 0, \
|
|
f"--max-delete=3 with 3 extras failed: {(result.stderr or result.stdout)[:300]}"
|
|
for i in range(3):
|
|
assert not os.path.exists(os.path.join(received, f"e{i}.txt")), \
|
|
f"extra e{i}.txt not deleted under --max-delete=3"
|
|
|
|
@pytest.mark.parametrize("mt", [False, True])
|
|
def test_force_replaces_nonempty_dir_with_file(self, mt):
|
|
"""--force lets an incoming regular file replace a non-empty destination
|
|
directory; without it the write (and the run) fails."""
|
|
source = os.path.join(TEST_DATA_DIR, f"force_{mt}_src")
|
|
clean_dir(source)
|
|
self._write(os.path.join(source, "sub", "old.txt"), b"old\n")
|
|
self._write(os.path.join(source, "keep.txt"), b"kept\n")
|
|
dest = os.path.join(TEST_DATA_DIR, f"force_{mt}_dst")
|
|
clean_dir(dest)
|
|
with ServerManager() as server:
|
|
server.start(extra_args=["--allow-delete"])
|
|
result, _ = run_client(source, dest, port=server.port)
|
|
assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
|
|
# The source path `sub` becomes a regular file (the dir is gone).
|
|
os.unlink(os.path.join(source, "sub", "old.txt"))
|
|
os.rmdir(os.path.join(source, "sub"))
|
|
self._write(os.path.join(source, "sub"), b"now a file\n")
|
|
|
|
result, _ = run_client(source, dest, port=server.port)
|
|
assert result.returncode != 0, \
|
|
"a file over a non-empty directory must fail without --force"
|
|
assert os.path.isdir(os.path.join(received, "sub")), \
|
|
"directory was destroyed although the run failed without --force"
|
|
assert os.path.exists(os.path.join(received, "sub", "old.txt")), \
|
|
"non-empty dir content was lost although the run failed without --force"
|
|
|
|
flags = ["--force"] + (["-m"] if mt else [])
|
|
result, _ = run_client(source, dest, flags=flags, port=server.port)
|
|
assert result.returncode == 0, \
|
|
f"--force run failed: {(result.stderr or result.stdout)[:300]}"
|
|
assert os.path.isfile(os.path.join(received, "sub")), \
|
|
"--force did not replace the directory with the file"
|
|
assert _read_file(os.path.join(received, "sub")) == b"now a file\n"
|
|
|
|
def test_force_inert_under_delay_updates(self):
|
|
"""Documented divergence: --force acts on the immediate-install path; a
|
|
--delay-updates run stages into its own tree and its publication renames
|
|
over regular files only, so a blocking directory is not cleared and the
|
|
run fails."""
|
|
source = os.path.join(TEST_DATA_DIR, "force_delay_src")
|
|
clean_dir(source)
|
|
self._write(os.path.join(source, "sub", "old.txt"), b"old\n")
|
|
self._write(os.path.join(source, "keep.txt"), b"kept\n")
|
|
dest = os.path.join(TEST_DATA_DIR, "force_delay_dst")
|
|
clean_dir(dest)
|
|
with ServerManager() as server:
|
|
server.start(extra_args=["--allow-delete"])
|
|
result, _ = run_client(source, dest, port=server.port)
|
|
assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
os.unlink(os.path.join(source, "sub", "old.txt"))
|
|
os.rmdir(os.path.join(source, "sub"))
|
|
self._write(os.path.join(source, "sub"), b"now a file\n")
|
|
result, _ = run_client(source, dest, flags=["--force", "--delay-updates"],
|
|
port=server.port)
|
|
assert result.returncode != 0, \
|
|
"--force --delay-updates unexpectedly replaced the blocking directory"
|
|
assert os.path.isdir(os.path.join(received, "sub")), \
|
|
"blocking directory was cleared although --delay-updates should keep --force inert"
|
|
assert os.path.exists(os.path.join(received, "sub", "old.txt")), \
|
|
"blocking directory content was lost"
|
|
|
|
@pytest.mark.parametrize("mt", [False, True])
|
|
def test_prune_empty_dirs_dirs_mode(self, mt):
|
|
"""--prune-empty-dirs omits an empty source directory's explicit entry in
|
|
--dirs mode (nothing is created, and an existing empty mirror is removed
|
|
by --delete). Recursive transfers never emit empty dirs, so the flag is
|
|
a no-op there (documented rsync -m parity)."""
|
|
source = os.path.join(TEST_DATA_DIR, f"prune_{mt}_src")
|
|
clean_dir(source)
|
|
os.makedirs(source, exist_ok=True) # physically empty source dir
|
|
|
|
dest = os.path.join(TEST_DATA_DIR, f"prune_{mt}_dst")
|
|
clean_dir(dest)
|
|
with ServerManager() as server:
|
|
server.start(extra_args=["--allow-delete"])
|
|
result, _ = run_client(source, dest, flags=["--dirs"], port=server.port)
|
|
assert result.returncode == 0, f"-d seed failed: {result.stderr[:200]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
assert os.path.isdir(received), "-d should create the empty mirror dir"
|
|
assert os.listdir(received) == []
|
|
|
|
# prune-empty-dirs: the empty mirror is pruned by --delete.
|
|
flags = ["--dirs", "--prune-empty-dirs", "--delete"] + (["-m"] if mt else [])
|
|
result, _ = run_client(source, dest, flags=flags, port=server.port)
|
|
assert result.returncode == 0, \
|
|
f"--dirs --prune-empty-dirs --delete failed: {(result.stderr or result.stdout)[:300]}"
|
|
assert not os.path.exists(received), \
|
|
"--prune-empty-dirs did not prune the empty dir (--delete left it)"
|
|
|
|
# A fresh destination: prune-empty-dirs means the empty dir is never sent.
|
|
dest2 = os.path.join(TEST_DATA_DIR, f"prune2_{mt}_dst")
|
|
clean_dir(dest2)
|
|
with ServerManager() as server:
|
|
server.start(extra_args=["--allow-delete"])
|
|
flags = ["--dirs", "--prune-empty-dirs", "-i"] + (["-m"] if mt else [])
|
|
result, _ = run_client(source, dest2, flags=flags, port=server.port)
|
|
assert result.returncode == 0, \
|
|
f"--dirs --prune-empty-dirs failed: {(result.stderr or result.stdout)[:300]}"
|
|
received2 = get_dest_received_dir(dest2, source)
|
|
assert not os.path.exists(received2), \
|
|
"--prune-empty-dirs transferred the empty directory"
|
|
assert result.stdout == "", \
|
|
f"--prune-empty-dirs leaked an itemize line: {result.stdout[:200]}"
|
|
|
|
@pytest.mark.parametrize("mt", [False, True])
|
|
def test_prune_empty_dirs_recursion_inherent(self, mt):
|
|
"""In recursive mode FastSync never transfers empty directories (rsync
|
|
-m parity): a truly-empty destination directory chain is removed by
|
|
--delete whether or not --prune-empty-dirs is given (the flag has no
|
|
additional effect there), while directories holding kept files survive.
|
|
A filter-excluded file's mirror is protected, so a directory that still
|
|
holds one is left intact (rsync default delete-excluded semantics)."""
|
|
source = os.path.join(TEST_DATA_DIR, f"prunerec_{mt}_src")
|
|
clean_dir(source)
|
|
self._write(os.path.join(source, "keep.txt"), b"kept\n")
|
|
self._write(os.path.join(source, "a", "keep.log"), b"a log\n")
|
|
self._write(os.path.join(source, "b", "deep", "kept.txt"), b"deep kept\n")
|
|
dest = os.path.join(TEST_DATA_DIR, f"prunerec_{mt}_dst")
|
|
clean_dir(dest)
|
|
with ServerManager() as server:
|
|
server.start(extra_args=["--allow-delete"])
|
|
result, _ = run_client(source, dest, port=server.port)
|
|
assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
# A stray empty chain (FastSync recursion never creates such dirs, so
|
|
# this models one left by an external tool / an earlier --dirs run).
|
|
os.makedirs(os.path.join(received, "empty", "chain"))
|
|
|
|
for prune in ([], ["--prune-empty-dirs"]):
|
|
flags = prune + ["--delete"] + (["-m"] if mt else [])
|
|
result, _ = run_client(source, dest, flags=flags, port=server.port)
|
|
assert result.returncode == 0, \
|
|
f"prune recursive sync failed: {(result.stderr or result.stdout)[:300]}"
|
|
assert not os.path.exists(os.path.join(received, "empty")), \
|
|
"truly-empty dir chain was not removed by --delete"
|
|
assert os.path.exists(os.path.join(received, "b", "deep", "kept.txt")), \
|
|
"non-empty dir subtree was wrongly removed"
|
|
assert _read_file(os.path.join(received, "keep.txt")) == b"kept\n"
|
|
|
|
# An excluded file's mirror is protected: the dir that holds it stays.
|
|
flags = ["--exclude", "*.log", "--delete", "--prune-empty-dirs"] + (["-m"] if mt else [])
|
|
result, _ = run_client(source, dest, flags=flags, port=server.port)
|
|
assert result.returncode == 0, \
|
|
f"prune recursive sync failed: {(result.stderr or result.stdout)[:300]}"
|
|
assert os.path.exists(os.path.join(received, "a", "keep.log")), \
|
|
"excluded file mirror was deleted under --delete (rsync protects it)"
|
|
|
|
def _run_client_as_nobody(self, source, dest, port, flags):
|
|
cmd = CLIENT_CMD + ["--source-dir", source, "--dest-dir", dest,
|
|
"--save-to-disk", "--server-port", str(port)] + flags
|
|
return subprocess.run(["setpriv", "--reuid=65534", "--regid=65534",
|
|
"--clear-groups"] + cmd, text=True, capture_output=True)
|
|
|
|
@pytest.mark.parametrize("mt", [False, True])
|
|
@pytest.mark.setpriv
|
|
def test_ignore_errors_keeps_deletion_active_on_scan_error(self, mt):
|
|
"""A source I/O error (unreadable subdirectory) aborts the run so no
|
|
deletion happens by default; --ignore-errors continues, still transfers
|
|
the readable tree and still deletes, single-threaded and under -m. Run
|
|
as an unprivileged user so the mode-000 directory is genuinely
|
|
unreadable."""
|
|
if os.geteuid() != 0 or shutil.which("setpriv") is None:
|
|
pytest.skip("requires root + setpriv to drop privileges for the client")
|
|
tag = f"ioerr_{os.getpid()}_{mt}"
|
|
source = os.path.join(TEST_DATA_DIR, f"{tag}_src")
|
|
clean_dir(source)
|
|
self._write(os.path.join(source, "top.txt"), b"top\n")
|
|
self._write(os.path.join(source, "ok", "inside.txt"), b"inside\n")
|
|
self._write(os.path.join(source, "locked", "blocked.txt"), b"blocked\n")
|
|
dest = os.path.join(TEST_DATA_DIR, f"{tag}_dst")
|
|
clean_dir(dest)
|
|
with ServerManager() as server:
|
|
server.start(extra_args=["--allow-delete"])
|
|
# Seed as root (server is root too).
|
|
result, _ = run_client(source, dest, port=server.port)
|
|
assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
try:
|
|
os.chmod(os.path.join(source, "locked"), 0)
|
|
|
|
# Default: scan error aborts the run; nothing is deleted.
|
|
self._write(os.path.join(received, "extra.txt"), b"extra\n")
|
|
flags = ["--delete"] + (["-m"] if mt else [])
|
|
result = self._run_client_as_nobody(source, dest, server.port, flags)
|
|
assert result.returncode != 0, "unreadable source dir did not fail the run"
|
|
assert os.path.exists(os.path.join(received, "extra.txt")), \
|
|
"default run deleted although the scan hit an I/O error"
|
|
|
|
# --ignore-errors: the readable tree transfers, deletion still runs.
|
|
self._write(os.path.join(received, "extra.txt"), b"extra\n")
|
|
flags = ["--delete", "--ignore-errors"] + (["-m"] if mt else [])
|
|
result = self._run_client_as_nobody(source, dest, server.port, flags)
|
|
assert not os.path.exists(os.path.join(received, "extra.txt")), \
|
|
f"--ignore-errors did not keep deletion active: {result.stderr[:300]}"
|
|
assert not os.path.exists(os.path.join(received, "locked")), \
|
|
"mirror of the unreadable dir was left behind (should be an extra)"
|
|
finally:
|
|
os.chmod(os.path.join(source, "locked"), 0o755)
|
|
|
|
@pytest.mark.parametrize("mt", [False, True])
|
|
@pytest.mark.parametrize("timing", ["--delete", "--delete-before"])
|
|
@pytest.mark.setpriv
|
|
def test_ignore_errors_unreadable_root_never_deletes(self, mt, timing):
|
|
"""An unreadable SOURCE ROOT must never be treated as a skippable scan
|
|
error: with --ignore-errors the sequential scanner treats the root as
|
|
fatal (matching the -m path, which cannot even create its scanner), so
|
|
no empty keep-set manifest is sent and the destination is never wiped.
|
|
Run as an unprivileged user so the mode-000 root is genuinely
|
|
unreadable."""
|
|
if os.geteuid() != 0 or shutil.which("setpriv") is None:
|
|
pytest.skip("requires root + setpriv to drop privileges for the client")
|
|
tag = f"rootio_{os.getpid()}_{mt}_{timing.strip('-')}"
|
|
source = os.path.join(TEST_DATA_DIR, f"{tag}_src")
|
|
clean_dir(source)
|
|
self._write(os.path.join(source, "file.txt"), b"content\n")
|
|
dest = os.path.join(TEST_DATA_DIR, f"{tag}_dst")
|
|
clean_dir(dest)
|
|
with ServerManager() as server:
|
|
server.start(extra_args=["--allow-delete"])
|
|
result, _ = run_client(source, dest, port=server.port)
|
|
assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
try:
|
|
os.chmod(source, 0)
|
|
self._write(os.path.join(received, "extra.txt"), b"extra\n")
|
|
flags = [timing, "--ignore-errors"] + (["-m"] if mt else [])
|
|
result = self._run_client_as_nobody(source, dest, server.port, flags)
|
|
assert result.returncode != 0, \
|
|
f"unreadable source root with {timing} (mt={mt}) unexpectedly succeeded"
|
|
assert os.path.exists(os.path.join(received, "file.txt")), \
|
|
f"{timing} (mt={mt}) wiped a kept destination file"
|
|
assert os.path.exists(os.path.join(received, "extra.txt")), \
|
|
f"{timing} (mt={mt}) deleted the extra although the scan could not read the root"
|
|
finally:
|
|
os.chmod(source, 0o755)
|
|
|
|
def test_delete_excluded_protection_is_sender_derived(self):
|
|
"""Plain --delete protects destination mirrors of files the SOURCE scan
|
|
excluded, but a destination-only file that merely matches an exclude
|
|
rule is still an extra and is removed (protection never re-applies rules
|
|
to the destination)."""
|
|
source = os.path.join(TEST_DATA_DIR, "senderderived_src")
|
|
clean_dir(source)
|
|
self._write(os.path.join(source, "keep.txt"), b"kept\n")
|
|
self._write(os.path.join(source, "secret.log"), b"secret\n")
|
|
dest = os.path.join(TEST_DATA_DIR, "senderderived_dst")
|
|
clean_dir(dest)
|
|
with ServerManager() as server:
|
|
server.start(extra_args=["--allow-delete"])
|
|
result, _ = run_client(source, dest, port=server.port)
|
|
assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
# A destination-only file that happens to match the exclude rule.
|
|
self._write(os.path.join(received, "stray.log"), b"never on the source\n")
|
|
result, _ = run_client(source, dest, flags=["--exclude", "*.log", "--delete"],
|
|
port=server.port)
|
|
assert result.returncode == 0, \
|
|
f"delete sync failed: {(result.stderr or result.stdout)[:300]}"
|
|
assert os.path.exists(os.path.join(received, "secret.log")), \
|
|
"source-excluded mirror was deleted under plain --delete"
|
|
assert not os.path.exists(os.path.join(received, "stray.log")), \
|
|
"destination-only file matching the exclude rule was left (should be deleted)"
|
|
|
|
|
|
def _pin_mtime(path, ts):
|
|
os.utime(path, (ts, ts))
|
|
|
|
|
|
class TestBasisDestDirs:
|
|
"""--compare-dest / --copy-dest / --link-dest alternate basis directories.
|
|
|
|
FastSync's basis directories are relative to the destination root and are
|
|
confined below it. The "unchanged" decision is receiver-side and requires
|
|
the per-file --incremental handshake (implied by these flags), so the basis
|
|
snapshot must reproduce the exact destination-relative mirror path of the
|
|
incoming files.
|
|
"""
|
|
|
|
STAGING = ".fastsync-stage"
|
|
TS = 1577836800 # 2020-01-01 00:00:00 UTC, used to pin matching mtimes
|
|
|
|
# fixture files: source and basis share the mtime pin, so a basis "match"
|
|
# is decided purely by content (xxHash). unchanged.txt is byte-identical;
|
|
# changed.txt is byte-DIFFERENT but has the SAME SIZE as the source (and
|
|
# the same pinned mtime), which is what forces the content-hash gate;
|
|
# added.txt does not exist in the basis at all.
|
|
UNCHANGED = "unchanged.txt"
|
|
CHANGED = "changed.txt"
|
|
ADDED = "added.txt"
|
|
|
|
def _make_source(self, name, source_files):
|
|
src = os.path.join(TEST_DATA_DIR, name)
|
|
clean_dir(src)
|
|
for rel, content in source_files.items():
|
|
full = os.path.join(src, rel)
|
|
os.makedirs(os.path.dirname(full), exist_ok=True)
|
|
with open(full, "wb") as fh:
|
|
fh.write(content)
|
|
_pin_mtime(full, self.TS)
|
|
return src
|
|
|
|
def _seed_basis_file(self, dest, source, basis_dir, rel, content, ts=None):
|
|
base = os.path.join(dest, basis_dir, os.path.relpath(
|
|
get_dest_received_dir(dest, source), dest))
|
|
full = os.path.join(base, rel)
|
|
os.makedirs(os.path.dirname(full), exist_ok=True)
|
|
with open(full, "wb") as fh:
|
|
fh.write(content)
|
|
_pin_mtime(full, self.TS if ts is None else ts)
|
|
return full
|
|
|
|
def _seed_basis(self, dest, source, basis_dir, basis_files):
|
|
for rel, content in basis_files.items():
|
|
self._seed_basis_file(dest, source, basis_dir, rel, content)
|
|
return os.path.join(dest, basis_dir, os.path.relpath(
|
|
get_dest_received_dir(dest, source), dest))
|
|
|
|
def _source_tree(self, prefix):
|
|
return {
|
|
self.UNCHANGED: b"stable content v1\n",
|
|
self.CHANGED: b"changed content now\n",
|
|
self.ADDED: b"brand new content\n",
|
|
}
|
|
|
|
def _basis_tree(self, prefix):
|
|
# unchanged.txt is identical to the source; changed.txt has the SAME
|
|
# byte size and pinned mtime but a different body (equal size forces
|
|
# the xxHash gate); added.txt is missing from the basis.
|
|
return {
|
|
self.UNCHANGED: b"stable content v1\n",
|
|
self.CHANGED: b"CHANGED CONTENT NOW\n",
|
|
}
|
|
|
|
def test_same_size_different_content_is_not_a_basis_match(self, shared_server):
|
|
# Core safety property: equal size + pinned mtime but different content
|
|
# must NEVER be hard-linked or copied from the basis -- the xxHash gate
|
|
# rejects it and the sender's data is transferred instead.
|
|
for flag, basis_dir in (("--link-dest", "szlb"), ("--copy-dest", "szcp"),
|
|
("--compare-dest", "szcmp")):
|
|
source = self._make_source("basis_same_size_src",
|
|
{self.UNCHANGED: b"same length body\n"})
|
|
dest = os.path.join(TEST_DATA_DIR, f"basis_same_size_dst_{basis_dir}")
|
|
clean_dir(dest)
|
|
basis_file = self._seed_basis_file(dest, source, basis_dir, self.UNCHANGED,
|
|
b"SAME LENGTH BODY!")
|
|
result, _ = run_client(source, dest, flags=[f"{flag}={basis_dir}"],
|
|
port=shared_server.port)
|
|
assert result.returncode == 0, \
|
|
f"{flag} same-size mismatch failed: {result.stderr[:300]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
dest_file = os.path.join(received, self.UNCHANGED)
|
|
assert _read_file(dest_file) == b"same length body\n", \
|
|
f"{flag}: basis content leaked into the destination on a hash mismatch"
|
|
if flag != "--compare-dest":
|
|
assert os.stat(dest_file).st_ino != os.stat(basis_file).st_ino, \
|
|
f"{flag}: linked/copied from a content-mismatched basis file"
|
|
|
|
@pytest.mark.ci
|
|
def test_compare_dest_skips_matching_and_transfers_missing(self, shared_server):
|
|
source = self._make_source("basis_compare_src", self._source_tree("c"))
|
|
dest = os.path.join(TEST_DATA_DIR, "basis_compare_dst")
|
|
clean_dir(dest)
|
|
self._seed_basis(dest, source, "cbasis", self._basis_tree("c"))
|
|
result, _ = run_client(source, dest,
|
|
flags=["--compare-dest=cbasis"],
|
|
port=shared_server.port)
|
|
assert result.returncode == 0, f"compare-dest failed: {result.stderr[:300]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
# compare-dest never copies: an exact basis match is skipped, leaving a
|
|
# sparse destination (rsync parity).
|
|
assert not os.path.exists(os.path.join(received, self.UNCHANGED)), \
|
|
"compare-dest materialized the unchanged file"
|
|
# Files the destination lacks AND the basis cannot satisfy are still
|
|
# transferred normally.
|
|
assert _read_file(os.path.join(received, self.CHANGED)) == \
|
|
self._source_tree("c")[self.CHANGED], "changed file not transferred"
|
|
assert _read_file(os.path.join(received, self.ADDED)) == \
|
|
self._source_tree("c")[self.ADDED], "added file not transferred"
|
|
|
|
def test_compare_dest_content_mismatch_forces_transfer(self, shared_server):
|
|
# The basis holds a file with a DIFFERENT body: even though it shares
|
|
# the mtime pin, the xxHash check fails and the data must be sent.
|
|
source = self._make_source("basis_compare_mismatch_src", {self.UNCHANGED: b"real data\n"})
|
|
dest = os.path.join(TEST_DATA_DIR, "basis_compare_mismatch_dst")
|
|
clean_dir(dest)
|
|
basis = self._seed_basis(dest, source, "cbasis", {self.UNCHANGED: b"stale data!!\n"})
|
|
result, _ = run_client(source, dest, flags=["--compare-dest=cbasis"],
|
|
port=shared_server.port)
|
|
assert result.returncode == 0, f"compare-dest mismatch failed: {result.stderr[:300]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
assert _read_file(os.path.join(received, self.UNCHANGED)) == b"real data\n", \
|
|
"content mismatch did not fall back to a normal transfer"
|
|
assert os.stat(os.path.join(received, self.UNCHANGED)).st_ino != \
|
|
os.stat(os.path.join(basis, self.UNCHANGED)).st_ino
|
|
|
|
def test_copy_dest_copies_unchanged_and_transfers_changed(self, shared_server):
|
|
source = self._make_source("basis_copy_src", self._source_tree("cp"))
|
|
dest = os.path.join(TEST_DATA_DIR, "basis_copy_dst")
|
|
clean_dir(dest)
|
|
basis = self._seed_basis(dest, source, "cpbasis", self._basis_tree("cp"))
|
|
result, _ = run_client(source, dest, flags=["--copy-dest=cpbasis"],
|
|
port=shared_server.port)
|
|
assert result.returncode == 0, f"copy-dest failed: {result.stderr[:300]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
unchanged = os.path.join(received, self.UNCHANGED)
|
|
assert _read_file(unchanged) == b"stable content v1\n", "unchanged file not materialized"
|
|
# A real local copy, NOT a hard link to the basis file.
|
|
assert os.stat(unchanged).st_ino != os.stat(os.path.join(basis, self.UNCHANGED)).st_ino
|
|
# Equal-size/different-content basis file falls back to the sender data.
|
|
assert _read_file(os.path.join(received, self.CHANGED)) == \
|
|
self._source_tree("cp")[self.CHANGED]
|
|
assert _read_file(os.path.join(received, self.ADDED)) == \
|
|
self._source_tree("cp")[self.ADDED]
|
|
|
|
def test_link_dest_hardlinks_and_falls_back(self, shared_server):
|
|
source = self._make_source("basis_link_src", self._source_tree("ln"))
|
|
dest = os.path.join(TEST_DATA_DIR, "basis_link_dst")
|
|
clean_dir(dest)
|
|
basis = self._seed_basis(dest, source, "lnbasis", self._basis_tree("ln"))
|
|
result, _ = run_client(source, dest, flags=["--link-dest=lnbasis"],
|
|
port=shared_server.port)
|
|
assert result.returncode == 0, f"link-dest failed: {result.stderr[:300]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
unchanged = os.path.join(received, self.UNCHANGED)
|
|
basis_file = os.path.join(basis, self.UNCHANGED)
|
|
# Real hard link: same inode as the DIR file, nlink >= 2, no data copy.
|
|
assert os.path.exists(unchanged)
|
|
assert os.stat(unchanged).st_ino == os.stat(basis_file).st_ino, \
|
|
"link-dest did not produce a hard link"
|
|
assert os.stat(unchanged).st_nlink >= 2
|
|
# Equal-size/different-content basis file must fall back to a plain
|
|
# transfer (not a link).
|
|
changed = os.path.join(received, self.CHANGED)
|
|
assert _read_file(changed) == self._source_tree("ln")[self.CHANGED]
|
|
assert os.stat(changed).st_ino != os.stat(os.path.join(basis, self.CHANGED)).st_ino
|
|
|
|
@pytest.mark.parametrize("flag", ["--compare-dest", "--copy-dest", "--link-dest"])
|
|
def test_basis_dir_missing_is_a_clean_noop(self, shared_server, flag):
|
|
# A basis directory that does not exist must simply transfer everything.
|
|
source = self._make_source("basis_missing_src", {self.UNCHANGED: b"content\n"})
|
|
dest = os.path.join(TEST_DATA_DIR, "basis_missing_dst")
|
|
clean_dir(dest)
|
|
result, _ = run_client(source, dest, flags=[f"{flag}=nope"],
|
|
port=shared_server.port)
|
|
assert result.returncode == 0, f"{flag} with missing dir failed: {result.stderr[:300]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
assert _read_file(os.path.join(received, self.UNCHANGED)) == b"content\n"
|
|
|
|
def test_link_dest_multithreaded(self, shared_server):
|
|
source = self._make_source("basis_link_mt_src", self._source_tree("mt"))
|
|
dest = os.path.join(TEST_DATA_DIR, "basis_link_mt_dst")
|
|
clean_dir(dest)
|
|
basis = self._seed_basis(dest, source, "mtbasis", self._basis_tree("mt"))
|
|
result, _ = run_client(source, dest, flags=["--link-dest=mtbasis", "-m"],
|
|
port=shared_server.port)
|
|
assert result.returncode == 0, f"-m link-dest failed: {result.stderr[:300]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
assert os.stat(os.path.join(received, self.UNCHANGED)).st_ino == \
|
|
os.stat(os.path.join(basis, self.UNCHANGED)).st_ino
|
|
assert _read_file(os.path.join(received, self.ADDED)) == \
|
|
self._source_tree("mt")[self.ADDED]
|
|
|
|
def test_link_dest_with_delay_updates_stages_and_publishes_link(self, shared_server):
|
|
source = self._make_source("basis_link_delay_src", {self.UNCHANGED: b"v1\n"})
|
|
dest = os.path.join(TEST_DATA_DIR, "basis_link_delay_dst")
|
|
clean_dir(dest)
|
|
basis = self._seed_basis(dest, source, "delaybasis", {self.UNCHANGED: b"v1\n"})
|
|
result, _ = run_client(source, dest,
|
|
flags=["--link-dest=delaybasis", "--delay-updates"],
|
|
port=shared_server.port)
|
|
assert result.returncode == 0, f"delay-updates link-dest failed: {result.stderr[:300]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
unchanged = os.path.join(received, self.UNCHANGED)
|
|
assert os.stat(unchanged).st_ino == \
|
|
os.stat(os.path.join(basis, self.UNCHANGED)).st_ino
|
|
assert not os.path.isdir(os.path.join(dest, self.STAGING)), \
|
|
"delay-updates staging tree was not cleaned up"
|
|
|
|
def test_delete_does_not_touch_basis_dir(self):
|
|
"""--delete removes genuine extras but must never treat a basis-dir
|
|
snapshot (which a --link-dest run just linked from) as destination
|
|
content."""
|
|
source = self._make_source("basis_delete_src", {self.UNCHANGED: b"v1\n"})
|
|
dest = os.path.join(TEST_DATA_DIR, "basis_delete_dst")
|
|
clean_dir(dest)
|
|
basis = self._seed_basis(dest, source, "delbasis", {self.UNCHANGED: b"v1\n"})
|
|
received = get_dest_received_dir(dest, source)
|
|
os.makedirs(received, exist_ok=True)
|
|
extra = os.path.join(received, "extra.txt")
|
|
with open(extra, "wb") as fh:
|
|
fh.write(b"extra")
|
|
with ServerManager() as server:
|
|
server.start(extra_args=["--allow-delete"])
|
|
result, _ = run_client(source, dest, flags=["--link-dest=delbasis", "--delete"],
|
|
port=server.port)
|
|
assert result.returncode == 0, \
|
|
f"delete+link-dest failed: {result.stderr[:300]}"
|
|
assert not os.path.exists(extra), "genuine extra file was not deleted"
|
|
assert _read_file(os.path.join(received, self.UNCHANGED)) == b"v1\n"
|
|
assert os.path.exists(os.path.join(basis, self.UNCHANGED)), \
|
|
"basis directory was deleted by --delete"
|
|
assert os.stat(os.path.join(received, self.UNCHANGED)).st_ino == \
|
|
os.stat(os.path.join(basis, self.UNCHANGED)).st_ino
|
|
|
|
def test_delay_delete_keeps_nested_staging_named_dir_as_content(self):
|
|
# The real --delay-updates staging directory is protected from --delete
|
|
# only as a DIRECT child of the receive root. A nested destination
|
|
# directory that merely shares the staging name is ordinary content, so
|
|
# its extras must still be deleted (regression guard for the walker).
|
|
source = self._make_source("basis_nested_stage_src",
|
|
{"top.txt": b"top\n", "sub/real.txt": b"real\n"})
|
|
dest = os.path.join(TEST_DATA_DIR, "basis_nested_stage_dst")
|
|
clean_dir(dest)
|
|
self._seed_basis(dest, source, "nstbasis",
|
|
{"top.txt": b"top\n", "sub/real.txt": b"real\n"})
|
|
received = get_dest_received_dir(dest, source)
|
|
nested = os.path.join(received, "sub", self.STAGING)
|
|
os.makedirs(nested, exist_ok=True)
|
|
extra = os.path.join(nested, "extra.txt")
|
|
with open(extra, "wb") as fh:
|
|
fh.write(b"nested extra")
|
|
with ServerManager() as server:
|
|
server.start(extra_args=["--allow-delete"])
|
|
result, _ = run_client(source, dest,
|
|
flags=["--link-dest=nstbasis", "--delete",
|
|
"--delay-updates"],
|
|
port=server.port)
|
|
assert result.returncode == 0, \
|
|
f"delay-delete nested staging failed: {result.stderr[:300]}"
|
|
assert not os.path.exists(extra), \
|
|
"extra inside a nested .fastsync-stage dir was not deleted"
|
|
assert not os.path.isdir(nested), \
|
|
"nested .fastsync-stage dir should have been removed after its extra"
|
|
assert _read_file(os.path.join(received, "sub", "real.txt")) == b"real\n"
|
|
assert not os.path.isdir(os.path.join(dest, self.STAGING)), \
|
|
"real delay-updates staging tree was not cleaned up"
|
|
|
|
def test_basis_priority_first_match_wins(self, shared_server):
|
|
# Two link-dest dirs both hold the exact file: the FIRST (command-line
|
|
# order) basis directory must win and supply the hard link.
|
|
source = self._make_source("basis_prio_src", {"f.txt": b"content\n"})
|
|
dest = os.path.join(TEST_DATA_DIR, "basis_prio_dst")
|
|
clean_dir(dest)
|
|
first = self._seed_basis_file(dest, source, "b1", "f.txt", b"content\n")
|
|
self._seed_basis_file(dest, source, "b2", "f.txt", b"content\n")
|
|
result, _ = run_client(source, dest, flags=["--link-dest=b1", "--link-dest=b2"],
|
|
port=shared_server.port)
|
|
assert result.returncode == 0, f"link-dest priority failed: {result.stderr[:300]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
assert os.stat(os.path.join(received, "f.txt")).st_ino == os.stat(first).st_ino, \
|
|
"first basis dir did not win over the second"
|
|
|
|
def test_basis_priority_across_compare_and_link(self, shared_server):
|
|
# A compare-dest entry listed BEFORE a link-dest entry shadows it (the
|
|
# exact match is found first and nothing is materialized); reversing the
|
|
# order lets the link-dest entry win and materialize a hard link.
|
|
source = self._make_source("basis_prio_mixed_src", {"f.txt": b"content\n"})
|
|
|
|
dest = os.path.join(TEST_DATA_DIR, "basis_prio_mixed_dst")
|
|
clean_dir(dest)
|
|
self._seed_basis_file(dest, source, "cmpb", "f.txt", b"content\n")
|
|
self._seed_basis_file(dest, source, "lnb", "f.txt", b"content\n")
|
|
result, _ = run_client(source, dest,
|
|
flags=["--compare-dest=cmpb", "--link-dest=lnb"],
|
|
port=shared_server.port)
|
|
assert result.returncode == 0, \
|
|
f"mixed priority (compare first) failed: {result.stderr[:300]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
assert not os.path.exists(os.path.join(received, "f.txt")), \
|
|
"compare-dest matched first, so the file must stay sparse (no link-dest materialize)"
|
|
|
|
dest = os.path.join(TEST_DATA_DIR, "basis_prio_mixed_dst2")
|
|
clean_dir(dest)
|
|
self._seed_basis_file(dest, source, "cmpb", "f.txt", b"content\n")
|
|
linkb2 = self._seed_basis_file(dest, source, "lnb", "f.txt", b"content\n")
|
|
result, _ = run_client(source, dest,
|
|
flags=["--link-dest=lnb", "--compare-dest=cmpb"],
|
|
port=shared_server.port)
|
|
assert result.returncode == 0, \
|
|
f"mixed priority (link first) failed: {result.stderr[:300]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
assert os.stat(os.path.join(received, "f.txt")).st_ino == os.stat(linkb2).st_ino, \
|
|
"link-dest did not materialize when listed before compare-dest"
|
|
|
|
def test_link_dest_size_only_ignores_mtime(self, shared_server):
|
|
# --size-only drops the mtime leg of the quick check: a basis file with
|
|
# the SAME content but a DIFFERENT mtime is still an exact match.
|
|
source = self._make_source("basis_sizeonly_src", {"f.txt": b"content\n"})
|
|
dest = os.path.join(TEST_DATA_DIR, "basis_sizeonly_dst")
|
|
clean_dir(dest)
|
|
basis_file = self._seed_basis_file(dest, source, "sob", "f.txt", b"content\n",
|
|
ts=self.TS + 500)
|
|
result, _ = run_client(source, dest, flags=["--link-dest=sob", "--size-only"],
|
|
port=shared_server.port)
|
|
assert result.returncode == 0, f"size-only link-dest failed: {result.stderr[:300]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
assert os.stat(os.path.join(received, "f.txt")).st_ino == os.stat(basis_file).st_ino, \
|
|
"--size-only should link a basis file whose mtime differs"
|
|
|
|
@pytest.mark.ci
|
|
def test_link_dest_ignore_times_never_links(self, shared_server):
|
|
# -I/--ignore-times forces every file to be updated, so a basis dir is
|
|
# never used to hard-link (rsync parity). The file is transferred and
|
|
# stored as a fresh inode even though it matches the basis exactly.
|
|
source = self._make_source("basis_igntimes_src", {"f.txt": b"content\n"})
|
|
dest = os.path.join(TEST_DATA_DIR, "basis_igntimes_dst")
|
|
clean_dir(dest)
|
|
basis_file = self._seed_basis_file(dest, source, "itb", "f.txt", b"content\n")
|
|
result, _ = run_client(source, dest, flags=["--link-dest=itb", "--ignore-times"],
|
|
port=shared_server.port)
|
|
assert result.returncode == 0, f"ignore-times link-dest failed: {result.stderr[:300]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
dest_file = os.path.join(received, "f.txt")
|
|
assert _read_file(dest_file) == b"content\n"
|
|
assert os.stat(dest_file).st_ino != os.stat(basis_file).st_ino, \
|
|
"--ignore-times must not hard-link to a basis file"
|
|
|
|
def test_basis_refuses_file_above_whole_file_limit(self, shared_server):
|
|
# Every whole-file payload path in FastSync (basis dirs included) is
|
|
# bounded by MAX_RECEIVE_WHOLE_FILE_SIZE. rsync supports basis dirs for
|
|
# arbitrary sizes; FastSync refuses such a run up front with a clear
|
|
# diagnostic instead of letting the receiver abort the whole transfer
|
|
# mid-stream with no client-side explanation.
|
|
source = self._make_source("basis_oversize_src", {"small.txt": b"ok\n"})
|
|
big = os.path.join(source, "huge.bin")
|
|
with open(big, "wb") as fh:
|
|
os.ftruncate(fh.fileno(), 256 * 1024 * 1024 + 4096)
|
|
dest = os.path.join(TEST_DATA_DIR, "basis_oversize_dst")
|
|
clean_dir(dest)
|
|
result, _ = run_client(source, dest, flags=["--link-dest=nope"],
|
|
port=shared_server.port)
|
|
assert result.returncode != 0, \
|
|
"basis run with an over-limit file unexpectedly succeeded"
|
|
assert "larger than" in result.stderr, \
|
|
f"no clear over-limit diagnostic: {result.stderr[:300]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
assert not os.path.exists(received), \
|
|
"over-limit basis run transferred files before failing"
|
|
|
|
|
|
def _random_payloads(size=2 * 1024 * 1024, changed=64 * 1024, seed=1234):
|
|
"""Return (old, new) byte strings of equal length where `new` differs from
|
|
`old` only in one contiguous `changed`-byte region. Incompressible (random)
|
|
data keeps the whole-file wire cost near the file size, so a delta transfer
|
|
is distinguishable from a whole-file one by its wire bytes."""
|
|
r = random.Random(seed)
|
|
data = bytearray(r.randbytes(size))
|
|
old = bytes(data)
|
|
off = size // 3
|
|
for i in range(off, off + changed):
|
|
data[i] = r.randrange(256)
|
|
return old, bytes(data)
|
|
|
|
|
|
class TestFuzzy:
|
|
"""-y/--fuzzy similar-file delta basis.
|
|
|
|
Scenario modelled on rsync's --fuzzy: a file is recreated under a similar
|
|
NEW basename in the same directory. The destination still holds the
|
|
old-named file (nothing deleted it), but the new path has no content of its
|
|
own at the destination, so without --fuzzy the receiver has no delta basis
|
|
and the whole file is sent. With --fuzzy the receiver searches the
|
|
destination directory, picks the similar-named sibling as the delta basis,
|
|
sends its block signature, and the sender transmits only the differences.
|
|
The reconstructed file must be byte-identical to the source in every mode;
|
|
only the wire usage changes (observed through CountingProxy, because client
|
|
--stats report source lengths, not wire bytes).
|
|
"""
|
|
|
|
OLD_NAME = "report-2025.dat"
|
|
NEW_NAME = "report-2026.dat"
|
|
TS = 1577836800 # 2020-01-01, used to pin stale destination mtimes
|
|
SIZE = 2 * 1024 * 1024
|
|
|
|
def _client_via_proxy(self, source, dest, flags, proxy):
|
|
cmd = (CLIENT_CMD + ["--source-dir", source, "--dest-dir", dest,
|
|
"--save-to-disk", "--server-port", str(proxy.port)] + flags)
|
|
return proxy.run(cmd)
|
|
|
|
def _seed_dest(self, source, dest, files, port):
|
|
"""Write `files` {rel: bytes} into source and mirror them to dest."""
|
|
for rel, content in files.items():
|
|
full = os.path.join(source, rel)
|
|
os.makedirs(os.path.dirname(full), exist_ok=True)
|
|
with open(full, "wb") as fh:
|
|
fh.write(content)
|
|
clean_dir(dest)
|
|
result, _ = run_client(source, dest, port=port)
|
|
assert result.returncode == 0, f"seed failed: {(result.stderr or result.stdout)[:300]}"
|
|
|
|
def _prepare(self, tag):
|
|
source = os.path.join(TEST_DATA_DIR, f"fuzzy_{tag}_src")
|
|
dest = os.path.join(TEST_DATA_DIR, f"fuzzy_{tag}_dst")
|
|
clean_dir(source)
|
|
return source, dest
|
|
|
|
def _run_measured(self, source, dest, flags, port):
|
|
"""Run a transfer through a byte-counting proxy. Returns (result, proxy)."""
|
|
proxy = CountingProxy(port)
|
|
result = self._client_via_proxy(source, dest, flags, proxy)
|
|
return result, proxy
|
|
|
|
def test_fuzzy_uses_similar_sibling_as_delta_basis(self, shared_server):
|
|
source, dest = self._prepare("basis")
|
|
old_bytes, new_bytes = _random_payloads()
|
|
self._seed_dest(source, dest, {self.OLD_NAME: old_bytes}, shared_server.port)
|
|
# Recreate the file under a similar new name; the old sibling stays on
|
|
# the destination (nothing deletes it).
|
|
os.unlink(os.path.join(source, self.OLD_NAME))
|
|
with open(os.path.join(source, self.NEW_NAME), "wb") as fh:
|
|
fh.write(new_bytes)
|
|
|
|
result, proxy = self._run_measured(source, dest, ["--fuzzy"], shared_server.port)
|
|
assert result.returncode == 0, \
|
|
f"--fuzzy rename transfer failed: {(result.stderr or result.stdout)[:300]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
assert _read_file(os.path.join(received, self.NEW_NAME)) == new_bytes, \
|
|
"fuzzy reconstruction is not byte-exact"
|
|
# The wire carried the delta, not the 2 MiB whole file.
|
|
assert proxy.client_to_server < len(new_bytes) // 4, \
|
|
f"fuzzy transfer sent {proxy.client_to_server} bytes; expected a delta"
|
|
|
|
def test_without_fuzzy_sends_the_whole_file(self, shared_server):
|
|
source, dest = self._prepare("whole")
|
|
old_bytes, new_bytes = _random_payloads()
|
|
self._seed_dest(source, dest, {self.OLD_NAME: old_bytes}, shared_server.port)
|
|
os.unlink(os.path.join(source, self.OLD_NAME))
|
|
with open(os.path.join(source, self.NEW_NAME), "wb") as fh:
|
|
fh.write(new_bytes)
|
|
|
|
result, proxy = self._run_measured(source, dest, ["--incremental", "--delta"], shared_server.port)
|
|
assert result.returncode == 0, f"no-fuzzy rename failed: {result.stderr[:300]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
assert _read_file(os.path.join(received, self.NEW_NAME)) == new_bytes
|
|
# No similar basis: the whole file goes over the wire.
|
|
assert proxy.client_to_server > len(new_bytes) // 2, \
|
|
f"expected a whole-file transfer, got {proxy.client_to_server} bytes"
|
|
|
|
@pytest.mark.parametrize("mt", [False, True])
|
|
def test_fuzzy_byte_exact_single_and_multithreaded(self, shared_server, mt):
|
|
source, dest = self._prepare(f"mt{'1' if mt else '0'}")
|
|
old_bytes, new_bytes = _random_payloads()
|
|
self._seed_dest(source, dest, {self.OLD_NAME: old_bytes}, shared_server.port)
|
|
os.unlink(os.path.join(source, self.OLD_NAME))
|
|
with open(os.path.join(source, self.NEW_NAME), "wb") as fh:
|
|
fh.write(new_bytes)
|
|
|
|
flags = ["--fuzzy"] + (["-m"] if mt else [])
|
|
result, proxy = self._run_measured(source, dest, flags, shared_server.port)
|
|
assert result.returncode == 0, \
|
|
f"--fuzzy {'-m ' if mt else ''}rename failed: {(result.stderr or result.stdout)[:300]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
assert _read_file(os.path.join(received, self.NEW_NAME)) == new_bytes, \
|
|
f"fuzzy {'-m ' if mt else ''}reconstruction is not byte-exact"
|
|
assert proxy.client_to_server < len(new_bytes) // 4
|
|
|
|
def test_no_candidate_falls_back_to_whole_file(self, shared_server):
|
|
# A brand-new destination directory holds no sibling at all, so --fuzzy
|
|
# finds nothing and the file is transferred whole (and correctly).
|
|
source, dest = self._prepare("nocand")
|
|
clean_dir(dest)
|
|
os.makedirs(source, exist_ok=True)
|
|
_, new_bytes = _random_payloads()
|
|
with open(os.path.join(source, self.NEW_NAME), "wb") as fh:
|
|
fh.write(new_bytes)
|
|
result, proxy = self._run_measured(source, dest, ["--fuzzy"], shared_server.port)
|
|
assert result.returncode == 0, \
|
|
f"--fuzzy no-candidate fallback failed: {(result.stderr or result.stdout)[:300]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
assert _read_file(os.path.join(received, self.NEW_NAME)) == new_bytes
|
|
assert proxy.client_to_server > len(new_bytes) // 2, \
|
|
"no-candidate fuzzy run should have sent the whole file"
|
|
|
|
def test_dissimilar_sibling_is_not_used(self, shared_server):
|
|
# The destination holds a large sibling whose basename is too different
|
|
# from the incoming name; the name gate must reject it and fall back to
|
|
# a whole-file transfer.
|
|
source, dest = self._prepare("dissim")
|
|
old_bytes, new_bytes = _random_payloads()
|
|
self._seed_dest(source, dest, {"totally-unrelated-notes.bin": old_bytes},
|
|
shared_server.port)
|
|
with open(os.path.join(source, self.NEW_NAME), "wb") as fh:
|
|
fh.write(new_bytes)
|
|
result, proxy = self._run_measured(source, dest, ["--fuzzy"], shared_server.port)
|
|
assert result.returncode == 0, \
|
|
f"--fuzzy dissimilar-sibling run failed: {(result.stderr or result.stdout)[:300]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
assert _read_file(os.path.join(received, self.NEW_NAME)) == new_bytes
|
|
assert proxy.client_to_server > len(new_bytes) // 2, \
|
|
"a dissimilar-named sibling must not be used as a fuzzy basis"
|
|
|
|
def test_fuzzy_helps_when_dest_holds_an_unsuitable_file(self, shared_server):
|
|
# The destination DOES hold the exact new name, but it is a tiny stale
|
|
# file (below the delta engine's minimum, ratio far outside its window),
|
|
# so it cannot serve as the basis. --fuzzy then falls back to the
|
|
# similar-named sibling.
|
|
source, dest = self._prepare("unsuitable")
|
|
old_bytes, new_bytes = _random_payloads()
|
|
self._seed_dest(source, dest,
|
|
{self.OLD_NAME: old_bytes, self.NEW_NAME: b"stale small file\n"},
|
|
shared_server.port)
|
|
with open(os.path.join(source, self.NEW_NAME), "wb") as fh:
|
|
fh.write(new_bytes)
|
|
result, proxy = self._run_measured(source, dest, ["--fuzzy"], shared_server.port)
|
|
assert result.returncode == 0, \
|
|
f"--fuzzy unsuitable-dest run failed: {(result.stderr or result.stdout)[:300]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
assert _read_file(os.path.join(received, self.NEW_NAME)) == new_bytes, \
|
|
"byte-exactness broken when the destination file was unsuitable"
|
|
assert proxy.client_to_server < len(new_bytes) // 4, \
|
|
"fuzzy should have reused the similar sibling as the basis"
|
|
|
|
def test_whole_file_makes_fuzzy_inert(self, shared_server):
|
|
# -W/--whole-file switches the delta machinery off, so --fuzzy has
|
|
# nothing to attach to and the file is transferred whole (rsync parity).
|
|
source, dest = self._prepare("wholefile")
|
|
old_bytes, new_bytes = _random_payloads()
|
|
self._seed_dest(source, dest, {self.OLD_NAME: old_bytes}, shared_server.port)
|
|
os.unlink(os.path.join(source, self.OLD_NAME))
|
|
with open(os.path.join(source, self.NEW_NAME), "wb") as fh:
|
|
fh.write(new_bytes)
|
|
result, proxy = self._run_measured(source, dest, ["--fuzzy", "-W"], shared_server.port)
|
|
assert result.returncode == 0, \
|
|
f"--fuzzy -W run failed: {(result.stderr or result.stdout)[:300]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
assert _read_file(os.path.join(received, self.NEW_NAME)) == new_bytes
|
|
assert proxy.client_to_server > len(new_bytes) // 2, \
|
|
"--whole-file must disable the fuzzy delta basis"
|
|
|
|
def test_fuzzy_via_short_y_alias(self, shared_server):
|
|
source, dest = self._prepare("shorty")
|
|
old_bytes, new_bytes = _random_payloads()
|
|
self._seed_dest(source, dest, {self.OLD_NAME: old_bytes}, shared_server.port)
|
|
os.unlink(os.path.join(source, self.OLD_NAME))
|
|
with open(os.path.join(source, self.NEW_NAME), "wb") as fh:
|
|
fh.write(new_bytes)
|
|
result, proxy = self._run_measured(source, dest, ["-y"], shared_server.port)
|
|
assert result.returncode == 0, f"-y rename failed: {(result.stderr or result.stdout)[:300]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
assert _read_file(os.path.join(received, self.NEW_NAME)) == new_bytes
|
|
assert proxy.client_to_server < len(new_bytes) // 4, "-y did not enable fuzzy"
|
|
|
|
def test_fuzzy_with_delay_updates_publishes_cleanly(self, shared_server):
|
|
# A fuzzy-reconstructed file goes through the normal store engine, so
|
|
# --delay-updates must stage and publish it with no staging leftovers.
|
|
source, dest = self._prepare("delay")
|
|
old_bytes, new_bytes = _random_payloads()
|
|
self._seed_dest(source, dest, {self.OLD_NAME: old_bytes}, shared_server.port)
|
|
os.unlink(os.path.join(source, self.OLD_NAME))
|
|
with open(os.path.join(source, self.NEW_NAME), "wb") as fh:
|
|
fh.write(new_bytes)
|
|
result, _ = run_client(source, dest,
|
|
flags=["--fuzzy", "--delay-updates"],
|
|
port=shared_server.port)
|
|
assert result.returncode == 0, \
|
|
f"--fuzzy --delay-updates failed: {(result.stderr or result.stdout)[:300]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
assert _read_file(os.path.join(received, self.NEW_NAME)) == new_bytes
|
|
assert not os.path.isdir(os.path.join(dest, ".fastsync-stage")), \
|
|
"delay-updates staging tree was not cleaned up"
|
|
|
|
def test_fuzzy_source_removed_after_transfer(self, shared_server):
|
|
# A fuzzy transfer is a real transfer (not a skip), so
|
|
# --remove-source-files must remove the renamed source file.
|
|
source, dest = self._prepare("rm")
|
|
old_bytes, new_bytes = _random_payloads()
|
|
self._seed_dest(source, dest, {self.OLD_NAME: old_bytes}, shared_server.port)
|
|
os.unlink(os.path.join(source, self.OLD_NAME))
|
|
with open(os.path.join(source, self.NEW_NAME), "wb") as fh:
|
|
fh.write(new_bytes)
|
|
result, _ = run_client(source, dest,
|
|
flags=["--fuzzy", "--remove-source-files"],
|
|
port=shared_server.port)
|
|
assert result.returncode == 0, \
|
|
f"--fuzzy --remove-source-files failed: {(result.stderr or result.stdout)[:300]}"
|
|
assert not os.path.exists(os.path.join(source, self.NEW_NAME)), \
|
|
"a fuzzy-transferred source should have been removed"
|
|
received = get_dest_received_dir(dest, source)
|
|
assert _read_file(os.path.join(received, self.NEW_NAME)) == new_bytes
|
|
assert _read_file(os.path.join(received, self.OLD_NAME)) == old_bytes
|
|
|
|
@staticmethod
|
|
def _rand_bytes(size, seed):
|
|
return random.Random(seed).randbytes(size)
|
|
|
|
def _replace_source_file(self, source, old_name, new_name, new_bytes):
|
|
"""Remove old_name from source and add new_name with new_bytes."""
|
|
os.unlink(os.path.join(source, old_name))
|
|
with open(os.path.join(source, new_name), "wb") as fh:
|
|
fh.write(new_bytes)
|
|
|
|
def test_worthless_fuzzy_basis_falls_back_inside_handshake(self, shared_server):
|
|
# The sibling passes the name AND size gates but shares no blocks with
|
|
# the incoming file, so the sender's delta is not worthwhile: it replies
|
|
# STATUS_NEXT and the receiver consumes the WHOLE file inside the delta
|
|
# handshake. This proves a bad fuzzy basis cannot desync the protocol
|
|
# or corrupt the result.
|
|
source, dest = self._prepare("worthless")
|
|
basis = self._rand_bytes(self.SIZE, 424242)
|
|
target = self._rand_bytes(self.SIZE, 777777)
|
|
self._seed_dest(source, dest, {self.OLD_NAME: basis}, shared_server.port)
|
|
self._replace_source_file(source, self.OLD_NAME, self.NEW_NAME, target)
|
|
result, proxy = self._run_measured(source, dest, ["--fuzzy"], shared_server.port)
|
|
assert result.returncode == 0, \
|
|
f"--fuzzy worthless-basis run failed: {(result.stderr or result.stdout)[:300]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
assert _read_file(os.path.join(received, self.NEW_NAME)) == target, \
|
|
"whole-file fallback after a worthless fuzzy basis is not byte-exact"
|
|
assert proxy.client_to_server > self.SIZE // 2, \
|
|
"a worthless basis should have made the sender fall back to the whole file"
|
|
|
|
def test_existing_dest_file_preferred_over_fuzzy_sibling(self, shared_server):
|
|
# Non-displacement: the destination holds a file at the exact path that
|
|
# is inside the delta size bounds (same size, different content, older
|
|
# mtime). FastSync must delta against THAT file -- even though it
|
|
# shares nothing with the source -- and must NOT reuse a similar-named
|
|
# sibling that is byte-identical to the source.
|
|
source, dest = self._prepare("nondisp")
|
|
sibling = self._rand_bytes(self.SIZE, 111) # will equal the incoming file
|
|
stale = self._rand_bytes(self.SIZE, 333) # worthless exact-path file
|
|
self._seed_dest(source, dest,
|
|
{self.OLD_NAME: sibling, self.NEW_NAME: stale},
|
|
shared_server.port)
|
|
# Force the exact-path destination file's mtime into the past so the
|
|
# quick check deterministically decides to transfer it.
|
|
os.utime(os.path.join(get_dest_received_dir(dest, source), self.NEW_NAME),
|
|
(self.TS, self.TS))
|
|
os.unlink(os.path.join(source, self.OLD_NAME))
|
|
with open(os.path.join(source, self.NEW_NAME), "wb") as fh:
|
|
fh.write(sibling)
|
|
result, proxy = self._run_measured(source, dest, ["--fuzzy"], shared_server.port)
|
|
assert result.returncode == 0, \
|
|
f"--fuzzy non-displacement run failed: {(result.stderr or result.stdout)[:300]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
assert _read_file(os.path.join(received, self.NEW_NAME)) == sibling
|
|
assert proxy.client_to_server > self.SIZE // 2, \
|
|
"the exact-path destination file must be the delta basis, not the fuzzy sibling"
|
|
|
|
def test_fuzzy_basis_larger_than_source(self, shared_server):
|
|
# The similar sibling is LARGER than the incoming file (within the delta
|
|
# engine's 10x ratio); the new file is an exact prefix of the basis, so
|
|
# every block matches and only a tiny delta travels.
|
|
source, dest = self._prepare("largerbasis")
|
|
big = self._rand_bytes(1536 * 1024, 1)
|
|
prefix = big[:1024 * 1024]
|
|
self._seed_dest(source, dest, {self.OLD_NAME: big}, shared_server.port)
|
|
self._replace_source_file(source, self.OLD_NAME, self.NEW_NAME, prefix)
|
|
result, proxy = self._run_measured(source, dest, ["--fuzzy"], shared_server.port)
|
|
assert result.returncode == 0, \
|
|
f"--fuzzy larger-basis run failed: {(result.stderr or result.stdout)[:300]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
assert _read_file(os.path.join(received, self.NEW_NAME)) == prefix, \
|
|
"shrunken file reconstructed from a larger fuzzy basis is not byte-exact"
|
|
assert proxy.client_to_server < len(prefix) // 4, \
|
|
"larger fuzzy basis should have carried most of the file as block matches"
|
|
|
|
def test_fuzzy_basis_smaller_than_source(self, shared_server):
|
|
# The similar sibling is SMALLER than the incoming file; the new file
|
|
# appends data past the basis, so the appended tail travels as literals
|
|
# while the shared prefix is block-matched.
|
|
source, dest = self._prepare("smallerbasis")
|
|
base = self._rand_bytes(self.SIZE, 2)
|
|
tail = self._rand_bytes(64 * 1024, 3)
|
|
new_bytes = base + tail
|
|
self._seed_dest(source, dest, {self.OLD_NAME: base}, shared_server.port)
|
|
self._replace_source_file(source, self.OLD_NAME, self.NEW_NAME, new_bytes)
|
|
result, proxy = self._run_measured(source, dest, ["--fuzzy"], shared_server.port)
|
|
assert result.returncode == 0, \
|
|
f"--fuzzy smaller-basis run failed: {(result.stderr or result.stdout)[:300]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
assert _read_file(os.path.join(received, self.NEW_NAME)) == new_bytes, \
|
|
"grown file reconstructed from a smaller fuzzy basis is not byte-exact"
|
|
assert proxy.client_to_server < len(new_bytes) // 4, \
|
|
"smaller fuzzy basis should have block-matched the shared prefix"
|
|
|
|
def test_no_fuzzy_end_to_end_equals_no_flag(self, shared_server):
|
|
# --no-fuzzy must not enable anything: a run with it behaves exactly
|
|
# like a run without it (whole-file transfer, byte-exact output).
|
|
source, dest = self._prepare("nofuzzye2e")
|
|
old_bytes, new_bytes = _random_payloads()
|
|
self._seed_dest(source, dest, {self.OLD_NAME: old_bytes}, shared_server.port)
|
|
self._replace_source_file(source, self.OLD_NAME, self.NEW_NAME, new_bytes)
|
|
result, proxy = self._run_measured(source, dest, ["--no-fuzzy"], shared_server.port)
|
|
assert result.returncode == 0, \
|
|
f"--no-fuzzy run failed: {(result.stderr or result.stdout)[:300]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
assert _read_file(os.path.join(received, self.NEW_NAME)) == new_bytes
|
|
assert proxy.client_to_server > len(new_bytes) // 2, \
|
|
"--no-fuzzy should leave the default whole-file behavior intact"
|
|
|
|
|
|
|
|
class TestIdentityMapping:
|
|
"""Ownership-application flags (--numeric-ids / --usermap / --groupmap /
|
|
--chown). In CI the receiver usually runs unprivileged, so ownership apply
|
|
is expected to fail from lack of privilege: the transfer must STILL succeed
|
|
and exit 0 (the receiver warns and continues, rsync parity). The only
|
|
assertion that requires the ownership to actually change is gated on
|
|
os.geteuid() == 0 so it is skipped (not failed) as a non-root user."""
|
|
|
|
def test_numeric_ids_transfer_succeeds_unprivileged(self, shared_server):
|
|
source = os.path.join(TEST_DATA_DIR, "identity_num_source")
|
|
dest = os.path.join(TEST_DATA_DIR, "identity_num_dest")
|
|
clean_dir(source)
|
|
clean_dir(dest)
|
|
with open(os.path.join(source, "f.txt"), "wb") as f:
|
|
f.write(b"hello identity")
|
|
result, _ = run_client(source, dest,
|
|
flags=["-M", "--numeric-ids"],
|
|
port=shared_server.port)
|
|
assert result.returncode == 0, \
|
|
f"exit {result.returncode}: {(result.stderr or '')[:200]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
with open(os.path.join(received, "f.txt"), "rb") as f:
|
|
assert f.read() == b"hello identity"
|
|
|
|
def test_usermap_and_groupmap_and_chown_succeed_unprivileged(self, shared_server):
|
|
source = os.path.join(TEST_DATA_DIR, "identity_map_source")
|
|
dest = os.path.join(TEST_DATA_DIR, "identity_map_dest")
|
|
clean_dir(source)
|
|
clean_dir(dest)
|
|
with open(os.path.join(source, "f.txt"), "wb") as f:
|
|
f.write(b"mapped")
|
|
result, _ = run_client(
|
|
source, dest,
|
|
flags=["-M", "--usermap=@1000:@1001", "--groupmap=@100:@101", "--chown=@2000:@2001"],
|
|
port=shared_server.port)
|
|
assert result.returncode == 0, \
|
|
f"exit {result.returncode}: {(result.stderr or '')[:200]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
with open(os.path.join(received, "f.txt"), "rb") as f:
|
|
assert f.read() == b"mapped"
|
|
|
|
@pytest.mark.skipif(os.geteuid() != 0, reason="only root can change ownership")
|
|
def test_numeric_ids_applies_ownership_as_root(self, shared_server):
|
|
source = os.path.join(TEST_DATA_DIR, "identity_root_source")
|
|
dest = os.path.join(TEST_DATA_DIR, "identity_root_dest")
|
|
clean_dir(source)
|
|
clean_dir(dest)
|
|
src_file = os.path.join(source, "f.txt")
|
|
with open(src_file, "wb") as f:
|
|
f.write(b"owner")
|
|
os.chown(src_file, 12345, 12346)
|
|
result, _ = run_client(source, dest,
|
|
flags=["-M", "--numeric-ids"],
|
|
port=shared_server.port)
|
|
assert result.returncode == 0, \
|
|
f"exit {result.returncode}: {(result.stderr or '')[:200]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
dst_file = os.path.join(received, "f.txt")
|
|
assert os.path.exists(dst_file)
|
|
st = os.stat(dst_file)
|
|
assert st.st_uid == 12345 and st.st_gid == 12346, \
|
|
f"owner not applied: uid={st.st_uid} gid={st.st_gid}"
|
|
|
|
@pytest.mark.skipif(os.geteuid() != 0, reason="only root can change ownership")
|
|
def test_chown_overrides_ownership_as_root(self, shared_server):
|
|
source = os.path.join(TEST_DATA_DIR, "identity_chown_root_source")
|
|
dest = os.path.join(TEST_DATA_DIR, "identity_chown_root_dest")
|
|
clean_dir(source)
|
|
clean_dir(dest)
|
|
src_file = os.path.join(source, "f.txt")
|
|
with open(src_file, "wb") as f:
|
|
f.write(b"root chown")
|
|
os.chown(src_file, 1, 1)
|
|
result, _ = run_client(source, dest,
|
|
flags=["-M", "--chown=@12345:@54321"],
|
|
port=shared_server.port)
|
|
assert result.returncode == 0, \
|
|
f"exit {result.returncode}: {(result.stderr or '')[:200]}"
|
|
received = get_dest_received_dir(dest, source)
|
|
dst_file = os.path.join(received, "f.txt")
|
|
assert os.path.exists(dst_file)
|
|
st = os.stat(dst_file)
|
|
assert st.st_uid == 12345 and st.st_gid == 54321, \
|
|
f"--chown not applied: uid={st.st_uid} gid={st.st_gid}"
|