TapTap
ffdbb6568f
- CLI: --append/--append-verify acceptance (parse + imply --incremental, validate) and incompatibility rejection with -s and --whole-file; both removed from the unimplemented reject list. - Config: on-the-wire append/append_verify round-trip. - Unit: append_resume_eligible / append_tail_length pure resume math. - Integration (test_append.py): matching-prefix resume is byte-identical and tail-only (wire bytes << source size); --append with a wrong prefix keeps prefix+tail (rsync parity) while --append-verify detects the mismatch and falls back to a byte-exact full transfer; --append with --inplace and -m.
141 lines
6.7 KiB
Python
141 lines
6.7 KiB
Python
"""--append / --append-verify tail-resume integration tests.
|
|
|
|
A shorter existing destination file is resumed by transferring only the tail:
|
|
--append sends it without verifying the retained prefix (rsync parity: a wrong
|
|
prefix is kept, so the result can differ from the source), while --append-verify
|
|
checksums the retained prefix against the source and, on a mismatch, falls back
|
|
to a clean full transfer so the result is always a byte-identical source copy.
|
|
"""
|
|
import os
|
|
import random
|
|
import shutil
|
|
|
|
from common import (
|
|
TEST_DATA_DIR,
|
|
run_client, CountingProxy, clean_dir,
|
|
get_dest_received_dir, CLIENT_CMD,
|
|
)
|
|
|
|
REL = "sub/grow.dat"
|
|
|
|
|
|
def _grow_payload(prefix_size, added_size, seed=99):
|
|
r = random.Random(seed)
|
|
return bytes(r.randbytes(prefix_size)), bytes(r.randbytes(added_size))
|
|
|
|
|
|
class TestAppend:
|
|
def _make(self, tag):
|
|
source = os.path.join(TEST_DATA_DIR, f"append_{tag}_src")
|
|
dest = os.path.join(TEST_DATA_DIR, f"append_{tag}_dst")
|
|
clean_dir(source)
|
|
shutil.rmtree(dest, ignore_errors=True)
|
|
return source, dest
|
|
|
|
def _place(self, root, rel, data):
|
|
p = os.path.join(root, rel)
|
|
os.makedirs(os.path.dirname(p), exist_ok=True)
|
|
with open(p, "wb") as fh:
|
|
fh.write(data)
|
|
return p
|
|
|
|
def _read(self, root, rel):
|
|
with open(os.path.join(root, rel), "rb") as fh:
|
|
return fh.read()
|
|
|
|
def _dest_file(self, source, dest, rel):
|
|
return os.path.join(get_dest_received_dir(dest, source), rel)
|
|
|
|
def test_append_resumes_short_dest_atomically(self, shared_server):
|
|
"""A shorter dest with a MATCHING prefix is resumed; the reconstructed
|
|
file is byte-identical to the source."""
|
|
source, dest = self._make("atomic")
|
|
prefix, added = _grow_payload(1 * 1024 * 1024, 64 * 1024)
|
|
self._place(source, REL, prefix + added)
|
|
self._place(self._dest_file(source, dest, ""), REL, prefix)
|
|
|
|
result, _ = run_client(source, dest, flags=["--append"], port=shared_server.port)
|
|
assert result.returncode == 0, \
|
|
f"--append failed: {(result.stderr or result.stdout)[:400]}"
|
|
assert self._read(self._dest_file(source, dest, ""), REL) == prefix + added
|
|
|
|
def test_append_verify_matching_prefix_succeeds(self, shared_server):
|
|
source, dest = self._make("verify_ok")
|
|
prefix, added = _grow_payload(512 * 1024, 32 * 1024)
|
|
self._place(source, REL, prefix + added)
|
|
self._place(self._dest_file(source, dest, ""), REL, prefix)
|
|
|
|
result, _ = run_client(source, dest, flags=["--append-verify"], port=shared_server.port)
|
|
assert result.returncode == 0, \
|
|
f"--append-verify failed: {(result.stderr or result.stdout)[:400]}"
|
|
assert self._read(self._dest_file(source, dest, ""), REL) == prefix + added
|
|
|
|
def test_append_sends_only_tail(self, shared_server):
|
|
"""Sorted transfer moves only the tail: wire bytes stay well below the
|
|
full source size (incompressible payload, no -c)."""
|
|
source, dest = self._make("tail")
|
|
prefix, added = _grow_payload(4 * 1024 * 1024, 8 * 1024, seed=7)
|
|
full = prefix + added
|
|
self._place(source, REL, full)
|
|
self._place(self._dest_file(source, dest, ""), REL, prefix)
|
|
|
|
proxy = CountingProxy(shared_server.port)
|
|
cmd = (CLIENT_CMD + ["--source-dir", source, "--dest-dir", dest,
|
|
"--save-to-disk", "--server-port", str(proxy.port), "--append"])
|
|
result = proxy.run(cmd)
|
|
assert result.returncode == 0, \
|
|
f"--append failed: {(result.stderr or result.stdout)[:400]}"
|
|
assert self._read(self._dest_file(source, dest, ""), REL) == full
|
|
assert proxy.client_to_server < full.__len__() // 2, \
|
|
f"expected a tail-only transfer, sent {proxy.client_to_server}B for {full.__len__()}B"
|
|
|
|
def test_plain_append_wrong_prefix_is_rsync_parity(self, shared_server):
|
|
"""--append does NOT verify the retained prefix: a wrong prefix is kept,
|
|
so the result is prefix+tail (differs from the source). This is the
|
|
documented rsync-parity risk of plain --append."""
|
|
source, dest = self._make("plain_wrong")
|
|
correct_prefix, added = _grow_payload(256 * 1024, 32 * 1024, seed=1)
|
|
wrong_prefix = bytes(b ^ 0xFF for b in correct_prefix)
|
|
self._place(source, REL, correct_prefix + added)
|
|
self._place(self._dest_file(source, dest, ""), REL, wrong_prefix)
|
|
|
|
result, _ = run_client(source, dest, flags=["--append"], port=shared_server.port)
|
|
assert result.returncode == 0
|
|
assert self._read(self._dest_file(source, dest, ""), REL) == wrong_prefix + added
|
|
|
|
def test_append_verify_wrong_prefix_never_corrupts(self, shared_server):
|
|
"""--append-verify detects the retained prefix mismatch and falls back to
|
|
a full transfer, so the result is a byte-identical source copy."""
|
|
source, dest = self._make("verify_wrong")
|
|
correct_prefix, added = _grow_payload(256 * 1024, 32 * 1024, seed=2)
|
|
wrong_prefix = bytes(b ^ 0xFF for b in correct_prefix)
|
|
self._place(source, REL, correct_prefix + added)
|
|
self._place(self._dest_file(source, dest, ""), REL, wrong_prefix)
|
|
|
|
result, _ = run_client(source, dest, flags=["--append-verify"], port=shared_server.port)
|
|
assert result.returncode == 0, \
|
|
f"--append-verify mismatch fallback failed: {(result.stderr or result.stdout)[:400]}"
|
|
assert self._read(self._dest_file(source, dest, ""), REL) == correct_prefix + added
|
|
|
|
def test_append_with_inplace(self, shared_server):
|
|
source, dest = self._make("inplace")
|
|
prefix, added = _grow_payload(128 * 1024, 16 * 1024, seed=3)
|
|
self._place(source, REL, prefix + added)
|
|
self._place(self._dest_file(source, dest, ""), REL, prefix)
|
|
|
|
result, _ = run_client(source, dest, flags=["--append", "--inplace"],
|
|
port=shared_server.port)
|
|
assert result.returncode == 0, \
|
|
f"--append --inplace failed: {(result.stderr or result.stdout)[:400]}"
|
|
assert self._read(self._dest_file(source, dest, ""), REL) == prefix + added
|
|
|
|
def test_append_multithreaded(self, shared_server):
|
|
source, dest = self._make("mthread")
|
|
prefix, added = _grow_payload(512 * 1024, 32 * 1024, seed=4)
|
|
self._place(source, REL, prefix + added)
|
|
self._place(self._dest_file(source, dest, ""), REL, prefix)
|
|
|
|
result, _ = run_client(source, dest, flags=["--append", "-m"], port=shared_server.port)
|
|
assert result.returncode == 0, \
|
|
f"--append -m failed: {(result.stderr or result.stdout)[:400]}"
|
|
assert self._read(self._dest_file(source, dest, ""), REL) == prefix + added |