"""Differential coverage for ``--delete-delay`` + ``--max-delete`` with a refilled deferred directory. FastSync snapshots a directory's extras at plan time (``defer_add``) and charges ``--max-delete`` then, and its deferred commit only removes the snapshot path, so a directory refilled before the commit survives ``ENOTEMPTY``. rsync computes the deferred deletions during the transfer, charges ``--max-delete`` on actual removals, and recursively removes a queued directory -- so content created after the plan inside an extra directory is removed too. These tests run both tools on the same fixture and pin the shared budget bound (the later extra survives, both exit 25) plus the documented residual (the refilled directory's late content survives under FastSync, not rsync). They are not part of the fast PR gate because the rsync side needs a wide real-time injection window (a throttled transfer), while the FastSync side uses the existing byte-deterministic slicing proxy. The refilled directory sits at the transfer ROOT, whose delete plan is always processed before any subdirectory's, so the budget is deterministically charged to the refilled entry; the second extra lives under ``b`` and is skipped. """ import os import shutil import subprocess import sys import threading import time import pytest sys.path.insert(0, os.path.dirname(__file__)) from common import ( # noqa: E402 TEST_DATA_DIR, ServerManager, clean_dir, get_dest_received_dir, run_client, ) from test_delete_timing_parity import _SlicingProxy # noqa: E402 RSYNC = shutil.which("rsync") requires_rsync = pytest.mark.skipif(RSYNC is None, reason="rsync 3.4.1 not installed") BIG_BYTES = 8 * 1024 * 1024 MID_TRANSFER_BYTES = 256 * 1024 PROXY_THROTTLE = 0.001 # rsync is driven locally, so the refill is injected on a wall-clock delay while # a throttled ~8 s transfer is in flight. 1.5 s is safely after rsync's plan # scan (t=0) and well before the deferred commit at the end. RSYNC_BWLIMIT = 1024 # 1 MiB/s RSYNC_INJECT_DELAY = 1.5 def _write(path, content): os.makedirs(os.path.dirname(path), exist_ok=True) with open(path, "wb") as fh: fh.write(content) def _seed_source(tag): source = os.path.join(TEST_DATA_DIR, f"ddb_{tag}_src") clean_dir(source) _write(os.path.join(source, "a", "keep.bin"), b"B" * BIG_BYTES) _write(os.path.join(source, "b", "keep.txt"), b"keep\n") return source def _seed_fastsync(tag): """FastSync mirrors the absolute source path under its receive root, so the extras live below ``received``.""" source = _seed_source(tag) dest = os.path.join(TEST_DATA_DIR, f"ddb_{tag}_dst") clean_dir(dest) received = get_dest_received_dir(dest, source) os.makedirs(os.path.join(received, "xdir"), exist_ok=True) os.makedirs(os.path.join(received, "b", "ydir"), exist_ok=True) return source, dest, received def _seed_rsync(tag): """rsync mirrors the source contents directly into the destination, so the extras are flat under ``rsync_dst``.""" source = _seed_source(tag) rsync_dst = os.path.join(TEST_DATA_DIR, f"ddb_{tag}_dst") clean_dir(rsync_dst) os.makedirs(os.path.join(rsync_dst, "xdir"), exist_ok=True) os.makedirs(os.path.join(rsync_dst, "b", "ydir"), exist_ok=True) return source, rsync_dst def _deleted_count(text): for line in text.splitlines(): if line.startswith("Number of deleted files:"): return int(line.split(":", 1)[1].split()[0]) return None def _rsync(args, timeout=120): env = dict(os.environ, LC_ALL="C") return subprocess.run([RSYNC] + args, capture_output=True, text=True, env=env, timeout=timeout) class TestDeleteDelayRefilledDirVsRsync: """The shared budget bound and the documented recursive-removal residual.""" def _fastsync_refilled(self, tag, max_delete=None): """Run FastSync with the refill injected deterministically by the proxy hook (fired once the receiver has processed the plan frames).""" source, dest, received = _seed_fastsync(tag) late = os.path.join(received, "xdir", "new.txt") def hook(): _write(late, b"created mid-transfer\n") flags = ["--delete-delay", "--incremental", "--ignore-times", "--stats"] if max_delete is not None: flags.append(f"--max-delete={max_delete}") with ServerManager() as server: server.start(extra_args=["--allow-delete"]) proxy = _SlicingProxy(server.port, hook=hook, hook_after=MID_TRANSFER_BYTES, throttle=PROXY_THROTTLE, wait_for_reply=True) result, _ = run_client(source, dest, flags=flags, port=proxy.port) proxy.finish() assert proxy.hook_called.is_set(), "refill hook never fired" return result, received, late @requires_rsync def test_max_delete_budget_bound_matches_and_residual_pinned(self): # --- FastSync: budget charged at snapshot; late content preserved --- result, received, late = self._fastsync_refilled("budget_fs", max_delete=1) assert result.returncode == 25, (result.stderr or result.stdout)[:300] assert _deleted_count(result.stdout) == 0, result.stdout assert os.path.exists(late), "FastSync removed the late content of a snapshotted dir" assert os.path.isdir(os.path.join(received, "xdir")) assert os.path.isdir(os.path.join(received, "b", "ydir")), ( "FastSync did not charge the plan-time budget: b/ydir was removed" ) # --- rsync: budget charged on actual removals; dirs removed recursively --- source, rsync_dst = _seed_rsync("budget_rsync") def inject(): time.sleep(RSYNC_INJECT_DELAY) _write(os.path.join(rsync_dst, "xdir", "new.txt"), b"created mid-transfer\n") t = threading.Thread(target=inject) t.start() rsync_result = _rsync( ["-a", "--delete-delay", "--max-delete=1", "--stats", f"--bwlimit={RSYNC_BWLIMIT}", source + "/", rsync_dst + "/"] ) t.join() assert rsync_result.returncode == 25, rsync_result.stderr # rsync removes the late content (recursive deferred removal); FastSync # keeps it and charges the snapshot directive instead. assert not os.path.exists(os.path.join(rsync_dst, "xdir", "new.txt")), ( "rsync kept late content inside a queued directory" ) # The shared observable: the later extra survives in both tools under # --max-delete=1, and both report the capped run with exit 25. assert os.path.isdir(os.path.join(rsync_dst, "b", "ydir")), ( "rsync did not bound the deletion with --max-delete=1" ) assert os.path.isdir(os.path.join(received, "b", "ydir")) @requires_rsync def test_refilled_extra_dir_recursive_removal_residual(self): """Without --max-delete the residual is a plain tree difference: rsync removes the refilled extra directory (and its late content), FastSync leaves the snapshot path in place on ENOTEMPTY.""" result, received, late = self._fastsync_refilled("recur_fs") assert result.returncode == 0, (result.stderr or result.stdout)[:300] assert os.path.exists(late), "FastSync removed the refilled directory's late content" source, rsync_dst = _seed_rsync("recur_rsync") def inject(): time.sleep(RSYNC_INJECT_DELAY) _write(os.path.join(rsync_dst, "xdir", "new.txt"), b"created mid-transfer\n") t = threading.Thread(target=inject) t.start() rsync_result = _rsync( ["-a", "--delete-delay", "--stats", f"--bwlimit={RSYNC_BWLIMIT}", source + "/", rsync_dst + "/"] ) t.join() assert rsync_result.returncode == 0, rsync_result.stderr assert not os.path.exists(os.path.join(rsync_dst, "xdir")), ( "rsync did not recursively remove the refilled extra directory" )