fix(parity): dry-run delete protections, delete-delay actual-removal budget, --info name2
- -n/--delete sends the same protected/size-skipped/scope as a real run, so the read-only would-delete walk no longer over-reports (row -> caveat) - --delete-delay charges --max-delete on actual removals and recursively re-scans a refilled deferred directory at commit; independent deferred cap - --info=name emits the leading ./ root line and name2 'is uptodate' lines - differential tests promoted from residual pins to rsync parity assertions
This commit is contained in:
@@ -1,19 +1,16 @@
|
||||
"""Differential coverage for ``--delete-delay`` + ``--max-delete`` with a
|
||||
refilled deferred directory.
|
||||
|
||||
FastSync snapshots a directory's extras at plan time (``defer_add``) and charges
|
||||
``--max-delete`` then, and its deferred commit only removes the snapshot path, so
|
||||
a directory refilled before the commit survives ``ENOTEMPTY``. rsync computes
|
||||
the deferred deletions during the transfer, charges ``--max-delete`` on actual
|
||||
removals, and recursively removes a queued directory -- so content created after
|
||||
the plan inside an extra directory is removed too.
|
||||
FastSync snapshots a directory's extras at plan time (``defer_add``) but charges
|
||||
``--max-delete`` only when a path is actually removed, and its deferred commit
|
||||
re-scans a queued directory and removes content created after the plan -- the
|
||||
same rules as rsync. These tests run both tools on the same fixture and assert
|
||||
both sides remove the late content (recursively) and bound the deletion with
|
||||
``--max-delete`` identically.
|
||||
|
||||
These tests run both tools on the same fixture and pin the shared budget bound
|
||||
(the later extra survives, both exit 25) plus the documented residual (the
|
||||
refilled directory's late content survives under FastSync, not rsync). They are
|
||||
not part of the fast PR gate because the rsync side needs a wide real-time
|
||||
injection window (a throttled transfer), while the FastSync side uses the
|
||||
existing byte-deterministic slicing proxy.
|
||||
They are not part of the fast PR gate because the rsync side needs a wide
|
||||
real-time injection window (a throttled transfer), while the FastSync side uses
|
||||
the existing byte-deterministic slicing proxy.
|
||||
|
||||
The refilled directory sits at the transfer ROOT, whose delete plan is always
|
||||
processed before any subdirectory's, so the budget is deterministically charged
|
||||
@@ -101,7 +98,7 @@ def _rsync(args, timeout=120):
|
||||
|
||||
|
||||
class TestDeleteDelayRefilledDirVsRsync:
|
||||
"""The shared budget bound and the documented recursive-removal residual."""
|
||||
"""Both tools charge --max-delete on actual removals and recurse."""
|
||||
|
||||
def _fastsync_refilled(self, tag, max_delete=None):
|
||||
"""Run FastSync with the refill injected deterministically by the proxy
|
||||
@@ -125,18 +122,18 @@ class TestDeleteDelayRefilledDirVsRsync:
|
||||
return result, received, late
|
||||
|
||||
@requires_rsync
|
||||
def test_max_delete_budget_bound_matches_and_residual_pinned(self):
|
||||
# --- FastSync: budget charged at snapshot; late content preserved ---
|
||||
def test_max_delete_budget_bound_matches(self):
|
||||
# --- FastSync: the one actual removal is the late file; dirs survive ---
|
||||
result, received, late = self._fastsync_refilled("budget_fs", max_delete=1)
|
||||
assert result.returncode == 25, (result.stderr or result.stdout)[:300]
|
||||
assert _deleted_count(result.stdout) == 0, result.stdout
|
||||
assert os.path.exists(late), "FastSync removed the late content of a snapshotted dir"
|
||||
assert _deleted_count(result.stdout) == 1, result.stdout
|
||||
assert not os.path.exists(late), "FastSync kept the late content of a queued dir"
|
||||
assert os.path.isdir(os.path.join(received, "xdir"))
|
||||
assert os.path.isdir(os.path.join(received, "b", "ydir")), (
|
||||
"FastSync did not charge the plan-time budget: b/ydir was removed"
|
||||
"FastSync did not bound the deletion with --max-delete=1"
|
||||
)
|
||||
|
||||
# --- rsync: budget charged on actual removals; dirs removed recursively ---
|
||||
# --- rsync: same budget rule and recursive removal ---
|
||||
source, rsync_dst = _seed_rsync("budget_rsync")
|
||||
|
||||
def inject():
|
||||
@@ -151,26 +148,23 @@ class TestDeleteDelayRefilledDirVsRsync:
|
||||
)
|
||||
t.join()
|
||||
assert rsync_result.returncode == 25, rsync_result.stderr
|
||||
# rsync removes the late content (recursive deferred removal); FastSync
|
||||
# keeps it and charges the snapshot directive instead.
|
||||
assert _deleted_count(rsync_result.stdout) == _deleted_count(result.stdout)
|
||||
assert not os.path.exists(os.path.join(rsync_dst, "xdir", "new.txt")), (
|
||||
"rsync kept late content inside a queued directory"
|
||||
)
|
||||
# The shared observable: the later extra survives in both tools under
|
||||
# --max-delete=1, and both report the capped run with exit 25.
|
||||
assert os.path.isdir(os.path.join(rsync_dst, "b", "ydir")), (
|
||||
"rsync did not bound the deletion with --max-delete=1"
|
||||
)
|
||||
assert os.path.isdir(os.path.join(received, "b", "ydir"))
|
||||
|
||||
@requires_rsync
|
||||
def test_refilled_extra_dir_recursive_removal_residual(self):
|
||||
"""Without --max-delete the residual is a plain tree difference: rsync
|
||||
removes the refilled extra directory (and its late content), FastSync
|
||||
leaves the snapshot path in place on ENOTEMPTY."""
|
||||
def test_refilled_extra_dir_recursive_removal_matches(self):
|
||||
"""Without --max-delete both tools remove the refilled extra directory
|
||||
(and its late content)."""
|
||||
result, received, late = self._fastsync_refilled("recur_fs")
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||
assert os.path.exists(late), "FastSync removed the refilled directory's late content"
|
||||
assert not os.path.exists(late), "FastSync kept the refilled directory's late content"
|
||||
assert not os.path.isdir(os.path.join(received, "xdir"))
|
||||
|
||||
source, rsync_dst = _seed_rsync("recur_rsync")
|
||||
|
||||
|
||||
@@ -320,10 +320,10 @@ class TestDeleteTimingFailure:
|
||||
class TestDeleteDelayDeletedCount:
|
||||
"""The reported deleted count must reflect entries actually removed."""
|
||||
|
||||
def test_refilled_deferred_dir_is_not_counted(self):
|
||||
def test_refilled_deferred_dir_is_recursively_removed_and_counted(self):
|
||||
"""A directory snapshotted into a --delete-delay plan that is refilled
|
||||
before the commit survives ENOTEMPTY and must NOT inflate "Number of
|
||||
deleted files" (regression for delete_plan.c counting at snapshot)."""
|
||||
before the commit is re-scanned and removed recursively (rsync parity):
|
||||
the late file and the directory are both counted as deleted."""
|
||||
source = os.path.join(TEST_DATA_DIR, "ddc_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "ddc_dst")
|
||||
clean_dir(source)
|
||||
@@ -347,12 +347,13 @@ class TestDeleteDelayDeletedCount:
|
||||
proxy.finish()
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:400]
|
||||
assert proxy.hook_called.is_set(), "hook never fired"
|
||||
assert os.path.exists(os.path.join(extra_dir, "new.txt")), "late file vanished"
|
||||
assert not os.path.exists(os.path.join(extra_dir, "new.txt")), "late file survived"
|
||||
assert not os.path.isdir(extra_dir), "refilled extra dir survived"
|
||||
deleted = None
|
||||
for line in result.stdout.splitlines():
|
||||
if line.startswith("Number of deleted files:"):
|
||||
deleted = int(line.split(":", 1)[1].split()[0])
|
||||
assert deleted == 0, (deleted, result.stdout)
|
||||
assert deleted == 2, (deleted, result.stdout)
|
||||
|
||||
|
||||
class TestDeleteDelayMaxDeleteParity:
|
||||
@@ -468,10 +469,9 @@ class TestDeleteAfterThreadsKeepSet:
|
||||
)
|
||||
|
||||
class TestDeleteDelayMaxDeleteRefilledDir:
|
||||
"""--delete-delay charges the --max-delete budget at plan/snapshot time, so a
|
||||
refilled snapshotted directory that survives ENOTEMPTY still spends its slot
|
||||
and a later extra is skipped, while the reported count stays at actual
|
||||
removals.
|
||||
"""--delete-delay charges the --max-delete budget on ACTUAL removals: the
|
||||
refilled directory's late content is removed first (consuming the one slot),
|
||||
so the directory itself and a later extra are skipped, matching rsync.
|
||||
|
||||
The refilled directory is at the destination ROOT (its plan is always sent
|
||||
first) and the skipped extra is under a separate source directory, so the
|
||||
@@ -479,7 +479,7 @@ class TestDeleteDelayMaxDeleteRefilledDir:
|
||||
order. The refill is injected through the byte-barrier proxy so it is
|
||||
causally after the plan frame."""
|
||||
|
||||
def test_budget_charged_at_plan_time(self):
|
||||
def test_budget_charged_on_actual_removal(self):
|
||||
source = os.path.join(TEST_DATA_DIR, "ddmb_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "ddmb_dst")
|
||||
clean_dir(source)
|
||||
@@ -504,9 +504,9 @@ class TestDeleteDelayMaxDeleteRefilledDir:
|
||||
proxy.finish()
|
||||
assert result.returncode == 25, (result.stderr or result.stdout)[:400]
|
||||
assert proxy.hook_called.is_set(), "hook never fired"
|
||||
# The refilled directory still consumes the plan-time budget, so the
|
||||
# later extra is skipped...
|
||||
assert os.path.exists(os.path.join(refilled_dir, "new.txt")), "late file vanished"
|
||||
assert os.path.isdir(later_dir), "later extra was not skipped by the plan-time budget"
|
||||
# ...while the reported count reflects only actual removals (none here).
|
||||
assert _deleted_count(result.stdout) == 0, result.stdout
|
||||
# The late content consumes the single budget slot; the refilled
|
||||
# directory itself and the later extra are skipped.
|
||||
assert not os.path.exists(os.path.join(refilled_dir, "new.txt")), "late file survived"
|
||||
assert os.path.isdir(later_dir), "later extra was not skipped by the budget"
|
||||
# The one actual removal is reported.
|
||||
assert _deleted_count(result.stdout) == 1, result.stdout
|
||||
|
||||
@@ -584,53 +584,53 @@ class TestRemoteDryRun:
|
||||
assert _snapshot_tree(received) == before, f"{flags} mutated the destination"
|
||||
|
||||
@pytest.mark.skipif(shutil.which("rsync") is None, reason="rsync not installed")
|
||||
def test_dry_run_delete_lines_over_report_residual(self):
|
||||
"""Documented residual (RSYNC_COMPAT.md `-n/--dry-run` row): FastSync's
|
||||
dry-run would-delete report includes the file that is merely being
|
||||
updated (derived from the receiver's STATUS_STATS extras) and, unlike
|
||||
rsync, also reports an excluded-but-protected extra. rsync `-n -i
|
||||
--delete` lists only genuine extras. Pins the residual that keeps the
|
||||
row Divergent."""
|
||||
def test_dry_run_delete_lines_match_rsync(self):
|
||||
"""`-n --delete` lists exactly the destination extras rsync would remove.
|
||||
|
||||
Covers the three cases that a real run protects: the file being updated
|
||||
(in the keep set), a filter-excluded source entry (protected prefix), and
|
||||
a --max-size-pruned source entry (always-protected prefix). Only the
|
||||
genuine destination-only extras may appear. Residual: a destination-only
|
||||
entry matching an exclude pattern is still removed (FastSync derives
|
||||
delete protection from the source scan, not a receiver filter engine);
|
||||
that divergence is pinned by TestOptionParity.
|
||||
"""
|
||||
source = os.path.join(TEST_DATA_DIR, "dryrep_src")
|
||||
rdst = os.path.join(TEST_DATA_DIR, "dryrep_rdst")
|
||||
fdst = os.path.join(TEST_DATA_DIR, "dryrep_fdst")
|
||||
clean_dir(source)
|
||||
clean_dir(rdst)
|
||||
clean_dir(fdst)
|
||||
with open(os.path.join(source, "a.txt"), "wb") as fh:
|
||||
fh.write(b"new content\n")
|
||||
for name, data in (("a.txt", b"new content\n"), ("keep.log", b"log\n"),
|
||||
("big.bin", b"B" * 2000)):
|
||||
with open(os.path.join(source, name), "wb") as fh:
|
||||
fh.write(data)
|
||||
os.utime(os.path.join(source, "a.txt"), (1_700_000_000, 1_700_000_000))
|
||||
for root in (rdst, fdst):
|
||||
with open(os.path.join(root, "a.txt"), "wb") as fh:
|
||||
fh.write(b"old\n")
|
||||
for name, data in (("extra.log", b"log\n"), ("extra.txt", b"extra\n")):
|
||||
received = get_dest_received_dir(fdst, source)
|
||||
os.makedirs(received, exist_ok=True)
|
||||
for root in (rdst, received):
|
||||
for name, data in (("a.txt", b"old\n"), ("keep.log", b"log\n"),
|
||||
("big.bin", b"B" * 2000), ("extra.txt", b"extra\n")):
|
||||
with open(os.path.join(root, name), "wb") as fh:
|
||||
fh.write(data)
|
||||
for p in (os.path.join(root, "a.txt"), os.path.join(root, "extra.log"),
|
||||
os.path.join(root, "extra.txt")):
|
||||
os.utime(p, (1_500_000_000, 1_500_000_000))
|
||||
os.utime(os.path.join(root, name), (1_500_000_000, 1_500_000_000))
|
||||
|
||||
flags = ["-a", "-n", "-i", "--delete", "--exclude=*.log", "--max-size=1000"]
|
||||
r = subprocess.run(["rsync", "-an", "-i", "--delete", "--exclude=*.log",
|
||||
source + "/", rdst + "/"],
|
||||
"--max-size=1000", source + "/", rdst + "/"],
|
||||
capture_output=True, text=True,
|
||||
env=dict(os.environ, LC_ALL="C"))
|
||||
assert r.returncode == 0, r.stderr
|
||||
rsync_del = {l.split(None, 1)[1] for l in r.stdout.splitlines()
|
||||
if l.startswith("*deleting")}
|
||||
assert rsync_del == {"extra.txt"}, f"unexpected rsync deleting set: {rsync_del}"
|
||||
rsync_del = sorted(l for l in r.stdout.splitlines() if l.startswith("*deleting"))
|
||||
assert rsync_del == ["*deleting extra.txt"], f"unexpected rsync set: {rsync_del}"
|
||||
|
||||
with ServerManager() as server:
|
||||
server.start(extra_args=["--allow-delete"])
|
||||
result, _ = run_client(source, fdst,
|
||||
flags=["-a", "-n", "-i", "--delete", "--exclude=*.log"],
|
||||
port=server.port)
|
||||
result, _ = run_client(source, fdst, flags=flags, port=server.port)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||
fs_del = {l.split(None, 1)[1] for l in (result.stdout or "").splitlines()
|
||||
if l.startswith("*deleting")}
|
||||
# Documented over-report: the transferred/updated file and the excluded
|
||||
# extra appear in FastSync's would-delete set.
|
||||
assert "a.txt" in fs_del, "residual changed: FastSync no longer over-reports the update"
|
||||
assert "extra.log" in fs_del, "residual changed: FastSync no longer reports excluded extra"
|
||||
fs_del = sorted(l for l in (result.stdout or "").splitlines()
|
||||
if l.startswith("*deleting"))
|
||||
assert fs_del == rsync_del, f"rsync={rsync_del}\nfastsync={fs_del}"
|
||||
|
||||
@pytest.mark.ci
|
||||
def test_remote_dry_run_quiet_is_silent(self, shared_server):
|
||||
|
||||
@@ -164,6 +164,64 @@ class TestInfoParity:
|
||||
f"rsync={entries(rsync_result.stdout)} fastsync={entries(result.stdout)}"
|
||||
)
|
||||
|
||||
@requires_rsync
|
||||
@pytest.mark.ci
|
||||
def test_info_name_root_line_matches_rsync(self, shared_server):
|
||||
"""A fresh destination: rsync prints `created directory`, then the
|
||||
transfer-root `./` name line before the entries; FastSync must emit the
|
||||
same `./` line."""
|
||||
source = os.path.join(TEST_DATA_DIR, "inf_root_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "inf_root_dst")
|
||||
rdst = os.path.join(TEST_DATA_DIR, "inf_root_rdst")
|
||||
clean_dir(source)
|
||||
_write(os.path.join(source, "f.bin"), b"payload\n")
|
||||
clean_dir(dest)
|
||||
shutil.rmtree(rdst, ignore_errors=True)
|
||||
rsync_result = _rsync(["-a", "--info=name", source + "/", rdst + "/"])
|
||||
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||
result, _ = run_client(source, dest, flags=["-a", "--info=name"],
|
||||
port=shared_server.port)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:200]
|
||||
|
||||
def names(text):
|
||||
return [l for l in text.splitlines()
|
||||
if l and not l.startswith("created directory")
|
||||
and not (l.endswith("/") and l != "./")]
|
||||
|
||||
assert names(rsync_result.stdout) == ["./", "f.bin"], names(rsync_result.stdout)
|
||||
assert names(result.stdout) == ["./", "f.bin"], names(result.stdout)
|
||||
|
||||
@requires_rsync
|
||||
@pytest.mark.ci
|
||||
def test_info_name2_uptodate_matches_rsync(self, shared_server):
|
||||
"""--info=name2 prints `NAME is uptodate` for entries the receiver
|
||||
already has, matching rsync byte-for-byte."""
|
||||
source = os.path.join(TEST_DATA_DIR, "inf_up_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "inf_up_dst")
|
||||
rdst = os.path.join(TEST_DATA_DIR, "inf_up_rdst")
|
||||
clean_dir(source)
|
||||
os.makedirs(os.path.join(source, "sub"))
|
||||
_write(os.path.join(source, "a.txt"), b"a\n")
|
||||
_write(os.path.join(source, "sub", "b.txt"), b"b\n")
|
||||
clean_dir(rdst)
|
||||
assert _rsync(["-a", source + "/", rdst + "/"]).returncode == 0
|
||||
rsync_result = _rsync(["-a", "--info=name2", source + "/", rdst + "/"])
|
||||
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||
|
||||
clean_dir(dest)
|
||||
seed, _ = run_client(source, dest, flags=["-a", "--incremental"],
|
||||
port=shared_server.port)
|
||||
assert seed.returncode == 0, (seed.stderr or seed.stdout)[:200]
|
||||
result, _ = run_client(source, dest, flags=["-a", "--incremental", "--info=name2"],
|
||||
port=shared_server.port)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:200]
|
||||
rsync_lines = sorted(l for l in rsync_result.stdout.splitlines()
|
||||
if l.endswith("is uptodate"))
|
||||
fast_lines = sorted(l for l in result.stdout.splitlines()
|
||||
if l.endswith("is uptodate"))
|
||||
assert fast_lines == rsync_lines, (rsync_lines, fast_lines)
|
||||
assert fast_lines == ["a.txt is uptodate", "sub/b.txt is uptodate"], fast_lines
|
||||
|
||||
@requires_rsync
|
||||
@pytest.mark.ci
|
||||
def test_info_nonreg_matches_rsync(self, shared_server):
|
||||
|
||||
Reference in New Issue
Block a user