feat(parity): rsync parity tracks 1-6 (protocol 2.28.0) #303

Merged
TapTap merged 14 commits from feat/parity-2.28 into dev 2026-09-19 17:14:37 +02:00
5 changed files with 151 additions and 8 deletions
Showing only changes of commit 711b7e50b3 - Show all commits
+18 -1
View File
@@ -75,7 +75,8 @@
heuristic with a 10× size window, not rsync's matcher), but its residual is the heuristic with a 10× size window, not rsync's matcher), but its residual is the
candidate-selection heuristic itself: the final tree is byte-exact by design, so candidate-selection heuristic itself: the final tree is byte-exact by design, so
it is pinned by the `TestFuzzy` threshold suite rather than a byte-level rsync it is pinned by the `TestFuzzy` threshold suite rather than a byte-level rsync
differential. The parity-review pass then moved `--delete-delay` to ⚠️ (the differential. (Track 5b later found the name heuristic is rsync's own and moved
the row ❌ → ⚠️, leaving only the narrower delta size window; see entry 15.) The parity-review pass then moved `--delete-delay` to ⚠️ (the
plan-time `--max-delete` charge and non-recursive deferred removal differ from plan-time `--max-delete` charge and non-recursive deferred removal differ from
rsync when a snapshotted entry fails removal). Differential-gate allowlist rsync when a snapshotted entry fails removal). Differential-gate allowlist
entries `min_size`/`empty_dirs_recursive`/`dirs_plain` were removed. The entries `min_size`/`empty_dirs_recursive`/`dirs_plain` were removed. The
@@ -157,6 +158,22 @@
resolution base and over-limit MISS refusal): matrix now resolution base and over-limit MISS refusal): matrix now
**116 ✅ / 13 ⚠️ / 28 ❌ = 157**. **116 ✅ / 13 ⚠️ / 28 ❌ = 157**.
15. **No-wire parity track 5b** on `feat/parity-2.28` (`PROTOCOL_VERSION` stays
`2.28.0` by project decision): `-y`/`--fuzzy` reclassified ❌ → ⚠️. A probe
against real rsync 3.4.1 (pinned `-B8192`, repeated-content 64 KiB corpus)
showed the name heuristic is already rsync's (`util1.c fuzzy_distance` /
`find_filename_suffix` + the exact size+mtime pass) and the output is always
byte-exact; the only residual is candidate ELIGIBILITY, because FastSync's
`delta_should_attempt` gate caps the size ratio at 10× and requires both
files ≥ 16 KiB while rsync will reuse a basis from 0.25× to 10000× and below
16 KiB. The choice is observable only as `--stats` bandwidth counters. Added
differential case `fuzzy_basis` (same-suffix sibling, one name edit,
identical content, block size pinned) asserting tree **and** normalized
`--stats` parity where the choices coincide, plus `TestFuzzy` pinning the
window boundary on both sides (>10× and <16 KiB siblings declined by
FastSync while rsync uses them, both trees byte-identical). Matrix now
**116 ✅ / 14 ⚠️ / 27 ❌ = 157**.
## Next steps ## Next steps
1. **Merge PR #284** (`dev` -> `main`) once reviewed (protected branch). 1. **Merge PR #284** (`dev` -> `main`) once reviewed (protected branch).
2. **Deferred security items** (documented, not implemented): 2. **Deferred security items** (documented, not implemented):
+12 -7
View File
File diff suppressed because one or more lines are too long
+13
View File
@@ -229,6 +229,18 @@ def corpus_iconv(root: str) -> None:
os.utime(full, (_SRC_MTIME, _SRC_MTIME)) os.utime(full, (_SRC_MTIME, _SRC_MTIME))
# Payload for the --fuzzy basis corpus: large enough for the delta engine's
# 16 KiB minimum and with repeated content so a coinciding basis yields a
# non-zero (and identical) Matched data count in both tools.
FUZZY_PAYLOAD = (b"the quick brown fox jumps over the lazy dog\n" * 2000)[:65536]
def corpus_fuzzy(root: str) -> None:
"""A named regular file; the `fuzzy` seed adds the similar-suffix sibling."""
clean_dir(root)
_write(os.path.join(root, "report_v2.txt"), FUZZY_PAYLOAD)
CORPORA: Dict[str, Callable[[str], None]] = { CORPORA: Dict[str, Callable[[str], None]] = {
"basic": corpus_basic, "basic": corpus_basic,
"unicode": corpus_unicode, "unicode": corpus_unicode,
@@ -240,6 +252,7 @@ CORPORA: Dict[str, Callable[[str], None]] = {
"multidir": corpus_multidir, "multidir": corpus_multidir,
"relative": corpus_relative, "relative": corpus_relative,
"iconv": corpus_iconv, "iconv": corpus_iconv,
"fuzzy": corpus_fuzzy,
} }
@@ -133,6 +133,16 @@ def seed_max_delete(_src, rroot, froot):
_mk(os.path.join(root, "extra2.txt"), b"e2\n", _OLD_MTIME) _mk(os.path.join(root, "extra2.txt"), b"e2\n", _OLD_MTIME)
def fuzzy_basis_seed(_src, rroot, froot):
"""Seed a same-suffix sibling whose name is one edit from the source and
whose content matches it, with a DIFFERENT mtime so rsync's exact
size+mtime pass cannot fire: both tools must select it via the
name-distance pass. Where the two tools' basis choices coincide the
block-level results are identical when the block size is pinned."""
for root in (rroot, froot):
_mk(os.path.join(root, "report_v1.txt"), H.FUZZY_PAYLOAD, _OLD_MTIME)
def max_delete_count_check(_src, rroot, froot, _rs, _fs): def max_delete_count_check(_src, rroot, froot, _rs, _fs):
"""The exact survivor set is order-dependent; the count must still match.""" """The exact survivor set is order-dependent; the count must still match."""
r = H.snapshot(rroot) r = H.snapshot(rroot)
@@ -208,6 +218,18 @@ _CASES = [
H.Case("chmod", "basic", ["-a", "--chmod=Fu+rwx"], compare_modes=True, H.Case("chmod", "basic", ["-a", "--chmod=Fu+rwx"], compare_modes=True,
ci=True, ref="--chmod"), ci=True, ref="--chmod"),
# --- delta / similar-file basis (--fuzzy) -----------------------------
# Basis choices coincide here (same-suffix sibling, name distance one edit,
# content identical); with the block size pinned both tools report the same
# Matched/Literal/transferred counters. The residual (FastSync's narrower
# delta size window) is covered by TestFuzzy in test_parity_quickwins.py.
H.Case("fuzzy_basis", "fuzzy",
["-a", "--no-whole-file", "--fuzzy", "--stats", "-B8192"],
fastsync_flags=["-a", "--incremental", "--delta", "--fuzzy",
"--stats", "--delta-block=8192"],
seed=fuzzy_basis_seed, stdout=H.STDOUT_STATS, ci=True,
ref="-y/--fuzzy similar-file basis"),
# --- deletion --------------------------------------------------------- # --- deletion ---------------------------------------------------------
H.Case("delete", "basic", ["-a", "--delete"], seed=seed_extras, H.Case("delete", "basic", ["-a", "--delete"], seed=seed_extras,
server_args=DELETE, ci=True, ref="--delete"), server_args=DELETE, ci=True, ref="--delete"),
@@ -847,6 +847,92 @@ class TestVerifyAndFlip:
"--verify-basis must reject the same-size/different-content basis" "--verify-basis must reject the same-size/different-content basis"
def _stat_bytes(output, key):
"""Parse a --stats byte counter (e.g. ``Matched data: 65,536 bytes``)."""
for line in output.splitlines():
if line.startswith(key + ":"):
raw = line.split(":", 1)[1].strip().split()[0]
return int(raw.replace(",", ""))
return None
class TestFuzzy:
"""Track 5b: `-y`/`--fuzzy` is an internal bandwidth optimization with a
byte-exact result. FastSync ports rsync 3.4.1's weighted-Levenshtein name
heuristic, so where both delta engines admit the candidate the tools pick
the same basis (the ``fuzzy_basis`` differential asserts the tree and the
Matched/Literal counters match with the block size pinned). The residual is
candidate ELIGIBILITY: FastSync's delta size gate (both files >= 16 KiB and
a <= 10x size ratio) is narrower than rsync's, which empirically uses a
fuzzy basis well beyond 10x and below 16 KiB. These tests pin the window
boundary and prove the byte-exact fallback on both sides of it."""
_BASE = b"the quick brown fox jumps over the lazy dog\n" * 4000
def _src(self, tag):
source = os.path.join(TEST_DATA_DIR, f"fz_{tag}_src")
clean_dir(source)
return source
def _dst(self, tag):
d = os.path.join(TEST_DATA_DIR, f"fz_{tag}_dst")
clean_dir(d)
return d
def _run_both(self, shared_server, source, dest, rdst, payload, sibling,
rs_extra=(), fs_extra=()):
with open(os.path.join(source, "report_v2.txt"), "wb") as fh:
fh.write(payload)
for root in (rdst, get_dest_received_dir(dest, source)):
os.makedirs(root, exist_ok=True)
with open(os.path.join(root, "report_v1.txt"), "wb") as fh:
fh.write(sibling)
rs = _rsync(["-a", "--no-whole-file", "--fuzzy", "--stats"] +
list(rs_extra) + [source + "/", rdst + "/"])
assert rs.returncode == 0, rs.stderr[:300]
result, _ = run_client(
source, dest,
flags=["-a", "--incremental", "--delta", "--fuzzy", "--stats"] +
list(fs_extra),
port=shared_server.port)
assert result.returncode == 0, result.stderr[:300]
_assert_same_tree(rdst, get_dest_received_dir(dest, source), "(--fuzzy)")
return rs, result
@requires_rsync
def test_fuzzy_above_size_window_declines_but_tree_exact(self, shared_server):
"""A sibling >10x the source is used by rsync but declined by FastSync's
delta size-ratio gate; both destinations stay byte-identical."""
n = 65536
payload = (self._BASE * ((n // len(self._BASE)) + 1))[:n]
sibling = (self._BASE * 200)[: n * 20]
source, dest, rdst = (self._src("big"), self._dst("big"),
self._dst("big_r"))
rs, result = self._run_both(shared_server, source, dest, rdst,
payload, sibling)
assert _stat_bytes(rs.stdout, "Matched data") > 0, \
"rsync should still use a >10x fuzzy basis"
assert _stat_bytes(result.stdout, "Matched data") == 0, \
"FastSync's 10x delta size-ratio gate must decline the oversized basis"
assert _stat_bytes(result.stdout, "Literal data") == n
@requires_rsync
def test_fuzzy_below_delta_minimum_declines_but_tree_exact(self, shared_server):
"""A sibling below the 16 KiB delta minimum is used by rsync but never
enters FastSync's delta/fuzzy path; both trees stay byte-identical."""
n = 8192
payload = (self._BASE * ((n // len(self._BASE)) + 1))[:n]
source, dest, rdst = (self._src("small"), self._dst("small"),
self._dst("small_r"))
rs, result = self._run_both(shared_server, source, dest, rdst,
payload, payload)
assert _stat_bytes(rs.stdout, "Matched data") > 0, \
"rsync applies --fuzzy below 16 KiB"
assert _stat_bytes(result.stdout, "Matched data") == 0, \
"FastSync's 16 KiB delta minimum must bypass the fuzzy basis"
assert _stat_bytes(result.stdout, "Literal data") == n
class TestIgnoreExistingShortCircuit: class TestIgnoreExistingShortCircuit:
"""#9: --ignore-existing is decided by the receiver during the per-file """#9: --ignore-existing is decided by the receiver during the per-file
check, before the sender streams any payload. A large destination file that check, before the sender streams any payload. A large destination file that