From c9f94ea46e51cf6810449a9dd2c9dce10eb45e3d Mon Sep 17 00:00:00 2001 From: TapTap Date: Sat, 19 Sep 2026 13:23:02 +0200 Subject: [PATCH] docs(parity): --checksum-choice observable-equivalent to rsync; reclassify Probe shows the block-checksum choice is not observable in the parity surface: %c, Matched/Literal data and the destination tree are invariant across xxh64/xxh128/xxh3/md5/md4/sha1 and the transfer,pre-transfer form; only %C changes, and it is byte-identical to rsync. FastSync's fixed xxHash32 block strong sum is collision-safe within the payload cap. Row -> parity (114/11/32). --- RSYNC_COMPAT.md | 8 +- tests/integration/test_codecs.py | 149 +++++++++++++++++++++++++++++++ 2 files changed, 154 insertions(+), 3 deletions(-) diff --git a/RSYNC_COMPAT.md b/RSYNC_COMPAT.md index 93e30ee..ac79895 100644 --- a/RSYNC_COMPAT.md +++ b/RSYNC_COMPAT.md @@ -6,8 +6,8 @@ This document maps rsync's full feature set to FastSync's current implementation | Status | Count | Description | |--------|-------|-------------| -| ✅ Parity | 113 | Reproduces rsync's semantics for this option's scope | -| ⚠️ Caveat | 12 | Wired and tested, but carries a documented behavioral difference from rsync (named in the row and/or the wave notes) | +| ✅ Parity | 114 | Reproduces rsync's semantics for this option's scope | +| ⚠️ Caveat | 11 | Wired and tested, but carries a documented behavioral difference from rsync (named in the row and/or the wave notes) | | ❌ Divergent | 32 | Rejected, an accepted no-op, deliberately non-rsync (native config/auth/batch, privileged namespaces, safe-subset privilege), or impossible on any portable filesystem call | | **Total** | **157** | One row per rsync option/feature group; a row may name several spellings | @@ -43,6 +43,8 @@ matrix is **111 ✅ / 13 ⚠️ / 33 ❌ = 157**. **Parity track 3a (no-wire, on `feat/parity-2.28`).** Three codec refinements, none of which change the wire layout (`PROTOCOL_VERSION` stays 2.28.0): (1) an omitted `--compress-level` now resolves to rsync 3.4.1's per-codec default (zstd 3, zlib/zlibx 6, lz4 ignored) and an explicit level is clamped per codec (zstd 1-22, zlib/zlibx 1-9), verified against `rsync --debug=NSTR1`; (2) `auto` now honors `RSYNC_COMPRESS_LIST`/`RSYNC_CHECKSUM_LIST` (rsync's whitespace-separated preference syntax, unknown names skipped, first supported wins, all-unknown is exit 4) before the compiled-in order, with an explicit `--zc`/`--cc` still winning; and (3) `--compress-choice=zlibx` is reclassified out of the caveat list — FastSync's zlib stream already carries only the literal/delta bytes (rsync's zlibx semantics) and is observably identical to `--zc=zlib`, so the zlib/zlibx aliasing is only an implementation detail. The matrix is now **113 ✅ / 12 ⚠️ / 32 ❌ = 157**. +**Parity track 3b (no-wire, on `feat/parity-2.28`).** `--checksum-choice`/`--cc` is reclassified out of the caveat list: its only documented residual was the delta BLOCK strong checksum (rsync applies the negotiated algorithm there; FastSync keeps a fixed xxHash32, `DeltaBlockSig`). A pre-seeded delta-transfer differential against rsync 3.4.1 (probe: `--no-whole-file -B8192 --stats --out-format=%c|%C %n` for rsync vs `--incremental --delta` for FastSync) shows the choice is **not observable** in the surface the parity gate compares — across `xxh64`/`xxh128`/`xxh3`/`md5`/`md4`/`sha1` and both two-name orders the destination tree is byte-identical, the `Matched data`/`Literal data`/`Total transferred file size` counters are unchanged (and, with the block size pinned, equal to rsync's), the `%c` block-checksum token is invariant, and the exit code is 0. The negotiated algorithm is only visible in `%C`, which renders the whole-file transfer digest and is already byte-identical to rsync. No wire field is added (`PROTOCOL_VERSION` stays 2.28.0). The matrix is now **114 ✅ / 11 ⚠️ / 32 ❌ = 157**. + **Parity completion wave (protocol 2.23.0 → 2.26.0).** This wave closed the remaining gaps the rsync-parity wave left open (delete timing, wire counters and output, codec breadth, general `-R`/`-d`, the full filter grammar, receiver-side @@ -651,7 +653,7 @@ targets verbatim, matching rsync. | Flag | Rsync Description | FastSync Status | Notes | |------|-------------------|-----------------|-------| | `--checksum` | Skip based on checksum | ✅ Parity | `-c`/`--checksum` compares per-file whole-file content digests to skip unchanged files. **As of protocol 2.23.0 the short `-c` implies the checksum quick-check**, so a plain `-c` run verifies content rather than only affecting the `--incremental` handshake. The digest algorithm is `xxh128` by default (protocol 2.26.0's negotiated default) and is selectable via `--checksum-choice`/`--cc` (`xxh128`/`xxh3`/`xxh64`/`xxhash`/`md5`/`md4`/`sha1`/`none`/`auto`, plus rsync's two-name form) and `--checksum-seed=NUM` (see those rows) | -| `--checksum-choice=STR`, `--cc=STR` | Choose checksum algorithm | ⚠️ Caveat | Real algorithm selection for the per-file whole-file digest used by the `--incremental`/`--checksum` handshake and basis-dir verification. **Protocol 2.26.0 accepts rsync 3.4.1's full set** — `xxh128` (the negotiated default), `xxh3`, `xxh64`, `xxhash`, `md5`, `md4`, `sha1`, `none`, `auto`, and the two-name `transfer,pre-transfer` form — with rsync's exit-4 rejection of an unknown name and of `none` on the transfer side when `--checksum` is on. `--cc=ALG` and space forms both parse. The algorithm id and seed cross the wire; the receiver hashes its old file with the same algorithm+seed and the per-file `STATUS_CHECK` handshake carries a bounded digest pinned to the negotiated length. `checksum_digest_file` now streams **every** supported algorithm (md4 via the self-contained RFC 1320 code, sha1/md5 via EVP, none as an empty digest), so the streaming path matches its contract, and `--out-format %C` uses the selected **transfer** half of a two-name choice and renders each algorithm byte-for-byte like rsync (xxh128 high-then-low, xxh64/xxh3 big-endian, md5/md4/sha1 standard hex, none a blank 2-char column) — differential-tested across all algorithms. **Remaining divergence:** rsync uses this choice for the block checksum on the wire too, while FastSync selects only the whole-file comparison digest and keeps the delta BLOCK strong checksum at xxHash32. `auto` now consults `RSYNC_CHECKSUM_LIST` (rsync's whitespace-separated preference list; unknown names skipped, first supported wins, all-unknown is exit 4) before the compiled-in order; because both peers run the identical build this deterministic resolution needs no rsync peer probe, and an explicit `--cc` still wins. The list is differential-tested through `--out-format %C` (byte-identical digests to rsync for md5/sha1/xxh3) | +| `--checksum-choice=STR`, `--cc=STR` | Choose checksum algorithm | ✅ Parity | Real algorithm selection for the per-file whole-file digest used by the `--incremental`/`--checksum` handshake and basis-dir verification. **Protocol 2.26.0 accepts rsync 3.4.1's full set** — `xxh128` (the negotiated default), `xxh3`, `xxh64`, `xxhash`, `md5`, `md4`, `sha1`, `none`, `auto`, and the two-name `transfer,pre-transfer` form — with rsync's exit-4 rejection of an unknown name and of `none` on the transfer side when `--checksum` is on. `--cc=ALG` and space forms both parse. The algorithm id and seed cross the wire; the receiver hashes its old file with the same algorithm+seed and the per-file `STATUS_CHECK` handshake carries a bounded digest pinned to the negotiated length. `checksum_digest_file` now streams **every** supported algorithm (md4 via the self-contained RFC 1320 code, sha1/md5 via EVP, none as an empty digest), so the streaming path matches its contract, and `--out-format %C` uses the selected **transfer** half of a two-name choice and renders each algorithm byte-for-byte like rsync (xxh128 high-then-low, xxh64/xxh3 big-endian, md5/md4/sha1 standard hex, none a blank 2-char column) — differential-tested across all algorithms. **Track-3b finding — the block-checksum residual is not observable.** rsync applies the choice to the block checksum on its wire too, while FastSync selects only the whole-file comparison digest and keeps the delta BLOCK strong checksum fixed at xxHash32 (`DeltaBlockSig`, `delta_signature_create_seeded`). Because FastSync does not interoperate with rsync on the wire, only the compared surface matters, and a pre-seeded delta differential against rsync 3.4.1 (`--no-whole-file -B8192 --stats --out-format=%c|%C %n` vs `--incremental --delta`) shows the choice does not move it: across `xxh64`, `xxh128`, `xxh3`, `md5`, `md4`, `sha1` and both two-name orders the destination tree is byte-identical, `Matched data`/`Literal data`/`Total transferred file size` are unchanged (and equal to rsync's with the block size pinned), the `%c` block-checksum token is invariant (rsync `16 + 6·ceil(size/block)`, already documented under `--out-format`; FastSync its own basis-read counter), and the exit code is 0. The negotiated algorithm is visible only in `%C`, which applies it to the whole-file transfer digest and matches rsync byte-for-byte. A false block match would require the 4-byte adler32 AND the 4-byte xxHash32 to collide; at the 256 MiB maximum with the 1 KiB minimum block size the expected false matches are ≤2⁻¹⁸, and the choice cannot change this because FastSync's block strong sum is fixed. `auto` now consults `RSYNC_CHECKSUM_LIST` (rsync's whitespace-separated preference list; unknown names skipped, first supported wins, all-unknown is exit 4) before the compiled-in order; because both peers run the identical build this deterministic resolution needs no rsync peer probe, and an explicit `--cc` still wins. The list is differential-tested through `--out-format %C` (byte-identical digests to rsync for md5/sha1/xxh3) | | `--compare-dest=DIR` | Compare dest files relative to DIR | ❌ Divergent | DIR is a receiver-side basis; protocol 2.26.0 uses an absolute path verbatim (rsync semantics) and resolves a relative path below the destination root (`..` components are rejected, `//` collapsed and trailing `/` dropped). On the receiver's per-file check (implies `--incremental`) an exact match = same size + mtime (unless `--size-only`; `-I` disables matching) **and** equal xxHash64 of the sender's file; a match suppresses the data transfer. compare-dest never copies: it only skips a file the destination does **not** already hold (sparse destination, rsync parity), and is consulted before the normal delta/full paths. Repeatable; searched in command-line order, first match wins. **Reclassified Divergent (differential evidence):** FastSync verifies a basis hit's content with xxHash64 while rsync's `--size-only` quick check trusts size (and mtime) alone, so with a same-size/different-content basis rsync skips/links the *wrong* basis content while FastSync transfers the source — a deliberate safety-stricter behavior that cannot match rsync (see `test_basis_dir_size_only_content_residual` in `tests/integration/test_parity_quickwins.py`). Attribute-only differences on a match are also not re-applied (data is skipped so the sender never sends metadata). Sizing: FastSync's whole-file payload limit is 256 MiB on **every** transfer path (not basis-specific); rsync applies basis dirs to arbitrary sizes, so FastSync refuses a basis run whose source contains a larger file up front with a clear error before any transfer. Wire: a basis-count field is always present on the config frame (protocol 2.9.0, so clients and servers must both be 2.9.0) | | `--copy-dest=DIR` | Include copies of unchanged files | ❌ Divergent | Same basis rules as `--compare-dest`, but an exact match materializes a **local copy** of the DIR file into the destination (via the normal atomic temp+rename store path, so `--existing`/`--ignore-existing`/`--update`/`--backup`/`--delay-updates` all still apply) instead of transferring data. Repeatable; command-line order = priority. Content is xxHash64-verified before the copy. **Reclassified Divergent** for the same basis-hit verification divergence as `--compare-dest`: rsync's `--size-only` size/quick-check match rings a same-size/different-content file as unchanged and copies the wrong basis bytes, while FastSync's xxHash verification transfers the source (differential test `test_basis_dir_size_only_content_residual`); a basis-hit also keeps whatever metadata the copy derived from the basis rather than rsync's "copy + fix attributes" in attribute-only cases. Requires `--incremental` (implied); incompatible with `-s`. Wire: protocol 2.9.0 | | `--link-dest=DIR` | Hardlink to files when unchanged | ❌ Divergent | Same basis rules as `--copy-dest`, but an exact match installs an atomic **hard link** to the DIR file (temp hard link + rename) so no data or disk space is used; where the link is impossible (basis on another filesystem, filesystem refuses links) it falls back cleanly to a byte-identical local copy, never a corrupt/partial file. `--delay-updates` stages the link and publishes by rename, so the final entry stays a real hard link. Repeatable (searched in command-line order, first match wins). Content is xxHash64-verified before linking. **Reclassified Divergent** for the shared basis-hit divergence: with `--size-only` a same-size/different-content basis is linked by rsync (installing wrong content) but FastSync detects the xxHash mismatch and transfers the source (differential test `test_basis_dir_size_only_content_residual`). Other inherent caveats: protocol 2.26.0 re-links an already up-to-date destination file to the basis; a link keeps the basis inode's own mode/uid/gid and mtime — metadata is never written through the shared inode (that would mutate the basis file), so a later `--inplace` run that rewrites such a destination path **will mutate the basis snapshot** through the shared inode (use `--copy-dest` when the destination must stay independently writable); a `--remove-source-files` source satisfied by a basis dir is treated as skipped and therefore **retained** (never removed); basis dirs are excluded from `--delete`. Requires `--incremental` (implied); incompatible with `-s`. Wire: protocol 2.9.0 | diff --git a/tests/integration/test_codecs.py b/tests/integration/test_codecs.py index 0e0dfbe..8c17f0d 100644 --- a/tests/integration/test_codecs.py +++ b/tests/integration/test_codecs.py @@ -9,6 +9,7 @@ directory when the shared test server is launched), so every scratch tree lives under ``TEST_DATA_DIR`` rather than pytest's ``tmp_path``. """ import os +import random import shutil import subprocess import sys @@ -68,6 +69,84 @@ def _tree_bytes(root): return out +_CC_DELTA_T0 = 1_600_000_000 +_CC_DELTA_T1 = 1_600_000_100 + +_DELTA_STATS_KEYS = ( + "Number of created files", + "Number of regular files transferred", + "Total transferred file size", + "Literal data", + "Matched data", +) + + +def _pin_tree(root, mtime): + for dirpath, dirnames, filenames in os.walk(root): + for name in dirnames + filenames: + path = os.path.join(dirpath, name) + if not os.path.islink(path): + os.utime(path, (mtime, mtime)) + os.utime(root, (mtime, mtime)) + + +def _make_delta_basis(src, size=512 * 1024): + """Build a source and a matching pre-modification basis tree. + + The source's ``big.bin`` is then modified in a few disjoint places and given + a newer mtime so both tools take the delta path. Returns the basis dir. + """ + clean_dir(src) + original = random.Random(20240101).randbytes(size) + with open(os.path.join(src, "big.bin"), "wb") as fh: + fh.write(original) + with open(os.path.join(src, "small.txt"), "wb") as fh: + fh.write(b"hello world\n") + _pin_tree(src, _CC_DELTA_T0) + + basis = src.rstrip("/") + "_basis" + clean_dir(basis) + shutil.copy2(os.path.join(src, "big.bin"), os.path.join(basis, "big.bin")) + shutil.copy2(os.path.join(src, "small.txt"), os.path.join(basis, "small.txt")) + _pin_tree(basis, _CC_DELTA_T0) + + modified = bytearray(original) + for off in (0, size // 3, 2 * size // 3, size - 64): + for i in range(32): + modified[off + i] ^= 0x5A + with open(os.path.join(src, "big.bin"), "wb") as fh: + fh.write(bytes(modified)) + os.utime(os.path.join(src, "big.bin"), (_CC_DELTA_T1, _CC_DELTA_T1)) + return basis + + +def _seed_from_basis(basis, target): + clean_dir(target) + for name in os.listdir(basis): + shutil.copy2(os.path.join(basis, name), os.path.join(target, name)) + + +def _delta_stats(text): + found = {} + for line in text.splitlines(): + for key in _DELTA_STATS_KEYS: + if line.startswith(key + ":"): + found[key] = line.split(":", 1)[1].strip() + return found + + +def _big_bin_outfmt(text): + """The ``(c, C)`` pair from the ``big.bin`` out-format line (`%c|%C %n`).""" + for line in text.splitlines(): + stripped = line.strip() + if "|" not in stripped or not stripped.endswith("big.bin"): + continue + c_field, rest = stripped.split("|", 1) + fields = rest.split() + return c_field.strip(), (fields[0] if fields else "") + return None, None + + class TestCodecChoiceMatrix: """The CLI accept/reject set and exit codes must match rsync 3.4.1.""" @@ -191,6 +270,76 @@ class TestCodecTransferDifferential: assert _tree_bytes(received) == _tree_bytes(rsync_dst) +class TestChecksumChoiceDeltaSurface: + """--checksum-choice does not move the delta-transfer parity surface. + + FastSync's delta BLOCK strong checksum is a fixed xxHash32, so the + negotiated algorithm only selects the whole-file comparison digest (and the + ``%C`` transfer digest). A pre-seeded delta transfer must therefore land + byte-identical bytes and report the same counters for every choice, while + ``%C`` -- the one token that tracks the choice -- stays byte-identical to + rsync. This pins the Track-3b reclassification in RSYNC_COMPAT.md. + """ + + CHOICES = ["xxh64", "xxh128", "xxh3", "md5", "md4", "sha1", + "xxh64,sha1", "sha1,xxh64"] + + @requires_rsync + @pytest.mark.ci + def test_delta_surface_invariant_to_checksum_choice(self, shared_server): + src = _scratch("ccdelta_src") + basis = _make_delta_basis(src) + source_bytes = _tree_bytes(src) + + rsync_c, fastsync_c, digests = {}, {}, {} + rsync_stats, fastsync_stats = {}, {} + for choice in self.CHOICES: + tag = choice.replace(",", "_") + rsync_dst = _scratch(f"ccdelta_rs_{tag}") + fs_dst = _scratch(f"ccdelta_fs_{tag}") + fs_root = get_dest_received_dir(fs_dst, src) + _seed_from_basis(basis, rsync_dst) + _seed_from_basis(basis, fs_root) + + # Pin the block size on both ends so the literal/matched split is + # comparable (rsync's adaptive default would otherwise differ from + # FastSync's 8192-byte default). + rsync_result = _rsync(["-a", "--no-whole-file", "-B8192", "--stats", + "--out-format=%c|%C %n", f"--cc={choice}", + src + "/", rsync_dst + "/"]) + assert rsync_result.returncode == 0, rsync_result.stderr + result, _ = run_client( + src, fs_dst, + flags=["-a", "--incremental", "--delta", "-B8192", "--stats", + "--out-format=%c|%C %n", f"--cc={choice}"], + port=shared_server.port) + assert result.returncode == 0, (result.stderr or result.stdout)[:300] + + assert _tree_bytes(rsync_dst) == source_bytes, choice + assert _tree_bytes(fs_root) == source_bytes, choice + assert _delta_stats(rsync_result.stdout) == _delta_stats(result.stdout), choice + + rs_c, rs_C = _big_bin_outfmt(rsync_result.stdout) + fs_c, fs_C = _big_bin_outfmt(result.stdout) + assert rs_C == fs_C, f"{choice}: %C rsync={rs_C!r} fastsync={fs_C!r}" + rsync_c[choice] = rs_c + fastsync_c[choice] = fs_c + digests[choice] = fs_C + rsync_stats[choice] = _delta_stats(rsync_result.stdout) + fastsync_stats[choice] = _delta_stats(result.stdout) + + # The choice is only observable in %C, and it is effective (the digests + # are not all the same algorithm's output). + assert len(set(digests.values())) > 1, digests + # The compared --stats counters are invariant across choices in each tool + # (and were asserted equal cross-tool inside the loop). + assert len({tuple(sorted(s.items())) for s in rsync_stats.values()}) == 1, rsync_stats + assert len({tuple(sorted(s.items())) for s in fastsync_stats.values()}) == 1, fastsync_stats + # The block-checksum token (%c) is invariant across choices in each tool. + assert len(set(rsync_c.values())) == 1, rsync_c + assert len(set(fastsync_c.values())) == 1, fastsync_c + + class TestCodecNegotiationFallback: """FastSync's auto negotiation and deterministic fallback order."""