feat(stats): receiver-observed created/literal counters (protocol 2.28.0)

- PROTOCOL_VERSION 2.27.0 -> 2.28.0; STATUS_STATS gains literal_bytes and the created reg/dir/link/special counters (golden wire updated) - receiver reports which destination entries it newly created, including implicitly-created parent directories below the logical transfer root, so Number of created files carries rsync's per-type breakdown - Literal data is now exact for a delta transfer (receiver counts the literal fragments it stored) - differential-tested vs rsync 3.4.1 for fresh-create, update and delta
2026-09-19 10:45:23 +02:00
parent 25062352f6
commit 6119e1e75c
21 changed files with 646 additions and 95 deletions
@@ -56,10 +56,13 @@ STDOUT_OUTFMT = "outfmt"
 STDOUT_STATS = "stats"

 # rsync --stats lines that are protocol-independent and must match exactly.
-# Deliberately excluded: the per-type "Number of files"/"Number of created
-# files" breakdown and Total bytes sent/received (documented residual, see the
-# `--stats` row in RSYNC_COMPAT.md).
+# `Number of files` and `Number of created files` carry rsync's per-type
+# breakdown; protocol 2.28.0 reports the receiver-created split over
+# STATUS_STATS.  Deliberately excluded: Total bytes sent/received (protocol
+# framing differs, see the `--stats` row in RSYNC_COMPAT.md).
 STATS_KEYS = (
+    "Number of files",
+    "Number of created files",
    "Number of deleted files",
    "Number of regular files transferred",
    "Total file size",
@@ -443,6 +446,11 @@ def run_differential(  # noqa: PLR0913 (explicit scenario parameters)
        rroot, froot = os.path.join(rdst, rel), get_dest_received_dir(fdst, src)
    else:
        rroot, froot = rdst, get_dest_received_dir(fdst, src)
+    # rsync's destination root always exists (clean_dir created it).  FastSync's
+    # logical transfer root is the mirror path below the destination argument,
+    # so pre-create it too: `Number of created files` counts the root only when
+    # it is genuinely absent, and the two tools must start from the same state.
+    os.makedirs(froot, exist_ok=True)
    if seed:
        seed(src, rroot, froot)

@@ -36,7 +36,7 @@ from common import (  # noqa: E402
    verify_transfer,
 )

-PROTOCOL_VERSION = b"2.27.0"
+PROTOCOL_VERSION = b"2.28.0"
 STATUS_MANIFEST = 5
 STATUS_OK = 0

@@ -289,6 +289,15 @@ def _make_one_file(root, name="f.bin", size=100):
        fh.write(bytes((i * 7 + 3) & 0xFF for i in range(size)))


+def _pick_stats(text, keys):
+    out = {}
+    for line in text.splitlines():
+        for key in keys:
+            if line.startswith(key + ":"):
+                out[key] = line
+    return out
+
+
 class TestWireStatsParity:
    """Wire-counter output parity: --out-format %b/%c/%C, --progress and
    --stats versus real rsync 3.4.1."""
@@ -496,6 +505,9 @@ class TestWireStatsParity:
        _make_one_file(source, "f.bin", 6000)
        clean_dir(dest)
        clean_dir(rdst)
+        # Start both tools from the same state: rsync's destination root exists,
+        # so pre-create FastSync's mirrored logical root as well.
+        os.makedirs(get_dest_received_dir(dest, source), exist_ok=True)
        rsync_result = _rsync(["-a", "--stats", source + "/", rdst + "/"])
        assert rsync_result.returncode == 0, rsync_result.stderr
        flags = ["-a", "--stats"] + (["--threads"] if mt else [])
@@ -526,17 +538,18 @@ class TestWireStatsParity:
    @requires_rsync
    @pytest.mark.ci
    def test_stats_file_count_breakdown_matches_rsync(self, shared_server):
-        """`Number of files` now carries rsync's per-type breakdown: the scanner
-        accounts directory entries (captured for -a/-t/-p) plus reg/link/special
-        from the transfer list.  `Number of created files` still lacks the type
-        breakdown (FastSync cannot tell which entries the receiver newly
-        created), so that residual is pinned separately."""
+        """`Number of files` and `Number of created files` both carry rsync's
+        per-type breakdown (protocol 2.28.0 reports the receiver-created
+        reg/dir/link/special split over STATUS_STATS)."""
        source = os.path.join(TEST_DATA_DIR, "wire_stc_src")
        dest = os.path.join(TEST_DATA_DIR, "wire_stc_dst")
        rdst = os.path.join(TEST_DATA_DIR, "wire_stc_rdst")
        _make_one_file(source, "f.bin", 6000)
        clean_dir(dest)
        clean_dir(rdst)
+        # Start both tools from the same state: rsync's destination root exists,
+        # so pre-create FastSync's mirrored logical root as well.
+        os.makedirs(get_dest_received_dir(dest, source), exist_ok=True)
        rsync_result = _rsync(["-a", "--stats", source + "/", rdst + "/"])
        assert rsync_result.returncode == 0, rsync_result.stderr
        result, _ = run_client(source, dest, flags=["-a", "--stats"],
@@ -556,10 +569,71 @@ class TestWireStatsParity:

        assert re.match(r"Number of files: 2 \(reg: 1, dir: 1\)$", r_files), r_files
        assert r_files == f_files, (r_files, f_files)
-        # rsync always carries the created type breakdown; FastSync prints the
-        # bare transferred-regular count (documented residual).
        assert re.match(r"Number of created files: 1 \(reg: 1\)$", r_created), r_created
-        assert re.fullmatch(r"Number of created files: 1", f_created), f_created
+        assert f_created == r_created, (r_created, f_created)
+
+    @requires_rsync
+    @pytest.mark.ci
+    @pytest.mark.parametrize("mt", [False, True])
+    def test_stats_created_and_literal_fresh_update_delta(self, shared_server, mt):
+        """The receiver-observed counters must match rsync for the three
+        transfer shapes: a fresh create (created breakdown + whole-file literal),
+        an update (created == 0, whole-file literal), and a delta update (only
+        the literal delta fragments are counted, not the whole file)."""
+        source = os.path.join(TEST_DATA_DIR, "wire_stcd_src")
+        dest = os.path.join(TEST_DATA_DIR, "wire_stcd_dst")
+        rdst = os.path.join(TEST_DATA_DIR, "wire_stcd_rdst")
+        clean_dir(source)
+        clean_dir(dest)
+        clean_dir(rdst)
+        os.makedirs(source, exist_ok=True)
+        os.makedirs(get_dest_received_dir(dest, source), exist_ok=True)
+        with open(os.path.join(source, "big.bin"), "wb") as fh:
+            fh.write(bytes(range(256)) * 4096)  # 1 MiB
+        mt_flag = ["--threads"] if mt else []
+
+        def compare(tag):
+            # Pin the delta block size on both ends: rsync's adaptive block size
+            # would otherwise make the literal/matched split non-comparable.
+            rsync_result = _rsync(["-a", "--stats", "--no-whole-file", "-B8192",
+                                   source + "/", rdst + "/"])
+            assert rsync_result.returncode == 0, rsync_result.stderr
+            result, _ = run_client(
+                source, dest,
+                flags=["-a", "--stats", "--incremental", "--delta", "-B8192"] + mt_flag,
+                port=shared_server.port)
+            assert result.returncode == 0, result.stderr[:300]
+            keys = ("Number of created files", "Literal data", "Matched data",
+                    "Total transferred file size")
+            r = _pick_stats(rsync_result.stdout, keys)
+            f = _pick_stats(result.stdout, keys)
+            assert r == f, f"{tag}: rsync={r} fastsync={f}"
+            return r
+
+        fresh = compare("fresh")
+        assert re.match(r"Number of created files: 1 \(reg: 1\)$",
+                        fresh["Number of created files"]), fresh
+
+        # Update the source and re-run: the destination already exists.
+        sleep_mtime = os.path.getmtime(os.path.join(source, "big.bin")) + 2
+        with open(os.path.join(source, "big.bin"), "r+b") as fh:
+            fh.seek(100)
+            fh.write(b"XXXXXXXXXX")
+        os.utime(os.path.join(source, "big.bin"), (sleep_mtime, sleep_mtime))
+        update = compare("update")
+        assert update["Number of created files"] == "Number of created files: 0", update
+
+        # Second delta update: change bytes far apart, so rsync ships only the
+        # literal fragments and FastSync must report the same Literal data.
+        sleep_mtime = os.path.getmtime(os.path.join(source, "big.bin")) + 2
+        with open(os.path.join(source, "big.bin"), "r+b") as fh:
+            fh.seek(500000)
+            fh.write(b"YYYYYYYYYY")
+        os.utime(os.path.join(source, "big.bin"), (sleep_mtime, sleep_mtime))
+        delta = compare("delta")
+        assert delta["Number of created files"] == "Number of created files: 0", delta
+        lit = int(delta["Literal data"].split(":", 1)[1].strip().split()[0].replace(",", ""))
+        assert 0 < lit < 1024 * 1024, delta

    @requires_rsync
    @pytest.mark.ci
@@ -94,14 +94,14 @@ def _seed_protocol_source(source):
 class TestProtocol:
    @pytest.mark.ci
    def test_protocol_current_version_accepted(self, shared_server):
-        """--protocol=2.27.0 (the current PROTOCOL_VERSION) is accepted and the
+        """--protocol=2.28.0 (the current PROTOCOL_VERSION) is accepted and the
        transfer completes normally."""
        source = os.path.join(TEST_DATA_DIR, "proto_ok_src")
        dest = os.path.join(TEST_DATA_DIR, "proto_ok_dst")
        shutil.rmtree(dest, ignore_errors=True)
        os.makedirs(dest)
        _seed_protocol_source(source)
-        result, _ = run_client(source, dest, flags=["--protocol=2.27.0"],
+        result, _ = run_client(source, dest, flags=["--protocol=2.28.0"],
                               port=shared_server.port)
        assert result.returncode == 0, \
            f"--protocol current run failed: {(result.stderr or result.stdout)[:400]}"