From 29f8be161c23b7ff18ba563d532cad998ae50a05 Mon Sep 17 00:00:00 2001 From: TapTap Date: Wed, 23 Sep 2026 21:52:40 +0200 Subject: [PATCH] feat(protocol): destination-state itemize for dirs/symlinks; --stats deleted breakdown --- CHANGELOG.md | 16 ++ RSYNC_COMPAT.md | 10 +- src/client/change_list.c | 71 ++++++- src/client/client_cli.c | 5 +- src/client/client_report.c | 149 ++++++++++++++- src/client/client_send.c | 61 +++++- src/client/client_send_internal.h | 8 + src/server/receiver.c | 180 +++++++++++++++--- src/server/receiver.h | 18 +- src/server/server.c | 15 +- src/shared/delete.c | 19 +- src/shared/delete.h | 24 ++- src/shared/delete_commit.c | 6 +- src/shared/delete_plan.c | 19 +- src/shared/format.c | 22 ++- src/shared/format.h | 31 ++- src/shared/protocol.h | 41 ++-- tests/integration/test_differential_parity.py | 152 +++++++++++++++ tests/test_format.c | 10 + tests/test_multiprocessing.c | 3 +- 20 files changed, 761 insertions(+), 99 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 417e701..5afd9cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,22 @@ Wire backlog cycle (protocol 2.29.0 → 2.30.0; config-frame layout unchanged). sends the terminal `STATUS_PARTIAL`; the client exits 23 like rsync and, under `--remove-source-files`, still removes the sources it successfully transferred. A clean run stays 0 and a fatal/connection error stays non-23. +- **Directory/symlink destination-state itemize (#314):** when `report_dest_info` + is negotiated (now also for `--progress`), the receiver answers `STATUS_MKDIR` + and `STATUS_SYMLINK` with the entry's pre-transfer destination snapshot + (existence, type, perms/owner/group/time, and whether an existing symlink's + target already matches), and the sender probes every ancestor directory before + the receiver creates it implicitly. A re-run over an unchanged tree no longer + emits per-directory `cd+++++++++` or unchanged-symlink lines, a changed + directory renders rsync's `.d..t......`, and a changed symlink renders + `cLc........` / `.L..t......`. Directory/symlink time comparison uses whole + seconds (rsync's `cmp_time`). The `STATUS_MKDIR` body gains a probe flag and + the `STATUS_DEST_INFO` record gains a symlink-target-match field; the + config-frame layout is unchanged. Differential-tested against rsync 3.4.1. +- **`--stats` deleted per-type breakdown (#316):** `STATUS_STATS` gains + `deleted_reg/dir/link/special`, tallied by the delete observers and rendered + as rsync's `Number of deleted files: X (reg: A, dir: B, link: C, special: D)`. + Differential-tested against rsync 3.4.1 for a mixed-type `--delete` tree. ## [2.29.0] - 2026-09-23 diff --git a/RSYNC_COMPAT.md b/RSYNC_COMPAT.md index d13ccb6..b0fdc48 100644 --- a/RSYNC_COMPAT.md +++ b/RSYNC_COMPAT.md @@ -121,12 +121,12 @@ Every one of those has an entry below with its remaining caveats. | Flag | Rsync Description | FastSync Status | Notes | |------|-------------------|-----------------|-------| -| `--stats` | Give transfer stats | ⚠️ Caveat | Prints transfer statistics. Protocol 2.25.0 populates the receiver-only counters the sender cannot observe (`Matched data`, `Number of deleted files`) from the receiver's `STATUS_STATS` report; the sender tracks the scanned file list per type so `Number of files` carries rsync's `(reg: X, dir: Y, link: Z, special: W)` breakdown (directories come from the scanner's captured directory list for `-a`/`-t`/`-p`, or from a lightweight traversed-directory counter on a plain `-r` run so the `dir:` category is present there too), `Number of regular files transferred` excludes symlinks/specials and up-to-date files, `Total file size` includes symlink target lengths, and `Total transferred file size` counts only transferred files. **Protocol 2.28.0 extends `STATUS_STATS`** with receiver-observed `literal_bytes` and the four `created_*` counters: `Number of created files` now carries rsync's `(reg/dir/link/special)` breakdown (the receiver reports which destination entries it newly created, including implicitly-created parent directories below the transfer root) and `Literal data` is exact for a delta transfer (the receiver counts the literal fragments it stored, not the whole source size) — all differential-tested in the sequential and `--threads` paths against rsync 3.4.1 for fresh-create, update and delta shapes. **Remaining divergences:** rsync's per-type breakdown on `Number of deleted files` is not reproduced; and `Total bytes sent`/`received` are FastSync wire bytes framed differently from rsync's, so they are not numerically comparable | +| `--stats` | Give transfer stats | ⚠️ Caveat | Prints transfer statistics. Protocol 2.25.0 populates the receiver-only counters the sender cannot observe (`Matched data`, `Number of deleted files`) from the receiver's `STATUS_STATS` report; the sender tracks the scanned file list per type so `Number of files` carries rsync's `(reg: X, dir: Y, link: Z, special: W)` breakdown (directories come from the scanner's captured directory list for `-a`/`-t`/`-p`, or from a lightweight traversed-directory counter on a plain `-r` run so the `dir:` category is present there too), `Number of regular files transferred` excludes symlinks/specials and up-to-date files, `Total file size` includes symlink target lengths, and `Total transferred file size` counts only transferred files. **Protocol 2.28.0 extends `STATUS_STATS`** with receiver-observed `literal_bytes` and the four `created_*` counters: `Number of created files` now carries rsync's `(reg/dir/link/special)` breakdown (the receiver reports which destination entries it newly created, including implicitly-created parent directories below the transfer root) and `Literal data` is exact for a delta transfer (the receiver counts the literal fragments it stored, not the whole source size) — all differential-tested in the sequential and `--threads` paths against rsync 3.4.1 for fresh-create, update and delta shapes. **Protocol 2.30.0 appends the four `deleted_*` counters**: the delete observers classify every entry the receiver ACTUALLY removed (regular/dir/symlink/special, a strict partition of the existing scalar), so `Number of deleted files: X (reg: A, dir: B, link: C, special: D)` matches rsync exactly — differential-tested for a mixed-type `--delete` tree (`test_differential_parity.py::test_stats_deleted_breakdown_matches_rsync`). **Remaining divergence:** `Total bytes sent`/`received` are FastSync wire bytes framed differently from rsync's, so they are not numerically comparable | | `-h`, `--human-readable` | Human-readable numbers | ✅ Parity | Formats transfer byte and rate counts using rsync's **decimal** (base-1000) units, matching rsync `-h` (e.g. `1.23M`), not binary units. **A lone `-h` with no transfer arguments prints help instead** (protocol 2.26.0), matching the rsync idiom; `-h` alongside a transfer remains human-readable | -| `-i`, `--itemize-changes` | Per-file change summary | ⚠️ Caveat | Prints rsync-style itemize lines to stdout for files actually sent (also under `-j`/`--threads`). Directory and transfer-root lines are now emitted too: a run produces rsync's `./` root line and per-directory `cd+++++++++`/`.d..t......` lines, rendered by the shared itemize code. **Residual:** the root `./` line is emitted unconditionally rather than keyed off rsync's root-attribute-change decision; **every** non-root directory is rendered as created (`cd+++++++++`) because the sender never probes a directory's destination state, so a pre-existing destination directory that rsync reports as unchanged (`.d..t......`) is still itemized as created — this is not limited to re-runs; an incremental re-run additionally itemizes directories/symlinks that lack a quick-check where rsync stays silent (unchanged regular files still print nothing, matching single-`-i`); and directory attribute columns (`%M`/`%U`/`%G`) come from the source | -| `--progress` | Show progress | ⚠️ Caveat | Protocol 2.25.0 prints rsync-style per-file progress blocks (percent, transferred/total bytes, rate, elapsed, `(xfr#N, to-chk=M/T)`) fed by the receiver's `STATUS_STATS`, in both the sequential and `--threads` send paths. FastSync also prints rsync's leading `./` transfer-root line and, when progress is requested (`--progress`/`-P`/`--info=progress`) and not `--quiet`, runs a **paths-only metadata pre-scan** (no file reads, no hashing) that supplies rsync's file-list total `T` for the `to-chk` denominator and the directory names; `--delete-during`/`--delete-delay` reuse their existing keep-set pre-scan instead of walking twice, and non-progress runs are untouched. Per-directory name lines are emitted (trailing `/`), and symlink (` -> target`) and special entries are named too, so a **fresh multi-directory tree's name set and `to-chk` denominator match rsync 3.4.1** (differential test, sequential and `--threads`) and a **single-file transfer's name lines and deterministic frames remain byte-identical** to rsync. **Order parity (parity-2.29):** the sequential scanner now emits entries in rsync's sorted depth-first flist order (non-directories ascending, then directories ascending), so the interleaving and the `to-chk` numerator match rsync for the default single-threaded transfer (differential `test_parity_order.py`; `--threads` has no rsync analogue and stays unordered). **Remaining divergences:** the leading `./` root line is emitted unconditionally rather than keyed off rsync's root-attribute-change decision, and an ancestor directory line is emitted whenever a child transfers (rsync suppresses it when the directory itself is unchanged); on a re-run, entries without a quick-check (symlinks, empty directories) are still named where rsync stays silent; and the rate/ETA are wall-clock dependent | +| `-i`, `--itemize-changes` | Per-file change summary | ⚠️ Caveat | Prints rsync-style itemize lines to stdout for files actually sent (also under `-j`/`--threads`). Directory and transfer-root lines are emitted too: a run produces rsync's `./` root line and per-directory `cd+++++++++`/`.d..t......` lines, rendered by the shared itemize code. **Protocol 2.30.0 closes the directory/symlink destination-state residual (#314):** when `report_dest_info` is negotiated (now also for `--progress`) the receiver answers `STATUS_MKDIR`/`STATUS_SYMLINK` with the entry's pre-transfer snapshot — including whether an existing symlink's target already matches — and the sender probes every ancestor directory before the receiver creates it implicitly. A re-run over an unchanged tree therefore emits no per-directory `cd+++++++++` and no unchanged-symlink line (matching rsync), a changed directory renders `.d..t......` (not `cd`), and a changed symlink renders `cLc........` / `.L..t......`. The time column compares whole seconds for directories/symlinks (rsync's `cmp_time`), so a sub-second-only difference is not a spurious `t`. Differential: `test_differential_parity.py::test_itemize_rerun_dirs_symlinks_matches_rsync`. **Residual:** the root `./` line is still emitted unconditionally rather than keyed off rsync's root-attribute-change decision; a directory whose ONLY change is an attribute and which has no transferred child is itemized at the end of the run (the pending-directory flush) rather than in rsync's depth-first position; and directory attribute columns (`%M`/`%U`/`%G`) come from the source | +| `--progress` | Show progress | ⚠️ Caveat | Protocol 2.25.0 prints rsync-style per-file progress blocks (percent, transferred/total bytes, rate, elapsed, `(xfr#N, to-chk=M/T)`) fed by the receiver's `STATUS_STATS`, in both the sequential and `--threads` send paths. FastSync also prints rsync's leading `./` transfer-root line and, when progress is requested (`--progress`/`-P`/`--info=progress`) and not `--quiet`, runs a **paths-only metadata pre-scan** (no file reads, no hashing) that supplies rsync's file-list total `T` for the `to-chk` denominator and the directory names; `--delete-during`/`--delete-delay` reuse their existing keep-set pre-scan instead of walking twice, and non-progress runs are untouched. Per-directory name lines are emitted (trailing `/`), and symlink (` -> target`) and special entries are named too, so a **fresh multi-directory tree's name set and `to-chk` denominator match rsync 3.4.1** (differential test, sequential and `--threads`) and a **single-file transfer's name lines and deterministic frames remain byte-identical** to rsync. **Order parity (parity-2.29):** the sequential scanner now emits entries in rsync's sorted depth-first flist order (non-directories ascending, then directories ascending), so the interleaving and the `to-chk` numerator match rsync for the default single-threaded transfer (differential `test_parity_order.py`; `--threads` has no rsync analogue and stays unordered). **Destination-state suppression (#314, protocol 2.30.0):** a pre-existing directory is no longer named (rsync names a directory only when it creates it) and an unchanged symlink is no longer named, so a re-run over an unchanged tree prints no directory/symlink name lines where rsync stays silent. **Remaining divergences:** the leading `./` root line is emitted unconditionally rather than keyed off rsync's root-attribute-change decision; and the rate/ETA are wall-clock dependent | | `-P` | Same as --partial --progress | ✅ Parity | Parses to `--partial` + `--progress`. The independent `--partial` retention semantics are rsync parity: an interrupted write retains the already-written temp at the destination (best-effort) so a later `--append`/`--append-verify` can resume. Progress presentation is owned by the `--progress` row; there is no separate `-P` divergence | -| `--out-format=FORMAT` | Custom output format | ❌ Divergent | Per-transfer template on stdout; tokens `%f` `%n` `%l` `%b` `%c` `%C` `%i` `%M` `%o` `%U` `%G` `%t` `%%`. `%C` now uses the negotiated transfer algorithm (`--checksum-choice`, default `xxh128`, seed 0) and renders every algorithm exactly like rsync — xxh128 high-then-low, xxh64/xxh3 big-endian, md5/md4/sha1 standard hex, `none` a blank 2-char column — differential-tested across all algorithms. `%f`/`%n`/`%l`/`%i`/`%M`/`%U`/`%G`/`%B` also match. **Reclassified because `%b`/`%c` are protocol-specific and cannot match:** a differential against rsync 3.4.1 shows whole-file `%c = 16` for both, but rsync whole-file `%b = filesize + 27 + transfer-digest-bytes` (39 for a 0-byte file; 43/35/47 for xxh128/xxh64/sha1 on a 12-byte file) while FastSync `%b` counts its own framing; in delta mode rsync `%c = 16 + 6·ceil(filesize/block_size)` (verified at block sizes 512/700/1024/2048) while FastSync counts its own signature handshake, and rsync `%b` is its token stream. FastSync's wire bytes are a different quantity, so exact `%b`/delta-`%c` equality is impossible. Directory and transfer-root lines are now emitted (rsync's `./` root line and per-directory `cd...`/`.d..t...` lines), with the same residual as `-i`: the root line is emitted unconditionally, every non-root directory renders as created because the sender does not probe directory destination state (so a pre-existing unchanged directory still shows `cd+++++++++`), and directory attribute columns (`%M`/`%U`/`%G`) come from the source | +| `--out-format=FORMAT` | Custom output format | ❌ Divergent | Per-transfer template on stdout; tokens `%f` `%n` `%l` `%b` `%c` `%C` `%i` `%M` `%o` `%U` `%G` `%t` `%%`. `%C` now uses the negotiated transfer algorithm (`--checksum-choice`, default `xxh128`, seed 0) and renders every algorithm exactly like rsync — xxh128 high-then-low, xxh64/xxh3 big-endian, md5/md4/sha1 standard hex, `none` a blank 2-char column — differential-tested across all algorithms. `%f`/`%n`/`%l`/`%i`/`%M`/`%U`/`%G`/`%B` also match. **Reclassified because `%b`/`%c` are protocol-specific and cannot match:** a differential against rsync 3.4.1 shows whole-file `%c = 16` for both, but rsync whole-file `%b = filesize + 27 + transfer-digest-bytes` (39 for a 0-byte file; 43/35/47 for xxh128/xxh64/sha1 on a 12-byte file) while FastSync `%b` counts its own framing; in delta mode rsync `%c = 16 + 6·ceil(filesize/block_size)` (verified at block sizes 512/700/1024/2048) while FastSync counts its own signature handshake, and rsync `%b` is its token stream. FastSync's wire bytes are a different quantity, so exact `%b`/delta-`%c` equality is impossible. Directory and transfer-root lines are now emitted (rsync's `./` root line and per-directory `cd...`/`.d..t...` lines); protocol 2.30.0 also gives them destination state (#314), so a pre-existing unchanged directory renders `.d..t......` (or nothing) instead of `cd+++++++++` and an unchanged symlink is suppressed, with the same `-i` residuals: the root line is emitted unconditionally and a directory whose only change is an attribute (no transferred child) is itemized at the end of the run; directory attribute columns (`%M`/`%U`/`%G`) come from the source | | `--log-file=FILE` | Log to file | ✅ Parity | `log_file` config field | | `--log-file-format=FMT` | Log format | ✅ Parity | Requires `--log-file`; writes one template line per transferred file using the same token set as `--out-format` (including `%b` as the wire byte count) | | `--8-bit-output`, `-8` | Leave high-bit chars unescaped | ✅ Parity | Applies to displayed paths and protocol debug output | @@ -964,7 +964,7 @@ These are the last compatibility items and the closing phase toward rsync flag p **Wire:** two trailing config-frame blocks after the `--iconv` spec, in fixed order — `send_privilege_options`/`receive_privilege_options` (one `super_mode` int, validated `0..2`), then `send_copy_as_options`/`receive_copy_as_options` (presence int + two int32 ids, validated `>= 0`, with `copy_as_set ⇒ use_metadata`). `PROTOCOL_VERSION` bumped **2.17.0 → 2.18.0**. **Divergences from rsync:** rsync's `--super` elevates the receiver and `--copy-as` actually switches its credentials; FastSync never elevates and only permits/forwards confined attempts, and `--copy-as` forces ownership rather than switching identity. -**Honest status after the parity 2.29 cycle (protocol 2.29.0 since the symlink-xattr wire wave, which adds no config-frame field and leaves this matrix unchanged), updated by the parity cycle 2.29 pass, the audit-cycle follow-ups, the triage cycle, and a later no-wire parity pass.** The wire backlog cycle (protocol 2.30.0) then moved `--stderr=MODE` ❌ → ⚠️ (the `client` mode is now accepted over the new `STATUS_CLIENT_MSG` channel; only the client->server direction is reproduced) and closed the `--devices` exit-code and `--remove-source-files` residuals via `STATUS_PARTIAL` (exit 23 with successful sources removed), leaving ✅ Parity 119 / ⚠️ Caveat 15 / ❌ Divergent 23 = 157 rows. The no-wire parity pass accepted `--inc-recursive`/`--no-inc-recursive` as inert no-ops (❌ → ✅, since FastSync's full scan is rsync's `--no-inc-recursive` and the destination is identical), narrowed the `--temp-dir` divergence by accepting an absolute path that canonicalizes inside the receive root (the row stays ❌ for out-of-root absolute paths), closed the `--delete-before` phase-0 divergence (⚠️ → ✅: both the single-threaded and the `--threads` data passes now replay the pre-scan file list, so a source file created after the scan is neither transferred nor kept, matching rsync), and moved `--fake-super` and `--devices` ❌ → ⚠️ (`--fake-super` now writes/reads rsync's exact `user.rsync.%stat` key and ` , :` grammar, interoperating with real rsync 3.4.1 for regular files and faking char/block devices as regular files carrying the real rdev; `--devices` now logs a failed device `mknod` as a per-entry failure that continues the transfer instead of a silent non-root skip — see those rows for the remaining directory-faking and exit-code residuals). A review pass then hardened the fake-super stat parser (strict range-checked parsing), made rsync-style daemon modules read-only by default with a startup warning for accepted-but-unenforced access-control keys, and extended the `--delete-before` replay to the `--threads` path. The 2.29 cycle closed the scanner-order, delete-timing, relative-basis, and fuzzy-eligibility residuals (moving `-n`/`--delete`/`--del`/`--delete-delay` to ✅) and improved the `--info`/`--stats`/`--debug` partial rows; the triage cycle moved `-F` and `-i`/`--itemize-changes` ✅ → ⚠️ for their documented residuals. The remaining ⚠️ rows are `--info`, `--debug`, `--msgs2stderr`, `--stats`, `--progress`, `-i`, `--filter`, `-F`, the three basis-dir options, `-y/--fuzzy`, `--fake-super`, and `--devices`. Earlier: **Honest status after the parity 2.28.0 cycle (protocol 2.28.0), updated by the rsync-parity-stats, rsync-parity-options, rsync-parity-fs, parity-review, no-wire parity-track-1/2b and wire parity-track-4a/5a passes.** ✅ Parity 116 / ⚠️ Caveat 14 / ❌ Divergent 27 = 157 rows. Earlier revisions of this document reported "143 ✅ / 0 divergence / 0 partial"; that conflated "parsed and tested" with "rsync parity", because many rows carried documented behavioral differences and some short options were not parsed at all. This reclassification makes every difference explicit. The completion wave closed 23 previously-caveated rows (9 that triage showed were already parity, plus 14 genuine fixes) and turned the 17 inherently non-rsync rows — native daemon config/auth, the FastSync batch container, the safe-subset device/privilege flags, `-X`'s privileged namespaces, `--fake-super`'s native xattr format, and the `--old-args` no-op — into explicit ❌ divergences. The stats pass flipped `--delete-delay` to ✅ (actual-removal accounting), but the parity-review pass moved it back to ⚠️ because FastSync charged the `--max-delete` budget at plan/snapshot time and left a refilled snapshotted directory in place, whereas rsync charges on actual removals and recursively removes a queued directory (including content created after its plan). The no-wire parity-track-1 pass fixed both (actual-removal charging plus recursive deferred removal with an independent deferred-list cap), narrowing the caveat to the partial-delete ordering. The stats pass also reclassified `--out-format` to ❌ (protocol-specific `%b`/delta-`%c`), and sharpened the `--stats`/`--progress`/`--checksum-choice` residuals. The options pass flipped `--bwlimit` and `--ignore-errors` to ✅ (rsync-exact size parsing and ~100 ms leaky-bucket throttling, and rsync's skip-unreadable-subdir plus IO-error-suppressed deletion with exit 23) and emits rsync-format `--info=name/flist/del/remove/nonreg/progress` lines (real-run `deleting`/`*deleting` carried over a new trailing `report_deletes` wire bool, `PROTOCOL_VERSION` 2.26.0 → 2.27.0), while reclassifying `-M` over daemon/TCP +**Honest status after the parity 2.29 cycle (protocol 2.29.0 since the symlink-xattr wire wave, which adds no config-frame field and leaves this matrix unchanged), updated by the parity cycle 2.29 pass, the audit-cycle follow-ups, the triage cycle, and a later no-wire parity pass.** The wire backlog cycle (protocol 2.30.0) then moved `--stderr=MODE` ❌ → ⚠️ (the `client` mode is now accepted over the new `STATUS_CLIENT_MSG` channel; only the client->server direction is reproduced) and closed the `--devices` exit-code and `--remove-source-files` residuals via `STATUS_PARTIAL` (exit 23 with successful sources removed), leaving ✅ Parity 119 / ⚠️ Caveat 15 / ❌ Divergent 23 = 157 rows. The same cycle then extended the destination-state report to directories and symlinks (#314: the receiver answers `STATUS_MKDIR`/`STATUS_SYMLINK` and an ancestor probe, so `-i`/`--progress`/`--out-format` render `.d..t......`/`cLc........` instead of `cd`/`cL` and suppress unchanged entries) and appended the per-type `deleted_*` counters to `STATUS_STATS` (#316: `--stats` now reproduces rsync's `Number of deleted files (reg/dir/link/special)` breakdown) — both differential-tested against rsync 3.4.1; the affected rows' caveats narrow but their classifications are unchanged, so the matrix stays **119 ✅ / 15 ⚠️ / 23 ❌ = 157**. The no-wire parity pass accepted `--inc-recursive`/`--no-inc-recursive` as inert no-ops (❌ → ✅, since FastSync's full scan is rsync's `--no-inc-recursive` and the destination is identical), narrowed the `--temp-dir` divergence by accepting an absolute path that canonicalizes inside the receive root (the row stays ❌ for out-of-root absolute paths), closed the `--delete-before` phase-0 divergence (⚠️ → ✅: both the single-threaded and the `--threads` data passes now replay the pre-scan file list, so a source file created after the scan is neither transferred nor kept, matching rsync), and moved `--fake-super` and `--devices` ❌ → ⚠️ (`--fake-super` now writes/reads rsync's exact `user.rsync.%stat` key and ` , :` grammar, interoperating with real rsync 3.4.1 for regular files and faking char/block devices as regular files carrying the real rdev; `--devices` now logs a failed device `mknod` as a per-entry failure that continues the transfer instead of a silent non-root skip — see those rows for the remaining directory-faking and exit-code residuals). A review pass then hardened the fake-super stat parser (strict range-checked parsing), made rsync-style daemon modules read-only by default with a startup warning for accepted-but-unenforced access-control keys, and extended the `--delete-before` replay to the `--threads` path. The 2.29 cycle closed the scanner-order, delete-timing, relative-basis, and fuzzy-eligibility residuals (moving `-n`/`--delete`/`--del`/`--delete-delay` to ✅) and improved the `--info`/`--stats`/`--debug` partial rows; the triage cycle moved `-F` and `-i`/`--itemize-changes` ✅ → ⚠️ for their documented residuals. The remaining ⚠️ rows are `--info`, `--debug`, `--msgs2stderr`, `--stats`, `--progress`, `-i`, `--filter`, `-F`, the three basis-dir options, `-y/--fuzzy`, `--fake-super`, and `--devices`. Earlier: **Honest status after the parity 2.28.0 cycle (protocol 2.28.0), updated by the rsync-parity-stats, rsync-parity-options, rsync-parity-fs, parity-review, no-wire parity-track-1/2b and wire parity-track-4a/5a passes.** ✅ Parity 116 / ⚠️ Caveat 14 / ❌ Divergent 27 = 157 rows. Earlier revisions of this document reported "143 ✅ / 0 divergence / 0 partial"; that conflated "parsed and tested" with "rsync parity", because many rows carried documented behavioral differences and some short options were not parsed at all. This reclassification makes every difference explicit. The completion wave closed 23 previously-caveated rows (9 that triage showed were already parity, plus 14 genuine fixes) and turned the 17 inherently non-rsync rows — native daemon config/auth, the FastSync batch container, the safe-subset device/privilege flags, `-X`'s privileged namespaces, `--fake-super`'s native xattr format, and the `--old-args` no-op — into explicit ❌ divergences. The stats pass flipped `--delete-delay` to ✅ (actual-removal accounting), but the parity-review pass moved it back to ⚠️ because FastSync charged the `--max-delete` budget at plan/snapshot time and left a refilled snapshotted directory in place, whereas rsync charges on actual removals and recursively removes a queued directory (including content created after its plan). The no-wire parity-track-1 pass fixed both (actual-removal charging plus recursive deferred removal with an independent deferred-list cap), narrowing the caveat to the partial-delete ordering. The stats pass also reclassified `--out-format` to ❌ (protocol-specific `%b`/delta-`%c`), and sharpened the `--stats`/`--progress`/`--checksum-choice` residuals. The options pass flipped `--bwlimit` and `--ignore-errors` to ✅ (rsync-exact size parsing and ~100 ms leaky-bucket throttling, and rsync's skip-unreadable-subdir plus IO-error-suppressed deletion with exit 23) and emits rsync-format `--info=name/flist/del/remove/nonreg/progress` lines (real-run `deleting`/`*deleting` carried over a new trailing `report_deletes` wire bool, `PROTOCOL_VERSION` 2.26.0 → 2.27.0), while reclassifying `-M` over daemon/TCP and receiver-side `protect`/`risk` re-derivation to ❌ (no argv channel / receiver filter engine); the wire parity-track-4a pass later added that receiver filter engine, flipping `--filter=RULE` back to ✅ (see above; the diff --git a/src/client/change_list.c b/src/client/change_list.c index 892cef7..ad4340b 100644 --- a/src/client/change_list.c +++ b/src/client/change_list.c @@ -123,8 +123,15 @@ static char itemize_type_char(const ChangeEvent* event) { static bool times_match(const Config* config, const ChangeEvent* event) { if (!event->dest.known || !event->dest.existed) return false; - if (event->mtime_sec == event->dest.mtime_sec) + if (event->mtime_sec == event->dest.mtime_sec) { + /* A regular file's sub-second mtime IS preserved by the receiver, so an nsec + difference is a real change. A directory or symlink has no preserved + sub-second mtime (rsync's quick-check compares whole seconds there), so a + nanosecond-only difference must not render a spurious `.d..t` / `.L..t`. */ + if (event->is_directory || event->is_symlink || event->is_special) + return true; return event->mtime_nsec == event->dest.mtime_nsec; + } long long delta = (long long)event->mtime_sec - (long long)event->dest.mtime_sec; if (delta < 0) delta = -delta; @@ -145,6 +152,10 @@ static void itemize_code(const Config* config, const ChangeEvent* event, char co /* rsync: an existing directory that only has attribute changes carries no transfer, so the update column is `.` rather than `>`. */ update = '.'; + else if (event->is_symlink) + /* rsync: an existing symlink whose target is unchanged is a `.` update + (attributes only); a changed target is `c` (the link value changed). */ + update = event->dest.target_matches ? '.' : 'c'; else update = '>'; code[0] = update; @@ -155,12 +166,20 @@ static void itemize_code(const Config* config, const ChangeEvent* event, char co code[11] = '\0'; return; } - bool size_diff = event->size != event->dest.size; - bool time_diff = !times_match(config, event); + /* rsync's value/checksum column: `c` for a symlink whose target changed (the + link value is the compared content); no destination digest is available for + a regular file. */ + bool value_diff = event->is_symlink && !event->dest.target_matches; + /* rsync itemizes size only for regular files: a directory's st_size and a + symlink's target length are not compared. */ + bool size_diff = !event->is_directory && !event->is_symlink && !event->is_special && + event->size != event->dest.size; + /* rsync itemizes the time column only when -t/--times is in effect. */ + bool time_diff = config->preserve_times && !times_match(config, event); bool perms_diff = (event->mode & 07777) != (event->dest.mode & 07777); bool owner_diff = event->uid != (uid_t)event->dest.uid; bool group_diff = event->gid != (gid_t)event->dest.gid; - code[2] = '.'; /* checksum: no destination digest available */ + code[2] = value_diff ? 'c' : '.'; code[3] = size_diff ? 's' : '.'; code[4] = time_diff ? 't' : '.'; code[5] = (config->preserve_perms && perms_diff) ? 'p' : '.'; @@ -172,6 +191,24 @@ static void itemize_code(const Config* config, const ChangeEvent* event, char co code[11] = '\0'; } +/* True when the itemized destination entry is unchanged, i.e. rsync would print + * no line at all. Reuses itemize_code so suppression is exactly consistent + * with what would have been rendered: the update column must be `.` and every + * attribute column must be `.`. */ +static bool itemize_is_unchanged(const Config* config, const ChangeEvent* event) { + if (!event->dest.known || !event->dest.existed) + return false; + char code[12]; + itemize_code(config, event, code); + if (code[0] != '.') + return false; + for (int i = 2; i < 11; i++) { + if (code[i] != '.') + return false; + } + return true; +} + /* rsync %n: the transfer-relative name, with a trailing slash for directories. * The transfer root is `.` (so `%n` renders `./`), matching rsync's root entry. */ static bool append_name(StrBuf* buf, const ChangeEvent* event) { @@ -678,6 +715,19 @@ void change_emit_file_sent_bytes(const Config* config, const File* file, char* name = NULL; char* path = NULL; fill_event_from_file(config, file, &event, &name, &path); + if (file->is_symlink) { + /* Output parity (protocol 2.30.0): an unchanged symlink is silent, like + rsync's quick check. The itemize/log stream suppresses it only when every + attribute matches; the name stream suppresses it whenever the link target + is unchanged (rsync names a symlink only when it relinks or creates it). */ + bool itemize_output = config->itemize_changes || config->out_format != NULL || + (config->log_file != NULL && config->log_file_format != NULL); + bool suppress = itemize_output + ? itemize_is_unchanged(config, &event) + : (event.dest.known && event.dest.existed && event.dest.target_matches); + if (suppress) + event.decision = CHANGE_UP_TO_DATE; + } if (name != NULL && path != NULL) { fill_event_checksum(config, file, &event); change_emit(config, &event); @@ -729,6 +779,19 @@ void change_emit_dir_sent(const Config* config, const File* file) { char* name = NULL; char* path = NULL; fill_event_from_file(config, file, &event, &name, &path); + /* Output parity (protocol 2.30.0): suppress a directory rsync would leave + silent. The itemize/log stream suppresses it only when every attribute + matches (`.d.........`); the name stream suppresses any pre-existing + directory (rsync names a directory only when it is created). */ + bool itemize_output = config->itemize_changes || config->out_format != NULL || + (config->log_file != NULL && config->log_file_format != NULL); + bool suppress = itemize_output ? itemize_is_unchanged(config, &event) + : (event.dest.known && event.dest.existed); + /* The transfer root's line is an unconditional FastSync residual (rsync keys + it off the root's own attribute change); keep emitting it. */ + bool is_root = event.name != NULL && event.name[0] == '\0'; + if (suppress && !is_root) + event.decision = CHANGE_UP_TO_DATE; if (name != NULL && path != NULL) change_emit(config, &event); free(name); diff --git a/src/client/client_cli.c b/src/client/client_cli.c index dd526ef..9063b56 100644 --- a/src/client/client_cli.c +++ b/src/client/client_cli.c @@ -2807,8 +2807,11 @@ static int cli_finalize_config(Config* config, bool verbose, bool no_delta, bool * need the pre-transfer destination snapshot (new vs modified and which * attributes differ), so ask the receiver to report it on every per-file * check. This is a wire field. */ + bool progress_active = + !config->quiet && (config->show_progress || (config->info_level & LOG_INFO_PROGRESS) != 0); config->report_dest_info = config->itemize_changes || config->out_format != NULL || - (config->log_file != NULL && config->log_file_format != NULL); + (config->log_file != NULL && config->log_file_format != NULL) || + progress_active; /* Wire-stats parity: --stats, --progress/-P, an --out-format token that needs * a wire counter (%b/%c), or a dry-run --delete need the receiver's * end-of-transfer STATUS_STATS report. This is a wire field (protocol diff --git a/src/client/client_report.c b/src/client/client_report.c index c29cba5..9c1e4c7 100644 --- a/src/client/client_report.c +++ b/src/client/client_report.c @@ -162,6 +162,9 @@ void report_transfer_stats(const Config* config, const TransferStats* stats, tim created_breakdown, sizeof(created_breakdown)); unsigned long long created_total = recv->created_reg + recv->created_dir + recv->created_link + recv->created_special; + char deleted_breakdown[128]; + type_breakdown(recv->deleted_reg, recv->deleted_dir, recv->deleted_link, recv->deleted_special, + deleted_breakdown, sizeof(deleted_breakdown)); printf("\n"); if (breakdown[0] != '\0') printf("Number of files: %llu %s\n", flist_total, breakdown); @@ -173,7 +176,13 @@ void report_transfer_stats(const Config* config, const TransferStats* stats, tim printf("Number of created files: %llu %s\n", created_total, created_breakdown); else printf("Number of created files: %llu\n", created_total); - printf("Number of deleted files: %llu\n", recv->deleted_files); + /* Protocol 2.30.0: the receiver reports the removed entries split by type, so + this line matches rsync's `Number of deleted files: X (reg: A, dir: B, + link: C, special: D)` (only the non-zero categories are listed). */ + if (deleted_breakdown[0] != '\0') + printf("Number of deleted files: %llu %s\n", recv->deleted_files, deleted_breakdown); + else + printf("Number of deleted files: %llu\n", recv->deleted_files); printf("Number of regular files transferred: %llu\n", stats->transferred_regular); printf("Total file size: %s bytes\n", total); printf("Total transferred file size: %s bytes\n", transferred); @@ -526,9 +535,16 @@ static void client_progress_emit_ancestors(const Config* config, const char* rel if (dir != NULL) change_emit_dir_sent(config, dir); } else { - char* escaped = output_escape(prefix, config->eight_bit_output); - printf("%s/\n", escaped ? escaped : prefix); - free(escaped); + /* --progress/-P names a directory only when it is newly created; + rsync stays silent for a pre-existing directory even when one of + its children changed (protocol 2.30.0 dest-state report). */ + const File* dir = progress_dir_lookup(prefix); + bool existed = dir != NULL && dir->dest_state.known && dir->dest_state.existed; + if (!existed) { + char* escaped = output_escape(prefix, config->eight_bit_output); + printf("%s/\n", escaped ? escaped : prefix); + free(escaped); + } } g_progress_index++; } else { @@ -553,6 +569,122 @@ void client_change_emit_ancestors(const Config* config, const File* file) { client_progress_emit_ancestors(config, rel); } +/* Send one STATUS_MKDIR probe (probe=1) for a pre-count directory and cache the + * receiver's pre-transfer destination snapshot in `dir->dest_state`. Returns + * false on a protocol/transport error. */ +static bool client_probe_dir_state(int fd, File* dir) { + if (dir == NULL || file_wire_path(dir) == NULL) + return true; + if (!send_status(fd, STATUS_MKDIR) || !send_int(fd, 1) || !send_wire_str(fd, file_wire_path(dir))) + return false; + Status status = STATUS_ERROR; + if (!receive_status(fd, &status) || status != STATUS_DEST_INFO || + !format_dest_state_receive(fd, &dir->dest_state)) { + log_message(LOG_LEVEL_ERROR, "Directory destination-state probe failed"); + return false; + } + return true; +} + +/* Output parity (protocol 2.30.0): ask the receiver for each not-yet-probed + * ancestor directory's pre-transfer state BEFORE the entry that first triggers + * it is sent, so the ancestor's -i/--out-format line renders rsync's + * `.d..t......` (existing, attributes changed) versus `cd+++++++++` (created) + * and an unchanged directory is suppressed. The probe is a STATUS_MKDIR frame + * with probe=1 (the receiver reports and creates nothing), so it must run before + * the receiver implicitly creates the parent for the child. Each directory is + * probed at most once; the result is cached in the pre-count File's dest_state. + * Returns false on a protocol/transport error (the caller aborts the transfer). */ +bool client_change_probe_ancestors(const Config* config, const File* file, int fd) { + if (config == NULL || file == NULL || fd < 0 || !config->report_dest_info) + return true; + if (!g_progress_dir_index_valid || !g_progress_precount.dir_refs) + return true; + const char* rel = delete_display_path(config, file_wire_path(file)); + if (rel == NULL) + return true; + size_t rel_len = strlen(rel); + for (size_t i = 1; i < rel_len; i++) { + if (rel[i] != '/') + continue; + char* prefix = malloc(i + 1); + if (prefix == NULL) + return false; + memcpy(prefix, rel, i); + prefix[i] = '\0'; + if (path_index_contains(&g_progress_dir_index, prefix)) { + File* dir = progress_dir_lookup(prefix); + /* The probe path is the same wire path the real STATUS_MKDIR would carry + (the pre-count File's send_path, or its absolute source path for a + plain recursive scan), not the display-relative prefix. */ + if (dir != NULL && !dir->dest_state.known && !client_probe_dir_state(fd, dir)) { + free(prefix); + return false; + } + } + free(prefix); + } + return true; +} + +/* Mark a directory the data pass already itemized/named so the end-of-transfer + * pending-directory flush does not report it a second time. `file` is a + * transferred directory entry (an empty-directory STATUS_MKDIR). */ +void client_change_mark_dir(const Config* config, const File* file) { + if (config == NULL || file == NULL || !g_progress_emitted_valid || + g_progress_emitted_keys == NULL) + return; + const char* rel = delete_display_path(config, file_wire_path(file)); + if (rel == NULL || rel[0] == '\0' || str_hash_set_lookup(&g_progress_emitted, rel)) + return; + char* key = str_dup(rel); + if (key == NULL) + return; + if (!array_list_add(g_progress_emitted_keys, key)) { + free(key); + return; + } + str_hash_set_insert_ref(&g_progress_emitted, key); +} + +/* Emit the itemize lines for source directories that CHANGED but had no + * transferred child, so no ancestor emission reached them (rsync reports a + * directory whose attributes changed even when its contents did not). Runs at + * the end of the data pass, BEFORE the deferred STATUS_DIR_TIMES apply, so the + * probe still observes each untouched directory's pre-transfer state. Only the + * itemize/out-format/log streams report attribute-only directory changes; + * --progress/-P stays silent for them, matching rsync. Best-effort: a probe + * failure simply stops the flush (the transfer's verdict is unaffected). */ +void client_change_emit_pending_dirs(const Config* config, int fd) { + if (config == NULL || fd < 0 || !config->report_dest_info) + return; + bool itemize_output = config->itemize_changes || config->out_format != NULL || + (config->log_file != NULL && config->log_file_format != NULL); + if (!itemize_output) + return; + if (!g_progress_dir_index_valid || g_progress_precount.dir_refs == NULL || + !g_progress_emitted_valid || g_progress_emitted_keys == NULL) + return; + for (int i = 0; i < g_progress_precount.dir_refs->size; i++) { + DirRef* ref = (DirRef*)g_progress_precount.dir_refs->items[i]; + if (ref == NULL || ref->name == NULL || ref->name[0] == '\0') + continue; + if (str_hash_set_lookup(&g_progress_emitted, ref->name)) + continue; + File* dir = ref->file; + if (dir == NULL) + continue; + if (!dir->dest_state.known && !client_probe_dir_state(fd, dir)) + return; + change_emit_dir_sent(config, dir); + char* key = str_dup(ref->name); + if (key != NULL && array_list_add(g_progress_emitted_keys, key)) + str_hash_set_insert_ref(&g_progress_emitted, key); + else + free(key); + } +} + /* rsync's --info=name/progress line for one entry: transfer-relative name (a * trailing slash for directories) plus the ` -> target` symlink suffix. */ static char* progress_entry_line(const File* file, const char* rel) { @@ -645,6 +777,15 @@ void client_progress_file(const Config* config, const File* file) { void client_progress_name(const Config* config, const File* file) { if (!g_progress_active || file == NULL) return; + /* rsync's --progress/-P name stream reports an entry only when it is created + or (for a symlink) actually relinked: a pre-existing directory or an + unchanged symlink is silent (protocol 2.30.0 dest-state report). */ + if (file->dest_state.known && file->dest_state.existed) { + if (file->is_dir || (file->is_symlink && file->dest_state.target_matches)) { + g_progress_index++; + return; + } + } const char* rel = delete_display_path(config, file_wire_path(file)); if (!config->itemize_changes && config->out_format == NULL) { char* line = progress_entry_line(file, rel ? rel : ""); diff --git a/src/client/client_send.c b/src/client/client_send.c index 2477ed7..bec5e6c 100644 --- a/src/client/client_send.c +++ b/src/client/client_send.c @@ -615,14 +615,34 @@ static bool send_file_direct(File* file, int fd, bool use_metadata, int compress static bool send_directory_entry(const Client* client, File* file, const Config* config) { if (!file || !file_wire_path(file)) return false; - if (!send_status(client->file_descriptor, STATUS_MKDIR) || - !send_wire_str(client->file_descriptor, file_wire_path(file))) + int fd = client->file_descriptor; + if (!send_status(fd, STATUS_MKDIR)) return false; - if (config->use_metadata && !metadata_send(client->file_descriptor, file->metadata)) + /* Output parity (protocol 2.30.0): when report_dest_info is negotiated every + STATUS_MKDIR body is prefixed with a probe flag (1 = probe only, 0 = a real + create), so the receiver knows whether to expect the metadata/xattr block. */ + if (config->report_dest_info && !send_int(fd, 0)) + return false; + if (!send_wire_str(fd, file_wire_path(file))) + return false; + if (config->use_metadata && !metadata_send(fd, file->metadata)) return false; /* Directory xattrs/ACLs (-X/-A) ride the same trailing block as regular files when the xattr transport was negotiated. */ - return !config->use_xattrs || xattr_send(client->file_descriptor, file->xattrs); + if (config->use_xattrs && !xattr_send(fd, file->xattrs)) + return false; + /* The receiver answers with the directory's pre-transfer destination state + BEFORE creating it, so the sender can render rsync's `.d..t......` versus + `cd+++++++++` and suppress an unchanged directory. */ + if (config->report_dest_info) { + Status status; + if (!receive_status(fd, &status) || status != STATUS_DEST_INFO || + !format_dest_state_receive(fd, &file->dest_state)) { + log_message(LOG_LEVEL_ERROR, "Unexpected reply to the directory destination-state report"); + return false; + } + } + return true; } /* P7 Wave D: transmit every captured source directory's metadata in terminal @@ -678,7 +698,21 @@ static bool send_symlink_entry(const Client* client, File* file, const Config* c return false; /* Symlink xattrs/ACLs (-X/-A) ride the same trailing block as regular files and directories when the xattr transport was negotiated. */ - return !config->use_xattrs || xattr_send(fd, file->xattrs); + if (config->use_xattrs && !xattr_send(fd, file->xattrs)) + return false; + /* The receiver answers with the symlink's pre-transfer destination state + (including whether the on-disk link target already matches) BEFORE creating + it, so the sender can render rsync's `cLc........` / `.L..t......` and + suppress an unchanged symlink. */ + if (config->report_dest_info) { + Status status; + if (!receive_status(fd, &status) || status != STATUS_DEST_INFO || + !format_dest_state_receive(fd, &file->dest_state)) { + log_message(LOG_LEVEL_ERROR, "Unexpected reply to the symlink destination-state report"); + return false; + } + } + return true; } // Send a single file directly via sendfile (non-incremental path). @@ -869,10 +903,12 @@ static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config, feed -i/--out-format its ancestor directory lines here. */ if (config->itemize_changes || config->out_format != NULL) client_change_emit_ancestors(config, chunk->items[i]); - if (chunk->items[i]->is_dir) + if (chunk->items[i]->is_dir) { change_emit_dir_sent(config, chunk->items[i]); - else + client_change_mark_dir(config, chunk->items[i]); + } else { change_emit_file_sent(config, chunk->items[i]); + } if (!chunk->items[i]->is_dir) transfer_stats_note_transferred(stats, chunk->items[i]); } @@ -884,6 +920,11 @@ static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config, if (f == NULL) continue; transfer_stats_note_entry(stats, f); + /* Output parity: probe this entry's ancestor directories' destination state + before the entry (or the first child below them) is sent, while the + receiver has not yet created them implicitly. */ + if (!client_change_probe_ancestors(config, f, client->file_descriptor)) + return -1; if (f->is_dir) { /* Explicit directory entry (--dirs): a MKDIR frame carrying the destination path (and metadata when negotiated). Directories have no @@ -892,6 +933,7 @@ static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config, return -1; client_change_emit_ancestors(config, f); change_emit_dir_sent(config, f); + client_change_mark_dir(config, f); client_progress_name(config, f); continue; } @@ -1137,6 +1179,7 @@ static int send_chunks_multithreaded(void* pipeline_context) { (and all parallel workers) has been joined before scanner_done was set, so the list is complete and race-free; on an early stop the list may be incomplete and is deliberately not sent. */ + client_change_emit_pending_dirs(context->config, client->file_descriptor); if (!context->scan_stopped_early && !send_dir_times(client, context->config, context->dir_entries)) goto send_fail; @@ -1844,6 +1887,10 @@ static int send_files_finalize(const Config* config, SendFilesState* state) { } } } + /* Output parity: report changed directories that had no transferred child + before the deferred directory times are applied (so the probe still sees + their pre-transfer state). */ + client_change_emit_pending_dirs(config, client->file_descriptor); /* P7 Wave D: every directory has now been traversed (or the scan stopped early), so transmit the captured directory times last. The receiver defers applying them until after its own deletion/publication phase. */ diff --git a/src/client/client_send_internal.h b/src/client/client_send_internal.h index d891b30..a7804fc 100644 --- a/src/client/client_send_internal.h +++ b/src/client/client_send_internal.h @@ -68,6 +68,14 @@ void client_progress_name(const Config* config, const File* file); /* Emit a transferred entry's ancestor directories (as -i/--out-format change * lines or --progress name lines) before the entry's own line. */ void client_change_emit_ancestors(const Config* config, const File* file); +/* Output parity (protocol 2.30.0): probe each not-yet-known ancestor directory's + * pre-transfer destination state before the entry that first triggers it is + * sent. Returns false on a protocol/transport error. */ +bool client_change_probe_ancestors(const Config* config, const File* file, int fd); +/* Mark a transferred directory entry as already reported, and flush the + * itemize lines for changed directories that had no transferred child. */ +void client_change_mark_dir(const Config* config, const File* file); +void client_change_emit_pending_dirs(const Config* config, int fd); void client_progress_uptodate(const Config* config, const File* file); void client_progress_prepare(const Config* config, const ArrayList* plan_dirs, unsigned long long plan_non_dir_count); diff --git a/src/server/receiver.c b/src/server/receiver.c index d99f7ff..a815391 100644 --- a/src/server/receiver.c +++ b/src/server/receiver.c @@ -11,10 +11,13 @@ #include "metadata.h" #include "protocol.h" #include "utils.h" +#include +#include #include #include #include #include +#include bool receiver_outcomes_append(ReceiverOutcomes* outcomes, unsigned char code) { if (!outcomes) @@ -101,14 +104,35 @@ static void receiver_tally_deleted(const ReceiverSink* sink, size_t deleted) { sink->stats->deleted_files += deleted; } -/* Observer for --info=del: record each truly-removed destination-relative path - in the ArrayList passed as the observer context, so the terminal STATUS_STATS - frame can list it. A failed append is best-effort (the deletion already - happened; output is cosmetic). Shared by the single-threaded receiver and - the -m pipeline's deferred commit. */ -void receiver_record_deleted_path(void* context, const char* rel_path) { - ArrayList* paths = context; - if (!paths || !rel_path) +/* Observer for --info=del/--stats: record each truly-removed destination- + relative path (when the context carries a path list) and tally it by type + (when it carries a stats record), so the terminal STATUS_STATS frame can list + the paths and render rsync's per-type `Number of deleted files` breakdown. A + failed append is best-effort (the deletion already happened; output is + cosmetic). Shared by the single-threaded receiver and the -m pipeline's + deferred commit. */ +void receiver_record_deleted_path(void* context, const char* rel_path, DeleteEntryType type) { + ReceiverDeleteContext* del = context; + if (!del || !rel_path) + return; + if (del->stats) { + switch (type) { + case DELETE_ENTRY_DIR: + del->stats->deleted_dir++; + break; + case DELETE_ENTRY_LINK: + del->stats->deleted_link++; + break; + case DELETE_ENTRY_SPECIAL: + del->stats->deleted_special++; + break; + default: + del->stats->deleted_reg++; + break; + } + } + ArrayList* paths = del->deleted_paths; + if (!paths) return; /* Bound the retained list like the keep-set manifest: only MAX_MANIFEST_ENTRIES paths are ever transmitted in the terminal STATUS_STATS frame, so recording @@ -120,6 +144,16 @@ void receiver_record_deleted_path(void* context, const char* rel_path) { free(copy); } +/* Install the delete observer (and its context) for one commit when the sink + carries a stats record or a path list. Returns NULL when neither is needed, + so the delete engines skip the observer entirely. */ +static DeletePathObserver receiver_delete_observer(const ReceiverSink* sink, + ReceiverDeleteContext* del) { + del->stats = sink ? sink->stats : NULL; + del->deleted_paths = sink ? sink->deleted_paths : NULL; + return (del->stats || del->deleted_paths) ? receiver_record_deleted_path : NULL; +} + static bool receiver_process_chunk(Chunk* chunk, const ReceiverSink* sink) { if (!chunk || !sink || !sink->store_file) return false; @@ -328,6 +362,9 @@ typedef struct { a successful FINISHED it is either committed here or handed to *pending_plans so the -m caller commits after its disk writer drained. */ DeletePlanSession* plan_session; + /* Observer context for the per-directory delete session, which outlives the + frame handler; must stay alive until the session commits. */ + ReceiverDeleteContext delete_ctx; bool early_delete; bool per_dir_delete; bool delete_limit_noted; @@ -403,9 +440,95 @@ static ReceiverStep receiver_handle_check_batch(ReceiverPendingState* state) { return RECEIVER_STEP_NEXT; } +/* Probe a destination entry's pre-transfer state for the output-parity + dest-info report (protocol 2.30.0). `wire_path` is the destination-relative + path; `incoming_target` is non-NULL only for a symlink probe, in which case + the on-disk link target is compared with the target the receiver is about to + store (after --munge-links). The final component is never followed and the + parent walk is confined below the receive root. Returns false only on an + allocation/secure-walk failure; a missing entry is reported as existed=false. */ +static bool receiver_probe_dest_state(const Config* config, const char* wire_path, + const char* incoming_target, OutputDestState* out) { + memset(out, 0, sizeof(*out)); + out->known = true; + if (!config || !wire_path || wire_path[0] == '\0') + return false; + char* full = path_cat(config->receive_root_directory, wire_path); + if (!full) + return false; + char* leaf = NULL; + int parent_fd = file_open_secure_parent(full, &leaf, false); + free(full); + if (parent_fd < 0) { + /* A missing/unreachable parent means the entry cannot exist yet. */ + free(leaf); + return true; + } + struct stat st; + if (fstatat(parent_fd, leaf, &st, AT_SYMLINK_NOFOLLOW) == 0) { + out->existed = true; + out->size = (unsigned long long)st.st_size; + out->mtime_sec = (long long)st.st_mtime; +#ifdef __linux__ + out->mtime_nsec = st.st_mtim.tv_nsec; +#endif + out->mode = (uint32_t)st.st_mode; + out->uid = (int32_t)st.st_uid; + out->gid = (int32_t)st.st_gid; + if (incoming_target && S_ISLNK(st.st_mode)) { + char target_buf[PATH_MAX]; + ssize_t n = readlinkat(parent_fd, leaf, target_buf, sizeof(target_buf) - 1); + if (n >= 0) { + target_buf[n] = '\0'; + char* expected = + config->munge_links ? file_symlink_munge(incoming_target) : str_dup(incoming_target); + if (expected) { + out->target_matches = strcmp(target_buf, expected) == 0; + free(expected); + } + } + } + } + close(parent_fd); + free(leaf); + return true; +} + static ReceiverStep receiver_handle_mkdir(ReceiverPendingState* state) { - File* dir = file_receive_directory(state->fd, state->config); - if (!dir || !state->sink->store_file(dir, state->sink->context)) + const Config* config = state->config; + int fd = state->fd; + if (config->report_dest_info) { + int probe = 0; + if (!receive_int(fd, &probe) || (probe != 0 && probe != 1)) + return RECEIVER_STEP_FAIL; + if (probe) { + /* Probe-only frame: report the destination state and create nothing. */ + char* path = receive_wire_str(fd); + if (!path || path[0] == '\0' || (!file_get_trust_sender() && has_path_traversal(path))) { + free(path); + send_status(fd, STATUS_ERROR); + return RECEIVER_STEP_FAIL; + } + OutputDestState info; + bool ok = receiver_probe_dest_state(config, path, NULL, &info); + free(path); + if (!ok || !send_status(fd, STATUS_DEST_INFO) || !format_dest_state_send(fd, &info)) + return RECEIVER_STEP_FAIL; + return RECEIVER_STEP_NEXT; + } + } + File* dir = file_receive_directory(fd, config); + if (!dir) + return RECEIVER_STEP_ERROR; + if (config->report_dest_info) { + OutputDestState info; + bool ok = receiver_probe_dest_state(config, file_wire_path(dir), NULL, &info); + if (!ok || !send_status(fd, STATUS_DEST_INFO) || !format_dest_state_send(fd, &info)) { + file_destroy(dir); + return RECEIVER_STEP_FAIL; + } + } + if (!state->sink->store_file(dir, state->sink->context)) return RECEIVER_STEP_ERROR; return RECEIVER_STEP_NEXT; } @@ -424,8 +547,20 @@ static ReceiverStep receiver_handle_hardlink(ReceiverPendingState* state) { } static ReceiverStep receiver_handle_symlink(ReceiverPendingState* state) { - File* sym = file_receive_symlink(state->fd, state->config); - if (!sym || !state->sink->store_file(sym, state->sink->context)) + const Config* config = state->config; + File* sym = file_receive_symlink(state->fd, config); + if (!sym) + return RECEIVER_STEP_ERROR; + if (config->report_dest_info) { + OutputDestState info; + bool ok = receiver_probe_dest_state(config, file_wire_path(sym), sym->symlink_target, &info); + if (!ok || !send_status(state->fd, STATUS_DEST_INFO) || + !format_dest_state_send(state->fd, &info)) { + file_destroy(sym); + return RECEIVER_STEP_FAIL; + } + } + if (!state->sink->store_file(sym, state->sink->context)) return RECEIVER_STEP_ERROR; return RECEIVER_STEP_NEXT; } @@ -468,12 +603,11 @@ static ReceiverStep receiver_handle_manifest(ReceiverPendingState* state) { --max-delete-capped commit still succeeds and the transfer proceeds; the terminal success frame reports the cap. */ size_t deleted = 0; - DeletePathObserver observer = - (config->report_deletes && sink->deleted_paths) ? receiver_record_deleted_path : NULL; + ReceiverDeleteContext delctx; + DeletePathObserver observer = receiver_delete_observer(sink, &delctx); DeleteCommitResult deletion = (config->use_delete || config->delete_missing_args) - ? manifest_delete_all_observed(config, manifest, &deleted, observer, - (void*)sink->deleted_paths) + ? manifest_delete_all_observed(config, manifest, &deleted, observer, &delctx) : DELETE_COMMIT_OK; receiver_tally_deleted(sink, deleted); delete_manifest_free(manifest); @@ -516,9 +650,9 @@ static ReceiverStep receiver_handle_delete_plan(ReceiverPendingState* state) { } if (!state->plan_session) { state->plan_session = delete_plan_session_create(config); - if (state->plan_session && config->report_deletes && sink->deleted_paths) + if (state->plan_session && (sink->stats || sink->deleted_paths)) delete_plan_session_set_delete_observer(state->plan_session, receiver_record_deleted_path, - (void*)sink->deleted_paths); + &state->delete_ctx); } if (!state->plan_session || delete_plan_session_receive(state->plan_session, config, fd) != 0) return RECEIVER_STEP_FAIL; @@ -623,6 +757,7 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver .pending_plans = pending_plans, .deferred_manifest = NULL, .plan_session = NULL, + .delete_ctx = {sink ? sink->stats : NULL, sink ? sink->deleted_paths : NULL}, .early_delete = config_delete_timing_early(config), .per_dir_delete = config_delete_timing_per_dir(config), .delete_limit_noted = false, @@ -662,10 +797,9 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver state.deferred_manifest = NULL; } else { size_t deleted = 0; - DeletePathObserver observer = - (config->report_deletes && sink->deleted_paths) ? receiver_record_deleted_path : NULL; + DeletePathObserver observer = receiver_delete_observer(sink, &state.delete_ctx); DeleteCommitResult deletion = manifest_delete_all_observed( - config, state.deferred_manifest, &deleted, observer, (void*)sink->deleted_paths); + config, state.deferred_manifest, &deleted, observer, &state.delete_ctx); receiver_tally_deleted(sink, deleted); delete_manifest_free(state.deferred_manifest); state.deferred_manifest = NULL; @@ -683,9 +817,9 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver hands the session to its caller instead, which commits after the disk writer drained. */ if (state.plan_session) { - if (config->report_deletes && sink->deleted_paths) + if (sink->stats || sink->deleted_paths) delete_plan_session_set_delete_observer(state.plan_session, receiver_record_deleted_path, - (void*)sink->deleted_paths); + &state.delete_ctx); if (state.pending_plans) { *state.pending_plans = state.plan_session; state.plan_session = NULL; diff --git a/src/server/receiver.h b/src/server/receiver.h index 3b2dcff..2f68527 100644 --- a/src/server/receiver.h +++ b/src/server/receiver.h @@ -55,10 +55,20 @@ typedef struct { bool receiver_outcomes_append(ReceiverOutcomes* outcomes, unsigned char code); void receiver_outcomes_destroy(ReceiverOutcomes* outcomes); -/* DeletePathObserver implementation for --info=del: `context` is an ArrayList* - that receives owned copies of every truly-removed destination-relative path. - Shared by the single-threaded receiver and the -m pipeline's deferred commit. */ -void receiver_record_deleted_path(void* context, const char* rel_path); +/* Delete observer context: `deleted_paths` (optional) receives owned copies of + every truly-removed destination-relative path for --info=del; `stats` + (optional) receives the per-type `Number of deleted files` tallies for + --stats. Both may be NULL, in which case the observer is a no-op. */ +typedef struct { + ReceiverStats* stats; + struct ArrayList* deleted_paths; +} ReceiverDeleteContext; + +/* DeletePathObserver implementation: records each truly-removed path (when the + context carries a path list) and tallies it by type (when it carries a stats + record). Shared by the single-threaded receiver and the -m pipeline's + deferred commit. */ +void receiver_record_deleted_path(void* context, const char* rel_path, DeleteEntryType type); /* Send the terminal success frame. `final_status` is usually STATUS_OK, or STATUS_DELETE_LIMIT when a --max-delete commit was capped. */ diff --git a/src/server/server.c b/src/server/server.c index 3e67385..29faaea 100644 --- a/src/server/server.c +++ b/src/server/server.c @@ -990,9 +990,11 @@ static void server_run_mt_receiver(ServerSession* state) { server-contacting --dry-run deletes nothing (no manifest is sent). */ if (context->deferred_manifest) { size_t deleted = 0; - DeletePathObserver observer = config->report_deletes ? receiver_record_deleted_path : NULL; - DeleteCommitResult deletion = manifest_delete_all_observed( - config, context->deferred_manifest, &deleted, observer, (void*)context->deleted_paths); + ReceiverDeleteContext delctx = {&context->stats, context->deleted_paths}; + DeletePathObserver observer = + (delctx.stats || delctx.deleted_paths) ? receiver_record_deleted_path : NULL; + DeleteCommitResult deletion = manifest_delete_all_observed(config, context->deferred_manifest, + &deleted, observer, &delctx); context->stats.deleted_files += deleted; if (deletion == DELETE_COMMIT_ERROR) { transfer_ok = false; @@ -1010,9 +1012,10 @@ static void server_run_mt_receiver(ServerSession* state) { if (context->deferred_plans) { /* Defence in depth (the enclosing block already excludes dry-run): a -n run never commits a deletion. */ - if (config->report_deletes) - delete_plan_session_set_delete_observer( - context->deferred_plans, receiver_record_deleted_path, (void*)context->deleted_paths); + ReceiverDeleteContext delctx = {&context->stats, context->deleted_paths}; + if (delctx.stats || delctx.deleted_paths) + delete_plan_session_set_delete_observer(context->deferred_plans, + receiver_record_deleted_path, &delctx); DeleteCommitResult deletion = config->dry_run ? DELETE_COMMIT_OK : delete_plan_session_commit(context->deferred_plans, config); diff --git a/src/shared/delete.c b/src/shared/delete.c index a47cc3b..66a1aca 100644 --- a/src/shared/delete.c +++ b/src/shared/delete.c @@ -32,6 +32,17 @@ static bool keep_is_file(const PathIndex* index, const char* rel_path) { return path_index_contains(index, rel_path); } +/* Classify a removed entry from its st_mode for the per-type delete counters. */ +DeleteEntryType delete_entry_type_of_mode(mode_t mode) { + if (S_ISDIR(mode)) + return DELETE_ENTRY_DIR; + if (S_ISLNK(mode)) + return DELETE_ENTRY_LINK; + if (S_ISREG(mode)) + return DELETE_ENTRY_REG; + return DELETE_ENTRY_SPECIAL; +} + /* True when child_rel is, or lies below, a protected entry. A prefix "a" therefore protects "a" and "a/b/c" but not "ab". Entries with top_level_only set only protect DIRECT children of the receive root (at_root); nested @@ -126,6 +137,7 @@ bool delete_dir_entries_collect(int dirfd, DeleteDirEntry** out, size_t* count, break; } entries[used].is_dir = S_ISDIR(st.st_mode); + entries[used].mode = st.st_mode; used++; } closedir(dir); @@ -329,10 +341,10 @@ static bool delete_walk_fd(int dirfd, const char* rel_path, const PathIndex* kee memcpy(with_slash, child_rel, len); with_slash[len] = '/'; with_slash[len + 1] = '\0'; - state->observer(state->observer_context, with_slash); + state->observer(state->observer_context, with_slash, DELETE_ENTRY_DIR); free(with_slash); } else { - state->observer(state->observer_context, child_rel); + state->observer(state->observer_context, child_rel, DELETE_ENTRY_DIR); } } } @@ -373,7 +385,8 @@ static bool delete_walk_fd(int dirfd, const char* rel_path, const PathIndex* kee char* child_rel = path_cat((char*)rel_path, entries[i].name); if (child_rel) { if (state->observer) - state->observer(state->observer_context, child_rel); + state->observer(state->observer_context, child_rel, + delete_entry_type_of_mode(entries[i].mode)); char* escaped_path = output_escape(child_rel, log_get_8_bit_output()); fprintf(stderr, " Deleted: %s\n", escaped_path ? escaped_path : ""); free(escaped_path); diff --git a/src/shared/delete.h b/src/shared/delete.h index f0921a1..dd77231 100644 --- a/src/shared/delete.h +++ b/src/shared/delete.h @@ -5,6 +5,7 @@ #include "config.h" #include #include +#include /* Delete engine. * @@ -66,6 +67,9 @@ bool path_under_skip_prefix(const char* child_rel, bool at_root, const DeleteSki typedef struct { char* name; bool is_dir; + /* The entry's full st_mode from the AT_SYMLINK_NOFOLLOW stat, so a delete + observer can classify a removed non-directory as reg/link/special. */ + mode_t mode; } DeleteDirEntry; /* Collect the entries of the directory open on `dirfd` (excluding "." and ".."), stat'ing each with AT_SYMLINK_NOFOLLOW. On success *out is a malloc'd array of @@ -99,10 +103,24 @@ DeleteWalkResult delete_extras_limited(const char* dest_root, const ArrayList* m const FilterRuleList* protect_rules, size_t* deleted_out, size_t* skipped_out); +/* Entry kind of a removed path, reported to the delete observer so the receiver + can build rsync's `--stats` `Number of deleted files` per-type breakdown. The + four categories are a strict partition of every removed entry. */ +typedef enum { + DELETE_ENTRY_REG = 0, + DELETE_ENTRY_DIR, + DELETE_ENTRY_LINK, + DELETE_ENTRY_SPECIAL +} DeleteEntryType; + /* Optional per-deletion observer: called for each destination-relative path - actually removed (a file, symlink, or directory), in removal order, so the - receiver can stream rsync's `--info=del`/`--info=remove` lines. */ -typedef void (*DeletePathObserver)(void* context, const char* rel_path); + actually removed (a file, symlink, or directory) with its entry kind, in + removal order, so the receiver can stream rsync's `--info=del`/`--info=remove` + lines and tally the per-type `--stats` counters. */ +typedef void (*DeletePathObserver)(void* context, const char* rel_path, DeleteEntryType type); + +/* Classify a removed entry from its st_mode for the per-type delete counters. */ +DeleteEntryType delete_entry_type_of_mode(mode_t mode); /* `delete_extras_limited_observed` is delete_extras_limited with an optional * observer; the observer is invoked only for entries truly removed. When diff --git a/src/shared/delete_commit.c b/src/shared/delete_commit.c index 0a2b66f..fac8fc2 100644 --- a/src/shared/delete_commit.c +++ b/src/shared/delete_commit.c @@ -191,13 +191,13 @@ typedef struct { const char* prefix; } PrefixedDeleteObserver; -static void prefixed_delete_observer(void* context, const char* rel) { +static void prefixed_delete_observer(void* context, const char* rel, DeleteEntryType type) { PrefixedDeleteObserver* prefixed = context; if (!prefixed->inner || !rel) return; char* joined = path_cat((char*)prefixed->prefix, rel); if (joined) { - prefixed->inner(prefixed->inner_context, joined); + prefixed->inner(prefixed->inner_context, joined, type); free(joined); } } @@ -359,7 +359,7 @@ static bool delete_missing_args_budgeted_observed(const Config* config, if (removed) { budget->deleted++; if (observer) - observer(observer_context, rel); + observer(observer_context, rel, delete_entry_type_of_mode(st.st_mode)); char* escaped = output_escape(rel, log_get_8_bit_output()); fprintf(stderr, " Deleted: %s\n", escaped ? escaped : ""); free(escaped); diff --git a/src/shared/delete_plan.c b/src/shared/delete_plan.c index f69213b..6b2b84d 100644 --- a/src/shared/delete_plan.c +++ b/src/shared/delete_plan.c @@ -478,9 +478,9 @@ struct DeletePlanSession { }; /* Report one path the session truly removed (no-op without an observer). */ -static void notify_deleted(DeletePlanSession* session, const char* rel) { +static void notify_deleted(DeletePlanSession* session, const char* rel, DeleteEntryType type) { if (session && session->observer && rel) - session->observer(session->observer_context, rel); + session->observer(session->observer_context, rel, type); } /* A removed directory is reported with rsync's trailing slash (`deleting dir/`) @@ -491,13 +491,13 @@ static void notify_deleted_dir(DeletePlanSession* session, const char* rel) { size_t len = strlen(rel); char* with_slash = malloc(len + 2); if (!with_slash) { - session->observer(session->observer_context, rel); + session->observer(session->observer_context, rel, DELETE_ENTRY_DIR); return; } memcpy(with_slash, rel, len); with_slash[len] = '/'; with_slash[len + 1] = '\0'; - session->observer(session->observer_context, with_slash); + session->observer(session->observer_context, with_slash, DELETE_ENTRY_DIR); free(with_slash); } @@ -711,8 +711,8 @@ static bool process_extra_dir(int dirfd, const char* name, const char* child_rel return errno == ENOTEMPTY || errno == EEXIST; } -static bool process_extra_file(int dirfd, const char* name, const char* child_rel, bool force_now, - DeletePlanSession* session) { +static bool process_extra_file(int dirfd, const char* name, const char* child_rel, mode_t mode, + bool force_now, DeletePlanSession* session) { if (session->defer && !force_now) { return defer_add(session, child_rel); } @@ -724,7 +724,7 @@ static bool process_extra_file(int dirfd, const char* name, const char* child_re session->deleted++; session->planned++; log_deleted(child_rel); - notify_deleted(session, child_rel); + notify_deleted(session, child_rel, delete_entry_type_of_mode(mode)); } else if (errno != ENOENT) { return false; } @@ -828,7 +828,8 @@ static bool process_children(int dirfd, const char* dir_rel, const ArrayList* ke operation_ok = false; continue; } - if (!process_extra_file(dirfd, entries[i].name, child_rel, force[i] || force_now, session)) + if (!process_extra_file(dirfd, entries[i].name, child_rel, entries[i].mode, + force[i] || force_now, session)) operation_ok = false; free(child_rel); } @@ -1032,7 +1033,7 @@ static bool apply_deferred_path(DeletePlanSession* session, const Config* config session->deleted++; session->planned++; log_deleted(rel); - notify_deleted(session, rel); + notify_deleted(session, rel, delete_entry_type_of_mode(st.st_mode)); } else if (errno != ENOENT) { close(parent_fd); free(leaf); diff --git a/src/shared/format.c b/src/shared/format.c index df2c463..91b4cc0 100644 --- a/src/shared/format.c +++ b/src/shared/format.c @@ -62,14 +62,16 @@ bool format_dest_state_send(int fd, const OutputDestState* state) { if (!state) return false; int32_t has_old = state->existed ? 1 : 0; + int32_t target_matches = state->target_matches ? 1 : 0; uint64_t size = (uint64_t)state->size; int64_t mtime = (int64_t)state->mtime_sec; int64_t mtime_nsec = state->mtime_nsec; uint32_t mode = state->mode; int32_t uid = state->uid; int32_t gid = state->gid; - return send_n_data(fd, &has_old, sizeof(has_old)) && send_n_data(fd, &size, sizeof(size)) && - send_n_data(fd, &mtime, sizeof(mtime)) && + return send_n_data(fd, &has_old, sizeof(has_old)) && + send_n_data(fd, &target_matches, sizeof(target_matches)) && + send_n_data(fd, &size, sizeof(size)) && send_n_data(fd, &mtime, sizeof(mtime)) && send_n_data(fd, &mtime_nsec, sizeof(mtime_nsec)) && send_n_data(fd, &mode, sizeof(mode)) && send_n_data(fd, &uid, sizeof(uid)) && send_n_data(fd, &gid, sizeof(gid)); } @@ -78,14 +80,16 @@ bool format_dest_state_receive(int fd, OutputDestState* state) { if (!state) return false; int32_t has_old = 0; + int32_t target_matches = 0; uint64_t size = 0; int64_t mtime = 0; int64_t mtime_nsec = 0; uint32_t mode = 0; int32_t uid = 0; int32_t gid = 0; - if (!receive_n_data(fd, &has_old, sizeof(has_old)) || !receive_n_data(fd, &size, sizeof(size)) || - !receive_n_data(fd, &mtime, sizeof(mtime)) || + if (!receive_n_data(fd, &has_old, sizeof(has_old)) || + !receive_n_data(fd, &target_matches, sizeof(target_matches)) || + !receive_n_data(fd, &size, sizeof(size)) || !receive_n_data(fd, &mtime, sizeof(mtime)) || !receive_n_data(fd, &mtime_nsec, sizeof(mtime_nsec)) || !receive_n_data(fd, &mode, sizeof(mode)) || !receive_n_data(fd, &uid, sizeof(uid)) || !receive_n_data(fd, &gid, sizeof(gid))) @@ -93,6 +97,7 @@ bool format_dest_state_receive(int fd, OutputDestState* state) { memset(state, 0, sizeof(*state)); state->known = true; state->existed = has_old != 0; + state->target_matches = target_matches != 0; state->size = size; state->mtime_sec = mtime; state->mtime_nsec = mtime_nsec; @@ -105,9 +110,10 @@ bool format_dest_state_receive(int fd, OutputDestState* state) { bool format_stats_send(int fd, const ReceiverStats* stats) { if (!stats) return false; - unsigned long long fields[8] = { + unsigned long long fields[12] = { stats->matched_data, stats->deleted_files, stats->would_delete_count, stats->literal_bytes, stats->created_reg, stats->created_dir, stats->created_link, stats->created_special, + stats->deleted_reg, stats->deleted_dir, stats->deleted_link, stats->deleted_special, }; return send_n_data(fd, fields, sizeof(fields)); } @@ -115,7 +121,7 @@ bool format_stats_send(int fd, const ReceiverStats* stats) { bool format_stats_receive(int fd, ReceiverStats* stats) { if (!stats) return false; - unsigned long long fields[8] = {0}; + unsigned long long fields[12] = {0}; if (!receive_n_data(fd, fields, sizeof(fields))) return false; memset(stats, 0, sizeof(*stats)); @@ -127,5 +133,9 @@ bool format_stats_receive(int fd, ReceiverStats* stats) { stats->created_dir = fields[5]; stats->created_link = fields[6]; stats->created_special = fields[7]; + stats->deleted_reg = fields[8]; + stats->deleted_dir = fields[9]; + stats->deleted_link = fields[10]; + stats->deleted_special = fields[11]; return true; } diff --git a/src/shared/format.h b/src/shared/format.h index 9da9a93..99d5dd5 100644 --- a/src/shared/format.h +++ b/src/shared/format.h @@ -17,10 +17,18 @@ /* Pre-transfer destination snapshot, reported by the receiver when the wire * config carries report_dest_info. `known` distinguishes "no report was * requested/received" from "the destination did not exist" (`existed == false` - * with `known == true`). */ + * with `known == true`). + * + * `target_matches` is meaningful only for a symlink destination (protocol + * 2.30.0): the receiver compares its on-disk link target with the incoming + * target and reports whether they are equal, so the sender can render rsync's + * `cLc........` (target changed) versus `.L..t......` (attributes only) and + * suppress an unchanged symlink's line entirely. It is always false for every + * other entry kind. */ typedef struct { bool known; bool existed; + bool target_matches; unsigned long long size; long long mtime_sec; long long mtime_nsec; @@ -57,17 +65,24 @@ bool format_dest_state_send(int fd, const OutputDestState* state); bool format_dest_state_receive(int fd, OutputDestState* state); /* End-of-transfer receiver counters reported through STATUS_STATS (protocol - * 2.25.0, extended in 2.28.0) when the wire config carries report_stats. - * `would_delete_count` is the number of destination-relative paths the receiver - * would have deleted in a -n/--dry-run --delete run; that many wire strings - * immediately follow the fixed record (sent/read by the caller). + * 2.25.0, extended in 2.28.0 and 2.30.0) when the wire config carries + * report_stats. `would_delete_count` is the number of destination-relative + * paths the receiver would have deleted in a -n/--dry-run --delete run; that + * many wire strings immediately follow the fixed record (sent/read by the + * caller). * * Protocol 2.28.0 adds the receiver-observed counters the sender cannot see: * `literal_bytes` is the file data the receiver actually stored literally * (whole files plus the literal fragments of a delta) and the four `created_*` * counters split the destination entries the receiver newly created by type, * reproducing rsync's `Number of created files` breakdown and an exact - * `Literal data` for a delta run. */ + * `Literal data` for a delta run. + * + * Protocol 2.30.0 appends the four `deleted_*` counters: the same reg/dir/link/ + * special split for the entries the receiver ACTUALLY removed, so `--stats` can + * render rsync's `Number of deleted files: X (reg: A, dir: B, link: C, + * special: D)` parenthetical. The scalar `deleted_files` stays the authoritative + * total (the breakdown is a strict partition of it). */ typedef struct { unsigned long long matched_data; unsigned long long deleted_files; @@ -77,6 +92,10 @@ typedef struct { unsigned long long created_dir; unsigned long long created_link; unsigned long long created_special; + unsigned long long deleted_reg; + unsigned long long deleted_dir; + unsigned long long deleted_link; + unsigned long long deleted_special; } ReceiverStats; /* Fixed-width STATUS_STATS counter record. The status frame and the optional diff --git a/src/shared/protocol.h b/src/shared/protocol.h index 97edddd..4a04def 100644 --- a/src/shared/protocol.h +++ b/src/shared/protocol.h @@ -129,8 +129,13 @@ enum NET_STATUS { STATUS_KEEPALIVE, STATUS_ABORT, STATUS_CHECK_BATCH, - /* An explicit directory entry (--dirs): the sender transmits only the path; - * the receiver creates the directory below the receive root. */ + /* An explicit directory entry (--dirs / an empty source directory): the sender + * transmits the path and, when metadata/xattrs are negotiated, their blocks; + * the receiver creates the directory below the receive root. Protocol 2.30.0 + * inserts an int32 probe flag right after the status when report_dest_info is + * negotiated: probe=1 is a report-only frame (path only; the receiver answers + * STATUS_DEST_INFO and creates nothing), probe=0 is a real create that is + * answered with the directory's pre-transfer state before it is created. */ STATUS_MKDIR, /* --append / --append-verify tail resume. STATUS_APPEND is sent by the * receiver after a per-file STATUS_CHECK when the existing destination file @@ -213,16 +218,22 @@ enum NET_STATUS { * limit stopped deletions"). Appended after STATUS_DRY_RUN_TRANSFER so no * existing status is renumbered. */ STATUS_DELETE_LIMIT, - /* Destination-state report for output parity (protocol 2.23.0). When the - * wire config carries report_dest_info=true, the receiver answers every - * per-file STATUS_CHECK request with STATUS_DEST_INFO FIRST, followed by a - * fixed record describing the pre-transfer destination entry - * (int32 has_old; uint64 size; int64 mtime; int64 mtime_nsec; uint32 mode; - * int32 uid; int32 gid). The ordinary STATUS_OK/STATUS_NEXT/... verdict - * follows, so the sender can render rsync-accurate -i/--out-format columns - * (new vs modified, and which of size/time/perms/owner/group differ) without - * changing the transfer decision itself. Appended after - * STATUS_DELETE_LIMIT so no existing status is renumbered. */ + /* Destination-state report for output parity (protocol 2.23.0; extended to + * directories/symlinks in 2.30.0). When the wire config carries + * report_dest_info=true, the receiver answers every per-file STATUS_CHECK + * request with STATUS_DEST_INFO FIRST, followed by a fixed record describing + * the pre-transfer destination entry (int32 has_old; int32 target_matches; + * uint64 size; int64 mtime; int64 mtime_nsec; uint32 mode; int32 uid; + * int32 gid). The ordinary STATUS_OK/STATUS_NEXT/... verdict follows, so the + * sender can render rsync-accurate -i/--out-format columns (new vs modified, + * and which of size/time/perms/owner/group differ) without changing the + * transfer decision itself. Protocol 2.30.0 also uses this record for + * STATUS_MKDIR and STATUS_SYMLINK: the sender consumes it into the entry's + * dest_state before emitting its change line, and target_matches reports + * whether an existing symlink's on-disk target already equals the incoming + * one (so the sender can render `cLc........` vs `.L..t......` and suppress + * an unchanged symlink). Appended after STATUS_DELETE_LIMIT so no existing + * status is renumbered. */ STATUS_DEST_INFO, /* Per-directory delete plan (protocol 2.24.0). The sender of a * --delete-during/--delete-delay transfer streams one frame per source @@ -244,8 +255,10 @@ enum NET_STATUS { * config carries report_stats=true, the receiver sends this status once, * immediately before its terminal success status, followed by a fixed stats * record (see format_stats_send/receive in format.h) and, when the run is a - * --dry-run with --delete, the would-delete path list. Appended after - * STATUS_DELETE_PLAN so no existing status is renumbered. */ + * --dry-run with --delete, the would-delete path list. Protocol 2.30.0 + * appends the four deleted_reg/dir/link/special counters to that record, so + * --stats can render rsync's `Number of deleted files` per-type breakdown. + * Appended after STATUS_DELETE_PLAN so no existing status is renumbered. */ STATUS_STATS, /* Client diagnostic channel (protocol 2.30.0, rsync's --stderr=client / * --no-msgs2stderr). When the client's --stderr mode is `client`, the diff --git a/tests/integration/test_differential_parity.py b/tests/integration/test_differential_parity.py index a5ee3d2..5e0261c 100644 --- a/tests/integration/test_differential_parity.py +++ b/tests/integration/test_differential_parity.py @@ -17,6 +17,7 @@ Run locally:: python3 -m pytest tests/integration/test_differential_parity.py -n 4 --dist=load -m parity """ import os +import re import shutil import sys import warnings @@ -666,6 +667,157 @@ def test_added_and_deleted_between_runs(parity_server_factory): _run_and_check(case_id, result) +def _seed_dest_tree(src, root): + """Copy `src`'s tree into `root` (the transfer mirror), preserving symlinks + and directory mtimes, so a second differential run starts from an existing + destination exactly like a seeded rsync run.""" + os.makedirs(root, exist_ok=True) + for dirpath, dirnames, filenames in os.walk(src): + rel = os.path.relpath(dirpath, src) + for name in dirnames: + s = os.path.join(dirpath, name) + d = os.path.join(root, rel, name) if rel != "." else os.path.join(root, name) + if os.path.islink(s): + continue + os.makedirs(d, exist_ok=True) + for name in filenames: + s = os.path.join(dirpath, name) + d = os.path.join(root, rel, name) if rel != "." else os.path.join(root, name) + os.makedirs(os.path.dirname(d), exist_ok=True) + if os.path.islink(s): + if os.path.lexists(d): + os.remove(d) + os.symlink(os.readlink(s), d) + else: + shutil.copy2(s, d) + if rel != ".": + os.utime(os.path.join(root, rel), None) + os.utime(root, None) + + +# A full rsync itemize code (11 columns) followed by the name. H._ITEMIZE_RE +# only matches created (`+`) entries, so the changed-attribute codes this test +# asserts need their own matcher. +_ITEMIZE_LINE_RE = re.compile(r"^[<>ch.*][fdLDS].{9} ") + + +def _itemize_dir_link_lines(text): + """The itemize lines for directory and symlink entries, excluding the + transfer-root `./` line (FastSync emits it unconditionally; a documented + residual).""" + out = [] + for line in (text or "").splitlines(): + line = line.rstrip() + if not line or not _ITEMIZE_LINE_RE.match(line): + continue + name = line.rsplit(" ", 1)[-1] + if name == "./": + continue + if name.endswith("/") or " -> " in line: + out.append(line) + return sorted(out) + + +@requires_rsync +@parity +def test_itemize_rerun_dirs_symlinks_matches_rsync(parity_server_factory): + """#314: a re-run reports directory/symlink destination state like rsync. + + On an unchanged tree FastSync emits no per-directory `cd+++++++++` (or + symlink) lines, and after a changed directory mtime / symlink target it + renders rsync's `.d..t......` / `cLc........` instead of `cd`/`cL`.""" + case_id = "itemize_rerun_dirs_symlinks" + src = os.path.join(TEST_DATA_DIR, "parity_itemds_src") + rdst = os.path.join(TEST_DATA_DIR, "parity_itemds_rdst") + fdst = os.path.join(TEST_DATA_DIR, "parity_itemds_fdst") + clean_dir(src) + _mk(os.path.join(src, "sub", "b.txt"), b"nested\n") + os.makedirs(os.path.join(src, "emptydir"), exist_ok=True) + os.symlink("a.txt", os.path.join(src, "link")) + _mk(os.path.join(src, "a.txt"), b"top\n") + clean_dir(rdst) + clean_dir(fdst) + server = parity_server_factory(SUPER) + rroot = rdst + froot = get_dest_received_dir(fdst, src) + _seed_dest_tree(src, rroot) + _seed_dest_tree(src, froot) + + # Unchanged re-run: no directory or symlink itemize lines from either tool. + rs = H.run_rsync(src, rdst, ["-a", "-i"]) + fs, _ = H.run_fastsync(src, fdst, ["-a", "-i", "--incremental"], server.port) + assert rs.returncode == 0, rs.stderr + assert fs.returncode == 0, fs.stderr + assert _itemize_dir_link_lines(rs.stdout) == [] + fast_unchanged = _itemize_dir_link_lines(fs.stdout) + assert fast_unchanged == [], f"unchanged re-run itemized dirs/links: {fast_unchanged}" + + # Change the directory mtime and the symlink target, then re-run. + _pin(os.path.join(src, "sub"), _OLD_MTIME) + os.remove(os.path.join(src, "link")) + os.symlink("b.txt", os.path.join(src, "link")) + rs = H.run_rsync(src, rdst, ["-a", "-i"]) + fs, _ = H.run_fastsync(src, fdst, ["-a", "-i", "--incremental"], server.port) + assert rs.returncode == 0, rs.stderr + assert fs.returncode == 0, fs.stderr + expected = _itemize_dir_link_lines(rs.stdout) + actual = _itemize_dir_link_lines(fs.stdout) + assert actual == expected, f"rsync={rs.stdout!r} fastsync={fs.stdout!r}" + assert any(line.endswith(" sub/") and line.startswith(".d..t") for line in actual), actual + assert any(line.startswith("cLc") and " -> b.txt" in line for line in actual), actual + + +def _deleted_breakdown_line(text): + for line in (text or "").splitlines(): + if line.startswith("Number of deleted files:"): + return " ".join(line.split()) + return "" + + +@requires_rsync +@parity +def test_stats_deleted_breakdown_matches_rsync(parity_server_factory): + """#316: `--stats` renders rsync's per-type `Number of deleted files` + breakdown for removed regular files, directories, symlinks and a special.""" + case_id = "stats_deleted_breakdown" + src = os.path.join(TEST_DATA_DIR, "parity_delbd_src") + rdst = os.path.join(TEST_DATA_DIR, "parity_delbd_rdst") + fdst = os.path.join(TEST_DATA_DIR, "parity_delbd_fdst") + clean_dir(src) + _mk(os.path.join(src, "keep.txt"), b"keep\n") + server = parity_server_factory(DELETE) + + def seed(_src, rroot, froot): + for root in (rroot, froot): + _mk(os.path.join(root, "extra1.txt"), b"e1\n", _OLD_MTIME) + _mk(os.path.join(root, "extradir", "inside.txt"), b"e2\n", _OLD_MTIME) + os.makedirs(os.path.join(root, "extradir"), exist_ok=True) + link = os.path.join(root, "extralink") + if not os.path.lexists(link): + os.symlink("keep.txt", link) + fifo = os.path.join(root, "extrafifo") + if not os.path.exists(fifo): + os.mkfifo(fifo) + + def extra(_src, _rroot, _froot, rs, fs): + rs_line = _deleted_breakdown_line(rs.stdout) + fs_line = _deleted_breakdown_line(fs.stdout) + if not rs_line: + return ["rsync printed no deleted-files line"] + if rs_line != fs_line: + return [f"deleted breakdown rsync={rs_line!r} fastsync={fs_line!r}"] + if "reg:" not in rs_line or "dir:" not in rs_line or \ + "link:" not in rs_line or "special:" not in rs_line: + return [f"breakdown missing a category: {rs_line!r}"] + return [] + + result = H.run_differential( + src, rdst, fdst, ["-a", "--delete", "--stats"], + ["-a", "--delete", "--stats", "--incremental"], server, + seed=seed, extra_check=extra) + _run_and_check(case_id, result, ref="--stats deleted per-type breakdown") + + @requires_rsync @parity def test_one_file_system(parity_server_factory): diff --git a/tests/test_format.c b/tests/test_format.c index 7343e26..820fb1c 100644 --- a/tests/test_format.c +++ b/tests/test_format.c @@ -64,6 +64,7 @@ static void test_dest_state_roundtrip() { memset(&out, 0, sizeof(out)); out.known = true; out.existed = true; + out.target_matches = true; out.size = 123456789ULL; out.mtime_sec = 1700000000; out.mtime_nsec = 123456789; @@ -76,6 +77,7 @@ static void test_dest_state_roundtrip() { EXPECT_TRUE(format_dest_state_receive(fds[1], &in)); EXPECT_TRUE(in.known); EXPECT_TRUE(in.existed); + EXPECT_TRUE(in.target_matches); EXPECT_TRUE(in.size == out.size); EXPECT_TRUE(in.mtime_sec == out.mtime_sec); EXPECT_TRUE(in.mtime_nsec == out.mtime_nsec); @@ -103,6 +105,10 @@ static void test_stats_roundtrip() { out.created_dir = 4; out.created_link = 2; out.created_special = 1; + out.deleted_reg = 9; + out.deleted_dir = 6; + out.deleted_link = 3; + out.deleted_special = 2; ReceiverStats in; memset(&in, 0, sizeof(in)); EXPECT_TRUE(format_stats_send(fds[0], &out)); @@ -115,6 +121,10 @@ static void test_stats_roundtrip() { EXPECT_TRUE(in.created_dir == out.created_dir); EXPECT_TRUE(in.created_link == out.created_link); EXPECT_TRUE(in.created_special == out.created_special); + EXPECT_TRUE(in.deleted_reg == out.deleted_reg); + EXPECT_TRUE(in.deleted_dir == out.deleted_dir); + EXPECT_TRUE(in.deleted_link == out.deleted_link); + EXPECT_TRUE(in.deleted_special == out.deleted_special); close(fds[0]); close(fds[1]); } diff --git a/tests/test_multiprocessing.c b/tests/test_multiprocessing.c index 5aa3be3..7581e41 100644 --- a/tests/test_multiprocessing.c +++ b/tests/test_multiprocessing.c @@ -153,7 +153,8 @@ static void test_receiver_deleted_paths_gated_by_report_deletes() { PipelineContextReceiver* ctx_info = pipeline_context_receiver_create(info, q_info, -1, NULL); EXPECT_NOT_NULL(ctx_info); EXPECT_NOT_NULL(ctx_info->deleted_paths); - receiver_record_deleted_path(ctx_info->deleted_paths, "d/old_extra"); + ReceiverDeleteContext delctx = {NULL, ctx_info->deleted_paths}; + receiver_record_deleted_path(&delctx, "d/old_extra", DELETE_ENTRY_REG); EXPECT_EQ_INT(ctx_info->deleted_paths->size, 1); EXPECT_EQ_STR((const char*)ctx_info->deleted_paths->items[0], "d/old_extra"); pipeline_context_receiver_destroy(ctx_info);