From cb2979fdf14ac680aaa843c1d89198ef68a4f253 Mon Sep 17 00:00:00 2001 From: TapTap Date: Wed, 23 Sep 2026 20:33:27 +0200 Subject: [PATCH] feat(protocol): client-message channel and rsync partial exit 23 (2.30.0) --- CHANGELOG.md | 14 +++ CMakeLists.txt | 2 +- README.md | 6 +- RSYNC_COMPAT.md | 12 +-- src/client/client_cli.c | 15 ++- src/client/client_report.c | 107 ++++++++++++++++++++++ src/client/client_send.c | 73 +++++++++++++-- src/client/client_send_internal.h | 7 ++ src/client/usage.c | 8 +- src/server/receiver.c | 33 ++++++- src/server/server.c | 2 +- src/shared/config.h | 17 +++- src/shared/log.c | 100 ++++++++++++++++---- src/shared/log.h | 21 ++++- src/shared/multiprocessing.c | 1 + src/shared/multiprocessing.h | 5 + src/shared/protocol.c | 21 ++++- src/shared/protocol.h | 31 ++++++- tests/integration/test_fault_injection.py | 2 +- tests/integration/test_features.py | 89 +++++++++++++++++- tests/integration/test_preflight.py | 6 +- tests/test_client_cli.c | 42 +++++++-- tests/test_config.c | 13 ++- tests/test_log.c | 49 ++++++++++ tests/test_protocol.c | 48 +++++++++- 25 files changed, 633 insertions(+), 91 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d2292e3..417e701 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,20 @@ run the same version because the handshake is strict. ## [Unreleased] +Wire backlog cycle (protocol 2.29.0 → 2.30.0; config-frame layout unchanged). + +- **`--stderr=client` client-message channel (#313):** the client now accepts + `--stderr=client` (and maps the deprecated `--no-msgs2stderr` to it), routing + its own diagnostics over the new bounded `STATUS_CLIENT_MSG` client->server + frame instead of writing them locally; the server writes each received + message to its stderr (respecting the server log destination). `errors`/`all` + behavior is unchanged. +- **Receiver partial failures exit 23 (#320):** a per-entry receiver failure + that does not abort the stream (e.g. an unprivileged `--devices` mknod) now + sends the terminal `STATUS_PARTIAL`; the client exits 23 like rsync and, under + `--remove-source-files`, still removes the sources it successfully + transferred. A clean run stays 0 and a fatal/connection error stays non-23. + ## [2.29.0] - 2026-09-23 The rsync-parity cycle 2.29 (no wire change; `PROTOCOL_VERSION` stays 2.28.0). diff --git a/CMakeLists.txt b/CMakeLists.txt index 308fa82..a75efe6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.22) -project(FastFileTransfer VERSION 2.29.0) +project(FastFileTransfer VERSION 2.30.0) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) set(CMAKE_C_STANDARD 11) diff --git a/README.md b/README.md index 67d548c..af2240f 100644 --- a/README.md +++ b/README.md @@ -674,9 +674,9 @@ remote SSH argv is already built injection-safe. | `--outbuf=MODE` | stdout/stderr buffering: `N` (none/unbuffered), `L` (line-buffered), or `B` (block-buffered, default). | | `--log-file ` | Write log output to a file. | | `--log-file-format=FORMAT` | Per-file log-line format (requires `--log-file`). | -| `--stderr=MODE` | Route logging to stderr: `errors` or `all`. | +| `--stderr=MODE` | Route logging: `errors` (default), `all`, or `client` (forward the client's diagnostics to the server's stderr over the client-message channel). | | `--msgs2stderr` | Route all messages to stderr (deprecated spelling of `--stderr=all`). | -| `--no-msgs2stderr` | Select errors-only stderr (deprecated spelling; the default). | +| `--no-msgs2stderr` | Forward the client's diagnostics to the server (deprecated spelling of `--stderr=client`). | | `-V`, `--version` | Print the FastSync protocol version. | | `--help` | Print command usage. | @@ -832,7 +832,7 @@ before the module list, before authentication, and the connecting peer address ## Protocol and Security -FastSync protocol version `2.29.0` is shared by the client and server. The +FastSync protocol version `2.30.0` is shared by the client and server. The current protocol is sender-driven and includes configuration negotiation, including the maximum allocation limit, incremental checks, checksums, manifests, keep-alives, abort handling, per-file remove-source results, and diff --git a/RSYNC_COMPAT.md b/RSYNC_COMPAT.md index 031c4f6..d13ccb6 100644 --- a/RSYNC_COMPAT.md +++ b/RSYNC_COMPAT.md @@ -110,8 +110,8 @@ Every one of those has an entry below with its remaining caveats. | `-V`, `--version` | Print version | ✅ Parity | | | `--info=FLAGS` | Fine-grained info verbosity | ⚠️ Caveat | Accepts rsync 3.4.1's full `--info` vocabulary — `backup`, `copy`, `del`, `flist`, `misc`, `mount`, `name`, `nonreg`, `progress`, `remove`, `skip`, `stats`, `symsafe`, `all`, `none` — with optional level suffixes (`--info=stats2`), so a valid rsync invocation is never rejected up front. Protocol 2.27.0 wires the categories that map to a real FastSync event, matching rsync's line format: `name` prints the updated entry names (with the ` -> target` link suffix), `flist` prints `sending incremental file list`, `del` prints `deleting PATH` (or `*deleting PATH` under `-i`/`--out-format`) for both dry-run would-delete and real deletions (real runs carry the removed paths over the new `report_deletes` wire bool), `remove` prints `sender removed PATH`, `nonreg` prints `skipping non-regular file "NAME"`, `progress` drives the per-file progress output, `copy`/`misc`/`skip` keep their existing channels, `stats` enables the same transfer-statistics block as `--stats`, and `mount` prints rsync's `[sender] skipping mount-point dir NAME` when `-xx` drops a mount-point directory (plain `-x` keeps the empty directory entry and stays silent, matching rsync; both differential-tested). `none` suppresses info output, explicit flags override `--verbose`, and a genuinely unknown name is still rejected by name (matching rsync). **Fixed (no-wire):** `--info=name2` (and higher) also prints rsync's `NAME is uptodate` lines for entries the receiver already has, and `--info=name` emits the leading transfer-root `./` name line before the first transferred entry (the marker rides in the existing `info_level` bitset; differential tests vs rsync 3.4.1). **Caveat:** the root `./` line is emitted before the first transferred name rather than keyed off rsync's root-attribute-change decision, so a pre-existing root that rsync leaves untouched can differ; the categories with no client-observable event stay accepted-but-silent — `symsafe` and `backup` (the backup happens on the receiver, which FastSync's protocol does not echo back); and `skip` maps to FastSync's sender-side skip logging rather than rsync's receiver-side "not creating new file" lines | | `--debug=FLAGS` | Fine-grained debug verbosity | ⚠️ Caveat | Protocol 2.26.0 accepts rsync 3.4.1's full `--debug` vocabulary with optional level suffixes. FastSync emits for its own channels (`io`, `proto`, `pack`, `util`, plus the aliases `hl`/`owner`) and maps the remaining categories that have a natural FastSync event onto real debug output: `flist` (per-directory scan progress), `del` (receiver-removed paths, riding the existing `report_deletes` wire bool), `hash`/`deltasum` (whole-file hashing and delta-sum generation), `recv` (receiver verdicts/signatures), `filter` (selection/exclusion decisions) and `send` (files handed to the sender). A normal run prints none of it; `--debug=help` lists the flags and a genuinely unknown name is rejected by name. **Caveat:** the output is FastSync's own timestamped debug format (it does not reproduce rsync's exact per-category lines), and the synthetic/rsync-internal categories (`acl`, `backup`, `bind`, `time`, ...) stay accepted-but-silent, so the row remains ⚠️ | -| `--stderr=MODE` | Change stderr output mode | ❌ Divergent | `errors` (default) and `all` are supported; `client` is rejected with a clear error (`--stderr=client is not supported`) because FastSync has no rsync client-message channel — the rejection itself is the documented behavior (Phase 7 Wave B decision). The modes that exist work; the missing rsync channel cannot be emulated without a wire change | -| `--msgs2stderr`, `--no-msgs2stderr` | Deprecated `--stderr` aliases | ⚠️ Caveat | `--msgs2stderr` maps to `--stderr=all` (supported, matching rsync). `--no-msgs2stderr` is rsync's spelling of `--stderr=client`, which FastSync has no client-message channel for, so it maps to the errors-only default instead of reproducing rsync's client mode. See `--stderr=MODE` | +| `--stderr=MODE` | Change stderr output mode | ⚠️ Caveat | `errors` (default) and `all` are supported and match rsync. As of protocol 2.30.0 `client` is accepted, and the client's own diagnostics are forwarded to the peer's stderr over the new bounded `STATUS_CLIENT_MSG` client-message channel (the server writes each received message to its stderr respecting the server log destination) instead of writing locally. Caveat: rsync's `client` mode is its historical single-client-process multiplexing of every process's messages (the client's errors on its own stderr, info on stdout), whereas FastSync's push-only protocol has no server->client message stream, so only the client->server direction is reproduced | +| `--msgs2stderr`, `--no-msgs2stderr` | Deprecated `--stderr` aliases | ⚠️ Caveat | `--msgs2stderr` maps to `--stderr=all` (supported, matching rsync). `--no-msgs2stderr` is rsync's spelling of `--stderr=client`; as of protocol 2.30.0 it maps to the `client` mode and forwards the client's diagnostics to the server's stderr over the `STATUS_CLIENT_MSG` channel instead of the old errors-only approximation. See `--stderr=MODE` for the one-direction caveat | | `--no-motd` | Suppress daemon MOTD | ✅ Parity | Client-only display switch (Wave C): the daemon still sends the configured `motd file` on a `host::module/path` connection; the client reads and discards the frame without showing it. Without the flag the MOTD is printed to stdout after the config/auth handshake and escaped so control bytes cannot inject terminal sequences | | `--exclude=PATTERN` | Exclude files matching pattern | ✅ Parity | Glob matching in scanner | | `--include=PATTERN` | Include files matching pattern | ✅ Parity | Glob matching in scanner | @@ -342,7 +342,7 @@ why plain `--append` works on the normal atomic path, not only with `--inplace`. | `-X`, `--xattrs` | Preserve extended attributes | ❌ Divergent | Deliberately restricted to unprivileged `user.*` extended attributes plus the two POSIX ACL xattrs; `security.*` (SELinux, capabilities, ...) and `trusted.*` are **never** captured or applied — a client can never force a privileged attribute onto the destination, and the receiver independently re-validates every incoming name against the whitelist. This is a security-policy divergence from rsync, which can preserve the privileged namespaces with the needed privilege; implementing them would defeat FastSync's privilege-escalation guard. `user.*` capture/apply matches rsync in a differential test. Payloads are bounded on both ends. Incompatible with `-s`. **Symlink xattrs are now carried (protocol 2.29.0):** a symlink entry appends the same bounded trailing xattr block to its `STATUS_SYMLINK` frame as every other entry kind, captured with `llistxattr`/`lgetxattr` so the link's OWN attributes are read and never the referent's, and re-applied no-follow with `lsetxattr` through the already-confined parent directory (`fsetxattr` cannot target a symlink: there is no `*at` xattr syscall and an `O_PATH` fd is rejected). On Linux the VFS refuses to associate xattrs with a symlink at all — every `lsetxattr` on a link fails with `EPERM` for `user.*`, `trusted.*` and `security.*`, even as root, verified in the CI container — so on FastSync's supported platforms the captured block is always empty and the apply is a no-op; the wire block is present for correctness and for a filesystem/platform that does support symlink xattrs. rsync 3.4.1's `--fake-super` is not a counterexample: it stores a symlink as a regular file whose `user.rsync.%stat` records the `S_IFLNK` mode bits, not an xattr on a real symlink. The row stays divergent only for the never-preserved privileged namespaces above | | `-H`, `--hard-links` | Preserve hard links | ✅ Parity | Files on the source that share an inode (`st_dev`+`st_ino`, e.g. a `cp -al` tree) are re-created as hard links to one another on the destination, so duplicate links stay deduplicated and only the first member's data is sent (later members are transmitted as payload-less `STATUS_HARDLINK` frames). The receiver links each sibling to the first member's installed file with an atomic link + rename; on `link()` failure it falls back to a byte-identical local copy of the first member, never a partial/corrupt file. Requires the sequential scan for ordering (the first member is always emitted and installed before any sibling is linked). Works single-threaded and under `-j`/`--threads`, `--inplace`, `--delay-updates` (links staged and published by rename) and `--partial`. Crosses the wire (`preserve_hard_links` bool; `PROTOCOL_VERSION` bumped **2.11.0 → 2.12.0**, peers must match). Incompatible with `-s` (chunk serialization) and `--append`/`--append-verify`, rejected up front with a distinct error. See the Phase-4 hard-links notes below | | `-D` | Same as --devices --specials | ✅ Parity | Implies `--devices --specials`. `-D` was unassigned in FastSync (verified: no collision), so it is free to imply both device-node and special-file preservation. As of protocol 2.23.0 `--specials` genuinely covers **both FIFOs and unix sockets**, so `-D` covers the full rsync set. See the `--devices`/`--specials` rows and the Phase-4 devices notes below | -| `--devices` | Preserve device files | ⚠️ Caveat | Recreates char/block device nodes with `mknodat` (type + rdev strictly validated, confined fd-relative below the receive root). A device whose `mknodat` fails with `EPERM`/`EACCES` (no `CAP_MKNOD`, or super-user activity forbidden) is a **per-entry failure**: FastSync logs `cannot create device ...` (rsync logs `mknod ... failed`), counts it, **continues with the remaining files**, and ends the run with a non-OK terminal status. rsync parity: rsync likewise continues and exits partial (23). Residuals: (1) FastSync's default AUTO still *attempts* the node on a non-root receiver and therefore reports the per-entry failure, whereas rsync without `--super` silently ignores `--devices` and skips the non-regular entry with exit 0 — use `--no-super` for rsync's silent-skip behavior; (2) FastSync's process exit code for a receiver-side per-entry failure is the general error code 1, not rsync's partial 23 (a client exit-code-mapping residual that applies to every receiver file error, not just this branch); (3) with `--remove-source-files`, the non-OK terminal status means successfully transferred sources are not removed on a partial run. `--specials` (FIFOs and unix sockets) keeps the unprivileged skip path and remains parity | +| `--devices` | Preserve device files | ⚠️ Caveat | Recreates char/block device nodes with `mknodat` (type + rdev strictly validated, confined fd-relative below the receive root). A device whose `mknodat` fails with `EPERM`/`EACCES` (no `CAP_MKNOD`, or super-user activity forbidden) is a **per-entry failure**: FastSync logs `cannot create device ...` (rsync logs `mknod ... failed`), counts it, **continues with the remaining files**, and ends the run with a partial terminal status (`STATUS_PARTIAL`, protocol 2.30.0) so the client exits 23 like rsync, and under `--remove-source-files` the successfully transferred sources are still removed. rsync parity: rsync likewise continues and exits partial (23). Residual: FastSync's default AUTO still *attempts* the node on a non-root receiver and therefore reports the per-entry failure, whereas rsync without `--super` silently ignores `--devices` and skips the non-regular entry with exit 0 — use `--no-super` for rsync's silent-skip behavior. `--specials` (FIFOs and unix sockets) keeps the unprivileged skip path and remains parity | | `--specials` | Preserve special files | ✅ Parity | **FIFO and unix-socket recreation work** (protocol 2.23.0): FIFOs are recreated with `mkfifoat`, and sockets with `mknodat(..., S_IFSOCK)` — the latter is unprivileged on Linux because it materializes the socket *node*, not a live bound socket, so it is a real, assertable behavior under CI (it matches rsync, which also recreates a socket by `mknod`). Node creation is confined below the receive root (fd-relative parent; no `..`, no symlink follow) and type/rdev are validated strictly; a matching existing node is left in place and an unrelated entry is never replaced. Crosses the wire like `--devices` (the `STATUS_SPECIAL` frame). See the Phase-4 devices notes | | `--copy-devices` | Copy device contents as file | ❌ Divergent | Copies a device/FIFO's reported `st_size` into an ordinary regular file and never reads an unbounded pseudo-device, so `--sendfile` cannot hang and the run always succeeds. Deliberate safe divergence from rsync's dd-like unbounded device read, which can block; the dangerous behavior will not be implemented | | `--write-devices` | Write to devices as files | ❌ Divergent | Writes only into an existing char/block node under the confined receive root (`O_NOFOLLOW` + `O_NONBLOCK`); a missing, symlinked, FIFO-with-no-reader, non-device, or otherwise unusable destination is skipped with a warning rather than allowed or aborted. Deliberate confinement divergence from rsync's more permissive behavior | @@ -804,7 +804,7 @@ modes or links. | `--stop-after=MINS` | Stop after N minutes | ✅ Parity | Client-only sender stop deadline (Phase 6): computing `--stop-after=MINS` (a positive minute count; 0/negative/garbage rejected) and `--stop-at=TIME` (`HH:MM`, `HH:MM:SS`, or `now+N[smhd]`; a past time stops immediately). The transfer stops ELEGANTLY at the next chunk boundary: everything already fully sent is kept and applied, the run returns 0, and --delete (late/delete-after timing) does NOT wipe the destination — when the scan is cut short the partial keep-set manifest is suppressed with a warning (the delete walk is skipped rather than acting on an incomplete keep-set, so unscanned source mirrors survive). `--delete-before`/`--delete-during` still run their complete pre-scan (which ignores the deadline). Local client-only fields: never serialized into the wire config frame, so no PROTOCOL_VERSION bump. `--stop-after` uses CLOCK_MONOTONIC; `--stop-at` uses the wall clock. Works single-threaded and under `-j`/`--threads` (multithreaded). Divergence: rsync computes `--stop-after` from the run start; FastSync likewise. When both are given, the earlier of the two deadlines wins (checked per iteration). See the Phase-6 stop notes below | | `--stop-at=TIME` | Stop at specified time | ✅ Parity | Deadline transfer stop (client-only, never serialized). Protocol 2.26.0 accepts rsync's full date/time grammar (`2030-12-31T23:59`, `2030/12/31T23:59`, `2030-12-31`, `12-31`, `14:00`, `:59`, `1`) in addition to FastSync's `HH:MM[:SS]` and `now+N[smhd]`; a past time stops immediately. Everything already transferred is kept and an early stop suppresses the late `--delete` keep-set so unscanned source mirrors survive. Works single-threaded and under `-j`/`--threads` | | `--fsync` | Fsync every written file before publication | ✅ Parity | | -| `--protocol=NUM` | Force older protocol version | ❌ Divergent | Forces the wire protocol version for this transfer. FastSync has exactly ONE wire format (`PROTOCOL_VERSION`, currently 2.29.0) with no downgrade/backward-compat code paths, so `--protocol=2.29.0` is accepted (it sets the version claim the client sends, which the server already requires to match exactly) and **every other value is rejected up front** with a clear error before any connection — it does not and cannot speak an older or virtual wire format. Divergence from rsync (which negotiates a range and downgrades to an integer 0..31): FastSync's honest contract is force-to-the-one-supported-value; a genuine downgrade would require a per-version compatibility layer that does not exist. Client-only; the server-side exact-match check is unchanged. `--protocol=2.28.0`/`2.27.0`/`2.26.0`/`2.25.0`/`2.24.0`/`2.23.0`/`2.22.0`/`2.21.0`/`2.20.0`/`2.19.0`/`2.18.0`/`2.17.0`/`2.16.0`/`2.15.0`/`216`/`31`/garbage are all rejected. See the Phase-6 protocol note below | +| `--protocol=NUM` | Force older protocol version | ❌ Divergent | Forces the wire protocol version for this transfer. FastSync has exactly ONE wire format (`PROTOCOL_VERSION`, currently 2.30.0) with no downgrade/backward-compat code paths, so `--protocol=2.30.0` is accepted (it sets the version claim the client sends, which the server already requires to match exactly) and **every other value is rejected up front** with a clear error before any connection — it does not and cannot speak an older or virtual wire format. Divergence from rsync (which negotiates a range and downgrades to an integer 0..31): FastSync's honest contract is force-to-the-one-supported-value; a genuine downgrade would require a per-version compatibility layer that does not exist. Client-only; the server-side exact-match check is unchanged. `--protocol=2.29.0`/`2.28.0`/`2.27.0`/`2.26.0`/`2.25.0`/`2.24.0`/`2.23.0`/`2.22.0`/`2.21.0`/`2.20.0`/`2.19.0`/`2.18.0`/`2.17.0`/`2.16.0`/`2.15.0`/`216`/`31`/garbage are all rejected. See the Phase-6 protocol note below | | `--iconv=CONVERT_SPEC` | Charset conversion | ✅ Parity | Charset conversion of FILE NAMES (not content) at the protocol boundary via iconv(3): `--iconv=LOCAL[,REMOTE]` — the sender converts each local filename LOCAL→REMOTE before transmitting, matching rsync's rule that the spec "stays the same whether you're pushing or pulling": on a PUSH the destination end's charset is the spec's REMOTE half, so the default receiver writes the wire bytes verbatim, and only a server started with its own `--iconv` (the daemon `charset` analog) declares a different destination charset and converts REMOTE→that LOCAL (rsync push parity, differential-tested with and without a server `--iconv`). The full CONVERT_SPEC is serialized into the config frame as a new trailing string field so the peer knows the wire charset; **PROTOCOL_VERSION bumped 2.15.0 → 2.16.0**. `LOCAL[,REMOTE]` parse: single charset ⇒ LOCAL==REMOTE (identity both ways); garbage rejected up front; protocol 2.26.0 additionally accepts `--iconv=.` (the locale's default charset for both directions), `--iconv=-` and `--no-iconv` (disable conversion). Validation probes BOTH directions (a spec that only opens one way is refused, as is a NUL-emitting target charset like utf-16/utf-32/ucs-2, since filenames cannot contain NUL). An unrepresentable name (EILSEQ/EINVAL) fails that path cleanly with a logged `--iconv: cannot convert file name ...` and is never written mangled/truncated. Conversion is applied at EVERY wire-path site (regular/MKDIR/hardlink path+target/symlink path+target/SPECIAL, the delete manifest, the incremental-check path, and the `-s`/`chunk_serialize` embedded blob path), on both client and server (`--iconv` is also a server/daemon option). Zero overhead when unset. See the Phase-6 iconv notes below | | `--checksum-seed=NUM` | Set checksum seed | ✅ Parity | Sets the seed for FastSync's whole-file xxHash digest (full 64-bit seed) and for the delta path's per-block xxHash32 strong checksum (low 32 bits of the seed). **As of protocol 2.23.0 a seed of `0` — the default when the flag is unset — is randomized per transfer and the chosen seed is sent to the receiver**, exactly like rsync, so two runs against different content do not share a predictable seed; an explicit non-zero seed is used verbatim, so an explicit seed deterministically reproduces every computed digest on BOTH endpoints (the seed crosses in the config frame). `--checksum-choice=md5` has no seed and ignores it (documented). The value is a strict decimal 0..2⁶⁴-1 (blank, signed, or non-numeric values are rejected). Like rsync, a seed only matters where a digest is actually computed (`--checksum` or a basis-dir run, or a delta transfer); it does not by itself enable `--checksum`/`--delta` | | `--secluded-args`, `-s` | Use protocol to send args | ❌ Divergent | Accepted for CLI compatibility (including the rsync short `-s`, Phase 7 Wave A) but a documented **no-op / divergence**. rsync's `-s` protects arguments from shell expansion by shipping them over the protocol; FastSync never passes remote arguments through a shell expansion boundary in the first place — its SSH transport builds the remote argv as **single-quote-escaped shell words** (`ssh_build_remote_command`), so the injection/leak that `-s` guards against does not exist and there is nothing to "seclude". Implementing a true arg-send protocol would mean replacing the argv-based SSH launch with an in-band argument channel, a large redesign of the transport that buys no security here. Chunk serialization remains the long-only `--chunk-serialization`. | @@ -944,7 +944,7 @@ These are the last compatibility items and the closing phase toward rsync flag p | `-T` / `--timeout` | `-T` = `--temp-dir` | → `--timeout` (long-only) | | `-a` / `--archive` (= `-c -m -M`) | `-a` = `-rlptD` | → becomes **real rsync `-a`** after the renames | -**Wave B — Output & filesystem completion (✅ implemented).** `-S`/`--sparse` (`⚠️→✅`): real hole preservation — a sparse-aware writer (`write_all_sparse`) skips all-zero runs ≥ 4096 bytes with `lseek(SEEK_CUR)` and `ftruncate`s the final size, wired into both the atomic temp+rename store and `--inplace` receiver-side with **no wire change** (the full file image is already in memory; the ftruncate presize is kept). `-P` (`⚠️→✅`): interrupted-write retention — on a save failure after data reached the temp fd, `--partial` now renames the already-written temp to the destination path (best-effort; falls through to the normal unlink on failure, never retains when `--partial` is off) so a later `--append`/`--append-verify` run can resume. `--block-size=SIZE` (`⚠️→✅`): promoted after verification — `--block-size` is now an alias for `--delta-block`, both set `config->delta_block_size`, which the delta engine already honored end-to-end (`delta_signature_create_seeded` + `delta_apply`); out-of-range values keep the default. `--fake-super` (`⚠️→✅`): added `fake_super_restore_fd` to parse and re-apply the recorded `user.fastsync.stat` record fd-relative (mode/time only — protocol 2.23.0: **never a real chown**; the resolved owner is recorded for a later privileged restore); a save under `--fake-super` now re-applies the recorded attrs instead of only recording them. (The later fake-super xattr-interop pass replaced that native `user.fastsync.stat` format with rsync's `user.rsync.%stat` grammar — see the row and Phase-4 notes.) `--stderr=client` (`⚠️→❌ Divergent`): FastSync has no rsync client-message channel, and `client` is rejected at CLI parse — the rejection is the documented behavior (unit-tested). `-N`/`--crtimes` (`⚠️→❌ Divergent`): birth-times cannot be set by any portable fs call (`utimensat` sets only atime/mtime); capture/transmit stays, setting is impossible, the flag is accepted and safely inert. Review-hardening (post-eval): fake-super replay applies the mode through the shared `metadata_mode_for_policy` helper (protocol 2.23.0: exactly the source mode under `-p`, with no masking); `--sparse` takes precedence over `--preallocate` (posix_fallocate skipped so holes survive) — **reversed by the parity-completion wave: `--preallocate` now wins, matching rsync**; `--partial` retention is disabled for `--no_replace` (ignore/existing) and only marks a write-attempt after the actual write begins; `--block-size=SIZE`/`--delta-block=SIZE` inline forms are accepted. +**Wave B — Output & filesystem completion (✅ implemented).** `-S`/`--sparse` (`⚠️→✅`): real hole preservation — a sparse-aware writer (`write_all_sparse`) skips all-zero runs ≥ 4096 bytes with `lseek(SEEK_CUR)` and `ftruncate`s the final size, wired into both the atomic temp+rename store and `--inplace` receiver-side with **no wire change** (the full file image is already in memory; the ftruncate presize is kept). `-P` (`⚠️→✅`): interrupted-write retention — on a save failure after data reached the temp fd, `--partial` now renames the already-written temp to the destination path (best-effort; falls through to the normal unlink on failure, never retains when `--partial` is off) so a later `--append`/`--append-verify` run can resume. `--block-size=SIZE` (`⚠️→✅`): promoted after verification — `--block-size` is now an alias for `--delta-block`, both set `config->delta_block_size`, which the delta engine already honored end-to-end (`delta_signature_create_seeded` + `delta_apply`); out-of-range values keep the default. `--fake-super` (`⚠️→✅`): added `fake_super_restore_fd` to parse and re-apply the recorded `user.fastsync.stat` record fd-relative (mode/time only — protocol 2.23.0: **never a real chown**; the resolved owner is recorded for a later privileged restore); a save under `--fake-super` now re-applies the recorded attrs instead of only recording them. (The later fake-super xattr-interop pass replaced that native `user.fastsync.stat` format with rsync's `user.rsync.%stat` grammar — see the row and Phase-4 notes.) `--stderr=client` (`⚠️→❌ Divergent` then, in the wire backlog cycle, `❌→⚠️ Caveat`): the Phase-7 Wave B decision rejected `client` at CLI parse because no client-message channel existed; protocol 2.30.0 adds one (`STATUS_CLIENT_MSG`), so `client` is now accepted and forwards the client's diagnostics to the server's stderr (see the row for the remaining one-direction caveat). `-N`/`--crtimes` (`⚠️→❌ Divergent`): birth-times cannot be set by any portable fs call (`utimensat` sets only atime/mtime); capture/transmit stays, setting is impossible, the flag is accepted and safely inert. Review-hardening (post-eval): fake-super replay applies the mode through the shared `metadata_mode_for_policy` helper (protocol 2.23.0: exactly the source mode under `-p`, with no masking); `--sparse` takes precedence over `--preallocate` (posix_fallocate skipped so holes survive) — **reversed by the parity-completion wave: `--preallocate` now wins, matching rsync**; `--partial` retention is disabled for `--no_replace` (ignore/existing) and only marks a write-attempt after the actual write begins; `--block-size=SIZE`/`--delta-block=SIZE` inline forms are accepted. **Wave C — Devices & special files (finalize statuses + tests) (✅ implemented).** The four special-file rows are finalized with coverage tests. `--devices`, `--copy-devices`, and `--write-devices` are **✅ Implemented**, each with a documented, safety-driven divergence: device-node creation is privilege-gated, so a receiver without `CAP_MKNOD` skips that entry with a warning (a per-entry skip, never a transfer failure); `--copy-devices` copies a device/FIFO's reported size into an ordinary regular file (a size-bounded safe divergence from rsync's unbounded dd-like read); `--write-devices` writes only into an existing char/block node under the confined receive root and skips every unusable target rather than clobbering or aborting. `--specials` reclassified from **⛔ Impossible/Divergence** to **✅ Parity** in protocol 2.23.0: **FIFO recreation works** (unprivileged `mkfifo`) **and unix sockets are recreated** with `mknod(S_IFSOCK)`, which Linux permits unprivileged (the flag previously assumed sockets were impossible — see the `--specials` row). Tests assert FIFO recreation, socket recreation, the regular-file result of `--copy-devices`, the skipped/missing and non-device `--write-devices` targets, and (root-gated) real device-node creation; a root runner additionally drops the receiver to an unprivileged user to assert the `CAP_MKNOD` skip is graceful. (The parity-completion wave later reclassified `--devices`, `--copy-devices`, and `--write-devices` as explicit **❌ Divergent** rows, because their safe subsets are deliberately not rsync's behavior; the implementation itself is unchanged.) @@ -964,7 +964,7 @@ These are the last compatibility items and the closing phase toward rsync flag p **Wire:** two trailing config-frame blocks after the `--iconv` spec, in fixed order — `send_privilege_options`/`receive_privilege_options` (one `super_mode` int, validated `0..2`), then `send_copy_as_options`/`receive_copy_as_options` (presence int + two int32 ids, validated `>= 0`, with `copy_as_set ⇒ use_metadata`). `PROTOCOL_VERSION` bumped **2.17.0 → 2.18.0**. **Divergences from rsync:** rsync's `--super` elevates the receiver and `--copy-as` actually switches its credentials; FastSync never elevates and only permits/forwards confined attempts, and `--copy-as` forces ownership rather than switching identity. -**Honest status after the parity 2.29 cycle (protocol 2.29.0 since the symlink-xattr wire wave, which adds no config-frame field and leaves this matrix unchanged), updated by the parity cycle 2.29 pass, the audit-cycle follow-ups, the triage cycle, and a later no-wire parity pass.** ✅ Parity 119 / ⚠️ Caveat 14 / ❌ Divergent 24 = 157 rows. The no-wire parity pass accepted `--inc-recursive`/`--no-inc-recursive` as inert no-ops (❌ → ✅, since FastSync's full scan is rsync's `--no-inc-recursive` and the destination is identical), narrowed the `--temp-dir` divergence by accepting an absolute path that canonicalizes inside the receive root (the row stays ❌ for out-of-root absolute paths), closed the `--delete-before` phase-0 divergence (⚠️ → ✅: both the single-threaded and the `--threads` data passes now replay the pre-scan file list, so a source file created after the scan is neither transferred nor kept, matching rsync), and moved `--fake-super` and `--devices` ❌ → ⚠️ (`--fake-super` now writes/reads rsync's exact `user.rsync.%stat` key and ` , :` grammar, interoperating with real rsync 3.4.1 for regular files and faking char/block devices as regular files carrying the real rdev; `--devices` now logs a failed device `mknod` as a per-entry failure that continues the transfer instead of a silent non-root skip — see those rows for the remaining directory-faking and exit-code residuals). A review pass then hardened the fake-super stat parser (strict range-checked parsing), made rsync-style daemon modules read-only by default with a startup warning for accepted-but-unenforced access-control keys, and extended the `--delete-before` replay to the `--threads` path. The 2.29 cycle closed the scanner-order, delete-timing, relative-basis, and fuzzy-eligibility residuals (moving `-n`/`--delete`/`--del`/`--delete-delay` to ✅) and improved the `--info`/`--stats`/`--debug` partial rows; the triage cycle moved `-F` and `-i`/`--itemize-changes` ✅ → ⚠️ for their documented residuals. The remaining ⚠️ rows are `--info`, `--debug`, `--msgs2stderr`, `--stats`, `--progress`, `-i`, `--filter`, `-F`, the three basis-dir options, `-y/--fuzzy`, `--fake-super`, and `--devices`. Earlier: **Honest status after the parity 2.28.0 cycle (protocol 2.28.0), updated by the rsync-parity-stats, rsync-parity-options, rsync-parity-fs, parity-review, no-wire parity-track-1/2b and wire parity-track-4a/5a passes.** ✅ Parity 116 / ⚠️ Caveat 14 / ❌ Divergent 27 = 157 rows. Earlier revisions of this document reported "143 ✅ / 0 divergence / 0 partial"; that conflated "parsed and tested" with "rsync parity", because many rows carried documented behavioral differences and some short options were not parsed at all. This reclassification makes every difference explicit. The completion wave closed 23 previously-caveated rows (9 that triage showed were already parity, plus 14 genuine fixes) and turned the 17 inherently non-rsync rows — native daemon config/auth, the FastSync batch container, the safe-subset device/privilege flags, `-X`'s privileged namespaces, `--fake-super`'s native xattr format, and the `--old-args` no-op — into explicit ❌ divergences. The stats pass flipped `--delete-delay` to ✅ (actual-removal accounting), but the parity-review pass moved it back to ⚠️ because FastSync charged the `--max-delete` budget at plan/snapshot time and left a refilled snapshotted directory in place, whereas rsync charges on actual removals and recursively removes a queued directory (including content created after its plan). The no-wire parity-track-1 pass fixed both (actual-removal charging plus recursive deferred removal with an independent deferred-list cap), narrowing the caveat to the partial-delete ordering. The stats pass also reclassified `--out-format` to ❌ (protocol-specific `%b`/delta-`%c`), and sharpened the `--stats`/`--progress`/`--checksum-choice` residuals. The options pass flipped `--bwlimit` and `--ignore-errors` to ✅ (rsync-exact size parsing and ~100 ms leaky-bucket throttling, and rsync's skip-unreadable-subdir plus IO-error-suppressed deletion with exit 23) and emits rsync-format `--info=name/flist/del/remove/nonreg/progress` lines (real-run `deleting`/`*deleting` carried over a new trailing `report_deletes` wire bool, `PROTOCOL_VERSION` 2.26.0 → 2.27.0), while reclassifying `-M` over daemon/TCP +**Honest status after the parity 2.29 cycle (protocol 2.29.0 since the symlink-xattr wire wave, which adds no config-frame field and leaves this matrix unchanged), updated by the parity cycle 2.29 pass, the audit-cycle follow-ups, the triage cycle, and a later no-wire parity pass.** The wire backlog cycle (protocol 2.30.0) then moved `--stderr=MODE` ❌ → ⚠️ (the `client` mode is now accepted over the new `STATUS_CLIENT_MSG` channel; only the client->server direction is reproduced) and closed the `--devices` exit-code and `--remove-source-files` residuals via `STATUS_PARTIAL` (exit 23 with successful sources removed), leaving ✅ Parity 119 / ⚠️ Caveat 15 / ❌ Divergent 23 = 157 rows. The no-wire parity pass accepted `--inc-recursive`/`--no-inc-recursive` as inert no-ops (❌ → ✅, since FastSync's full scan is rsync's `--no-inc-recursive` and the destination is identical), narrowed the `--temp-dir` divergence by accepting an absolute path that canonicalizes inside the receive root (the row stays ❌ for out-of-root absolute paths), closed the `--delete-before` phase-0 divergence (⚠️ → ✅: both the single-threaded and the `--threads` data passes now replay the pre-scan file list, so a source file created after the scan is neither transferred nor kept, matching rsync), and moved `--fake-super` and `--devices` ❌ → ⚠️ (`--fake-super` now writes/reads rsync's exact `user.rsync.%stat` key and ` , :` grammar, interoperating with real rsync 3.4.1 for regular files and faking char/block devices as regular files carrying the real rdev; `--devices` now logs a failed device `mknod` as a per-entry failure that continues the transfer instead of a silent non-root skip — see those rows for the remaining directory-faking and exit-code residuals). A review pass then hardened the fake-super stat parser (strict range-checked parsing), made rsync-style daemon modules read-only by default with a startup warning for accepted-but-unenforced access-control keys, and extended the `--delete-before` replay to the `--threads` path. The 2.29 cycle closed the scanner-order, delete-timing, relative-basis, and fuzzy-eligibility residuals (moving `-n`/`--delete`/`--del`/`--delete-delay` to ✅) and improved the `--info`/`--stats`/`--debug` partial rows; the triage cycle moved `-F` and `-i`/`--itemize-changes` ✅ → ⚠️ for their documented residuals. The remaining ⚠️ rows are `--info`, `--debug`, `--msgs2stderr`, `--stats`, `--progress`, `-i`, `--filter`, `-F`, the three basis-dir options, `-y/--fuzzy`, `--fake-super`, and `--devices`. Earlier: **Honest status after the parity 2.28.0 cycle (protocol 2.28.0), updated by the rsync-parity-stats, rsync-parity-options, rsync-parity-fs, parity-review, no-wire parity-track-1/2b and wire parity-track-4a/5a passes.** ✅ Parity 116 / ⚠️ Caveat 14 / ❌ Divergent 27 = 157 rows. Earlier revisions of this document reported "143 ✅ / 0 divergence / 0 partial"; that conflated "parsed and tested" with "rsync parity", because many rows carried documented behavioral differences and some short options were not parsed at all. This reclassification makes every difference explicit. The completion wave closed 23 previously-caveated rows (9 that triage showed were already parity, plus 14 genuine fixes) and turned the 17 inherently non-rsync rows — native daemon config/auth, the FastSync batch container, the safe-subset device/privilege flags, `-X`'s privileged namespaces, `--fake-super`'s native xattr format, and the `--old-args` no-op — into explicit ❌ divergences. The stats pass flipped `--delete-delay` to ✅ (actual-removal accounting), but the parity-review pass moved it back to ⚠️ because FastSync charged the `--max-delete` budget at plan/snapshot time and left a refilled snapshotted directory in place, whereas rsync charges on actual removals and recursively removes a queued directory (including content created after its plan). The no-wire parity-track-1 pass fixed both (actual-removal charging plus recursive deferred removal with an independent deferred-list cap), narrowing the caveat to the partial-delete ordering. The stats pass also reclassified `--out-format` to ❌ (protocol-specific `%b`/delta-`%c`), and sharpened the `--stats`/`--progress`/`--checksum-choice` residuals. The options pass flipped `--bwlimit` and `--ignore-errors` to ✅ (rsync-exact size parsing and ~100 ms leaky-bucket throttling, and rsync's skip-unreadable-subdir plus IO-error-suppressed deletion with exit 23) and emits rsync-format `--info=name/flist/del/remove/nonreg/progress` lines (real-run `deleting`/`*deleting` carried over a new trailing `report_deletes` wire bool, `PROTOCOL_VERSION` 2.26.0 → 2.27.0), while reclassifying `-M` over daemon/TCP and receiver-side `protect`/`risk` re-derivation to ❌ (no argv channel / receiver filter engine); the wire parity-track-4a pass later added that receiver filter engine, flipping `--filter=RULE` back to ✅ (see above; the diff --git a/src/client/client_cli.c b/src/client/client_cli.c index d89177a..dd526ef 100644 --- a/src/client/client_cli.c +++ b/src/client/client_cli.c @@ -433,12 +433,10 @@ static int set_stderr_mode(const char* value) { log_set_stderr_mode(LOG_STDERR_ERRORS); else if (strcmp(value, "all") == 0 || strcmp(value, "a") == 0) log_set_stderr_mode(LOG_STDERR_ALL); - else if (strcmp(value, "client") == 0 || strcmp(value, "c") == 0) { - log_message(LOG_LEVEL_ERROR, - "--stderr=client is not supported: FastSync has no client message channel"); - return -1; - } else { - log_message(LOG_LEVEL_ERROR, "--stderr must be errors or all"); + else if (strcmp(value, "client") == 0 || strcmp(value, "c") == 0) + log_set_stderr_mode(LOG_STDERR_CLIENT); + else { + log_message(LOG_LEVEL_ERROR, "--stderr must be errors, all, or client"); return -1; } return 0; @@ -1353,10 +1351,9 @@ static bool cli_handle_pre_negation(CliParseCtx* ctx) { return true; } /* "--no-msgs2stderr" is the deprecated spelling of --stderr=client (rsync - * 3.4.1). FastSync has no separate client message channel, so the closest - * supported mode is the errors-only default. */ + * 3.4.1); the client-message channel now exists, so it maps to `client`. */ if (strcmp(arg, "--no-msgs2stderr") == 0) - return set_stderr_mode("errors") == 0; + return set_stderr_mode("client") == 0; /* "--no-motd" is a real rsync option name (client-side daemon MOTD display * suppression), not a negation of a "--motd" flag, so it is handled before * the generic --no-* negation branch. */ diff --git a/src/client/client_report.c b/src/client/client_report.c index ff45260..c29cba5 100644 --- a/src/client/client_report.c +++ b/src/client/client_report.c @@ -12,6 +12,7 @@ #include #include #include +#include #include /* Surface a server rejection to the user. When the last status exchange @@ -1051,3 +1052,109 @@ const char* delete_display_path(const Config* config, const char* path) { return path; return utils_strip_transfer_root(path, config->send_directory); } + +/* ---- --stderr=client diagnostic channel (protocol 2.30.0) ---- + * + * When the client's --stderr mode is `client`, log_message() hands each of the + * client's own diagnostics to the sink installed here instead of writing them + * locally. The sink QUEUES the text (it may be called from scanner worker + * threads while the sender is streaming) and the sender thread -- the sole + * writer of the protocol stream -- drains the queue over the wire at frame + * boundaries via client_flush_client_messages(). A bounded queue caps the + * memory a chatty run can pin; overflow falls back to local output so a + * diagnostic is never silently dropped. */ +#define CLIENT_MSG_MAX_QUEUED 256 +#define CLIENT_MSG_MAX_BYTES (256 * 1024) + +static mtx_t client_msg_mutex; +static once_flag client_msg_mutex_once = ONCE_FLAG_INIT; +static ArrayList* client_msg_queue = NULL; /* owns char* */ +static size_t client_msg_bytes = 0; +/* True only while a live transfer session exists: before the connection is up + (or after it drops) the sink declines so log_message falls back to local + output, matching rsync's documented fallback. */ +static bool client_msg_active = false; + +static void client_msg_mutex_init(void) { + mtx_init(&client_msg_mutex, mtx_plain); +} + +static bool client_msg_enqueue(const char* message); + +/* Install the global log sink for the duration of one transfer. Safe to call + * more than once; the queue is created lazily. */ +void client_messages_install(void) { + call_once(&client_msg_mutex_once, client_msg_mutex_init); + mtx_lock(&client_msg_mutex); + if (!client_msg_queue) + client_msg_queue = array_list_create(free); + mtx_unlock(&client_msg_mutex); + log_set_client_msg_sink(client_msg_enqueue); +} + +void client_messages_activate(bool active) { + client_msg_active = active; +} + +/* log_message sink: takes ownership (queues) the message when a session is + * live; returns false otherwise so the caller writes it locally. */ +static bool client_msg_enqueue(const char* message) { + if (!message || message[0] == '\0') + return client_msg_active; + if (!client_msg_active) + return false; + size_t len = strlen(message); + call_once(&client_msg_mutex_once, client_msg_mutex_init); + mtx_lock(&client_msg_mutex); + bool queued = false; + if (client_msg_queue && (size_t)client_msg_queue->size < CLIENT_MSG_MAX_QUEUED && + client_msg_bytes + len <= CLIENT_MSG_MAX_BYTES) { + char* copy = str_dup(message); + if (copy) { + if (array_list_add(client_msg_queue, copy)) { + client_msg_bytes += len; + queued = true; + } else { + free(copy); + } + } + } + mtx_unlock(&client_msg_mutex); + return queued; +} + +/* Drain the queued diagnostics as STATUS_CLIENT_MSG frames on the sender + * thread. Swaps the queue out under the mutex so a concurrent worker logging + * never blocks on the wire. Must be called at a protocol frame boundary. */ +void client_flush_client_messages(int fd) { + if (fd < 0) + return; + call_once(&client_msg_mutex_once, client_msg_mutex_init); + mtx_lock(&client_msg_mutex); + ArrayList* pending = client_msg_queue; + client_msg_queue = array_list_create(free); + client_msg_bytes = 0; + mtx_unlock(&client_msg_mutex); + if (!pending) + return; + for (int i = 0; i < pending->size; i++) { + const char* message = pending->items[i]; + if (message && message[0] != '\0' && !send_client_message(fd, message)) + break; /* peer is gone; the rest would fail too */ + } + array_list_delete(pending); +} + +/* Tear down the sink after a transfer and free anything still queued. */ +void client_messages_end(void) { + log_set_client_msg_sink(NULL); + client_msg_active = false; + call_once(&client_msg_mutex_once, client_msg_mutex_init); + mtx_lock(&client_msg_mutex); + ArrayList* pending = client_msg_queue; + client_msg_queue = NULL; + client_msg_bytes = 0; + mtx_unlock(&client_msg_mutex); + if (pending) + array_list_delete(pending); +} diff --git a/src/client/client_send.c b/src/client/client_send.c index 16e386c..2477ed7 100644 --- a/src/client/client_send.c +++ b/src/client/client_send.c @@ -127,6 +127,11 @@ Client* connect_transfer_client(const Config* config) { void disconnect_transfer_client(Client* client) { if (!client) return; + /* --stderr=client: push any diagnostics logged during the transfer to the + peer before the socket closes; once deactivated, later messages fall back + to local output instead of being lost. */ + client_flush_client_messages(client->file_descriptor); + client_messages_activate(false); client_disconnect(client); client_delete(client); } @@ -240,11 +245,16 @@ static void mark_sender_done(PipelineContextSender* context) { When --remove-source-files is active the receiver acknowledges each data file it processed, in send order: STATUS_NEXT means the file was written, STATUS_OK means the file was skipped/unchanged. Skipped sources are marked - so the later removal pass keeps them. */ + so the later removal pass keeps them. `partial_out` is set when the receiver + reported STATUS_PARTIAL (a per-entry receiver failure): the transfer is + otherwise complete, so successfully stored sources are still removed and the + caller exits 23 (rsync's partial transfer) instead of a fatal non-zero. */ static bool finalize_transfer(Client* client, const Config* config, ArrayList* remove_sources, - bool* delete_limit_out, ReceiverStats* stats_out) { + bool* delete_limit_out, bool* partial_out, ReceiverStats* stats_out) { if (delete_limit_out) *delete_limit_out = false; + if (partial_out) + *partial_out = false; if (!send_status(client->file_descriptor, STATUS_FINISHED)) return false; /* The receiver emits its optional wire-stats frame (protocol 2.25.0) FIRST, @@ -298,6 +308,16 @@ static bool finalize_transfer(Client* client, const Config* config, ArrayList* r *delete_limit_out = true; return true; } + /* A per-entry receiver failure the receiver chose to continue past is a + rsync PARTIAL transfer: everything else succeeded and the stored sources + may be removed, but the client must exit 23. */ + if (status == STATUS_PARTIAL) { + log_message(LOG_LEVEL_WARNING, + "some files could not be transferred (see the server log for details)"); + if (partial_out) + *partial_out = true; + return true; + } if (status != STATUS_OK) { log_server_rejection("Receiver reported transfer failure"); return false; @@ -815,6 +835,9 @@ static bool source_is_regular_file(const File* file) { static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config, ArrayList* remove_sources, TransferStats* stats) { + /* --stderr=client: this is a frame boundary, so forward any diagnostics the + scanner/log emitted since the previous chunk before the next frame. */ + client_flush_client_messages(client->file_descriptor); if (config->use_chunk_serialization) { if (remove_sources) { for (int i = 0; i < chunk->element_count; i++) { @@ -954,6 +977,7 @@ static int send_chunks_multithreaded(void* pipeline_context) { protocol_session_set_io_timeout(&session, context->config->timeout); protocol_session_set_ssl(&session, (SSL*)client->ssl); protocol_session_bind(&session); + client_messages_activate(true); if (!config_send(client->file_descriptor, context->config)) { pipeline_cancel(context); disconnect_transfer_client(client); @@ -1117,11 +1141,14 @@ static int send_chunks_multithreaded(void* pipeline_context) { !send_dir_times(client, context->config, context->dir_entries)) goto send_fail; bool delete_limit = false; + bool partial = false; ReceiverStats recv_stats; memset(&recv_stats, 0, sizeof(recv_stats)); + client_flush_client_messages(client->file_descriptor); bool ok = finalize_transfer(client, context->config, context->remove_source_files, &delete_limit, - &recv_stats); + &partial, &recv_stats); context->delete_limit = delete_limit; + context->partial = partial; if (!ok && context->config->use_delete) log_message(LOG_LEVEL_ERROR, "server reported a deletion failure (--delete); see the server log for the reason"); @@ -1823,9 +1850,12 @@ static int send_files_finalize(const Config* config, SendFilesState* state) { if (!send_dir_times(client, config, state->dir_entries)) return 1; bool delete_limit = false; + bool partial = false; ReceiverStats recv_stats; memset(&recv_stats, 0, sizeof(recv_stats)); - bool ok = finalize_transfer(client, config, state->remove_sources, &delete_limit, &recv_stats); + client_flush_client_messages(client->file_descriptor); + bool ok = finalize_transfer(client, config, state->remove_sources, &delete_limit, &partial, + &recv_stats); if (!ok && config->use_delete) log_message(LOG_LEVEL_ERROR, "server reported a deletion failure (--delete); see the server log for the reason"); @@ -1843,12 +1873,12 @@ static int send_files_finalize(const Config* config, SendFilesState* state) { (double)state->transfer_stats.transferred_file_size / (double)BYTES_PER_MIB); /* A skipped source entry (--ignore-errors past an unreadable directory, or a dereferenced symlink with no referent) makes rsync report a partial - transfer (exit 23) even though the rest of the run succeeded. A - --max-delete-capped commit is a successful transfer that rsync reports + transfer (exit 23), as does a receiver per-entry failure (STATUS_PARTIAL). + A --max-delete-capped commit is a successful transfer that rsync reports with exit code 25. */ if (!ok) return 1; - if (state->had_scan_io) + if (state->had_scan_io || partial) return 23; return delete_limit ? 25 : 0; } @@ -1885,7 +1915,18 @@ static void send_files_cleanup(SendFilesState* state) { client_set_abort_armed(false); } +static int send_files_impl(Config* config); + int send_files(Config* config) { + /* Install the --stderr=client sink for the whole run (it only queues while a + session is live) and release its queue on every return path. */ + client_messages_install(); + int rc = send_files_impl(config); + client_messages_end(); + return rc; +} + +static int send_files_impl(Config* config) { if (config->list_only) return send_list_only(config); if (config->dry_run) @@ -1931,6 +1972,7 @@ int send_files(Config* config) { protocol_session_set_io_timeout(&session, config->timeout); protocol_session_set_ssl(&session, (SSL*)client->ssl); protocol_session_bind(&session); + client_messages_activate(true); int ret = 1; if (!send_files_prepare(config, &state)) @@ -1946,7 +1988,16 @@ send_fail: return ret; } +static int send_files_multithreaded_impl(Config* config); + int send_files_multithreaded(Config* config) { + client_messages_install(); + int rc = send_files_multithreaded_impl(config); + client_messages_end(); + return rc; +} + +static int send_files_multithreaded_impl(Config* config) { if (!config) return 1; if (config->list_only) @@ -2202,16 +2253,18 @@ int send_files_multithreaded(Config* config) { mtx_unlock(&context->mutex_scanner); bool sender_ok = sender_result == thrd_success; bool delete_limit = context->delete_limit; + bool partial = context->partial; /* A skipped source entry (--ignore-errors past an unreadable directory, or a dereferenced symlink with no referent) makes rsync report a partial - transfer (exit 23). A --max-delete-capped commit is a successful transfer - that rsync reports with exit code 25. */ + transfer (exit 23), as does a receiver per-entry failure (STATUS_PARTIAL). + A --max-delete-capped commit is a successful transfer that rsync reports + with exit code 25. */ pipeline_context_sender_destroy(context); client_progress_cleanup(); client_set_abort_armed(false); if (!sender_ok) return 1; - if (scan_io) + if (scan_io || partial) return 23; return delete_limit ? 25 : 0; } diff --git a/src/client/client_send_internal.h b/src/client/client_send_internal.h index 7ce71bc..d891b30 100644 --- a/src/client/client_send_internal.h +++ b/src/client/client_send_internal.h @@ -72,6 +72,13 @@ void client_progress_uptodate(const Config* config, const File* file); void client_progress_prepare(const Config* config, const ArrayList* plan_dirs, unsigned long long plan_non_dir_count); bool receive_stats_record(int fd, ReceiverStats* stats, ArrayList* would_delete); +/* --stderr=client diagnostic channel (client_report.c): install the queueing + * log sink for a transfer, mark the session live, flush queued diagnostics over + * the wire at a frame boundary, and tear the sink down. */ +void client_messages_install(void); +void client_messages_activate(bool active); +void client_flush_client_messages(int fd); +void client_messages_end(void); /* client_send.c */ void receive_daemon_motd(Client* client, const Config* config); diff --git a/src/client/usage.c b/src/client/usage.c index 328df37..71af5a9 100644 --- a/src/client/usage.c +++ b/src/client/usage.c @@ -297,11 +297,13 @@ void print_usage(void) { printf(" --max-depth Maximum directory depth (0=unlimited)\n"); printf(" -x, --one-file-system Do not cross filesystem boundaries\n"); printf(" --log-file , --log-file= Write log messages to file\n"); - printf(" --stderr=MODE Route logging to stderr: errors or all\n"); + printf(" --stderr=MODE Route logging: errors (default), all, or client\n"); + printf(" (forward the client's diagnostics to the server's\n"); + printf(" stderr)\n"); printf(" --msgs2stderr Route all messages to stderr (deprecated spelling of\n"); printf(" --stderr=all)\n"); - printf(" --no-msgs2stderr Select errors-only stderr (deprecated spelling; the\n"); - printf(" default)\n"); + printf(" --no-msgs2stderr Forward the client's diagnostics to the server\n"); + printf(" (deprecated spelling of --stderr=client)\n"); printf(" --partial Keep partial files on interrupted transfer\n"); printf(" --partial-dir Directory for partial files (implies --partial)\n"); printf(" -T, --temp-dir Scratch dir for temp files before atomic install.\n"); diff --git a/src/server/receiver.c b/src/server/receiver.c index 765501d..d99f7ff 100644 --- a/src/server/receiver.c +++ b/src/server/receiver.c @@ -277,6 +277,7 @@ static bool status_counts_as_progress(Status status) { case STATUS_ABORT: case STATUS_CHECK_BATCH: case STATUS_DIR_TIMES: + case STATUS_CLIENT_MSG: return false; default: return true; @@ -347,6 +348,25 @@ static ReceiverStep receiver_handle_keepalive(ReceiverPendingState* state) { return RECEIVER_STEP_NEXT; } +/* rsync --stderr=client: a client diagnostic forwarded over the wire. Read the + * bounded string and write it to the server's stderr (respecting the server log + * destination). The body is peer-controlled text, so it is logged verbatim + * (log_client_message adds the standard prefix); trailing newlines are stripped + * so a message cannot inject a blank line. A malformed string (over-long or + * embedded NUL) is a framing error and tears the connection down. */ +static ReceiverStep receiver_handle_client_msg(ReceiverPendingState* state) { + char* message = receive_str(state->fd); + if (!message) + return RECEIVER_STEP_FAIL; + size_t len = strlen(message); + while (len > 0 && (message[len - 1] == '\n' || message[len - 1] == '\r')) + message[--len] = '\0'; + if (message[0] != '\0') + log_client_message(message); + free(message); + return RECEIVER_STEP_NEXT; +} + static ReceiverStep receiver_handle_abort(ReceiverPendingState* state) { (void)state; log_message(LOG_LEVEL_INFO, "Received abort from client, cleaning up"); @@ -527,6 +547,8 @@ static ReceiverStep receiver_dispatch_status(ReceiverPendingState* state, Status switch (status) { case STATUS_KEEPALIVE: return receiver_handle_keepalive(state); + case STATUS_CLIENT_MSG: + return receiver_handle_client_msg(state); case STATUS_ABORT: return receiver_handle_abort(state); case STATUS_CHECK: @@ -610,7 +632,7 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver status == STATUS_KEEPALIVE || status == STATUS_ABORT || status == STATUS_CHECK_BATCH || status == STATUS_MKDIR || status == STATUS_MANIFEST || status == STATUS_HARDLINK || status == STATUS_SYMLINK || status == STATUS_SPECIAL || status == STATUS_DIR_TIMES || - status == STATUS_DELETE_PLAN) { + status == STATUS_DELETE_PLAN || status == STATUS_CLIENT_MSG) { ReceiverStep step = receiver_dispatch_status(&state, status); if (step == RECEIVER_STEP_FAIL) goto fail; @@ -793,13 +815,14 @@ static void receiver_note_delete_limit(void* context_pointer) { /* Terminal status for a run. A capped --delete limit wins (rsync exit 25); otherwise any per-entry failure (for example an unprivileged --devices - mknod) makes the terminal frame non-OK so the client exits non-zero. rsync - reports 23 here; mapping the client's exact exit code to 23 is a separate, - pre-existing concern. A clean run keeps STATUS_OK. */ + mknod) makes the terminal frame STATUS_PARTIAL so the client exits 23 + (rsync's "partial transfer due to error") while still removing the sources + it successfully transferred under --remove-source-files. A fatal stream + error keeps STATUS_ERROR (a non-23 exit). A clean run keeps STATUS_OK. */ static Status receiver_final_status(bool delete_limit_reached, size_t failed_entries) { if (delete_limit_reached) return STATUS_DELETE_LIMIT; - return failed_entries > 0 ? STATUS_ERROR : STATUS_OK; + return failed_entries > 0 ? STATUS_PARTIAL : STATUS_OK; } static bool receiver_send_success_frame(int fd, void* context_pointer) { diff --git a/src/server/server.c b/src/server/server.c index f84a616..3e67385 100644 --- a/src/server/server.c +++ b/src/server/server.c @@ -1050,7 +1050,7 @@ static void server_run_mt_receiver(ServerSession* state) { context->failed_entries, context->failed_entries == 1 ? "y" : "ies"); Status final_status = context->delete_limit_reached ? STATUS_DELETE_LIMIT - : (context->failed_entries > 0 ? STATUS_ERROR : STATUS_OK); + : (context->failed_entries > 0 ? STATUS_PARTIAL : STATUS_OK); /* Emit the optional wire-stats record first (protocol 2.25.0), then the success/outcome frame, exactly like the single-threaded receiver. */ if (!receiver_send_stats_frame(state->fd, config, &context->stats, context->would_delete, diff --git a/src/shared/config.h b/src/shared/config.h index a7e5bf7..924a090 100644 --- a/src/shared/config.h +++ b/src/shared/config.h @@ -83,7 +83,7 @@ typedef struct { typedef enum SuperMode { SUPER_MODE_AUTO = 0, SUPER_MODE_ON = 1, SUPER_MODE_OFF = 2 } SuperMode; /* =========================================================================== - * Config wire-field table (single source of truth for protocol 2.29.0). + * Config wire-field table (single source of truth for protocol 2.30.0). * * Every field below crosses the wire. The table is the ONLY place a * serialized field is named: config.h expands CONFIG_WIRE_FIELDS() to declare @@ -1084,7 +1084,20 @@ typedef struct Config { * version must bump; the strict same-version handshake (config_receive rejects a * mismatched version before parsing anything else) keeps a 2.29 client and a * 2.28 server from ever reaching that state. */ -#define PROTOCOL_VERSION "2.29.0" +/* (11) Client-message channel + partial exit (protocol 2.30.0): the + * config-frame LAYOUT is unchanged (no new config field), but the frame stream + * gains two statuses. STATUS_CLIENT_MSG (client->server) carries a bounded, + * length-prefixed diagnostic string so a client running with --stderr=client + * (rsync's --no-msgs2stderr spelling) can forward its own diagnostics to the + * server's stderr. STATUS_PARTIAL (receiver->client) is the terminal status + * sent instead of STATUS_OK when a per-entry receiver failure (e.g. an + * unprivileged --devices mknod) did not abort the stream; the sender exits 23 + * (rsync's partial transfer) and still removes successfully transferred + * --remove-source-files sources. A 2.29 peer that does not know these status + * values would reject them as an unknown status and tear the connection down, + * so the protocol version must bump; the strict same-version handshake keeps a + * 2.30 client and a 2.29 server from ever reaching that state. */ +#define PROTOCOL_VERSION "2.30.0" #define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024) /* Upper bound on total basis-dir entries (rsync caps --link-dest at 20). */ #define MAX_BASIS_DIRS 64 diff --git a/src/shared/log.c b/src/shared/log.c index 4e69a05..1467828 100644 --- a/src/shared/log.c +++ b/src/shared/log.c @@ -16,6 +16,7 @@ static bool info_flags_explicit = false; static FILE* log_fp = NULL; static _Thread_local bool eight_bit_output; static LogStderrMode stderr_mode = LOG_STDERR_ERRORS; +static LogClientMsgSink client_msg_sink = NULL; /* Serializes access to log_fp and makes each emitted line atomic: the * timestamp prefix, formatted body, and trailing newline are written as one @@ -77,6 +78,51 @@ LogStderrMode log_get_stderr_mode(void) { return stderr_mode; } +void log_set_client_msg_sink(LogClientMsgSink sink) { + client_msg_sink = sink; +} + +LogClientMsgSink log_get_client_msg_sink(void) { + return client_msg_sink; +} + +/* Format just the message body (no prefix/newline) into a freshly allocated + * buffer. Shared by log_message (which may hand the body to a client-message + * sink) and log_client_message. Returns NULL on allocation/format failure. */ +static char* format_log_body(const char* format, va_list args) { + va_list copy; + va_copy(copy, args); + int body_len = vsnprintf(NULL, 0, format, copy); + va_end(copy); + if (body_len < 0) + return NULL; + char* body = malloc((size_t)body_len + 1); + if (!body) + return NULL; + vsnprintf(body, (size_t)body_len + 1, format, args); + return body; +} + +/* Assemble a complete log line (prefix + body + newline) from an already + * formatted body. Returns NULL on allocation failure. */ +static char* format_log_line_from_body(LogLevel log_level, const struct tm* t, const char* body) { + char prefix[64]; + int prefix_len = snprintf( + prefix, sizeof(prefix), "%04d-%02d-%02d %02d:%02d:%02d [%s]: ", t->tm_year + 1900, + t->tm_mon + 1, t->tm_mday, t->tm_hour, t->tm_min, t->tm_sec, log_level_strings[log_level]); + if (prefix_len < 0 || prefix_len >= (int)sizeof(prefix)) + return NULL; + size_t body_len = strlen(body); + char* line = malloc((size_t)prefix_len + body_len + 2); /* body + '\n' + NUL */ + if (!line) + return NULL; + memcpy(line, prefix, (size_t)prefix_len); + memcpy(line + prefix_len, body, body_len); + line[(size_t)prefix_len + body_len] = '\n'; + line[(size_t)prefix_len + body_len + 1] = '\0'; + return line; +} + /* Format one complete log line (timestamp prefix + body + newline) into a * freshly allocated buffer. This is pure CPU/malloc work and must happen * OUTSIDE the log mutex: the mutex only guards the log_fp pointer, so a @@ -84,26 +130,11 @@ LogStderrMode log_get_stderr_mode(void) { * on allocation/formatting failure. */ static char* format_log_line(LogLevel log_level, const struct tm* t, const char* format, va_list args) { - char prefix[64]; - int prefix_len = snprintf( - prefix, sizeof(prefix), "%04d-%02d-%02d %02d:%02d:%02d [%s]: ", t->tm_year + 1900, - t->tm_mon + 1, t->tm_mday, t->tm_hour, t->tm_min, t->tm_sec, log_level_strings[log_level]); - if (prefix_len < 0 || prefix_len >= (int)sizeof(prefix)) + char* body = format_log_body(format, args); + if (!body) return NULL; - va_list copy; - va_copy(copy, args); - int body_len = vsnprintf(NULL, 0, format, copy); - va_end(copy); - if (body_len < 0) - return NULL; - size_t total = (size_t)prefix_len + (size_t)body_len; - char* line = malloc(total + 2); /* body bytes + '\n' + NUL */ - if (!line) - return NULL; - memcpy(line, prefix, (size_t)prefix_len); - vsnprintf(line + prefix_len, (size_t)body_len + 1, format, args); - line[total] = '\n'; - line[total + 1] = '\0'; + char* line = format_log_line_from_body(log_level, t, body); + free(body); return line; } @@ -121,6 +152,20 @@ static void emit_log_line(FILE* console, const char* line) { mtx_unlock(&log_mutex); } +void log_client_message(const char* message) { + if (!message) + return; + time_t now = time(NULL); + struct tm t; + if (!localtime_r(&now, &t)) + return; + char* line = format_log_line_from_body(LOG_LEVEL_INFO, &t, message); + if (!line) + return; + emit_log_line(stderr, line); + free(line); +} + void log_message(LogLevel log_level, const char* format, ...) { if (log_level < current_log_level) return; @@ -138,8 +183,23 @@ void log_message(LogLevel log_level, const char* format, ...) { va_list args; va_start(args, format); - char* line = format_log_line(log_level, &t, format, args); + char* body = format_log_body(format, args); va_end(args); + if (!body) + return; + /* LOG_STDERR_CLIENT: hand the diagnostic to the client-message channel. A + sink that takes ownership suppresses the local write; otherwise (no sink + yet, or the peer connection is not up) fall through to local output so the + diagnostic is never lost. */ + if (stderr_mode == LOG_STDERR_CLIENT) { + LogClientMsgSink sink = client_msg_sink; + if (sink && sink(body)) { + free(body); + return; + } + } + char* line = format_log_line_from_body(log_level, &t, body); + free(body); if (!line) return; emit_log_line(dest_io, line); diff --git a/src/shared/log.h b/src/shared/log.h index d64c467..f7ea308 100644 --- a/src/shared/log.h +++ b/src/shared/log.h @@ -15,7 +15,11 @@ #endif typedef enum { LOG_LEVEL_DEBUG, LOG_LEVEL_INFO, LOG_LEVEL_WARNING, LOG_LEVEL_ERROR } LogLevel; -typedef enum { LOG_STDERR_ERRORS, LOG_STDERR_ALL } LogStderrMode; +/* --stderr=MODE destinations. ERRORS keeps errors on stderr and everything + * else on stdout; ALL sends every message to stderr; CLIENT routes the client's + * own diagnostics over the protocol stream to the peer's stderr (rsync's + * --stderr=client / --no-msgs2stderr). */ +typedef enum { LOG_STDERR_ERRORS, LOG_STDERR_ALL, LOG_STDERR_CLIENT } LogStderrMode; typedef enum { LOG_DEBUG_IO = 1u << 0, @@ -86,5 +90,20 @@ void log_set_8_bit_output(bool enabled); bool log_get_8_bit_output(void); void log_set_stderr_mode(LogStderrMode mode); LogStderrMode log_get_stderr_mode(void); +/* Write a message a peer forwarded over the client-message channel to this + * process's stderr (and log file), with the standard log prefix. Used by the + * server side of rsync's --stderr=client. */ +void log_client_message(const char* message); + +/* Sink for LOG_STDERR_CLIENT. log_message() passes the un-prefixed message + * body to the installed sink; a `true` return means the sink took ownership + * (e.g. queued it for protocol transmission) and the message must NOT also be + * written locally. A `false` return (or a NULL sink) makes log_message fall + * back to the normal local destination, so a diagnostic emitted before the peer + * connection exists is never lost (rsync's documented fallback). The sink may + * be called from any thread and must be tolerant of that. */ +typedef bool (*LogClientMsgSink)(const char* message); +void log_set_client_msg_sink(LogClientMsgSink sink); +LogClientMsgSink log_get_client_msg_sink(void); #endif diff --git a/src/shared/multiprocessing.c b/src/shared/multiprocessing.c index 3e54e87..6bad215 100644 --- a/src/shared/multiprocessing.c +++ b/src/shared/multiprocessing.c @@ -53,6 +53,7 @@ PipelineContextSender* pipeline_context_sender_create(Config* config, Queue* que context->dir_entries_mutex_init = false; atomic_init(&context->dir_count, 0); context->delete_limit = false; + context->partial = false; int init = 0; if (config->use_metadata) { context->dir_entries = array_list_create(file_destroy); diff --git a/src/shared/multiprocessing.h b/src/shared/multiprocessing.h index 92ae4cc..4e75d26 100644 --- a/src/shared/multiprocessing.h +++ b/src/shared/multiprocessing.h @@ -134,6 +134,11 @@ typedef struct { deletion (STATUS_DELETE_LIMIT): the transfer succeeded and the process must exit 25 like rsync. Read by the caller after the sender thread is joined. */ bool delete_limit; + /* Set by the sender thread when the receiver reported STATUS_PARTIAL (a + per-entry receiver failure that did not abort the stream): the transfer + otherwise succeeded, successfully stored --remove-source-files sources were + removed, and the process must exit 23 like rsync. Read after join. */ + bool partial; } PipelineContextSender; /* `config` is borrowed and must outlive the context: destroy does NOT free it, diff --git a/src/shared/protocol.c b/src/shared/protocol.c index 6bce72d..035c5fc 100644 --- a/src/shared/protocol.c +++ b/src/shared/protocol.c @@ -664,6 +664,10 @@ static const char* status_to_string(Status status) { return "DELETE_LIMIT"; case STATUS_DEST_INFO: return "DEST_INFO"; + case STATUS_CLIENT_MSG: + return "CLIENT_MSG"; + case STATUS_PARTIAL: + return "PARTIAL"; default: return "UNKNOWN"; } @@ -672,10 +676,10 @@ static const char* status_to_string(Status status) { /* Reject a raw wire status outside the known enum range before it is handed to * callers, so an unknown/corrupt frame fails as a protocol error instead of * being silently interpreted as an unexpected-but-valid verdict. STATUS_OK is - * the first enumerator and STATUS_STATS the last, so the range check accepts + * the first enumerator and STATUS_PARTIAL the last, so the range check accepts * every status the protocol defines. */ static bool status_is_valid(Status status) { - return status >= STATUS_OK && status <= STATUS_STATS; + return status >= STATUS_OK && status <= STATUS_PARTIAL; } /* Shared string send/receive implementation. `redact` selects whether the @@ -1144,6 +1148,19 @@ bool send_error_detail(int fd, const char* message) { return send_status(fd, STATUS_ERROR_DETAIL) && send_str(fd, message); } +bool send_client_message(int fd, const char* message) { + if (!message) + message = ""; + char bounded[MAX_CLIENT_MSG_BYTES + 1]; + size_t len = strlen(message); + if (len > MAX_CLIENT_MSG_BYTES) { + memcpy(bounded, message, MAX_CLIENT_MSG_BYTES); + bounded[MAX_CLIENT_MSG_BYTES] = '\0'; + message = bounded; + } + return send_status(fd, STATUS_CLIENT_MSG) && send_str(fd, message); +} + const char* protocol_last_error(void) { return io_error_detail; } diff --git a/src/shared/protocol.h b/src/shared/protocol.h index d852fb1..97edddd 100644 --- a/src/shared/protocol.h +++ b/src/shared/protocol.h @@ -15,6 +15,12 @@ * this for a rejection and the detail frame stays a small, fixed bound. */ #define MAX_ERROR_DETAIL_BYTES 4096 +/* Hard cap on a client diagnostic forwarded over the STATUS_CLIENT_MSG channel + * (protocol 2.30.0, rsync's --stderr=client). The body is reused from the + * bounded-string wire helper and sliced to this many bytes before it is sent, + * so a peer can never be made to retain more than this per message. */ +#define MAX_CLIENT_MSG_BYTES 4096 + /* Maximum uncompressed file payload accepted by the receiver's whole-file * paths. A single whole file is charged against the per-connection memory * reservation (MAX_CONNECTION_MEMORY) and against the server allocation @@ -240,7 +246,25 @@ enum NET_STATUS { * record (see format_stats_send/receive in format.h) and, when the run is a * --dry-run with --delete, the would-delete path list. Appended after * STATUS_DELETE_PLAN so no existing status is renumbered. */ - STATUS_STATS + STATUS_STATS, + /* Client diagnostic channel (protocol 2.30.0, rsync's --stderr=client / + * --no-msgs2stderr). When the client's --stderr mode is `client`, the + * client forwards its own diagnostics over this client->server frame + * (STATUS_CLIENT_MSG followed by a bounded length-prefixed string, capped at + * MAX_CLIENT_MSG_BYTES) instead of writing them to its local stderr. The + * receiver reads the string and writes it to the server's stderr (respecting + * the server log destination). Appended after STATUS_STATS so no existing + * status is renumbered. */ + STATUS_CLIENT_MSG, + /* Receiver-side partial transfer (protocol 2.30.0). Sent by the receiver as + * the terminal status INSTEAD of STATUS_OK when one or more entries failed + * per-entry without aborting the stream (currently a --devices mknod + * EPERM/EACCES). The transfer otherwise succeeded and every successfully + * stored file was acknowledged, so the sender may still remove + * --remove-source-files sources; the sender maps this to rsync's exit code + * 23 ("partial transfer due to error"), distinct from a fatal STATUS_ERROR. + * Appended after STATUS_CLIENT_MSG so no existing status is renumbered. */ + STATUS_PARTIAL }; void io_set_fds(int read_fd, int write_fd); @@ -341,6 +365,11 @@ bool receive_status(int file_descriptor, Status* status); * length-prefixed string. Over-long messages are sliced and NULL is treated * as "". Returns false if the status or the string could not be sent. */ bool send_error_detail(int file_descriptor, const char* message); +/* Send STATUS_CLIENT_MSG followed by a bounded (<= MAX_CLIENT_MSG_BYTES) + * length-prefixed string carrying a client diagnostic. Over-long messages are + * sliced and NULL is treated as "". Returns false if the status or the string + * could not be sent. */ +bool send_client_message(int file_descriptor, const char* message); /* Human-readable reason captured from the most recent STATUS_ERROR_DETAIL * received on this thread, or "" when the last status was a bare STATUS_ERROR * (or no detail was seen). Thread-local, and valid until the next non-keepalive diff --git a/tests/integration/test_fault_injection.py b/tests/integration/test_fault_injection.py index 8c67f6d..1509735 100644 --- a/tests/integration/test_fault_injection.py +++ b/tests/integration/test_fault_injection.py @@ -36,7 +36,7 @@ from common import ( # noqa: E402 verify_transfer, ) -PROTOCOL_VERSION = b"2.29.0" +PROTOCOL_VERSION = b"2.30.0" STATUS_MANIFEST = 5 STATUS_OK = 0 diff --git a/tests/integration/test_features.py b/tests/integration/test_features.py index 5730af0..cf28bb5 100644 --- a/tests/integration/test_features.py +++ b/tests/integration/test_features.py @@ -203,9 +203,9 @@ class TestDeviceSpecial: flags=["--devices"], port=port) finally: out, err = _stop_captured_server(server) - assert result.returncode != 0, ( - f"a failed device mknod must be a transfer error like rsync (got exit 0): " - f"{(out + err)[:300]}" + assert result.returncode == 23, ( + f"a failed device mknod must exit 23 (rsync partial transfer), got " + f"{result.returncode}: {(out + err)[:300]}" ) received = get_dest_received_dir(DEVICE_DEST, DEVICE_SOURCE) assert not os.path.lexists(os.path.join(received, "chardev")), ( @@ -215,6 +215,38 @@ class TestDeviceSpecial: f"receiver did not log the device creation error: out={out!r} err={err!r}" ) + @pytest.mark.skipif(os.geteuid() != 0, reason="requires root to create device nodes") + def test_devices_nonroot_partial_removes_transferred_sources(self): + """rsync parity for a partial receiver run under --remove-source-files: + the successfully transferred regular source is still removed, the + un-creatable device source is kept, and the client exits 23 (verified + against rsync 3.4.1: it removes ok.txt/ok2.txt, keeps the device, and + exits 23).""" + if os.geteuid() != 0 or shutil.which("setpriv") is None: + pytest.skip("requires root + setpriv to run the receiver unprivileged") + self._setup() + os.mknod(os.path.join(DEVICE_SOURCE, "chardev"), stat.S_IFCHR | 0o666, + os.makedev(1, 3)) + os.makedirs(DEVICE_DEST, exist_ok=True) + os.chmod(DEVICE_DEST, 0o777) + server, port = _start_captured_server( + prefix=["setpriv", "--reuid=65534", "--regid=65534", "--clear-groups"]) + try: + result, _ = run_client(DEVICE_SOURCE, DEVICE_DEST, + flags=["--devices", "--remove-source-files"], port=port) + finally: + out, err = _stop_captured_server(server) + assert result.returncode == 23, ( + f"a partial receiver run must exit 23, got {result.returncode}: " + f"{(out + err)[:300]}" + ) + assert not os.path.exists(os.path.join(DEVICE_SOURCE, "plain.txt")), ( + "a successfully transferred source must be removed even on a partial run" + ) + assert os.path.exists(os.path.join(DEVICE_SOURCE, "chardev")), ( + "the source device that failed to materialize must be kept" + ) + @pytest.mark.skipif(os.geteuid() != 0, reason="requires root to create device nodes") def test_devices_recreates_real_char_device(self, shared_server): """Root-only: a source char device node is recreated on the destination @@ -334,6 +366,57 @@ def setup_test_data(): shutil.rmtree(DEST_DIR, ignore_errors=True) +class TestClientStderrChannel: + """--stderr=client: the client's own diagnostics go to the peer's stderr.""" + + @pytest.mark.ci + def test_client_diagnostic_reaches_server_stderr(self): + """A client-side warning emitted during the transfer is forwarded over + the STATUS_CLIENT_MSG channel and printed on the server's stderr, not the + client's. A dangling symlink under -L is the deterministic trigger.""" + source = os.path.join(TEST_DATA_DIR, "client_msg_src") + dest = os.path.join(TEST_DATA_DIR, "client_msg_dst") + clean_dir(source) + clean_dir(dest) + with open(os.path.join(source, "plain.txt"), "wb") as f: + f.write(b"payload\n") + os.symlink("no-such-referent", os.path.join(source, "dangling")) + server, port = _start_captured_server() + try: + result, _ = run_client(source, dest, flags=["-L", "--stderr=client"], + port=port) + finally: + out, err = _stop_captured_server(server) + assert "symlink has no referent" in (out + err), ( + f"client diagnostic did not reach the server stderr: out={out!r} err={err!r}" + ) + assert "symlink has no referent" not in (result.stderr or ""), ( + f"client diagnostic must not also be written locally: {result.stderr!r}" + ) + + @pytest.mark.ci + def test_no_msgs2stderr_alias_uses_client_channel(self): + """--no-msgs2stderr is rsync's spelling of --stderr=client and now + forwards the client's diagnostics to the server too.""" + source = os.path.join(TEST_DATA_DIR, "client_msg_alias_src") + dest = os.path.join(TEST_DATA_DIR, "client_msg_alias_dst") + clean_dir(source) + clean_dir(dest) + with open(os.path.join(source, "plain.txt"), "wb") as f: + f.write(b"payload\n") + os.symlink("no-such-referent", os.path.join(source, "dangling")) + server, port = _start_captured_server() + try: + result, _ = run_client(source, dest, flags=["-L", "--no-msgs2stderr"], + port=port) + finally: + out, err = _stop_captured_server(server) + assert "symlink has no referent" in (out + err), ( + f"--no-msgs2stderr did not route to the server: out={out!r} err={err!r}" + ) + assert "symlink has no referent" not in (result.stderr or "") + + class TestDryRun: def test_trust_sender_transfer_completes(self, shared_server): """--trust-sender is a receiver-local policy (never sent to the peer). diff --git a/tests/integration/test_preflight.py b/tests/integration/test_preflight.py index 1763800..6397ae4 100644 --- a/tests/integration/test_preflight.py +++ b/tests/integration/test_preflight.py @@ -133,14 +133,14 @@ def _seed_protocol_source(source): class TestProtocol: @pytest.mark.ci def test_protocol_current_version_accepted(self, shared_server): - """--protocol=2.29.0 (the current PROTOCOL_VERSION) is accepted and the + """--protocol=2.30.0 (the current PROTOCOL_VERSION) is accepted and the transfer completes normally.""" source = os.path.join(TEST_DATA_DIR, "proto_ok_src") dest = os.path.join(TEST_DATA_DIR, "proto_ok_dst") shutil.rmtree(dest, ignore_errors=True) os.makedirs(dest) _seed_protocol_source(source) - result, _ = run_client(source, dest, flags=["--protocol=2.29.0"], + result, _ = run_client(source, dest, flags=["--protocol=2.30.0"], port=shared_server.port) assert result.returncode == 0, \ f"--protocol current run failed: {(result.stderr or result.stdout)[:400]}" @@ -157,7 +157,7 @@ class TestProtocol: shutil.rmtree(dest, ignore_errors=True) os.makedirs(dest) _seed_protocol_source(source) - for bad in ("2.28.0", "2.27.0", "2.26.0", "2.25.0", "2.24.0", "2.23.0", "2.22.0", "2.21.0", "2.20.0", + for bad in ("2.29.0", "2.28.0", "2.27.0", "2.26.0", "2.25.0", "2.24.0", "2.23.0", "2.22.0", "2.21.0", "2.20.0", "2.19.0", "2.18.0", "2.17.0", "2.15.0", "2.16.0", "216", "31"): result, _ = run_client(source, dest, flags=[f"--protocol={bad}"], port=shared_server.port) diff --git a/tests/test_client_cli.c b/tests/test_client_cli.c index d43a9a1..49c7f02 100644 --- a/tests/test_client_cli.c +++ b/tests/test_client_cli.c @@ -352,7 +352,7 @@ static void test_parse_args_protocol_accept_current() { Config* cfg = valid_client_config(); EXPECT_NOT_NULL(cfg); char* argv_equals[] = {"fastsync", "--source-dir", "/src", - "--dest-dir", "/dst", "--protocol=2.29.0"}; + "--dest-dir", "/dst", "--protocol=2.30.0"}; int positional_args[2]; int positional_count = 0; EXPECT_EQ_INT(parse_args(cfg, 6, argv_equals, positional_args, &positional_count), 0); @@ -362,7 +362,7 @@ static void test_parse_args_protocol_accept_current() { cfg = valid_client_config(); EXPECT_NOT_NULL(cfg); char* argv_space[] = {"fastsync", "--source-dir", "/src", "--dest-dir", - "/dst", "--protocol", "2.29.0"}; + "/dst", "--protocol", "2.30.0"}; positional_count = 0; EXPECT_EQ_INT(parse_args(cfg, 7, argv_space, positional_args, &positional_count), 0); EXPECT_EQ_STR(cfg->version, PROTOCOL_VERSION); @@ -372,10 +372,10 @@ static void test_parse_args_protocol_accept_current() { /* Any --protocol value other than the current PROTOCOL_VERSION must end in * failure (parse_args simply stores it; validate_config rejects it up front). */ static void test_parse_args_protocol_rejects_other_versions() { - static const char* const bad_versions[] = {"2.17", "2.16", "2.15.0", "2.16.0", "2.17.0", - "2.18.0", "2.19.0", "2.20.0", "2.21.0", "2.22.0", - "2.23.0", "2.24.0", "2.25.0", "2.26.0", "2.27.0", - "2.28.0", "216", "31", "abc", ""}; + static const char* const bad_versions[] = { + "2.17", "2.16", "2.15.0", "2.16.0", "2.17.0", "2.18.0", "2.19.0", + "2.20.0", "2.21.0", "2.22.0", "2.23.0", "2.24.0", "2.25.0", "2.26.0", + "2.27.0", "2.28.0", "2.29.0", "216", "31", "abc", ""}; for (size_t i = 0; i < sizeof(bad_versions) / sizeof(bad_versions[0]); i++) { Config* cfg = valid_client_config(); EXPECT_NOT_NULL(cfg); @@ -2323,9 +2323,9 @@ static void test_parse_args_8_bit_output() { } static void test_parse_args_stderr_modes() { - static const char* const modes[] = {"errors", "all", "e", "a"}; - static const LogStderrMode expected[] = {LOG_STDERR_ERRORS, LOG_STDERR_ALL, LOG_STDERR_ERRORS, - LOG_STDERR_ALL}; + static const char* const modes[] = {"errors", "all", "client", "e", "a", "c"}; + static const LogStderrMode expected[] = {LOG_STDERR_ERRORS, LOG_STDERR_ALL, LOG_STDERR_CLIENT, + LOG_STDERR_ERRORS, LOG_STDERR_ALL, LOG_STDERR_CLIENT}; for (size_t i = 0; i < sizeof(modes) / sizeof(modes[0]); i++) { Config* cfg = config_create(); char option[32]; @@ -2340,8 +2340,29 @@ static void test_parse_args_stderr_modes() { log_set_stderr_mode(LOG_STDERR_ERRORS); } +/* rsync's deprecated --msgs2stderr / --no-msgs2stderr spellings map to + * --stderr=all and --stderr=client respectively; the client-message channel + * that `client` needs now exists (protocol 2.30.0). */ +static void test_parse_args_msgs2stderr_aliases() { + Config* cfg = config_create(); + char* argv_all[] = {"fastsync", "--msgs2stderr", "/src", "/dst"}; + int positional_args[2]; + int positional_count = 0; + EXPECT_EQ_INT(parse_args(cfg, 4, argv_all, positional_args, &positional_count), 0); + EXPECT_EQ_INT(log_get_stderr_mode(), LOG_STDERR_ALL); + config_delete(cfg); + + cfg = config_create(); + char* argv_client[] = {"fastsync", "--no-msgs2stderr", "/src", "/dst"}; + positional_count = 0; + EXPECT_EQ_INT(parse_args(cfg, 4, argv_client, positional_args, &positional_count), 0); + EXPECT_EQ_INT(log_get_stderr_mode(), LOG_STDERR_CLIENT); + config_delete(cfg); + log_set_stderr_mode(LOG_STDERR_ERRORS); +} + static void test_parse_args_rejects_unsupported_stderr_modes() { - static const char* const modes[] = {"client", "c", "invalid"}; + static const char* const modes[] = {"invalid", "x", ""}; for (size_t i = 0; i < sizeof(modes) / sizeof(modes[0]); i++) { Config* cfg = config_create(); char option[32]; @@ -5242,6 +5263,7 @@ void test_client_cli() { test_parse_args_ignore_times(); test_parse_args_8_bit_output(); test_parse_args_stderr_modes(); + test_parse_args_msgs2stderr_aliases(); test_parse_args_rejects_unsupported_stderr_modes(); test_parse_args_secluded_args(); test_parse_args_chunk_serialization_long_form(); diff --git a/tests/test_config.c b/tests/test_config.c index 890de8e..0b94d00 100644 --- a/tests/test_config.c +++ b/tests/test_config.c @@ -2924,7 +2924,7 @@ static void golden_config_populate(Config* c) { array_list_add(c->filters, str_dup("- /sub/dir/")); } -/* The pinned golden frame (protocol 2.29.0). The values below are the only +/* The pinned golden frame (protocol 2.30.0). The values below are the only * thing that ties the generated table to the historical wire format; update * them ONLY with a PROTOCOL_VERSION bump and a documented reason. The 2.24.0 * delete-plan wave changed only the version string; 2.25.0 appended the @@ -2936,10 +2936,13 @@ static void golden_config_populate(Config* c) { * (project decision), so the frame grew by one int to 886 bytes. The 2.29.0 * symlink-xattr wave changes only the version string: the config-frame layout * is unchanged (use_xattrs already crosses the wire); the STATUS_SYMLINK frame - * body grows instead. The byte-exact values are recomputed for the merged - * layout. */ + * body grows instead. The 2.30.0 client-message/partial wave changes only the + * version string: the config-frame layout is unchanged (the new + * STATUS_CLIENT_MSG and STATUS_PARTIAL statuses are not part of this frame), so + * the length stays 886 and only the hash moves. The byte-exact values are + * recomputed for the merged layout. */ #define GOLDEN_WIRE_LEN 886 -#define GOLDEN_WIRE_HASH 17827864270611927842ULL +#define GOLDEN_WIRE_HASH 4169866417069573876ULL static unsigned long long fnv1a_64(const unsigned char* buf, size_t len) { unsigned long long h = 1469598103934665603ULL; @@ -3021,7 +3024,7 @@ static unsigned long long capture_wire_hash(const Config* cfg, size_t* out_len) return h; } -/* Byte-for-byte wire compatibility guard (protocol 2.29.0). The expected hash +/* Byte-for-byte wire compatibility guard (protocol 2.30.0). The expected hash * pins the pre-X-macro byte stream; the refactor MUST NOT change it. */ static void test_config_wire_golden() { if (is_running_under_valgrind()) diff --git a/tests/test_log.c b/tests/test_log.c index b5acedd..75829c6 100644 --- a/tests/test_log.c +++ b/tests/test_log.c @@ -117,6 +117,54 @@ static void test_log_stderr_mode_all() { log_set_stderr_mode(LOG_STDERR_ERRORS); } +/* --stderr=client: an installed sink takes the message body and suppresses the + * local write; a declining sink (or no sink) falls back to stderr. */ +static char g_client_msg_capture[256]; + +static bool client_msg_capture_sink(const char* message) { + snprintf(g_client_msg_capture, sizeof(g_client_msg_capture), "%s", message); + return true; +} + +static bool client_msg_decline_sink(const char* message) { + (void)message; + return false; +} + +static void test_log_stderr_mode_client() { + int pipe_fds[2]; + EXPECT_EQ_INT(pipe(pipe_fds), 0); + int saved_stderr = dup(STDERR_FILENO); + EXPECT_TRUE(saved_stderr >= 0); + EXPECT_TRUE(dup2(pipe_fds[1], STDERR_FILENO) >= 0); + close(pipe_fds[1]); + + set_log_level(LOG_LEVEL_WARNING); + log_set_stderr_mode(LOG_STDERR_CLIENT); + + g_client_msg_capture[0] = '\0'; + log_set_client_msg_sink(client_msg_capture_sink); + log_message(LOG_LEVEL_ERROR, "routed to peer %d", 7); + fflush(stderr); + EXPECT_EQ_STR(g_client_msg_capture, "routed to peer 7"); + + /* A sink that declines makes the message fall back to local stderr. */ + log_set_client_msg_sink(client_msg_decline_sink); + log_message(LOG_LEVEL_ERROR, "fallback local"); + fflush(stderr); + + log_set_client_msg_sink(NULL); + log_set_stderr_mode(LOG_STDERR_ERRORS); + EXPECT_TRUE(dup2(saved_stderr, STDERR_FILENO) >= 0); + close(saved_stderr); + char output[256] = {0}; + ssize_t length = read(pipe_fds[0], output, sizeof(output) - 1); + close(pipe_fds[0]); + EXPECT_TRUE(length > 0); + EXPECT_TRUE(strstr(output, "fallback local") != NULL); + EXPECT_TRUE(strstr(output, "routed to peer") == NULL); +} + /* Test that log_message handles various format strings */ static void test_log_message_formats() { set_log_level(LOG_LEVEL_DEBUG); @@ -271,6 +319,7 @@ void test_log() { test_log_set_level_error(); test_log_filtering(); test_log_stderr_mode_all(); + test_log_stderr_mode_client(); test_log_message_formats(); test_log_debug_enabled_matches_gate(); test_log_concurrent_no_torn_lines(); diff --git a/tests/test_protocol.c b/tests/test_protocol.c index 5852e3f..164a9fc 100644 --- a/tests/test_protocol.c +++ b/tests/test_protocol.c @@ -228,7 +228,7 @@ static void test_send_receive_status() { /* An unknown wire status outside the enum range must be rejected as a protocol * error instead of being handed to the caller as an unexpected verdict. The - * last known enumerator (STATUS_STATS) must still be accepted, proving the + * last known enumerator (STATUS_PARTIAL) must still be accepted, proving the * validation does not reject legitimate statuses. */ static void test_receive_status_rejects_unknown() { int p[2]; @@ -236,7 +236,7 @@ static void test_receive_status_rejects_unknown() { ProtocolSession session; protocol_session_init(&session, p[0], p[1]); - Status bogus = (Status)(STATUS_STATS + 1); + Status bogus = (Status)(STATUS_PARTIAL + 1); EXPECT_EQ_INT((int)write(p[1], &bogus, sizeof(bogus)), (int)sizeof(bogus)); Status received = STATUS_OK; EXPECT_FALSE(protocol_receive_status(&session, &received)); @@ -245,12 +245,12 @@ static void test_receive_status_rejects_unknown() { EXPECT_EQ_INT((int)write(p[1], &negative, sizeof(negative)), (int)sizeof(negative)); EXPECT_FALSE(protocol_receive_status(&session, &received)); - Status top = STATUS_STATS; + Status top = STATUS_PARTIAL; EXPECT_EQ_INT((int)write(p[1], &top, sizeof(top)), (int)sizeof(top)); EXPECT_TRUE(protocol_receive_status(&session, &received)); - EXPECT_EQ_INT((int)received, (int)STATUS_STATS); + EXPECT_EQ_INT((int)received, (int)STATUS_PARTIAL); - Status timed_bogus = (Status)(STATUS_STATS + 7); + Status timed_bogus = (Status)(STATUS_PARTIAL + 7); EXPECT_EQ_INT((int)write(p[1], &timed_bogus, sizeof(timed_bogus)), (int)sizeof(timed_bogus)); EXPECT_FALSE(protocol_receive_status_timed(&session, &received, 5)); @@ -258,6 +258,43 @@ static void test_receive_status_rejects_unknown() { close(p[1]); } +/* STATUS_CLIENT_MSG carries a bounded, length-prefixed diagnostic string + * (protocol 2.30.0, --stderr=client). An over-long message must be sliced to + * MAX_CLIENT_MSG_BYTES rather than sent whole. */ +static void test_send_client_message_bounded() { + int p[2]; + EXPECT_EQ_INT(pipe(p), 0); + io_set_fds(p[0], p[1]); + io_set_bwlimit(0); + + const char message[] = "client diagnostic line"; + EXPECT_TRUE(send_client_message(0, message)); + Status received = STATUS_OK; + EXPECT_TRUE(receive_status(0, &received)); + EXPECT_EQ_INT((int)received, (int)STATUS_CLIENT_MSG); + char* body = receive_str(0); + EXPECT_NOT_NULL(body); + EXPECT_EQ_STR(body, message); + free(body); + + size_t big_len = MAX_CLIENT_MSG_BYTES + 100; + char* big = malloc(big_len + 1); + EXPECT_NOT_NULL(big); + memset(big, 'x', big_len); + big[big_len] = '\0'; + EXPECT_TRUE(send_client_message(0, big)); + EXPECT_TRUE(receive_status(0, &received)); + EXPECT_EQ_INT((int)received, (int)STATUS_CLIENT_MSG); + char* big_body = receive_str(0); + EXPECT_NOT_NULL(big_body); + EXPECT_EQ_INT((int)strlen(big_body), (int)MAX_CLIENT_MSG_BYTES); + free(big_body); + free(big); + + close(p[0]); + close(p[1]); +} + static void test_receive_n_data_truncated() { int p[2]; EXPECT_EQ_INT(pipe(p), 0); @@ -1250,6 +1287,7 @@ void test_protocol() { test_send_receive_int(); test_send_receive_status(); test_receive_status_rejects_unknown(); + test_send_client_message_bounded(); test_protocol_session_io_timeout(); test_protocol_server_io_timeout_floor(); test_send_receive_status_timed();