From 47de05d2150d3bd609f9bd9a33867f904220cdf8 Mon Sep 17 00:00:00 2001 From: TapTap Date: Fri, 11 Sep 2026 15:58:20 +0200 Subject: [PATCH] feat(p7-output-fs): sparse hole preservation (-S), partial retention (-P), fake-super replay; block-size verified; crtimes/stderr -> Impossible/Divergence (Wave B) - write_all_sparse: skips all-zero runs >= 4096 bytes via lseek(SEEK_CUR) and ftruncates the final size, wired into the atomic temp+rename and --inplace paths with no wire change (full image already in memory). - --partial retention: on a save failure after the temp held data, rename the already-written temp to the destination path (best-effort; falls through to unlink; never retains when --partial is off) so --append/--append-verify can resume; tested by forcing futimens EINVAL with an out-of-range nsec. - --block-size aliases --delta-block; verified config->delta_block_size is honored by the delta engine end-to-end (unit + integration tests). - fake_super_restore_fd: parses and re-applies user.fastsync.stat fd-relative (fchown best-effort/non-root skipped, fchmod, futimens); a save under --fake-super now both records and re-applies. - -N/--crtimes and --stderr=client promoted to a new 'Impossible/Divergence' status bucket (Summary: 136/2/4/3/2 = 147). - cppcheck/clang-format clean; unit 37/37; integration 407 passed. --- README.md | 30 ++++--- RSYNC_COMPAT.md | 21 ++--- src/client/client_cli.c | 2 +- src/client/usage.c | 10 +-- src/shared/file.c | 94 ++++++++++++++++++---- src/shared/file.h | 6 +- src/shared/file_receive.c | 18 ++--- src/shared/file_store.c | 47 ++++++++++- src/shared/xattr.c | 39 +++++++++ src/shared/xattr.h | 8 ++ tests/integration/test_features.py | 112 ++++++++++++++++++++++++++ tests/test_client_cli.c | 51 ++++++++++++ tests/test_file.c | 123 +++++++++++++++++++++++++++++ tests/test_xattr.c | 44 +++++++++++ 14 files changed, 549 insertions(+), 56 deletions(-) diff --git a/README.md b/README.md index 438e04b..aaa687a 100644 --- a/README.md +++ b/README.md @@ -65,10 +65,13 @@ replacement for every rsync feature or protocol mode. modes. - Owner/group, ACL, xattr, hard-link, device, and special-file handling is incomplete or unavailable. -- Sparse-file handling does not yet preserve all holes correctly. -- `--partial`, `--partial-dir`, `-P`, `--append`, and `--append-verify` are not - yet full rsync-style resumable transfers. Interrupted files are not retained - for resumption. +- Sparse-file hole preservation (`-S`, `--sparse`) is implemented receiver-side: + long all-zero runs are written as holes (no wire change; the full file image + is already in memory). +- `--partial`, `--partial-dir`, `-P`, `--append`, and `--append-verify` keep + the write atomic (temp + rename). With `--partial`, a failed/interrupted write + now retains the already-written temp at the destination path (best-effort) so + a later `--append`/`--append-verify` run can resume it. - `--dirs` is not implemented. Its compatibility aliases `--old-dirs` and `--old-d` are recognized but rejected explicitly rather than silently using FastSync's recursive directory behavior. @@ -103,7 +106,7 @@ partial, alternate, and planned behavior. | `-v, --verbose` | Enable debug logging | | `-q, --quiet` | Suppress non-error output | | `--progress` | Show real-time transfer speed | -| `-P` | Enables partial-transfer mode and progress output (partial retention is incomplete) | +| `-P` | Enables partial-transfer mode + progress output; interrupted writes retain the already-written temp for resumption | | `--delete` | Delete files on receiver not present in source (default timing: delete-after, i.e. only after the whole transfer succeeded) | | `--delete-before` | Delete extras before the transfer starts (implies `--delete`) | | `--delete-during`, `--del` | Delete extras once the keep-set is known, before data is applied (implies `--delete`) | @@ -363,7 +366,7 @@ features without changing the meaning of ordinary compatibility options. | `--chunk-serialization` | Enable FastSync chunk serialization (long form only; `-s` is rsync's `--secluded-args`). | | `--sendfile` | Use TCP `sendfile()` zero-copy transfer. Incompatible with compression and chunk serialization. Long form only. | | `--delta` | Use FastSync-native block delta transfer. Requires `--incremental`. | -| `--delta-block ` | Set the FastSync delta block size. | +| `--delta-block ` | Set the FastSync delta block size (`--block-size` is an alias). | | `--delta-max ` | Limit files eligible for FastSync delta transfer. | | `--server-host ` | Select the TCP server host. | | `--server-port ` | Select the TCP server port. | @@ -410,12 +413,13 @@ remote SSH argv is already built injection-safe. zero means unlimited.| | `--incremental` | Skip files matching destination size and mtime.| | `--checksum` | Include xxHash64 content checks in incremental comparisons.| | `--backup` | Back up overwritten files.| | `--backup - dir` | Store backups under a separate directory.| - | `--suffix` | Set the backup filename suffix.| | `--partial` | - Select partial - transfer handling.With `--partial - dir`, - completed files are written there; -resumable transfers are not implemented.| | `--partial - dir` | - Set a relative partial - transfer directory below the server destination root; -use with `--partial`. | +| `--suffix` | Set the backup filename suffix.| | `--partial` | + Select partial - transfer handling. On failed/interrupted writes the + already-written temp file is retained (best-effort) for resumption.| + With `--partial --partial-dir `, completed files are written under the + partial directory and installed atomically. | | `--partial - dir` | + Set a relative partial - transfer directory below the server destination root. + Use with `--partial`. | | `--inplace` | Write directly to the destination instead of using a temporary file. | ### Metadata and links @@ -428,7 +432,7 @@ link-target transfer remains incomplete. | | `--copy-links` | Copy symlink referents. | | `--safe-links` | Skip symlinks that point outside the transfer tree. | | `--copy-unsafe-links` | Copy unsafe symlink referents. | -| `-S`, `--sparse` | Request sparse-file handling; full hole preservation is planned. | +| `-S`, `--sparse` | Sparse-file handling: receiver preserves holes (zero runs are written as holes; no wire change). | ### Output and logging diff --git a/RSYNC_COMPAT.md b/RSYNC_COMPAT.md index 9961680..738e83b 100644 --- a/RSYNC_COMPAT.md +++ b/RSYNC_COMPAT.md @@ -6,9 +6,10 @@ This document maps rsync's full feature set to FastSync's current implementation | Status | Count | Description | |--------|-------|-------------| -| βœ… Implemented | 132 | Feature works end-to-end | +| βœ… Implemented | 136 | Feature works end-to-end | | πŸ”€ Alt Arg | 0 | Functionality exists but under different flag/semantics | -| ⚠️ Partial | 10 | Flag parsed/stored but behavior incomplete | +| β›” Impossible/Divergence | 2 | Flag is a documented divergence or cannot be implemented on any portable filesystem call | +| ⚠️ Partial | 4 | Flag parsed/stored but behavior incomplete | | πŸ”„ Compatibility No-op | 3 | Flag is accepted for CLI compatibility but has no effect | | ❌ Not Implemented | 2 | Flag not recognized or no behavior | | **Total** | **147** | | @@ -26,7 +27,7 @@ This document maps rsync's full feature set to FastSync's current implementation | `-V`, `--version` | Print version | βœ… Implemented | | | `--info=FLAGS` | Fine-grained info verbosity | βœ… Implemented | Supports `copy`, `misc`, `skip`, `stats`, `all`, and `none`; explicit flags override `--verbose`, and `none` suppresses info output; unsupported names are rejected | | `--debug=FLAGS` | Fine-grained debug verbosity | βœ… Implemented | `io`, `proto`, `pack`, and `util` are supported; `--debug=help` lists flags; other rsync categories are rejected | -| `--stderr=MODE` | Change stderr output mode | ⚠️ Partial | `errors` (default) and `all` are supported; `client` is rejected because FastSync has no rsync message channel | +| `--stderr=MODE` | Change stderr output mode | β›” Impossible/Divergence | `errors` (default) and `all` are supported; `client` is rejected with a clear error (`--stderr=client is not supported`) because FastSync has no rsync client-message channel β€” the rejection itself is the documented behavior (Phase 7 Wave B decision). The modes that exist work; the missing rsync channel cannot be emulated without a wire change | | `--no-motd` | Suppress daemon MOTD | βœ… Implemented | Client-only display switch (Wave C): the daemon still sends the configured `motd file` on a `host::module/path` connection; the client reads and discards the frame without showing it. Without the flag the MOTD is printed to stdout after the config/auth handshake and escaped so control bytes cannot inject terminal sequences | | `--exclude=PATTERN` | Exclude files matching pattern | βœ… Implemented | Glob matching in scanner | | `--include=PATTERN` | Include files matching pattern | βœ… Implemented | Glob matching in scanner | @@ -40,7 +41,7 @@ This document maps rsync's full feature set to FastSync's current implementation | `-h`, `--human-readable` | Human-readable numbers | βœ… Implemented | Formats transfer byte sizes using binary units | | `-i`, `--itemize-changes` | Per-file change summary | βœ… Implemented | Prints rsync-style `>f+++++++++` lines to stdout only for files actually sent (also under `-j`/`--threads`); unchanged files print nothing, matching single-`-i` behavior | | `--progress` | Show progress | βœ… Implemented | Progress callback in sender | -| `-P` | Same as --partial --progress | ⚠️ Partial | Parses and enables progress, but interrupted files are not retained for resumable transfers | +| `-P` | Same as --partial --progress | βœ… Implemented | Phase 7 Wave B: `-P` parses to `--partial` + `--progress`. On a failed/interrupted write the receiver now retains the already-written temp file at the destination path (best-effort rename instead of unlink when configured), so a later `--append`/`--append-verify` run can resume it; `--partial-dir` still stages completed files under the confined partial dir and installs them atomically. The retention never runs when `--partial` is off and only ever renames the already-written temp (never a corrupt blend) | | `--out-format=FORMAT` | Custom output format | βœ… Implemented | Per-transfer template on stdout; tokens `%f` `%n` `%l` `%b` `%M` `%%` (`%b` is the source length, always `== %l`; post-compression/delta wire bytes are not counted); unknown escapes preserved | | `--log-file=FILE` | Log to file | βœ… Implemented | `log_file` config field | | `--log-file-format=FMT` | Log format | βœ… Implemented | Requires `--log-file`; writes one template line per transferred file using the same token set as `--out-format` (including `%b` `==` source length) | @@ -86,7 +87,7 @@ This document maps rsync's full feature set to FastSync's current implementation | `--append` | Append data to shorter files | βœ… Implemented | Tail-only resume. When an existing destination file is SHORTER than the source, the receiver negotiates a resume offset with the sender and only the tail is transferred; the receiver rebuilds the full file (retained prefix + tail) and installs it through the normal atomic store path, so the result is byte-identical to the source whenever the retained prefix matches. Plain `--append` does NOT content-verify that prefix (rsync parity): a destination whose prefix differs from the source is resumed anyway, so the result (wrong prefix + correct tail) is NOT byte-identical and the file is effectively left corrupt β€” the documented rsync-parity risk (use `--append-verify` when the prefix cannot be trusted). Non-content attributes (permissions/ownership/mtime, via `-M`) are still applied. Requires the per-file `STATUS_CHECK` handshake, so it implies `--incremental`; it takes precedence over block delta for a growing file and falls back to delta/full when the destination is not shorter. Incompatible with `-s` (chunk serialization) and `--whole-file` (both rejected up front so the mode never silently degrades to a full transfer). Combines with `--inplace`, `--partial`/`--partial-dir`, and `--delay-updates` (the reconstructed full file flows through those paths unchanged). Divergence: rsync appends in place; FastSync reconstructs and atomically installs, so an interrupted or failed resume never leaves a half-written file at the destination (no corruption window), and `--append` is thus safe to use with the normal atomic path β€” not only with in-place writes | | `--append-verify` | Append with old-data checksum | βœ… Implemented | Like `--append`, but the retained prefix IS verified before resuming: the sender transmits the source prefix checksum and the receiver compares it to the xxHash64 of the retained destination prefix; on a match only the tail is transferred, on a MISMATCH the run falls back to a clean full transfer so the result is always a byte-identical source copy (never a corrupt prefix+tail blend). Wire/protocol: the append handshake adds `STATUS_APPEND` / `STATUS_APPEND_SIG` / `STATUS_APPEND_OK` / `STATUS_APPEND_DATA` frames and `PROTOCOL_VERSION` was bumped **2.9.0 β†’ 2.10.0** (peers must match, and both must be 2.10.0 or the run fails the version check). Same implications/incompatibilities as `--append`; when both spellings are given `--append-verify` wins (the safer semantics). See the Phase-3 append notes below | | `-W`, `--whole-file` | Copy whole file (no delta) | βœ… Implemented | `whole_file` config field. Forces a full (whole-file) copy, disabling the block-level delta machinery: the sender only sends `STATUS_NEXT` + full data (client_send.c) and the receiver never requests a delta signature/reconstruction β€” the receiver's `try_delta = use_delta && !whole_file && ...` short-circuits. `whole_file` crosses the wire folded into `use_delta` (the wire carries `use_delta && !whole_file`), so no separate field/bump is needed. Delta is opt-in (`--delta` needs `--incremental`); `-W` additionally makes `--fuzzy` inert (no similar-file delta basis). `--append`/`--append-verify` are incompatible with `-W` and rejected up front (both sides). See the delta/append notes below | -| `--block-size=SIZE` | Force checksum block-size | ⚠️ Partial | Parsed as `--delta-block`; controls delta transfer block size | +| `--block-size=SIZE` | Force checksum block-size | βœ… Implemented | Phase 7 Wave B: `--block-size` is an alias for `--delta-block`; both set `config->delta_block_size` (default `DELTA_BLOCK_SIZE_DEFAULT`, bounds `DELTA_BLOCK_SIZE_MIN..MAX`, out-of-range values are rejected with the default kept). The value is genuinely honored by the delta engine end-to-end: `delta_signature_create_seeded(old, size, config->delta_block_size, seed)` on the sender and receiver, `delta_apply(old, ...)` with the same size, so a non-default block size changes the block count of every signature the harnesses exchange (verified by unit + integration tests) | ## 6. Destination Handling @@ -253,11 +254,11 @@ why plain `--append` works on the normal atomic path, not only with `--inplace`. | `--copy-devices` | Copy device contents as file | ⚠️ Partial | Copy a device's CONTENT into an ordinary regular file on the destination instead of recreating the node β€” non-privileged and safe. FastSync scans a device/FIFO as a regular file: its reported size (`st_size`, typically 0 for char devices and FIFOs) is copied, so a FIFO or a non-readable device becomes an empty (or size-bounded) regular file without ever blocking or reading unbounded pseudo-device streams. The run always succeeds and never crashes on such input. **Deliberate, safe divergence from rsync's dd-like unbounded device read.** See the Phase-4 devices notes | | `--write-devices` | Write to devices as files | ⚠️ Partial | Write the received data directly into an **existing** device node on the destination instead of creating a regular file. Restricted and best-effort: the destination must already exist and be a char/block device (opened only under the confined receive root, with `O_NOFOLLOW` + `O_NONBLOCK`); a missing, symlinked, FIFO-with-no-reader (`ENXIO`), non-device destination, or any write failure is **skipped with a warning** rather than allowed, so a run can never clobber the system, never blocks on a special-file target, and never aborts on an unusable target. See the Phase-4 devices notes | | `-U`, `--atimes` | Preserve access times | βœ… Implemented | Captures the source access time (from the scanner's pre-read stat, so it is not clobbered by reading the file for transfer) and transmits it over the wire; the receiver restores it together with the mtime via `futimens`/`utimensat`. Implies metadata transmission (the times travel inside the `-M` metadata payload), but does not enable ownership application (that stays opt-in via the identity flags). Wire: new `atime` fields on the metadata frame + a `preserve_atimes` config boolean; `PROTOCOL_VERSION` bumped **2.11.0 β†’ 2.12.0** | -| `-N`, `--crtimes` | Preserve create times | ⚠️ Partial | Captures the source birth time via `statx(STATX_BTIME)` on Linux and transmits it (recorded as a wire field), but there is **no portable way to set a birth time** (`utimensat` can only set atime/mtime), so the receiver explicitly does NOT apply it: it logs a debug note and continues β€” never failing the transfer and never pretending it worked. On platforms without `statx` it parses as a documented no-op (flag accepted; nothing is captured). Implies metadata transmission. Wire: new `crtime` fields + a `preserve_crtimes` config boolean; `PROTOCOL_VERSION` bumped **2.11.0 β†’ 2.12.0** (see the Phase-4 metadata-time notes) | +| `-N`, `--crtimes` | Preserve create times | β›” Impossible/Divergence | Birth-times cannot be set by any portable filesystem call (`utimensat`/`futimens` only set atime/mtime), so this row is an explicit **Impossible/Divergence** (Phase 7 Wave B). Capture + transmit stays: `statx(STATX_BTIME)` on Linux records the source birth time as a wire field; the receiver logs a debug note that it cannot be applied and continues β€” never failing the transfer and never pretending it worked. On platforms without `statx` it parses as a documented no-op (flag accepted; nothing is captured). Implies metadata transmission. Wire: new `crtime` fields + a `preserve_crtimes` config boolean; `PROTOCOL_VERSION` bumped **2.11.0 β†’ 2.12.0** (see the Phase-4 metadata-time notes) | | `-O`, `--omit-dir-times` | Omit dirs from --times | πŸ”„ Compatibility No-op | Accepted and parsed for CLI compatibility, and the config boolean crosses the wire, but it has **no effect**: FastSync never preserves directory mtimes in the first place (directories are created via `mkdir` with no metadata, a documented divergence under `-d`/recursive), so there is nothing for an "omit" to suppress. It never breaks a normal run | | `-J`, `--omit-link-times` | Omit symlinks from --times | πŸ”„ Compatibility No-op | Accepted and parsed for CLI compatibility, and the config boolean crosses the wire, but it has **no effect**: FastSync never sets symlink times (`-l`/`--links` copies symlinks as symlinks but the receiver does not apply timestamps/owner to symlink entries), so there is nothing for an "omit" to suppress. It never breaks a normal run | | `--super` | Receiver attempts super-user activities | ❌ Not Implemented | | -| `--fake-super` | Store/recover privileged attrs via xattrs | ⚠️ Partial | Honest, limited subset. The receiver records the source `uid:gid:mode:mtime_sec:mtime_nsec` into a reserved `user.fastsync.stat` xattr on each written file (best-effort, fd-relative), so a later privileged restore could re-apply them β€” without attempting the (typically failing as non-root) `chown`. Full rsync fake-super **replay** (parsing that xattr to actually re-apply ownership on a later privileged run) is out of scope and is **divergent** from rsync, which uses its own `user.rsync.%stat%` format; no cross-tool conversion is attempted. Implies metadata transmission so the source uid/gid/mode/mtime are available. Both it and `-X`/`-A` are incompatible with `-s` (chunk serialization), rejected up front | +| `--fake-super` | Store/recover privileged attrs via xattrs | βœ… Implemented | Phase 7 Wave B: full record **and replay**. The receiver writes the source `uid:gid:mode:mtime_sec:mtime_nsec` into a reserved `user.fastsync.stat` xattr on each written file (best-effort, fd-relative, format unchanged), then immediately re-applies it via `fake_super_restore_fd`: `fchown` (only where privileged β€” a non-root EPERM is logged-and-skipped, matching FastSync's identity philosophy), `fchmod`, and `futimens`. Absence or a malformed record is a silent no-op, never fatal. The recording format diverges from rsync's `user.rsync.%stat%`; no cross-tool conversion is attempted. Implies metadata transmission so the source uid/gid/mode/mtime are available. Both it and `-X`/`-A` are incompatible with `-s` (chunk serialization), rejected up front | | `--open-noatime` | Avoid changing access time when opening files | βœ… Implemented | Sender-side policy: the sender opens source files with `O_NOATIME` (Linux) when reading them for transfer, so the open/read does NOT bump the source's on-disk access time. Degrades safely when `O_NOATIME` is unavailable (not defined) or refused (`EPERM`, since it needs `CAP_FOWNER` or file ownership): the code falls back to a normal open, so the data always transfers β€” only the atime-bump is skipped. It does not itself capture/preserve atime; it only avoids modifying it. **Client-only, never crosses the wire.** Exposed as `file_open_for_read()` and applied to both the buffered data path and the sendfile path | | `--numeric-ids` | Do not map uid/gid by name | βœ… Implemented | Ownership is applied through FastSync's opt-in identity path (see the Phase-4 identity notes below). `--numeric-ids` is a mapping-policy modifier: when applying ownership it uses the transmitted numeric uid/gid directly, skipping the name lookup. Without an ownership-affecting option it is inert (FastSync only applies ownership when the user opts in). It does not need `-M` to be parsed, but ownership is only applied when metadata (hence the source uid/gid) is actually transmitted (see the notes) | | `--usermap=STRING` | Map usernames | βœ… Implemented | Opt-in ownership application. rsync subset implemented: comma-separated `FROM:TO` rules evaluated in order, first match wins; `FROM`/`TO` are group/user names (resolved on the SOURCE machine at parse time), `*` (FROM matches any id / TO = the receiving process's current euid), and an `@N` or bare `N` numeric id. Rules are carried over the wire as resolved numeric id pairs; the receiver applies a matching rule (else falls back to `--chown`, `--numeric-ids`, then a best-effort name lookup) via an fd-relative `fchown`. Malformed/unresolvable specs are rejected with a clear error, never a silent no-op. Implies metadata preservation so the source uid/gid travel. Only effective when the receiver can actually change ownership (root or membership); otherwise it warns and continues | @@ -576,7 +577,7 @@ now transmits targets (the prior behavior was broken/partial); its status moved | Flag | Rsync Description | FastSync Status | Notes | |------|-------------------|-----------------|-------| -| `-S`, `--sparse` | Sparse block handling | ⚠️ Partial | Flag is accepted, but full hole preservation is not implemented | +| `-S`, `--sparse` | Sparse block handling | βœ… Implemented | Phase 7 Wave B: real hole preservation with no wire change. The receiver's sparse-aware writer (`write_all_sparse`, next to `write_all` in `src/shared/file.c` and `src/shared/file_store.c`) walks the in-memory file image and emits any all-zero run β‰₯ 4096 bytes as a hole via `lseek(SEEK_CUR)` (the pre-size `ftruncate` guarantees the offset bookkeeping and logical size), `ftruncate(size)` after the last run pins the final size even with a hole tail. Wired into both the atomic temp+rename store and `--inplace` when `sparse` is set; the non-sparse path is byte-identical to before | | `--preallocate` | Allocate dest files before writing | βœ… Implemented | The receiver preallocates the destination file's full expected space before any data is written, so a transfer that would overflow disk fails fast at allocation time (a clean error, not a half-written file) and the file is laid out contiguously, avoiding fragmentation. Crosses the wire (the config frame carries a `preallocate` boolean; `PROTOCOL_VERSION` bumped **2.10.0 β†’ 2.11.0**, peers must match) so the sender knows the receiver will preallocate and the receiver performs it. **Allocation approach:** `posix_fallocate()` is preferred because it reserves *real* disk blocks (true fail-fast on ENOSPC), falling back to plain `ftruncate()` only when the filesystem reports the allocation is unsupported (`EOPNOTSUPP`/`ENOSYS`); `ftruncate` still extends the logical size so the intent degrades gracefully. **Fallback/error semantics:** `EOPNOTSUPP`/`ENOSYS` β†’ clean fallback to `ftruncate` (best-effort, preallocates the logical size and never fails a transfer on filesystems that lack `posix_fallocate`); a genuine allocation failure (`ENOSPC`/`EDQUOT`/`EFBIG`/…) aborts the file/receive with a distinct `preallocate failed ... transfer aborted` error β€” it does **not** fall back to a normal non-preallocated write, preserving the fail-fast purpose. **Size-known requirement:** preallocation only runs when the final size is already known up front (the normal regular-file case); unknown-length data is skipped (never failed). **Orthogonality:** applies uniformly across the atomic temp+rename store path, `--inplace`, `--partial`/`--partial-dir`, `--delay-updates` (the staged temp file is preallocated before data flows) and the `--link-dest` copy fallback; it neither implies nor conflicts with `-s`, `--append`, or delta. rsync-divergence: rsync signals that `--preallocate` is ignored with `--sparse`; FastSync simply preallocates first and still honours `--sparse`'s `ftruncate` sizing/trim, so the two combine rather than one being silently ignored. See the Phase-4 preallocate notes below | **Preallocate notes (Phase 4, preallocate wave):** `--preallocate` is implemented as a real receiver-side allocation of the destination file's space before data is written. It is a plain boolean config flag that crosses the wire (serialized in the config frame's selection-options block, mirroring `--inplace`/`--append`/`--force`), so the run requires matching ends: `PROTOCOL_VERSION` was bumped **2.10.0 β†’ 2.11.0** (peers must match or the version check fails). The allocation is performed on the exact destination fd, immediately after it is opened, before any bytes are streamed; `posix_fallocate` (and the `ftruncate` fallback) leave the fd's file offset untouched, so the subsequent data write at offset 0 is unaffected and complete. Because FastSync writes each file's byte payload in one in-memory batch, the "full expected size" is exactly the known `data_size`, which is what gets preallocated. Unknown-length/streamed payloads are skipped rather than failed. A failed allocation logs a distinct `preallocate failed` error and aborts the file (the atomic temp is unlinked, the inplace target is left untrimmed) so the run fails cleanly and never silently degrades to a non-preallocated write β€” preserving rsync's fail-fast intent on a full disk. @@ -811,7 +812,7 @@ These are the last compatibility items and the closing phase toward rsync flag p | `-T` / `--timeout` | `-T` = `--temp-dir` | β†’ `--timeout` (long-only) | | `-a` / `--archive` (= `-c -m -M`) | `-a` = `-rlptgoD` | β†’ becomes **real rsync `-a`** after the renames | -**Wave B β€” Output & filesystem completion.** `-S`/`--sparse` (`βš οΈβ†’βœ…`): real hole preservation (skip zero runs / `SEEK_HOLE` read, `ftruncate` sizing) instead of accepted-and-stored. `-P` (`βš οΈβ†’βœ…`): interrupted-file retention enabling true resumable `--partial` transfers (today only `--progress` is honored). `--block-size=SIZE` (`βš οΈβ†’βœ…`): make the delta checksum block-size genuinely configurable/honored rather than merely parsed as `--delta-block`. `--fake-super` replay (`βš οΈβ†’βœ…` or `Impossible/Divergence`): the FastSync-native `user.fastsync.stat` xattr already records uid/gid/mode/mtime β†’ parse and re-apply it on a later privileged run (possible β†’ implement). `--stderr=client` (`⚠️`): FastSync has no client-side rsync message channel β†’ implement a minimal message classification **or** mark **Impossible/Divergence** (decide in-wave). `-N`/`--crtimes` (`βš οΈβ†’Impossible/Divergence`): birth-times cannot be set by any portable fs call β†’ promote from "Partial" to explicit **Impossible/Divergence** (capture + transmit stays). +**Wave B β€” Output & filesystem completion (βœ… implemented).** `-S`/`--sparse` (`βš οΈβ†’βœ…`): real hole preservation β€” a sparse-aware writer (`write_all_sparse`) skips all-zero runs β‰₯ 4096 bytes with `lseek(SEEK_CUR)` and `ftruncate`s the final size, wired into both the atomic temp+rename store and `--inplace` receiver-side with **no wire change** (the full file image is already in memory; the ftruncate presize is kept). `-P` (`βš οΈβ†’βœ…`): interrupted-write retention β€” on a save failure after data reached the temp fd, `--partial` now renames the already-written temp to the destination path (best-effort; falls through to the normal unlink on failure, never retains when `--partial` is off) so a later `--append`/`--append-verify` run can resume. `--block-size=SIZE` (`βš οΈβ†’βœ…`): promoted after verification β€” `--block-size` is now an alias for `--delta-block`, both set `config->delta_block_size`, which the delta engine already honored end-to-end (`delta_signature_create_seeded` + `delta_apply`); out-of-range values keep the default. `--fake-super` (`βš οΈβ†’βœ…`): added `fake_super_restore_fd` to parse and re-apply the recorded `user.fastsync.stat` record fd-relative (fchown best-effort/non-root skipped, fchmod, futimens); a save under `--fake-super` now re-applies the recorded attrs instead of only recording them, with the recording format unchanged. `--stderr=client` (`βš οΈβ†’β›” Impossible/Divergence`): FastSync has no rsync client-message channel, and `client` is rejected at CLI parse β€” the rejection is the documented behavior (unit-tested). `-N`/`--crtimes` (`βš οΈβ†’β›” Impossible/Divergence`): birth-times cannot be set by any portable fs call (`utimensat` sets only atime/mtime); capture/transmit stays, setting is impossible, the flag is accepted and safely inert. **Wave C β€” Devices & special files (finalize statuses + tests).** `--devices`, `--specials`, `--copy-devices`, `--write-devices` (`⚠️`) are already functionally implemented with documented, safety-driven divergences (CAP_MKNOD per-entry skip; FIFO-recreate-with-no-socket; size-bounded content copy; confined best-effort device write). During this wave each is promoted to its final status with coverage tests: `--specials` **sockets** cannot be recreated by any standard filesystem call β†’ mark **Impossible/Divergence**; the rest are complete β†’ **βœ…**. @@ -819,7 +820,7 @@ These are the last compatibility items and the closing phase toward rsync flag p **Wave E (LAST) β€” Privilege (deferred decision, `❌`).** `--super`, `--copy-as=USER[:GROUP]`: **deferred by explicit project decision β€” the privilege model must be decided when this wave starts.** Candidate directions to fix then: a **safe** receiver model β€” `--copy-as` performs a drop-to-uid/group only when the process is privileged (and a clear refusal otherwise, never blind elevation); `--super` lifts only within the confined receive root β€” versus a **full setuid/elevation** model (higher security-review burden). Recommended: the safe-subset + clear-refusal direction, consistent with FastSync's confinement philosophy. These are the only remaining `❌` rows. -**Post-Phase-7 Summary targets.** The 3 `πŸ”€ Alt Arg` rows (`-a`, `-p`, `-z`) β†’ **βœ…** (real rsync semantics; `-z` divergence shrinks to "zstd-only", matching `--checksum-choice`). `⚠️ Partial` (10) β†’ real `βœ…` or explicit **Impossible/Divergence**. `πŸ”„ Compatibility No-op` (3) β†’ real `βœ…` (dir/symlink times) or **Impossible/Divergence** (`--secluded-args`). `❌ Not Implemented` (2) β†’ still deferred to Wave E. A new **Impossible/Divergence** status bucket is added to the Summary table; everything else lands at `βœ…`. +**Post-Phase-7 Summary targets.** The 3 `πŸ”€ Alt Arg` rows (`-a`, `-p`, `-z`) β†’ **βœ…** (real rsync semantics; `-z` divergence shrinks to "zstd-only", matching `--checksum-choice`). `⚠️ Partial` (4 after Wave B) β†’ real `βœ…` or explicit **Impossible/Divergence**. `πŸ”„ Compatibility No-op` (3) β†’ real `βœ…` (dir/symlink times) or **Impossible/Divergence** (`--secluded-args`). `❌ Not Implemented` (2) β†’ still deferred to Wave E. The **Impossible/Divergence** status bucket was added to the Summary table (Wave B: `--stderr` + `-N`); everything else lands at `βœ…`. ### Recommended Delivery Order diff --git a/src/client/client_cli.c b/src/client/client_cli.c index 4cbfb33..3b831f9 100644 --- a/src/client/client_cli.c +++ b/src/client/client_cli.c @@ -1079,7 +1079,7 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args, if (config_add_pattern(&config->include_patterns, &config->include_count, argv[++i], "--include") != 0) return -1; - } else if (opt_is(argv[i], "--delta-block", NULL)) { + } else if (opt_is(argv[i], "--delta-block", "--block-size")) { if (i + 1 >= argc) { log_message(LOG_LEVEL_ERROR, "missing argument for %s", argv[i]); return -1; diff --git a/src/client/usage.c b/src/client/usage.c index 2ecbfe6..779ad70 100644 --- a/src/client/usage.c +++ b/src/client/usage.c @@ -140,8 +140,8 @@ void print_usage(void) { printf(" --incremental and --delta; inert with --whole-file,\n"); printf(" --no-delta, or --no-incremental)\n"); printf(" --no-fuzzy Disable --fuzzy\n"); - printf(" --delta-block Delta block size in bytes (default: %d)\n", - DELTA_BLOCK_SIZE_DEFAULT); + printf(" --delta-block , --block-size \n"); + printf(" Delta block size in bytes (default: %d)\n", DELTA_BLOCK_SIZE_DEFAULT); printf(" --delta-max Max file size for delta transfer (default: %llu)\n", DELTA_MAX_FILE_SIZE); printf(" -j, --threads Enable multithreading\n"); @@ -165,9 +165,9 @@ void print_usage(void) { printf(" setting an ACL the receiver is not permitted to\n"); printf(" set is warned and skipped, never fatal)\n"); printf(" --fake-super Store the source uid/gid/mode/mtime in a reserved\n"); - printf(" user.fastsync.stat xattr on each written file instead\n"); - printf(" of applying ownership (for a later privileged restore);\n"); - printf(" partial: full rsync fake-super replay is out of scope\n"); + printf(" user.fastsync.stat xattr on each written file and\n"); + printf(" re-apply it (fd-relative) on a privileged run; the\n"); + printf(" recording format diverges from rsync's user.rsync.%%stat%%\n"); printf(" --chmod Modify transferred permissions (rsync syntax)\n"); printf(" --numeric-ids Do not map uid/gid by name: use the source numeric\n"); printf(" ids directly when applying ownership\n"); diff --git a/src/shared/file.c b/src/shared/file.c index 9bc8b9e..ae5ed5f 100644 --- a/src/shared/file.c +++ b/src/shared/file.c @@ -36,6 +36,44 @@ static bool write_all(int fd, const void* data, unsigned long long size) { return true; } +/* A run of NUL bytes at least this long is emitted as a hole (lseek) rather + * than written, so the resulting file is genuinely sparse on the filesystem. */ +#define SPARSE_HOLE_MIN 4096U + +/* Sparse-aware writer (--sparse/-S). Walks `data`; any all-zero run of at + * least SPARSE_HOLE_MIN bytes is skipped with lseek(SEEK_CUR) so the block is + * never allocated (a real hole on the destination); every other byte is written + * normally. The file is pre-sized with ftruncate by the callers before this + * runs, so holes are guaranteed and the offset bookkeeping stays correct + * (each lseek advances the fd offset exactly as a write of that many bytes + * would). After the final run, ftruncate(size) guarantees the logical size is + * exactly `size` even when the tail was a hole. The full file image is in + * memory, so no wire change is needed. Returns false on I/O error. */ +static bool write_all_sparse(int fd, const unsigned char* data, unsigned long long size) { + unsigned long long i = 0; + while (i < size) { + if (data[i] == 0) { + unsigned long long run_start = i; + while (i < size && data[i] == 0) + i++; + unsigned long long run_len = i - run_start; + if (run_len >= SPARSE_HOLE_MIN) { + if (lseek(fd, (off_t)run_len, SEEK_CUR) < 0) + return false; + } else if (!write_all(fd, data + run_start, run_len)) { + return false; + } + } else { + unsigned long long run_start = i; + while (i < size && data[i] != 0) + i++; + if (!write_all(fd, data + run_start, i - run_start)) + return false; + } + } + return ftruncate(fd, (off_t)size) == 0; +} + /* Preallocate `size` bytes on `fd` before any data is written (--preallocate). * posix_fallocate reserves real disk blocks, so an out-of-space condition * (ENOSPC/EDQUOT) surfaces up front instead of partway through a transfer; @@ -805,9 +843,15 @@ int file_open_private_dir(const char* dir_path) { static void restore_extra_fd(int fd, const FileMetadata* metadata, const FileXattrList* xattrs, bool fake_super) { xattr_apply_fd(fd, xattrs); - if (fake_super && metadata) + if (fake_super && metadata) { fake_super_store_fd(fd, (uint32_t)metadata->uid, (uint32_t)metadata->gid, (uint32_t)metadata->mode, metadata->mtime_sec, metadata->mtime_nsec); + /* Replay: re-apply the recorded uid/gid/mode/mtime fd-relative so a save + under --fake-super restores the attrs (when privileged) instead of only + recording them. Best-effort; a non-root fchown failure is logged/skipped + by fake_super_restore_fd, never fatal. */ + fake_super_restore_fd(fd); + } } static bool file_to_disk_secure_impl(const char* path, const void* data, @@ -815,7 +859,8 @@ static bool file_to_disk_secure_impl(const char* path, const void* data, bool preallocate, const FileMetadata* metadata, bool preserve_executability, bool update, bool no_replace, bool use_fsync, const char* temp_dir, - const FileXattrList* xattrs, bool fake_super) { + const FileXattrList* xattrs, bool fake_super, + bool keep_partial) { char* leaf = NULL; int dirfd = file_open_secure_parent(path, &leaf, true); if (dirfd < 0) @@ -852,7 +897,9 @@ static bool file_to_disk_secure_impl(const char* path, const void* data, if (sparse && data_size > 0) ok = ftruncate(fd, (off_t)data_size) == 0; if (ok || !sparse || data_size == 0) - ok = write_all(fd, data, data_size); + ok = sparse && data_size > 0 + ? write_all_sparse(fd, (const unsigned char*)data, data_size) + : write_all(fd, data, data_size); if (ok) ok = ftruncate(fd, (off_t)data_size) == 0; /* Normalize the mode: apply the metadata-derived safe mode when the @@ -875,6 +922,10 @@ static bool file_to_disk_secure_impl(const char* path, const void* data, } else { /* The --update newer-destination check runs first so a skipped file never creates an empty scratch directory behind it. */ + /* True once the temp is being written: distinguishes a mid-write/metadata/ + install failure (partial data may exist, --partial may retain it) from a + pre-write validation failure (nothing to retain). */ + bool write_attempted = false; if (update && metadata) { /* This check protects the normal atomic path as far as possible. A concurrent replacement can still occur before the final rename. */ @@ -947,11 +998,13 @@ static bool file_to_disk_secure_impl(const char* path, const void* data, strerror(prealloc_rc)); } if (prealloc_rc == 0) { + write_attempted = true; lseek(fd, 0, SEEK_SET); if (sparse && data_size > 0) ok = ftruncate(fd, (off_t)data_size) == 0; if (ok || (!sparse || data_size == 0)) - ok = write_all(fd, data, data_size); + ok = sparse && data_size > 0 ? write_all_sparse(fd, (const unsigned char*)data, data_size) + : write_all(fd, data, data_size); if (ok && metadata) ok = file_restore_metadata_fd(fd, metadata, preserve_executability); if (ok) @@ -986,8 +1039,19 @@ static bool file_to_disk_secure_impl(const char* path, const void* data, ok = false; } } - if (!ok) - unlinkat(scratch_dirfd >= 0 ? scratch_dirfd : dirfd, tmp, 0); + if (!ok) { + /* --partial retention (best-effort): on a failure that happened after + the temp held data (mid-write / metadata / fsync / install error), + keep the already-written temp at the final destination path instead + of unlinking it, so a later --append / --append-verify run can resume. + This only ever renames the already-written temp (never a corrupt + blend); the rename can fail (cross-device, permissions) and we then + fall through to the normal unlink cleanup. Never retains when + keep_partial is off. */ + if (!keep_partial || !write_attempted || + renameat(scratch_dirfd >= 0 ? scratch_dirfd : dirfd, tmp, dirfd, leaf) != 0) + unlinkat(scratch_dirfd >= 0 ? scratch_dirfd : dirfd, tmp, 0); + } /* Once the temp fd was created the outcome is permanent: a write, metadata, fsync, close, linkat or renameat failure will not be fixed by retrying under a fresh name, so stop here. Only the open-failure @@ -1010,7 +1074,7 @@ bool file_to_disk_secure(const char* path, const void* data, unsigned long long bool preserve_executability, const char* temp_dir) { return file_to_disk_secure_impl(path, data, data_size, inplace, sparse, preallocate, metadata, preserve_executability, false, false, false, temp_dir, NULL, - false); + false, false); } bool file_to_disk_secure_update(const char* path, const void* data, unsigned long long data_size, @@ -1018,7 +1082,7 @@ bool file_to_disk_secure_update(const char* path, const void* data, unsigned lon const FileMetadata* metadata, bool preserve_executability, const char* temp_dir) { return file_to_disk_secure_impl(path, data, data_size, inplace, sparse, preallocate, metadata, - preserve_executability, true, false, false, temp_dir, NULL, + preserve_executability, true, false, false, temp_dir, NULL, false, false); } @@ -1029,7 +1093,7 @@ bool file_to_disk_secure_with_fsync(const char* path, const void* data, const char* temp_dir) { return file_to_disk_secure_impl(path, data, data_size, inplace, sparse, preallocate, metadata, preserve_executability, false, false, use_fsync, temp_dir, NULL, - false); + false, false); } bool file_to_disk_secure_no_replace(const char* path, const void* data, @@ -1037,22 +1101,24 @@ bool file_to_disk_secure_no_replace(const char* path, const void* data, const FileMetadata* metadata, bool preserve_executability, const char* temp_dir) { return file_to_disk_secure_impl(path, data, data_size, false, sparse, preallocate, metadata, - preserve_executability, false, true, false, temp_dir, NULL, + preserve_executability, false, true, false, temp_dir, NULL, false, false); } /* Receiver write-path variant that also applies the per-file xattrs (-X/-A) * and, under --fake-super, parks the source stat in the reserved xattr, on the * just-written file descriptor before the final rename. `no_replace` / `update` - * mirror the plain wrappers; see file_to_disk_secure_impl for the semantics. */ + * mirror the plain wrappers; `keep_partial` enables --partial retention of a + * failed write's temp. See file_to_disk_secure_impl for the semantics. */ bool file_to_disk_secure_attrs(const char* path, const void* data, unsigned long long data_size, bool inplace, bool sparse, bool preallocate, const FileMetadata* metadata, bool preserve_executability, bool update, bool no_replace, bool use_fsync, - const FileXattrList* xattrs, bool fake_super, const char* temp_dir) { + const FileXattrList* xattrs, bool fake_super, bool keep_partial, + const char* temp_dir) { return file_to_disk_secure_impl(path, data, data_size, inplace, sparse, preallocate, metadata, preserve_executability, update, no_replace, use_fsync, temp_dir, - xattrs, fake_super); + xattrs, fake_super, keep_partial); } /* Atomic --link-dest install. The destination is replaced (via a temporary @@ -1155,7 +1221,7 @@ static bool file_to_disk_secure_link_impl(const char* path, const char* basis_pa by the filesystem). Write a byte-identical local copy instead. */ return file_to_disk_secure_attrs(path, data, data_size, false, false, preallocate, metadata, preserve_executability, false, false, use_fsync, xattrs, - fake_super, temp_dir); + fake_super, false, temp_dir); } if (scratch_dirfd >= 0) diff --git a/src/shared/file.h b/src/shared/file.h index 37da41b..554170b 100644 --- a/src/shared/file.h +++ b/src/shared/file.h @@ -113,12 +113,14 @@ bool file_to_disk_secure_no_replace(const char* path, const void* data, const char* temp_dir); /* Receiver write-path variant that also applies per-file xattrs (-X/-A) and the * --fake-super stat xattr fd-relative before the final rename. `update` / - * `no_replace` / `use_fsync` mirror the plain wrappers above. */ + * `no_replace` / `use_fsync` mirror the plain wrappers above; `keep_partial` + * enables --partial best-effort retention of a failed write's temp. */ bool file_to_disk_secure_attrs(const char* path, const void* data, unsigned long long data_size, bool inplace, bool sparse, bool preallocate, const FileMetadata* metadata, bool preserve_executability, bool update, bool no_replace, bool use_fsync, - const FileXattrList* xattrs, bool fake_super, const char* temp_dir); + const FileXattrList* xattrs, bool fake_super, bool keep_partial, + const char* temp_dir); /* Atomic --link-dest install: replace `path` with a hard link to `basis_path` (via a temp name + rename); fall back to a byte-identical local copy from `data` when the link is impossible (EXDEV/EPERM/unsupported filesystem). diff --git a/src/shared/file_receive.c b/src/shared/file_receive.c index 95ee6aa..b4fff74 100644 --- a/src/shared/file_receive.c +++ b/src/shared/file_receive.c @@ -87,10 +87,10 @@ static FileSaveResult file_stage_delayed_update(const char* root_directory, config->preallocate, metadata, preserve_executability, config->use_fsync, NULL); } else { - ok = - file_to_disk_secure_attrs(staged_path, file->data->data, file->data->size, false, sparse, - config->preallocate, metadata, preserve_executability, false, - false, config->use_fsync, file->xattrs, config->fake_super, NULL); + ok = file_to_disk_secure_attrs(staged_path, file->data->data, file->data->size, false, sparse, + config->preallocate, metadata, preserve_executability, false, + false, config->use_fsync, file->xattrs, config->fake_super, + false, NULL); } if (!ok) { free(staged_path); @@ -796,11 +796,11 @@ FileSaveResult file_save_to_disk_full(const char* root_directory, const File* fi } else { /* The plain no-replace / update / with-fsync engines, plus per-file xattr (-X/-A) and --fake-super application on the written fd. */ - ok = file_to_disk_secure_attrs(disk_path, file->data->data, file->data->size, inplace, sparse, - config && config->preallocate, metadata, preserve_executability, - config && config->update, config && config->ignore_existing, - config && config->use_fsync, file->xattrs, - config ? config->fake_super : false, confined_temp); + ok = file_to_disk_secure_attrs( + disk_path, file->data->data, file->data->size, inplace, sparse, + config && config->preallocate, metadata, preserve_executability, config && config->update, + config && config->ignore_existing, config && config->use_fsync, file->xattrs, + config ? config->fake_super : false, config ? config->partial : false, confined_temp); } free(confined_temp); confined_temp = NULL; diff --git a/src/shared/file_store.c b/src/shared/file_store.c index 8bd637d..842e15e 100644 --- a/src/shared/file_store.c +++ b/src/shared/file_store.c @@ -139,6 +139,44 @@ static bool write_all(int fd, const void* data, unsigned long long size) { return true; } +/* A run of NUL bytes at least this long is emitted as a hole (lseek) rather + * than written, so the resulting file is genuinely sparse on the filesystem. */ +#define SPARSE_HOLE_MIN 4096U + +/* Sparse-aware writer (--sparse/-S). Walks `data`; any all-zero run of at + * least SPARSE_HOLE_MIN bytes is skipped with lseek(SEEK_CUR) so the block is + * never allocated (a real hole on the destination); every other byte is written + * normally. The file is pre-sized with ftruncate by the callers before this + * runs, so holes are guaranteed and the offset bookkeeping stays correct + * (each lseek advances the fd offset exactly as a write of that many bytes + * would). After the final run, ftruncate(size) guarantees the logical size is + * exactly `size` even when the tail was a hole. The full file image is in + * memory, so no wire change is needed. Returns false on I/O error. */ +static bool write_all_sparse(int fd, const unsigned char* data, unsigned long long size) { + unsigned long long i = 0; + while (i < size) { + if (data[i] == 0) { + unsigned long long run_start = i; + while (i < size && data[i] == 0) + i++; + unsigned long long run_len = i - run_start; + if (run_len >= SPARSE_HOLE_MIN) { + if (lseek(fd, (off_t)run_len, SEEK_CUR) < 0) + return false; + } else if (!write_all(fd, data + run_start, run_len)) { + return false; + } + } else { + unsigned long long run_start = i; + while (i < size && data[i] != 0) + i++; + if (!write_all(fd, data + run_start, i - run_start)) + return false; + } + } + return ftruncate(fd, (off_t)size) == 0; +} + bool file_store_write_secure(const char* path, const void* data, unsigned long long data_size, bool inplace, bool sparse, const FileMetadata* metadata, bool preserve_executability) { @@ -151,8 +189,12 @@ bool file_store_write_secure(const char* path, const void* data, unsigned long l if (inplace) { fd = openat(dirfd, leaf, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC | O_NOFOLLOW, 0644); if (fd >= 0) { - if (!sparse || data_size == 0 || ftruncate(fd, (off_t)data_size) == 0) + if (sparse && data_size > 0) { + if (ftruncate(fd, (off_t)data_size) == 0) + ok = write_all_sparse(fd, data, data_size); + } else { ok = write_all(fd, data, data_size); + } if (ok && metadata) ok = file_restore_metadata_fd(fd, metadata, preserve_executability); } @@ -177,7 +219,8 @@ bool file_store_write_secure(const char* path, const void* data, unsigned long l if (sparse && data_size > 0) ok = ftruncate(fd, (off_t)data_size) == 0; if (ok || (!sparse || data_size == 0)) - ok = write_all(fd, data, data_size); + ok = (sparse && data_size > 0) ? write_all_sparse(fd, (const unsigned char*)data, data_size) + : write_all(fd, data, data_size); if (ok && metadata) ok = file_restore_metadata_fd(fd, metadata, preserve_executability); if (close(fd) != 0) diff --git a/src/shared/xattr.c b/src/shared/xattr.c index 14f60bb..a4d1287 100644 --- a/src/shared/xattr.c +++ b/src/shared/xattr.c @@ -9,7 +9,10 @@ #include #include #include +#include #include +#include +#include /* ---- lifecycle ---- */ @@ -328,4 +331,40 @@ void fake_super_store_fd(int fd, uint32_t uid, uint32_t gid, uint32_t mode, int6 log_message(LOG_LEVEL_WARNING, "--fake-super: could not store %s on destination file: %s", FAKESUPER_XATTR, strerror(errno)); } +} + +/* --fake-super replay: read the freshly-stored record and re-apply the source + * stat fd-relative. A privileged (root) run can actually change the owner; + * a non-root run logs-and-skips the fchown (never fatal, mirroring the normal + * metadata identity path) and still applies mode/mtime where permitted. */ +bool fake_super_restore_fd(int fd) { + if (fd < 0) + return false; + char record[128]; + ssize_t len = fgetxattr(fd, FAKESUPER_XATTR, record, sizeof(record) - 1); + if (len < 0) + return false; /* absent or filesystem without xattrs: silent no-op */ + record[len] = '\0'; + unsigned long ul_uid, ul_gid, ul_mode; + long long mtime_sec; + long mtime_nsec; + if (sscanf(record, "%lu:%lu:%lo:%lld:%ld", &ul_uid, &ul_gid, &ul_mode, &mtime_sec, &mtime_nsec) != + 5) + return false; /* malformed record: skip, never fatal */ + + /* Owner is applied best-effort only: a non-root process cannot chown and must + not abort the transfer for that reason (FastSync identity philosophy). */ + if (fchown(fd, (uid_t)ul_uid, (gid_t)ul_gid) != 0 && errno != EPERM && errno != EACCES && + errno != EINVAL) + log_message(LOG_LEVEL_WARNING, "--fake-super: could not restore owner on destination file: %s", + strerror(errno)); + if (fchmod(fd, (mode_t)(ul_mode & 07777U)) != 0) + log_message(LOG_LEVEL_WARNING, "--fake-super: could not restore mode on destination file: %s", + strerror(errno)); + struct timespec times[2] = {{.tv_sec = 0, .tv_nsec = UTIME_OMIT}, + {.tv_sec = (time_t)mtime_sec, .tv_nsec = mtime_nsec}}; + if (futimens(fd, times) != 0) + log_message(LOG_LEVEL_WARNING, "--fake-super: could not restore mtime on destination file: %s", + strerror(errno)); + return true; } \ No newline at end of file diff --git a/src/shared/xattr.h b/src/shared/xattr.h index c61c2d1..6f98d7c 100644 --- a/src/shared/xattr.h +++ b/src/shared/xattr.h @@ -86,4 +86,12 @@ bool xattr_apply_fd(int fd, const FileXattrList* list); void fake_super_store_fd(int fd, uint32_t uid, uint32_t gid, uint32_t mode, int64_t mtime_sec, int64_t mtime_nsec); +/* --fake-super replay: parse the FAKESUPER_XATTR record previously written on + * `fd` by fake_super_store_fd and re-apply uid/gid/mode/mtime fd-relative. + * Best-effort: absence of the xattr or a malformed record is a silent no-op + * that never fails the transfer, and fchown is applied only when permitted + * (a non-root EPERM is logged and skipped, matching FastSync's identity + * philosophy). Returns true when the xattr was present and parsed. */ +bool fake_super_restore_fd(int fd); + #endif \ No newline at end of file diff --git a/tests/integration/test_features.py b/tests/integration/test_features.py index df3b0f6..bfa7b39 100644 --- a/tests/integration/test_features.py +++ b/tests/integration/test_features.py @@ -4255,6 +4255,118 @@ class TestCrtimes: f"combined -U -N dest atime {dst_st.st_atime} != {atime}" +class TestSparse: + """-S/--sparse: the receiver preserves holes by skipping long zero runs with + lseek (no wire change; the full image is in memory). The destination file + must round-trip its logical size and content byte-for-byte; on filesystems + that report holes (SEEK_HOLE/SEEK_DATA) we additionally assert the file is + genuinely sparse via st_blocks, but that check is tolerant (CI filesystems + may report no holes).""" + + def _make_sparse_source(self, name, total, zero_start, zero_len): + source = os.path.join(TEST_DATA_DIR, name) + clean_dir(source) + sfile = os.path.join(source, "blob.bin") + with open(sfile, "wb") as f: + head = os.urandom(zero_start) + tail = os.urandom(total - zero_start - zero_len) + f.write(head) + f.write(b"\x00" * zero_len) + f.write(tail) + assert f.tell() == total + return source, sfile + + @pytest.mark.parametrize("flag", ["-S", "--sparse"]) + @pytest.mark.parametrize("mt", [False, True]) + def test_sparse_transfer_round_trips(self, shared_server, flag, mt): + total = 4 * 1024 * 1024 + source = os.path.join(TEST_DATA_DIR, f"sparse_mt{mt}_{flag.lstrip('-')}_src") + dest = os.path.join(TEST_DATA_DIR, f"sparse_mt{mt}_{flag.lstrip('-')}_dst") + clean_dir(source) + clean_dir(dest) + zero_start = 1 * 1024 * 1024 + zero_len = 2 * 1024 * 1024 + _, sfile = self._make_sparse_source(os.path.basename(source), total, zero_start, zero_len) + with open(sfile, "rb") as f: + src_bytes = f.read() + + flags = [flag] + (["--threads"] if mt else []) + result, _ = run_client(source, dest, flags=flags, port=shared_server.port) + assert result.returncode == 0, \ + f"{flag} transfer failed: {(result.stderr or result.stdout)[:300]}" + + received = get_dest_received_dir(dest, source) + dfile = os.path.join(received, "blob.bin") + assert os.path.getsize(dfile) == total, "logical size must match data_size" + with open(dfile, "rb") as f: + assert f.read() == src_bytes, "sparse destination content must round-trip exactly" + + # Tolerant sparseness assert: if the filesystem reports holes, the file + # must actually be sparse (fewer allocated blocks than its size). + with open(dfile, "rb") as f: + off = os.lseek(f.fileno(), zero_start, os.SEEK_DATA) + if off >= 0: + hole = os.lseek(f.fileno(), off, os.SEEK_HOLE) + else: + hole = -1 + if hole > zero_start: + st = os.stat(dfile) + assert st.st_blocks * 512 < total, \ + f"-S file not sparse: {st.st_blocks} blocks for {total} bytes" + + def test_sparse_inplace(self, shared_server): + """--sparse must also preserve holes in the --inplace write path.""" + total = 2 * 1024 * 1024 + source = os.path.join(TEST_DATA_DIR, "sparse_inplace_src") + dest = os.path.join(TEST_DATA_DIR, "sparse_inplace_dst") + clean_dir(source) + clean_dir(dest) + _, sfile = self._make_sparse_source(os.path.basename(source), total, total // 2, + total // 4) + with open(sfile, "rb") as f: + src_bytes = f.read() + result, _ = run_client(source, dest, flags=["-S", "--inplace"], port=shared_server.port) + assert result.returncode == 0, \ + f"-S --inplace failed: {(result.stderr or result.stdout)[:300]}" + received = get_dest_received_dir(dest, source) + dfile = os.path.join(received, "blob.bin") + assert os.path.getsize(dfile) == total + with open(dfile, "rb") as f: + assert f.read() == src_bytes + + +class TestBlockSize: + """--block-size / --delta-block: the checksum block size is genuinely honored + by the delta engine (both spellings parse to config->delta_block_size). An + end-to-end delta transfer with a non-default block size must still be + byte-exact.""" + + @pytest.mark.parametrize("flag", ["--block-size", "--delta-block"]) + def test_non_default_block_size_delta_transfer(self, shared_server, flag): + source = os.path.join(TEST_DATA_DIR, "blocksize_delta_src") + dest = os.path.join(TEST_DATA_DIR, "blocksize_delta_dst") + clean_dir(source) + clean_dir(dest) + payload = os.urandom(300 * 1024) # enough for several 1 KiB blocks + with open(os.path.join(source, "big.bin"), "wb") as f: + f.write(payload) + # First run installs the file; second run with delta + a small block size. + result, _ = run_client(source, dest, flags=["-S"], + port=shared_server.port) + assert result.returncode == 0 + received = get_dest_received_dir(dest, source) + # Change the source, then delta-transfer with a non-default block size. + with open(os.path.join(source, "big.bin"), "ab") as f: + f.write(os.urandom(4096)) + clean_dir(dest) + result, _ = run_client(source, dest, + flags=["--incremental", "--delta", flag, "1024"], + port=shared_server.port) + assert result.returncode == 0, \ + f"{flag} 1024 delta transfer failed: {(result.stderr or result.stdout)[:300]}" + with open(os.path.join(received, "big.bin"), "rb") as f: + assert f.read() == open(os.path.join(source, "big.bin"), "rb").read() + class TestOmitTimes: """-O/--omit-dir-times and -J/--omit-link-times are recognized and cross the wire as receiver-side preferences. FastSync does not currently apply dir or diff --git a/tests/test_client_cli.c b/tests/test_client_cli.c index 03deceb..ffce540 100644 --- a/tests/test_client_cli.c +++ b/tests/test_client_cli.c @@ -3,6 +3,7 @@ #include "client_validation.h" #include "chmod.h" #include "config.h" +#include "delta.h" #include "file_list.h" #include "log.h" #include "test_utils.h" @@ -2908,6 +2909,55 @@ static void test_parse_args_password_file() { config_delete(cfg); } +/* --block-size (Delta block size): --block-size/--delta-block set + * config->delta_block_size, out-of-range values are rejected with the default + * kept, and the configured size genuinely reaches the delta engine (a larger + * block yields fewer signature blocks for identical data). */ +static void test_parse_args_block_size() { + Config* cfg = config_create(); + cfg->send_directory = str_dup("/src"); + cfg->receive_root_directory = str_dup("/dst"); + int positional_args[2]; + int positional_count = 0; + + char* argv_long[] = {"fastsync", "--block-size", "4096", "/src", "/dst"}; + EXPECT_EQ_INT(parse_args(cfg, 5, argv_long, positional_args, &positional_count), 0); + EXPECT_EQ_INT((int)cfg->delta_block_size, 4096); + + cfg->delta_block_size = DELTA_BLOCK_SIZE_DEFAULT; + char* argv_delta[] = {"fastsync", "--delta-block", "2048", "/src", "/dst"}; + positional_count = 0; + EXPECT_EQ_INT(parse_args(cfg, 5, argv_delta, positional_args, &positional_count), 0); + EXPECT_EQ_INT((int)cfg->delta_block_size, 2048); + + /* Out of range: parsed, warned, and the default is kept. */ + cfg->delta_block_size = DELTA_BLOCK_SIZE_DEFAULT; + char* argv_bad[] = {"fastsync", "--block-size", "1", "/src", "/dst"}; + positional_count = 0; + EXPECT_EQ_INT(parse_args(cfg, 5, argv_bad, positional_args, &positional_count), 0); + EXPECT_EQ_INT((int)cfg->delta_block_size, (int)DELTA_BLOCK_SIZE_DEFAULT); + + /* A non-default block size changes the number of signature blocks for + identical data: block_count = ceil(size / block_size). */ + const char data[10000] = {0}; + DeltaSignature* small = delta_signature_create_seeded(data, sizeof(data), 1024, 0); + DeltaSignature* large = delta_signature_create_seeded(data, sizeof(data), 8192, 0); + EXPECT_NOT_NULL(small); + EXPECT_NOT_NULL(large); + /* cppcheck-suppress knownConditionTrueFalse -- EXPECT_NOT_NULL above asserts, + but cppcheck cannot see through the macro; the guard is defensive. */ + if (small && large) { + EXPECT_TRUE(large->block_size == 8192 && small->block_size == 1024); + EXPECT_TRUE(large->block_count < small->block_count); + EXPECT_EQ_INT((int)small->block_count, 10); /* ceil(10000/1024) */ + EXPECT_EQ_INT((int)large->block_count, 2); /* ceil(10000/8192) */ + } + delta_signature_destroy(small); + delta_signature_destroy(large); + + config_delete(cfg); +} + void test_client_cli() { test_validate_config_required_paths(); test_parse_args_numeric_ids(); @@ -2918,6 +2968,7 @@ void test_client_cli() { test_parse_args_rejects_malformed_identity(); test_parse_args_preallocate(); test_parse_args_metadata_times(); + test_parse_args_block_size(); test_parse_args_devices_specials(); test_parse_args_atimes_long_and_short(); test_parse_args_omit_link_times_long(); diff --git a/tests/test_file.c b/tests/test_file.c index d771b75..ad1d002 100644 --- a/tests/test_file.c +++ b/tests/test_file.c @@ -1,5 +1,9 @@ +#ifndef _GNU_SOURCE +#define _GNU_SOURCE /* SEEK_HOLE/SEEK_DATA for the sparse-hole sparseness check */ +#endif #include "test_file.h" #include "file.h" +#include "file_store.h" #include "data.h" #include "config.h" #include "utils.h" @@ -1216,6 +1220,123 @@ void test_trust_sender() { file_set_authorized_root(-1, NULL); } +/* --sparse/-S hole preservation: a buffer with a long zero run written via + * file_store_write_secure(sparse=true) must round-trip its content exactly and + * have the right logical size, and should additionally be genuinely sparse on + * filesystems that support holes. The sparseness assertion is tolerant: if the + * filesystem reports no holes (SEEK_HOLE/SEEK_DATA -> ENXIO) we skip the strict + * block-count check, but content and size always hold. */ +static void test_file_write_to_disk_sparse_preserves_holes() { + const char* path = "test_sparse_file.bin"; + unlink(path); + /* 256 KiB with a 128 KiB zero run in the middle, bracketed by headers/tails. */ + const unsigned long long size = 256u * 1024u; + unsigned char* buf = malloc(size); + EXPECT_NOT_NULL(buf); + /* cppcheck-suppress knownConditionTrueFalse -- EXPECT_NOT_NULL above asserts, + but cppcheck cannot see through the macro; the guard is defensive. */ + if (!buf) + return; + memset(buf, 0, size); + for (unsigned long long i = 0; i < 4096; i++) { + buf[i] = (unsigned char)(i % 251); + buf[size - 1 - i] = (unsigned char)((i * 7) % 253); + } + + EXPECT_TRUE(file_store_write_secure(path, buf, size, false, true, NULL, false)); + free(buf); + + /* Logical size must equal data_size exactly. */ + struct stat st; + EXPECT_EQ_INT(stat(path, &st), 0); + EXPECT_EQ_INT((int)st.st_size, (int)size); + + /* Content must round-trip exactly. */ + int fd = open(path, O_RDONLY); + EXPECT_TRUE(fd >= 0); + /* cppcheck-suppress knownConditionTrueFalse -- EXPECT_TRUE above asserts, + but cppcheck cannot see through the macro; the guard is defensive. */ + if (fd >= 0) { + unsigned char* readback = malloc(size); + if (readback) { + unsigned long long got = 0; + while (got < size) { + ssize_t n = read(fd, readback + got, (size_t)(size - got)); + if (n <= 0) + break; + got += (unsigned long long)n; + } + EXPECT_EQ_INT((int)got, (int)size); + if (got == size) { + /* The middle hole region stays all-zero. */ + for (unsigned long long i = 4096; i < size - 4096; i++) + if (readback[i] != 0) { + EXPECT_EQ_INT(0, 1); + break; + } + } + free(readback); + } + /* Tolerant sparseness check: seek for holes; skip if unsupported. */ + off_t hole_off = lseek(fd, (off_t)4096, SEEK_HOLE); + if (hole_off >= 0 && hole_off < (off_t)size) { + off_t next_data = lseek(fd, hole_off, SEEK_DATA); + fstat(fd, &st); + int blocks = (int)(st.st_blocks * 512); + if (next_data > hole_off) + EXPECT_TRUE(blocks < (int)size); + } + close(fd); + } + unlink(path); +} + +/* --partial retention is hard to provoke end-to-end mid-transfer (the whole + * image is in one in-memory write), so this drives the failure path directly: + * a metadata whose mtime_nsec is out of the legal [0,999999999] range makes + * futimens (in file_restore_metadata_fd) fail with EINVAL AFTER the temp has + * been fully written. With keep_partial=true the written temp must be renamed + * to the destination path (a resumable partial); with keep_partial=false the + * same failure must leave NOTHING behind. The retention is always best-effort + * (never a corrupt blend), and this asserts the both-on/off behavior. */ +static void test_file_write_to_disk_partial_retention() { + const char* path = "test_partial_retention.bin"; + unlink(path); + const char content[] = "partial-retention payload"; + FileMetadata m; + memset(&m, 0, sizeof(m)); + m.mode = 0644; + m.uid = (uid_t)geteuid(); + m.gid = (gid_t)getegid(); + m.mtime_sec = 1700000000; + m.mtime_nsec = 2000000000; /* invalid: forces futimens EINVAL after the write */ + m.atime_valid = false; + m.crtime_valid = false; + bool ok = file_to_disk_secure_attrs(path, content, strlen(content), false, false, true, &m, false, + false, false, false, NULL, false, true, NULL); + EXPECT_FALSE(ok); /* the write itself succeeded, but metadata restore failed */ + /* Retained: the already-written temp now sits at the destination path. */ + int fd = open(path, O_RDONLY); + EXPECT_TRUE(fd >= 0); + /* cppcheck-suppress knownConditionTrueFalse -- EXPECT_TRUE above asserts, + but cppcheck cannot see through the macro; the guard is defensive. */ + if (fd >= 0) { + char buf[64]; + ssize_t n = read(fd, buf, sizeof(buf)); + close(fd); + EXPECT_EQ_INT((int)strlen(content), (int)n); + if (n == (ssize_t)strlen(content)) + EXPECT_TRUE(memcmp(buf, content, strlen(content)) == 0); + } + unlink(path); + + /* Same failure with keep_partial=false: temp is unlinked, nothing retained. */ + ok = file_to_disk_secure_attrs(path, content, strlen(content), false, false, true, &m, false, + false, false, false, NULL, false, false, NULL); + EXPECT_FALSE(ok); + EXPECT_TRUE(access(path, F_OK) == -1); +} + void test_file() { test_file_create(); test_file_special_rdev_valid(); @@ -1230,6 +1351,8 @@ void test_file() { test_file_save_to_disk_ignore_existing_entry_types(); test_file_save_to_disk_partial_install(); test_file_save_to_disk_reports_skips(); + test_file_write_to_disk_sparse_preserves_holes(); + test_file_write_to_disk_partial_retention(); test_file_write_to_disk_basic(); test_file_write_to_disk_with_fsync(); test_file_write_to_disk_preallocate_atomic(); diff --git a/tests/test_xattr.c b/tests/test_xattr.c index 44846c4..ed26996 100644 --- a/tests/test_xattr.c +++ b/tests/test_xattr.c @@ -222,6 +222,49 @@ static void test_link_copy_fallback_preserves_xattrs() { rmdir(basis_dir); } +/* --fake-super replay: fake_super_store_fd records the source stat into the + * reserved xattr, and fake_super_restore_fd re-applies mode/mtime (and owner, + * when the process may) fd-relative. Restore must also be a safe no-op with no + * xattr present. Guarded on filesystem xattr support. */ +static void test_fake_super_restore() { + const char* path = "test_fake_super_restore.txt"; + unlink(path); + int fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0600); + if (fd < 0) + return; + bool has_xattr = setxattr(path, "user.fastsync.xprobe", "p", 1, 0) == 0; + if (has_xattr) + removexattr(path, "user.fastsync.xprobe"); + if (!has_xattr) { + close(fd); + unlink(path); + return; /* skip silently when the filesystem has no xattr support */ + } + + /* No xattr present yet: restore is a silent no-op (returns false, no crash). */ + EXPECT_FALSE(fake_super_restore_fd(fd)); + + fake_super_store_fd(fd, 1001, 1002, 0751, 1700000000, 123456789); + EXPECT_TRUE(fake_super_restore_fd(fd)); + struct stat st; + EXPECT_EQ_INT(fstat(fd, &st), 0); + EXPECT_EQ_INT((int)(st.st_mode & 07777), 0751); + + /* Restore with a malformed record must skip without failing. */ + time_t before = st.st_mtime; + int wfd = open(path, O_RDONLY); + if (wfd >= 0) { + EXPECT_EQ_INT((int)fsetxattr(wfd, FAKESUPER_XATTR, "not-a-valid-record", 19, 0), 0); + close(wfd); + } + EXPECT_FALSE(fake_super_restore_fd(fd)); + fstat(fd, &st); + EXPECT_EQ_INT((int)st.st_mtime, (int)before); + + close(fd); + unlink(path); +} + void test_xattr() { test_xattr_wire_roundtrip(); test_xattr_reject_privileged_namespace(); @@ -229,4 +272,5 @@ void test_xattr() { test_xattr_count_bound(); test_xattr_capture_and_appliable(); test_link_copy_fallback_preserves_xattrs(); + test_fake_super_restore(); } \ No newline at end of file