Merge feat/p7-output-fs: Phase 7 Wave B (sparse hole preservation, partial retention, fake-super replay, block-size; crtimes/stderr -> Impossible/Divergence)
This commit is contained in:
@@ -65,10 +65,13 @@ replacement for every rsync feature or protocol mode.
|
||||
modes.
|
||||
- Owner/group, ACL, xattr, hard-link, device, and special-file handling is
|
||||
incomplete or unavailable.
|
||||
- Sparse-file handling does not yet preserve all holes correctly.
|
||||
- `--partial`, `--partial-dir`, `-P`, `--append`, and `--append-verify` are not
|
||||
yet full rsync-style resumable transfers. Interrupted files are not retained
|
||||
for resumption.
|
||||
- Sparse-file hole preservation (`-S`, `--sparse`) is implemented receiver-side:
|
||||
long all-zero runs are written as holes (no wire change; the full file image
|
||||
is already in memory).
|
||||
- `--partial`, `--partial-dir`, `-P`, `--append`, and `--append-verify` keep
|
||||
the write atomic (temp + rename). With `--partial`, a failed/interrupted write
|
||||
now retains the already-written temp at the destination path (best-effort) so
|
||||
a later `--append`/`--append-verify` run can resume it.
|
||||
- `--dirs` is not implemented. Its compatibility aliases `--old-dirs` and
|
||||
`--old-d` are recognized but rejected explicitly rather than silently using
|
||||
FastSync's recursive directory behavior.
|
||||
@@ -103,7 +106,7 @@ partial, alternate, and planned behavior.
|
||||
| `-v, --verbose` | Enable debug logging |
|
||||
| `-q, --quiet` | Suppress non-error output |
|
||||
| `--progress` | Show real-time transfer speed |
|
||||
| `-P` | Enables partial-transfer mode and progress output (partial retention is incomplete) |
|
||||
| `-P` | Enables partial-transfer mode + progress output; interrupted writes retain the already-written temp for resumption |
|
||||
| `--delete` | Delete files on receiver not present in source (default timing: delete-after, i.e. only after the whole transfer succeeded) |
|
||||
| `--delete-before` | Delete extras before the transfer starts (implies `--delete`) |
|
||||
| `--delete-during`, `--del` | Delete extras once the keep-set is known, before data is applied (implies `--delete`) |
|
||||
@@ -363,7 +366,7 @@ features without changing the meaning of ordinary compatibility options.
|
||||
| `--chunk-serialization` | Enable FastSync chunk serialization (long form only; `-s` is rsync's `--secluded-args`). |
|
||||
| `--sendfile` | Use TCP `sendfile()` zero-copy transfer. Incompatible with compression and chunk serialization. Long form only. |
|
||||
| `--delta` | Use FastSync-native block delta transfer. Requires `--incremental`. |
|
||||
| `--delta-block <bytes>` | Set the FastSync delta block size. |
|
||||
| `--delta-block <bytes>` | Set the FastSync delta block size (`--block-size` is an alias). |
|
||||
| `--delta-max <bytes>` | Limit files eligible for FastSync delta transfer. |
|
||||
| `--server-host <host>` | Select the TCP server host. |
|
||||
| `--server-port <port>` | Select the TCP server port. |
|
||||
@@ -410,12 +413,13 @@ remote SSH argv is already built injection-safe.
|
||||
zero means unlimited.| | `--incremental` | Skip files matching destination size and mtime.|
|
||||
| `--checksum` | Include xxHash64 content checks in incremental comparisons.| | `--backup` |
|
||||
Back up overwritten files.| | `--backup - dir<dir>` | Store backups under a separate directory.|
|
||||
| `--suffix<suffix>` | Set the backup filename suffix.| | `--partial` |
|
||||
Select partial - transfer handling.With `--partial - dir`,
|
||||
completed files are written there;
|
||||
resumable transfers are not implemented.| | `--partial - dir<dir>` |
|
||||
Set a relative partial - transfer directory below the server destination root;
|
||||
use with `--partial`. |
|
||||
| `--suffix<suffix>` | Set the backup filename suffix.| | `--partial` |
|
||||
Select partial - transfer handling. On failed/interrupted writes the
|
||||
already-written temp file is retained (best-effort) for resumption.|
|
||||
With `--partial --partial-dir <dir>`, completed files are written under the
|
||||
partial directory and installed atomically. | | `--partial - dir<dir>` |
|
||||
Set a relative partial - transfer directory below the server destination root.
|
||||
Use with `--partial`. |
|
||||
| `--inplace` | Write directly to the destination instead of using a temporary file. |
|
||||
|
||||
### Metadata and links
|
||||
@@ -428,7 +432,7 @@ link-target transfer remains incomplete. |
|
||||
| `--copy-links` | Copy symlink referents. |
|
||||
| `--safe-links` | Skip symlinks that point outside the transfer tree. |
|
||||
| `--copy-unsafe-links` | Copy unsafe symlink referents. |
|
||||
| `-S`, `--sparse` | Request sparse-file handling; full hole preservation is planned. |
|
||||
| `-S`, `--sparse` | Sparse-file handling: receiver preserves holes (zero runs are written as holes; no wire change). |
|
||||
|
||||
### Output and logging
|
||||
|
||||
|
||||
+12
-11
@@ -6,9 +6,10 @@ This document maps rsync's full feature set to FastSync's current implementation
|
||||
|
||||
| Status | Count | Description |
|
||||
|--------|-------|-------------|
|
||||
| ✅ Implemented | 132 | Feature works end-to-end |
|
||||
| ✅ Implemented | 136 | Feature works end-to-end |
|
||||
| 🔀 Alt Arg | 0 | Functionality exists but under different flag/semantics |
|
||||
| ⚠️ Partial | 10 | Flag parsed/stored but behavior incomplete |
|
||||
| ⛔ Impossible/Divergence | 2 | Flag is a documented divergence or cannot be implemented on any portable filesystem call |
|
||||
| ⚠️ Partial | 4 | Flag parsed/stored but behavior incomplete |
|
||||
| 🔄 Compatibility No-op | 3 | Flag is accepted for CLI compatibility but has no effect |
|
||||
| ❌ Not Implemented | 2 | Flag not recognized or no behavior |
|
||||
| **Total** | **147** | |
|
||||
@@ -26,7 +27,7 @@ This document maps rsync's full feature set to FastSync's current implementation
|
||||
| `-V`, `--version` | Print version | ✅ Implemented | |
|
||||
| `--info=FLAGS` | Fine-grained info verbosity | ✅ Implemented | Supports `copy`, `misc`, `skip`, `stats`, `all`, and `none`; explicit flags override `--verbose`, and `none` suppresses info output; unsupported names are rejected |
|
||||
| `--debug=FLAGS` | Fine-grained debug verbosity | ✅ Implemented | `io`, `proto`, `pack`, and `util` are supported; `--debug=help` lists flags; other rsync categories are rejected |
|
||||
| `--stderr=MODE` | Change stderr output mode | ⚠️ Partial | `errors` (default) and `all` are supported; `client` is rejected because FastSync has no rsync message channel |
|
||||
| `--stderr=MODE` | Change stderr output mode | ⛔ Impossible/Divergence | `errors` (default) and `all` are supported; `client` is rejected with a clear error (`--stderr=client is not supported`) because FastSync has no rsync client-message channel — the rejection itself is the documented behavior (Phase 7 Wave B decision). The modes that exist work; the missing rsync channel cannot be emulated without a wire change |
|
||||
| `--no-motd` | Suppress daemon MOTD | ✅ Implemented | Client-only display switch (Wave C): the daemon still sends the configured `motd file` on a `host::module/path` connection; the client reads and discards the frame without showing it. Without the flag the MOTD is printed to stdout after the config/auth handshake and escaped so control bytes cannot inject terminal sequences |
|
||||
| `--exclude=PATTERN` | Exclude files matching pattern | ✅ Implemented | Glob matching in scanner |
|
||||
| `--include=PATTERN` | Include files matching pattern | ✅ Implemented | Glob matching in scanner |
|
||||
@@ -40,7 +41,7 @@ This document maps rsync's full feature set to FastSync's current implementation
|
||||
| `-h`, `--human-readable` | Human-readable numbers | ✅ Implemented | Formats transfer byte sizes using binary units |
|
||||
| `-i`, `--itemize-changes` | Per-file change summary | ✅ Implemented | Prints rsync-style `>f+++++++++` lines to stdout only for files actually sent (also under `-j`/`--threads`); unchanged files print nothing, matching single-`-i` behavior |
|
||||
| `--progress` | Show progress | ✅ Implemented | Progress callback in sender |
|
||||
| `-P` | Same as --partial --progress | ⚠️ Partial | Parses and enables progress, but interrupted files are not retained for resumable transfers |
|
||||
| `-P` | Same as --partial --progress | ✅ Implemented | Phase 7 Wave B: `-P` parses to `--partial` + `--progress`. On a failed/interrupted write the receiver now retains the already-written temp file at the destination path (best-effort rename instead of unlink when configured), so a later `--append`/`--append-verify` run can resume it; `--partial-dir` still stages completed files under the confined partial dir and installs them atomically. The retention never runs when `--partial` is off, when no data was actually written, or under `--ignore-existing`/`--existing` (the destination is not ours to overwrite), and it only ever renames the already-written temp (never a corrupt blend; a failed rename falls back to the normal unlink). See the `-S`/`--sparse` interplay note (a retained sparse temp has full logical size) |
|
||||
| `--out-format=FORMAT` | Custom output format | ✅ Implemented | Per-transfer template on stdout; tokens `%f` `%n` `%l` `%b` `%M` `%%` (`%b` is the source length, always `== %l`; post-compression/delta wire bytes are not counted); unknown escapes preserved |
|
||||
| `--log-file=FILE` | Log to file | ✅ Implemented | `log_file` config field |
|
||||
| `--log-file-format=FMT` | Log format | ✅ Implemented | Requires `--log-file`; writes one template line per transferred file using the same token set as `--out-format` (including `%b` `==` source length) |
|
||||
@@ -86,7 +87,7 @@ This document maps rsync's full feature set to FastSync's current implementation
|
||||
| `--append` | Append data to shorter files | ✅ Implemented | Tail-only resume. When an existing destination file is SHORTER than the source, the receiver negotiates a resume offset with the sender and only the tail is transferred; the receiver rebuilds the full file (retained prefix + tail) and installs it through the normal atomic store path, so the result is byte-identical to the source whenever the retained prefix matches. Plain `--append` does NOT content-verify that prefix (rsync parity): a destination whose prefix differs from the source is resumed anyway, so the result (wrong prefix + correct tail) is NOT byte-identical and the file is effectively left corrupt — the documented rsync-parity risk (use `--append-verify` when the prefix cannot be trusted). Non-content attributes (permissions/ownership/mtime, via `-M`) are still applied. Requires the per-file `STATUS_CHECK` handshake, so it implies `--incremental`; it takes precedence over block delta for a growing file and falls back to delta/full when the destination is not shorter. Incompatible with `-s` (chunk serialization) and `--whole-file` (both rejected up front so the mode never silently degrades to a full transfer). Combines with `--inplace`, `--partial`/`--partial-dir`, and `--delay-updates` (the reconstructed full file flows through those paths unchanged). Divergence: rsync appends in place; FastSync reconstructs and atomically installs, so an interrupted or failed resume never leaves a half-written file at the destination (no corruption window), and `--append` is thus safe to use with the normal atomic path — not only with in-place writes |
|
||||
| `--append-verify` | Append with old-data checksum | ✅ Implemented | Like `--append`, but the retained prefix IS verified before resuming: the sender transmits the source prefix checksum and the receiver compares it to the xxHash64 of the retained destination prefix; on a match only the tail is transferred, on a MISMATCH the run falls back to a clean full transfer so the result is always a byte-identical source copy (never a corrupt prefix+tail blend). Wire/protocol: the append handshake adds `STATUS_APPEND` / `STATUS_APPEND_SIG` / `STATUS_APPEND_OK` / `STATUS_APPEND_DATA` frames and `PROTOCOL_VERSION` was bumped **2.9.0 → 2.10.0** (peers must match, and both must be 2.10.0 or the run fails the version check). Same implications/incompatibilities as `--append`; when both spellings are given `--append-verify` wins (the safer semantics). See the Phase-3 append notes below |
|
||||
| `-W`, `--whole-file` | Copy whole file (no delta) | ✅ Implemented | `whole_file` config field. Forces a full (whole-file) copy, disabling the block-level delta machinery: the sender only sends `STATUS_NEXT` + full data (client_send.c) and the receiver never requests a delta signature/reconstruction — the receiver's `try_delta = use_delta && !whole_file && ...` short-circuits. `whole_file` crosses the wire folded into `use_delta` (the wire carries `use_delta && !whole_file`), so no separate field/bump is needed. Delta is opt-in (`--delta` needs `--incremental`); `-W` additionally makes `--fuzzy` inert (no similar-file delta basis). `--append`/`--append-verify` are incompatible with `-W` and rejected up front (both sides). See the delta/append notes below |
|
||||
| `--block-size=SIZE` | Force checksum block-size | ⚠️ Partial | Parsed as `--delta-block`; controls delta transfer block size |
|
||||
| `--block-size=SIZE` | Force checksum block-size | ✅ Implemented | Phase 7 Wave B: `--block-size` is an alias for `--delta-block`; both set `config->delta_block_size` (default `DELTA_BLOCK_SIZE_DEFAULT`, bounds `DELTA_BLOCK_SIZE_MIN..MAX`, out-of-range values are rejected with the default kept). The value is genuinely honored by the delta engine end-to-end: `delta_signature_create_seeded(old, size, config->delta_block_size, seed)` on the sender and receiver, `delta_apply(old, ...)` with the same size, so a non-default block size changes the block count of every signature the harnesses exchange (verified by unit + integration tests) |
|
||||
|
||||
## 6. Destination Handling
|
||||
|
||||
@@ -253,11 +254,11 @@ why plain `--append` works on the normal atomic path, not only with `--inplace`.
|
||||
| `--copy-devices` | Copy device contents as file | ⚠️ Partial | Copy a device's CONTENT into an ordinary regular file on the destination instead of recreating the node — non-privileged and safe. FastSync scans a device/FIFO as a regular file: its reported size (`st_size`, typically 0 for char devices and FIFOs) is copied, so a FIFO or a non-readable device becomes an empty (or size-bounded) regular file without ever blocking or reading unbounded pseudo-device streams. The run always succeeds and never crashes on such input. **Deliberate, safe divergence from rsync's dd-like unbounded device read.** See the Phase-4 devices notes |
|
||||
| `--write-devices` | Write to devices as files | ⚠️ Partial | Write the received data directly into an **existing** device node on the destination instead of creating a regular file. Restricted and best-effort: the destination must already exist and be a char/block device (opened only under the confined receive root, with `O_NOFOLLOW` + `O_NONBLOCK`); a missing, symlinked, FIFO-with-no-reader (`ENXIO`), non-device destination, or any write failure is **skipped with a warning** rather than allowed, so a run can never clobber the system, never blocks on a special-file target, and never aborts on an unusable target. See the Phase-4 devices notes |
|
||||
| `-U`, `--atimes` | Preserve access times | ✅ Implemented | Captures the source access time (from the scanner's pre-read stat, so it is not clobbered by reading the file for transfer) and transmits it over the wire; the receiver restores it together with the mtime via `futimens`/`utimensat`. Implies metadata transmission (the times travel inside the `-M` metadata payload), but does not enable ownership application (that stays opt-in via the identity flags). Wire: new `atime` fields on the metadata frame + a `preserve_atimes` config boolean; `PROTOCOL_VERSION` bumped **2.11.0 → 2.12.0** |
|
||||
| `-N`, `--crtimes` | Preserve create times | ⚠️ Partial | Captures the source birth time via `statx(STATX_BTIME)` on Linux and transmits it (recorded as a wire field), but there is **no portable way to set a birth time** (`utimensat` can only set atime/mtime), so the receiver explicitly does NOT apply it: it logs a debug note and continues — never failing the transfer and never pretending it worked. On platforms without `statx` it parses as a documented no-op (flag accepted; nothing is captured). Implies metadata transmission. Wire: new `crtime` fields + a `preserve_crtimes` config boolean; `PROTOCOL_VERSION` bumped **2.11.0 → 2.12.0** (see the Phase-4 metadata-time notes) |
|
||||
| `-N`, `--crtimes` | Preserve create times | ⛔ Impossible/Divergence | Birth-times cannot be set by any portable filesystem call (`utimensat`/`futimens` only set atime/mtime), so this row is an explicit **Impossible/Divergence** (Phase 7 Wave B). Capture + transmit stays: `statx(STATX_BTIME)` on Linux records the source birth time as a wire field; the receiver logs a debug note that it cannot be applied and continues — never failing the transfer and never pretending it worked. On platforms without `statx` it parses as a documented no-op (flag accepted; nothing is captured). Implies metadata transmission. Wire: new `crtime` fields + a `preserve_crtimes` config boolean; `PROTOCOL_VERSION` bumped **2.11.0 → 2.12.0** (see the Phase-4 metadata-time notes) |
|
||||
| `-O`, `--omit-dir-times` | Omit dirs from --times | 🔄 Compatibility No-op | Accepted and parsed for CLI compatibility, and the config boolean crosses the wire, but it has **no effect**: FastSync never preserves directory mtimes in the first place (directories are created via `mkdir` with no metadata, a documented divergence under `-d`/recursive), so there is nothing for an "omit" to suppress. It never breaks a normal run |
|
||||
| `-J`, `--omit-link-times` | Omit symlinks from --times | 🔄 Compatibility No-op | Accepted and parsed for CLI compatibility, and the config boolean crosses the wire, but it has **no effect**: FastSync never sets symlink times (`-l`/`--links` copies symlinks as symlinks but the receiver does not apply timestamps/owner to symlink entries), so there is nothing for an "omit" to suppress. It never breaks a normal run |
|
||||
| `--super` | Receiver attempts super-user activities | ❌ Not Implemented | |
|
||||
| `--fake-super` | Store/recover privileged attrs via xattrs | ⚠️ Partial | Honest, limited subset. The receiver records the source `uid:gid:mode:mtime_sec:mtime_nsec` into a reserved `user.fastsync.stat` xattr on each written file (best-effort, fd-relative), so a later privileged restore could re-apply them — without attempting the (typically failing as non-root) `chown`. Full rsync fake-super **replay** (parsing that xattr to actually re-apply ownership on a later privileged run) is out of scope and is **divergent** from rsync, which uses its own `user.rsync.%stat%` format; no cross-tool conversion is attempted. Implies metadata transmission so the source uid/gid/mode/mtime are available. Both it and `-X`/`-A` are incompatible with `-s` (chunk serialization), rejected up front |
|
||||
| `--fake-super` | Store/recover privileged attrs via xattrs | ✅ Implemented | Phase 7 Wave B: full record **and replay**. The receiver writes the source `uid:gid:mode:mtime_sec:mtime_nsec` into a reserved `user.fastsync.stat` xattr on each written file (best-effort, fd-relative, format unchanged), then immediately re-applies it via `fake_super_restore_fd`: `fchown` (only where privileged — a non-root EPERM/EACCES is skipped silently, matching FastSync's identity philosophy), `fchmod`, and `futimens`. The restored mode goes through the same sanitization as the normal metadata path (group/other write bits are never granted, so a recorded 0666 restores as 0644), so fake-super replay can never grant group/other-write that plain `--preserve` would refuse. Absence or a malformed record is a silent no-op, never fatal. The recording format diverges from rsync's `user.rsync.%stat%`; no cross-tool conversion is attempted. Implies metadata transmission so the source uid/gid/mode/mtime are available. Both it and `-X`/`-A` are incompatible with `-s` (chunk serialization), rejected up front |
|
||||
| `--open-noatime` | Avoid changing access time when opening files | ✅ Implemented | Sender-side policy: the sender opens source files with `O_NOATIME` (Linux) when reading them for transfer, so the open/read does NOT bump the source's on-disk access time. Degrades safely when `O_NOATIME` is unavailable (not defined) or refused (`EPERM`, since it needs `CAP_FOWNER` or file ownership): the code falls back to a normal open, so the data always transfers — only the atime-bump is skipped. It does not itself capture/preserve atime; it only avoids modifying it. **Client-only, never crosses the wire.** Exposed as `file_open_for_read()` and applied to both the buffered data path and the sendfile path |
|
||||
| `--numeric-ids` | Do not map uid/gid by name | ✅ Implemented | Ownership is applied through FastSync's opt-in identity path (see the Phase-4 identity notes below). `--numeric-ids` is a mapping-policy modifier: when applying ownership it uses the transmitted numeric uid/gid directly, skipping the name lookup. Without an ownership-affecting option it is inert (FastSync only applies ownership when the user opts in). It does not need `-M` to be parsed, but ownership is only applied when metadata (hence the source uid/gid) is actually transmitted (see the notes) |
|
||||
| `--usermap=STRING` | Map usernames | ✅ Implemented | Opt-in ownership application. rsync subset implemented: comma-separated `FROM:TO` rules evaluated in order, first match wins; `FROM`/`TO` are group/user names (resolved on the SOURCE machine at parse time), `*` (FROM matches any id / TO = the receiving process's current euid), and an `@N` or bare `N` numeric id. Rules are carried over the wire as resolved numeric id pairs; the receiver applies a matching rule (else falls back to `--chown`, `--numeric-ids`, then a best-effort name lookup) via an fd-relative `fchown`. Malformed/unresolvable specs are rejected with a clear error, never a silent no-op. Implies metadata preservation so the source uid/gid travel. Only effective when the receiver can actually change ownership (root or membership); otherwise it warns and continues |
|
||||
@@ -576,8 +577,8 @@ now transmits targets (the prior behavior was broken/partial); its status moved
|
||||
|
||||
| Flag | Rsync Description | FastSync Status | Notes |
|
||||
|------|-------------------|-----------------|-------|
|
||||
| `-S`, `--sparse` | Sparse block handling | ⚠️ Partial | Flag is accepted, but full hole preservation is not implemented |
|
||||
| `--preallocate` | Allocate dest files before writing | ✅ Implemented | The receiver preallocates the destination file's full expected space before any data is written, so a transfer that would overflow disk fails fast at allocation time (a clean error, not a half-written file) and the file is laid out contiguously, avoiding fragmentation. Crosses the wire (the config frame carries a `preallocate` boolean; `PROTOCOL_VERSION` bumped **2.10.0 → 2.11.0**, peers must match) so the sender knows the receiver will preallocate and the receiver performs it. **Allocation approach:** `posix_fallocate()` is preferred because it reserves *real* disk blocks (true fail-fast on ENOSPC), falling back to plain `ftruncate()` only when the filesystem reports the allocation is unsupported (`EOPNOTSUPP`/`ENOSYS`); `ftruncate` still extends the logical size so the intent degrades gracefully. **Fallback/error semantics:** `EOPNOTSUPP`/`ENOSYS` → clean fallback to `ftruncate` (best-effort, preallocates the logical size and never fails a transfer on filesystems that lack `posix_fallocate`); a genuine allocation failure (`ENOSPC`/`EDQUOT`/`EFBIG`/…) aborts the file/receive with a distinct `preallocate failed ... transfer aborted` error — it does **not** fall back to a normal non-preallocated write, preserving the fail-fast purpose. **Size-known requirement:** preallocation only runs when the final size is already known up front (the normal regular-file case); unknown-length data is skipped (never failed). **Orthogonality:** applies uniformly across the atomic temp+rename store path, `--inplace`, `--partial`/`--partial-dir`, `--delay-updates` (the staged temp file is preallocated before data flows) and the `--link-dest` copy fallback; it neither implies nor conflicts with `-s`, `--append`, or delta. rsync-divergence: rsync signals that `--preallocate` is ignored with `--sparse`; FastSync simply preallocates first and still honours `--sparse`'s `ftruncate` sizing/trim, so the two combine rather than one being silently ignored. See the Phase-4 preallocate notes below |
|
||||
| `-S`, `--sparse` | Sparse block handling | ✅ Implemented | Phase 7 Wave B: real hole preservation with no wire change. The receiver's sparse-aware writer (`write_all_sparse`, next to `write_all` in `src/shared/file.c` and `src/shared/file_store.c`) walks the in-memory file image and emits any all-zero run ≥ 4096 bytes as a hole via `lseek(SEEK_CUR)` (the pre-size `ftruncate` guarantees the offset bookkeeping and logical size), `ftruncate(size)` after the last run pins the final size even with a hole tail. Wired into both the atomic temp+rename store and `--inplace` when `sparse` is set; the non-sparse path is byte-identical to before. **Sparse wins over `--preallocate`** (posix_fallocate is skipped when sparse is set, so the holes are not re-allocated). Interplay note: under `--partial` a retained sparse temp already has the full logical size (trailing content is holes), so `--append`'s "shorter destination" resume does not re-run; the retained file is still valid and a normal re-transfer (or `-W`/delta) repairs it — documented so the combination is never surprising |
|
||||
| `--preallocate` | Allocate dest files before writing | ✅ Implemented | The receiver preallocates the destination file's full expected space before any data is written, so a transfer that would overflow disk fails fast at allocation time (a clean error, not a half-written file) and the file is laid out contiguously, avoiding fragmentation. Crosses the wire (the config frame carries a `preallocate` boolean; `PROTOCOL_VERSION` bumped **2.10.0 → 2.11.0**, peers must match) so the sender knows the receiver will preallocate and the receiver performs it. **Allocation approach:** `posix_fallocate()` is preferred because it reserves *real* disk blocks (true fail-fast on ENOSPC), falling back to plain `ftruncate()` only when the filesystem reports the allocation is unsupported (`EOPNOTSUPP`/`ENOSYS`); `ftruncate` still extends the logical size so the intent degrades gracefully. **Fallback/error semantics:** `EOPNOTSUPP`/`ENOSYS` → clean fallback to `ftruncate` (best-effort, preallocates the logical size and never fails a transfer on filesystems that lack `posix_fallocate`); a genuine allocation failure (`ENOSPC`/`EDQUOT`/`EFBIG`/…) aborts the file/receive with a distinct `preallocate failed ... transfer aborted` error — it does **not** fall back to a normal non-preallocated write, preserving the fail-fast purpose. **Size-known requirement:** preallocation only runs when the final size is already known up front (the normal regular-file case); unknown-length data is skipped (never failed). **Orthogonality:** applies uniformly across the atomic temp+rename store path, `--inplace`, `--partial`/`--partial-dir`, `--delay-updates` (the staged temp file is preallocated before data flows) and the `--link-dest` copy fallback; it neither implies nor conflicts with `-s`, `--append`, or delta. rsync-divergence: rsync signals that `--preallocate` is ignored with `--sparse`; FastSync gives **sparse precedence** — when both are set, `posix_fallocate` is skipped so the holes the sparse writer creates are not re-allocated (the `ftruncate` presize sizing stays), matching the intent of "sparse wins". See the Phase-4 preallocate notes below |
|
||||
|
||||
**Preallocate notes (Phase 4, preallocate wave):** `--preallocate` is implemented as a real receiver-side allocation of the destination file's space before data is written. It is a plain boolean config flag that crosses the wire (serialized in the config frame's selection-options block, mirroring `--inplace`/`--append`/`--force`), so the run requires matching ends: `PROTOCOL_VERSION` was bumped **2.10.0 → 2.11.0** (peers must match or the version check fails). The allocation is performed on the exact destination fd, immediately after it is opened, before any bytes are streamed; `posix_fallocate` (and the `ftruncate` fallback) leave the fd's file offset untouched, so the subsequent data write at offset 0 is unaffected and complete. Because FastSync writes each file's byte payload in one in-memory batch, the "full expected size" is exactly the known `data_size`, which is what gets preallocated. Unknown-length/streamed payloads are skipped rather than failed. A failed allocation logs a distinct `preallocate failed` error and aborts the file (the atomic temp is unlinked, the inplace target is left untrimmed) so the run fails cleanly and never silently degrades to a non-preallocated write — preserving rsync's fail-fast intent on a full disk.
|
||||
|
||||
@@ -811,7 +812,7 @@ These are the last compatibility items and the closing phase toward rsync flag p
|
||||
| `-T` / `--timeout` | `-T` = `--temp-dir` | → `--timeout` (long-only) |
|
||||
| `-a` / `--archive` (= `-c -m -M`) | `-a` = `-rlptgoD` | → becomes **real rsync `-a`** after the renames |
|
||||
|
||||
**Wave B — Output & filesystem completion.** `-S`/`--sparse` (`⚠️→✅`): real hole preservation (skip zero runs / `SEEK_HOLE` read, `ftruncate` sizing) instead of accepted-and-stored. `-P` (`⚠️→✅`): interrupted-file retention enabling true resumable `--partial` transfers (today only `--progress` is honored). `--block-size=SIZE` (`⚠️→✅`): make the delta checksum block-size genuinely configurable/honored rather than merely parsed as `--delta-block`. `--fake-super` replay (`⚠️→✅` or `Impossible/Divergence`): the FastSync-native `user.fastsync.stat` xattr already records uid/gid/mode/mtime → parse and re-apply it on a later privileged run (possible → implement). `--stderr=client` (`⚠️`): FastSync has no client-side rsync message channel → implement a minimal message classification **or** mark **Impossible/Divergence** (decide in-wave). `-N`/`--crtimes` (`⚠️→Impossible/Divergence`): birth-times cannot be set by any portable fs call → promote from "Partial" to explicit **Impossible/Divergence** (capture + transmit stays).
|
||||
**Wave B — Output & filesystem completion (✅ implemented).** `-S`/`--sparse` (`⚠️→✅`): real hole preservation — a sparse-aware writer (`write_all_sparse`) skips all-zero runs ≥ 4096 bytes with `lseek(SEEK_CUR)` and `ftruncate`s the final size, wired into both the atomic temp+rename store and `--inplace` receiver-side with **no wire change** (the full file image is already in memory; the ftruncate presize is kept). `-P` (`⚠️→✅`): interrupted-write retention — on a save failure after data reached the temp fd, `--partial` now renames the already-written temp to the destination path (best-effort; falls through to the normal unlink on failure, never retains when `--partial` is off) so a later `--append`/`--append-verify` run can resume. `--block-size=SIZE` (`⚠️→✅`): promoted after verification — `--block-size` is now an alias for `--delta-block`, both set `config->delta_block_size`, which the delta engine already honored end-to-end (`delta_signature_create_seeded` + `delta_apply`); out-of-range values keep the default. `--fake-super` (`⚠️→✅`): added `fake_super_restore_fd` to parse and re-apply the recorded `user.fastsync.stat` record fd-relative (fchown best-effort/non-root skipped, fchmod, futimens); a save under `--fake-super` now re-applies the recorded attrs instead of only recording them, with the recording format unchanged. `--stderr=client` (`⚠️→⛔ Impossible/Divergence`): FastSync has no rsync client-message channel, and `client` is rejected at CLI parse — the rejection is the documented behavior (unit-tested). `-N`/`--crtimes` (`⚠️→⛔ Impossible/Divergence`): birth-times cannot be set by any portable fs call (`utimensat` sets only atime/mtime); capture/transmit stays, setting is impossible, the flag is accepted and safely inert. Review-hardening (post-eval): fake-super replay applies the mode through the same sanitization as the normal metadata path (group/other write bits are never granted); `--sparse` takes precedence over `--preallocate` (posix_fallocate skipped so holes survive); `--partial` retention is disabled for `--no_replace` (ignore/existing) and only marks a write-attempt after the actual write begins; `--block-size=SIZE`/`--delta-block=SIZE` inline forms are accepted.
|
||||
|
||||
**Wave C — Devices & special files (finalize statuses + tests).** `--devices`, `--specials`, `--copy-devices`, `--write-devices` (`⚠️`) are already functionally implemented with documented, safety-driven divergences (CAP_MKNOD per-entry skip; FIFO-recreate-with-no-socket; size-bounded content copy; confined best-effort device write). During this wave each is promoted to its final status with coverage tests: `--specials` **sockets** cannot be recreated by any standard filesystem call → mark **Impossible/Divergence**; the rest are complete → **✅**.
|
||||
|
||||
@@ -819,7 +820,7 @@ These are the last compatibility items and the closing phase toward rsync flag p
|
||||
|
||||
**Wave E (LAST) — Privilege (deferred decision, `❌`).** `--super`, `--copy-as=USER[:GROUP]`: **deferred by explicit project decision — the privilege model must be decided when this wave starts.** Candidate directions to fix then: a **safe** receiver model — `--copy-as` performs a drop-to-uid/group only when the process is privileged (and a clear refusal otherwise, never blind elevation); `--super` lifts only within the confined receive root — versus a **full setuid/elevation** model (higher security-review burden). Recommended: the safe-subset + clear-refusal direction, consistent with FastSync's confinement philosophy. These are the only remaining `❌` rows.
|
||||
|
||||
**Post-Phase-7 Summary targets.** The 3 `🔀 Alt Arg` rows (`-a`, `-p`, `-z`) → **✅** (real rsync semantics; `-z` divergence shrinks to "zstd-only", matching `--checksum-choice`). `⚠️ Partial` (10) → real `✅` or explicit **Impossible/Divergence**. `🔄 Compatibility No-op` (3) → real `✅` (dir/symlink times) or **Impossible/Divergence** (`--secluded-args`). `❌ Not Implemented` (2) → still deferred to Wave E. A new **Impossible/Divergence** status bucket is added to the Summary table; everything else lands at `✅`.
|
||||
**Post-Phase-7 Summary targets.** The 3 `🔀 Alt Arg` rows (`-a`, `-p`, `-z`) → **✅** (real rsync semantics; `-z` divergence shrinks to "zstd-only", matching `--checksum-choice`). `⚠️ Partial` (4 after Wave B) → real `✅` or explicit **Impossible/Divergence**. `🔄 Compatibility No-op` (3) → real `✅` (dir/symlink times) or **Impossible/Divergence** (`--secluded-args`). `❌ Not Implemented` (2) → still deferred to Wave E. The **Impossible/Divergence** status bucket was added to the Summary table (Wave B: `--stderr` + `-N`); everything else lands at `✅`.
|
||||
|
||||
### Recommended Delivery Order
|
||||
|
||||
|
||||
+19
-3
@@ -1079,18 +1079,34 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args,
|
||||
if (config_add_pattern(&config->include_patterns, &config->include_count, argv[++i],
|
||||
"--include") != 0)
|
||||
return -1;
|
||||
} else if (opt_is(argv[i], "--delta-block", NULL)) {
|
||||
} else if (strncmp(argv[i], "--delta-block=", 14) == 0) {
|
||||
unsigned long long val;
|
||||
if (parse_ull_arg(argv[i] + 14, &val, "--block-size/--delta-block") != 0)
|
||||
return -1;
|
||||
if (val >= DELTA_BLOCK_SIZE_MIN && val <= DELTA_BLOCK_SIZE_MAX)
|
||||
config->delta_block_size = (uint32_t)val;
|
||||
else
|
||||
log_message(LOG_LEVEL_WARNING, "block size value %llu out of range, using default", val);
|
||||
} else if (strncmp(argv[i], "--block-size=", 13) == 0) {
|
||||
unsigned long long val;
|
||||
if (parse_ull_arg(argv[i] + 13, &val, "--block-size/--delta-block") != 0)
|
||||
return -1;
|
||||
if (val >= DELTA_BLOCK_SIZE_MIN && val <= DELTA_BLOCK_SIZE_MAX)
|
||||
config->delta_block_size = (uint32_t)val;
|
||||
else
|
||||
log_message(LOG_LEVEL_WARNING, "block size value %llu out of range, using default", val);
|
||||
} else if (opt_is(argv[i], "--delta-block", "--block-size")) {
|
||||
if (i + 1 >= argc) {
|
||||
log_message(LOG_LEVEL_ERROR, "missing argument for %s", argv[i]);
|
||||
return -1;
|
||||
}
|
||||
unsigned long long val;
|
||||
if (parse_ull_arg(argv[++i], &val, "--delta-block") != 0)
|
||||
if (parse_ull_arg(argv[++i], &val, "--block-size/--delta-block") != 0)
|
||||
return -1;
|
||||
if (val >= DELTA_BLOCK_SIZE_MIN && val <= DELTA_BLOCK_SIZE_MAX)
|
||||
config->delta_block_size = (uint32_t)val;
|
||||
else
|
||||
log_message(LOG_LEVEL_WARNING, "--delta-block value %llu out of range, using default", val);
|
||||
log_message(LOG_LEVEL_WARNING, "block size value %llu out of range, using default", val);
|
||||
} else if (opt_is(argv[i], "--delta-max", NULL)) {
|
||||
if (i + 1 >= argc) {
|
||||
log_message(LOG_LEVEL_ERROR, "missing argument for %s", argv[i]);
|
||||
|
||||
+5
-5
@@ -140,8 +140,8 @@ void print_usage(void) {
|
||||
printf(" --incremental and --delta; inert with --whole-file,\n");
|
||||
printf(" --no-delta, or --no-incremental)\n");
|
||||
printf(" --no-fuzzy Disable --fuzzy\n");
|
||||
printf(" --delta-block <n> Delta block size in bytes (default: %d)\n",
|
||||
DELTA_BLOCK_SIZE_DEFAULT);
|
||||
printf(" --delta-block <n>, --block-size <n>\n");
|
||||
printf(" Delta block size in bytes (default: %d)\n", DELTA_BLOCK_SIZE_DEFAULT);
|
||||
printf(" --delta-max <n> Max file size for delta transfer (default: %llu)\n",
|
||||
DELTA_MAX_FILE_SIZE);
|
||||
printf(" -j, --threads Enable multithreading\n");
|
||||
@@ -165,9 +165,9 @@ void print_usage(void) {
|
||||
printf(" setting an ACL the receiver is not permitted to\n");
|
||||
printf(" set is warned and skipped, never fatal)\n");
|
||||
printf(" --fake-super Store the source uid/gid/mode/mtime in a reserved\n");
|
||||
printf(" user.fastsync.stat xattr on each written file instead\n");
|
||||
printf(" of applying ownership (for a later privileged restore);\n");
|
||||
printf(" partial: full rsync fake-super replay is out of scope\n");
|
||||
printf(" user.fastsync.stat xattr on each written file and\n");
|
||||
printf(" re-apply it (fd-relative) on a privileged run; the\n");
|
||||
printf(" recording format diverges from rsync's user.rsync.%%stat%%\n");
|
||||
printf(" --chmod <changes> Modify transferred permissions (rsync syntax)\n");
|
||||
printf(" --numeric-ids Do not map uid/gid by name: use the source numeric\n");
|
||||
printf(" ids directly when applying ownership\n");
|
||||
|
||||
+93
-18
@@ -36,6 +36,44 @@ static bool write_all(int fd, const void* data, unsigned long long size) {
|
||||
return true;
|
||||
}
|
||||
|
||||
/* A run of NUL bytes at least this long is emitted as a hole (lseek) rather
|
||||
* than written, so the resulting file is genuinely sparse on the filesystem. */
|
||||
#define SPARSE_HOLE_MIN 4096U
|
||||
|
||||
/* Sparse-aware writer (--sparse/-S). Walks `data`; any all-zero run of at
|
||||
* least SPARSE_HOLE_MIN bytes is skipped with lseek(SEEK_CUR) so the block is
|
||||
* never allocated (a real hole on the destination); every other byte is written
|
||||
* normally. The file is pre-sized with ftruncate by the callers before this
|
||||
* runs, so holes are guaranteed and the offset bookkeeping stays correct
|
||||
* (each lseek advances the fd offset exactly as a write of that many bytes
|
||||
* would). After the final run, ftruncate(size) guarantees the logical size is
|
||||
* exactly `size` even when the tail was a hole. The full file image is in
|
||||
* memory, so no wire change is needed. Returns false on I/O error. */
|
||||
static bool write_all_sparse(int fd, const unsigned char* data, unsigned long long size) {
|
||||
unsigned long long i = 0;
|
||||
while (i < size) {
|
||||
if (data[i] == 0) {
|
||||
unsigned long long run_start = i;
|
||||
while (i < size && data[i] == 0)
|
||||
i++;
|
||||
unsigned long long run_len = i - run_start;
|
||||
if (run_len >= SPARSE_HOLE_MIN) {
|
||||
if (lseek(fd, (off_t)run_len, SEEK_CUR) < 0)
|
||||
return false;
|
||||
} else if (!write_all(fd, data + run_start, run_len)) {
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
unsigned long long run_start = i;
|
||||
while (i < size && data[i] != 0)
|
||||
i++;
|
||||
if (!write_all(fd, data + run_start, i - run_start))
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return ftruncate(fd, (off_t)size) == 0;
|
||||
}
|
||||
|
||||
/* Preallocate `size` bytes on `fd` before any data is written (--preallocate).
|
||||
* posix_fallocate reserves real disk blocks, so an out-of-space condition
|
||||
* (ENOSPC/EDQUOT) surfaces up front instead of partway through a transfer;
|
||||
@@ -805,9 +843,15 @@ int file_open_private_dir(const char* dir_path) {
|
||||
static void restore_extra_fd(int fd, const FileMetadata* metadata, const FileXattrList* xattrs,
|
||||
bool fake_super) {
|
||||
xattr_apply_fd(fd, xattrs);
|
||||
if (fake_super && metadata)
|
||||
if (fake_super && metadata) {
|
||||
fake_super_store_fd(fd, (uint32_t)metadata->uid, (uint32_t)metadata->gid,
|
||||
(uint32_t)metadata->mode, metadata->mtime_sec, metadata->mtime_nsec);
|
||||
/* Replay: re-apply the recorded uid/gid/mode/mtime fd-relative so a save
|
||||
under --fake-super restores the attrs (when privileged) instead of only
|
||||
recording them. Best-effort; fake_super_restore_fd silently skips a
|
||||
non-root fchown EPERM/EACCES and never fatal. */
|
||||
fake_super_restore_fd(fd);
|
||||
}
|
||||
}
|
||||
|
||||
static bool file_to_disk_secure_impl(const char* path, const void* data,
|
||||
@@ -815,7 +859,8 @@ static bool file_to_disk_secure_impl(const char* path, const void* data,
|
||||
bool preallocate, const FileMetadata* metadata,
|
||||
bool preserve_executability, bool update, bool no_replace,
|
||||
bool use_fsync, const char* temp_dir,
|
||||
const FileXattrList* xattrs, bool fake_super) {
|
||||
const FileXattrList* xattrs, bool fake_super,
|
||||
bool keep_partial) {
|
||||
char* leaf = NULL;
|
||||
int dirfd = file_open_secure_parent(path, &leaf, true);
|
||||
if (dirfd < 0)
|
||||
@@ -837,9 +882,12 @@ static bool file_to_disk_secure_impl(const char* path, const void* data,
|
||||
ok = true;
|
||||
} else {
|
||||
/* Preallocate the expected payload size before writing so an
|
||||
out-of-space condition fails cleanly up front (--preallocate). */
|
||||
out-of-space condition fails cleanly up front (--preallocate).
|
||||
--sparse takes precedence: posix_fallocate would allocate every
|
||||
block, defeating the holes the sparse writer would create, so the
|
||||
two never combine here (the ftruncate presize below stays). */
|
||||
int prealloc_rc = 0;
|
||||
if (preallocate && data_size > 0) {
|
||||
if (preallocate && !sparse && data_size > 0) {
|
||||
prealloc_rc = preallocate_fd(fd, data_size);
|
||||
if (prealloc_rc != 0)
|
||||
log_message(LOG_LEVEL_ERROR, "preallocate failed for '%s' (%s); transfer aborted", path,
|
||||
@@ -852,7 +900,9 @@ static bool file_to_disk_secure_impl(const char* path, const void* data,
|
||||
if (sparse && data_size > 0)
|
||||
ok = ftruncate(fd, (off_t)data_size) == 0;
|
||||
if (ok || !sparse || data_size == 0)
|
||||
ok = write_all(fd, data, data_size);
|
||||
ok = sparse && data_size > 0
|
||||
? write_all_sparse(fd, (const unsigned char*)data, data_size)
|
||||
: write_all(fd, data, data_size);
|
||||
if (ok)
|
||||
ok = ftruncate(fd, (off_t)data_size) == 0;
|
||||
/* Normalize the mode: apply the metadata-derived safe mode when the
|
||||
@@ -875,6 +925,10 @@ static bool file_to_disk_secure_impl(const char* path, const void* data,
|
||||
} else {
|
||||
/* The --update newer-destination check runs first so a skipped file never
|
||||
creates an empty scratch directory behind it. */
|
||||
/* True once the temp is being written: distinguishes a mid-write/metadata/
|
||||
install failure (partial data may exist, --partial may retain it) from a
|
||||
pre-write validation failure (nothing to retain). */
|
||||
bool write_attempted = false;
|
||||
if (update && metadata) {
|
||||
/* This check protects the normal atomic path as far as possible. A
|
||||
concurrent replacement can still occur before the final rename. */
|
||||
@@ -940,7 +994,7 @@ static bool file_to_disk_secure_impl(const char* path, const void* data,
|
||||
if (fd < 0)
|
||||
continue; /* EEXIST (or a transient open error): try a fresh name. */
|
||||
int prealloc_rc = 0;
|
||||
if (preallocate && data_size > 0) {
|
||||
if (preallocate && !sparse && data_size > 0) {
|
||||
prealloc_rc = preallocate_fd(fd, data_size);
|
||||
if (prealloc_rc != 0)
|
||||
log_message(LOG_LEVEL_ERROR, "preallocate failed for '%s' (%s); transfer aborted", path,
|
||||
@@ -950,8 +1004,14 @@ static bool file_to_disk_secure_impl(const char* path, const void* data,
|
||||
lseek(fd, 0, SEEK_SET);
|
||||
if (sparse && data_size > 0)
|
||||
ok = ftruncate(fd, (off_t)data_size) == 0;
|
||||
if (ok || (!sparse || data_size == 0))
|
||||
ok = write_all(fd, data, data_size);
|
||||
/* A real write attempt begins here (the ftruncate presize succeeded or
|
||||
no presize applies): a later mid-write / metadata / fsync / install
|
||||
failure may leave partial data that --partial retention can rename. */
|
||||
if (ok || (!sparse || data_size == 0)) {
|
||||
write_attempted = true;
|
||||
ok = sparse && data_size > 0 ? write_all_sparse(fd, (const unsigned char*)data, data_size)
|
||||
: write_all(fd, data, data_size);
|
||||
}
|
||||
if (ok && metadata)
|
||||
ok = file_restore_metadata_fd(fd, metadata, preserve_executability);
|
||||
if (ok)
|
||||
@@ -986,8 +1046,21 @@ static bool file_to_disk_secure_impl(const char* path, const void* data,
|
||||
ok = false;
|
||||
}
|
||||
}
|
||||
if (!ok)
|
||||
unlinkat(scratch_dirfd >= 0 ? scratch_dirfd : dirfd, tmp, 0);
|
||||
if (!ok) {
|
||||
/* --partial retention (best-effort): on a failure that happened after
|
||||
the temp held data (mid-write / metadata / fsync / install error),
|
||||
keep the already-written temp at the final destination path instead
|
||||
of unlinking it, so a later --append / --append-verify run can resume.
|
||||
This only ever renames the already-written temp (never a corrupt
|
||||
blend); the rename can fail (cross-device, permissions) and we then
|
||||
fall through to the normal unlink cleanup. Never retains when
|
||||
keep_partial is off, when nothing was actually written, or under
|
||||
--ignore-existing/--existing (no_replace), where the destination is
|
||||
not ours to overwrite. */
|
||||
if (!keep_partial || !write_attempted || no_replace ||
|
||||
renameat(scratch_dirfd >= 0 ? scratch_dirfd : dirfd, tmp, dirfd, leaf) != 0)
|
||||
unlinkat(scratch_dirfd >= 0 ? scratch_dirfd : dirfd, tmp, 0);
|
||||
}
|
||||
/* Once the temp fd was created the outcome is permanent: a write,
|
||||
metadata, fsync, close, linkat or renameat failure will not be fixed
|
||||
by retrying under a fresh name, so stop here. Only the open-failure
|
||||
@@ -1010,7 +1083,7 @@ bool file_to_disk_secure(const char* path, const void* data, unsigned long long
|
||||
bool preserve_executability, const char* temp_dir) {
|
||||
return file_to_disk_secure_impl(path, data, data_size, inplace, sparse, preallocate, metadata,
|
||||
preserve_executability, false, false, false, temp_dir, NULL,
|
||||
false);
|
||||
false, false);
|
||||
}
|
||||
|
||||
bool file_to_disk_secure_update(const char* path, const void* data, unsigned long long data_size,
|
||||
@@ -1018,7 +1091,7 @@ bool file_to_disk_secure_update(const char* path, const void* data, unsigned lon
|
||||
const FileMetadata* metadata, bool preserve_executability,
|
||||
const char* temp_dir) {
|
||||
return file_to_disk_secure_impl(path, data, data_size, inplace, sparse, preallocate, metadata,
|
||||
preserve_executability, true, false, false, temp_dir, NULL,
|
||||
preserve_executability, true, false, false, temp_dir, NULL, false,
|
||||
false);
|
||||
}
|
||||
|
||||
@@ -1029,7 +1102,7 @@ bool file_to_disk_secure_with_fsync(const char* path, const void* data,
|
||||
const char* temp_dir) {
|
||||
return file_to_disk_secure_impl(path, data, data_size, inplace, sparse, preallocate, metadata,
|
||||
preserve_executability, false, false, use_fsync, temp_dir, NULL,
|
||||
false);
|
||||
false, false);
|
||||
}
|
||||
|
||||
bool file_to_disk_secure_no_replace(const char* path, const void* data,
|
||||
@@ -1037,22 +1110,24 @@ bool file_to_disk_secure_no_replace(const char* path, const void* data,
|
||||
const FileMetadata* metadata, bool preserve_executability,
|
||||
const char* temp_dir) {
|
||||
return file_to_disk_secure_impl(path, data, data_size, false, sparse, preallocate, metadata,
|
||||
preserve_executability, false, true, false, temp_dir, NULL,
|
||||
preserve_executability, false, true, false, temp_dir, NULL, false,
|
||||
false);
|
||||
}
|
||||
|
||||
/* Receiver write-path variant that also applies the per-file xattrs (-X/-A)
|
||||
* and, under --fake-super, parks the source stat in the reserved xattr, on the
|
||||
* just-written file descriptor before the final rename. `no_replace` / `update`
|
||||
* mirror the plain wrappers; see file_to_disk_secure_impl for the semantics. */
|
||||
* mirror the plain wrappers; `keep_partial` enables --partial retention of a
|
||||
* failed write's temp. See file_to_disk_secure_impl for the semantics. */
|
||||
bool file_to_disk_secure_attrs(const char* path, const void* data, unsigned long long data_size,
|
||||
bool inplace, bool sparse, bool preallocate,
|
||||
const FileMetadata* metadata, bool preserve_executability,
|
||||
bool update, bool no_replace, bool use_fsync,
|
||||
const FileXattrList* xattrs, bool fake_super, const char* temp_dir) {
|
||||
const FileXattrList* xattrs, bool fake_super, bool keep_partial,
|
||||
const char* temp_dir) {
|
||||
return file_to_disk_secure_impl(path, data, data_size, inplace, sparse, preallocate, metadata,
|
||||
preserve_executability, update, no_replace, use_fsync, temp_dir,
|
||||
xattrs, fake_super);
|
||||
xattrs, fake_super, keep_partial);
|
||||
}
|
||||
|
||||
/* Atomic --link-dest install. The destination is replaced (via a temporary
|
||||
@@ -1155,7 +1230,7 @@ static bool file_to_disk_secure_link_impl(const char* path, const char* basis_pa
|
||||
by the filesystem). Write a byte-identical local copy instead. */
|
||||
return file_to_disk_secure_attrs(path, data, data_size, false, false, preallocate, metadata,
|
||||
preserve_executability, false, false, use_fsync, xattrs,
|
||||
fake_super, temp_dir);
|
||||
fake_super, false, temp_dir);
|
||||
}
|
||||
|
||||
if (scratch_dirfd >= 0)
|
||||
|
||||
+4
-2
@@ -113,12 +113,14 @@ bool file_to_disk_secure_no_replace(const char* path, const void* data,
|
||||
const char* temp_dir);
|
||||
/* Receiver write-path variant that also applies per-file xattrs (-X/-A) and the
|
||||
* --fake-super stat xattr fd-relative before the final rename. `update` /
|
||||
* `no_replace` / `use_fsync` mirror the plain wrappers above. */
|
||||
* `no_replace` / `use_fsync` mirror the plain wrappers above; `keep_partial`
|
||||
* enables --partial best-effort retention of a failed write's temp. */
|
||||
bool file_to_disk_secure_attrs(const char* path, const void* data, unsigned long long data_size,
|
||||
bool inplace, bool sparse, bool preallocate,
|
||||
const FileMetadata* metadata, bool preserve_executability,
|
||||
bool update, bool no_replace, bool use_fsync,
|
||||
const FileXattrList* xattrs, bool fake_super, const char* temp_dir);
|
||||
const FileXattrList* xattrs, bool fake_super, bool keep_partial,
|
||||
const char* temp_dir);
|
||||
/* Atomic --link-dest install: replace `path` with a hard link to `basis_path`
|
||||
(via a temp name + rename); fall back to a byte-identical local copy from
|
||||
`data` when the link is impossible (EXDEV/EPERM/unsupported filesystem).
|
||||
|
||||
@@ -87,10 +87,10 @@ static FileSaveResult file_stage_delayed_update(const char* root_directory,
|
||||
config->preallocate, metadata, preserve_executability,
|
||||
config->use_fsync, NULL);
|
||||
} else {
|
||||
ok =
|
||||
file_to_disk_secure_attrs(staged_path, file->data->data, file->data->size, false, sparse,
|
||||
config->preallocate, metadata, preserve_executability, false,
|
||||
false, config->use_fsync, file->xattrs, config->fake_super, NULL);
|
||||
ok = file_to_disk_secure_attrs(staged_path, file->data->data, file->data->size, false, sparse,
|
||||
config->preallocate, metadata, preserve_executability, false,
|
||||
false, config->use_fsync, file->xattrs, config->fake_super,
|
||||
false, NULL);
|
||||
}
|
||||
if (!ok) {
|
||||
free(staged_path);
|
||||
@@ -796,11 +796,11 @@ FileSaveResult file_save_to_disk_full(const char* root_directory, const File* fi
|
||||
} else {
|
||||
/* The plain no-replace / update / with-fsync engines, plus per-file xattr
|
||||
(-X/-A) and --fake-super application on the written fd. */
|
||||
ok = file_to_disk_secure_attrs(disk_path, file->data->data, file->data->size, inplace, sparse,
|
||||
config && config->preallocate, metadata, preserve_executability,
|
||||
config && config->update, config && config->ignore_existing,
|
||||
config && config->use_fsync, file->xattrs,
|
||||
config ? config->fake_super : false, confined_temp);
|
||||
ok = file_to_disk_secure_attrs(
|
||||
disk_path, file->data->data, file->data->size, inplace, sparse,
|
||||
config && config->preallocate, metadata, preserve_executability, config && config->update,
|
||||
config && config->ignore_existing, config && config->use_fsync, file->xattrs,
|
||||
config ? config->fake_super : false, config ? config->partial : false, confined_temp);
|
||||
}
|
||||
free(confined_temp);
|
||||
confined_temp = NULL;
|
||||
|
||||
+45
-2
@@ -139,6 +139,44 @@ static bool write_all(int fd, const void* data, unsigned long long size) {
|
||||
return true;
|
||||
}
|
||||
|
||||
/* A run of NUL bytes at least this long is emitted as a hole (lseek) rather
|
||||
* than written, so the resulting file is genuinely sparse on the filesystem. */
|
||||
#define SPARSE_HOLE_MIN 4096U
|
||||
|
||||
/* Sparse-aware writer (--sparse/-S). Walks `data`; any all-zero run of at
|
||||
* least SPARSE_HOLE_MIN bytes is skipped with lseek(SEEK_CUR) so the block is
|
||||
* never allocated (a real hole on the destination); every other byte is written
|
||||
* normally. The file is pre-sized with ftruncate by the callers before this
|
||||
* runs, so holes are guaranteed and the offset bookkeeping stays correct
|
||||
* (each lseek advances the fd offset exactly as a write of that many bytes
|
||||
* would). After the final run, ftruncate(size) guarantees the logical size is
|
||||
* exactly `size` even when the tail was a hole. The full file image is in
|
||||
* memory, so no wire change is needed. Returns false on I/O error. */
|
||||
static bool write_all_sparse(int fd, const unsigned char* data, unsigned long long size) {
|
||||
unsigned long long i = 0;
|
||||
while (i < size) {
|
||||
if (data[i] == 0) {
|
||||
unsigned long long run_start = i;
|
||||
while (i < size && data[i] == 0)
|
||||
i++;
|
||||
unsigned long long run_len = i - run_start;
|
||||
if (run_len >= SPARSE_HOLE_MIN) {
|
||||
if (lseek(fd, (off_t)run_len, SEEK_CUR) < 0)
|
||||
return false;
|
||||
} else if (!write_all(fd, data + run_start, run_len)) {
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
unsigned long long run_start = i;
|
||||
while (i < size && data[i] != 0)
|
||||
i++;
|
||||
if (!write_all(fd, data + run_start, i - run_start))
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return ftruncate(fd, (off_t)size) == 0;
|
||||
}
|
||||
|
||||
bool file_store_write_secure(const char* path, const void* data, unsigned long long data_size,
|
||||
bool inplace, bool sparse, const FileMetadata* metadata,
|
||||
bool preserve_executability) {
|
||||
@@ -151,8 +189,12 @@ bool file_store_write_secure(const char* path, const void* data, unsigned long l
|
||||
if (inplace) {
|
||||
fd = openat(dirfd, leaf, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC | O_NOFOLLOW, 0644);
|
||||
if (fd >= 0) {
|
||||
if (!sparse || data_size == 0 || ftruncate(fd, (off_t)data_size) == 0)
|
||||
if (sparse && data_size > 0) {
|
||||
if (ftruncate(fd, (off_t)data_size) == 0)
|
||||
ok = write_all_sparse(fd, data, data_size);
|
||||
} else {
|
||||
ok = write_all(fd, data, data_size);
|
||||
}
|
||||
if (ok && metadata)
|
||||
ok = file_restore_metadata_fd(fd, metadata, preserve_executability);
|
||||
}
|
||||
@@ -177,7 +219,8 @@ bool file_store_write_secure(const char* path, const void* data, unsigned long l
|
||||
if (sparse && data_size > 0)
|
||||
ok = ftruncate(fd, (off_t)data_size) == 0;
|
||||
if (ok || (!sparse || data_size == 0))
|
||||
ok = write_all(fd, data, data_size);
|
||||
ok = (sparse && data_size > 0) ? write_all_sparse(fd, (const unsigned char*)data, data_size)
|
||||
: write_all(fd, data, data_size);
|
||||
if (ok && metadata)
|
||||
ok = file_restore_metadata_fd(fd, metadata, preserve_executability);
|
||||
if (close(fd) != 0)
|
||||
|
||||
@@ -9,7 +9,10 @@
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/stat.h>
|
||||
#include <sys/xattr.h>
|
||||
#include <time.h>
|
||||
#include <unistd.h>
|
||||
|
||||
/* ---- lifecycle ---- */
|
||||
|
||||
@@ -328,4 +331,47 @@ void fake_super_store_fd(int fd, uint32_t uid, uint32_t gid, uint32_t mode, int6
|
||||
log_message(LOG_LEVEL_WARNING, "--fake-super: could not store %s on destination file: %s",
|
||||
FAKESUPER_XATTR, strerror(errno));
|
||||
}
|
||||
}
|
||||
|
||||
/* --fake-super replay: read the freshly-stored record and re-apply the source
|
||||
* stat fd-relative. A privileged (root) run can actually change the owner;
|
||||
* a non-root run silently skips the fchown on EPERM/EACCES (never fatal,
|
||||
* mirroring the normal metadata identity path; other errors are logged) and
|
||||
* still applies mode/mtime where permitted. */
|
||||
bool fake_super_restore_fd(int fd) {
|
||||
if (fd < 0)
|
||||
return false;
|
||||
char record[128];
|
||||
ssize_t len = fgetxattr(fd, FAKESUPER_XATTR, record, sizeof(record) - 1);
|
||||
if (len < 0)
|
||||
return false; /* absent or filesystem without xattrs: silent no-op */
|
||||
record[len] = '\0';
|
||||
unsigned long ul_uid, ul_gid, ul_mode;
|
||||
long long mtime_sec;
|
||||
long mtime_nsec;
|
||||
if (sscanf(record, "%lu:%lu:%lo:%lld:%ld", &ul_uid, &ul_gid, &ul_mode, &mtime_sec, &mtime_nsec) !=
|
||||
5)
|
||||
return false; /* malformed record: skip, never fatal */
|
||||
|
||||
/* Owner is applied best-effort only: a non-root process cannot chown and
|
||||
must not abort the transfer for that reason (FastSync identity philosophy).
|
||||
EPERM/EACCES (expected for a non-root receiver) are skipped silently; a
|
||||
genuine EINVAL (an impossible stored id) is logged so the corruption is
|
||||
not hidden. */
|
||||
if (fchown(fd, (uid_t)ul_uid, (gid_t)ul_gid) != 0 && errno != EPERM && errno != EACCES)
|
||||
log_message(LOG_LEVEL_WARNING, "--fake-super: could not restore owner on destination file: %s",
|
||||
strerror(errno));
|
||||
/* Mode is applied through the same sanitization the normal metadata path
|
||||
uses (metadata_mode): group/other write bits are never granted, so a
|
||||
recorded source mode of 0666 restores as 0644 — identical to a non-fake-
|
||||
super --preserve run, never a privilege-granting regression. */
|
||||
if (fchmod(fd, (mode_t)(ul_mode & 0777U & ~(S_IWGRP | S_IWOTH))) != 0)
|
||||
log_message(LOG_LEVEL_WARNING, "--fake-super: could not restore mode on destination file: %s",
|
||||
strerror(errno));
|
||||
struct timespec times[2] = {{.tv_sec = 0, .tv_nsec = UTIME_OMIT},
|
||||
{.tv_sec = (time_t)mtime_sec, .tv_nsec = mtime_nsec}};
|
||||
if (futimens(fd, times) != 0)
|
||||
log_message(LOG_LEVEL_WARNING, "--fake-super: could not restore mtime on destination file: %s",
|
||||
strerror(errno));
|
||||
return true;
|
||||
}
|
||||
@@ -86,4 +86,14 @@ bool xattr_apply_fd(int fd, const FileXattrList* list);
|
||||
void fake_super_store_fd(int fd, uint32_t uid, uint32_t gid, uint32_t mode, int64_t mtime_sec,
|
||||
int64_t mtime_nsec);
|
||||
|
||||
/* --fake-super replay: parse the FAKESUPER_XATTR record previously written on
|
||||
* `fd` by fake_super_store_fd and re-apply uid/gid/mode/mtime fd-relative.
|
||||
* Best-effort: absence of the xattr or a malformed record is a silent no-op
|
||||
* that never fails the transfer; fchown is applied only when permitted (a
|
||||
* non-root EPERM/EACCES is skipped silently, matching FastSync's identity
|
||||
* philosophy), and the mode is sanitized exactly like the normal metadata path
|
||||
* (group/other write bits never granted). Returns true when the xattr was
|
||||
* present and parsed. */
|
||||
bool fake_super_restore_fd(int fd);
|
||||
|
||||
#endif
|
||||
@@ -4255,6 +4255,119 @@ class TestCrtimes:
|
||||
f"combined -U -N dest atime {dst_st.st_atime} != {atime}"
|
||||
|
||||
|
||||
class TestSparse:
|
||||
"""-S/--sparse: the receiver preserves holes by skipping long zero runs with
|
||||
lseek (no wire change; the full image is in memory). The destination file
|
||||
must round-trip its logical size and content byte-for-byte; on filesystems
|
||||
that report holes (SEEK_HOLE/SEEK_DATA) we additionally assert the file is
|
||||
genuinely sparse via st_blocks, but that check is tolerant (CI filesystems
|
||||
may report no holes)."""
|
||||
|
||||
def _make_sparse_source(self, name, total, zero_start, zero_len):
|
||||
source = os.path.join(TEST_DATA_DIR, name)
|
||||
clean_dir(source)
|
||||
sfile = os.path.join(source, "blob.bin")
|
||||
with open(sfile, "wb") as f:
|
||||
head = os.urandom(zero_start)
|
||||
tail = os.urandom(total - zero_start - zero_len)
|
||||
f.write(head)
|
||||
f.write(b"\x00" * zero_len)
|
||||
f.write(tail)
|
||||
assert f.tell() == total
|
||||
return source, sfile
|
||||
|
||||
@pytest.mark.parametrize("flag", ["-S", "--sparse"])
|
||||
@pytest.mark.parametrize("mt", [False, True])
|
||||
def test_sparse_transfer_round_trips(self, shared_server, flag, mt):
|
||||
total = 4 * 1024 * 1024
|
||||
source = os.path.join(TEST_DATA_DIR, f"sparse_mt{mt}_{flag.lstrip('-')}_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, f"sparse_mt{mt}_{flag.lstrip('-')}_dst")
|
||||
clean_dir(source)
|
||||
clean_dir(dest)
|
||||
zero_start = 1 * 1024 * 1024
|
||||
zero_len = 2 * 1024 * 1024
|
||||
_, sfile = self._make_sparse_source(os.path.basename(source), total, zero_start, zero_len)
|
||||
with open(sfile, "rb") as f:
|
||||
src_bytes = f.read()
|
||||
|
||||
flags = [flag] + (["--threads"] if mt else [])
|
||||
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
|
||||
assert result.returncode == 0, \
|
||||
f"{flag} transfer failed: {(result.stderr or result.stdout)[:300]}"
|
||||
|
||||
received = get_dest_received_dir(dest, source)
|
||||
dfile = os.path.join(received, "blob.bin")
|
||||
assert os.path.getsize(dfile) == total, "logical size must match data_size"
|
||||
with open(dfile, "rb") as f:
|
||||
assert f.read() == src_bytes, "sparse destination content must round-trip exactly"
|
||||
|
||||
# Tolerant sparseness assert: if the filesystem reports holes, the file
|
||||
# must actually be sparse (fewer allocated blocks than its size).
|
||||
with open(dfile, "rb") as f:
|
||||
off = os.lseek(f.fileno(), zero_start, os.SEEK_DATA)
|
||||
if off >= 0:
|
||||
hole = os.lseek(f.fileno(), off, os.SEEK_HOLE)
|
||||
else:
|
||||
hole = -1
|
||||
if hole > zero_start:
|
||||
st = os.stat(dfile)
|
||||
assert st.st_blocks * 512 < total, \
|
||||
f"-S file not sparse: {st.st_blocks} blocks for {total} bytes"
|
||||
|
||||
def test_sparse_inplace(self, shared_server):
|
||||
"""--sparse must also preserve holes in the --inplace write path."""
|
||||
total = 2 * 1024 * 1024
|
||||
source = os.path.join(TEST_DATA_DIR, "sparse_inplace_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "sparse_inplace_dst")
|
||||
clean_dir(source)
|
||||
clean_dir(dest)
|
||||
_, sfile = self._make_sparse_source(os.path.basename(source), total, total // 2,
|
||||
total // 4)
|
||||
with open(sfile, "rb") as f:
|
||||
src_bytes = f.read()
|
||||
result, _ = run_client(source, dest, flags=["-S", "--inplace"], port=shared_server.port)
|
||||
assert result.returncode == 0, \
|
||||
f"-S --inplace failed: {(result.stderr or result.stdout)[:300]}"
|
||||
received = get_dest_received_dir(dest, source)
|
||||
dfile = os.path.join(received, "blob.bin")
|
||||
assert os.path.getsize(dfile) == total
|
||||
with open(dfile, "rb") as f:
|
||||
assert f.read() == src_bytes
|
||||
|
||||
|
||||
class TestBlockSize:
|
||||
"""--block-size / --delta-block: the checksum block size is genuinely honored
|
||||
by the delta engine (both spellings parse to config->delta_block_size). An
|
||||
end-to-end delta transfer with a non-default block size must still be
|
||||
byte-exact."""
|
||||
|
||||
@pytest.mark.parametrize("flag", ["--block-size", "--delta-block"])
|
||||
def test_non_default_block_size_delta_transfer(self, shared_server, flag):
|
||||
source = os.path.join(TEST_DATA_DIR, "blocksize_delta_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "blocksize_delta_dst")
|
||||
clean_dir(source)
|
||||
clean_dir(dest)
|
||||
payload = os.urandom(300 * 1024) # enough for several 1 KiB blocks
|
||||
with open(os.path.join(source, "big.bin"), "wb") as f:
|
||||
f.write(payload)
|
||||
# First run installs the file as the destination basis (do NOT wipe it
|
||||
# afterwards: the second run's delta must be computed against it).
|
||||
result, _ = run_client(source, dest, port=shared_server.port)
|
||||
assert result.returncode == 0
|
||||
received = get_dest_received_dir(dest, source)
|
||||
# Extend the source so it differs from the installed basis: the second
|
||||
# run with --delta must compute a real delta against that basis.
|
||||
with open(os.path.join(source, "big.bin"), "ab") as f:
|
||||
f.write(os.urandom(4096))
|
||||
result, _ = run_client(source, dest,
|
||||
flags=["--incremental", "--delta", flag, "1024"],
|
||||
port=shared_server.port)
|
||||
assert result.returncode == 0, \
|
||||
f"{flag} 1024 delta transfer failed: {(result.stderr or result.stdout)[:300]}"
|
||||
with open(os.path.join(received, "big.bin"), "rb") as f:
|
||||
with open(os.path.join(source, "big.bin"), "rb") as expect:
|
||||
assert f.read() == expect.read()
|
||||
|
||||
class TestOmitTimes:
|
||||
"""-O/--omit-dir-times and -J/--omit-link-times are recognized and cross the
|
||||
wire as receiver-side preferences. FastSync does not currently apply dir or
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
#include "client_validation.h"
|
||||
#include "chmod.h"
|
||||
#include "config.h"
|
||||
#include "delta.h"
|
||||
#include "file_list.h"
|
||||
#include "log.h"
|
||||
#include "test_utils.h"
|
||||
@@ -2908,6 +2909,62 @@ static void test_parse_args_password_file() {
|
||||
config_delete(cfg);
|
||||
}
|
||||
|
||||
/* --block-size (Delta block size): --block-size/--delta-block set
|
||||
* config->delta_block_size, out-of-range values are rejected with the default
|
||||
* kept, and the configured size genuinely reaches the delta engine (a larger
|
||||
* block yields fewer signature blocks for identical data). */
|
||||
static void test_parse_args_block_size() {
|
||||
Config* cfg = config_create();
|
||||
cfg->send_directory = str_dup("/src");
|
||||
cfg->receive_root_directory = str_dup("/dst");
|
||||
int positional_args[2];
|
||||
int positional_count = 0;
|
||||
|
||||
char* argv_long[] = {"fastsync", "--block-size", "4096", "/src", "/dst"};
|
||||
EXPECT_EQ_INT(parse_args(cfg, 5, argv_long, positional_args, &positional_count), 0);
|
||||
EXPECT_EQ_INT((int)cfg->delta_block_size, 4096);
|
||||
|
||||
cfg->delta_block_size = DELTA_BLOCK_SIZE_DEFAULT;
|
||||
char* argv_delta[] = {"fastsync", "--delta-block", "2048", "/src", "/dst"};
|
||||
positional_count = 0;
|
||||
EXPECT_EQ_INT(parse_args(cfg, 5, argv_delta, positional_args, &positional_count), 0);
|
||||
EXPECT_EQ_INT((int)cfg->delta_block_size, 2048);
|
||||
|
||||
/* Inline =SIZE forms (the documented rsync spelling) are accepted too. */
|
||||
cfg->delta_block_size = DELTA_BLOCK_SIZE_DEFAULT;
|
||||
char* argv_eq[] = {"fastsync", "--block-size=8192", "/src", "/dst"};
|
||||
positional_count = 0;
|
||||
EXPECT_EQ_INT(parse_args(cfg, 4, argv_eq, positional_args, &positional_count), 0);
|
||||
EXPECT_EQ_INT((int)cfg->delta_block_size, 8192);
|
||||
|
||||
/* Out of range: parsed, warned, and the default is kept. */
|
||||
cfg->delta_block_size = DELTA_BLOCK_SIZE_DEFAULT;
|
||||
char* argv_bad[] = {"fastsync", "--block-size", "1", "/src", "/dst"};
|
||||
positional_count = 0;
|
||||
EXPECT_EQ_INT(parse_args(cfg, 5, argv_bad, positional_args, &positional_count), 0);
|
||||
EXPECT_EQ_INT((int)cfg->delta_block_size, (int)DELTA_BLOCK_SIZE_DEFAULT);
|
||||
|
||||
/* A non-default block size changes the number of signature blocks for
|
||||
identical data: block_count = ceil(size / block_size). */
|
||||
const char data[10000] = {0};
|
||||
DeltaSignature* small = delta_signature_create_seeded(data, sizeof(data), 1024, 0);
|
||||
DeltaSignature* large = delta_signature_create_seeded(data, sizeof(data), 8192, 0);
|
||||
EXPECT_NOT_NULL(small);
|
||||
EXPECT_NOT_NULL(large);
|
||||
/* cppcheck-suppress knownConditionTrueFalse -- EXPECT_NOT_NULL above asserts,
|
||||
but cppcheck cannot see through the macro; the guard is defensive. */
|
||||
if (small && large) {
|
||||
EXPECT_TRUE(large->block_size == 8192 && small->block_size == 1024);
|
||||
EXPECT_TRUE(large->block_count < small->block_count);
|
||||
EXPECT_EQ_INT((int)small->block_count, 10); /* ceil(10000/1024) */
|
||||
EXPECT_EQ_INT((int)large->block_count, 2); /* ceil(10000/8192) */
|
||||
}
|
||||
delta_signature_destroy(small);
|
||||
delta_signature_destroy(large);
|
||||
|
||||
config_delete(cfg);
|
||||
}
|
||||
|
||||
void test_client_cli() {
|
||||
test_validate_config_required_paths();
|
||||
test_parse_args_numeric_ids();
|
||||
@@ -2918,6 +2975,7 @@ void test_client_cli() {
|
||||
test_parse_args_rejects_malformed_identity();
|
||||
test_parse_args_preallocate();
|
||||
test_parse_args_metadata_times();
|
||||
test_parse_args_block_size();
|
||||
test_parse_args_devices_specials();
|
||||
test_parse_args_atimes_long_and_short();
|
||||
test_parse_args_omit_link_times_long();
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
#ifndef _GNU_SOURCE
|
||||
#define _GNU_SOURCE /* SEEK_HOLE/SEEK_DATA for the sparse-hole sparseness check */
|
||||
#endif
|
||||
#include "test_file.h"
|
||||
#include "file.h"
|
||||
#include "file_store.h"
|
||||
#include "data.h"
|
||||
#include "config.h"
|
||||
#include "utils.h"
|
||||
@@ -1216,6 +1220,119 @@ void test_trust_sender() {
|
||||
file_set_authorized_root(-1, NULL);
|
||||
}
|
||||
|
||||
/* --sparse/-S hole preservation: a buffer with a long zero run written via
|
||||
* file_store_write_secure(sparse=true) must round-trip its content exactly and
|
||||
* have the right logical size, and should additionally be genuinely sparse on
|
||||
* filesystems that support holes. The sparseness assertion is tolerant: if the
|
||||
* filesystem reports no holes (SEEK_HOLE/SEEK_DATA -> ENXIO) we skip the strict
|
||||
* block-count check, but content and size always hold. */
|
||||
static void test_file_write_to_disk_sparse_preserves_holes() {
|
||||
const char* path = "test_sparse_file.bin";
|
||||
unlink(path);
|
||||
/* 256 KiB with a 128 KiB zero run in the middle, bracketed by headers/tails. */
|
||||
const unsigned long long size = 256u * 1024u;
|
||||
unsigned char* buf = malloc(size);
|
||||
EXPECT_NOT_NULL(buf);
|
||||
/* cppcheck-suppress knownConditionTrueFalse -- EXPECT_NOT_NULL above asserts,
|
||||
but cppcheck cannot see through the macro; the guard is defensive. */
|
||||
if (!buf)
|
||||
return;
|
||||
memset(buf, 0, size);
|
||||
for (unsigned long long i = 0; i < 4096; i++) {
|
||||
buf[i] = (unsigned char)(i % 251);
|
||||
buf[size - 1 - i] = (unsigned char)((i * 7) % 253);
|
||||
}
|
||||
|
||||
EXPECT_TRUE(file_store_write_secure(path, buf, size, false, true, NULL, false));
|
||||
|
||||
/* Logical size must equal data_size exactly. */
|
||||
struct stat st;
|
||||
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||
EXPECT_EQ_INT((int)st.st_size, (int)size);
|
||||
|
||||
/* Content must round-trip exactly: the full readback must equal the original
|
||||
buffer byte-for-byte (header, the hole region staying zero, and tail) —
|
||||
a writer bug in the lseek-offset bookkeeping would show up here. */
|
||||
int fd = open(path, O_RDONLY);
|
||||
EXPECT_TRUE(fd >= 0);
|
||||
/* cppcheck-suppress knownConditionTrueFalse -- EXPECT_TRUE above asserts,
|
||||
but cppcheck cannot see through the macro; the guard is defensive. */
|
||||
if (fd >= 0) {
|
||||
unsigned char* readback = malloc(size);
|
||||
if (readback) {
|
||||
unsigned long long got = 0;
|
||||
while (got < size) {
|
||||
ssize_t n = read(fd, readback + got, (size_t)(size - got));
|
||||
if (n <= 0)
|
||||
break;
|
||||
got += (unsigned long long)n;
|
||||
}
|
||||
EXPECT_EQ_INT((int)got, (int)size);
|
||||
if (got == size)
|
||||
EXPECT_EQ_INT(memcmp(readback, buf, size), 0);
|
||||
free(readback);
|
||||
}
|
||||
/* Tolerant sparseness check: seek for holes; skip if unsupported. */
|
||||
off_t hole_off = lseek(fd, (off_t)4096, SEEK_HOLE);
|
||||
if (hole_off >= 0 && hole_off < (off_t)size) {
|
||||
off_t next_data = lseek(fd, hole_off, SEEK_DATA);
|
||||
fstat(fd, &st);
|
||||
int blocks = (int)(st.st_blocks * 512);
|
||||
if (next_data > hole_off)
|
||||
EXPECT_TRUE(blocks < (int)size);
|
||||
}
|
||||
close(fd);
|
||||
}
|
||||
free(buf);
|
||||
unlink(path);
|
||||
}
|
||||
|
||||
/* --partial retention is hard to provoke end-to-end mid-transfer (the whole
|
||||
* image is in one in-memory write), so this drives the failure path directly:
|
||||
* a metadata whose mtime_nsec is out of the legal [0,999999999] range makes
|
||||
* futimens (in file_restore_metadata_fd) fail with EINVAL AFTER the temp has
|
||||
* been fully written. With keep_partial=true the written temp must be renamed
|
||||
* to the destination path (a resumable partial); with keep_partial=false the
|
||||
* same failure must leave NOTHING behind. The retention is always best-effort
|
||||
* (never a corrupt blend), and this asserts the both-on/off behavior. */
|
||||
static void test_file_write_to_disk_partial_retention() {
|
||||
const char* path = "test_partial_retention.bin";
|
||||
unlink(path);
|
||||
const char content[] = "partial-retention payload";
|
||||
FileMetadata m;
|
||||
memset(&m, 0, sizeof(m));
|
||||
m.mode = 0644;
|
||||
m.uid = (uid_t)geteuid();
|
||||
m.gid = (gid_t)getegid();
|
||||
m.mtime_sec = 1700000000;
|
||||
m.mtime_nsec = 2000000000; /* invalid: forces futimens EINVAL after the write */
|
||||
m.atime_valid = false;
|
||||
m.crtime_valid = false;
|
||||
bool ok = file_to_disk_secure_attrs(path, content, strlen(content), false, false, true, &m, false,
|
||||
false, false, false, NULL, false, true, NULL);
|
||||
EXPECT_FALSE(ok); /* the write itself succeeded, but metadata restore failed */
|
||||
/* Retained: the already-written temp now sits at the destination path. */
|
||||
int fd = open(path, O_RDONLY);
|
||||
EXPECT_TRUE(fd >= 0);
|
||||
/* cppcheck-suppress knownConditionTrueFalse -- EXPECT_TRUE above asserts,
|
||||
but cppcheck cannot see through the macro; the guard is defensive. */
|
||||
if (fd >= 0) {
|
||||
char buf[64];
|
||||
ssize_t n = read(fd, buf, sizeof(buf));
|
||||
close(fd);
|
||||
EXPECT_EQ_INT((int)strlen(content), (int)n);
|
||||
if (n == (ssize_t)strlen(content))
|
||||
EXPECT_TRUE(memcmp(buf, content, strlen(content)) == 0);
|
||||
}
|
||||
unlink(path);
|
||||
|
||||
/* Same failure with keep_partial=false: temp is unlinked, nothing retained. */
|
||||
ok = file_to_disk_secure_attrs(path, content, strlen(content), false, false, true, &m, false,
|
||||
false, false, false, NULL, false, false, NULL);
|
||||
EXPECT_FALSE(ok);
|
||||
EXPECT_TRUE(access(path, F_OK) == -1);
|
||||
}
|
||||
|
||||
void test_file() {
|
||||
test_file_create();
|
||||
test_file_special_rdev_valid();
|
||||
@@ -1230,6 +1347,8 @@ void test_file() {
|
||||
test_file_save_to_disk_ignore_existing_entry_types();
|
||||
test_file_save_to_disk_partial_install();
|
||||
test_file_save_to_disk_reports_skips();
|
||||
test_file_write_to_disk_sparse_preserves_holes();
|
||||
test_file_write_to_disk_partial_retention();
|
||||
test_file_write_to_disk_basic();
|
||||
test_file_write_to_disk_with_fsync();
|
||||
test_file_write_to_disk_preallocate_atomic();
|
||||
|
||||
@@ -222,6 +222,57 @@ static void test_link_copy_fallback_preserves_xattrs() {
|
||||
rmdir(basis_dir);
|
||||
}
|
||||
|
||||
/* --fake-super replay: fake_super_store_fd records the source stat into the
|
||||
* reserved xattr, and fake_super_restore_fd re-applies mode/mtime (and owner,
|
||||
* when the process may) fd-relative. Restore must also be a safe no-op with no
|
||||
* xattr present. Guarded on filesystem xattr support. */
|
||||
static void test_fake_super_restore() {
|
||||
const char* path = "test_fake_super_restore.txt";
|
||||
unlink(path);
|
||||
int fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0600);
|
||||
if (fd < 0)
|
||||
return;
|
||||
bool has_xattr = setxattr(path, "user.fastsync.xprobe", "p", 1, 0) == 0;
|
||||
if (has_xattr)
|
||||
removexattr(path, "user.fastsync.xprobe");
|
||||
if (!has_xattr) {
|
||||
close(fd);
|
||||
unlink(path);
|
||||
return; /* skip silently when the filesystem has no xattr support */
|
||||
}
|
||||
|
||||
/* No xattr present yet: restore is a silent no-op (returns false, no crash). */
|
||||
EXPECT_FALSE(fake_super_restore_fd(fd));
|
||||
|
||||
fake_super_store_fd(fd, 1001, 1002, 0751, 1700000000, 123456789);
|
||||
EXPECT_TRUE(fake_super_restore_fd(fd));
|
||||
struct stat st;
|
||||
EXPECT_EQ_INT(fstat(fd, &st), 0);
|
||||
EXPECT_EQ_INT((int)(st.st_mode & 07777), 0751);
|
||||
|
||||
/* Mode sanitization: the normal metadata path never grants group/other write
|
||||
bits, and fake-super replay must not re-add them (a recorded 0666 restores
|
||||
as 0644, never as world-writable). */
|
||||
fake_super_store_fd(fd, 1001, 1002, 0666, 1700000000, 0);
|
||||
EXPECT_TRUE(fake_super_restore_fd(fd));
|
||||
EXPECT_EQ_INT(fstat(fd, &st), 0);
|
||||
EXPECT_EQ_INT((int)(st.st_mode & 0777), 0644);
|
||||
|
||||
/* Restore with a malformed record must skip without failing. */
|
||||
time_t before = st.st_mtime;
|
||||
int wfd = open(path, O_RDONLY);
|
||||
if (wfd >= 0) {
|
||||
EXPECT_EQ_INT((int)fsetxattr(wfd, FAKESUPER_XATTR, "not-a-valid-record", 19, 0), 0);
|
||||
close(wfd);
|
||||
}
|
||||
EXPECT_FALSE(fake_super_restore_fd(fd));
|
||||
fstat(fd, &st);
|
||||
EXPECT_EQ_INT((int)st.st_mtime, (int)before);
|
||||
|
||||
close(fd);
|
||||
unlink(path);
|
||||
}
|
||||
|
||||
void test_xattr() {
|
||||
test_xattr_wire_roundtrip();
|
||||
test_xattr_reject_privileged_namespace();
|
||||
@@ -229,4 +280,5 @@ void test_xattr() {
|
||||
test_xattr_count_bound();
|
||||
test_xattr_capture_and_appliable();
|
||||
test_link_copy_fallback_preserves_xattrs();
|
||||
test_fake_super_restore();
|
||||
}
|
||||
Reference in New Issue
Block a user