Release v2.28.0 #304
@@ -65,6 +65,14 @@ python3 -m pytest tests/integration/test_differential_parity.py -n 4 --dist=load
|
||||
See `tests/integration/README.md` for the differential parity gate and its
|
||||
`parity_caveats.py` allowlist (the residual burn-down mechanism).
|
||||
|
||||
Unit tests under valgrind must set `FASTSYNC_UNDER_VALGRIND=1` (CI does): the
|
||||
tests use it to skip fork-based tests, because valgrind 3.22 does not expose
|
||||
`vgpreload` in the guest's `/proc/self/maps`.
|
||||
|
||||
```bash
|
||||
FASTSYNC_UNDER_VALGRIND=1 valgrind --leak-check=full --show-leak-kinds=definite --error-exitcode=1 ./build/tests
|
||||
```
|
||||
|
||||
## CI Workflow — Waiting for Results
|
||||
|
||||
When running the CI workflow via `tea` (the task execution agent), always set a sufficient timeout (e.g., 600000ms) to allow CI to finish. After CI completes, check the results yourself — do not assume success. Monitor CI status via the Gitea API (see below) or `tea actions`, then inspect logs on failure.
|
||||
|
||||
+2
-1
@@ -1,6 +1,6 @@
|
||||
cmake_minimum_required(VERSION 3.22)
|
||||
|
||||
project(FastFileTransfer VERSION 2.26.0)
|
||||
project(FastFileTransfer VERSION 2.27.0)
|
||||
|
||||
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
|
||||
set(CMAKE_C_STANDARD 11)
|
||||
@@ -221,6 +221,7 @@ set(TEST_SRCS
|
||||
tests/test_daemon_limits.c
|
||||
tests/test_data.c
|
||||
tests/test_delay_updates.c
|
||||
tests/test_delete_plan.c
|
||||
tests/test_delta.c
|
||||
tests/test_file.c
|
||||
tests/test_file_list.c
|
||||
|
||||
+43
-3
@@ -8,8 +8,8 @@
|
||||
- **Release PR #284 (`dev` -> `main`)** open, CI green (run 553).
|
||||
`main` is protected: it needs review/approval to merge.
|
||||
https://gitea.tap-tap.win/TapTap/FastSync/pulls/284
|
||||
- **`PROTOCOL_VERSION` = `"2.26.0"`** (`src/shared/config.h`); CMake
|
||||
`project(FastFileTransfer VERSION 2.26.0)`.
|
||||
- **`PROTOCOL_VERSION` = `"2.27.0"`** (`src/shared/config.h`); CMake
|
||||
`project(FastFileTransfer VERSION 2.27.0)`.
|
||||
- Working tree clean; no wave worktrees remain.
|
||||
|
||||
## What landed this session
|
||||
@@ -39,7 +39,47 @@
|
||||
docs state push-only / remote-source unsupported.
|
||||
5. **Preserve-attribute split (protocol 2.22.0)** landed on `feat/preserve-attr-split`: per-attribute `-p/-t/-o/-g` + `--no-*` negations, `-a` = `-rlptgoD`, and the 2.21.0 → 2.22.0 wire bump.
|
||||
6. **Rsync-parity wave (protocol 2.23.0)** on `feat/rsync-parity`: rsync short options/clustering/attached values (`-r`/`-b`/`-L`/`-B`, `-av`, `-aAX`, `-B1000`, `-essh`, `-MOPT`), `-c` checksum quick-check, `--checksum-choice`/`--compress-choice` validation and seed randomization, rsync timeout/max-alloc defaults, temp-dir confinement + `EXDEV` fallback, ownership/mapping parity (numeric-ids modifier, map ranges/`*`/empty-FROM, `--chown`+map conflicts, fake-super resolved-owner record), verbatim symlink storage with rsync `--safe-links`/`--munge-links`, socket recreation under `--specials`, `--chmod` 3.4.1 semantics, and delete scoping + `--max-delete` partial/exit-25. Wire: appended delete-manifest synchronized-directory section and `STATUS_DELETE_LIMIT`.
|
||||
7. **Parity-completion wave (protocol 2.24.0 → 2.26.0)** on `feat/parity-completion`: per-directory delete plans (`STATUS_DELETE_PLAN`) for `--delete-during`/`--delete-delay`; receiver `STATUS_STATS` counters feeding `--stats`/`--progress` and `--out-format %b/%c/%C`, plus `-n --delete` lines; `lz4`/`zlib`/`zlibx` compression and `md4`/`sha1`/`none` checksums with `auto` negotiation (default `xxh128`/`zstd`); general `-R`/`--no-implied-dirs`/`-d`; the full filter grammar (`merge`/`dir-merge`/`hide`/`show`/`protect`/`risk`/`clear` + modifiers) and corrected `-F`/`-FF`; receiver-side `--chown`/map TO-name resolution; absolute basis dirs + `--link-dest` relink; receiver-side `--ignore-existing` short-circuit; `--preallocate` over `--sparse` via `fallocate(2)`; `--iconv=.`/`-`/`--no-iconv`; lone `-h` help; aliases `--ignore-non-existing`/`--protect-args`/`--msgs2stderr`; and the full `--info`/`--debug` vocabulary. `RSYNC_COMPAT.md` reclassifies the matrix to 106 ✅ / 27 ⚠️ / 23 ❌.
|
||||
7. **Parity-completion wave (protocol 2.24.0 → 2.26.0)** on `feat/parity-completion`: per-directory delete plans (`STATUS_DELETE_PLAN`) for `--delete-during`/`--delete-delay`; receiver `STATUS_STATS` counters feeding `--stats`/`--progress` and `--out-format %b/%c/%C`, plus `-n --delete` lines; `lz4`/`zlib`/`zlibx` compression and `md4`/`sha1`/`none` checksums with `auto` negotiation (default `xxh128`/`zstd`); general `-R`/`--no-implied-dirs`/`-d`; the full filter grammar (`merge`/`dir-merge`/`hide`/`show`/`protect`/`risk`/`clear` + modifiers) and corrected `-F`/`-FF`; receiver-side `--chown`/map TO-name resolution; absolute basis dirs + `--link-dest` relink; receiver-side `--ignore-existing` short-circuit; `--preallocate` over `--sparse` via `fallocate(2)`; `--iconv=.`/`-`/`--no-iconv`; lone `-h` help; aliases `--ignore-non-existing`/`--protect-args`/`--msgs2stderr`; and the full `--info`/`--debug` vocabulary. `RSYNC_COMPAT.md` reclassifies the matrix to 106 ✅ / 27 ⚠️ / 23 ❌; the later rsync-parity-stats pass (`fix/parity-stats`) moves it to 107 ✅ / 25 ⚠️ / 24 ❌ (see item 8).
|
||||
8. **rsync-parity-stats pass** on `fix/parity-stats` (no wire change, `PROTOCOL_VERSION` stays `2.26.0`): `--delete-delay` now reports only entries it actually removes, while the `--max-delete` budget is charged at plan/snapshot time (`planned`, via `defer_add`) to bound the deferred list (a refilled deferred directory that survives `ENOTEMPTY` is not reported but still consumes budget); `--stats` gained the `(reg/dir/link/special)` `Number of files` breakdown and now counts only regular files actually stored for `Number of regular files transferred`/transferred size/literal data (up-to-date re-runs report 0); `Total file size` includes symlink target lengths; `--progress` prints the leading `./` root line and counts it in `to-chk` so a single-file transfer matches rsync; and `%C` uses the selected transfer checksum with `checksum_digest_file` supporting md4/sha1/none, byte-identical to rsync for every algorithm. `--out-format` reclassified ❌ (`%b`/delta-`%c` are protocol-specific). Differential + regression tests added; full suite + ASan + clang-format + cppcheck clean.
|
||||
9. **Option-parity wave (protocol 2.26.0 → 2.27.0, on `fix/parity-options`):**
|
||||
`--bwlimit` now ports rsync 3.4.1's units/quantization and paces like its
|
||||
leaky bucket; `--ignore-errors` reproduces rsync's default (an I/O error
|
||||
skips deletion unless the flag is set; the readable tree still transfers and
|
||||
the run exits 23) across every delete timing; the `--info` categories with a
|
||||
FastSync event (`name`/`flist`/`del`/`remove`/`nonreg`/`progress`) emit
|
||||
rsync's line format, with real-run `deleting`/`*deleting` lines carried over
|
||||
the new trailing config bool `report_deletes` (golden wire updated by
|
||||
`tests/test_config.c`). Two residuals were reclassified **divergent**: `-M`
|
||||
over daemon/TCP (no argv channel in FastSync's binary config handshake;
|
||||
rsync-daemon differential pins the rsync behavior) and receiver-side
|
||||
`protect`/`risk` re-derivation for destination-only entries (would need a
|
||||
receiver filter engine; differential pins the divergence). The options pass
|
||||
stands at **110 ✅ / 21 ⚠️ / 26 ❌**. New `tests/integration/test_option_parity.py`
|
||||
holds the rsync differentials (bwlimit parse+rate, info lines, real-setpriv
|
||||
`--ignore-errors`, rsync-daemon `-M`, filter-protect pin).
|
||||
|
||||
10. **rsync-parity-fs pass** on `fix/parity-fs` (no wire change of its own; integrated
|
||||
on top of the 2.27.0 options wave): recursive transfers now recreate empty source directories (and
|
||||
`-m/--prune-empty-dirs` still suppresses them), a directory entry replaces a
|
||||
blocking destination regular file, and `-R --no-implied-dirs --files-from`
|
||||
places a listed file under a missing implied parent with default attributes
|
||||
instead of refusing (real rsync 3.4.1 parity, differential-tested). `--iconv`
|
||||
now reproduces rsync's push direction (destination charset = the spec's REMOTE
|
||||
half; a server `--iconv` overrides), and `-T/--temp-dir` relative semantics are
|
||||
confirmed identical while the absolute-path confinement is a deliberate
|
||||
divergence. The basis-dir options, `--delay-updates` and `--dry-run` were
|
||||
reclassified to ❌ after a differential test reproduced each exact residual
|
||||
(basis content verification, fixed staging-name collision, and dry-run
|
||||
would-delete over-report). `--fuzzy` was also reclassified to ❌ (deterministic
|
||||
heuristic with a 10× size window, not rsync's matcher), but its residual is the
|
||||
candidate-selection heuristic itself: the final tree is byte-exact by design, so
|
||||
it is pinned by the `TestFuzzy` threshold suite rather than a byte-level rsync
|
||||
differential. The parity-review pass then moved `--delete-delay` to ⚠️ (the
|
||||
plan-time `--max-delete` charge and non-recursive deferred removal differ from
|
||||
rsync when a snapshotted entry fails removal). Differential-gate allowlist
|
||||
entries `min_size`/`empty_dirs_recursive`/`dirs_plain` were removed. The
|
||||
integrated stats+options+fs branch stands at **111 ✅ / 13 ⚠️ / 33 ❌ = 157**;
|
||||
full suite + ASan + clang-format + cppcheck clean.
|
||||
|
||||
## Next steps
|
||||
1. **Merge PR #284** (`dev` -> `main`) once reviewed (protected branch).
|
||||
|
||||
@@ -789,7 +789,7 @@ before the module list, before authentication, and the connecting peer address
|
||||
|
||||
## Protocol and Security
|
||||
|
||||
FastSync protocol version `2.26.0` is shared by the client and server. The
|
||||
FastSync protocol version `2.27.0` is shared by the client and server. The
|
||||
current protocol is sender-driven and includes configuration negotiation,
|
||||
including the maximum allocation limit, incremental checks, checksums,
|
||||
manifests, keep-alives, abort handling, per-file remove-source results, and
|
||||
|
||||
+60
-39
@@ -6,10 +6,10 @@ This document maps rsync's full feature set to FastSync's current implementation
|
||||
|
||||
| Status | Count | Description |
|
||||
|--------|-------|-------------|
|
||||
| ✅ Parity | 106 | Reproduces rsync's semantics for this option's scope |
|
||||
| ⚠️ Caveat | 27 | Wired and tested, but carries a documented behavioral difference from rsync (named in the row and/or the wave notes) |
|
||||
| ❌ Divergent | 23 | Rejected, an accepted no-op, deliberately non-rsync (native config/auth/batch, privileged namespaces, safe-subset privilege), or impossible on any portable filesystem call |
|
||||
| **Total** | **156** | One row per rsync option/feature group; a row may name several spellings |
|
||||
| ✅ Parity | 111 | Reproduces rsync's semantics for this option's scope |
|
||||
| ⚠️ Caveat | 13 | Wired and tested, but carries a documented behavioral difference from rsync (named in the row and/or the wave notes) |
|
||||
| ❌ Divergent | 33 | Rejected, an accepted no-op, deliberately non-rsync (native config/auth/batch, privileged namespaces, safe-subset privilege), or impossible on any portable filesystem call |
|
||||
| **Total** | **157** | One row per rsync option/feature group; a row may name several spellings |
|
||||
|
||||
This matrix reports honest rsync parity, not "implemented" as a synonym for
|
||||
"parsed". A ✅ row matches rsync for the option's scope. A ⚠️ row is real and
|
||||
@@ -21,6 +21,22 @@ the batch container, `--fake-super`'s xattr format, `--copy-as` credential
|
||||
switching), or impossible (`-N`/`--crtimes`). The counts are derived from the
|
||||
rows below; update them together with the table.
|
||||
|
||||
**Option wave (protocol 2.26.0 → 2.27.0).** A differential pass against rsync
|
||||
3.4.1 over the remaining option caveats. `--bwlimit` now parses rsync's units
|
||||
exactly and paces like rsync's leaky bucket; `--ignore-errors` reproduces
|
||||
rsync's default (an I/O error skips deletion unless the flag is set, while the
|
||||
readable tree still transfers and the run exits 23); the `--info` categories
|
||||
that map to a FastSync event (`name`, `flist`, `del`, `remove`, `nonreg`,
|
||||
`progress`) now emit rsync's line format, including real-run `deleting PATH` /
|
||||
`*deleting` lines carried over a new `report_deletes` wire bool; and two
|
||||
genuinely non-interoperable residuals are reclassified divergent (`-M` over a
|
||||
daemon/TCP connection, which FastSync's binary config handshake has no argv
|
||||
channel for, and receiver-side `protect`/`risk` re-derivation for
|
||||
destination-only entries, which would need a receiver filter engine). After the
|
||||
combined `fix/parity-stats` + `fix/parity-options` + `fix/parity-fs` passes (and
|
||||
the later `fix/parity-review` correction that moved `--delete-delay` to ⚠️) the
|
||||
matrix is **111 ✅ / 13 ⚠️ / 33 ❌ = 157**.
|
||||
|
||||
**Parity completion wave (protocol 2.23.0 → 2.26.0).** This wave closed the
|
||||
remaining gaps the rsync-parity wave left open (delete timing, wire counters and
|
||||
output, codec breadth, general `-R`/`-d`, the full filter grammar, receiver-side
|
||||
@@ -51,7 +67,7 @@ Every one of those has an entry below with its remaining caveats.
|
||||
| `-q`, `--quiet` | Suppress non-error messages | ✅ Parity | Suppresses client output while preserving errors |
|
||||
| `--help` | Show help | ✅ Parity | Prints usage and exits. A lone `-h` with no other transfer arguments also prints help (protocol 2.26.0), matching the rsync idiom; `-h` alongside a transfer keeps its rsync meaning of `--human-readable` (see that row) |
|
||||
| `-V`, `--version` | Print version | ✅ Parity | |
|
||||
| `--info=FLAGS` | Fine-grained info verbosity | ⚠️ Caveat | Protocol 2.26.0 accepts rsync 3.4.1's full `--info` vocabulary — `backup`, `copy`, `del`, `flist`, `misc`, `mount`, `name`, `nonreg`, `progress`, `remove`, `skip`, `stats`, `symsafe`, `all`, `none` — with optional level suffixes (`--info=stats2`), so a valid rsync invocation is never rejected up front. The categories that map to a FastSync channel emit (`copy`, `name`, `misc`, `skip`, `stats`); the remaining rsync categories are accepted silently, with no output. `none` suppresses info output, explicit flags override `--verbose`, and a genuinely unknown name is still rejected by name (matching rsync). **Caveat:** many accepted rsync categories produce no output (e.g. `del`, `flist`, `remove`, `progress`, `symsafe`, `mount`, `nonreg`, `backup`), so e.g. `--info=progress` is accepted for CLI compatibility only; `name` maps to the `copy` channel rather than rsync's per-file name output |
|
||||
| `--info=FLAGS` | Fine-grained info verbosity | ⚠️ Caveat | Accepts rsync 3.4.1's full `--info` vocabulary — `backup`, `copy`, `del`, `flist`, `misc`, `mount`, `name`, `nonreg`, `progress`, `remove`, `skip`, `stats`, `symsafe`, `all`, `none` — with optional level suffixes (`--info=stats2`), so a valid rsync invocation is never rejected up front. Protocol 2.27.0 wires the categories that map to a real FastSync event, matching rsync's line format: `name` prints the updated entry names (with the ` -> target` link suffix), `flist` prints `sending incremental file list`, `del` prints `deleting PATH` (or `*deleting PATH` under `-i`/`--out-format`) for both dry-run would-delete and real deletions (real runs carry the removed paths over the new `report_deletes` wire bool), `remove` prints `sender removed PATH`, `nonreg` prints `skipping non-regular file "NAME"`, `progress` drives the per-file progress output, and `copy`/`misc`/`skip`/`stats` keep their existing channels. `none` suppresses info output, explicit flags override `--verbose`, and a genuinely unknown name is still rejected by name (matching rsync). **Caveat:** the categories with no client-observable event stay accepted-but-silent — `symsafe`, `mount`, and `backup` (the backup happens on the receiver, which FastSync's protocol does not echo back); `name` level 2 (`is uptodate` lines) and the leading `./` root name line are not emitted; and `skip` maps to FastSync's sender-side skip logging rather than rsync's receiver-side "not creating new file" lines |
|
||||
| `--debug=FLAGS` | Fine-grained debug verbosity | ⚠️ Caveat | Protocol 2.26.0 accepts rsync 3.4.1's full `--debug` vocabulary with optional level suffixes. FastSync emits for its own channels (`io`, `proto`, `pack`, `util`, plus the aliases `hl`/`owner`); the rsync-only categories (`acl`, `filter`, `send`, ...) are accepted silently. `--debug=help` lists the flags; a genuinely unknown name is rejected by name. **Caveat:** most accepted rsync categories produce no output (e.g. `acl`, `filter`, `send`, `flist`, `del`, `deltasum`, `hash`, `recv`, `time`), so they are accepted for CLI compatibility only |
|
||||
| `--stderr=MODE` | Change stderr output mode | ❌ Divergent | `errors` (default) and `all` are supported; `client` is rejected with a clear error (`--stderr=client is not supported`) because FastSync has no rsync client-message channel — the rejection itself is the documented behavior (Phase 7 Wave B decision). The modes that exist work; the missing rsync channel cannot be emulated without a wire change |
|
||||
| `--msgs2stderr`, `--no-msgs2stderr` | Deprecated `--stderr` aliases | ⚠️ Caveat | `--msgs2stderr` maps to `--stderr=all` (supported, matching rsync). `--no-msgs2stderr` is rsync's spelling of `--stderr=client`, which FastSync has no client-message channel for, so it maps to the errors-only default instead of reproducing rsync's client mode. See `--stderr=MODE` |
|
||||
@@ -64,12 +80,12 @@ Every one of those has an entry below with its remaining caveats.
|
||||
|
||||
| Flag | Rsync Description | FastSync Status | Notes |
|
||||
|------|-------------------|-----------------|-------|
|
||||
| `--stats` | Give transfer stats | ⚠️ Caveat | Prints transfer statistics. Protocol 2.25.0 populates the receiver-only counters the sender cannot observe: `Matched data` (a delta basis's reused bytes) and `Number of deleted files` come from the receiver's `STATUS_STATS` report, and the protocol-independent lines (regular files transferred, total/transferred file size, literal data, matched data, deleted files, file-list size) match rsync exactly in both the sequential and `--threads` paths. **Remaining divergence:** rsync prints `Number of files` and `Number of created files` with a per-type breakdown (`(reg: X, dir: Y, link: Z)`); FastSync prints the bare transferred-entry count because its scanner does not put directory entries in the transfer list and the sender cannot tell which entries the receiver newly created. `Total bytes sent`/`received` are FastSync wire bytes and are not numerically comparable to rsync's |
|
||||
| `--stats` | Give transfer stats | ⚠️ Caveat | Prints transfer statistics. Protocol 2.25.0 populates the receiver-only counters the sender cannot observe: `Matched data` (a delta basis's reused bytes) and `Number of deleted files` come from the receiver's `STATUS_STATS` report. The sender now tracks the scanned file list per type and only counts regular files the receiver actually stored, so `Number of files` carries rsync's `(reg: X, dir: Y, link: Z, special: W)` breakdown (directories come from the scanner's captured directory list, present for `-a`/`-t`/`-p`), `Number of regular files transferred` excludes symlinks/specials and up-to-date files, `Total file size` includes symlink target lengths, and `Total transferred file size`/`Literal data` count only transferred files — all differential-tested in the sequential and `--threads` paths. **Remaining divergences:** `Number of created files` is the transferred-regular count (FastSync cannot tell which entries the receiver newly created, so on an update where rsync reports 0 created FastSync can report the transferred file) and lacks the type breakdown; a recursive scan that preserves no directory attribute (`-r` without `-t`/`-p`) captures no directory entries, so the `dir:` category is then omitted; `Literal data` is exact for a whole-file transfer but an upper bound for a delta transfer (the sender counts each stored file's whole source size rather than only the literal fragments rsync ships, since it does not measure the delta payload it sends); rsync's per-type breakdown on `Number of deleted files` is not reproduced; and `Total bytes sent`/`received` are FastSync wire bytes framed differently from rsync's, so they are not numerically comparable |
|
||||
| `-h`, `--human-readable` | Human-readable numbers | ✅ Parity | Formats transfer byte and rate counts using rsync's **decimal** (base-1000) units, matching rsync `-h` (e.g. `1.23M`), not binary units. **A lone `-h` with no transfer arguments prints help instead** (protocol 2.26.0), matching the rsync idiom; `-h` alongside a transfer remains human-readable |
|
||||
| `-i`, `--itemize-changes` | Per-file change summary | ✅ Parity | Prints rsync-style `>f+++++++++` lines to stdout only for files actually sent (also under `-j`/`--threads`); unchanged files print nothing, matching single-`-i` behavior |
|
||||
| `--progress` | Show progress | ⚠️ Caveat | Protocol 2.25.0 prints rsync-style per-file progress blocks (percent, transferred/total bytes, rate, elapsed, `(xfr#N, to-chk=M/T)`) fed by the receiver's `STATUS_STATS`, in both the sequential and `--threads` send paths; the first frame for a sub-32 KiB file is byte-identical to rsync. **Remaining divergences:** FastSync does not print rsync's leading `./` whole-transfer line, its `to-chk` total differs by the source-root entry (the scanner does not emit the root directory as a transfer entry), and the rate/ETA are wall-clock dependent, so only the first frame is pinned against rsync |
|
||||
| `--progress` | Show progress | ⚠️ Caveat | Protocol 2.25.0 prints rsync-style per-file progress blocks (percent, transferred/total bytes, rate, elapsed, `(xfr#N, to-chk=M/T)`) fed by the receiver's `STATUS_STATS`, in both the sequential and `--threads` send paths. FastSync now also prints rsync's leading `./` transfer-root line and counts that root entry in the `to-chk` denominator, so a **single-file transfer's name lines and deterministic frames are byte-identical to rsync** (differential test). **Remaining divergences:** for a multi-directory tree rsync prints a per-directory name line and its `to-chk` denominator includes every directory/symlink/special entry; FastSync's streaming scan emits only file-name lines and counts just the root plus transferred files (a full flist pre-count would be needed), and the rate/ETA are wall-clock dependent |
|
||||
| `-P` | Same as --partial --progress | ✅ Parity | Parses to `--partial` + `--progress`. The independent `--partial` retention semantics are rsync parity: an interrupted write retains the already-written temp at the destination (best-effort) so a later `--append`/`--append-verify` can resume. Progress presentation is owned by the `--progress` row; there is no separate `-P` divergence |
|
||||
| `--out-format=FORMAT` | Custom output format | ⚠️ Caveat | Per-transfer template on stdout; tokens `%f` `%n` `%l` `%b` `%c` `%C` `%i` `%M` `%o` `%U` `%G` `%t` `%%`. Protocol 2.25.0 adds the wire counters: `%C` is the whole-file digest (default `xxh128`, seed 0), so `%C %l %n` matches rsync byte-for-byte for a whole-file transfer. **Remaining divergences:** `%b` counts FastSync's own wire bytes (framing and checksum trailer), not rsync's protocol-specific count, so the two are not numerically equal; `%c` matches rsync's 16-byte block-sum header for whole-file transfers but differs in delta mode (each counts its own handshake bytes). **Also:** when `--checksum-choice=xxh64` is selected explicitly, `%C` still prints an xxh128 digest rather than the selected xxh64 (`change_list.c:208-220`) |
|
||||
| `--out-format=FORMAT` | Custom output format | ❌ Divergent | Per-transfer template on stdout; tokens `%f` `%n` `%l` `%b` `%c` `%C` `%i` `%M` `%o` `%U` `%G` `%t` `%%`. `%C` now uses the negotiated transfer algorithm (`--checksum-choice`, default `xxh128`, seed 0) and renders every algorithm exactly like rsync — xxh128 high-then-low, xxh64/xxh3 big-endian, md5/md4/sha1 standard hex, `none` a blank 2-char column — differential-tested across all algorithms. `%f`/`%n`/`%l`/`%i`/`%M`/`%U`/`%G`/`%B` also match. **Reclassified because `%b`/`%c` are protocol-specific and cannot match:** a differential against rsync 3.4.1 shows whole-file `%c = 16` for both, but rsync whole-file `%b = filesize + 27 + transfer-digest-bytes` (39 for a 0-byte file; 43/35/47 for xxh128/xxh64/sha1 on a 12-byte file) while FastSync `%b` counts its own framing; in delta mode rsync `%c = 16 + 6·ceil(filesize/block_size)` (verified at block sizes 512/700/1024/2048) while FastSync counts its own signature handshake, and rsync `%b` is its token stream. FastSync's wire bytes are a different quantity, so exact `%b`/delta-`%c` equality is impossible |
|
||||
| `--log-file=FILE` | Log to file | ✅ Parity | `log_file` config field |
|
||||
| `--log-file-format=FMT` | Log format | ✅ Parity | Requires `--log-file`; writes one template line per transferred file using the same token set as `--out-format` (including `%b` as the wire byte count) |
|
||||
| `--8-bit-output`, `-8` | Leave high-bit chars unescaped | ✅ Parity | Applies to displayed paths and protocol debug output |
|
||||
@@ -81,7 +97,7 @@ Every one of those has an entry below with its remaining caveats.
|
||||
|------|-------------------|-----------------|-------|
|
||||
| `--exclude-from=FILE` | Read exclude patterns from file | ✅ Parity | Reads patterns from file |
|
||||
| `--include-from=FILE` | Read include patterns from file | ✅ Parity | Reads patterns from file |
|
||||
| `--filter=RULE` | Add file-filtering rule | ⚠️ Caveat | The short `-f` **is** bound to `--filter` (the old FastSync sendfile conflict is gone; sendfile is long-only `--sendfile`), and `-f RULE`, `-f=RULE`, `--filter=RULE` and the two-argument form all parse. Protocol 2.26.0 implements rsync's filter grammar: `+`/`-`, `include`/`exclude`, a leading `/` anchor (to the transfer root or a `.rsync-filter` file's directory), a trailing `/` dir-only rule, and the `merge`/`.`, `dir-merge`/`:`, `hide`/`H`, `show`/`S`, `protect`/`P`, `risk`/`R` and `clear`/`!` words, including the `:`/`.` modifiers. First match wins; the filter layer is independent of `--exclude`/`--include`. **Remaining divergence:** the receiver-mirror protection a `protect`/`risk` rule produces is derived from the sender's source traversal, so a rule that would match only a destination-only entry is not re-derived on the receiver; destination-only deletion protection continues to come from the ordinary sender-derived protected-prefix mechanism |
|
||||
| `--filter=RULE` | Add file-filtering rule | ❌ Divergent | The short `-f` **is** bound to `--filter` (the old FastSync sendfile conflict is gone; sendfile is long-only `--sendfile`), and `-f RULE`, `-f=RULE`, `--filter=RULE` and the two-argument form all parse. Protocol 2.26.0 implements rsync's filter grammar: `+`/`-`, `include`/`exclude`, a leading `/` anchor (to the transfer root or a `.rsync-filter` file's directory), a trailing `/` dir-only rule, and the `merge`/`.`, `dir-merge`/`:`, `hide`/`H`, `show`/`S`, `protect`/`P`, `risk`/`R` and `clear`/`!` words, including the `:`/`.` modifiers. First match wins; the filter layer is independent of `--exclude`/`--include`. **Reclassified because the receiver-side `protect`/`risk` semantics cannot be reproduced:** rsync maintains an independent filter engine on the receiver and re-applies every rule to the destination during deletion, so a `P *.log` rule protects a destination-only `extra.log` (differential test vs rsync 3.4.1 pins this). FastSync is sender-derived: its delete protection is the set of source paths the scan actually pruned, so a rule that matches only a destination-only entry is never re-derived and the extra is deleted. Closing this would require shipping the whole (including per-directory merge) filter grammar to, and re-implementing rsync's dual-sided engine on, the receiver — a protocol/architecture change out of proportion to the residual |
|
||||
| `--files-from=FILE` | Read source file list from file | ✅ Parity | Entries are paths relative to the source root (leading `./` stripped, `..`/absolute rejected at parse time, blank lines ignored; NUL-delimited with `-0`). A listed regular file is transferred; a listed directory transfers its whole subtree (FastSync recursion is always on). Non-listed paths are pruned by the scanner; the delete manifest is scoped to the listed directory subtrees. A listed entry that does not exist is a hard error unless `--ignore-missing-args`/`--delete-missing-args` is given. **An empty list is a zero-transfer success (exit 0), matching rsync 3.4.1** — the earlier claim that rsync reports "no source files specified" was wrong. Scalability note: `file_list_affects` is O(list size) per scanned entry, so a very large list against a huge tree is quadratic (the documented bound) |
|
||||
| `-0`, `--from0` | Delimit *-from files with NULs | ✅ Parity | `--files-from` entries become NUL-delimited; the flag may appear before or after `--files-from` on the command line. NUL mode preserves entry bytes exactly (trailing CR/LF are part of the name; only newline mode trims them) |
|
||||
| `--max-size=SIZE` | Skip files larger than SIZE | ✅ Parity | `max_size` in scanner |
|
||||
@@ -101,8 +117,8 @@ Every one of those has an entry below with its remaining caveats.
|
||||
|------|-------------------|-----------------|-------|
|
||||
| `-r`, `--recursive` | Recurse into directories | ✅ Parity | Default behavior |
|
||||
| `-R`, `--relative` | Use relative path names | ✅ Parity | Protocol 2.26.0 implements rsync's general `-R` path semantics: without a cut the source argument is mirrored in full below the destination root; a `/./` cut in the source argument (`src/./foo`) makes everything after the cut the destination prefix, so the layout matches rsync's relative reconstruction; and `--files-from` entries land under their bare relative path. The delete manifest derives from the sent (relative) paths and is scoped to the transferred prefix subtree, so `--delete` cannot remove destination content outside that prefix (a blocker fix). Works single-threaded and under `-j`/`--threads` |
|
||||
| `--no-implied-dirs` | Don't send implied dirs with -R | ✅ Parity | With `-R`, rsync creates the ancestor directories implied by a listed path and, with `--no-implied-dirs`, omits them from the transfer so the destination directories keep the destination's own mode/mtime. Protocol 2.26.0 matches this: the implied-dir walk applies the transfer's per-attribute metadata only to explicitly transferred directories, and a differential test verifies the modes and mtimes of the implied parents against rsync with and without the flag. Works single-threaded and under `-j`/`--threads` |
|
||||
| `-d`, `--dirs`, `--old-dirs`, `--old-d` | Transfer dirs without recursing | ⚠️ Caveat | Protocol 2.26.0 implements rsync's one-level `-d` listing for `dir`, `dir/` and `.`: the source's immediate contents are transferred (files with content, directories as explicit entries), matching rsync's destination tree in a differential test. `--dirs --files-from` transfers exactly the listed items — a listed directory is created empty and a listed file with content — under the same `-R` layout rules. Directory entries cross as `STATUS_MKDIR` and appear in the delete manifest, so `--delete` prunes correctly and an empty listed directory survives. Directory times are applied at the end of the transfer; modes/ownership follow the per-attribute policy. **Remaining divergence:** a plain recursive `-a` scan still does not create empty source directories (directory entries are record-only unless `-d`/`--files-from` explicitly lists a directory), and under `--delay-updates` directories are created immediately while only regular files are staged (see the recursive-empty-directory residual in the completion-wave section) |
|
||||
| `--no-implied-dirs` | Don't send implied dirs with -R | ✅ Parity | With `-R`, rsync creates the ancestor directories implied by a listed path and, with `--no-implied-dirs`, omits their attributes from the transfer so they keep the destination's own state (or are created with default attributes when absent). Protocol 2.26.0 matches this: without `--files-from` the implied-dir walk applies per-attribute metadata only to explicitly transferred directories, and with `-R --files-from` a listed file whose parent is not itself listed is placed normally — the missing implied parent is created with default attributes (not the source's) and the file transfers with `rc 0`, exactly like rsync 3.4.1 (a differential test verifies the modes and mtimes with and without the flag). Works single-threaded and under `-j`/`--threads` |
|
||||
| `-d`, `--dirs`, `--old-dirs`, `--old-d` | Transfer dirs without recursing | ✅ Parity | Protocol 2.26.0 implements rsync's one-level `-d` listing for `dir`, `dir/` and `.`: the source's immediate contents are transferred (files with content, directories as explicit entries), matching rsync's destination tree in a differential test. `--dirs --files-from` transfers exactly the listed items — a listed directory is created empty and a listed file with content — under the same `-R` layout rules. A plain recursive scan also recreates empty source directories now: the scanner emits a payload-less directory entry (with metadata) for every traversed directory that produced no transferred or descended child, unless `-m/--prune-empty-dirs` suppresses it or the run is `--files-from`/`--list-only` (a directory emptied by filtering is recreated too, matching rsync). Directory entries cross as `STATUS_MKDIR` and appear in the delete manifest, so `--delete` prunes correctly and an empty listed directory survives; an incoming directory replaces a destination regular file (rsync removes the non-directory and creates the directory), verified differentially. Directory times are applied at the end of the transfer; modes/ownership follow the per-attribute policy. Under `--delay-updates` directories are created immediately while only regular files are staged, exactly as rsync does |
|
||||
| `--mkpath` | Create missing path components | ✅ Parity | Wire option (client → server). At connection start the server creates the client's destination root directory (and any missing leading components below its own authorized root) when `--mkpath` is set, failing the connection cleanly if it cannot. Without `--mkpath` a destination root that does not exist yet is rejected up front (rsync semantics), so the flag is the only way to transfer into a not-yet-created destination directory. Creation is confined by the same secure mkdir walk as file writes (`O_NOFOLLOW`, no `..`) |
|
||||
| `--inc-recursive`, `--no-inc-recursive` | Incremental recursion mode | ❌ Divergent | rsync's man-page-only scanning-mode switch (and its short aliases). FastSync always performs a single full recursive scan, so both spellings are rejected as unknown options rather than accepted as a no-op; there is no incremental-recursion engine to toggle. A genuine implementation would be a scan-architecture change with no benefit for FastSync's push model |
|
||||
|
||||
@@ -122,12 +138,12 @@ Every one of those has an entry below with its remaining caveats.
|
||||
|
||||
| Flag | Rsync Description | FastSync Status | Notes |
|
||||
|------|-------------------|-----------------|-------|
|
||||
| `-n`, `--dry-run` | Trial run with no changes | ⚠️ Caveat | Server-contacting since protocol 2.21.0. The routing predicate `dry_run_targets_server()` selects the server-contacting path for any target a real run would reach over the wire (SSH, daemon `host::module`, explicit `--server-host`/`--server-port`, TLS, source-bind `--address`); the client handshakes with the receiver, which runs the normal read-only per-file check and answers `STATUS_DRY_RUN_TRANSFER`/`STATUS_OK` without mutating anything. Protocol 2.25.0 also reports would-delete lines: with `--delete` the receiver's `STATUS_STATS` carries the extras it would have removed and the client prints rsync-style `*deleting` lines (sequential and `--threads`; control bytes escaped). Dry-run never deletes. **Remaining divergences:** the `*deleting` line ordering can differ from rsync's delete-during walk, and a filtered dry-run can over-report what the real commit would remove |
|
||||
| `-n`, `--dry-run` | Trial run with no changes | ❌ Divergent | Server-contacting since protocol 2.21.0. The routing predicate `dry_run_targets_server()` selects the server-contacting path for any target a real run would reach over the wire (SSH, daemon `host::module`, explicit `--server-host`/`--server-port`, TLS, source-bind `--address`); the client handshakes with the receiver, which runs the normal read-only per-file check and answers `STATUS_DRY_RUN_TRANSFER`/`STATUS_OK` without mutating anything. Protocol 2.25.0 also reports would-delete lines: with `--delete` the receiver's `STATUS_STATS` carries the extras it would have removed and the client prints rsync-style `*deleting` lines (sequential and `--threads`; control bytes escaped). Dry-run never deletes. **Reclassified Divergent (differential evidence):** the would-delete report over-reports — it includes the file that is merely being updated (the receiver's extras walk does not see the would-be-transferred file in its keep-set) and, unlike rsync, also lists an excluded-but-protected extra under `--exclude`, and its line ordering differs from rsync's delete-during walk (`test_dry_run_delete_lines_over_report_residual`). A real run remains correct; only the dry-run report diverges |
|
||||
| `-b`, `--backup` | Make backups of overwritten files | ✅ Parity | Backup before overwrite |
|
||||
| `--backup-dir=DIR` | Backup directory hierarchy | ✅ Parity | `backup_dir` config field |
|
||||
| `--suffix=SUFFIX` | Backup suffix (default ~) | ✅ Parity | `suffix` config field |
|
||||
| `--delay-updates` | Put updated files in place at end | ⚠️ Caveat | Successfully received files are staged under a private 0700 `.fastsync-stage` dir inside the receive root and atomically renamed into their final destinations only after the whole transfer (manifest/delete handling included) succeeds, just before the success/outcome frame is sent. The delete walker deliberately skips the staging dir at the receive root, so `--delete` removes genuine extras but never the staged files (deletion runs before publication; rsync's delete-after ordering is not implemented). `--existing`/`--ignore-existing`/`--update` decide against the final destination path at stage time; `--backup` moves the old file aside at publication, and **`--force` is honored at publication** (protocol 2.23.0): a staged regular file or symlink may replace a destination directory that blocks it. Incompatible with `--inplace` and with `--backup-dir=.fastsync-stage` (the internal staging name is reserved; both are rejected). The staging dir name is fixed, so two simultaneous delayed transfers to the same destination root are serialized with an exclusive advisory lock held for the whole transfer: the second session fails cleanly instead of corrupting the first. Aborting or failing before publication installs nothing and removes the staging tree; a crash between stage and publish leaves staged leftovers that the next delayed run wipes at start (process death releases the lock). A stage→publish failure aborts the transfer (best-effort cleanup of the not-yet-published staged files; already-published files are not rolled back). Works in single-threaded and `-j`/`--threads` modes |
|
||||
| `-T`, `--temp-dir=DIR` | Create temporary files in DIR | ⚠️ Caveat | `--temp-dir` with the rsync short `-T` (the timeout alias moved to long-only `--timeout`). **Protocol 2.23.0 receiver policy: the scratch dir is confined to the receive root — a relative dir is resolved below it, and an absolute path or one containing `..` is rejected by the receiver** (an absolute/foreign-filesystem scratch dir was the divergence; rsync's standalone mode would follow an absolute `--temp-dir`, while its daemon also confines). Temp copies use a unique name there and are atomically renamed into place. **On `EXDEV` (scratch dir and destination on different filesystems) the receiver falls back to a non-atomic copy instead of aborting the transfer**, matching rsync. `--inplace` and `--partial-dir` writes bypass the scratch dir |
|
||||
| `--delay-updates` | Put updated files in place at end | ❌ Divergent | Successfully received files are staged under a private 0700 `.fastsync-stage` dir inside the receive root and atomically renamed into their final destinations only after the whole transfer (manifest/delete handling included) succeeds, just before the success/outcome frame is sent. The delete walker deliberately skips the staging dir at the receive root, so `--delete` removes genuine extras but never the staged files (deletion runs before publication; rsync's delete-after ordering is not implemented). `--existing`/`--ignore-existing`/`--update` decide against the final destination path at stage time; `--backup` moves the old file aside at publication, and **`--force` is honored at publication** (protocol 2.23.0): a staged regular file or symlink may replace a destination directory that blocks it. Incompatible with `--inplace` and with `--backup-dir=.fastsync-stage` (the internal staging name is reserved; both are rejected). The staging dir name is fixed, so two simultaneous delayed transfers to the same destination root are serialized with an exclusive advisory lock held for the whole transfer: the second session fails cleanly instead of corrupting the first. Aborting or failing before publication installs nothing and removes the staging tree; a crash between stage and publish leaves staged leftovers that the next delayed run wipes at start (process death releases the lock). A stage→publish failure aborts the transfer (best-effort cleanup of the not-yet-published staged files; already-published files are not rolled back). **Reclassified Divergent (differential evidence):** the staging name is fixed and a delayed run wipes a pre-existing destination tree of that name at start even without `--delete`, whereas rsync uses its own internal temp name and leaves a genuine destination entry named `.fastsync-stage` untouched (`test_delay_updates_staging_name_collision_residual`); deletion also runs before publication while rsync's `--delay-updates` implies `--delete-after`. Works in single-threaded and `-j`/`--threads` modes |
|
||||
| `-T`, `--temp-dir=DIR` | Create temporary files in DIR | ❌ Divergent | `--temp-dir` with the rsync short `-T` (the timeout alias moved to long-only `--timeout`). A **relative** dir matches rsync exactly: it is resolved below the receive/destination root and must already exist (differentially verified: `rsync -a --temp-dir=scratch src/ dst/` and FastSync produce identical trees and an empty scratch dir). **Reclassified as a deliberate divergence because an absolute `--temp-dir` is rejected by the receiver** — it is resolved verbatim by rsync standalone (which will use `/tmp` or any other absolute directory, including one outside the destination), but FastSync's security-reviewed receiver confines the scratch dir to the authorized receive root and rejects any absolute path or one containing `..`. A differential test confirms rsync exits 0 using an absolute scratch dir while FastSync refuses before writing anything into it (the scratch dir stays empty). Its daemon mode also confines relative to the module, but standalone rsync's absolute-temp-dir behavior is not reproduced because it would let a client place receiver scratch files outside the sandbox. Temp copies use a unique name in the scratch dir and are atomically renamed into place; **on `EXDEV` (scratch dir and destination on different filesystems, reachable via a confined relative symlink) the receiver falls back to a non-atomic copy instead of aborting**, matching rsync. `--inplace` and `--partial-dir` writes bypass the scratch dir |
|
||||
| `--partial` | Keep partially transferred files | ✅ Parity | On a failed/interrupted write the already-written temp file is retained at the destination path (best-effort rename instead of unlink) so a later `--append`/`--append-verify` run can resume it. Retention never runs when no data was actually written or under `--ignore-existing`/`--existing` (the destination is not ours to overwrite), and it only ever renames the already-written temp. A failed rename falls back to the normal unlink |
|
||||
| `--partial-dir=DIR` | Keep partial files in DIR | ✅ Parity | With `--partial`, the working file is written under the confined partial directory (a relative dir below the receive root) and atomically renamed into place once complete, so an interrupted transfer leaves a resumable copy there and completed transfers do not linger under it. `--inplace` bypasses the partial dir (rsync parity). Requires `--partial` |
|
||||
|
||||
@@ -138,13 +154,13 @@ Every one of those has an entry below with its remaining caveats.
|
||||
| `--delete` | Delete extraneous files from dest | ⚠️ Caveat | `use_delete` config field. Deletion is always derived from the transmitted keep-set manifest of the paths the sender sent/keeps (never from unchecked input), runs through the symlink-safe walker bounded by `MAX_SERVER_DELETE_COUNT`, and skips the `.fastsync-stage` staging dir under `--delay-updates`. FastSync's default timing when no timing flag is given is **delete-after** (extras are removed only once the whole transfer succeeded) — intentionally NOT rsync's `--del`/delete-during default, to preserve FastSync's commit-style safety. By default the destination mirror of a path the source scan pruned (filter/exclude/size rules) is **protected** from deletion — matching rsync, which does not delete excluded files under `--delete`; `--delete-excluded` opts back into deleting them (see below). Deletion is scoped to the **synchronized directories** sent in the manifest (protocol 2.23.0), so a `--files-from` subset no longer deletes untransmitted paths outside the listed directory subtrees. The walk is bounded: a client `--max-delete=NUM` (or the 100000-entry server bound) makes it **partial** — entries up to the bound are removed, the rest are skipped, and the client exits **25** (`RERR_PARTIAL`), matching rsync, rather than failing the transfer. Extraneous destination symlinks are unlinked by name (never followed); a directory still holding a kept/protected entry is left behind rather than failing |
|
||||
| `--delete-before` | Delete before transfer | ⚠️ Caveat | Implies `--delete`. The sender runs a full source pre-scan (paths only) and transmits the keep-set manifest BEFORE any file data; the receiver validates it, removes every destination entry not listed (bounded walk, staging-dir skip, protected prefixes honored), then acks `STATUS_OK`. The sender only starts streaming after the deletion committed, or aborts if the receiver reported a deletion error. By definition the deletions already happened when a later transfer phase fails — rsync's delete-before is destructive the same way; a subsequent failure does not restore the removed files. Divergence: the keep-set is the pre-scan snapshot, so a file that appears on the source between the pre-scan and the data pass is still transferred but was not protected from deletion |
|
||||
| `--del`, `--delete-during` | Delete during transfer | ⚠️ Caveat | Both spellings accepted; imply `--delete`. **Protocol 2.24.0 implements per-directory delete plans:** as the sender finishes each source directory it streams a `STATUS_DELETE_PLAN` for that directory and the receiver removes that directory's extras before applying the next directory's data, so a mid-transfer failure has already removed the extras of the directories reached (verified with a byte-slicing proxy). **Remaining divergence:** the exact abort boundary and the progressive ordering of removals versus rsync's generator can differ, and `-d`/`--dirs` (no descent) falls back to the end-of-transfer commit. `-R` plans are scoped to the transferred prefix subtree |
|
||||
| `--delete-delay` | Find deletions during, delete after | ⚠️ Caveat | Implies `--delete`. **Protocol 2.24.0 implements rsync's delete-delay timing:** the sender records each directory's delete plan while scanning and the receiver commits those removals only after the whole transfer succeeds (per plan), so an extra created in the destination after its directory's plan survives while `--delete-after` re-scans and removes it, and a failed transfer removes nothing. **Remaining divergence:** exact ordering/abort boundaries can differ from rsync's generator, and `-d` falls back to the end commit. **Also:** the reported "Number of deleted files" can be inflated because a directory snapshotted into the delete plan that later fails to delete (ENOTEMPTY) is still counted (`delete_plan.c:583-593`, `866`, `891`) |
|
||||
| `--delete-delay` | Find deletions during, delete after | ⚠️ Caveat | Implies `--delete`. **Protocol 2.24.0 implements rsync's delete-delay timing:** the sender records each directory's delete plan while scanning and the receiver commits those removals only after the whole transfer succeeds (per plan), so an extra created in the destination after its directory's plan survives while `--delete-after` re-scans and removes it, and a failed transfer removes nothing. The **reported** deleted count advances only on an actual removal; the `--max-delete` budget is charged at plan/snapshot time (the `planned` counter, incremented by `defer_add`) to bound the deferred list, so a snapshotted extra that is later skipped or fails removal still consumes budget. A directory snapshotted into the plan that is refilled before the commit and survives `ENOTEMPTY` is **not** reported, and a `--max-delete=2` partial delete reports exactly 2 (differential test vs rsync 3.4.1, exit 25 both). **Caveat:** rsync charges `--max-delete` on actual removals and its deferred removal recurses into a queued directory, so when a snapshotted entry fails removal FastSync skips a later extra that rsync would still delete, and content created in a refilled extra directory is removed by rsync but left in place by FastSync (the differential test pins both sides). Unit tests cover the refilled-directory, removed-file, and plan-time budget accounting. Exact ordering of which extras are removed first can still differ from rsync's generator |
|
||||
| `--delete-after` | Delete after transfer | ✅ Parity | Implies `--delete`. The delete-after timing is also what plain `--delete` does: the keep-set manifest closes the data stream and the receiver commits the bounded deletion only after the terminal `STATUS_FINISHED` proves the whole transfer (every data frame received and stored) succeeded. A failed or aborted transfer removes nothing |
|
||||
| `--delete-excluded` | Also delete excluded files | ⚠️ Caveat | `delete_excluded` config field. Under `--delete` FastSync protects (rsync's default) the destination mirror of paths the sender's source scan pruned by the user-selection rules — the `--filter`/`-F`/`-C` layer and the legacy `--exclude`/`--include` layer. The sender transmits those concrete pruned paths as **protected prefixes** in the delete-manifest frame (see the Phase-3 notes below); the walker never descends into or removes them. `--delete-excluded` opts back in: the sender sends an empty protected list, so the excluded destination mirrors become ordinary extras and are removed. **`--max-size`/`--min-size` pruned mirrors are a separate, always-on protection** (protocol 2.23.0, rsync parity): size-pruned source mirrors survive `--delete` even with `--delete-excluded`. Divergences (documented): protection is derived only from what the source scan actually pruned — a stray destination-only file that happens to match an exclude rule is not protected (FastSync never re-applies rules to the destination, keeping deletion sender-derived) |
|
||||
| `--max-delete=NUM` | Max files to delete | ✅ Parity | `max_delete` config field (default -1 = no client limit; 0 = delete nothing). **Protocol 2.23.0 matches rsync's partial semantics:** the receiver deletes up to NUM entries (regular files, symlinks and empty directories; each directory removal counts as one) and then **stops deleting, skips the rest, and reports the run as partial**. The client prints a "deletions stopped due to `--max-delete` limit" message and exits **25** (rsync's `RERR_PARTIAL`), not a hard failure — the transfer itself succeeded. NUM only applies together with `--delete` (it is inert otherwise, matching rsync). A client NUM below the server hard bound `MAX_SERVER_DELETE_COUNT` (100000) replaces it; a NUM above it never raises that cap. Deleting an entire destination with no limit is still bounded by the server's 100000-entry ceiling. `--delete-missing-args` exact-path deletions and the ordinary extras walk draw from the same budget, matching rsync |
|
||||
| `--ignore-errors` | Delete even with I/O errors | ⚠️ Caveat | Sender-side, client-only config field. rsync suppresses `--delete` when the transfer had I/O errors; FastSync's equivalent is a source-scan I/O error (an unreadable directory, e.g. EACCES). By default the scan aborts the run so no deletion happens. With `--ignore-errors` the scan continues past the unreadable directory, the readable tree is transferred, deletion still runs (the unreadable directory's mirror is treated as an extra), and the run exits 23 (`RERR_PARTIAL`), matching rsync. **Remaining divergence / uncertainty:** the EACCES differential is not exercised in CI because the runner is root (mode 000 is still readable), so this rests on source inspection plus the setpriv integration test |
|
||||
| `--ignore-errors` | Delete even with I/O errors | ✅ Parity | Sender-side, client-only config field. Matches rsync's semantics exactly: an unreadable source subdirectory is always skipped so the readable tree transfers (the transfer root itself stays fatal), and the run reports rsync's partial-transfer exit **23**. Deletion policy follows rsync: by default an I/O error suppresses deletion (`IO error encountered -- skipping file deletion`), while `--ignore-errors` lets the deletion commit. The decision applies to every timing (`--delete`, `--delete-before`, `--delete-during`, `--delete-delay`, `--delete-after`) in both the sequential and `--threads` send paths. Differential-tested against rsync 3.4.1 with both tools run as an unprivileged user (mode-000 source directory); the reference build's root-only gate still excludes the EACCES differential, but the setpriv differential test exercises it. The piece that stays FastSync-specific is documented under the recursive-empty-directory residual: FastSync never emits an unreadable (or empty) directory entry, so that mirror is an extra that a run with `--ignore-errors` removes, where rsync emits the directory and keeps its mirror |
|
||||
| `--force` | Force deletion of non-empty dirs | ✅ Parity | `force_delete` receiver config field (crosses the wire). rsync's `--force` lets an incoming non-directory replace a destination directory; FastSync implements exactly that: when a regular file (or symlink) is written to a path that is currently a (possibly non-empty) destination directory, `--force` removes that directory tree first — confined to the receive root and symlink-safe (O_NOFOLLOW fd walk, symlinks removed by name, never followed) — so the install can place the file. **Protocol 2.23.0 honors `--force` on the `--delay-updates` publication path too**, not only the immediate-install path. Without `--force` such a write fails and the run aborts. Gated by the server `--allow-delete` policy (a client cannot use `--force` to remove a destination tree on a server that forbids deletion) |
|
||||
| `-m`, `--prune-empty-dirs` | Prune empty dir chains | ✅ Parity | `-m`/`--prune-empty-dirs` (Phase 7 Wave A freed the rsync short `-m`; FastSync multithreading is now `-j`/`--threads`). FastSync's recursive transfer records directory times but never CREATES an empty directory (a `STATUS_DIR_TIMES` entry is record-only, and `--dirs` empty entries are pruned by this flag), so empty directories are inherently never transferred (which is rsync's `-m` behavior) and truly-empty destination directory chains are removed by `--delete` regardless of this flag. The flag's additional real effect is on the `--dirs` explicit directory-entry generator: a plain `-d <empty-dir>` run omits the empty source directory's entry, so nothing is created at the destination (no `STATUS_MKDIR`, no `-i`/`--out-format` change line, and an existing empty mirror becomes an extra that `--delete` prunes). Explicitly `--files-from`-listed directories always pass through (documented `--files-from` behavior). A directory that still holds an excluded-but-protected file survives, matching the `--delete-excluded` default |
|
||||
| `-m`, `--prune-empty-dirs` | Prune empty dir chains | ✅ Parity | `-m`/`--prune-empty-dirs` (Phase 7 Wave A freed the rsync short `-m`; FastSync multithreading is now `-j`/`--threads`). A recursive transfer now recreates empty source directories by default (rsync parity); this flag suppresses that emission, so an empty directory (physically empty, or emptied by filtering) is not created and a true empty-directory chain is removed by `--delete`, matching rsync's `-m`. It also affects the `--dirs` explicit directory-entry generator: a plain `-d <empty-dir>` run omits the empty source directory's entry, so nothing is created at the destination (no `STATUS_MKDIR`, no `-i`/`--out-format` change line, and an existing empty mirror becomes an extra that `--delete` prunes). Explicitly `--files-from`-listed directories always pass through (documented `--files-from` behavior; `--files-from` runs never emit implicit empty directories). A directory that still holds an excluded-but-protected file survives, matching the `--delete-excluded` default |
|
||||
|
||||
**Deletion-timing implementation notes (Phase 3):** the delete flags above are
|
||||
real. Two new config booleans (`delete_during`, `delete_delay`) join the already
|
||||
@@ -288,7 +304,7 @@ why plain `--append` works on the normal atomic path, not only with `--inplace`.
|
||||
| `--write-devices` | Write to devices as files | ❌ Divergent | Writes only into an existing char/block node under the confined receive root (`O_NOFOLLOW` + `O_NONBLOCK`); a missing, symlinked, FIFO-with-no-reader, non-device, or otherwise unusable destination is skipped with a warning rather than allowed or aborted. Deliberate confinement divergence from rsync's more permissive behavior |
|
||||
| `-U`, `--atimes` | Preserve access times | ✅ Parity | Captures the source access time (from the scanner's pre-read stat, so it is not clobbered by reading the file for transfer) and transmits it over the wire; the receiver restores it together with the mtime via `futimens`/`utimensat`. Implies metadata transmission (the times travel inside the shared metadata payload), but does not enable ownership application (that stays opt-in via the identity flags). Wire: `atime` fields on the metadata frame + a `preserve_atimes` config boolean; `PROTOCOL_VERSION` bumped **2.11.0 → 2.12.0** |
|
||||
| `-N`, `--crtimes` | Preserve create times | ❌ Divergent | Birth-times cannot be set by any portable filesystem call (`utimensat`/`futimens` only set atime/mtime), so this row is an explicit **Divergent** entry (Phase 7 Wave B). Capture + transmit stays: `statx(STATX_BTIME)` on Linux records the source birth time as a wire field; the receiver logs a debug note that it cannot be applied and continues — never failing the transfer and never pretending it worked. On platforms without `statx` it parses as a documented no-op (flag accepted; nothing is captured). Implies metadata transmission. Wire: new `crtime` fields + a `preserve_crtimes` config boolean; `PROTOCOL_VERSION` bumped **2.11.0 → 2.12.0** (see the Phase-4 metadata-time notes) |
|
||||
| `-O`, `--omit-dir-times` | Omit dirs from --times | ✅ Parity | Real modifier now that FastSync preserves directory times. With metadata on, the scanner captures every traversed source directory's mtime (and atime under `-U`) and the sender transmits them in trailing `STATUS_DIR_TIMES` frame(s) **after all file data and the optional delete manifest** (chunked at the receiver's `MAX_MANIFEST_ENTRIES` per-frame cap); a dir-time entry only RECORDS metadata and never creates the directory, so empty source directories stay untransferred. The receiver defers applying them until its delete / `--delay-updates` publication phases have committed, so writing or removing a child never clobbers a parent directory's mtime (rsync applies directory times at the end for exactly this reason). When `-O` is set (the boolean crosses the wire) the receiver does not apply any of them; without `-O` an `-a`/`--preserve` transfer now restores directory times (reversing the old "never preserves dir times" divergence). Wire change: the terminal `STATUS_DIR_TIMES` frame; `PROTOCOL_VERSION` bumped **2.16.0 → 2.17.0** |
|
||||
| `-O`, `--omit-dir-times` | Omit dirs from --times | ✅ Parity | Real modifier now that FastSync preserves directory times. With metadata on, the scanner captures every traversed source directory's mtime (and atime under `-U`) and the sender transmits them in trailing `STATUS_DIR_TIMES` frame(s) **after all file data and the optional delete manifest** (chunked at the receiver's `MAX_MANIFEST_ENTRIES` per-frame cap); a dir-time entry only RECORDS metadata and never creates the directory (an empty source directory is created by the separate `STATUS_MKDIR` entry the scanner now emits, and `-m/--prune-empty-dirs` suppresses that; the trailing dir-time simply re-applies the metadata). The receiver defers applying them until its delete / `--delay-updates` publication phases have committed, so writing or removing a child never clobbers a parent directory's mtime (rsync applies directory times at the end for exactly this reason). When `-O` is set (the boolean crosses the wire) the receiver does not apply any of them; without `-O` an `-a`/`--preserve` transfer now restores directory times (reversing the old "never preserves dir times" divergence). Wire change: the terminal `STATUS_DIR_TIMES` frame; `PROTOCOL_VERSION` bumped **2.16.0 → 2.17.0** |
|
||||
| `-J`, `--omit-link-times` | Omit symlinks from --times | ✅ Parity | Real modifier now that FastSync preserves symlink times. Symlink entries already carried their metadata on `STATUS_SYMLINK`; the receiver now applies it with **no-follow primitives only** (`utimensat(..., AT_SYMLINK_NOFOLLOW)`, plus best-effort `fchmodat(..., AT_SYMLINK_NOFOLLOW)` and policy-gated `fchownat(..., AT_SYMLINK_NOFOLLOW)`), so the link itself is stamped without ever dereferencing it, confined fd-relative below the authorized receive root. A symlink has no children, so the times are applied immediately at creation. When `-J` is set (the boolean crosses the wire) the receiver skips the timestamps (mode/ownership are unaffected); without `-J` an `-a`/`-l` transfer restores symlink mtimes. Wire change alongside `-O`: the shared `STATUS_DIR_TIMES` frame; `PROTOCOL_VERSION` bumped **2.16.0 → 2.17.0** |
|
||||
| `--super` | Receiver attempts super-user activities | ❌ Divergent | Safe-subset privilege model. `--super` permits the receiver to attempt already-confined super-user activities (ownership application, char/block device-node creation, `--write-devices`); `--no-super` forbids them even for root; `auto` keeps the historical best-effort attempt. **FastSync never elevates** — no `setuid`/`seteuid`/`setgid` — and `--super` never bypasses the confinement floor, so it diverges from rsync's real elevation. A server `--no-super` veto forces it off for every connection; a privileged standalone listener defaults off without `--allow-super`; daemon modules opt in with `client owner = yes` |
|
||||
| `--fake-super` | Store/recover privileged attrs via xattrs | ❌ Divergent | Records the resolved `uid:gid:mode:mtime_sec:mtime_nsec` in a reserved `user.fastsync.stat` xattr and immediately replays mode/times fd-relative, but **never performs a real `chown`** (the owner is recorded for a later privileged restore). The on-disk key and format are FastSync-native, not rsync's `user.rsync.%stat%`, so recordings are not interoperable with rsync — the same class as the native auth and batch formats. Implies metadata transmission; incompatible with `-s` |
|
||||
@@ -629,11 +645,11 @@ targets verbatim, matching rsync.
|
||||
| Flag | Rsync Description | FastSync Status | Notes |
|
||||
|------|-------------------|-----------------|-------|
|
||||
| `--checksum` | Skip based on checksum | ✅ Parity | `-c`/`--checksum` compares per-file whole-file content digests to skip unchanged files. **As of protocol 2.23.0 the short `-c` implies the checksum quick-check**, so a plain `-c` run verifies content rather than only affecting the `--incremental` handshake. The digest algorithm is `xxh128` by default (protocol 2.26.0's negotiated default) and is selectable via `--checksum-choice`/`--cc` (`xxh128`/`xxh3`/`xxh64`/`xxhash`/`md5`/`md4`/`sha1`/`none`/`auto`, plus rsync's two-name form) and `--checksum-seed=NUM` (see those rows) |
|
||||
| `--checksum-choice=STR`, `--cc=STR` | Choose checksum algorithm | ⚠️ Caveat | Real algorithm selection for the per-file whole-file digest used by the `--incremental`/`--checksum` handshake and basis-dir verification. **Protocol 2.26.0 accepts rsync 3.4.1's full set** — `xxh128` (the negotiated default), `xxh3`, `xxh64`, `xxhash`, `md5`, `md4`, `sha1`, `none`, `auto`, and the two-name `transfer,pre-transfer` form — with rsync's exit-4 rejection of an unknown name and of `none` on the transfer side when `--checksum` is on. `--cc=ALG` and space forms both parse. The algorithm id and seed cross the wire; the receiver hashes its old file with the same algorithm+seed and the per-file `STATUS_CHECK` handshake carries a bounded digest pinned to the negotiated length. **Remaining divergences:** rsync uses this choice for the transfer checksum on the wire as well, while FastSync selects only the whole-file comparison digest and keeps the delta BLOCK strong checksum at xxHash32; the `RSYNC_CHECKSUM_LIST` environment variable is not consulted; and `auto` always resolves deterministically to the first supported entry in rsync's preference order rather than probing the peer |
|
||||
| `--compare-dest=DIR` | Compare dest files relative to DIR | ⚠️ Caveat | DIR is a receiver-side basis; protocol 2.26.0 uses an absolute path verbatim (rsync semantics) and resolves a relative path below the destination root (`..` components are rejected, `//` collapsed and trailing `/` dropped). On the receiver's per-file check (implies `--incremental`) an exact match = same size + mtime (unless `--size-only`; `-I` disables matching) **and** equal xxHash64 of the sender's file; a match suppresses the data transfer. compare-dest never copies: it only skips a file the destination does **not** already hold (sparse destination, rsync parity), and is consulted before the normal delta/full paths. Repeatable; searched in command-line order, first match wins. Divergences: when the destination already holds a *different* version rsync deletes it but FastSync instead transfers the data (keeps the mirror complete; never deletes without `--delete`); attribute-only differences on a match are not re-applied (data is skipped so the sender never sends metadata); content is verified by xxHash64, stricter than rsync's default quick check. Sizing: FastSync's whole-file payload limit is 256 MiB on **every** transfer path (not basis-specific); rsync applies basis dirs to arbitrary sizes, so FastSync refuses a basis run whose source contains a larger file up front with a clear error before any transfer. Wire: a basis-count field is always present on the config frame (protocol 2.9.0, so clients and servers must both be 2.9.0) |
|
||||
| `--copy-dest=DIR` | Include copies of unchanged files | ⚠️ Caveat | Same basis rules as `--compare-dest`, but an exact match materializes a **local copy** of the DIR file into the destination (via the normal atomic temp+rename store path, so `--existing`/`--ignore-existing`/`--update`/`--backup`/`--delay-updates` all still apply) instead of transferring data. Repeatable; command-line order = priority. Content is xxHash64-verified before the copy. Divergences: a basis-hit destination keeps the basis file's own mode/uid/gid and mtime (the sender sends no metadata on a skip), so with `--size-only` its mtime can differ from the source and attribute-only differences are copied with the basis attributes rather than rsync's "copy + fix attributes". Requires `--incremental` (implied); incompatible with `-s`. Wire: protocol 2.9.0 |
|
||||
| `--link-dest=DIR` | Hardlink to files when unchanged | ⚠️ Caveat | Same basis rules as `--copy-dest`, but an exact match installs an atomic **hard link** to the DIR file (temp hard link + rename) so no data or disk space is used; where the link is impossible (basis on another filesystem, filesystem refuses links) it falls back cleanly to a byte-identical local copy, never a corrupt/partial file. `--delay-updates` stages the link and publishes by rename, so the final entry stays a real hard link. Repeatable (searched in command-line order, first match wins). Content is xxHash64-verified before linking. Divergences and caveats: protocol 2.26.0 re-links an already up-to-date destination file to the basis (the relink path installs the hard link when the content matches); a link keeps the basis inode's own mode/uid/gid and mtime — metadata is never written through the shared inode (that would mutate the basis file), so a later `--inplace` run that rewrites such a destination path **will mutate the basis snapshot** through the shared inode (use `--copy-dest` when the destination must stay independently writable); with `--size-only` the linked mtime can differ from the source; a `--remove-source-files` source satisfied by a basis dir is treated as skipped and therefore **retained** (never removed); basis dirs are excluded from `--delete`. Requires `--incremental` (implied); incompatible with `-s`. Wire: protocol 2.9.0 |
|
||||
| `-y`, `--fuzzy`, `--no-fuzzy` | Find similar file for basis | ⚠️ Caveat | `-y/--fuzzy` is a pure bandwidth optimization on the existing receiver-driven delta path: when a file must be transferred and the destination holds no usable content at the exact path (file absent, or the destination file is outside the delta engine's size bounds), the receiver searches the SAME destination directory for an existing regular file whose basename is similar to the incoming name and uses it as the delta basis, so the sender transmits only the differences instead of the whole file. The output is always byte-exact regardless of which (or whether any) basis is chosen. Decision location: the receiver performs the candidate search inside `receive_incremental_check` and sends the normal `STATUS_DELTA_SIGNATURE`; the sender never learns the basis was a different file, so no new frame type or sender logic was needed — only the config frame grew a `fuzzy` boolean, so `PROTOCOL_VERSION` was bumped **2.8.0 → 2.9.0** (peers must match). Similarity heuristic (deterministic, simpler than rsync's deliberately-fuzzy matching, and documented precisely): candidates are the target's sibling entries in its destination directory, opened `O_NOFOLLOW`/`AT_SYMLINK_NOFOLLOW` under the confined root (symlinks never followed; nothing outside the destination root is ever read or hashed); dotfiles, directories, the target's own name, and the `.fastsync-stage`/temp scratch names are excluded; like the ordinary delta path, the block signature the receiver transmits is derived from on-disk content it may not otherwise send, so a negotiated `--fuzzy` run exposes the destination's sibling files (at block granularity) to the sender as a known-plaintext oracle — the same information class as the normal delta handshake over the file being replaced; the size gate is the delta engine's own bounds (both files ≥ 16 KiB, ≤ `--delta-max`, ratio ≤ 10×) rather than rsync's ~1.5× size window; protocol 2.26.0 uses a name-distance/suffix heuristic modelled on rsync's plus an exact size+mtime pass, and reads a single best candidate; the exact tie-break order can still differ from rsync's; the directory scan is capped at 4096 entries so a pathological directory cannot stall a transfer. When fuzzy applies: only to files the receiver would otherwise send whole — the destination's own file is always preferred as the delta basis when it exists and fits the delta size bounds, so fuzzy does NOT replace an existing-but-different destination basis; FastSync's 10× delta size-ratio bound means an existing destination file that is too far away in size still lets the fuzzy search run. When no similar candidate exists the transfer falls back to the normal whole-file transfer. rsync-divergence note: rsync's own matching uses a fuzzy name/size rule set; FastSync implements the closest safe deterministic approximation above. Because FastSync's delta machinery is off by default (rsync's is on), `--fuzzy` implies `--incremental` + `--delta` (unless `--whole-file`/`-W` or an explicit `--no-delta` switched delta off, in which case fuzzy is inert — matching rsync where `--whole-file` makes fuzzy irrelevant). Unlike the basis-dir options, `--fuzzy` honors an explicit `--no-incremental` (it does not force the handshake back on); an explicit `--no-incremental` also suppresses the delta implication so no invalid `--delta requires --incremental` config results. `--no-fuzzy` negates it. All surrounding semantics are untouched: a fuzzy-reconstructed file is stored as a normal file, so `--remove-source-files`, itemize/`-i`, `--stats`, `--backup`, `--delay-updates`, `--existing`/`--ignore-existing`/`--update` behave exactly as for a whole-file transfer (the fuzzy delta does not skip the file) |
|
||||
| `--checksum-choice=STR`, `--cc=STR` | Choose checksum algorithm | ⚠️ Caveat | Real algorithm selection for the per-file whole-file digest used by the `--incremental`/`--checksum` handshake and basis-dir verification. **Protocol 2.26.0 accepts rsync 3.4.1's full set** — `xxh128` (the negotiated default), `xxh3`, `xxh64`, `xxhash`, `md5`, `md4`, `sha1`, `none`, `auto`, and the two-name `transfer,pre-transfer` form — with rsync's exit-4 rejection of an unknown name and of `none` on the transfer side when `--checksum` is on. `--cc=ALG` and space forms both parse. The algorithm id and seed cross the wire; the receiver hashes its old file with the same algorithm+seed and the per-file `STATUS_CHECK` handshake carries a bounded digest pinned to the negotiated length. `checksum_digest_file` now streams **every** supported algorithm (md4 via the self-contained RFC 1320 code, sha1/md5 via EVP, none as an empty digest), so the streaming path matches its contract, and `--out-format %C` uses the selected **transfer** half of a two-name choice and renders each algorithm byte-for-byte like rsync (xxh128 high-then-low, xxh64/xxh3 big-endian, md5/md4/sha1 standard hex, none a blank 2-char column) — differential-tested across all algorithms. **Remaining divergences:** rsync uses this choice for the block checksum on the wire too, while FastSync selects only the whole-file comparison digest and keeps the delta BLOCK strong checksum at xxHash32; the `RSYNC_CHECKSUM_LIST` environment variable is not consulted; and `auto` always resolves deterministically to the first supported entry in rsync's preference order rather than probing the peer |
|
||||
| `--compare-dest=DIR` | Compare dest files relative to DIR | ❌ Divergent | DIR is a receiver-side basis; protocol 2.26.0 uses an absolute path verbatim (rsync semantics) and resolves a relative path below the destination root (`..` components are rejected, `//` collapsed and trailing `/` dropped). On the receiver's per-file check (implies `--incremental`) an exact match = same size + mtime (unless `--size-only`; `-I` disables matching) **and** equal xxHash64 of the sender's file; a match suppresses the data transfer. compare-dest never copies: it only skips a file the destination does **not** already hold (sparse destination, rsync parity), and is consulted before the normal delta/full paths. Repeatable; searched in command-line order, first match wins. **Reclassified Divergent (differential evidence):** FastSync verifies a basis hit's content with xxHash64 while rsync's `--size-only` quick check trusts size (and mtime) alone, so with a same-size/different-content basis rsync skips/links the *wrong* basis content while FastSync transfers the source — a deliberate safety-stricter behavior that cannot match rsync (see `test_basis_dir_size_only_content_residual` in `tests/integration/test_parity_quickwins.py`). Attribute-only differences on a match are also not re-applied (data is skipped so the sender never sends metadata). Sizing: FastSync's whole-file payload limit is 256 MiB on **every** transfer path (not basis-specific); rsync applies basis dirs to arbitrary sizes, so FastSync refuses a basis run whose source contains a larger file up front with a clear error before any transfer. Wire: a basis-count field is always present on the config frame (protocol 2.9.0, so clients and servers must both be 2.9.0) |
|
||||
| `--copy-dest=DIR` | Include copies of unchanged files | ❌ Divergent | Same basis rules as `--compare-dest`, but an exact match materializes a **local copy** of the DIR file into the destination (via the normal atomic temp+rename store path, so `--existing`/`--ignore-existing`/`--update`/`--backup`/`--delay-updates` all still apply) instead of transferring data. Repeatable; command-line order = priority. Content is xxHash64-verified before the copy. **Reclassified Divergent** for the same basis-hit verification divergence as `--compare-dest`: rsync's `--size-only` size/quick-check match rings a same-size/different-content file as unchanged and copies the wrong basis bytes, while FastSync's xxHash verification transfers the source (differential test `test_basis_dir_size_only_content_residual`); a basis-hit also keeps whatever metadata the copy derived from the basis rather than rsync's "copy + fix attributes" in attribute-only cases. Requires `--incremental` (implied); incompatible with `-s`. Wire: protocol 2.9.0 |
|
||||
| `--link-dest=DIR` | Hardlink to files when unchanged | ❌ Divergent | Same basis rules as `--copy-dest`, but an exact match installs an atomic **hard link** to the DIR file (temp hard link + rename) so no data or disk space is used; where the link is impossible (basis on another filesystem, filesystem refuses links) it falls back cleanly to a byte-identical local copy, never a corrupt/partial file. `--delay-updates` stages the link and publishes by rename, so the final entry stays a real hard link. Repeatable (searched in command-line order, first match wins). Content is xxHash64-verified before linking. **Reclassified Divergent** for the shared basis-hit divergence: with `--size-only` a same-size/different-content basis is linked by rsync (installing wrong content) but FastSync detects the xxHash mismatch and transfers the source (differential test `test_basis_dir_size_only_content_residual`). Other inherent caveats: protocol 2.26.0 re-links an already up-to-date destination file to the basis; a link keeps the basis inode's own mode/uid/gid and mtime — metadata is never written through the shared inode (that would mutate the basis file), so a later `--inplace` run that rewrites such a destination path **will mutate the basis snapshot** through the shared inode (use `--copy-dest` when the destination must stay independently writable); a `--remove-source-files` source satisfied by a basis dir is treated as skipped and therefore **retained** (never removed); basis dirs are excluded from `--delete`. Requires `--incremental` (implied); incompatible with `-s`. Wire: protocol 2.9.0 |
|
||||
| `-y`, `--fuzzy`, `--no-fuzzy` | Find similar file for basis | ❌ Divergent | `-y/--fuzzy` is a pure bandwidth optimization on the existing receiver-driven delta path: when a file must be transferred and the destination holds no usable content at the exact path (file absent, or the destination file is outside the delta engine's size bounds), the receiver searches the SAME destination directory for an existing regular file whose basename is similar to the incoming name and uses it as the delta basis, so the sender transmits only the differences instead of the whole file. The output is always byte-exact regardless of which (or whether any) basis is chosen. Decision location: the receiver performs the candidate search inside `receive_incremental_check` and sends the normal `STATUS_DELTA_SIGNATURE`; the sender never learns the basis was a different file, so no new frame type or sender logic was needed — only the config frame grew a `fuzzy` boolean, so `PROTOCOL_VERSION` was bumped **2.8.0 → 2.9.0** (peers must match). Similarity heuristic (deterministic, simpler than rsync's deliberately-fuzzy matching, and documented precisely): candidates are the target's sibling entries in its destination directory, opened `O_NOFOLLOW`/`AT_SYMLINK_NOFOLLOW` under the confined root (symlinks never followed; nothing outside the destination root is ever read or hashed); dotfiles, directories, the target's own name, and the `.fastsync-stage`/temp scratch names are excluded; like the ordinary delta path, the block signature the receiver transmits is derived from on-disk content it may not otherwise send, so a negotiated `--fuzzy` run exposes the destination's sibling files (at block granularity) to the sender as a known-plaintext oracle — the same information class as the normal delta handshake over the file being replaced; the size gate is the delta engine's own bounds (both files ≥ 16 KiB, ≤ `--delta-max`, ratio ≤ 10×) rather than rsync's ~1.5× size window; protocol 2.26.0 uses a name-distance/suffix heuristic modelled on rsync's plus an exact size+mtime pass, and reads a single best candidate; the exact tie-break order can still differ from rsync's; the directory scan is capped at 4096 entries so a pathological directory cannot stall a transfer. When fuzzy applies: only to files the receiver would otherwise send whole — the destination's own file is always preferred as the delta basis when it exists and fits the delta size bounds, so fuzzy does NOT replace an existing-but-different destination basis; FastSync's 10× delta size-ratio bound means an existing destination file that is too far away in size still lets the fuzzy search run. When no similar candidate exists the transfer falls back to the normal whole-file transfer. rsync-divergence note: rsync's own matching uses a fuzzy name/size rule set; FastSync implements the closest safe deterministic approximation above. Because FastSync's delta machinery is off by default (rsync's is on), `--fuzzy` implies `--incremental` + `--delta` (unless `--whole-file`/`-W` or an explicit `--no-delta` switched delta off, in which case fuzzy is inert — matching rsync where `--whole-file` makes fuzzy irrelevant). Unlike the basis-dir options, `--fuzzy` honors an explicit `--no-incremental` (it does not force the handshake back on); an explicit `--no-incremental` also suppresses the delta implication so no invalid `--delta requires --incremental` config results. `--no-fuzzy` negates it. All surrounding semantics are untouched: a fuzzy-reconstructed file is stored as a normal file, so `--remove-source-files`, itemize/`-i`, `--stats`, `--backup`, `--delay-updates`, `--existing`/`--ignore-existing`/`--update` behave exactly as for a whole-file transfer (the fuzzy delta does not skip the file). **Reclassified Divergent:** because the output is always byte-exact, the residual is the candidate-selection heuristic itself — a deterministic name-distance/suffix rule with a 10× size-ratio window (vs rsync's ~1.5× window), not rsync's deliberately fuzzy matcher, so the chosen basis (and thus the wire bytes) can differ from rsync even though the final tree cannot. The Integration fuzzy suite (`TestFuzzy`) pins FastSync's thresholds (exact-size+mtime pass, name-distance rejection, 10× unsuitable-destination fallback); a bit-identical basis choice is not achievable without porting rsync's matcher |
|
||||
|
||||
## 12. Compression
|
||||
|
||||
@@ -659,8 +675,8 @@ targets verbatim, matching rsync.
|
||||
| `--address=ADDRESS` | Bind address for outgoing socket | ✅ Parity | Binds the outgoing client socket to a local source address before `connect()` (resolved with the same `-4`/`-6` family hints as the destination). Local socket concern: never crosses the wire |
|
||||
| `-4`, `--ipv4` | Prefer IPv4 | ✅ Parity | Forces `AF_INET` in the `getaddrinfo` hints for client destination/source resolution and the server bind (see the Phase 5, Wave B note). Mutually exclusive with `-6` |
|
||||
| `-6`, `--ipv6` | Prefer IPv6 | ✅ Parity | Forces `AF_INET6` in the `getaddrinfo` hints for client destination/source resolution and the server bind. Mutually exclusive with `-4` |
|
||||
| `--remote-option=OPT`, `-M` | Send an option only to the remote side | ⚠️ Caveat | Each value is appended to the remote server invocation over SSH as an individually single-quote-escaped shell word in `ssh_build_remote_command()`. Values are validated (non-empty, no control characters) and shell metacharacters cannot break out of the quoting (`;`, `&`, `\|`, <code>`</code>, `$`, `(`, `)`, quotes are neutralized), so a value cannot inject an arbitrary remote command and a subsequent `--` on the client line cannot be turned into one. The short `-M` form (`-M OPT`, `-M=OPT`, and rsync-style attached `-MOPT`) is available, matching rsync; metadata mode moved to long-only `--preserve`. **Divergence:** `-M` is only meaningful for the SSH transport (`user@host:path`); a daemon (`host::module/path`) or local TCP destination **rejects** it (there is no remote command line to append to), whereas rsync applies it to its own remote process on every transport. The options never cross the binary config frame |
|
||||
| `--bwlimit=RATE` | Limit I/O bandwidth | ⚠️ Caveat | Token-bucket throttling of the transfer I/O (Kibibytes/second). **Divergence:** FastSync accepts only a positive integer; rsync additionally accepts `0` (no limit) and decimal/suffixed rates (`1.5`, `1.5m`, `100K`), so those rsync spellings are rejected. The limit is a local I/O concern and is not negotiated on the wire |
|
||||
| `--remote-option=OPT`, `-M` | Send an option only to the remote side | ❌ Divergent | Each value is appended to the remote server invocation over SSH as an individually single-quote-escaped shell word in `ssh_build_remote_command()`. Values are validated (non-empty, no control characters) and shell metacharacters cannot break out of the quoting (`;`, `&`, `\|`, <code>`</code>, `$`, `(`, `)`, quotes are neutralized), so a value cannot inject an arbitrary remote command and a subsequent `--` on the client line cannot be turned into one. The short `-M` form (`-M OPT`, `-M=OPT`, and rsync-style attached `-MOPT`) is available, matching rsync; metadata mode moved to long-only `--preserve`. **Reclassified because the daemon/TCP case cannot be reproduced:** `-M` is only meaningful for the SSH transport (`user@host:path`); a daemon (`host::module/path`) or local TCP destination **rejects** it, whereas rsync forwards it to its own remote process on every transport. A differential test starts a real rsync daemon and shows `-M--totally-bogus` reaching the remote parser (`unknown option`) while a valid `-M--safe-links` is accepted. FastSync's daemon handshake is a fixed binary config frame with no per-connection argv channel; adding one would let a client set arbitrary server-side options (the same class of divergence as the native daemon config/auth), so the safe subset stays SSH-only |
|
||||
| `--bwlimit=RATE` | Limit I/O bandwidth | ✅ Parity | A faithful port of rsync 3.4.1's `parse_size_arg(bwlimit_arg, 'K', "bwlimit", 512, -1, True)`: a bare value is KiB/s, `K`/`M`/`G`/`T`/`P` are binary suffixes, `KB`/`MB` are decimal, `KiB`/`MiB` are binary, decimals are accepted and quantized to whole KiB exactly like rsync's `(size + 512) / 1024`, `0` (or an empty value) means "no limit", and any other value below the 512-byte floor is rejected. The token bucket's burst capacity is ~100 ms of bandwidth, matching the point at which rsync's leaky bucket starts sleeping, so a throttled transfer paces like rsync (4 MiB at `--bwlimit=1024`/`2048` matches rsync within ~4%). Differential-tested: the accept/reject matrix and the wall-clock rate both match rsync 3.4.1. The limit is a local I/O concern and is not negotiated on the wire |
|
||||
|
||||
## 14. Daemon Mode
|
||||
|
||||
@@ -733,8 +749,8 @@ modes or links.
|
||||
| `--stop-after=MINS` | Stop after N minutes | ✅ Parity | Client-only sender stop deadline (Phase 6): computing `--stop-after=MINS` (a positive minute count; 0/negative/garbage rejected) and `--stop-at=TIME` (`HH:MM`, `HH:MM:SS`, or `now+N[smhd]`; a past time stops immediately). The transfer stops ELEGANTLY at the next chunk boundary: everything already fully sent is kept and applied, the run returns 0, and --delete (late/delete-after timing) does NOT wipe the destination — when the scan is cut short the partial keep-set manifest is suppressed with a warning (the delete walk is skipped rather than acting on an incomplete keep-set, so unscanned source mirrors survive). `--delete-before`/`--delete-during` still run their complete pre-scan (which ignores the deadline). Local client-only fields: never serialized into the wire config frame, so no PROTOCOL_VERSION bump. `--stop-after` uses CLOCK_MONOTONIC; `--stop-at` uses the wall clock. Works single-threaded and under `-j`/`--threads` (multithreaded). Divergence: rsync computes `--stop-after` from the run start; FastSync likewise. When both are given, the earlier of the two deadlines wins (checked per iteration). See the Phase-6 stop notes below |
|
||||
| `--stop-at=TIME` | Stop at specified time | ✅ Parity | Deadline transfer stop (client-only, never serialized). Protocol 2.26.0 accepts rsync's full date/time grammar (`2030-12-31T23:59`, `2030/12/31T23:59`, `2030-12-31`, `12-31`, `14:00`, `:59`, `1`) in addition to FastSync's `HH:MM[:SS]` and `now+N[smhd]`; a past time stops immediately. Everything already transferred is kept and an early stop suppresses the late `--delete` keep-set so unscanned source mirrors survive. Works single-threaded and under `-j`/`--threads` |
|
||||
| `--fsync` | Fsync every written file before publication | ✅ Parity | |
|
||||
| `--protocol=NUM` | Force older protocol version | ❌ Divergent | Forces the wire protocol version for this transfer. FastSync has exactly ONE wire format (`PROTOCOL_VERSION`, currently 2.26.0) with no downgrade/backward-compat code paths, so `--protocol=2.26.0` is accepted (it sets the version claim the client sends, which the server already requires to match exactly) and **every other value is rejected up front** with a clear error before any connection — it does not and cannot speak an older or virtual wire format. Divergence from rsync (which negotiates a range and downgrades to an integer 0..31): FastSync's honest contract is force-to-the-one-supported-value; a genuine downgrade would require a per-version compatibility layer that does not exist. Client-only; the server-side exact-match check is unchanged. `--protocol=2.25.0`/`2.24.0`/`2.23.0`/`2.22.0`/`2.21.0`/`2.20.0`/`2.19.0`/`2.18.0`/`2.17.0`/`2.16.0`/`2.15.0`/`216`/`31`/garbage are all rejected. See the Phase-6 protocol note below |
|
||||
| `--iconv=CONVERT_SPEC` | Charset conversion | ⚠️ Caveat | Charset conversion of FILE NAMES (not content) at the protocol boundary via iconv(3): `--iconv=LOCAL[,REMOTE]` — the sender converts each local filename LOCAL→REMOTE before transmitting, and the receiver converts each wire filename REMOTE→LOCAL before creating/writing. The full CONVERT_SPEC is serialized into the config frame as a new trailing string field so the peer knows the wire charset; **PROTOCOL_VERSION bumped 2.15.0 → 2.16.0**. `LOCAL[,REMOTE]` parse: single charset ⇒ LOCAL==REMOTE (identity both ways); garbage rejected up front; protocol 2.26.0 additionally accepts `--iconv=.` (the locale's default charset for both directions), `--iconv=-` and `--no-iconv` (disable conversion). Validation probes BOTH directions (a spec that only opens one way is refused, as is a NUL-emitting target charset like utf-16/utf-32/ucs-2, since filenames cannot contain NUL). An unrepresentable name (EILSEQ/EINVAL) fails that path cleanly with a logged `--iconv: cannot convert file name ...` and is never written mangled/truncated. Conversion is applied at EVERY wire-path site (regular/MKDIR/hardlink path+target/symlink path+target/SPECIAL, the delete manifest, the incremental-check path, and the `-s`/`chunk_serialize` embedded blob path), on both client and server (`--iconv` is also a server/daemon option). Zero overhead when unset. See the Phase-6 iconv notes below |
|
||||
| `--protocol=NUM` | Force older protocol version | ❌ Divergent | Forces the wire protocol version for this transfer. FastSync has exactly ONE wire format (`PROTOCOL_VERSION`, currently 2.27.0) with no downgrade/backward-compat code paths, so `--protocol=2.27.0` is accepted (it sets the version claim the client sends, which the server already requires to match exactly) and **every other value is rejected up front** with a clear error before any connection — it does not and cannot speak an older or virtual wire format. Divergence from rsync (which negotiates a range and downgrades to an integer 0..31): FastSync's honest contract is force-to-the-one-supported-value; a genuine downgrade would require a per-version compatibility layer that does not exist. Client-only; the server-side exact-match check is unchanged. `--protocol=2.26.0`/`2.25.0`/`2.24.0`/`2.23.0`/`2.22.0`/`2.21.0`/`2.20.0`/`2.19.0`/`2.18.0`/`2.17.0`/`2.16.0`/`2.15.0`/`216`/`31`/garbage are all rejected. See the Phase-6 protocol note below |
|
||||
| `--iconv=CONVERT_SPEC` | Charset conversion | ✅ Parity | Charset conversion of FILE NAMES (not content) at the protocol boundary via iconv(3): `--iconv=LOCAL[,REMOTE]` — the sender converts each local filename LOCAL→REMOTE before transmitting, matching rsync's rule that the spec "stays the same whether you're pushing or pulling": on a PUSH the destination end's charset is the spec's REMOTE half, so the default receiver writes the wire bytes verbatim, and only a server started with its own `--iconv` (the daemon `charset` analog) declares a different destination charset and converts REMOTE→that LOCAL (rsync push parity, differential-tested with and without a server `--iconv`). The full CONVERT_SPEC is serialized into the config frame as a new trailing string field so the peer knows the wire charset; **PROTOCOL_VERSION bumped 2.15.0 → 2.16.0**. `LOCAL[,REMOTE]` parse: single charset ⇒ LOCAL==REMOTE (identity both ways); garbage rejected up front; protocol 2.26.0 additionally accepts `--iconv=.` (the locale's default charset for both directions), `--iconv=-` and `--no-iconv` (disable conversion). Validation probes BOTH directions (a spec that only opens one way is refused, as is a NUL-emitting target charset like utf-16/utf-32/ucs-2, since filenames cannot contain NUL). An unrepresentable name (EILSEQ/EINVAL) fails that path cleanly with a logged `--iconv: cannot convert file name ...` and is never written mangled/truncated. Conversion is applied at EVERY wire-path site (regular/MKDIR/hardlink path+target/symlink path+target/SPECIAL, the delete manifest, the incremental-check path, and the `-s`/`chunk_serialize` embedded blob path), on both client and server (`--iconv` is also a server/daemon option). Zero overhead when unset. See the Phase-6 iconv notes below |
|
||||
| `--checksum-seed=NUM` | Set checksum seed | ✅ Parity | Sets the seed for FastSync's whole-file xxHash digest (full 64-bit seed) and for the delta path's per-block xxHash32 strong checksum (low 32 bits of the seed). **As of protocol 2.23.0 a seed of `0` — the default when the flag is unset — is randomized per transfer and the chosen seed is sent to the receiver**, exactly like rsync, so two runs against different content do not share a predictable seed; an explicit non-zero seed is used verbatim, so an explicit seed deterministically reproduces every computed digest on BOTH endpoints (the seed crosses in the config frame). `--checksum-choice=md5` has no seed and ignores it (documented). The value is a strict decimal 0..2⁶⁴-1 (blank, signed, or non-numeric values are rejected). Like rsync, a seed only matters where a digest is actually computed (`--checksum` or a basis-dir run, or a delta transfer); it does not by itself enable `--checksum`/`--delta` |
|
||||
| `--secluded-args`, `-s` | Use protocol to send args | ❌ Divergent | Accepted for CLI compatibility (including the rsync short `-s`, Phase 7 Wave A) but a documented **no-op / divergence**. rsync's `-s` protects arguments from shell expansion by shipping them over the protocol; FastSync never passes remote arguments through a shell expansion boundary in the first place — its SSH transport builds the remote argv as **single-quote-escaped shell words** (`ssh_build_remote_command`), so the injection/leak that `-s` guards against does not exist and there is nothing to "seclude". Implementing a true arg-send protocol would mean replacing the argv-based SSH launch with an in-band argument channel, a large redesign of the transport that buys no security here. Chunk serialization remains the long-only `--chunk-serialization`. |
|
||||
| `--protect-args` | Old name of --secluded-args | ❌ Divergent | Accepted for CLI compatibility as a documented no-op; the same rationale as `--secluded-args`/`-s` (FastSync's remote SSH argv is already built injection-safe, so there is no argument-leak to close) |
|
||||
@@ -848,7 +864,7 @@ These are the hardest compatibility items because they require durable formats o
|
||||
|
||||
**Phase 6, Wave A (stop deadline) shipping note:** `--stop-after=MINS` and `--stop-at=TIME` are client-only sender stop deadlines. `--stop-after` takes a positive minute count (0/negative/garbage rejected); `--stop-at` takes `HH:MM`, `HH:MM:SS`, or `now+N[smhd]` (a past time stops immediately, a garbage spec is rejected at parse time). The deadline is computed once at the start of the transfer (CLOCK_MONOTONIC for `--stop-after`, wall clock via `time()` for `--stop-at`) and checked at every chunk boundary in both the single-threaded `send_files` loop and the multithreaded `send_chunks_multithreaded` path, and inside the scanner loops so a busy scan itself stops. When it fires, the transfer stops ELEGANTLY: the in-flight chunk completes, the existing completion tail runs (summary, `disconnect`), and the run returns 0 — exactly like rsync's clean early stop. Because the deadline is client-only and never crosses the wire config frame, no PROTOCOL_VERSION bump is required. The safety-critical interaction is with `--delete`: FastSync streams while scanning, so a deadline can cut the source scan short and yield a PARTIAL keep-set manifest; committing that would make the receiver delete destination mirrors of source files not yet scanned. So the sender tracks `scan_stopped_early` and, when it is true on the late/delete-after (`--delete`/`--delete-after`/`--delete-delay`) path, SUPPRESSES the keep-set manifest (logs a warning) so no deletion happens from an incomplete set — this is the safe direction (preserves data; the delete simply does not run). `--delete-before`/`--delete-during` are unaffected: their complete pre-scan runs before any data and ignores the deadline (a stop can be exceeded by that pre-scan). Under `-j`/`--threads` the stop is symmetric and the scanner thread's still-in-progress manifest appends can never race the tail because the tail does not read the manifest on the early-stop path.
|
||||
|
||||
**Phase 6, Wave B (iconv) shipping note (PROTOCOL 2.15.0 → 2.16.0):** `--iconv=LOCAL[,REMOTE]` converts file NAMES at the wire boundary (never content). The full CONVERT_SPEC is serialized into the config frame as a new trailing string field (empty→NULL canonicalized), so both ends share the same wire charset interpretation; this required the PROTOCOL bump because the frame is a strict ordered sequence and a peer that does not parse the new trailing field would desynchronize. Each end derives LOCAL (its own charset) and REMOTE (the wire charset): the sender opens LOCAL→REMOTE and converts every transmitted filename; the receiver opens REMOTE→LOCAL and converts every received filename before creating/writing. Conversion is applied at every wire-path site (regular/MKDIR/hardlink path+target/symlink path+target/SPECIAL, the delete manifest keep/protected/missing entries, the incremental-check path, and the embedded `-s`/chunk-blob path). A name it cannot convert (EILSEQ/EINVAL) is failed cleanly with a logged `--iconv: cannot convert file name ...` and is never written truncated/mangled. Validation probes both directions up front (both the sender local→remote and the receiver remote→local, and, for a server/daemon with its own `--iconv`, the client-REMOTE→server-LOCAL pair) so an unusable spec is rejected before the connection rather than mid-transfer, and NUL-emitting target charsets (utf-16/utf-32/ucs-2) are refused because filenames cannot contain NUL. Divergence documented upstream: the receiver does NOT half-swap; the wire charset always comes from the sender's REMOTE half, so a server whose local charset differs from the client's LOCAL must declare it with its own `--iconv`. Conversion is process-global and runs on a single thread per process (sender thread / receiver-loop thread), initialized before worker threads start and freed after they join.
|
||||
**Phase 6, Wave B (iconv) shipping note (PROTOCOL 2.15.0 → 2.16.0):** `--iconv=LOCAL[,REMOTE]` converts file NAMES at the wire boundary (never content). The full CONVERT_SPEC is serialized into the config frame as a new trailing string field (empty→NULL canonicalized), so both ends share the same wire charset interpretation; this required the PROTOCOL bump because the frame is a strict ordered sequence and a peer that does not parse the new trailing field would desynchronize. Each end derives its charset and the wire charset: the sender opens LOCAL→REMOTE and converts every transmitted filename; on a push the receiver's destination charset is the spec's REMOTE half, so it writes the wire bytes verbatim, unless the server was started with its own `--iconv` naming a different LOCAL charset (then it opens REMOTE→that LOCAL). Conversion is applied at every wire-path site (regular/MKDIR/hardlink path+target/symlink path+target/SPECIAL, the delete manifest keep/protected/missing entries, the incremental-check path, and the embedded `-s`/chunk-blob path). A name it cannot convert (EILSEQ/EINVAL) is failed cleanly with a logged `--iconv: cannot convert file name ...` and is never written truncated/mangled. Validation probes both directions up front (both the sender local→remote and the receiver remote→destination, and, for a server/daemon with its own `--iconv`, the client-REMOTE→server-LOCAL pair) so an unusable spec is rejected before the connection rather than mid-transfer, and NUL-emitting target charsets (utf-16/utf-32/ucs-2) are refused because filenames cannot contain NUL. The wire charset always comes from the sender's REMOTE half; a server whose local charset differs from the client's REMOTE must declare it with its own `--iconv` (the daemon `charset` analog). Conversion is process-global and runs on a single thread per process (sender thread / receiver-loop thread), initialized before worker threads start and freed after they join.
|
||||
|
||||
**Phase 6, Wave C (protocol-version) shipping note (no PROTOCOL_VERSION change):** `--protocol=NUM` lets the client force the wire protocol version for a transfer. FastSync's protocol is a single lockstep format: the config frame is a strict ordered sequence and the server requires the client's version string to equal `PROTOCOL_VERSION` exactly (`config_receive_with_validate`, src/shared/config.c) — there are no older-format code paths and no downgrade/negotiation machinery, so a lower/higher/virtual version can never be spoken. The honest contract is therefore: the current `PROTOCOL_VERSION` (2.26.0 as of the parity-completion wave) is accepted and stored into the client's `version` claim (which `config_send` already transmits), and every other value — `2.25.0`, `2.24.0`, `2.23.0`, `2.22.0`, `2.21.0`, `2.20.0`, `2.19.0`, `2.18.0`, `2.18`, `2.17.0`, `2.16.0`, `2.15.0`, `3.0.0`, rsync-integer spellings like `216`/`31`, garbage, empty — is rejected up front in `validate_config()` before any connection, with a clear error that FastSync supports only its current wire protocol and cannot speak an older or virtual one. Implementation is client-only: a server-side `--protocol` is intentionally not added because the server has no negotiation (it only enforces exact match), and it could only ever be the current version. This preserves (and slightly tightens) existing validation: the client now also refuses to launch with a version it cannot actually speak, rather than only the server rejecting it later. A genuine downgrade would require a per-version compatibility layer for every frame/feature added since (append 2.10, preallocate 2.11, hardlinks 2.12, devices/specials/symlink-trust/xattr 2.13, remote-option 2.14, daemon module/auth 2.15, iconv 2.16, dir/symlink times 2.17, privilege flags --super/--copy-as 2.18, SCRAM daemon auth 2.19, packed metadata 2.20, error-detail/dry-run 2.21, preserve-attribute split 2.22, rsync-parity wave 2.23) and is intentionally out of scope — documented divergences from rsync's integer-negotiated downgrade remain.
|
||||
|
||||
@@ -879,7 +895,7 @@ These are the last compatibility items and the closing phase toward rsync flag p
|
||||
|
||||
**Wave D — Times superstructure & arg-protection no-ops (✅ implemented, `--secluded-args` ❌).** `-O`/`--omit-dir-times` and `-J`/`--omit-link-times` are now **real modifiers** (both `🔄 → ✅ Implemented`), reversing the old "never preserves directory/symlink times" divergence:
|
||||
|
||||
- **Directory times.** The recursive scanner captures every traversed source directory's metadata (mtime, plus atime under `-U`) into a per-transfer list — two paths are covered: the sequential `DirectoryScanner` captures each opened directory (including the transfer root), and the parallel scanner captures both the root in `parallel_scanner_create_with_options` and each worker's subdirectories in `open_next_directory` (appends are guarded by a mutex shared with the sender's pipeline context). The sender transmits them in trailing `STATUS_DIR_TIMES` frames (each: int count + count × (wire path, metadata) pairs) sent **after all file data and after the optional delete manifest**, just before `STATUS_FINISHED`. A tree larger than `MAX_MANIFEST_ENTRIES` (1 048 576) directories is chunked into repeated frames, each within the receiver's per-frame bound. A dir-time entry is RECORD-ONLY (`file->dir_time_only`): `file_save_to_disk_full` returns `FILE_SAVE_SKIPPED` without creating anything, so a source directory that was empty (or pruned by `-m/--prune-empty-dirs`) is never resurrected. The receiver accumulates received directory metadata in a `DirTimeList` and applies it only at the very end — after the entire stream, after the commit-style `--delete` deletion, and after `--delay-updates` publication — because creating or removing a child bumps the parent's mtime. Application is fd-relative/walk-confined (`file_open_secure_parent` + `utimensat(..., AT_SYMLINK_NOFOLLOW)`) and best-effort per entry: an absent path (an intentionally uncreated empty dir) is skipped QUIETLY and only a real existing directory is stamped. `-O` (config boolean, already on the wire) makes the receiver skip the whole set. The single-threaded sink applies in `receiver_send_success_frame`; the `-j`/`--threads` sink accumulates in `write_thread` and server.c applies after both threads join and the deletion commits.
|
||||
- **Directory times.** The recursive scanner captures every traversed source directory's metadata (mtime, plus atime under `-U`) into a per-transfer list — two paths are covered: the sequential `DirectoryScanner` captures each opened directory (including the transfer root), and the parallel scanner captures both the root in `parallel_scanner_create_with_options` and each worker's subdirectories in `open_next_directory` (appends are guarded by a mutex shared with the sender's pipeline context). The sender transmits them in trailing `STATUS_DIR_TIMES` frames (each: int count + count × (wire path, metadata) pairs) sent **after all file data and after the optional delete manifest**, just before `STATUS_FINISHED`. A tree larger than `MAX_MANIFEST_ENTRIES` (1 048 576) directories is chunked into repeated frames, each within the receiver's per-frame bound. A dir-time entry is RECORD-ONLY (`file->dir_time_only`): `file_save_to_disk_full` returns `FILE_SAVE_SKIPPED` without creating anything (the directory's creation, when it is empty, is now carried by a separate `STATUS_MKDIR` entry the scanner emits for every directory that produced no transferred child, and `-m/--prune-empty-dirs` suppresses that). The receiver accumulates received directory metadata in a `DirTimeList` and applies it only at the very end — after the entire stream, after the commit-style `--delete` deletion, and after `--delay-updates` publication — because creating or removing a child bumps the parent's mtime. Application is fd-relative/walk-confined (`file_open_secure_parent` + `utimensat(..., AT_SYMLINK_NOFOLLOW)`) and best-effort per entry: an absent path (an intentionally uncreated empty dir) is skipped QUIETLY and only a real existing directory is stamped. `-O` (config boolean, already on the wire) makes the receiver skip the whole set. The single-threaded sink applies in `receiver_send_success_frame`; the `-j`/`--threads` sink accumulates in `write_thread` and server.c applies after both threads join and the deletion commits.
|
||||
- **Symlink times/owner/mode.** `STATUS_SYMLINK` already carried metadata; the receiver now applies it with no-follow primitives only: `utimensat(..., AT_SYMLINK_NOFOLLOW)`, best-effort `fchmodat(..., AT_SYMLINK_NOFOLLOW)` (honest no-op where unsupported, e.g. Linux), and policy-gated `fchownat(..., AT_SYMLINK_NOFOLLOW)` via a new `identity_apply_ownership_link` that shares the identity resolver with the fd path. `-J` suppresses only the timestamps; ownership stays governed by the identity opt-in (`--numeric-ids`/`--usermap`/`--groupmap`/`--chown`) exactly like regular files. A symlink has no children, so this is applied immediately at creation.
|
||||
- **Wire:** the shared `STATUS_DIR_TIMES` frame (and metadata on `STATUS_MKDIR` for `--dirs` entries) is a frame-sequence change, so `PROTOCOL_VERSION` was bumped **2.16.0 → 2.17.0**; every version-sensitive test (`--protocol` accepted/rejected values) was updated. The config-frame layout itself is unchanged (the omit booleans already crossed). Non-metadata and `--no-preserve` transfers send no `STATUS_DIR_TIMES` frame and no directory metadata, keeping them byte-identical.
|
||||
|
||||
@@ -893,7 +909,7 @@ These are the last compatibility items and the closing phase toward rsync flag p
|
||||
|
||||
**Wire:** two trailing config-frame blocks after the `--iconv` spec, in fixed order — `send_privilege_options`/`receive_privilege_options` (one `super_mode` int, validated `0..2`), then `send_copy_as_options`/`receive_copy_as_options` (presence int + two int32 ids, validated `>= 0`, with `copy_as_set ⇒ use_metadata`). `PROTOCOL_VERSION` bumped **2.17.0 → 2.18.0**. **Divergences from rsync:** rsync's `--super` elevates the receiver and `--copy-as` actually switches its credentials; FastSync never elevates and only permits/forwards confined attempts, and `--copy-as` forces ownership rather than switching identity.
|
||||
|
||||
**Honest status after the parity-completion wave (protocol 2.26.0).** ✅ Parity 106 / ⚠️ Caveat 27 / ❌ Divergent 23 = 156 rows. Earlier revisions of this document reported "143 ✅ / 0 divergence / 0 partial"; that conflated "parsed and tested" with "rsync parity", because many rows carried documented behavioral differences and some short options were not parsed at all. This reclassification makes every difference explicit. The completion wave closed 23 previously-caveated rows (9 that triage showed were already parity, plus 14 genuine fixes) and turned the 17 inherently non-rsync rows — native daemon config/auth, the FastSync batch container, the safe-subset device/privilege flags, `-X`'s privileged namespaces, `--fake-super`'s native xattr format, and the `--old-args` no-op — into explicit ❌ divergences. The remaining ⚠️ rows are the ones with a documented residual (see the row notes and the **Parity Completion Wave (protocol 2.26.0)** section below).
|
||||
**Honest status after the parity-completion wave (protocol 2.27.0), updated by the rsync-parity-stats, rsync-parity-options, rsync-parity-fs, and parity-review passes.** ✅ Parity 111 / ⚠️ Caveat 13 / ❌ Divergent 33 = 157 rows. Earlier revisions of this document reported "143 ✅ / 0 divergence / 0 partial"; that conflated "parsed and tested" with "rsync parity", because many rows carried documented behavioral differences and some short options were not parsed at all. This reclassification makes every difference explicit. The completion wave closed 23 previously-caveated rows (9 that triage showed were already parity, plus 14 genuine fixes) and turned the 17 inherently non-rsync rows — native daemon config/auth, the FastSync batch container, the safe-subset device/privilege flags, `-X`'s privileged namespaces, `--fake-super`'s native xattr format, and the `--old-args` no-op — into explicit ❌ divergences. The stats pass flipped `--delete-delay` to ✅ (actual-removal accounting), but the parity-review pass moved it back to ⚠️: FastSync charges the `--max-delete` budget at plan/snapshot time and leaves a refilled snapshotted directory in place, whereas rsync charges on actual removals and recursively removes a queued directory (including content created after its plan), so the two diverge when a snapshotted entry fails removal (see the `--delete-delay` row and the differential test). The stats pass also reclassified `--out-format` to ❌ (protocol-specific `%b`/delta-`%c`), and sharpened the `--stats`/`--progress`/`--checksum-choice` residuals. The options pass flipped `--bwlimit` and `--ignore-errors` to ✅ (rsync-exact size parsing and ~100 ms leaky-bucket throttling, and rsync's skip-unreadable-subdir plus IO-error-suppressed deletion with exit 23) and emits rsync-format `--info=name/flist/del/remove/nonreg/progress` lines (real-run `deleting`/`*deleting` carried over a new trailing `report_deletes` wire bool, `PROTOCOL_VERSION` 2.26.0 → 2.27.0), while reclassifying `-M` over daemon/TCP and receiver-side `protect`/`risk` re-derivation to ❌ (no argv channel / receiver filter engine). The fs pass flipped `-d/--dirs` and `--iconv` to ✅ — recursive transfers now recreate empty source directories (and replace a blocking destination non-directory with an incoming directory); `-R --no-implied-dirs --files-from` places a listed file under a missing implied parent with default attributes instead of refusing; and `--iconv` now reproduces rsync's push direction (destination charset = the spec's REMOTE half) — and reclassified six rows to ❌ after reproducing their exact residual with differential tests: `--temp-dir` (the receiver confines the scratch dir to the receive root, so an absolute temp dir is deliberately rejected although standalone rsync follows it), the three basis-dir options (FastSync xxHash-verifies a basis hit while rsync's `--size-only` quick check installs the wrong basis content), `--delay-updates` (fixed staging name wipes an unrelated destination entry of that name), and `--dry-run` (would-delete report over-reports). `--fuzzy` was also reclassified to ❌ (deterministic heuristic with a 10× size window, not rsync's matcher), but its residual is the candidate-selection heuristic itself: the final tree is byte-exact by design, so no destination differential can expose it and the row is pinned by the `TestFuzzy` threshold suite rather than a byte-level rsync differential. The remaining ⚠️ rows are the ones with a documented residual (see the row notes and the **Parity Completion Wave (protocol 2.26.0)** section below).
|
||||
|
||||
**Preserve-attribute split (protocol 2.21.0 → 2.22.0) — ✅ implemented.** FastSync splits the former single metadata bundle into four independent, rsync-compatible per-attribute flags — `-p/--perms`, `-t/--times`, `-o/--owner`, `-g/--group` — each with a negation (`--no-perms`/`--no-times`/`--no-owner`/`--no-group`, short `--no-p`/`--no-t`/`--no-o`/`--no-g`), plus `--no-preserve` clearing all four. `-a/--archive` is now full rsync `-rlptgoD` (owner and group included, though their application stays privilege-gated), `-A/--acls` implies `-p`, `-X/--xattrs` does not, `-E/--executability` sets only executability, and `-U`/`-N` do not imply `-t`. `--incremental`/`--delta` still auto-preserve perms+times unless the user explicitly negated them. Wire: the binary config frame gains four appended booleans (`preserve_perms`/`preserve_times`/`preserve_owner`/`preserve_group`) after `omit_link_times`, so `PROTOCOL_VERSION` is bumped **2.21.0 → 2.22.0**; the fixed-width `FileMetadata` layout is unchanged and the receiver gates the metadata frame on a derived `use_metadata`. Receiver behavior: each attribute is applied independently, directory modes are applied under `-p` (at the end of the transfer, alongside dir times), symlink mode under `-p`, and `-O/--omit-dir-times` suppresses directory times only. Documented divergences as of 2.22.0, **all but (d)/(e) removed by the rsync-parity wave (protocol 2.23.0)**: (a) the mode-masking divergence is **gone** — under `-p` the source mode is now copied exactly, including `S_IWGRP`/`S_IWOTH` and setuid/setgid/sticky; (b) a brand-new file without `-p` still gets `source_mode & ~umask` when metadata is present (else the historical fixed `0644`), and a new *directory* without `-p` still uses FastSync's `0755` default; (c) the `--chmod`-implies-`-p` divergence is **gone** — `--chmod` no longer implies `-p` (rsync parity); (d) `-o`/`-g` map by name on the receiver with a raw-numeric fallback (only numeric ids cross the wire); (e) a daemon module without `client owner = yes` does not refuse a plain `-a`/`-o`/`-g` — it forces super off, applies no ownership, and logs a warning, while explicit `--chown`/`--usermap`/`--groupmap`/`--numeric-ids`/`--copy-as`/`--super` are still refused.
|
||||
|
||||
@@ -1123,10 +1139,14 @@ wire protocol three times (full rationale in `src/shared/config.h`):
|
||||
|
||||
These remain after the wave; the individual rows carry the precise wording.
|
||||
|
||||
- **`--stats`** lacks rsync's `(reg/dir/link)` breakdown on `Number of files`
|
||||
and `Number of created files`; **`--progress`** omits the leading `./` line
|
||||
and its `to-chk` total differs by the root entry; **`--out-format`** `%b`/`%c`
|
||||
count FastSync wire bytes.
|
||||
- **`--stats`** now reproduces rsync's `(reg/dir/link/special)` breakdown on
|
||||
`Number of files`, the regular-transferred count and the size totals, but
|
||||
`Number of created files` is the transferred-regular count without a type
|
||||
breakdown and wire-byte totals differ; **`--progress`** now prints the leading
|
||||
`./` line and includes the root in `to-chk` (single-file output is
|
||||
byte-identical), but a multi-directory `to-chk` denominator and per-directory
|
||||
name lines still differ; **`--out-format`** `%C` matches for every algorithm,
|
||||
but `%b`/`%c` count FastSync wire bytes (protocol-specific, hence ❌).
|
||||
- **`-n --delete`** ordering can differ from rsync's delete-during walk and a
|
||||
filtered dry-run can over-report.
|
||||
- **Delete timing:** the default `--delete` remains delete-after rather than
|
||||
@@ -1136,13 +1156,14 @@ These remain after the wave; the individual rows carry the precise wording.
|
||||
sender-derived). `--ignore-errors` exits 23 but its EACCES differential is not
|
||||
exercised in CI.
|
||||
- **`--delay-updates`** uses a fixed staging name with an advisory lock and
|
||||
deletes before publication; **`--temp-dir`** rejects absolute/foreign paths;
|
||||
**`--remote-option`** is SSH-only; **`--iconv`** keeps the receiver
|
||||
half-swap/charset-declaration caveat.
|
||||
deletes before publication; **`--temp-dir`** rejects absolute/foreign paths
|
||||
(deliberately confined, see the row); **`--remote-option`** is SSH-only.
|
||||
**`--iconv`** now matches rsync's push direction (destination charset = the
|
||||
spec's REMOTE half; a server `--iconv` overrides it).
|
||||
- **Basis dirs** do not re-apply attributes on a match, keep the
|
||||
`--size-only` mtime caveat, and share the 256 MiB whole-file cap; **`--fuzzy`**
|
||||
has a different tie-break order; recursive transfers still do not create empty
|
||||
directories; and **`--bwlimit`** rejects rsync's `0`/decimal/suffixed rates.
|
||||
has a different tie-break order; and **`--bwlimit`** rejects rsync's
|
||||
`0`/decimal/suffixed rates.
|
||||
- **`--inc-recursive`/`--no-inc-recursive`** are not implemented (rejected).
|
||||
|
||||
### Intentional divergences (explicit ❌ rows)
|
||||
|
||||
+54
-21
@@ -1,5 +1,6 @@
|
||||
#include "change_list.h"
|
||||
#include "checksum.h"
|
||||
#include "log.h"
|
||||
#include "utils.h"
|
||||
#include <fcntl.h>
|
||||
#include <limits.h>
|
||||
@@ -70,7 +71,8 @@ static bool strbuf_append(StrBuf* buf, const char* text) {
|
||||
|
||||
bool change_list_enabled(const Config* config) {
|
||||
return config != NULL && (config->itemize_changes || config->out_format != NULL ||
|
||||
(config->log_file != NULL && config->log_file_format != NULL));
|
||||
(config->log_file != NULL && config->log_file_format != NULL) ||
|
||||
(config->info_level & LOG_INFO_NAME) != 0);
|
||||
}
|
||||
|
||||
/* ---- Itemize code ---- */
|
||||
@@ -192,27 +194,38 @@ char* change_render_itemize(const Config* config, const ChangeEvent* event) {
|
||||
return line.data;
|
||||
}
|
||||
|
||||
/* ---- --out-format / --log-file-format ---- */
|
||||
|
||||
/* rsync 3.4.1's `%C` uses the negotiated transfer checksum; with the default
|
||||
* "auto" choice on both ends that is xxh128. FastSync's internal XXH64 default
|
||||
* is not an rsync algorithm, so map it to xxh128 for parity. */
|
||||
static ChecksumAlgo out_format_checksum_algo(const Config* config) {
|
||||
switch ((ChecksumAlgo)config->checksum_algo) {
|
||||
case CHECKSUM_ALGO_MD5:
|
||||
return CHECKSUM_ALGO_MD5;
|
||||
case CHECKSUM_ALGO_XXH3:
|
||||
return CHECKSUM_ALGO_XXH3;
|
||||
case CHECKSUM_ALGO_XXH128:
|
||||
return CHECKSUM_ALGO_XXH128;
|
||||
case CHECKSUM_ALGO_XXH64:
|
||||
default:
|
||||
return CHECKSUM_ALGO_XXH128;
|
||||
/* rsync's `--info=name` line for an updated entry: the transfer-relative name
|
||||
* (trailing slash for directories) plus the ` -> target` / ` => target` link
|
||||
* suffix. `--info=name` does not alter an itemize/out-format run. */
|
||||
static char* change_render_name(const ChangeEvent* event) {
|
||||
StrBuf line = {0};
|
||||
bool ok = append_name(&line, event) && append_link_suffix(&line, event);
|
||||
if (!ok) {
|
||||
strbuf_free(&line);
|
||||
return NULL;
|
||||
}
|
||||
if (line.data == NULL) {
|
||||
line.data = str_dup("");
|
||||
if (!line.data)
|
||||
return NULL;
|
||||
}
|
||||
return line.data;
|
||||
}
|
||||
|
||||
/* Render a digest as rsync's sum_as_hex: for xxh128 the HIGH 64-bit half is
|
||||
* printed before the low half; every other algorithm prints its bytes in order. */
|
||||
/* ---- --out-format / --log-file-format ---- */
|
||||
|
||||
/* rsync 3.4.1's `%C` uses the negotiated TRANSFER checksum (the first name of a
|
||||
* two-name "transfer,pre-transfer" --checksum-choice), not the pre-transfer
|
||||
* whole-file digest FastSync compares against on the wire. The default "auto"
|
||||
* resolves to xxh128, so an explicit selection and the default both render the
|
||||
* selected algorithm's digest. */
|
||||
static ChecksumAlgo out_format_checksum_algo(const Config* config) {
|
||||
return (ChecksumAlgo)config->checksum_transfer_algo;
|
||||
}
|
||||
|
||||
/* Render a digest as rsync's sum_as_hex: xxh128 prints the HIGH 64-bit half
|
||||
* before the low half, and xxh64/xxh3 print their 64-bit value big-endian; every
|
||||
* other algorithm prints its bytes in order. */
|
||||
static void digest_to_hex(ChecksumAlgo algo, const uint8_t* digest, size_t len, char* out) {
|
||||
if (algo == CHECKSUM_ALGO_XXH128 && len == 16) {
|
||||
uint64_t low = 0;
|
||||
@@ -222,6 +235,12 @@ static void digest_to_hex(ChecksumAlgo algo, const uint8_t* digest, size_t len,
|
||||
snprintf(out, len * 2 + 1, "%016llx%016llx", (unsigned long long)high, (unsigned long long)low);
|
||||
return;
|
||||
}
|
||||
if ((algo == CHECKSUM_ALGO_XXH64 || algo == CHECKSUM_ALGO_XXH3) && len == 8) {
|
||||
uint64_t value = 0;
|
||||
memcpy(&value, digest, sizeof(value));
|
||||
snprintf(out, len * 2 + 1, "%016llx", (unsigned long long)value);
|
||||
return;
|
||||
}
|
||||
static const char hex[] = "0123456789abcdef";
|
||||
for (size_t i = 0; i < len; i++) {
|
||||
out[i * 2] = hex[(digest[i] >> 4) & 0xf];
|
||||
@@ -260,6 +279,9 @@ static void fill_event_checksum(const Config* config, const File* file, ChangeEv
|
||||
if (file->path == NULL)
|
||||
return;
|
||||
ChecksumAlgo algo = out_format_checksum_algo(config);
|
||||
/* rsync renders `--checksum-choice=none` as a blank 2-character column. */
|
||||
if (algo == CHECKSUM_ALGO_NONE)
|
||||
return;
|
||||
uint8_t digest[CHECKSUM_MAX_DIGEST_LEN];
|
||||
size_t len = 0;
|
||||
/* rsync's %C is the transfer checksum, which is always seeded with 0 (it is
|
||||
@@ -329,9 +351,10 @@ char* change_render_format(const char* format, const Config* config, const Chang
|
||||
if (event->checksum_known) {
|
||||
ok = strbuf_append(&line, event->checksum);
|
||||
} else {
|
||||
/* rsync pads a non-regular / untransferred entry with spaces. */
|
||||
/* rsync pads a non-regular / untransferred / `none` entry with spaces;
|
||||
`none` renders as a blank 2-character column. */
|
||||
ChecksumAlgo algo = out_format_checksum_algo(config);
|
||||
int width = checksum_digest_len(algo) * 2;
|
||||
int width = algo == CHECKSUM_ALGO_NONE ? 2 : checksum_digest_len(algo) * 2;
|
||||
for (int i = 0; i < width && ok; i++)
|
||||
ok = strbuf_append_char(&line, ' ');
|
||||
}
|
||||
@@ -450,6 +473,16 @@ void change_emit(const Config* config, const ChangeEvent* event) {
|
||||
print_escaped_line(stdout, line, config->eight_bit_output);
|
||||
free(line);
|
||||
}
|
||||
} else if ((config->info_level & LOG_INFO_NAME) != 0 &&
|
||||
!(config->show_progress || (config->info_level & LOG_INFO_PROGRESS))) {
|
||||
/* --info=name without -i/--out-format: print the updated entry's name. The
|
||||
--progress path owns the name line when progress output is active (it
|
||||
emits the same names before the progress frames), so do not duplicate. */
|
||||
char* line = change_render_name(event);
|
||||
if (line != NULL) {
|
||||
print_escaped_line(stdout, line, config->eight_bit_output);
|
||||
free(line);
|
||||
}
|
||||
}
|
||||
if (to_log) {
|
||||
char* line = change_render_format(config->log_file_format, config, event);
|
||||
|
||||
+149
-18
@@ -23,6 +23,7 @@
|
||||
#include <langinfo.h>
|
||||
#include <limits.h>
|
||||
#include <locale.h>
|
||||
#include <math.h>
|
||||
#include <time.h>
|
||||
#include <signal.h>
|
||||
#include <stdbool.h>
|
||||
@@ -501,7 +502,10 @@ static bool is_accepted_debug_category(const char* name) {
|
||||
|
||||
static bool is_accepted_info_category(const char* name) {
|
||||
static const char* const categories[] = {
|
||||
"backup", "del", "flist", "mount", "nonreg", "progress", "remove", "syms", "symsafe",
|
||||
"backup",
|
||||
"mount",
|
||||
"syms",
|
||||
"symsafe",
|
||||
};
|
||||
for (size_t i = 0; i < sizeof(categories) / sizeof(categories[0]); i++) {
|
||||
if (strcmp(name, categories[i]) == 0)
|
||||
@@ -608,14 +612,26 @@ static int parse_info_flags(const char* value, Config* config) {
|
||||
free(flags);
|
||||
return 1;
|
||||
}
|
||||
if (strcmp(name, "copy") == 0 || strcmp(name, "name") == 0)
|
||||
if (strcmp(name, "copy") == 0)
|
||||
flag = LOG_INFO_COPY;
|
||||
else if (strcmp(name, "name") == 0)
|
||||
flag = LOG_INFO_NAME;
|
||||
else if (strcmp(name, "misc") == 0)
|
||||
flag = LOG_INFO_MISC;
|
||||
else if (strcmp(name, "skip") == 0)
|
||||
flag = LOG_INFO_SKIP;
|
||||
else if (strcmp(name, "stats") == 0)
|
||||
flag = LOG_INFO_STATS;
|
||||
else if (strcmp(name, "del") == 0)
|
||||
flag = LOG_INFO_DEL;
|
||||
else if (strcmp(name, "remove") == 0)
|
||||
flag = LOG_INFO_REMOVE;
|
||||
else if (strcmp(name, "flist") == 0)
|
||||
flag = LOG_INFO_FLIST;
|
||||
else if (strcmp(name, "nonreg") == 0)
|
||||
flag = LOG_INFO_NONREG;
|
||||
else if (strcmp(name, "progress") == 0)
|
||||
flag = LOG_INFO_PROGRESS;
|
||||
else if (is_accepted_info_category(name))
|
||||
continue;
|
||||
else {
|
||||
@@ -1819,22 +1835,130 @@ static int set_log_file_option(Config* config, const char* log_path) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Apply a --bwlimit value (kilobytes per second). Returns 0 on success, -1 on
|
||||
* error. */
|
||||
/* Faithful port of rsync 3.4.1's `parse_size_arg(bwlimit_arg, 'K', "bwlimit",
|
||||
* 512, -1, True)`: a default KiB suffix, binary (1024) multipliers unless a
|
||||
* `b`/`B` decimal suffix or explicit `iB` is given, an optional decimal
|
||||
* fraction, the P/T/G/M/K suffixes, and the special rules that a value of 0
|
||||
* means "no limit" while any other value below 512 bytes is rejected. The
|
||||
* parsed byte count is then quantized to whole KiB exactly like rsync's
|
||||
* `bwlimit = (size + 512) / 1024`. Returns 0 on success, -1 on a parse error. */
|
||||
static int parse_bwlimit_value(const char* value, unsigned long long* bytes_per_sec_out) {
|
||||
if (!value || !bytes_per_sec_out)
|
||||
return -1;
|
||||
const char* arg = value;
|
||||
int reps;
|
||||
long long mult;
|
||||
while (*arg >= '0' && *arg <= '9')
|
||||
arg++;
|
||||
if (*arg != '\0' && (*arg == '.' || *arg == localeconv()->decimal_point[0]))
|
||||
for (arg++; *arg >= '0' && *arg <= '9'; arg++) {
|
||||
}
|
||||
|
||||
char suffix = *arg && *arg != '+' && *arg != '-' ? *arg++ : 'K';
|
||||
switch (suffix) {
|
||||
case 'b':
|
||||
case 'B':
|
||||
reps = 0;
|
||||
break;
|
||||
case 'k':
|
||||
case 'K':
|
||||
reps = 1;
|
||||
break;
|
||||
case 'm':
|
||||
case 'M':
|
||||
reps = 2;
|
||||
break;
|
||||
case 'g':
|
||||
case 'G':
|
||||
reps = 3;
|
||||
break;
|
||||
case 't':
|
||||
case 'T':
|
||||
reps = 4;
|
||||
break;
|
||||
case 'p':
|
||||
case 'P':
|
||||
reps = 5;
|
||||
break;
|
||||
default:
|
||||
log_message(LOG_LEVEL_ERROR, "--bwlimit=%s is invalid", value);
|
||||
return -1;
|
||||
}
|
||||
if (*arg == 'b' || *arg == 'B') {
|
||||
mult = 1000;
|
||||
arg++;
|
||||
} else if (*arg == '\0' || *arg == '+' || *arg == '-') {
|
||||
mult = 1024;
|
||||
} else if ((arg[0] == 'i' || arg[0] == 'I') && (arg[1] == 'b' || arg[1] == 'B')) {
|
||||
mult = 1024;
|
||||
arg += 2;
|
||||
} else {
|
||||
log_message(LOG_LEVEL_ERROR, "--bwlimit=%s is invalid", value);
|
||||
return -1;
|
||||
}
|
||||
|
||||
long long base = 1;
|
||||
for (int i = 0; i < reps; i++) {
|
||||
if (base > LLONG_MAX / mult) {
|
||||
log_message(LOG_LEVEL_ERROR, "--bwlimit=%s is too large", value);
|
||||
return -1;
|
||||
}
|
||||
base *= mult;
|
||||
}
|
||||
/* rsync multiplies the numeric prefix (atof) by mult^reps in a signed
|
||||
* ssize_t, which is undefined on overflow. Scale in double and range-check
|
||||
* before converting, so a huge value is rejected as "too large" (where
|
||||
* rsync's overflow happens to land on a negative result) without invoking
|
||||
* signed-overflow UB. */
|
||||
double scaled = (double)base * strtod(value, NULL);
|
||||
/* (double)LLONG_MAX rounds up to 2^63, which is itself out of range for the
|
||||
* cast, so reject at >= that bound; LLONG_MIN == -2^63 is exactly
|
||||
* representable and thus castable, so the lower bound stays strict. */
|
||||
if (!isfinite(scaled) || scaled >= (double)LLONG_MAX || scaled < (double)LLONG_MIN) {
|
||||
log_message(LOG_LEVEL_ERROR, "--bwlimit=%s is too large", value);
|
||||
return -1;
|
||||
}
|
||||
long long size = (long long)scaled;
|
||||
if ((*arg == '+' || *arg == '-') && arg[1] == '1' && arg != value) {
|
||||
/* The only form accepted here is "+1"/"-1" (a longer number leaves a
|
||||
trailing byte and is rejected below), so apply the delta directly and
|
||||
guard the one overflow direction. */
|
||||
if (*arg == '+') {
|
||||
if (size == LLONG_MAX) {
|
||||
log_message(LOG_LEVEL_ERROR, "--bwlimit=%s is too large", value);
|
||||
return -1;
|
||||
}
|
||||
size += 1;
|
||||
} else {
|
||||
size -= 1;
|
||||
}
|
||||
arg += 2;
|
||||
}
|
||||
if (*arg != '\0' || size < 0) {
|
||||
log_message(LOG_LEVEL_ERROR, "--bwlimit=%s is %s", value, size < 0 ? "too large" : "invalid");
|
||||
return -1;
|
||||
}
|
||||
if (size != 0 && size < 512) {
|
||||
log_message(LOG_LEVEL_ERROR, "--bwlimit=%s is too small (min: 512 or 0 for unlimited)", value);
|
||||
return -1;
|
||||
}
|
||||
long long kib = size == 0 ? 0 : (size + 512) / 1024;
|
||||
if (kib > (long long)(ULLONG_MAX / 1024)) {
|
||||
log_message(LOG_LEVEL_ERROR, "--bwlimit=%s is too large", value);
|
||||
return -1;
|
||||
}
|
||||
*bytes_per_sec_out = (unsigned long long)kib * 1024;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Apply a --bwlimit value using rsync 3.4.1's units/semantics. Returns 0 on
|
||||
* success, -1 on error. */
|
||||
static int set_bwlimit_option(const char* value) {
|
||||
unsigned long long kbps;
|
||||
if (parse_ull_arg(value, &kbps, "--bwlimit") != 0)
|
||||
unsigned long long bytes_per_sec;
|
||||
if (parse_bwlimit_value(value, &bytes_per_sec) != 0)
|
||||
return -1;
|
||||
if (kbps == 0) {
|
||||
log_message(LOG_LEVEL_ERROR, "--bwlimit must be a positive integer");
|
||||
return -1;
|
||||
}
|
||||
if (kbps > ULLONG_MAX / 1024) {
|
||||
log_message(LOG_LEVEL_ERROR, "--bwlimit value too large");
|
||||
return -1;
|
||||
}
|
||||
io_set_bwlimit(kbps * 1024);
|
||||
log_info_message(LOG_INFO_MISC, "Set bandwidth limit to %llu KB/s", kbps);
|
||||
io_set_bwlimit(bytes_per_sec);
|
||||
log_info_message(LOG_INFO_MISC, "Set bandwidth limit to %llu KB/s", bytes_per_sec / 1024);
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -2549,8 +2673,15 @@ static int cli_finalize_config(Config* config, bool verbose, bool no_delta, bool
|
||||
}
|
||||
}
|
||||
}
|
||||
config->report_stats = config->stats || config->show_progress || format_needs_wire ||
|
||||
(config->dry_run && config->use_delete);
|
||||
/* --info=del on a real --delete run asks the receiver to report the paths it
|
||||
actually removed; the report rides the STATUS_STATS path list, so the wire
|
||||
stats frame must be negotiated too. */
|
||||
config->report_deletes = config->use_delete && !config->dry_run &&
|
||||
((config->info_level & LOG_INFO_DEL) != 0 || config->itemize_changes ||
|
||||
config->out_format != NULL);
|
||||
config->report_stats = config->stats || config->show_progress ||
|
||||
(config->info_level & LOG_INFO_PROGRESS) || format_needs_wire ||
|
||||
config->report_deletes || (config->dry_run && config->use_delete);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
+325
-164
@@ -66,6 +66,16 @@ static void log_server_rejection(const char* context) {
|
||||
}
|
||||
}
|
||||
|
||||
/* rsync's --ignore-errors semantics: an I/O error during the transfer normally
|
||||
* suppresses deletion entirely ("IO error encountered -- skipping file
|
||||
* deletion"); --ignore-errors lets the deletion run anyway. FastSync always
|
||||
* continues past an unreadable subdirectory so the readable tree transfers, and
|
||||
* always reports the partial transfer (exit 23); this only decides whether the
|
||||
* deletion phase is skipped. Returns true when deletion may proceed. */
|
||||
bool ignore_errors_allows_delete(const Config* config, bool had_io_error) {
|
||||
return !had_io_error || (config && config->ignore_errors);
|
||||
}
|
||||
|
||||
static const char* display_bytes(unsigned long long bytes, bool human_readable, char* buffer,
|
||||
size_t buffer_size) {
|
||||
if (human_readable && format_human_size_decimal(bytes, buffer, buffer_size))
|
||||
@@ -83,15 +93,53 @@ static const char* stats_bytes(const Config* config, unsigned long long bytes, c
|
||||
return buffer;
|
||||
}
|
||||
|
||||
/* Print the rsync `--stats` block on stdout. Byte totals use the process-wide
|
||||
wire counters and the receiver-only counters come from the STATUS_STATS frame;
|
||||
the labels, layout and rate/speedup formulas match rsync 3.4.1. Shared by the
|
||||
single-threaded and multithreaded send paths. */
|
||||
static void report_transfer_stats(const Config* config, int total_files,
|
||||
unsigned long long total_bytes, time_t start,
|
||||
/* Build rsync's `Number of files` parenthetical: each non-zero category, in
|
||||
reg/dir/link/special order. Empty when the flist counted nothing. */
|
||||
static void stats_type_breakdown(const TransferStats* stats, char* out, size_t out_size) {
|
||||
unsigned long long total =
|
||||
stats->flist_reg + stats->flist_dir + stats->flist_link + stats->flist_special;
|
||||
if (total == 0) {
|
||||
out[0] = '\0';
|
||||
return;
|
||||
}
|
||||
out[0] = '\0';
|
||||
size_t used = 0;
|
||||
const struct {
|
||||
const char* name;
|
||||
unsigned long long count;
|
||||
} parts[4] = {{"reg", stats->flist_reg},
|
||||
{"dir", stats->flist_dir},
|
||||
{"link", stats->flist_link},
|
||||
{"special", stats->flist_special}};
|
||||
bool first = true;
|
||||
for (size_t i = 0; i < 4; i++) {
|
||||
if (parts[i].count == 0)
|
||||
continue;
|
||||
int written = snprintf(out + used, out_size - used, "%s%s: %llu", first ? "(" : ", ",
|
||||
parts[i].name, parts[i].count);
|
||||
if (written < 0 || (size_t)written >= out_size - used)
|
||||
break;
|
||||
used += (size_t)written;
|
||||
first = false;
|
||||
}
|
||||
if (!first && used + 1 < out_size)
|
||||
out[used++] = ')';
|
||||
out[used] = '\0';
|
||||
}
|
||||
|
||||
/* Print the rsync `--stats` block on stdout. The source-side flist and
|
||||
transferred counters come from `stats` (filled while scanning/sending), the
|
||||
receiver-only counters from the STATUS_STATS frame, and the wire byte totals
|
||||
from the process-wide protocol counters. The labels, layout and
|
||||
rate/speedup formulas match rsync 3.4.1. Shared by the single-threaded and
|
||||
multithreaded send paths. */
|
||||
static void report_transfer_stats(const Config* config, const TransferStats* stats, time_t start,
|
||||
const ReceiverStats* recv) {
|
||||
if (!config->stats || config->quiet)
|
||||
return;
|
||||
TransferStats empty = {0};
|
||||
if (stats == NULL)
|
||||
stats = ∅
|
||||
ReceiverStats none = {0};
|
||||
if (recv == NULL)
|
||||
recv = &none;
|
||||
@@ -101,11 +149,18 @@ static void report_transfer_stats(const Config* config, int total_files,
|
||||
double elapsed = difftime(time(NULL), start);
|
||||
double rate = (double)(sent + received) / (0.5 + elapsed);
|
||||
char total_buffer[32];
|
||||
char transferred_buffer[32];
|
||||
char literal_buffer[32];
|
||||
char sent_buffer[32];
|
||||
char recv_buffer[32];
|
||||
char rate_buffer[32] = {0};
|
||||
char human_rate[32] = {0};
|
||||
const char* total = stats_bytes(config, total_bytes, total_buffer, sizeof(total_buffer));
|
||||
const char* total =
|
||||
stats_bytes(config, stats->total_file_size, total_buffer, sizeof(total_buffer));
|
||||
const char* transferred = stats_bytes(config, stats->transferred_file_size, transferred_buffer,
|
||||
sizeof(transferred_buffer));
|
||||
const char* literal =
|
||||
stats_bytes(config, stats->literal_data, literal_buffer, sizeof(literal_buffer));
|
||||
const char* sent_s = stats_bytes(config, sent, sent_buffer, sizeof(sent_buffer));
|
||||
const char* recv_s = stats_bytes(config, received, recv_buffer, sizeof(recv_buffer));
|
||||
const char* rate_str = rate_buffer;
|
||||
@@ -116,15 +171,26 @@ static void report_transfer_stats(const Config* config, int total_files,
|
||||
} else {
|
||||
snprintf(rate_buffer, sizeof(rate_buffer), "%.2f", rate);
|
||||
}
|
||||
double speedup = (sent + received) > 0 ? (double)total_bytes / (double)(sent + received) : 0.0;
|
||||
double speedup =
|
||||
(sent + received) > 0 ? (double)stats->total_file_size / (double)(sent + received) : 0.0;
|
||||
char breakdown[128];
|
||||
stats_type_breakdown(stats, breakdown, sizeof(breakdown));
|
||||
unsigned long long flist_total =
|
||||
stats->flist_reg + stats->flist_dir + stats->flist_link + stats->flist_special;
|
||||
printf("\n");
|
||||
printf("Number of files: %d\n", total_files);
|
||||
printf("Number of created files: %d\n", total_files);
|
||||
if (breakdown[0] != '\0')
|
||||
printf("Number of files: %llu %s\n", flist_total, breakdown);
|
||||
else
|
||||
printf("Number of files: %llu\n", flist_total);
|
||||
/* FastSync cannot tell which entries the receiver newly created, so it
|
||||
reports the transferred regular files (which are created on a fresh
|
||||
destination). See RSYNC_COMPAT.md for the documented residual. */
|
||||
printf("Number of created files: %llu\n", stats->transferred_regular);
|
||||
printf("Number of deleted files: %llu\n", recv->deleted_files);
|
||||
printf("Number of regular files transferred: %d\n", total_files);
|
||||
printf("Number of regular files transferred: %llu\n", stats->transferred_regular);
|
||||
printf("Total file size: %s bytes\n", total);
|
||||
printf("Total transferred file size: %s bytes\n", total);
|
||||
printf("Literal data: %s bytes\n", total);
|
||||
printf("Total transferred file size: %s bytes\n", transferred);
|
||||
printf("Literal data: %s bytes\n", literal);
|
||||
printf("Matched data: %llu bytes\n", recv->matched_data);
|
||||
printf("File list size: 0\n");
|
||||
printf("File list generation time: 0.000 seconds\n");
|
||||
@@ -138,6 +204,48 @@ static void report_transfer_stats(const Config* config, int total_files,
|
||||
fflush(stdout);
|
||||
}
|
||||
|
||||
/* Classify one scanned source entry into the rsync flist counters. Called for
|
||||
every entry the sender walks, transferred or skipped. Directory entries are
|
||||
counted here only for the explicit -d/--dirs generator; a recursive scan's
|
||||
directories are accounted from the scanner's dir_entries list at report time. */
|
||||
static void transfer_stats_note_entry(TransferStats* stats, const File* file) {
|
||||
if (stats == NULL || file == NULL)
|
||||
return;
|
||||
if (file->is_dir) {
|
||||
stats->flist_dir++;
|
||||
return;
|
||||
}
|
||||
if (file->is_symlink) {
|
||||
stats->flist_link++;
|
||||
stats->total_file_size += file->symlink_target ? strlen(file->symlink_target) : 0;
|
||||
return;
|
||||
}
|
||||
if (file->is_special) {
|
||||
stats->flist_special++;
|
||||
return;
|
||||
}
|
||||
stats->flist_reg++;
|
||||
stats->total_file_size += file->data ? file->data->size : 0;
|
||||
}
|
||||
|
||||
/* Account for a regular file (or a whole-file append) the receiver actually
|
||||
stored: rsync's transferred-file count and transferred/literal byte totals.
|
||||
`literal_data` counts the whole source size, which is exact for a whole-file
|
||||
send but an upper bound for a delta send (the receiver reuses basis blocks
|
||||
the sender never ships); see TransferStats.literal_data in format.h. */
|
||||
static void transfer_stats_note_transferred(TransferStats* stats, const File* file) {
|
||||
if (stats == NULL || file == NULL)
|
||||
return;
|
||||
if (file->is_dir || file->is_symlink || file->is_special)
|
||||
return;
|
||||
if (file->link_group != 0 && !file->link_first)
|
||||
return;
|
||||
unsigned long long size = file->data ? file->data->size : 0;
|
||||
stats->transferred_regular++;
|
||||
stats->transferred_file_size += size;
|
||||
stats->literal_data += size;
|
||||
}
|
||||
|
||||
/* ---- rsync-style per-file --progress ------------------------------------
|
||||
* rsync prints, for each transferred regular file, the file name followed by a
|
||||
* two-frame progress line: the first at the initial 32 KiB read window (always
|
||||
@@ -188,19 +296,77 @@ static void progress_final_frame(unsigned long long size, char* out, size_t out_
|
||||
unsigned long long remain = (unsigned long long)(diff_ms / 1000);
|
||||
snprintf(rembuf, sizeof(rembuf), "%4u:%02u:%02u", (unsigned)(remain / 3600),
|
||||
(unsigned)((remain / 60) % 60), (unsigned)(remain % 60));
|
||||
/* rsync's `to-chk` denominator is the whole file list, which includes the
|
||||
transfer-root directory FastSync never emits as a transfer entry. Count
|
||||
that root entry so a single-file transfer matches rsync exactly. */
|
||||
unsigned long long total = g_progress_seen + 1;
|
||||
unsigned long long to_chk =
|
||||
g_progress_seen > g_progress_xferred ? g_progress_seen - g_progress_xferred : 0;
|
||||
snprintf(out, out_size, "\r%15s %3d%% %7.2f%s %s (xfr#%llu, to-chk=%llu/%llu)\n", ofs_buf, 100,
|
||||
rate, units, rembuf, g_progress_xferred, to_chk, g_progress_seen);
|
||||
rate, units, rembuf, g_progress_xferred, to_chk, total);
|
||||
}
|
||||
|
||||
static bool info_flag_enabled(const Config* config, LogInfoFlag flag) {
|
||||
return config != NULL && (config->info_level & flag) != 0;
|
||||
}
|
||||
|
||||
/* Print rsync's deletion lines for a received list of destination-relative
|
||||
* paths: `*deleting PATH` when itemizing, the --out-format expansion when a
|
||||
* format is set, else `deleting PATH` for --info=del. Used by both the dry-run
|
||||
* would-delete report and the real --info=del report. */
|
||||
static void print_delete_reports(const Config* config, const ArrayList* paths) {
|
||||
if (!config || !paths || config->quiet)
|
||||
return;
|
||||
if (!(config->itemize_changes || config->out_format != NULL ||
|
||||
info_flag_enabled(config, LOG_INFO_DEL)))
|
||||
return;
|
||||
for (int i = 0; i < paths->size; i++) {
|
||||
const char* raw = (const char*)paths->items[i];
|
||||
const char* path = delete_display_path(config, raw);
|
||||
if (config->out_format != NULL) {
|
||||
ChangeEvent event;
|
||||
memset(&event, 0, sizeof(event));
|
||||
event.decision = CHANGE_SENT;
|
||||
event.deleted = true;
|
||||
event.name = path;
|
||||
event.path = path;
|
||||
char* line = change_render_format(config->out_format, config, &event);
|
||||
if (line) {
|
||||
char* escaped = output_escape(line, config->eight_bit_output);
|
||||
printf("%s\n", escaped ? escaped : line);
|
||||
free(escaped);
|
||||
free(line);
|
||||
}
|
||||
} else {
|
||||
char* escaped = output_escape(path, config->eight_bit_output);
|
||||
if (config->itemize_changes)
|
||||
printf("*deleting %s\n", escaped ? escaped : path);
|
||||
else
|
||||
printf("deleting %s\n", escaped ? escaped : path);
|
||||
free(escaped);
|
||||
}
|
||||
}
|
||||
fflush(stdout);
|
||||
}
|
||||
|
||||
static void client_progress_begin(const Config* config) {
|
||||
g_progress_active = config->show_progress && !config->quiet;
|
||||
g_progress_active =
|
||||
(config->show_progress || info_flag_enabled(config, LOG_INFO_PROGRESS)) && !config->quiet;
|
||||
g_progress_xferred = 0;
|
||||
g_progress_seen = 0;
|
||||
if (!g_progress_active)
|
||||
if (!g_progress_active) {
|
||||
/* `--info=flist` prints rsync's file-list header even without progress. */
|
||||
if (!config->quiet && info_flag_enabled(config, LOG_INFO_FLIST)) {
|
||||
printf("sending incremental file list\n");
|
||||
fflush(stdout);
|
||||
}
|
||||
return;
|
||||
}
|
||||
printf("sending incremental file list\n");
|
||||
/* rsync prints the transfer-root directory's name before the first file when
|
||||
that directory is created; FastSync mirrors the source root below the
|
||||
receive root and creates it on a fresh destination, so emit it here. */
|
||||
printf("./\n");
|
||||
fflush(stdout);
|
||||
}
|
||||
|
||||
@@ -309,6 +475,13 @@ static bool prepare_scanner(const Config* config, int num_threads, PreparedScann
|
||||
options->exclude_per_dir_filter_files = config->per_dir_filter_count >= 2;
|
||||
options->dirs = config->dirs;
|
||||
options->relative = config->relative;
|
||||
/* A real recursive transfer recreates empty source directories (rsync
|
||||
parity); low-level scanner users leave this off. */
|
||||
options->emit_empty_dirs = true;
|
||||
/* --no-implied-dirs only has meaning with -R (rsync): without it the option
|
||||
is a documented no-op, so the scanner must not suppress directory
|
||||
metadata. */
|
||||
options->no_implied_dirs = config->no_implied_dirs && config->relative;
|
||||
/* -R/--relative outside --files-from reconstructs every destination path from
|
||||
* the source spec (rsync's '/./' cut point). With --files-from the listed
|
||||
* entry already supplies the bare relative path, so no prefix is built. */
|
||||
@@ -325,6 +498,9 @@ static bool prepare_scanner(const Config* config, int num_threads, PreparedScann
|
||||
options->prune_empty_dirs = config->prune_empty_dirs;
|
||||
options->ignore_io_errors = config->ignore_errors;
|
||||
options->ignore_missing_args = config->ignore_missing_args || config->delete_missing_args;
|
||||
options->note_nonreg = (config->info_level & LOG_INFO_NONREG) != 0 && !config->quiet;
|
||||
options->send_directory = config->send_directory;
|
||||
options->eight_bit_output = config->eight_bit_output;
|
||||
options->excluded_paths = NULL;
|
||||
options->excluded_mutex = NULL;
|
||||
options->size_skipped_paths = NULL;
|
||||
@@ -469,72 +645,6 @@ static const char* delete_plan_walk_root(const Config* config, const ArrayList*
|
||||
return marker;
|
||||
}
|
||||
|
||||
/* True when some --files-from entry is an ancestor-or-equal directory of
|
||||
* `rel` (an empty entry -- the whole tree "." -- counts as the root). */
|
||||
static bool file_list_ancestor_listed(const FileListSet* set, const char* rel) {
|
||||
if (!set)
|
||||
return true;
|
||||
for (int i = 0; i < set->count; i++) {
|
||||
const char* listed = set->entries[i];
|
||||
if (listed[0] == '\0')
|
||||
return true;
|
||||
size_t n = strlen(listed);
|
||||
if (strncmp(rel, listed, n) == 0 && (rel[n] == '/' || rel[n] == '\0'))
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/* --no-implied-dirs (meaningful only with -R + --files-from): a listed file
|
||||
* may only be placed when its parent directory (or one of its ancestors) is
|
||||
* itself an explicitly listed entry. rsync omits a file whose implied parent
|
||||
* directory is suppressed, and an explicitly listed file that cannot be placed
|
||||
* fails the transfer; FastSync fails the whole run up front with a clear error
|
||||
* (it has no per-entry skip channel). Without -R or --files-from the option
|
||||
* has no effect. */
|
||||
static bool no_implied_dirs_files_from_valid(const Config* config) {
|
||||
if (!config->no_implied_dirs || !config->relative)
|
||||
return true;
|
||||
const FileListSet* set = (const FileListSet*)config->files_from_set;
|
||||
if (!set)
|
||||
return true;
|
||||
for (int i = 0; i < set->count; i++) {
|
||||
const char* entry = set->entries[i];
|
||||
if (entry[0] == '\0')
|
||||
continue;
|
||||
char* full = path_cat(config->send_directory, entry);
|
||||
if (!full)
|
||||
return false;
|
||||
struct stat st;
|
||||
bool is_file = lstat(full, &st) == 0 && S_ISREG(st.st_mode);
|
||||
free(full);
|
||||
if (!is_file)
|
||||
continue;
|
||||
const char* slash = strrchr(entry, '/');
|
||||
if (!slash)
|
||||
continue; /* top-level file: its parent is the receive root */
|
||||
size_t parent_len = (size_t)(slash - entry);
|
||||
if (parent_len == 0)
|
||||
continue;
|
||||
char* parent = malloc(parent_len + 1);
|
||||
if (!parent)
|
||||
return false;
|
||||
memcpy(parent, entry, parent_len);
|
||||
parent[parent_len] = '\0';
|
||||
bool listed = file_list_ancestor_listed(set, parent);
|
||||
if (!listed) {
|
||||
log_message(LOG_LEVEL_ERROR,
|
||||
"--no-implied-dirs: cannot place file '%s': parent directory '%s' is not "
|
||||
"explicitly listed (list the directory or drop --no-implied-dirs)",
|
||||
entry, parent);
|
||||
}
|
||||
free(parent);
|
||||
if (!listed)
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/* The destination-relative mirror path for a missing --files-from entry: where
|
||||
a PRESENT entry with the same name would have been written. With -R that is
|
||||
the entry's bare relative path (the bare wire path the receiver uses);
|
||||
@@ -648,7 +758,7 @@ static bool files_from_list_check(const Config* config, ArrayList* missing_dest,
|
||||
"--ignore-missing-args: ignored %d missing --files-from entr%s", *skipped_out,
|
||||
*skipped_out == 1 ? "y" : "ies");
|
||||
}
|
||||
return no_implied_dirs_files_from_valid(config);
|
||||
return true;
|
||||
}
|
||||
|
||||
/* Basis directories are honored by the receiver's per-file incremental check,
|
||||
@@ -870,6 +980,13 @@ static void remove_transferred_sources(const Config* config, ArrayList* paths) {
|
||||
log_message(LOG_LEVEL_WARNING, "Could not remove source file %s",
|
||||
escaped_path ? escaped_path : "<allocation failed>");
|
||||
free(escaped_path);
|
||||
} else if (info_flag_enabled(config, LOG_INFO_REMOVE) && !config->quiet) {
|
||||
/* rsync's --info=remove line: the transfer-relative name. */
|
||||
const char* rel = delete_display_path(config, source->path);
|
||||
char* escaped = output_escape(rel, config->eight_bit_output);
|
||||
printf("sender removed %s\n", escaped ? escaped : rel);
|
||||
free(escaped);
|
||||
fflush(stdout);
|
||||
}
|
||||
close(dirfd);
|
||||
}
|
||||
@@ -959,20 +1076,7 @@ static bool receive_stats_record(int fd, ReceiverStats* stats, ArrayList* would_
|
||||
static const char* delete_display_path(const Config* config, const char* path) {
|
||||
if (!config || !path || !config->send_directory)
|
||||
return path;
|
||||
const char* root = config->send_directory;
|
||||
while (*root == '/')
|
||||
root++;
|
||||
const char* rel = path;
|
||||
while (*rel == '/')
|
||||
rel++;
|
||||
size_t root_len = strlen(root);
|
||||
while (root_len > 0 && root[root_len - 1] == '/')
|
||||
root_len--;
|
||||
if (root_len == 0)
|
||||
return rel;
|
||||
if (strncmp(rel, root, root_len) == 0 && (rel[root_len] == '/' || rel[root_len] == '\0'))
|
||||
return rel + root_len + (rel[root_len] == '/' ? 1 : 0);
|
||||
return rel;
|
||||
return utils_strip_transfer_root(path, config->send_directory);
|
||||
}
|
||||
|
||||
/* Send the final STATUS_FINISHED frame and await the receiver's verdict.
|
||||
@@ -993,8 +1097,19 @@ static bool finalize_transfer(Client* client, const Config* config, ArrayList* r
|
||||
return false;
|
||||
if (status == STATUS_STATS) {
|
||||
ReceiverStats scratch;
|
||||
if (!receive_stats_record(client->file_descriptor, stats_out ? stats_out : &scratch, NULL))
|
||||
/* A real --info=del run carries the actually-removed paths in the stats
|
||||
frame's path list; collect and print them in rsync's format. */
|
||||
ArrayList* deleted = config->report_deletes ? array_list_create(free) : NULL;
|
||||
if (config->report_deletes && !deleted)
|
||||
return false;
|
||||
if (!receive_stats_record(client->file_descriptor, stats_out ? stats_out : &scratch, deleted)) {
|
||||
array_list_delete(deleted);
|
||||
return false;
|
||||
}
|
||||
if (deleted) {
|
||||
print_delete_reports(config, deleted);
|
||||
array_list_delete(deleted);
|
||||
}
|
||||
if (!receive_status(client->file_descriptor, &status))
|
||||
return false;
|
||||
}
|
||||
@@ -1437,8 +1552,11 @@ static bool scan_paths_only(const Config* config, const ScannerOptions* options,
|
||||
ArrayList* manifest, DeletePlanSender* plans, bool* io_error_out) {
|
||||
if (io_error_out)
|
||||
*io_error_out = false;
|
||||
DirectoryScanner* scanner =
|
||||
directory_scanner_create_with_options(config->send_directory, options);
|
||||
ScannerOptions local = *options;
|
||||
/* The pre-scan is a paths-only pass with no client output; it must not emit
|
||||
--info=nonreg lines (the data pass does that once). */
|
||||
local.note_nonreg = false;
|
||||
DirectoryScanner* scanner = directory_scanner_create_with_options(config->send_directory, &local);
|
||||
if (!scanner)
|
||||
return false;
|
||||
bool ok = true;
|
||||
@@ -1942,37 +2060,7 @@ static int send_dry_run_remote(Config* config) {
|
||||
array_list_delete(would_delete);
|
||||
goto dry_fail;
|
||||
}
|
||||
/* rsync prints `*deleting PATH` when itemizing (or `deleting PATH` with
|
||||
--out-format / -v); the plain-total output used here has no delete
|
||||
counterpart, so only the itemize/out-format cases are rendered. */
|
||||
if (!config->quiet && (config->itemize_changes || config->out_format != NULL)) {
|
||||
for (int i = 0; i < would_delete->size; i++) {
|
||||
const char* raw = (const char*)would_delete->items[i];
|
||||
const char* path = delete_display_path(config, raw);
|
||||
if (config->out_format != NULL) {
|
||||
ChangeEvent event;
|
||||
memset(&event, 0, sizeof(event));
|
||||
event.decision = CHANGE_SENT;
|
||||
event.deleted = true;
|
||||
event.name = path;
|
||||
event.path = path;
|
||||
char* line = change_render_format(config->out_format, config, &event);
|
||||
if (line) {
|
||||
/* Escape the whole rendered line, exactly like change_emit() does
|
||||
for a real transfer, so a control byte in the peer-supplied path
|
||||
cannot forge output. */
|
||||
char* escaped = output_escape(line, config->eight_bit_output);
|
||||
printf("%s\n", escaped ? escaped : line);
|
||||
free(escaped);
|
||||
free(line);
|
||||
}
|
||||
} else {
|
||||
char* escaped = output_escape(path, config->eight_bit_output);
|
||||
printf("*deleting %s\n", escaped ? escaped : path);
|
||||
free(escaped);
|
||||
}
|
||||
}
|
||||
}
|
||||
print_delete_reports(config, would_delete);
|
||||
array_list_delete(would_delete);
|
||||
if (!receive_status(client->file_descriptor, &status))
|
||||
goto dry_fail;
|
||||
@@ -1986,7 +2074,16 @@ static int send_dry_run_remote(Config* config) {
|
||||
else
|
||||
printf("Total: %d files, %.1f MB\n", file_count, (double)total_bytes / (double)BYTES_PER_MIB);
|
||||
}
|
||||
report_transfer_stats(config, file_count, total_bytes, dry_start, &dry_stats);
|
||||
{
|
||||
TransferStats dry_transfer;
|
||||
memset(&dry_transfer, 0, sizeof(dry_transfer));
|
||||
dry_transfer.flist_reg = (unsigned long long)file_count;
|
||||
dry_transfer.total_file_size = total_bytes;
|
||||
dry_transfer.transferred_regular = (unsigned long long)file_count;
|
||||
dry_transfer.transferred_file_size = total_bytes;
|
||||
dry_transfer.literal_data = total_bytes;
|
||||
report_transfer_stats(config, &dry_transfer, dry_start, &dry_stats);
|
||||
}
|
||||
ret = io_error ? 1 : 0;
|
||||
|
||||
dry_fail:
|
||||
@@ -2236,7 +2333,7 @@ static bool source_is_regular_file(const File* file) {
|
||||
}
|
||||
|
||||
static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config,
|
||||
ArrayList* remove_sources) {
|
||||
ArrayList* remove_sources, TransferStats* stats) {
|
||||
if (config->use_chunk_serialization) {
|
||||
if (remove_sources) {
|
||||
for (int i = 0; i < chunk->element_count; i++) {
|
||||
@@ -2263,10 +2360,13 @@ static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config,
|
||||
for (int i = 0; i < chunk->element_count; i++) {
|
||||
if (chunk->items[i] == NULL)
|
||||
continue;
|
||||
transfer_stats_note_entry(stats, chunk->items[i]);
|
||||
if (chunk->items[i]->is_dir)
|
||||
change_emit_dir_sent(config, chunk->items[i]);
|
||||
else
|
||||
change_emit_file_sent(config, chunk->items[i]);
|
||||
if (!chunk->items[i]->is_dir)
|
||||
transfer_stats_note_transferred(stats, chunk->items[i]);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
@@ -2275,6 +2375,7 @@ static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config,
|
||||
File* f = chunk->items[i];
|
||||
if (f == NULL)
|
||||
continue;
|
||||
transfer_stats_note_entry(stats, f);
|
||||
if (f->is_dir) {
|
||||
/* Explicit directory entry (--dirs): a MKDIR frame carrying the
|
||||
destination path (and metadata when negotiated). Directories have no
|
||||
@@ -2328,6 +2429,7 @@ static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config,
|
||||
source_file_destroy(source);
|
||||
return -1;
|
||||
}
|
||||
transfer_stats_note_transferred(stats, f);
|
||||
change_emit_file_sent_bytes(config, f, protocol_bytes_written() - bytes_before,
|
||||
protocol_bytes_read() - read_before);
|
||||
client_progress_file(config, f);
|
||||
@@ -2436,7 +2538,7 @@ static int send_chunks_multithreaded(void* pipeline_context) {
|
||||
return thrd_error;
|
||||
}
|
||||
if (send_chunk_with_removal(client, current_chunk, context->config,
|
||||
context->remove_source_files) != 0) {
|
||||
context->remove_source_files, &context->stats) != 0) {
|
||||
log_message(LOG_LEVEL_ERROR, "unexpected error while sending chunk");
|
||||
chunk_destroy(current_chunk);
|
||||
pipeline_cancel(context);
|
||||
@@ -2482,7 +2584,8 @@ static int send_chunks_multithreaded(void* pipeline_context) {
|
||||
"unscanned source mirrors are not deleted");
|
||||
else
|
||||
log_message(LOG_LEVEL_WARNING, "transfer stopped early (stop deadline)");
|
||||
} else if (context->config->use_delete && !context->early_delete && !context->delete_plans) {
|
||||
} else if (context->config->use_delete && !context->early_delete && !context->delete_plans &&
|
||||
!context->delete_suppressed) {
|
||||
/* Empty keep-set + scan I/O error must not delete the whole destination
|
||||
(the source may not be genuinely empty -- see send_files). */
|
||||
bool empty_io;
|
||||
@@ -2495,12 +2598,20 @@ static int send_chunks_multithreaded(void* pipeline_context) {
|
||||
"with an empty keep-set (--delete)");
|
||||
goto send_fail;
|
||||
}
|
||||
if (send_delete_manifest(client->file_descriptor, context->manifest, context->excluded_paths,
|
||||
context->size_skipped_paths, context->missing_args,
|
||||
context->synced_dirs) != 0)
|
||||
/* rsync default: an I/O error suppresses deletion unless --ignore-errors.
|
||||
The keep-set manifest is not sent, so the receiver removes nothing. */
|
||||
mtx_lock(&context->mutex_scanner);
|
||||
bool scan_io_now = context->scan_had_io_error;
|
||||
mtx_unlock(&context->mutex_scanner);
|
||||
if (!ignore_errors_allows_delete(context->config, scan_io_now)) {
|
||||
log_message(LOG_LEVEL_WARNING, "IO error encountered -- skipping file deletion");
|
||||
} else if (send_delete_manifest(client->file_descriptor, context->manifest,
|
||||
context->excluded_paths, context->size_skipped_paths,
|
||||
context->missing_args, context->synced_dirs) != 0) {
|
||||
goto send_fail;
|
||||
}
|
||||
} else if (context->config->delete_missing_args && !context->early_delete &&
|
||||
!context->delete_plans) {
|
||||
!context->delete_suppressed && !context->delete_plans) {
|
||||
/* --delete-missing-args without --delete: no keep-set is built, but the
|
||||
exact-delete paths still ride the same manifest frame (commit once the
|
||||
transfer succeeded). */
|
||||
@@ -2533,13 +2644,12 @@ static int send_chunks_multithreaded(void* pipeline_context) {
|
||||
"server reported a deletion failure (--delete); see the server log for the reason");
|
||||
if (ok)
|
||||
remove_transferred_sources(context->config, context->remove_source_files);
|
||||
mtx_lock(&context->mutex_progress);
|
||||
int total_files = context->total_files;
|
||||
unsigned long long total_bytes = context->total_bytes;
|
||||
mtx_unlock(&context->mutex_progress);
|
||||
report_transfer_stats(context->config, total_files, total_bytes, start, &recv_stats);
|
||||
log_info_message(LOG_INFO_STATS, "Transfer summary: %d files, %.1f MB", total_files,
|
||||
(double)total_bytes / (double)BYTES_PER_MIB);
|
||||
if (context->dir_entries)
|
||||
context->stats.flist_dir += (unsigned long long)context->dir_entries->size;
|
||||
report_transfer_stats(context->config, &context->stats, start, &recv_stats);
|
||||
log_info_message(LOG_INFO_STATS, "Transfer summary: %llu files, %.1f MB",
|
||||
context->stats.transferred_regular,
|
||||
(double)context->stats.transferred_file_size / (double)BYTES_PER_MIB);
|
||||
disconnect_transfer_client(client);
|
||||
mark_sender_done(context);
|
||||
protocol_session_unbind();
|
||||
@@ -2628,7 +2738,8 @@ static int scan_directory_multithreaded(void* pipeline_context) {
|
||||
failed = use_dscanner ? directory_scanner_failed(dscanner) : parallel_scanner_failed(scanner);
|
||||
break;
|
||||
}
|
||||
if (context->config->use_delete && !context->early_delete && !context->delete_plans) {
|
||||
if (context->config->use_delete && !context->early_delete && !context->delete_plans &&
|
||||
!context->delete_suppressed) {
|
||||
mtx_lock(&context->mutex_scanner);
|
||||
bool manifest_ok = add_chunk_to_manifest(context->manifest, current_chunk);
|
||||
mtx_unlock(&context->mutex_scanner);
|
||||
@@ -2921,6 +3032,7 @@ int send_files(Config* config) {
|
||||
bool prescan_ok =
|
||||
scan_paths_only(config, &prepared.options, early_manifest, NULL, &had_scan_io);
|
||||
bool early_ok = false;
|
||||
bool skip_delete = false;
|
||||
if (prescan_ok) {
|
||||
/* A scan that hit an I/O error and produced NO keep entries is ambiguous
|
||||
(the source may not be genuinely empty -- part of it was unreadable),
|
||||
@@ -2932,6 +3044,11 @@ int send_files(Config* config) {
|
||||
"source scan hit an I/O error before finding any file; refusing to delete "
|
||||
"with an empty keep-set (--delete)");
|
||||
prescan_ok = false;
|
||||
} else if (!ignore_errors_allows_delete(config, had_scan_io)) {
|
||||
/* rsync default: an I/O error suppresses deletion unless
|
||||
--ignore-errors. Skip the manifest; the transfer still proceeds. */
|
||||
log_message(LOG_LEVEL_WARNING, "IO error encountered -- skipping file deletion");
|
||||
skip_delete = true;
|
||||
} else {
|
||||
early_ok = send_delete_manifest_early(client, early_manifest, excluded, size_skipped,
|
||||
missing_args, synced_dirs);
|
||||
@@ -2943,7 +3060,7 @@ int send_files(Config* config) {
|
||||
prepared.options.excluded_paths = NULL;
|
||||
prepared.options.size_skipped_paths = NULL;
|
||||
prepared.options.synced_dirs = NULL;
|
||||
if (!prescan_ok || !early_ok)
|
||||
if (!prescan_ok || (!early_ok && !skip_delete))
|
||||
goto send_fail;
|
||||
} else if (delete_per_dir) {
|
||||
/* --delete-during/--delete-delay: build one plan per source directory from a
|
||||
@@ -2957,6 +3074,7 @@ int send_files(Config* config) {
|
||||
prepared.options.plan_dirs = plan_dirs;
|
||||
bool prescan_ok = scan_paths_only(config, &prepared.options, NULL, plan_sender, &had_scan_io);
|
||||
bool plans_ok = false;
|
||||
bool skip_delete = false;
|
||||
if (prescan_ok) {
|
||||
const char* walk_root = delete_plan_walk_root(config, synced_dirs);
|
||||
const ArrayList* scope =
|
||||
@@ -2968,6 +3086,15 @@ int send_files(Config* config) {
|
||||
"source scan hit an I/O error before finding any file; refusing to delete "
|
||||
"with an empty keep-set (--delete)");
|
||||
prescan_ok = false;
|
||||
} else if (!ignore_errors_allows_delete(config, had_scan_io)) {
|
||||
/* rsync default: an I/O error suppresses deletion unless
|
||||
--ignore-errors. Drop the plans; the transfer still proceeds. */
|
||||
log_message(LOG_LEVEL_WARNING, "IO error encountered -- skipping file deletion");
|
||||
delete_plan_sender_destroy(plan_sender);
|
||||
plan_sender = NULL;
|
||||
array_list_delete(plan_dirs);
|
||||
plan_dirs = NULL;
|
||||
skip_delete = true;
|
||||
} else {
|
||||
plans_ok = delete_plan_send_root(client->file_descriptor, plan_sender) == 0;
|
||||
}
|
||||
@@ -2976,7 +3103,7 @@ int send_files(Config* config) {
|
||||
prepared.options.size_skipped_paths = NULL;
|
||||
prepared.options.synced_dirs = NULL;
|
||||
prepared.options.plan_dirs = NULL;
|
||||
if (!prescan_ok || !plans_ok)
|
||||
if (!prescan_ok || (!plans_ok && !skip_delete))
|
||||
goto send_fail;
|
||||
} else if (config->use_delete) {
|
||||
manifest = array_list_create(free);
|
||||
@@ -3003,8 +3130,8 @@ int send_files(Config* config) {
|
||||
goto send_fail;
|
||||
|
||||
Chunk* current_chunk;
|
||||
unsigned long long total_bytes = 0;
|
||||
int total_files = 0;
|
||||
TransferStats transfer_stats;
|
||||
memset(&transfer_stats, 0, sizeof(transfer_stats));
|
||||
time_t start = time(NULL);
|
||||
client_progress_begin(config);
|
||||
/* True when the stop deadline cut the scan short so the keep-set manifest is
|
||||
@@ -3030,11 +3157,6 @@ int send_files(Config* config) {
|
||||
scan_stopped_early = true;
|
||||
break;
|
||||
}
|
||||
unsigned long long chunk_bytes = 0;
|
||||
for (int i = 0; i < current_chunk->element_count; i++) {
|
||||
chunk_bytes += current_chunk->items[i]->data->size;
|
||||
total_files++;
|
||||
}
|
||||
if (manifest && !add_chunk_to_manifest(manifest, current_chunk)) {
|
||||
chunk_destroy(current_chunk);
|
||||
goto send_fail;
|
||||
@@ -3061,13 +3183,13 @@ int send_files(Config* config) {
|
||||
send_failed = true;
|
||||
break;
|
||||
}
|
||||
if (send_chunk_with_removal(client, current_chunk, config, remove_sources) != 0) {
|
||||
if (send_chunk_with_removal(client, current_chunk, config, remove_sources, &transfer_stats) !=
|
||||
0) {
|
||||
log_message(LOG_LEVEL_ERROR, "Failed to send chunk");
|
||||
chunk_destroy(current_chunk);
|
||||
send_failed = true;
|
||||
break;
|
||||
}
|
||||
total_bytes += chunk_bytes;
|
||||
chunk_destroy(current_chunk);
|
||||
}
|
||||
if (send_failed) {
|
||||
@@ -3111,7 +3233,18 @@ int send_files(Config* config) {
|
||||
"an empty keep-set (--delete)");
|
||||
goto send_fail;
|
||||
}
|
||||
if ((manifest || config->delete_missing_args) && !delete_early && !delete_per_dir) {
|
||||
/* rsync default: a scan I/O error suppresses deletion unless
|
||||
--ignore-errors, even in the late (commit) modes. Drop the keep-set so
|
||||
the receiver removes nothing; the readable tree still transferred. */
|
||||
bool late_delete =
|
||||
(manifest || config->delete_missing_args) && !delete_early && !delete_per_dir;
|
||||
if (late_delete && !ignore_errors_allows_delete(config, had_scan_io)) {
|
||||
log_message(LOG_LEVEL_WARNING, "IO error encountered -- skipping file deletion");
|
||||
if (manifest) {
|
||||
array_list_delete(manifest);
|
||||
manifest = NULL;
|
||||
}
|
||||
} else if (late_delete) {
|
||||
/* Late (commit) ordering: all file data is out; transmit the manifest so
|
||||
the receiver commits the extras walk (--delete) and/or the
|
||||
--delete-missing-args exact-path deletions only after the transfer
|
||||
@@ -3152,9 +3285,16 @@ int send_files(Config* config) {
|
||||
"server reported a deletion failure (--delete); see the server log for the reason");
|
||||
if (ok)
|
||||
remove_transferred_sources(config, remove_sources);
|
||||
report_transfer_stats(config, total_files, total_bytes, start, &recv_stats);
|
||||
log_info_message(LOG_INFO_STATS, "Transfer summary: %d files, %.1f MB", total_files,
|
||||
(double)total_bytes / (double)BYTES_PER_MIB);
|
||||
/* A recursive -a scan has no directory entries in its chunks; account them
|
||||
from the scanner's captured directory list (present whenever a directory
|
||||
attribute is preserved, e.g. -a/-t/-p). The -d generator counts its
|
||||
explicit directory entries inline instead. */
|
||||
if (dir_entries)
|
||||
transfer_stats.flist_dir += (unsigned long long)dir_entries->size;
|
||||
report_transfer_stats(config, &transfer_stats, start, &recv_stats);
|
||||
log_info_message(LOG_INFO_STATS, "Transfer summary: %llu files, %.1f MB",
|
||||
transfer_stats.transferred_regular,
|
||||
(double)transfer_stats.transferred_file_size / (double)BYTES_PER_MIB);
|
||||
/* A skipped source entry (--ignore-errors past an unreadable directory, or a
|
||||
dereferenced symlink with no referent) makes rsync report a partial
|
||||
transfer (exit 23) even though the rest of the run succeeded. A
|
||||
@@ -3358,8 +3498,28 @@ int send_files_multithreaded(Config** config_ptr) {
|
||||
pipeline_context_sender_destroy(context);
|
||||
return 1;
|
||||
}
|
||||
if (!per_dir)
|
||||
if (context->scan_had_io_error && !ignore_errors_allows_delete(config, true)) {
|
||||
/* rsync default: an I/O error suppresses deletion unless
|
||||
--ignore-errors. Drop the prebuilt keep-set so nothing is sent; the
|
||||
data pass still transfers the readable tree and exits 23. */
|
||||
log_message(LOG_LEVEL_WARNING, "IO error encountered -- skipping file deletion");
|
||||
if (context->manifest) {
|
||||
array_list_delete(context->manifest);
|
||||
context->manifest = NULL;
|
||||
}
|
||||
if (context->delete_plans) {
|
||||
delete_plan_sender_destroy(context->delete_plans);
|
||||
context->delete_plans = NULL;
|
||||
}
|
||||
if (context->plan_dirs) {
|
||||
array_list_delete(context->plan_dirs);
|
||||
context->plan_dirs = NULL;
|
||||
}
|
||||
/* A later --delete pass must not try to rebuild/send a keep-set. */
|
||||
context->delete_suppressed = true;
|
||||
} else if (!per_dir) {
|
||||
context->early_delete = true;
|
||||
}
|
||||
} else {
|
||||
context->manifest = array_list_create(free);
|
||||
if (!context->manifest) {
|
||||
@@ -3370,7 +3530,8 @@ int send_files_multithreaded(Config** config_ptr) {
|
||||
}
|
||||
if (config->remove_source_files)
|
||||
context->remove_source_files = array_list_create(source_file_destroy);
|
||||
if ((config->use_delete && !context->manifest && !context->delete_plans) ||
|
||||
if ((config->use_delete && !context->manifest && !context->delete_plans &&
|
||||
!context->delete_suppressed) ||
|
||||
(config->remove_source_files && !context->remove_source_files)) {
|
||||
pipeline_context_sender_destroy(context);
|
||||
return 1;
|
||||
|
||||
@@ -23,6 +23,12 @@ void client_set_abort_armed(bool armed);
|
||||
* config_delete() once the call returns). */
|
||||
int send_files(Config* config);
|
||||
int send_files_multithreaded(Config** config);
|
||||
/* rsync's --ignore-errors deletion gate: with no I/O error during the scan the
|
||||
* deletion phase always proceeds; with one it is suppressed unless
|
||||
* `--ignore-errors` was given. Exposed so the decision can be unit-tested
|
||||
* without a privileged (mode-000) source directory. See client_send.c. */
|
||||
bool ignore_errors_allows_delete(const Config* config, bool had_io_error);
|
||||
|
||||
/* Phase 6 residual-batch (client-only). See client_send.c. */
|
||||
int write_batch_from_source(const Config* config, const char* batch_path);
|
||||
int apply_batch_to_dest(const Config* config, const char* batch_path, const char* dest_root);
|
||||
|
||||
+95
-10
@@ -432,6 +432,19 @@ static void scanner_record_protected(DirectoryScanner* scanner, const char* fs_p
|
||||
scanner->failed = true;
|
||||
}
|
||||
|
||||
/* rsync's `--info=nonreg` line for a non-regular entry that is not being
|
||||
* preserved: `skipping non-regular file "NAME"`. The name is the path relative
|
||||
* to the transfer root, so it matches rsync's displayed name. */
|
||||
static void scanner_note_nonreg(const ScannerOptions* options, const char* fs_path) {
|
||||
if (!options || !options->note_nonreg || !fs_path)
|
||||
return;
|
||||
const char* rel = utils_strip_transfer_root(fs_path, options->send_directory);
|
||||
char* escaped = output_escape(rel, options->eight_bit_output);
|
||||
printf("skipping non-regular file \"%s\"\n", escaped ? escaped : rel);
|
||||
free(escaped);
|
||||
fflush(stdout);
|
||||
}
|
||||
|
||||
/* A user-selection exclusion (--filter/-C/per-dir or --exclude/--include). */
|
||||
static void scanner_record_excluded(DirectoryScanner* scanner, const char* fs_path) {
|
||||
scanner_record_protected(scanner, fs_path, scanner->options.excluded_paths);
|
||||
@@ -830,7 +843,8 @@ static bool scanner_capture_dir_time(ArrayList* dir_entries, mtx_t* mutex, const
|
||||
const char* fs_path, bool relative_mode,
|
||||
const char* relative_prefix, bool preserve_atimes,
|
||||
bool preserve_crtimes, bool preserve_xattrs,
|
||||
bool preserve_acls) {
|
||||
bool preserve_acls, bool no_implied_dirs,
|
||||
const FileListSet* file_list) {
|
||||
if (!dir_entries || !root_path || !fs_path)
|
||||
return true;
|
||||
struct stat st;
|
||||
@@ -839,6 +853,13 @@ static bool scanner_capture_dir_time(ArrayList* dir_entries, mtx_t* mutex, const
|
||||
char* rel = scanner_path_relative(root_path, fs_path);
|
||||
if (!rel)
|
||||
return true;
|
||||
/* --no-implied-dirs: an implied parent directory (not listed, and not under
|
||||
a listed directory) keeps the destination's own/default attributes, so its
|
||||
source metadata is not transmitted. */
|
||||
if (no_implied_dirs && file_list && !file_list_dir_in_scope(file_list, rel)) {
|
||||
free(rel);
|
||||
return true;
|
||||
}
|
||||
if (relative_mode && rel[0] == '\0') {
|
||||
/* -R + --files-from: the transfer root itself has no bare relative wire
|
||||
path (matches the -R scan, which never emits the root). */
|
||||
@@ -902,6 +923,43 @@ static bool scanner_capture_dir_time(ArrayList* dir_entries, mtx_t* mutex, const
|
||||
return true;
|
||||
}
|
||||
|
||||
/* Recursive scan: emit a payload-less directory entry for the directory that
|
||||
* just finished scanning. rsync creates every source directory at the
|
||||
* destination; FastSync otherwise creates one only implicitly through a
|
||||
* transferred child, so a directory emptied on the transfer side (physically
|
||||
* empty, or all of its entries filtered out) would never appear. The transfer
|
||||
* root is skipped (it maps to the receive root, which already exists), as are
|
||||
* --files-from (only listed items and their implied parents transfer),
|
||||
* --list-only (directory lines are emitted by the caller) and
|
||||
* -m/--prune-empty-dirs. Returns false on allocation failure. */
|
||||
static bool scanner_emit_empty_dir(DirectoryScanner* scanner, ArrayList* chunk_data) {
|
||||
if (!scanner->current_path || !scanner->current_rel || scanner->current_rel[0] == '\0')
|
||||
return true;
|
||||
struct stat st;
|
||||
if (lstat(scanner->current_path, &st) != 0 || !S_ISDIR(st.st_mode))
|
||||
return true;
|
||||
File* dir = scanner_build_dir_file(scanner->current_path, &st, &scanner->options);
|
||||
if (!dir)
|
||||
return false;
|
||||
if (scanner->relative_mode) {
|
||||
dir->send_path = str_dup(scanner->current_rel);
|
||||
} else if (scanner->options.relative_prefix) {
|
||||
dir->send_path =
|
||||
scanner_prefix_send_path(scanner->options.relative_prefix, scanner->current_rel);
|
||||
}
|
||||
if ((scanner->relative_mode || scanner->options.relative_prefix) && !dir->send_path) {
|
||||
file_destroy(dir);
|
||||
return false;
|
||||
}
|
||||
if (scanner->options.preserve_xattrs || scanner->options.preserve_acls)
|
||||
dir->xattrs = xattr_capture_path(scanner->current_path, scanner->options.preserve_acls);
|
||||
if (!array_list_add(chunk_data, dir)) {
|
||||
file_destroy(dir);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/* Open the next queued directory and set up its filter context. Returns 1 when
|
||||
a directory is open, 0 when the queue is exhausted, and -1 on a fatal error.
|
||||
A directory that cannot be opened is an I/O error: it is recorded on the
|
||||
@@ -925,6 +983,7 @@ static int open_next_directory(DirectoryScanner* scanner) {
|
||||
* the directory that enqueued them. */
|
||||
const FilterNode* inherited = scanner->at_seed_dir ? scanner->seed_node : de->context;
|
||||
scanner->at_seed_dir = false;
|
||||
scanner->current_dir_produced = false;
|
||||
free(de);
|
||||
|
||||
free(scanner->current_rel);
|
||||
@@ -954,11 +1013,18 @@ static int open_next_directory(DirectoryScanner* scanner) {
|
||||
scanner->current_rel = NULL;
|
||||
free(scanner->current_path);
|
||||
scanner->current_path = NULL;
|
||||
if (!scanner->options.ignore_io_errors || is_root_seed) {
|
||||
if (is_root_seed) {
|
||||
/* The transfer ROOT being unreadable is always fatal: an empty keep-set
|
||||
would delete the whole destination. Mark the scan as errored so the
|
||||
client can report the partial-transfer exit code (rsync's 23). */
|
||||
scanner->root_io_error = true;
|
||||
scanner->failed = true;
|
||||
return -1;
|
||||
}
|
||||
/* --ignore-errors: record the I/O error and keep scanning the rest. */
|
||||
/* A subdirectory that cannot be opened is always skipped (rsync continues
|
||||
with a partial transfer), whether or not --ignore-errors is set. The
|
||||
error is recorded so the client exits 23; --ignore-errors only changes
|
||||
what the deletion phase does with the recorded error. */
|
||||
continue;
|
||||
}
|
||||
if (open_directory_filter_context(scanner, inherited) != 0) {
|
||||
@@ -985,7 +1051,8 @@ static int open_next_directory(DirectoryScanner* scanner) {
|
||||
scanner->options.dir_entries, scanner->options.dir_entries_mutex, scanner->root_path,
|
||||
scanner->current_path, scanner->relative_mode, scanner->options.relative_prefix,
|
||||
scanner->options.preserve_atimes, scanner->options.preserve_crtimes,
|
||||
scanner->options.preserve_xattrs, scanner->options.preserve_acls)) {
|
||||
scanner->options.preserve_xattrs, scanner->options.preserve_acls,
|
||||
scanner->options.no_implied_dirs, scanner->options.file_list)) {
|
||||
closedir(scanner->current_dir);
|
||||
scanner->current_dir = NULL;
|
||||
free(scanner->current_path);
|
||||
@@ -1350,10 +1417,22 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
|
||||
|
||||
const struct dirent* entry = readdir(scanner->current_dir);
|
||||
if (entry == NULL) {
|
||||
/* The directory is exhausted: if nothing was transferred or descended
|
||||
from it, recreate it at the destination as an explicit entry. */
|
||||
if (scanner->options.emit_empty_dirs && !scanner->current_dir_produced &&
|
||||
!scanner->options.prune_empty_dirs && !scanner->options.list_dirs &&
|
||||
scanner->options.file_list == NULL) {
|
||||
if (!scanner_emit_empty_dir(scanner, chunk_data))
|
||||
scanner->failed = true;
|
||||
}
|
||||
closedir(scanner->current_dir);
|
||||
scanner->current_dir = NULL;
|
||||
free(scanner->current_path);
|
||||
scanner->current_path = NULL;
|
||||
if (scanner->failed) {
|
||||
array_list_delete(chunk_data);
|
||||
return NULL;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -1488,6 +1567,7 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
|
||||
scanner->failed = true;
|
||||
break;
|
||||
}
|
||||
scanner->current_dir_produced = true;
|
||||
free(cur_path);
|
||||
continue;
|
||||
}
|
||||
@@ -1502,6 +1582,7 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
scanner->current_dir_produced = true;
|
||||
int next_depth = scanner->current_depth + 1;
|
||||
if (scanner->options.max_depth <= 0 || next_depth < scanner->options.max_depth) {
|
||||
DirEntry* de = dir_entry_create(cur_path, next_depth, scanner->current_node);
|
||||
@@ -1554,6 +1635,7 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
|
||||
scanner->options.preserve_specials,
|
||||
scanner->options.copy_devices, file, &stats);
|
||||
if (special == SCANNER_SPECIAL_SKIP) {
|
||||
scanner_note_nonreg(&scanner->options, file->path);
|
||||
free(rel_copy);
|
||||
file_destroy(file);
|
||||
continue;
|
||||
@@ -1577,6 +1659,7 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
|
||||
scanner->failed = true;
|
||||
break;
|
||||
}
|
||||
scanner->current_dir_produced = true;
|
||||
chunk_data_size += file->data->size;
|
||||
if (chunk_data_size > scanner->options.chunk_size) {
|
||||
free(rel_copy);
|
||||
@@ -1604,7 +1687,7 @@ bool directory_scanner_failed(const DirectoryScanner* scanner) {
|
||||
}
|
||||
|
||||
bool directory_scanner_had_io_error(const DirectoryScanner* scanner) {
|
||||
return scanner != NULL && scanner->io_error;
|
||||
return scanner != NULL && (scanner->io_error || scanner->root_io_error);
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
@@ -1974,6 +2057,7 @@ static void scan_root_entry(const ScannerOptions* options, const FilterNode* roo
|
||||
ScannerSpecial special = scanner_prepare_special(
|
||||
options->preserve_devices, options->preserve_specials, options->copy_devices, file, &st);
|
||||
if (special == SCANNER_SPECIAL_SKIP) {
|
||||
scanner_note_nonreg(ps->options, file->path);
|
||||
free(rel);
|
||||
file_destroy(file);
|
||||
return;
|
||||
@@ -2134,6 +2218,7 @@ ParallelScanner* parallel_scanner_create_with_options(const char* root_directory
|
||||
return NULL;
|
||||
}
|
||||
ps->allocation_session = allocation_session;
|
||||
ps->options = options;
|
||||
|
||||
ArrayList* root_files = array_list_create(file_destroy);
|
||||
ArrayList* subdirs = array_list_create(free);
|
||||
@@ -2202,11 +2287,11 @@ ParallelScanner* parallel_scanner_create_with_options(const char* root_directory
|
||||
transfer root itself (it hands the root's immediate subdirectories to
|
||||
workers), so capture the root's directory time here. */
|
||||
if (options->capture_dir_times &&
|
||||
!scanner_capture_dir_time(options->dir_entries, options->dir_entries_mutex, root_directory,
|
||||
root_directory, options->relative && options->file_list != NULL,
|
||||
options->relative_prefix, options->preserve_atimes,
|
||||
options->preserve_crtimes, options->preserve_xattrs,
|
||||
options->preserve_acls)) {
|
||||
!scanner_capture_dir_time(
|
||||
options->dir_entries, options->dir_entries_mutex, root_directory, root_directory,
|
||||
options->relative && options->file_list != NULL, options->relative_prefix,
|
||||
options->preserve_atimes, options->preserve_crtimes, options->preserve_xattrs,
|
||||
options->preserve_acls, options->no_implied_dirs, options->file_list)) {
|
||||
array_list_delete(root_files);
|
||||
array_list_delete(subdirs);
|
||||
parallel_scanner_destroy(ps);
|
||||
|
||||
+33
-3
@@ -121,9 +121,18 @@ typedef struct {
|
||||
* it) and to emit its plan after the data stream, when no file frame would
|
||||
* otherwise trigger it. Guarded by `excluded_mutex`. */
|
||||
ArrayList* plan_dirs;
|
||||
/* --ignore-errors: an unreadable directory during the scan is recorded as an
|
||||
* I/O error and skipped instead of aborting the scan. Client-only. */
|
||||
/* --ignore-errors: an unreadable subdirectory no longer aborts the scan (it
|
||||
* is always skipped so the rest of the tree transfers); this flag is kept so
|
||||
* the client can distinguish the option state when deciding deletion policy.
|
||||
* Client-only. */
|
||||
bool ignore_io_errors;
|
||||
/* --info=nonreg: print rsync's `skipping non-regular file "NAME"` line for a
|
||||
* non-regular entry that is not being preserved. Client-only. */
|
||||
bool note_nonreg;
|
||||
/* Source root and 8-bit-output policy used to render a `--info=nonreg` name
|
||||
* relative to the transfer root. Borrowed read-only. */
|
||||
const char* send_directory;
|
||||
bool eight_bit_output;
|
||||
/* --ignore-missing-args (implied by --delete-missing-args): an explicitly
|
||||
* --files-from-listed entry that does not exist under the source is skipped
|
||||
* instead of failing (the --dirs generator is the only scanner path that
|
||||
@@ -151,6 +160,17 @@ typedef struct {
|
||||
bool capture_dir_times;
|
||||
ArrayList* dir_entries;
|
||||
mtx_t* dir_entries_mutex;
|
||||
/* Recreate empty source directories on a recursive transfer: emit a
|
||||
* payload-less directory entry for every traversed directory that produced
|
||||
* no transferred/descended child. Off by default so low-level scanner users
|
||||
* (unit helpers, --list-only) see only the historical file list; the real
|
||||
* sender sets it in prepare_scanner. */
|
||||
bool emit_empty_dirs;
|
||||
/* --no-implied-dirs with -R + --files-from: a directory that is only an
|
||||
* implied parent of a listed entry (not itself listed, nor below a listed
|
||||
* directory) must not carry source metadata; it is created with default
|
||||
* attributes at the destination, matching rsync. */
|
||||
bool no_implied_dirs;
|
||||
} ScannerOptions;
|
||||
|
||||
/* Internal per-scanner filter state. FilterNode chains represent the ordered
|
||||
@@ -168,6 +188,11 @@ typedef struct {
|
||||
int current_depth;
|
||||
dev_t root_dev;
|
||||
bool failed;
|
||||
/* Recursive scan: whether the open directory yielded any transferred or
|
||||
descended entry. When it did not, closing it emits a directory entry so
|
||||
the empty source directory is recreated at the destination (rsync
|
||||
parity). */
|
||||
bool current_dir_produced;
|
||||
/* Phase 2 (files-from / filter layer). */
|
||||
char* root_path; /* transfer root (fs path) for rel computation */
|
||||
char* current_rel; /* rel path of the open directory ("" == root) */
|
||||
@@ -186,6 +211,10 @@ typedef struct {
|
||||
--ignore-errors the scan continues past it and the caller decides what to
|
||||
do; `failed` is reserved for fatal errors that always abort the scan. */
|
||||
bool io_error;
|
||||
/* The transfer ROOT could not be opened. It is always fatal, even under
|
||||
--ignore-errors, but the client still maps it to rsync's partial-transfer
|
||||
exit (23) rather than a generic failure. */
|
||||
bool root_io_error;
|
||||
} DirectoryScanner;
|
||||
|
||||
typedef struct {
|
||||
@@ -205,7 +234,8 @@ typedef struct {
|
||||
int completed;
|
||||
Chunk* initial_chunk;
|
||||
ProtocolSession* allocation_session;
|
||||
FilterNode* root_filter_node; /* root .rsync-filter context (owned by ps) */
|
||||
FilterNode* root_filter_node; /* root .rsync-filter context (owned by ps) */
|
||||
const ScannerOptions* options; /* borrowed scan options (--info=nonreg output) */
|
||||
} ParallelScanner;
|
||||
|
||||
DirectoryScanner* directory_scanner_create(const char* root_directory, bool use_metadata,
|
||||
|
||||
+8
-5
@@ -90,8 +90,8 @@ void print_usage(void) {
|
||||
printf(" entry's destination mirror receiver-side. Independent of\n");
|
||||
printf(" --delete (it does not imply --delete; a non-empty directory\n");
|
||||
printf(" mirror is removed only with --force or --delete)\n");
|
||||
printf(" -m, --prune-empty-dirs Do not transfer empty directory entries (--dirs mode);\n");
|
||||
printf(" recursive transfers never send empty dirs\n");
|
||||
printf(" -m, --prune-empty-dirs Do not create empty directories (a recursive transfer\n");
|
||||
printf(" otherwise recreates them, like rsync)\n");
|
||||
printf(" Note: each timing flag implies --delete. Combining a timing flag with\n");
|
||||
printf(" --no-delete (in either order) is rejected as a config error.\n");
|
||||
printf(" --ignore-existing Skip files that already exist on receiver\n");
|
||||
@@ -103,8 +103,9 @@ void print_usage(void) {
|
||||
printf(" -R, --relative With --files-from, preserve each listed entry's relative path\n");
|
||||
printf(" below the destination root instead of mirroring the full\n");
|
||||
printf(" source path (no effect without --files-from)\n");
|
||||
printf(" --no-implied-dirs With -R --files-from, refuse to place a listed file whose\n");
|
||||
printf(" parent directory is not itself listed\n");
|
||||
printf(" --no-implied-dirs With -R, do not apply the source metadata of a listed file's\n");
|
||||
printf(" implied parent directories (they are still created with\n");
|
||||
printf(" default attributes)\n");
|
||||
printf(" --mkpath Create the destination root directory on the server when it\n");
|
||||
printf(" does not exist yet\n");
|
||||
printf(" --exclude <pattern>, --exclude=<pattern> Exclude files matching pattern\n");
|
||||
@@ -243,7 +244,9 @@ void print_usage(void) {
|
||||
printf(" reusable digest is sent (keep the file mode 0600)\n");
|
||||
printf(" --no-motd Suppress display of the daemon's MOTD (the server\n");
|
||||
printf(" still sends it; the client just does not show it)\n");
|
||||
printf(" --bwlimit <KB/s> Bandwidth limit in kilobytes per second\n");
|
||||
printf(" --bwlimit=RATE Limit socket I/O bandwidth (default unit KiB/s,\n");
|
||||
printf(" rsync-style: 0 = no limit; K/M/G/T/P suffixes are\n");
|
||||
printf(" binary, KB/MB decimal, KiB/MiB binary; decimals allowed)\n");
|
||||
printf(" --tls Enable TLS encryption\n");
|
||||
printf(" --cert <path> TLS certificate file (PEM)\n");
|
||||
printf(" --key <path> TLS private key file (PEM)\n");
|
||||
|
||||
+61
-12
@@ -61,13 +61,18 @@ bool receiver_send_final_success(int fd, const Config* config, const ReceiverOut
|
||||
}
|
||||
|
||||
bool receiver_send_stats_frame(int fd, const Config* config, const ReceiverStats* stats,
|
||||
const struct ArrayList* would_delete) {
|
||||
const struct ArrayList* would_delete,
|
||||
const struct ArrayList* deleted_paths) {
|
||||
if (!config->report_stats)
|
||||
return true;
|
||||
ReceiverStats local;
|
||||
memset(&local, 0, sizeof(local));
|
||||
const ReceiverStats* out = stats ? stats : &local;
|
||||
size_t count = would_delete ? (size_t)would_delete->size : 0;
|
||||
/* The path list carries the dry-run would-delete set for a -n run and the
|
||||
actually-removed set for a real --info=del run. */
|
||||
const struct ArrayList* paths =
|
||||
config->dry_run ? would_delete : (config->report_deletes ? deleted_paths : NULL);
|
||||
size_t count = paths ? (size_t)paths->size : 0;
|
||||
if (count > (size_t)MAX_MANIFEST_ENTRIES)
|
||||
count = MAX_MANIFEST_ENTRIES;
|
||||
ReceiverStats record = *out;
|
||||
@@ -76,7 +81,7 @@ bool receiver_send_stats_frame(int fd, const Config* config, const ReceiverStats
|
||||
!send_int(fd, (int)count))
|
||||
return false;
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
const char* path = (const char*)would_delete->items[i];
|
||||
const char* path = (const char*)paths->items[i];
|
||||
if (!send_wire_str(fd, path ? path : ""))
|
||||
return false;
|
||||
}
|
||||
@@ -90,6 +95,25 @@ static void receiver_tally_deleted(const ReceiverSink* sink, size_t deleted) {
|
||||
sink->stats->deleted_files += deleted;
|
||||
}
|
||||
|
||||
/* Observer for --info=del: record each truly-removed destination-relative path
|
||||
in the ArrayList passed as the observer context, so the terminal STATUS_STATS
|
||||
frame can list it. A failed append is best-effort (the deletion already
|
||||
happened; output is cosmetic). Shared by the single-threaded receiver and
|
||||
the -m pipeline's deferred commit. */
|
||||
void receiver_record_deleted_path(void* context, const char* rel_path) {
|
||||
ArrayList* paths = context;
|
||||
if (!paths || !rel_path)
|
||||
return;
|
||||
/* Bound the retained list like the keep-set manifest: only MAX_MANIFEST_ENTRIES
|
||||
paths are ever transmitted in the terminal STATUS_STATS frame, so recording
|
||||
more only grows memory. A hostile/huge deletion set is therefore capped. */
|
||||
if ((size_t)paths->size >= (size_t)MAX_MANIFEST_ENTRIES)
|
||||
return;
|
||||
char* copy = str_dup(rel_path);
|
||||
if (copy && !array_list_add(paths, copy))
|
||||
free(copy);
|
||||
}
|
||||
|
||||
static bool receiver_process_chunk(Chunk* chunk, const ReceiverSink* sink) {
|
||||
if (!chunk || !sink || !sink->store_file)
|
||||
return false;
|
||||
@@ -398,9 +422,13 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver
|
||||
--max-delete-capped commit still succeeds and the transfer proceeds;
|
||||
the terminal success frame reports the cap. */
|
||||
size_t deleted = 0;
|
||||
DeleteCommitResult deletion = (config->use_delete || config->delete_missing_args)
|
||||
? manifest_delete_all_counted(config, manifest, &deleted)
|
||||
: DELETE_COMMIT_OK;
|
||||
DeletePathObserver observer =
|
||||
(config->report_deletes && sink->deleted_paths) ? receiver_record_deleted_path : NULL;
|
||||
DeleteCommitResult deletion =
|
||||
(config->use_delete || config->delete_missing_args)
|
||||
? manifest_delete_all_observed(config, manifest, &deleted, observer,
|
||||
(void*)sink->deleted_paths)
|
||||
: DELETE_COMMIT_OK;
|
||||
receiver_tally_deleted(sink, deleted);
|
||||
delete_manifest_free(manifest);
|
||||
if (deletion == DELETE_COMMIT_ERROR) {
|
||||
@@ -435,8 +463,12 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver
|
||||
send_status(file_descriptor, STATUS_ERROR);
|
||||
goto fail;
|
||||
}
|
||||
if (!plan_session)
|
||||
if (!plan_session) {
|
||||
plan_session = delete_plan_session_create(config);
|
||||
if (plan_session && config->report_deletes && sink->deleted_paths)
|
||||
delete_plan_session_set_delete_observer(plan_session, receiver_record_deleted_path,
|
||||
(void*)sink->deleted_paths);
|
||||
}
|
||||
if (!plan_session || delete_plan_session_receive(plan_session, config, file_descriptor) != 0)
|
||||
goto fail;
|
||||
if (delete_plan_session_limit_reached(plan_session) && !delete_limit_noted &&
|
||||
@@ -479,8 +511,10 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver
|
||||
deferred_manifest = NULL;
|
||||
} else {
|
||||
size_t deleted = 0;
|
||||
DeleteCommitResult deletion =
|
||||
manifest_delete_all_counted(config, deferred_manifest, &deleted);
|
||||
DeletePathObserver observer =
|
||||
(config->report_deletes && sink->deleted_paths) ? receiver_record_deleted_path : NULL;
|
||||
DeleteCommitResult deletion = manifest_delete_all_observed(
|
||||
config, deferred_manifest, &deleted, observer, (void*)sink->deleted_paths);
|
||||
receiver_tally_deleted(sink, deleted);
|
||||
delete_manifest_free(deferred_manifest);
|
||||
deferred_manifest = NULL;
|
||||
@@ -498,6 +532,9 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver
|
||||
hands the session to its caller instead, which commits after the disk
|
||||
writer drained. */
|
||||
if (plan_session) {
|
||||
if (config->report_deletes && sink->deleted_paths)
|
||||
delete_plan_session_set_delete_observer(plan_session, receiver_record_deleted_path,
|
||||
(void*)sink->deleted_paths);
|
||||
if (pending_plans) {
|
||||
*pending_plans = plan_session;
|
||||
plan_session = NULL;
|
||||
@@ -569,6 +606,8 @@ typedef struct {
|
||||
--delete would-delete path list collected while processing the manifest. */
|
||||
ReceiverStats stats;
|
||||
ArrayList* would_delete;
|
||||
/* --info=del: actually-removed paths collected during the delete commit. */
|
||||
ArrayList* deleted_paths;
|
||||
} ReceiverSaveContext;
|
||||
|
||||
static bool receiver_save_file(File* file, void* context_pointer) {
|
||||
@@ -621,7 +660,8 @@ static void receiver_note_delete_limit(void* context_pointer) {
|
||||
static bool receiver_send_success_frame(int fd, void* context_pointer) {
|
||||
ReceiverSaveContext* context = context_pointer;
|
||||
Status final_status = context->delete_limit_reached ? STATUS_DELETE_LIMIT : STATUS_OK;
|
||||
if (!receiver_send_stats_frame(fd, context->config, &context->stats, context->would_delete))
|
||||
if (!receiver_send_stats_frame(fd, context->config, &context->stats, context->would_delete,
|
||||
context->deleted_paths))
|
||||
return false;
|
||||
/* Server-contacting --dry-run: nothing was staged or written, so there is
|
||||
nothing to publish and no directory times to stamp. */
|
||||
@@ -651,8 +691,15 @@ int receiver_receive_files(Config* config, int file_descriptor) {
|
||||
ReceiverSaveContext context = {.config = config, .outcomes = {0}};
|
||||
dir_time_list_init(&context.dir_times);
|
||||
context.would_delete = array_list_create(free);
|
||||
if (!context.would_delete)
|
||||
/* report_deletes (--info=del / -i / --out-format under --delete) is the only
|
||||
reason to retain the actually-removed paths; a plain --delete must not
|
||||
str_dup every removal. NULL is handled by every consumer. */
|
||||
context.deleted_paths = config->report_deletes ? array_list_create(free) : NULL;
|
||||
if (!context.would_delete || (config->report_deletes && !context.deleted_paths)) {
|
||||
array_list_delete(context.would_delete);
|
||||
array_list_delete(context.deleted_paths);
|
||||
return -1;
|
||||
}
|
||||
ReceiverSink sink = {receiver_save_file,
|
||||
&context,
|
||||
true,
|
||||
@@ -660,12 +707,14 @@ int receiver_receive_files(Config* config, int file_descriptor) {
|
||||
receiver_send_success_frame,
|
||||
receiver_note_delete_limit,
|
||||
&context.stats,
|
||||
context.would_delete};
|
||||
context.would_delete,
|
||||
context.deleted_paths};
|
||||
int ret = receiver_process(config, file_descriptor, &sink);
|
||||
if (ret != 0 && config->delay_updates && config->delay_context)
|
||||
delay_updates_cleanup(config->delay_context);
|
||||
receiver_outcomes_destroy(&context.outcomes);
|
||||
dir_time_list_free(&context.dir_times);
|
||||
array_list_delete(context.would_delete);
|
||||
array_list_delete(context.deleted_paths);
|
||||
return ret;
|
||||
}
|
||||
|
||||
+11
-1
@@ -46,11 +46,20 @@ typedef struct {
|
||||
carries the -n/--dry-run --delete path list. */
|
||||
ReceiverStats* stats;
|
||||
struct ArrayList* would_delete;
|
||||
/* When --info=del requested it, receiver-owned strings for every path the
|
||||
deletion commit ACTUALLY removed, sent in the terminal STATUS_STATS frame's
|
||||
path list so the sender can print rsync's `deleting PATH` lines. */
|
||||
struct ArrayList* deleted_paths;
|
||||
} ReceiverSink;
|
||||
|
||||
bool receiver_outcomes_append(ReceiverOutcomes* outcomes, unsigned char code);
|
||||
void receiver_outcomes_destroy(ReceiverOutcomes* outcomes);
|
||||
|
||||
/* DeletePathObserver implementation for --info=del: `context` is an ArrayList*
|
||||
that receives owned copies of every truly-removed destination-relative path.
|
||||
Shared by the single-threaded receiver and the -m pipeline's deferred commit. */
|
||||
void receiver_record_deleted_path(void* context, const char* rel_path);
|
||||
|
||||
/* Send the terminal success frame. `final_status` is usually STATUS_OK, or
|
||||
STATUS_DELETE_LIMIT when a --max-delete commit was capped. */
|
||||
bool receiver_send_final_success(int fd, const Config* config, const ReceiverOutcomes* outcomes,
|
||||
@@ -60,7 +69,8 @@ bool receiver_send_final_success(int fd, const Config* config, const ReceiverOut
|
||||
non-NULL, a count and that many wire strings) when the wire config requested
|
||||
report_stats. A no-op otherwise. */
|
||||
bool receiver_send_stats_frame(int fd, const Config* config, const ReceiverStats* stats,
|
||||
const struct ArrayList* would_delete);
|
||||
const struct ArrayList* would_delete,
|
||||
const struct ArrayList* deleted_paths);
|
||||
|
||||
int receiver_process(Config* config, int file_descriptor, const ReceiverSink* sink);
|
||||
/* receiver_process with an escape hatch for the commit-style (late) deletion:
|
||||
|
||||
@@ -31,6 +31,7 @@ PipelineContextReceiver* pipeline_context_receiver_create(Config* config, Queue*
|
||||
context->delete_limit_reached = false;
|
||||
memset(&context->stats, 0, sizeof(context->stats));
|
||||
context->would_delete = NULL;
|
||||
context->deleted_paths = NULL;
|
||||
atomic_init(&context->cancelled, false);
|
||||
int init = 0;
|
||||
if (mtx_init(&context->mutex, mtx_plain) != thrd_success)
|
||||
@@ -46,6 +47,15 @@ PipelineContextReceiver* pipeline_context_receiver_create(Config* config, Queue*
|
||||
context->would_delete = array_list_create(free);
|
||||
if (!context->would_delete)
|
||||
goto fail;
|
||||
/* The actually-removed path list is only needed to render rsync's
|
||||
`deleting PATH` lines, which the client requests via report_deletes
|
||||
(--info=del / -i / --out-format under --delete). A plain --delete run must
|
||||
not allocate it or observe every removal. */
|
||||
if (config->report_deletes) {
|
||||
context->deleted_paths = array_list_create(free);
|
||||
if (!context->deleted_paths)
|
||||
goto fail;
|
||||
}
|
||||
return context;
|
||||
|
||||
fail:
|
||||
@@ -56,6 +66,12 @@ fail:
|
||||
cnd_destroy(&context->condition_not_full);
|
||||
if (init >= 1)
|
||||
mtx_destroy(&context->mutex);
|
||||
/* Free every list that was already created before the failing allocation:
|
||||
`context` itself is freed below, so they would otherwise leak. */
|
||||
if (context->would_delete)
|
||||
array_list_delete(context->would_delete);
|
||||
if (context->deleted_paths)
|
||||
array_list_delete(context->deleted_paths);
|
||||
free(context);
|
||||
return NULL;
|
||||
}
|
||||
@@ -71,6 +87,8 @@ void pipeline_context_receiver_destroy(PipelineContextReceiver* context) {
|
||||
dir_time_list_free(&context->dir_times);
|
||||
if (context->would_delete)
|
||||
array_list_delete(context->would_delete);
|
||||
if (context->deleted_paths)
|
||||
array_list_delete(context->deleted_paths);
|
||||
mtx_destroy(&context->mutex);
|
||||
cnd_destroy(&context->condition_not_full);
|
||||
cnd_destroy(&context->condition_not_empty);
|
||||
@@ -184,7 +202,8 @@ int receive_thread(void* pipeline_context) {
|
||||
NULL,
|
||||
receiver_pipeline_note_delete_limit,
|
||||
&context->stats,
|
||||
context->would_delete};
|
||||
context->would_delete,
|
||||
context->deleted_paths};
|
||||
if (receiver_process_pending((Config*)config, file_descriptor, &sink, &context->deferred_manifest,
|
||||
&context->deferred_plans) != 0) {
|
||||
receiver_thread_fail(context);
|
||||
|
||||
@@ -61,6 +61,9 @@ typedef struct PipelineContextReceiver {
|
||||
/* -n/--dry-run --delete would-delete path list, collected by receive_thread
|
||||
and reported in the STATUS_STATS frame. */
|
||||
struct ArrayList* would_delete;
|
||||
/* --info=del actually-removed path list, collected by the deferred delete
|
||||
commit in server.c and reported in the STATUS_STATS frame. */
|
||||
struct ArrayList* deleted_paths;
|
||||
} PipelineContextReceiver;
|
||||
|
||||
PipelineContextReceiver* pipeline_context_receiver_create(Config* config, Queue* queue_receiver,
|
||||
|
||||
+7
-3
@@ -956,8 +956,9 @@ void handler(int file_descriptor) {
|
||||
server-contacting --dry-run deletes nothing (no manifest is sent). */
|
||||
if (context->deferred_manifest) {
|
||||
size_t deleted = 0;
|
||||
DeleteCommitResult deletion =
|
||||
manifest_delete_all_counted(config, context->deferred_manifest, &deleted);
|
||||
DeletePathObserver observer = config->report_deletes ? receiver_record_deleted_path : NULL;
|
||||
DeleteCommitResult deletion = manifest_delete_all_observed(
|
||||
config, context->deferred_manifest, &deleted, observer, (void*)context->deleted_paths);
|
||||
context->stats.deleted_files += deleted;
|
||||
if (deletion == DELETE_COMMIT_ERROR) {
|
||||
transfer_ok = false;
|
||||
@@ -975,6 +976,9 @@ void handler(int file_descriptor) {
|
||||
if (context->deferred_plans) {
|
||||
/* Defence in depth (the enclosing block already excludes dry-run): a
|
||||
-n run never commits a deletion. */
|
||||
if (config->report_deletes)
|
||||
delete_plan_session_set_delete_observer(
|
||||
context->deferred_plans, receiver_record_deleted_path, (void*)context->deleted_paths);
|
||||
DeleteCommitResult deletion =
|
||||
config->dry_run ? DELETE_COMMIT_OK
|
||||
: delete_plan_session_commit(context->deferred_plans, config);
|
||||
@@ -1010,7 +1014,7 @@ void handler(int file_descriptor) {
|
||||
/* Emit the optional wire-stats record first (protocol 2.25.0), then the
|
||||
success/outcome frame, exactly like the single-threaded receiver. */
|
||||
if (!receiver_send_stats_frame(file_descriptor, config, &context->stats,
|
||||
context->would_delete) ||
|
||||
context->would_delete, context->deleted_paths) ||
|
||||
!receiver_send_final_success(file_descriptor, config, &context->outcomes, final_status))
|
||||
transfer_ok = false;
|
||||
} else {
|
||||
|
||||
+15
-10
@@ -168,9 +168,14 @@ bool charset_spec_valid_direction(const char* from_charset, const char* to_chars
|
||||
return direction_probe_valid(from_charset, to_charset);
|
||||
}
|
||||
|
||||
/* The receiver's real conversion is wire(client REMOTE) -> server-local (the
|
||||
* server's own --iconv LOCAL half, or the client's LOCAL half when the server
|
||||
* has no --iconv). A dedicated pre-ack check so an impossible direction is
|
||||
/* The receiver's conversion is wire charset -> destination charset. rsync's
|
||||
* CONVERT_SPEC is LOCAL,REMOTE and "stays the same whether you're pushing or
|
||||
* pulling", so for a PUSH (FastSync's only direction) the destination end's
|
||||
* charset is the spec's REMOTE half: the client converts LOCAL -> REMOTE on the
|
||||
* sender and the receiver writes the wire bytes verbatim. Only a server that
|
||||
* declares its OWN --iconv (the daemon "charset" analog) has a different local
|
||||
* charset, and then it is that spec's LOCAL half and the receiver converts
|
||||
* wire -> server-local. A dedicated pre-ack check so an impossible direction is
|
||||
* rejected before the connection instead of refusing mid-transfer. */
|
||||
bool charset_wire_receiver_spec_valid(const char* spec, const char* server_spec) {
|
||||
if (!spec)
|
||||
@@ -180,7 +185,7 @@ bool charset_wire_receiver_spec_valid(const char* spec, const char* server_spec)
|
||||
if (charset_spec_parse(spec, &local, &remote) != 0)
|
||||
return false;
|
||||
const char* wire = remote;
|
||||
const char* target_local = local;
|
||||
const char* target_local = remote;
|
||||
char* server_local = NULL;
|
||||
char* server_remote = NULL;
|
||||
if (server_spec) {
|
||||
@@ -302,13 +307,13 @@ bool charset_wire_init_receiver(const char* spec, const char* server_spec) {
|
||||
char* remote;
|
||||
if (charset_spec_parse(spec, &local, &remote) != 0)
|
||||
return false;
|
||||
/* The wire charset is the client spec's REMOTE half; the local charset is
|
||||
* the client spec's LOCAL half unless the server was itself started with
|
||||
* --iconv naming a different local charset (the server halves above never
|
||||
* travel, so the server's own flag is the only way its local charset can
|
||||
* differ from what the client assumed). */
|
||||
/* The wire charset is the client spec's REMOTE half (rsync's LOCAL,REMOTE
|
||||
* spec stays the same push or pull, so on a push the destination end's
|
||||
* charset is REMOTE and the receiver writes the wire bytes verbatim). Only a
|
||||
* server started with its own --iconv declares a different local charset (the
|
||||
* server halves above never travel), and then it is that spec's LOCAL half. */
|
||||
const char* wire = remote;
|
||||
const char* target_local = local;
|
||||
const char* target_local = remote;
|
||||
char* server_local = NULL;
|
||||
char* server_remote = NULL;
|
||||
if (server_spec) {
|
||||
|
||||
@@ -57,8 +57,9 @@ void charset_conversion_close(void* conversion);
|
||||
/* Process-wide wire conversion. charset_wire_init_sender (client side) opens
|
||||
* LOCAL->REMOTE; charset_wire_init_receiver (server side) opens
|
||||
* wire(REMOTE)->server-local. server_spec is the server's own --iconv, whose
|
||||
* LOCAL half may override the local charset the client assumed; NULL reuses
|
||||
* the client spec's LOCAL half. Both return false on an unsupported spec.
|
||||
* LOCAL half overrides the destination charset; NULL means the destination
|
||||
* charset is the client spec's REMOTE half (rsync's push semantics: the wire
|
||||
* bytes are written verbatim). Both return false on an unsupported spec.
|
||||
* The state is freed with charset_wire_free. */
|
||||
bool charset_wire_init_sender(const char* spec);
|
||||
bool charset_wire_init_receiver(const char* spec, const char* server_spec);
|
||||
@@ -66,9 +67,9 @@ void charset_wire_free(void);
|
||||
bool charset_wire_active(void);
|
||||
|
||||
/* Pre-ack receiver-direction sanity (see charset_wire_init_receiver): true
|
||||
* when the exact wire->server-local conversion the receiver will use (client
|
||||
* spec's REMOTE half into the server's own LOCAL half, or the client's LOCAL
|
||||
* half when the server has no --iconv) opens and produces NUL-free output. */
|
||||
* when the exact wire->destination conversion the receiver will use (client
|
||||
* spec's REMOTE half into the server's own LOCAL half, or REMOTE->REMOTE when
|
||||
* the server has no --iconv) opens and produces NUL-free output. */
|
||||
bool charset_wire_receiver_spec_valid(const char* spec, const char* server_spec);
|
||||
|
||||
/* Convert a path across the wire in the process direction. Returns a malloc'd
|
||||
|
||||
+27
-2
@@ -223,17 +223,25 @@ bool checksum_digest_file(ChecksumAlgo algo, uint64_t seed, const char* path, ui
|
||||
if (fd < 0)
|
||||
return false;
|
||||
|
||||
if (algo == CHECKSUM_ALGO_NONE) {
|
||||
/* No checksum requested: nothing to read; an empty digest succeeds. */
|
||||
close(fd);
|
||||
*out_len = 0;
|
||||
return true;
|
||||
}
|
||||
|
||||
uint8_t buffer[64 * 1024];
|
||||
bool ok = false;
|
||||
|
||||
if (algo == CHECKSUM_ALGO_MD5) {
|
||||
if (algo == CHECKSUM_ALGO_MD5 || algo == CHECKSUM_ALGO_SHA1) {
|
||||
const EVP_MD* md = algo == CHECKSUM_ALGO_MD5 ? EVP_md5() : EVP_sha1();
|
||||
EVP_MD_CTX* ctx = EVP_MD_CTX_new();
|
||||
if (!ctx) {
|
||||
close(fd);
|
||||
return false;
|
||||
}
|
||||
unsigned int digest_len = 0;
|
||||
if (EVP_DigestInit_ex(ctx, EVP_md5(), NULL) == 1) {
|
||||
if (EVP_DigestInit_ex(ctx, md, NULL) == 1) {
|
||||
ok = true;
|
||||
ssize_t got;
|
||||
while ((got = read(fd, buffer, sizeof(buffer))) > 0) {
|
||||
@@ -254,6 +262,23 @@ bool checksum_digest_file(ChecksumAlgo algo, uint64_t seed, const char* path, ui
|
||||
return ok;
|
||||
}
|
||||
|
||||
if (algo == CHECKSUM_ALGO_MD4) {
|
||||
Md4Ctx ctx;
|
||||
md4_init(&ctx);
|
||||
ok = true;
|
||||
ssize_t got;
|
||||
while ((got = read(fd, buffer, sizeof(buffer))) > 0)
|
||||
md4_update(&ctx, buffer, (size_t)got);
|
||||
if (got < 0)
|
||||
ok = false;
|
||||
if (ok) {
|
||||
md4_final(&ctx, out);
|
||||
*out_len = 16;
|
||||
}
|
||||
close(fd);
|
||||
return ok;
|
||||
}
|
||||
|
||||
XXH64_state_t xxh64;
|
||||
XXH3_state_t* xxh3 = NULL;
|
||||
if (algo == CHECKSUM_ALGO_XXH64) {
|
||||
|
||||
+2
-1
@@ -206,7 +206,8 @@ static bool validate_received_config(const Config* config) {
|
||||
valid_wire_bool(config->preserve_perms) && valid_wire_bool(config->preserve_times) &&
|
||||
valid_wire_bool(config->preserve_owner) && valid_wire_bool(config->preserve_group) &&
|
||||
valid_wire_bool(config->munge_links) && valid_wire_bool(config->keep_dirlinks) &&
|
||||
valid_wire_bool(config->fake_super) &&
|
||||
valid_wire_bool(config->fake_super) && valid_wire_bool(config->report_dest_info) &&
|
||||
valid_wire_bool(config->report_stats) && valid_wire_bool(config->report_deletes) &&
|
||||
(!config->copy_as_set || (config->copy_as_uid >= 0 && config->copy_as_gid >= 0)) &&
|
||||
(!config->use_compression ||
|
||||
(config->compression_level >= 1 && config->compression_level <= 22)) &&
|
||||
|
||||
+24
-6
@@ -82,7 +82,7 @@ typedef struct {
|
||||
typedef enum SuperMode { SUPER_MODE_AUTO = 0, SUPER_MODE_ON = 1, SUPER_MODE_OFF = 2 } SuperMode;
|
||||
|
||||
/* ===========================================================================
|
||||
* Config wire-field table (single source of truth for protocol 2.26.0).
|
||||
* Config wire-field table (single source of truth for protocol 2.27.0).
|
||||
*
|
||||
* Every field below crosses the wire. The table is the ONLY place a
|
||||
* serialized field is named: config.h expands CONFIG_WIRE_FIELDS() to declare
|
||||
@@ -259,9 +259,18 @@ typedef enum SuperMode { SUPER_MODE_AUTO = 0, SUPER_MODE_ON = 1, SUPER_MODE_OFF
|
||||
* for -n/--dry-run --delete, the destination-relative paths it WOULD have
|
||||
* deleted. It is set by the client only when --stats, --progress/-P, an
|
||||
* --out-format token needs a wire counter (%b/%c), or a dry-run carries
|
||||
* --delete; the transfer decision itself is unchanged. */
|
||||
* --delete; the transfer decision itself is unchanged.
|
||||
*
|
||||
* --info wave (protocol 2.27.0). report_deletes tells the receiver to include
|
||||
* the destination-relative paths it ACTUALLY removed in its terminal
|
||||
* STATUS_STATS record (the same path-list field the dry-run would-delete report
|
||||
* uses), so the sender can print rsync's `deleting PATH`/`*deleting` lines for a
|
||||
* real (non-dry-run) deletion. It is set when --delete is active and any of
|
||||
* --info=del, -i/--itemize-changes or --out-format requests per-file change
|
||||
* output; the transfer decision itself is unchanged. */
|
||||
#define CONFIG_WIRE_OUTPUT_FIELDS(X) \
|
||||
X(report_dest_info, bool, false, BOOL) X(report_stats, bool, false, BOOL)
|
||||
X(report_dest_info, bool, false, BOOL) \
|
||||
X(report_stats, bool, false, BOOL) X(report_deletes, bool, false, BOOL)
|
||||
|
||||
/* Codec-negotiation wave (protocol 2.26.0). compression_algo is the concrete
|
||||
* codec the client selected for this transfer (a CompressionAlgo id) and is the
|
||||
@@ -424,8 +433,10 @@ typedef struct Config {
|
||||
* repeated -F adds --filter='- .rsync-filter' so they are excluded too. */
|
||||
int per_dir_filter_count;
|
||||
bool one_file_system; /* -x/--one-file-system: do not cross filesystem boundaries */
|
||||
/* --no-implied-dirs: client-only. With -R + --files-from, refuse to place a
|
||||
* listed file whose ancestor directory is not itself explicitly listed. */
|
||||
/* --no-implied-dirs: client-only. With -R, do not transfer the source
|
||||
* metadata of the parent directories implied by a listed path; an unlisted
|
||||
* implied parent is still created (with default attributes) so the listed
|
||||
* file can be placed, matching rsync. */
|
||||
bool no_implied_dirs;
|
||||
/* -d/--dirs: client-only. Transfer the directory entries named by the
|
||||
* source argument / --files-from list without recursing into contents. */
|
||||
@@ -988,7 +999,14 @@ typedef struct Config {
|
||||
* boundary, and the strict same-version handshake (config_receive rejects a
|
||||
* mismatched version before parsing anything else) keeps mixed deployments from
|
||||
* ever reaching that state. */
|
||||
#define PROTOCOL_VERSION "2.26.0"
|
||||
/* (7) --info=del report (protocol 2.27.0): the config frame gains one trailing
|
||||
* bool, report_deletes, appended after report_stats. When set, the receiver
|
||||
* lists the paths it actually removed in the terminal STATUS_STATS path list
|
||||
* (the same count-delimited list the -n/--dry-run would-delete report uses), so
|
||||
* the sender can print rsync's `deleting PATH` lines for a real deletion. No
|
||||
* change to the fixed STATUS_STATS record itself; only a new trailing config
|
||||
* bool, which still requires the version bump for the strict lockstep. */
|
||||
#define PROTOCOL_VERSION "2.27.0"
|
||||
#define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024)
|
||||
/* Upper bound on total basis-dir entries (rsync caps --link-dest at 20). */
|
||||
#define MAX_BASIS_DIRS 64
|
||||
|
||||
@@ -412,6 +412,11 @@ struct DeletePlanSession {
|
||||
bool dry_run;
|
||||
size_t max_delete;
|
||||
size_t deleted;
|
||||
/* Removals charged against --max-delete. In --delete-delay mode a path is
|
||||
planned (and the budget consumed) while scanning, but `deleted` advances
|
||||
only when the commit actually unlinks it, so an entry that survives the
|
||||
commit (a directory refilled mid-transfer -> ENOTEMPTY) is not reported. */
|
||||
size_t planned;
|
||||
size_t skipped;
|
||||
bool limit_hit;
|
||||
bool limit_logged;
|
||||
@@ -421,8 +426,16 @@ struct DeletePlanSession {
|
||||
ArrayList* size_skipped;
|
||||
ArrayList* missing;
|
||||
ArrayList* deferred;
|
||||
DeletePathObserver observer;
|
||||
void* observer_context;
|
||||
};
|
||||
|
||||
/* Report one path the session truly removed (no-op without an observer). */
|
||||
static void notify_deleted(DeletePlanSession* session, const char* rel) {
|
||||
if (session && session->observer && rel)
|
||||
session->observer(session->observer_context, rel);
|
||||
}
|
||||
|
||||
DeletePlanSession* delete_plan_session_create(const Config* config) {
|
||||
if (!config)
|
||||
return NULL;
|
||||
@@ -565,7 +578,7 @@ static bool build_plan_skips(const Config* config, const DeletePlanSession* sess
|
||||
}
|
||||
|
||||
static bool budget_available(const DeletePlanSession* session) {
|
||||
return session->deleted < session->max_delete;
|
||||
return session->planned < session->max_delete;
|
||||
}
|
||||
|
||||
static void note_skipped(DeletePlanSession* session) {
|
||||
@@ -579,7 +592,9 @@ static void log_deleted(const char* rel) {
|
||||
free(escaped);
|
||||
}
|
||||
|
||||
/* Append a snapshot path for --delete-delay. */
|
||||
/* Append a snapshot path for --delete-delay. The budget is charged here, but
|
||||
* `deleted` is not: the path counts only once apply_deferred_path truly
|
||||
* unlinks it. */
|
||||
static bool defer_add(DeletePlanSession* session, const char* rel) {
|
||||
char* copy = str_dup(rel);
|
||||
if (!copy)
|
||||
@@ -588,7 +603,7 @@ static bool defer_add(DeletePlanSession* session, const char* rel) {
|
||||
free(copy);
|
||||
return false;
|
||||
}
|
||||
session->deleted++;
|
||||
session->planned++;
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -633,7 +648,9 @@ static bool process_extra_dir(int dirfd, const char* name, const char* child_rel
|
||||
}
|
||||
if (unlinkat(dirfd, name, AT_REMOVEDIR) == 0) {
|
||||
session->deleted++;
|
||||
session->planned++;
|
||||
log_deleted(child_rel);
|
||||
notify_deleted(session, child_rel);
|
||||
*removed = true;
|
||||
return true;
|
||||
}
|
||||
@@ -657,7 +674,9 @@ static bool process_extra_file(int dirfd, const char* name, const char* child_re
|
||||
}
|
||||
if (unlinkat(dirfd, name, 0) == 0) {
|
||||
session->deleted++;
|
||||
session->planned++;
|
||||
log_deleted(child_rel);
|
||||
notify_deleted(session, child_rel);
|
||||
} else if (errno != ENOENT) {
|
||||
return false;
|
||||
}
|
||||
@@ -768,13 +787,15 @@ static bool apply_missing(DeletePlanSession* session, const Config* config) {
|
||||
return true;
|
||||
DeleteManifest manifest = {
|
||||
.keeps = NULL, .protected = NULL, .missing = session->missing, .dirs = NULL};
|
||||
size_t remaining = budget_available(session) ? session->max_delete - session->deleted : 0;
|
||||
size_t remaining = budget_available(session) ? session->max_delete - session->planned : 0;
|
||||
size_t deleted = 0;
|
||||
size_t skipped = 0;
|
||||
bool limit = false;
|
||||
bool ok = manifest_delete_missing_args_limited(config, &manifest, remaining, &deleted, &skipped,
|
||||
&limit);
|
||||
bool ok = manifest_delete_missing_args_limited_observed(config, &manifest, remaining, &deleted,
|
||||
&skipped, &limit, session->observer,
|
||||
session->observer_context);
|
||||
session->deleted += deleted;
|
||||
session->planned += deleted;
|
||||
session->skipped += skipped;
|
||||
if (limit)
|
||||
session->limit_hit = true;
|
||||
@@ -839,7 +860,6 @@ int delete_plan_session_receive(DeletePlanSession* session, const Config* config
|
||||
/* Apply one snapshotted --delete-delay path (post-order: children precede their
|
||||
* parent directory). */
|
||||
static bool apply_deferred_path(DeletePlanSession* session, const Config* config, const char* rel) {
|
||||
(void)session;
|
||||
char* full = path_cat(config->receive_root_directory, rel);
|
||||
if (!full)
|
||||
return false;
|
||||
@@ -863,13 +883,24 @@ static bool apply_deferred_path(DeletePlanSession* session, const Config* config
|
||||
else
|
||||
rc = unlinkat(parent_fd, leaf, 0);
|
||||
bool ok = rc == 0 || errno == ENOENT || errno == ENOTEMPTY || errno == EEXIST;
|
||||
if (rc == 0)
|
||||
if (rc == 0) {
|
||||
session->deleted++;
|
||||
log_deleted(rel);
|
||||
notify_deleted(session, rel);
|
||||
}
|
||||
close(parent_fd);
|
||||
free(leaf);
|
||||
return ok;
|
||||
}
|
||||
|
||||
void delete_plan_session_set_delete_observer(DeletePlanSession* session,
|
||||
DeletePathObserver observer, void* context) {
|
||||
if (!session)
|
||||
return;
|
||||
session->observer = observer;
|
||||
session->observer_context = context;
|
||||
}
|
||||
|
||||
DeleteCommitResult delete_plan_session_commit(DeletePlanSession* session, const Config* config) {
|
||||
if (!session || !config)
|
||||
return DELETE_COMMIT_ERROR;
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
#include "config.h"
|
||||
#include "file_receive.h"
|
||||
#include "protocol.h"
|
||||
#include "utils.h"
|
||||
#include <stdbool.h>
|
||||
|
||||
/* Per-directory delete plans (protocol 2.24.0).
|
||||
@@ -76,8 +77,16 @@ int delete_plan_session_receive(DeletePlanSession* session, const Config* config
|
||||
DeleteCommitResult delete_plan_session_commit(DeletePlanSession* session, const Config* config);
|
||||
/* True once the shared --max-delete budget stopped part of a deletion. */
|
||||
bool delete_plan_session_limit_reached(const DeletePlanSession* session);
|
||||
/* Number of destination entries the session's plans removed (or, for
|
||||
--delete-delay, snapshotted for removal), for the end-of-transfer stats. */
|
||||
/* Number of destination entries the session actually removed, for the
|
||||
end-of-transfer stats. For --delete-delay this excludes a snapshotted entry
|
||||
that survived (e.g. a refilled directory that failed ENOTEMPTY), even though
|
||||
that entry already consumed --max-delete budget at snapshot time. */
|
||||
size_t delete_plan_session_deleted(const DeletePlanSession* session);
|
||||
/* Install an observer invoked for every destination-relative path the session
|
||||
truly removes (including the deferred --delete-delay commit), so the receiver
|
||||
can report rsync's `deleting PATH` lines through the terminal STATUS_STATS
|
||||
record. Pass NULL/0 to clear. */
|
||||
void delete_plan_session_set_delete_observer(DeletePlanSession* session,
|
||||
DeletePathObserver observer, void* context);
|
||||
|
||||
#endif
|
||||
|
||||
@@ -807,6 +807,25 @@ bool file_ensure_directory_secure(const char* path) {
|
||||
} else if (errno == EEXIST) {
|
||||
dir_fd = openat(parent_fd, leaf, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||
}
|
||||
} else if (dir_fd < 0 && errno == ENOTDIR) {
|
||||
/* rsync replaces a destination non-directory (regular file) with an
|
||||
incoming directory. Confined to the already-opened secure parent fd:
|
||||
the leaf is unlinked by name (never followed) and only a non-directory
|
||||
is ever removed, so this cannot escape the authorized root or remove a
|
||||
pre-existing directory tree. A symlink is left alone (openat with
|
||||
O_NOFOLLOW reports ELOOP, which takes no branch here), since replacing
|
||||
it is not required for FastSync's transferred directories and keeps
|
||||
--keep-dirlinks semantics untouched. */
|
||||
struct stat leaf_st;
|
||||
if (fstatat(parent_fd, leaf, &leaf_st, AT_SYMLINK_NOFOLLOW) == 0 && !S_ISDIR(leaf_st.st_mode) &&
|
||||
!S_ISLNK(leaf_st.st_mode)) {
|
||||
if (unlinkat(parent_fd, leaf, 0) == 0) {
|
||||
if (mkdirat(parent_fd, leaf, (mode_t)(0777 & ~(mode_t)file_process_umask())) == 0)
|
||||
created = true;
|
||||
/* On failure dir_fd stays < 0 below, so the caller still sees it. */
|
||||
dir_fd = openat(parent_fd, leaf, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||
}
|
||||
}
|
||||
}
|
||||
bool ok = dir_fd >= 0;
|
||||
/* --copy-as owns a directory this call just created (the final component;
|
||||
|
||||
+64
-13
@@ -3195,8 +3195,9 @@ char* file_receive_basis_delete_relative(const Config* config, const char* path)
|
||||
alternate basis directories are never destination content and are skipped at
|
||||
any depth. Returns true unless a traversal/unlink error aborted the walk;
|
||||
the budget's limit_hit/skipped fields report a cap-stopped run. */
|
||||
static bool delete_extras_budgeted(const Config* config, DeleteManifest* manifest,
|
||||
DeleteBudgetState* budget) {
|
||||
static bool delete_extras_budgeted_observed(const Config* config, DeleteManifest* manifest,
|
||||
DeleteBudgetState* budget, DeletePathObserver observer,
|
||||
void* observer_context) {
|
||||
if (!config || !manifest || !manifest->keeps)
|
||||
return false;
|
||||
fprintf(stderr, "Deleting files not in manifest...\n");
|
||||
@@ -3260,9 +3261,9 @@ static bool delete_extras_budgeted(const Config* config, DeleteManifest* manifes
|
||||
remaining = budget->max_delete - budget->deleted;
|
||||
size_t deleted = 0;
|
||||
size_t skipped = 0;
|
||||
DeleteWalkResult result =
|
||||
delete_extras_limited(config->receive_root_directory, manifest->keeps, manifest->dirs,
|
||||
remaining, skips, used, &deleted, &skipped);
|
||||
DeleteWalkResult result = delete_extras_limited_observed(
|
||||
config->receive_root_directory, manifest->keeps, manifest->dirs, remaining, skips, used,
|
||||
&deleted, &skipped, observer, observer_context);
|
||||
if (owned_prefixes) {
|
||||
for (int i = 0; i < config->basis_count; i++)
|
||||
free(owned_prefixes[i]);
|
||||
@@ -3282,6 +3283,31 @@ static bool delete_extras_budgeted(const Config* config, DeleteManifest* manifes
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool delete_extras_budgeted(const Config* config, DeleteManifest* manifest,
|
||||
DeleteBudgetState* budget) {
|
||||
return delete_extras_budgeted_observed(config, manifest, budget, NULL, NULL);
|
||||
}
|
||||
|
||||
/* Prefixes every observed path with a fixed subtree root, so a nested walk
|
||||
(a recursively removed missing-arg directory) reports receive-root-relative
|
||||
names like the rest of the delete output. */
|
||||
typedef struct {
|
||||
DeletePathObserver inner;
|
||||
void* inner_context;
|
||||
const char* prefix;
|
||||
} PrefixedDeleteObserver;
|
||||
|
||||
static void prefixed_delete_observer(void* context, const char* rel) {
|
||||
PrefixedDeleteObserver* prefixed = context;
|
||||
if (!prefixed->inner || !rel)
|
||||
return;
|
||||
char* joined = path_cat((char*)prefixed->prefix, rel);
|
||||
if (joined) {
|
||||
prefixed->inner(prefixed->inner_context, joined);
|
||||
free(joined);
|
||||
}
|
||||
}
|
||||
|
||||
/* --delete-missing-args exact-path deletions: each destination mirror in
|
||||
manifest->missing is an explicit user request, so it is removed even when the
|
||||
ordinary extras walk (with its protected prefixes) would leave it alone. The
|
||||
@@ -3295,8 +3321,10 @@ static bool delete_extras_budgeted(const Config* config, DeleteManifest* manifes
|
||||
--max-delete budget: once it is exhausted the remaining requests are skipped
|
||||
and counted. Returns false only on a genuine error (a confinement failure on
|
||||
a validated path or an I/O error), which fails the run. */
|
||||
static bool delete_missing_args_budgeted(const Config* config, DeleteManifest* manifest,
|
||||
DeleteBudgetState* budget) {
|
||||
static bool delete_missing_args_budgeted_observed(const Config* config, DeleteManifest* manifest,
|
||||
DeleteBudgetState* budget,
|
||||
DeletePathObserver observer,
|
||||
void* observer_context) {
|
||||
if (!config || !manifest)
|
||||
return false;
|
||||
if (!manifest->missing || manifest->missing->size == 0)
|
||||
@@ -3413,9 +3441,12 @@ static bool delete_missing_args_budgeted(const Config* config, DeleteManifest* m
|
||||
budget->deleted >= budget->max_delete ? 0 : budget->max_delete - budget->deleted;
|
||||
size_t contents_deleted = 0;
|
||||
size_t contents_skipped = 0;
|
||||
PrefixedDeleteObserver nested = {observer, observer_context, rel};
|
||||
DeleteWalkResult walk =
|
||||
no_keeps ? delete_extras_limited(full, no_keeps, NULL, remaining, NULL, 0,
|
||||
&contents_deleted, &contents_skipped)
|
||||
no_keeps ? delete_extras_limited_observed(full, no_keeps, NULL, remaining, NULL, 0,
|
||||
&contents_deleted, &contents_skipped,
|
||||
observer ? prefixed_delete_observer : NULL,
|
||||
observer ? &nested : NULL)
|
||||
: DELETE_WALK_ERROR;
|
||||
if (no_keeps)
|
||||
array_list_delete(no_keeps);
|
||||
@@ -3455,6 +3486,8 @@ static bool delete_missing_args_budgeted(const Config* config, DeleteManifest* m
|
||||
}
|
||||
if (removed) {
|
||||
budget->deleted++;
|
||||
if (observer)
|
||||
observer(observer_context, rel);
|
||||
char* escaped = output_escape(rel, log_get_8_bit_output());
|
||||
fprintf(stderr, " Deleted: %s\n", escaped ? escaped : "<allocation failed>");
|
||||
free(escaped);
|
||||
@@ -3541,15 +3574,25 @@ bool manifest_delete_extras(const Config* config, DeleteManifest* manifest) {
|
||||
bool manifest_delete_missing_args(const Config* config, DeleteManifest* manifest) {
|
||||
DeleteBudgetState budget = {
|
||||
.max_delete = SIZE_MAX, .deleted = 0, .skipped = 0, .limit_hit = false};
|
||||
return delete_missing_args_budgeted(config, manifest, &budget);
|
||||
return delete_missing_args_budgeted_observed(config, manifest, &budget, NULL, NULL);
|
||||
}
|
||||
|
||||
bool manifest_delete_missing_args_limited(const Config* config, DeleteManifest* manifest,
|
||||
size_t max_delete, size_t* deleted, size_t* skipped,
|
||||
bool* limit_hit) {
|
||||
return manifest_delete_missing_args_limited_observed(config, manifest, max_delete, deleted,
|
||||
skipped, limit_hit, NULL, NULL);
|
||||
}
|
||||
|
||||
bool manifest_delete_missing_args_limited_observed(const Config* config, DeleteManifest* manifest,
|
||||
size_t max_delete, size_t* deleted,
|
||||
size_t* skipped, bool* limit_hit,
|
||||
DeletePathObserver observer,
|
||||
void* observer_context) {
|
||||
DeleteBudgetState budget = {
|
||||
.max_delete = max_delete, .deleted = 0, .skipped = 0, .limit_hit = false};
|
||||
bool ok = delete_missing_args_budgeted(config, manifest, &budget);
|
||||
bool ok =
|
||||
delete_missing_args_budgeted_observed(config, manifest, &budget, observer, observer_context);
|
||||
if (deleted)
|
||||
*deleted = budget.deleted;
|
||||
if (skipped)
|
||||
@@ -3572,6 +3615,12 @@ DeleteCommitResult manifest_delete_all(const Config* config, DeleteManifest* man
|
||||
|
||||
DeleteCommitResult manifest_delete_all_counted(const Config* config, DeleteManifest* manifest,
|
||||
size_t* deleted) {
|
||||
return manifest_delete_all_observed(config, manifest, deleted, NULL, NULL);
|
||||
}
|
||||
|
||||
DeleteCommitResult manifest_delete_all_observed(const Config* config, DeleteManifest* manifest,
|
||||
size_t* deleted, DeletePathObserver observer,
|
||||
void* observer_context) {
|
||||
if (deleted)
|
||||
*deleted = 0;
|
||||
if (!config || !manifest)
|
||||
@@ -3590,9 +3639,11 @@ DeleteCommitResult manifest_delete_all_counted(const Config* config, DeleteManif
|
||||
.deleted = 0,
|
||||
.skipped = 0,
|
||||
.limit_hit = false};
|
||||
if (config->delete_missing_args && !delete_missing_args_budgeted(config, manifest, &budget))
|
||||
if (config->delete_missing_args &&
|
||||
!delete_missing_args_budgeted_observed(config, manifest, &budget, observer, observer_context))
|
||||
return DELETE_COMMIT_ERROR;
|
||||
if (config->use_delete && !delete_extras_budgeted(config, manifest, &budget))
|
||||
if (config->use_delete &&
|
||||
!delete_extras_budgeted_observed(config, manifest, &budget, observer, observer_context))
|
||||
return DELETE_COMMIT_ERROR;
|
||||
if (deleted)
|
||||
*deleted = budget.deleted;
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
#include "config.h"
|
||||
#include "file_types.h"
|
||||
#include "utils.h"
|
||||
#include <stdbool.h>
|
||||
|
||||
/* Server-side file receive/save path. */
|
||||
@@ -126,6 +127,13 @@ bool manifest_delete_missing_args(const Config* config, DeleteManifest* manifest
|
||||
bool manifest_delete_missing_args_limited(const Config* config, DeleteManifest* manifest,
|
||||
size_t max_delete, size_t* deleted, size_t* skipped,
|
||||
bool* limit_hit);
|
||||
/* Observer-aware form of manifest_delete_missing_args_limited: `observer` (may
|
||||
be NULL) is invoked for every destination-relative path truly removed. */
|
||||
bool manifest_delete_missing_args_limited_observed(const Config* config, DeleteManifest* manifest,
|
||||
size_t max_delete, size_t* deleted,
|
||||
size_t* skipped, bool* limit_hit,
|
||||
DeletePathObserver observer,
|
||||
void* observer_context);
|
||||
/* Outcome of committing a delete manifest. LIMIT_REACHED reports rsync's
|
||||
partial --max-delete result: the budget allowed some deletions and the rest
|
||||
were skipped (the run still stores all file data but the client exits 25). */
|
||||
@@ -146,6 +154,11 @@ DeleteCommitResult manifest_delete_all(const Config* config, DeleteManifest* man
|
||||
removed (for the end-of-transfer wire stats). `deleted` may be NULL. */
|
||||
DeleteCommitResult manifest_delete_all_counted(const Config* config, DeleteManifest* manifest,
|
||||
size_t* deleted);
|
||||
/* Observer-aware form of manifest_delete_all_counted: `observer` (may be NULL)
|
||||
is invoked for every destination-relative path truly removed. */
|
||||
DeleteCommitResult manifest_delete_all_observed(const Config* config, DeleteManifest* manifest,
|
||||
size_t* deleted, DeletePathObserver observer,
|
||||
void* observer_context);
|
||||
|
||||
/* -n/--dry-run --delete would-delete reporting: walk the destination exactly as
|
||||
the delete pass would and append (strdup'd) destination-relative paths that
|
||||
|
||||
@@ -73,4 +73,27 @@ typedef struct {
|
||||
bool format_stats_send(int fd, const ReceiverStats* stats);
|
||||
bool format_stats_receive(int fd, ReceiverStats* stats);
|
||||
|
||||
/* Sender-side file-list accounting for rsync's `--stats` block. Filled while
|
||||
* the scan/send loops walk each entry: the flist counters describe every
|
||||
* scanned source entry (transferred or skipped), while the transferred/literal
|
||||
* counters describe only the regular files the receiver actually stored. The
|
||||
* type split lets the client print rsync's `Number of files` breakdown; the
|
||||
* receiver-only counters (matched data, deleted, created) come from
|
||||
* STATUS_STATS. */
|
||||
typedef struct {
|
||||
unsigned long long flist_reg;
|
||||
unsigned long long flist_dir;
|
||||
unsigned long long flist_link;
|
||||
unsigned long long flist_special;
|
||||
unsigned long long total_file_size; /* sum of entry sizes (link target len) */
|
||||
unsigned long long transferred_regular; /* regular files actually stored */
|
||||
unsigned long long transferred_file_size; /* source size of those files */
|
||||
/* Whole-file accuracy: the `--stats` "Literal data" row. The sender counts
|
||||
* the source size of every stored file, so a whole-file transfer matches
|
||||
* rsync. A delta run actually ships only the literal fragments of the diff
|
||||
* (the rest is matched/copied), so here the value is an upper bound, not
|
||||
* rsync's literal-byte total; see RSYNC_COMPAT.md's `--stats` row. */
|
||||
unsigned long long literal_data;
|
||||
} TransferStats;
|
||||
|
||||
#endif
|
||||
|
||||
+15
-1
@@ -21,7 +21,21 @@ typedef enum {
|
||||
LOG_INFO_MISC = 1u << 1,
|
||||
LOG_INFO_SKIP = 1u << 2,
|
||||
LOG_INFO_STATS = 1u << 3,
|
||||
LOG_INFO_ALL = LOG_INFO_COPY | LOG_INFO_MISC | LOG_INFO_SKIP | LOG_INFO_STATS,
|
||||
/* rsync categories that map to a FastSync event (emitted in rsync's line
|
||||
* format): del (deletions), remove (sender-side source removal), name
|
||||
* (transferred entry names), flist (file-list header), nonreg (skipped
|
||||
* non-regular files), progress (per-file progress). rsync's `backup`
|
||||
* category is accepted for CLI parity but stays silent: the receiver does the
|
||||
* backing-up and FastSync has no backup event to report from the sender. */
|
||||
LOG_INFO_DEL = 1u << 4,
|
||||
LOG_INFO_REMOVE = 1u << 5,
|
||||
LOG_INFO_NAME = 1u << 6,
|
||||
LOG_INFO_FLIST = 1u << 7,
|
||||
LOG_INFO_NONREG = 1u << 8,
|
||||
LOG_INFO_PROGRESS = 1u << 9,
|
||||
LOG_INFO_ALL = LOG_INFO_COPY | LOG_INFO_MISC | LOG_INFO_SKIP | LOG_INFO_STATS | LOG_INFO_DEL |
|
||||
LOG_INFO_REMOVE | LOG_INFO_NAME | LOG_INFO_FLIST | LOG_INFO_NONREG |
|
||||
LOG_INFO_PROGRESS,
|
||||
} LogInfoFlag;
|
||||
|
||||
void log_message(LogLevel log_level, const char* message, ...);
|
||||
|
||||
@@ -38,10 +38,12 @@ PipelineContextSender* pipeline_context_sender_create(Config* config, Queue* que
|
||||
context->remove_source_files = NULL;
|
||||
context->early_delete = false;
|
||||
context->delete_plans = NULL;
|
||||
context->delete_suppressed = false;
|
||||
context->scan_stopped_early = false;
|
||||
context->total_files = 0;
|
||||
context->progress_bytes = 0;
|
||||
context->total_bytes = 0;
|
||||
memset(&context->stats, 0, sizeof(context->stats));
|
||||
context->sender_done = false;
|
||||
atomic_init(&context->cancelled, false);
|
||||
protocol_session_init(&context->allocation_session, -1, -1);
|
||||
|
||||
@@ -9,6 +9,7 @@
|
||||
#include "config.h"
|
||||
#include "delete_plan.h"
|
||||
#include "file.h"
|
||||
#include "format.h"
|
||||
#include "protocol.h"
|
||||
#include "queue.h"
|
||||
#include "stop_condition.h"
|
||||
@@ -82,10 +83,22 @@ typedef struct {
|
||||
thread transmits the root plan before any data and the remaining plans
|
||||
alongside the chunks. Set once before the worker threads start. */
|
||||
DeletePlanSender* delete_plans;
|
||||
/* A scan I/O error without --ignore-errors suppressed deletion: the prebuilt
|
||||
keep-set/plans were dropped, and the streaming scanner must not build a
|
||||
fresh manifest or re-send the per-directory plans. Set once before the
|
||||
worker threads start. */
|
||||
bool delete_suppressed;
|
||||
mtx_t mutex_progress;
|
||||
int total_files;
|
||||
unsigned long long progress_bytes;
|
||||
unsigned long long total_bytes;
|
||||
/* Per-type flist / transferred accounting for the rsync --stats breakdown and
|
||||
the progress `to-chk` denominator. Owned by the sender thread: it is the
|
||||
only writer (the entry/transfer notes in send_chunks_multithreaded) and it
|
||||
reads the totals in its completion tail, so no lock is needed. This is NOT
|
||||
guarded by mutex_progress (which covers total_files/progress_bytes/
|
||||
total_bytes/sender_done). */
|
||||
TransferStats stats;
|
||||
bool sender_done;
|
||||
atomic_bool cancelled;
|
||||
ProtocolSession allocation_session;
|
||||
|
||||
+24
-3
@@ -183,12 +183,28 @@ void io_set_bwlimit(unsigned long long bytes_per_sec) {
|
||||
mtx_unlock(&bw_mutex);
|
||||
}
|
||||
|
||||
unsigned long long io_get_bwlimit(void) {
|
||||
return global_bwlimit();
|
||||
}
|
||||
|
||||
/* rsync's throttle (io.c sleep_for_bwlimit) sleeps once its unslept debt
|
||||
* reaches ~100 ms of bandwidth, so its effective initial burst is about 0.1 s
|
||||
* worth of bytes, not a full second. FastSync models the same with a token
|
||||
* bucket whose capacity is bwlimit/10, so a throttled run paces like rsync
|
||||
* instead of sending a full second's worth up front. */
|
||||
static long long bw_burst_capacity(unsigned long long bwlimit) {
|
||||
if (bwlimit == 0)
|
||||
return 0;
|
||||
long long burst = (long long)(bwlimit / 10);
|
||||
return burst > 0 ? burst : 1;
|
||||
}
|
||||
|
||||
void protocol_session_set_bwlimit(ProtocolSession* session, unsigned long long bytes_per_sec) {
|
||||
if (!session)
|
||||
return;
|
||||
session->bwlimit =
|
||||
bytes_per_sec > (unsigned long long)LLONG_MAX ? (unsigned long long)LLONG_MAX : bytes_per_sec;
|
||||
session->bw_tokens = (long long)session->bwlimit;
|
||||
session->bw_tokens = bw_burst_capacity(session->bwlimit);
|
||||
struct timespec now;
|
||||
clock_gettime(CLOCK_MONOTONIC, &now);
|
||||
session->bw_last_refill_sec = now.tv_sec;
|
||||
@@ -222,8 +238,9 @@ static void bw_throttle_session(ProtocolSession* session, size_t bytes_written)
|
||||
|
||||
long long tokens_to_add = (long long)((double)session->bwlimit * elapsed_ns / 1000000000.0);
|
||||
session->bw_tokens += tokens_to_add;
|
||||
if (session->bw_tokens > (long long)session->bwlimit)
|
||||
session->bw_tokens = (long long)session->bwlimit;
|
||||
long long burst = bw_burst_capacity(session->bwlimit);
|
||||
if (session->bw_tokens > burst)
|
||||
session->bw_tokens = burst;
|
||||
|
||||
session->bw_tokens -= bytes_written;
|
||||
|
||||
@@ -234,7 +251,11 @@ static void bw_throttle_session(ProtocolSession* session, size_t bytes_written)
|
||||
poll(NULL, 0, (int)(deficit_us / 1000));
|
||||
else
|
||||
usleep((useconds_t)deficit_us);
|
||||
/* Reset the bucket AFTER the sleep: crediting the sleep duration as elapsed
|
||||
refill time would cancel half the throttle (the next call would see the
|
||||
whole sleep as refill and immediately grant a fresh burst). */
|
||||
session->bw_tokens = 0;
|
||||
clock_gettime(CLOCK_MONOTONIC, &now);
|
||||
session->bw_last_refill_sec = now.tv_sec;
|
||||
session->bw_last_refill_nsec = now.tv_nsec;
|
||||
}
|
||||
|
||||
@@ -207,6 +207,7 @@ enum NET_STATUS {
|
||||
|
||||
void io_set_fds(int read_fd, int write_fd);
|
||||
void io_set_bwlimit(unsigned long long bytes_per_sec);
|
||||
unsigned long long io_get_bwlimit(void);
|
||||
void io_set_ssl(SSL* ssl);
|
||||
SSL* io_get_ssl(void);
|
||||
|
||||
|
||||
+47
-7
@@ -52,6 +52,31 @@ bool path_is_within_root(const char* root, const char* path) {
|
||||
return strncmp(root, path, root_len) == 0 && (path[root_len] == '\0' || path[root_len] == '/');
|
||||
}
|
||||
|
||||
/* Borrowed transfer-relative view of `path`: strip any leading '/' and then a
|
||||
* `root` prefix (its own leading/trailing slashes tolerated), returning a
|
||||
* pointer into `path`. Non-allocating, so it is safe on the hot scan/print
|
||||
* paths. A NULL/empty root, or a path not under `root`, leaves only the
|
||||
* leading-slash strip. `path` must be NUL-terminated and live in the caller. */
|
||||
const char* utils_strip_transfer_root(const char* path, const char* root) {
|
||||
if (path == NULL)
|
||||
return NULL;
|
||||
const char* rel = path;
|
||||
while (*rel == '/')
|
||||
rel++;
|
||||
if (root == NULL)
|
||||
return rel;
|
||||
while (*root == '/')
|
||||
root++;
|
||||
size_t root_len = strlen(root);
|
||||
while (root_len > 0 && root[root_len - 1] == '/')
|
||||
root_len--;
|
||||
if (root_len == 0)
|
||||
return rel;
|
||||
if (strncmp(rel, root, root_len) == 0 && (rel[root_len] == '/' || rel[root_len] == '\0'))
|
||||
return rel + root_len + (rel[root_len] == '/' ? 1 : 0);
|
||||
return rel;
|
||||
}
|
||||
|
||||
/* Open the destination root directory itself, confined to the authorized root.
|
||||
* NOTE (do not merge with file_open_secure_parent): this walk opens dest_root
|
||||
* (a directory that must already exist) and returns its fd, whereas
|
||||
@@ -616,7 +641,8 @@ static bool is_synced_dir(const PathIndex* dirs, const char* rel) {
|
||||
static bool delete_extras_fd(int dirfd, const char* rel_path, const PathIndex* keep,
|
||||
const PathIndex* dirs, DeleteBudget* budget,
|
||||
const DeleteSkipEntry* skips, int skip_count, bool parent_deletable,
|
||||
bool* all_removed) {
|
||||
bool* all_removed, DeletePathObserver observer,
|
||||
void* observer_context) {
|
||||
/* openat(dirfd, ".") opens an independent file description: a dup() would
|
||||
share dirfd's file offset and a prior pass could leave the stream drained. */
|
||||
int scanfd = openat(dirfd, ".", O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||
@@ -666,7 +692,7 @@ static bool delete_extras_fd(int dirfd, const char* rel_path, const PathIndex* k
|
||||
bool child_all_removed = false;
|
||||
if (childfd >= 0) {
|
||||
if (!delete_extras_fd(childfd, child_rel, keep, dirs, budget, skips, skip_count, deletable,
|
||||
&child_all_removed))
|
||||
&child_all_removed, observer, observer_context))
|
||||
operation_ok = false;
|
||||
close(childfd);
|
||||
} else if (errno != ENOENT) {
|
||||
@@ -693,6 +719,8 @@ static bool delete_extras_fd(int dirfd, const char* rel_path, const PathIndex* k
|
||||
local_survives = true;
|
||||
} else {
|
||||
budget->deleted++;
|
||||
if (observer)
|
||||
observer(observer_context, child_rel);
|
||||
}
|
||||
} else {
|
||||
local_survives = true;
|
||||
@@ -713,6 +741,8 @@ static bool delete_extras_fd(int dirfd, const char* rel_path, const PathIndex* k
|
||||
local_survives = true;
|
||||
} else {
|
||||
budget->deleted++;
|
||||
if (observer)
|
||||
observer(observer_context, child_rel);
|
||||
char* escaped_path = output_escape(child_rel, log_get_8_bit_output());
|
||||
fprintf(stderr, " Deleted: %s\n", escaped_path ? escaped_path : "<allocation failed>");
|
||||
free(escaped_path);
|
||||
@@ -867,10 +897,12 @@ bool delete_extras_list(const char* dest_root, const ArrayList* manifest,
|
||||
return ok;
|
||||
}
|
||||
|
||||
DeleteWalkResult delete_extras_limited(const char* dest_root, const ArrayList* manifest,
|
||||
const ArrayList* synced_dirs, size_t max_delete,
|
||||
const DeleteSkipEntry* skips, int skip_count,
|
||||
size_t* deleted_out, size_t* skipped_out) {
|
||||
DeleteWalkResult delete_extras_limited_observed(const char* dest_root, const ArrayList* manifest,
|
||||
const ArrayList* synced_dirs, size_t max_delete,
|
||||
const DeleteSkipEntry* skips, int skip_count,
|
||||
size_t* deleted_out, size_t* skipped_out,
|
||||
DeletePathObserver observer,
|
||||
void* observer_context) {
|
||||
if (deleted_out)
|
||||
*deleted_out = 0;
|
||||
if (skipped_out)
|
||||
@@ -911,7 +943,7 @@ DeleteWalkResult delete_extras_limited(const char* dest_root, const ArrayList* m
|
||||
DeleteBudget budget = {.max_delete = max_delete, .deleted = 0, .skipped = 0, .limit_hit = false};
|
||||
bool all_removed = false;
|
||||
bool ok = delete_extras_fd(rootfd, "", &keep, have_dirs ? &dirs : NULL, &budget, skips,
|
||||
skip_count, false, &all_removed);
|
||||
skip_count, false, &all_removed, observer, observer_context);
|
||||
if (close(rootfd) != 0)
|
||||
ok = false;
|
||||
path_index_free(&keep);
|
||||
@@ -926,6 +958,14 @@ DeleteWalkResult delete_extras_limited(const char* dest_root, const ArrayList* m
|
||||
return budget.limit_hit ? DELETE_WALK_LIMIT_REACHED : DELETE_WALK_OK;
|
||||
}
|
||||
|
||||
DeleteWalkResult delete_extras_limited(const char* dest_root, const ArrayList* manifest,
|
||||
const ArrayList* synced_dirs, size_t max_delete,
|
||||
const DeleteSkipEntry* skips, int skip_count,
|
||||
size_t* deleted_out, size_t* skipped_out) {
|
||||
return delete_extras_limited_observed(dest_root, manifest, synced_dirs, max_delete, skips,
|
||||
skip_count, deleted_out, skipped_out, NULL, NULL);
|
||||
}
|
||||
|
||||
bool delete_extras(const char* dest_root, const ArrayList* manifest) {
|
||||
return delete_extras_limited(dest_root, manifest, NULL, SIZE_MAX, NULL, 0, NULL, NULL) ==
|
||||
DELETE_WALK_OK;
|
||||
|
||||
@@ -138,6 +138,20 @@ DeleteWalkResult delete_extras_limited(const char* dest_root, const ArrayList* m
|
||||
const ArrayList* synced_dirs, size_t max_delete,
|
||||
const DeleteSkipEntry* skips, int skip_count,
|
||||
size_t* deleted_out, size_t* skipped_out);
|
||||
|
||||
/* Optional per-deletion observer: called for each destination-relative path
|
||||
actually removed (a file, symlink, or directory), in removal order, so the
|
||||
receiver can stream rsync's `--info=del`/`--info=remove` lines. */
|
||||
typedef void (*DeletePathObserver)(void* context, const char* rel_path);
|
||||
|
||||
/* `delete_extras_limited_observed` is delete_extras_limited with an optional
|
||||
observer; the observer is invoked only for entries truly removed. */
|
||||
DeleteWalkResult delete_extras_limited_observed(const char* dest_root, const ArrayList* manifest,
|
||||
const ArrayList* synced_dirs, size_t max_delete,
|
||||
const DeleteSkipEntry* skips, int skip_count,
|
||||
size_t* deleted_out, size_t* skipped_out,
|
||||
DeletePathObserver observer,
|
||||
void* observer_context);
|
||||
/* Read-only companion to delete_extras_limited: walk the destination exactly as
|
||||
the delete pass would and APPEND (strdup'd) destination-relative paths that
|
||||
WOULD be removed, without touching disk. Used for -n/--dry-run --delete
|
||||
@@ -177,6 +191,11 @@ const char* utils_get_authorized_root_path(void);
|
||||
* callers guarantee this); this is containment by string, not by resolved
|
||||
* symlinks. Shared by the utils and file secure-walk root confinement. */
|
||||
bool path_is_within_root(const char* root, const char* path);
|
||||
/* Non-allocating transfer-relative view of `path`: strip any leading '/' and
|
||||
* then a `root` prefix (leading/trailing slashes tolerated), returning a
|
||||
* borrowed pointer into `path`. A NULL/empty root, or a path not under
|
||||
* `root`, yields just the leading-slash strip. `path`/`root` must stay alive. */
|
||||
const char* utils_strip_transfer_root(const char* path, const char* root);
|
||||
/* True when `path` contains a ".." component. This is a purely lexical
|
||||
* dot-dot check: an absolute path is NOT rejected here, because default
|
||||
* (non-relative) transfers legitimately put the sender's absolute source path
|
||||
|
||||
@@ -38,11 +38,17 @@ FastSync mirrors the absolute source path under its receive root (see
|
||||
# Fast subset that guards the ✅ surface on pull requests
|
||||
python3 -m pytest tests/integration/test_differential_parity.py -n 4 --dist=load -m parity_ci
|
||||
|
||||
# Full set (all ✅ cases plus the documented ⚠️/❌ residuals)
|
||||
# Full differential case table (`_CASES`): every row is marked `parity`, and a
|
||||
# case with an allowlisted residual in `parity_caveats.py` is included too.
|
||||
python3 -m pytest tests/integration/test_differential_parity.py -n 4 --dist=load -m parity
|
||||
```
|
||||
|
||||
The suite skips cleanly when `rsync` is not installed.
|
||||
`-m parity` selects only the `_CASES` table in this module. Differential
|
||||
coverage for options outside that table (`--temp-dir`, `--delay-updates`,
|
||||
`--dry-run`, `--fuzzy`, the basis-dir options, `-M` over daemon/TCP, and
|
||||
receiver filter-protect) lives in dedicated modules (`test_option_parity.py`,
|
||||
`test_parity_blockers.py`, `test_parity_quickwins.py`, ...) and is not part of
|
||||
this gate. The suite skips cleanly when `rsync` is not installed.
|
||||
|
||||
## Allowlist (`parity_caveats.py`)
|
||||
|
||||
@@ -52,10 +58,11 @@ Each entry maps a case id to the aspects that may differ (`tree`, `stdout`,
|
||||
|
||||
```python
|
||||
CAVEATS = {
|
||||
"min_size": {
|
||||
"tree": "recursive transfer does not create a source directory that "
|
||||
"becomes empty after --min-size filtering. ref: RSYNC_COMPAT.md "
|
||||
"`-d/--dirs` row and completion-wave residual.",
|
||||
"max_delete": {
|
||||
"tree": "which destination extras survive a partial --max-delete abort "
|
||||
"is deletion-order dependent and unspecified; rc=25 and the "
|
||||
"number of survivors match rsync. ref: RSYNC_COMPAT.md "
|
||||
"`--max-delete=NUM` row.",
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
@@ -25,40 +25,6 @@ re-triaged when the row moves.
|
||||
|
||||
# case id -> {aspect: "reason (ref: RSYNC_COMPAT.md ...)"}
|
||||
CAVEATS = {
|
||||
# A source subtree whose only files are all filtered out (here, by
|
||||
# --min-size) is left behind as an empty directory by rsync but not by
|
||||
# FastSync: the recursive scanner keeps directory entries record-only, so
|
||||
# `sub/deep` is only implicit through the (skipped) file. This is the
|
||||
# documented recursive-empty-directory residual, not a payload/selection
|
||||
# bug.
|
||||
"min_size": {
|
||||
"tree": "recursive transfer does not create a source directory that "
|
||||
"becomes empty after --min-size filtering (FastSync directory "
|
||||
"entries are record-only). ref: RSYNC_COMPAT.md `-d/--dirs` "
|
||||
"row and completion-wave residual ('recursive transfers still "
|
||||
"do not create empty directories').",
|
||||
},
|
||||
# FastSync's recursive scanner keeps directory entries record-only, so a
|
||||
# plain `-d`/recursive source whose only role for a directory is that entry
|
||||
# (empty dir, or a dir emptied by filtering) is not created on the
|
||||
# destination. rsync creates it. `--dirs`/`--files-from`-listed
|
||||
# directories DO cross as explicit entries (covered by the passing
|
||||
# `empty_dirs_files_from` / `files_from` cases).
|
||||
"empty_dirs_recursive": {
|
||||
"tree": "recursive transfer does not create empty source directories. "
|
||||
"ref: RSYNC_COMPAT.md `-d/--dirs` row and completion-wave "
|
||||
"residual ('recursive transfers still do not create empty "
|
||||
"directories').",
|
||||
},
|
||||
# A plain `-d` invocation: FastSync's `--source-dir` treats the argument as
|
||||
# the directory entry itself (creates the empty source-root mirror), while
|
||||
# rsync's `src/` trailing-slash form lists the immediate contents.
|
||||
"dirs_plain": {
|
||||
"tree": "plain -d semantics: FastSync creates the source-root directory "
|
||||
"entry (its documented --dirs files-from behavior) instead of "
|
||||
"rsync's one-level contents listing for a `src/` argument. "
|
||||
"ref: RSYNC_COMPAT.md `-d/--dirs` row (⚠️).",
|
||||
},
|
||||
# --max-delete stops the extras walk part-way and exits 25 in both
|
||||
# implementations; which of the remaining extras survives depends on
|
||||
# deletion order, which neither tool specifies. The exit code and the
|
||||
|
||||
@@ -89,6 +89,9 @@ class Case:
|
||||
ignore_paths: Tuple[str, ...] = ()
|
||||
extra_check: Optional[Callable] = None
|
||||
files_from: Optional[Tuple[str, ...]] = None
|
||||
# rsync receives ``src + "/"``; FastSync mirrors the path it is given, so a
|
||||
# trailing-slash-sensitive case must hand FastSync the same form.
|
||||
fs_src_suffix: str = ""
|
||||
ci: bool = False
|
||||
ref: str = ""
|
||||
|
||||
@@ -418,6 +421,7 @@ def run_differential( # noqa: PLR0913 (explicit scenario parameters)
|
||||
ignore_paths: Tuple[str, ...] = (),
|
||||
extra_check: Optional[Callable] = None,
|
||||
files_from: Optional[Tuple[str, ...]] = None,
|
||||
fs_src_suffix: str = "",
|
||||
) -> Dict[str, object]:
|
||||
"""Run one rsync/FastSync pair and return the diff aspects.
|
||||
|
||||
@@ -447,7 +451,7 @@ def run_differential( # noqa: PLR0913 (explicit scenario parameters)
|
||||
fs_flags.append(f"--files-from={list_path}")
|
||||
|
||||
rs = run_rsync(src, rdst, rs_flags)
|
||||
fs_result, _ = run_fastsync(src, fdst, fs_flags, server.port)
|
||||
fs_result, _ = run_fastsync(src + fs_src_suffix, fdst, fs_flags, server.port)
|
||||
|
||||
class _View:
|
||||
"""Adapter so tree_diff/extra_check keep the Case-shaped interface."""
|
||||
@@ -486,7 +490,7 @@ def execute_case(case: Case, server) -> Dict[str, object]:
|
||||
layout=case.layout, seed=case.seed, stdout=case.stdout,
|
||||
compare_modes=case.compare_modes, compare_hardlinks=case.compare_hardlinks,
|
||||
ignore_paths=case.ignore_paths, extra_check=case.extra_check,
|
||||
files_from=case.files_from,
|
||||
files_from=case.files_from, fs_src_suffix=case.fs_src_suffix,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
"""Differential coverage for ``--delete-delay`` + ``--max-delete`` with a
|
||||
refilled deferred directory.
|
||||
|
||||
FastSync snapshots a directory's extras at plan time (``defer_add``) and charges
|
||||
``--max-delete`` then, and its deferred commit only removes the snapshot path, so
|
||||
a directory refilled before the commit survives ``ENOTEMPTY``. rsync computes
|
||||
the deferred deletions during the transfer, charges ``--max-delete`` on actual
|
||||
removals, and recursively removes a queued directory -- so content created after
|
||||
the plan inside an extra directory is removed too.
|
||||
|
||||
These tests run both tools on the same fixture and pin the shared budget bound
|
||||
(the later extra survives, both exit 25) plus the documented residual (the
|
||||
refilled directory's late content survives under FastSync, not rsync). They are
|
||||
not part of the fast PR gate because the rsync side needs a wide real-time
|
||||
injection window (a throttled transfer), while the FastSync side uses the
|
||||
existing byte-deterministic slicing proxy.
|
||||
|
||||
The refilled directory sits at the transfer ROOT, whose delete plan is always
|
||||
processed before any subdirectory's, so the budget is deterministically charged
|
||||
to the refilled entry; the second extra lives under ``b`` and is skipped.
|
||||
"""
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
from common import ( # noqa: E402
|
||||
TEST_DATA_DIR,
|
||||
ServerManager,
|
||||
clean_dir,
|
||||
get_dest_received_dir,
|
||||
run_client,
|
||||
)
|
||||
from test_delete_timing_parity import _SlicingProxy # noqa: E402
|
||||
|
||||
RSYNC = shutil.which("rsync")
|
||||
requires_rsync = pytest.mark.skipif(RSYNC is None, reason="rsync 3.4.1 not installed")
|
||||
|
||||
BIG_BYTES = 8 * 1024 * 1024
|
||||
MID_TRANSFER_BYTES = 256 * 1024
|
||||
PROXY_THROTTLE = 0.001
|
||||
# rsync is driven locally, so the refill is injected on a wall-clock delay while
|
||||
# a throttled ~8 s transfer is in flight. 1.5 s is safely after rsync's plan
|
||||
# scan (t=0) and well before the deferred commit at the end.
|
||||
RSYNC_BWLIMIT = 1024 # 1 MiB/s
|
||||
RSYNC_INJECT_DELAY = 1.5
|
||||
|
||||
|
||||
def _write(path, content):
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
with open(path, "wb") as fh:
|
||||
fh.write(content)
|
||||
|
||||
|
||||
def _seed_source(tag):
|
||||
source = os.path.join(TEST_DATA_DIR, f"ddb_{tag}_src")
|
||||
clean_dir(source)
|
||||
_write(os.path.join(source, "a", "keep.bin"), b"B" * BIG_BYTES)
|
||||
_write(os.path.join(source, "b", "keep.txt"), b"keep\n")
|
||||
return source
|
||||
|
||||
|
||||
def _seed_fastsync(tag):
|
||||
"""FastSync mirrors the absolute source path under its receive root, so the
|
||||
extras live below ``received``."""
|
||||
source = _seed_source(tag)
|
||||
dest = os.path.join(TEST_DATA_DIR, f"ddb_{tag}_dst")
|
||||
clean_dir(dest)
|
||||
received = get_dest_received_dir(dest, source)
|
||||
os.makedirs(os.path.join(received, "xdir"), exist_ok=True)
|
||||
os.makedirs(os.path.join(received, "b", "ydir"), exist_ok=True)
|
||||
return source, dest, received
|
||||
|
||||
|
||||
def _seed_rsync(tag):
|
||||
"""rsync mirrors the source contents directly into the destination, so the
|
||||
extras are flat under ``rsync_dst``."""
|
||||
source = _seed_source(tag)
|
||||
rsync_dst = os.path.join(TEST_DATA_DIR, f"ddb_{tag}_dst")
|
||||
clean_dir(rsync_dst)
|
||||
os.makedirs(os.path.join(rsync_dst, "xdir"), exist_ok=True)
|
||||
os.makedirs(os.path.join(rsync_dst, "b", "ydir"), exist_ok=True)
|
||||
return source, rsync_dst
|
||||
|
||||
|
||||
def _deleted_count(text):
|
||||
for line in text.splitlines():
|
||||
if line.startswith("Number of deleted files:"):
|
||||
return int(line.split(":", 1)[1].split()[0])
|
||||
return None
|
||||
|
||||
|
||||
def _rsync(args, timeout=120):
|
||||
env = dict(os.environ, LC_ALL="C")
|
||||
return subprocess.run([RSYNC] + args, capture_output=True, text=True, env=env, timeout=timeout)
|
||||
|
||||
|
||||
class TestDeleteDelayRefilledDirVsRsync:
|
||||
"""The shared budget bound and the documented recursive-removal residual."""
|
||||
|
||||
def _fastsync_refilled(self, tag, max_delete=None):
|
||||
"""Run FastSync with the refill injected deterministically by the proxy
|
||||
hook (fired once the receiver has processed the plan frames)."""
|
||||
source, dest, received = _seed_fastsync(tag)
|
||||
late = os.path.join(received, "xdir", "new.txt")
|
||||
|
||||
def hook():
|
||||
_write(late, b"created mid-transfer\n")
|
||||
|
||||
flags = ["--delete-delay", "--incremental", "--ignore-times", "--stats"]
|
||||
if max_delete is not None:
|
||||
flags.append(f"--max-delete={max_delete}")
|
||||
with ServerManager() as server:
|
||||
server.start(extra_args=["--allow-delete"])
|
||||
proxy = _SlicingProxy(server.port, hook=hook, hook_after=MID_TRANSFER_BYTES,
|
||||
throttle=PROXY_THROTTLE, wait_for_reply=True)
|
||||
result, _ = run_client(source, dest, flags=flags, port=proxy.port)
|
||||
proxy.finish()
|
||||
assert proxy.hook_called.is_set(), "refill hook never fired"
|
||||
return result, received, late
|
||||
|
||||
@requires_rsync
|
||||
def test_max_delete_budget_bound_matches_and_residual_pinned(self):
|
||||
# --- FastSync: budget charged at snapshot; late content preserved ---
|
||||
result, received, late = self._fastsync_refilled("budget_fs", max_delete=1)
|
||||
assert result.returncode == 25, (result.stderr or result.stdout)[:300]
|
||||
assert _deleted_count(result.stdout) == 0, result.stdout
|
||||
assert os.path.exists(late), "FastSync removed the late content of a snapshotted dir"
|
||||
assert os.path.isdir(os.path.join(received, "xdir"))
|
||||
assert os.path.isdir(os.path.join(received, "b", "ydir")), (
|
||||
"FastSync did not charge the plan-time budget: b/ydir was removed"
|
||||
)
|
||||
|
||||
# --- rsync: budget charged on actual removals; dirs removed recursively ---
|
||||
source, rsync_dst = _seed_rsync("budget_rsync")
|
||||
|
||||
def inject():
|
||||
time.sleep(RSYNC_INJECT_DELAY)
|
||||
_write(os.path.join(rsync_dst, "xdir", "new.txt"), b"created mid-transfer\n")
|
||||
|
||||
t = threading.Thread(target=inject)
|
||||
t.start()
|
||||
rsync_result = _rsync(
|
||||
["-a", "--delete-delay", "--max-delete=1", "--stats",
|
||||
f"--bwlimit={RSYNC_BWLIMIT}", source + "/", rsync_dst + "/"]
|
||||
)
|
||||
t.join()
|
||||
assert rsync_result.returncode == 25, rsync_result.stderr
|
||||
# rsync removes the late content (recursive deferred removal); FastSync
|
||||
# keeps it and charges the snapshot directive instead.
|
||||
assert not os.path.exists(os.path.join(rsync_dst, "xdir", "new.txt")), (
|
||||
"rsync kept late content inside a queued directory"
|
||||
)
|
||||
# The shared observable: the later extra survives in both tools under
|
||||
# --max-delete=1, and both report the capped run with exit 25.
|
||||
assert os.path.isdir(os.path.join(rsync_dst, "b", "ydir")), (
|
||||
"rsync did not bound the deletion with --max-delete=1"
|
||||
)
|
||||
assert os.path.isdir(os.path.join(received, "b", "ydir"))
|
||||
|
||||
@requires_rsync
|
||||
def test_refilled_extra_dir_recursive_removal_residual(self):
|
||||
"""Without --max-delete the residual is a plain tree difference: rsync
|
||||
removes the refilled extra directory (and its late content), FastSync
|
||||
leaves the snapshot path in place on ENOTEMPTY."""
|
||||
result, received, late = self._fastsync_refilled("recur_fs")
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||
assert os.path.exists(late), "FastSync removed the refilled directory's late content"
|
||||
|
||||
source, rsync_dst = _seed_rsync("recur_rsync")
|
||||
|
||||
def inject():
|
||||
time.sleep(RSYNC_INJECT_DELAY)
|
||||
_write(os.path.join(rsync_dst, "xdir", "new.txt"), b"created mid-transfer\n")
|
||||
|
||||
t = threading.Thread(target=inject)
|
||||
t.start()
|
||||
rsync_result = _rsync(
|
||||
["-a", "--delete-delay", "--stats", f"--bwlimit={RSYNC_BWLIMIT}",
|
||||
source + "/", rsync_dst + "/"]
|
||||
)
|
||||
t.join()
|
||||
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||
assert not os.path.exists(os.path.join(rsync_dst, "xdir")), (
|
||||
"rsync did not recursively remove the refilled extra directory"
|
||||
)
|
||||
@@ -229,10 +229,13 @@ class TestDeleteTimingFinalStateParity:
|
||||
@pytest.mark.parametrize("timing", ["--delete-during", "--delete-delay"])
|
||||
@requires_rsync
|
||||
def test_success_final_state_matches_rsync(self, timing):
|
||||
# Worker-safe names: xdist may run both parametrizations concurrently, so
|
||||
# the timing is part of every fixture path.
|
||||
label = timing.lstrip("-")
|
||||
# Build the rsync fixture from the same seed so both sides start equal.
|
||||
source, dest, received = _seed_pair("parity_rsync")
|
||||
source, dest, received = _seed_pair(f"parity_rsync_{label}")
|
||||
source2 = source
|
||||
rsync_dst = os.path.join(TEST_DATA_DIR, "dtp_parity_rsync_dst")
|
||||
rsync_dst = os.path.join(TEST_DATA_DIR, f"dtp_rsync_{label}_dst")
|
||||
clean_dir(rsync_dst)
|
||||
# rsync mirrors src/ into dst/; seed the same extra.
|
||||
_write(os.path.join(rsync_dst, "d", "old_extra"), b"stale extra\n")
|
||||
@@ -258,7 +261,8 @@ class TestDeleteTimingTypeConflictParity:
|
||||
@pytest.mark.parametrize("timing", ["--delete-during", "--delete-delay"])
|
||||
@requires_rsync
|
||||
def test_type_conflicts_match_rsync(self, timing):
|
||||
source = os.path.join(TEST_DATA_DIR, "dtc_src")
|
||||
label = timing.lstrip("-")
|
||||
source = os.path.join(TEST_DATA_DIR, f"dtc_{label}_src")
|
||||
clean_dir(source)
|
||||
_write(os.path.join(source, "foo"), b"now a file\n")
|
||||
_write(os.path.join(source, "bar", "inner.txt"), b"now a dir\n")
|
||||
@@ -268,13 +272,13 @@ class TestDeleteTimingTypeConflictParity:
|
||||
_write(os.path.join(root, "foo", "inner.txt"), b"was a dir\n")
|
||||
_write(os.path.join(root, "bar"), b"was a file\n")
|
||||
|
||||
rsync_dst = os.path.join(TEST_DATA_DIR, "dtc_rsync_dst")
|
||||
rsync_dst = os.path.join(TEST_DATA_DIR, f"dtc_{label}_rsync_dst")
|
||||
seed_dest(rsync_dst)
|
||||
rsync_result = _rsync(["-a", timing, source + "/", rsync_dst + "/"])
|
||||
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||
rsync_tree = _tree(rsync_dst)
|
||||
|
||||
dest = os.path.join(TEST_DATA_DIR, "dtc_dst")
|
||||
dest = os.path.join(TEST_DATA_DIR, f"dtc_{label}_dst")
|
||||
clean_dir(dest)
|
||||
received = get_dest_received_dir(dest, source)
|
||||
seed_dest(received)
|
||||
@@ -292,7 +296,7 @@ class TestDeleteTimingFailure:
|
||||
|
||||
@pytest.mark.parametrize("mt", [False, True])
|
||||
def test_during_removes_delay_preserves_on_failure(self, mt):
|
||||
source, dest, received = _seed_pair("failure", big=True)
|
||||
source, dest, received = _seed_pair(f"failure_mt{int(mt)}", big=True)
|
||||
extra = os.path.join(received, "d", "old_extra")
|
||||
assert os.path.exists(extra)
|
||||
with ServerManager() as server:
|
||||
@@ -313,13 +317,98 @@ class TestDeleteTimingFailure:
|
||||
)
|
||||
|
||||
|
||||
class TestDeleteDelayDeletedCount:
|
||||
"""The reported deleted count must reflect entries actually removed."""
|
||||
|
||||
def test_refilled_deferred_dir_is_not_counted(self):
|
||||
"""A directory snapshotted into a --delete-delay plan that is refilled
|
||||
before the commit survives ENOTEMPTY and must NOT inflate "Number of
|
||||
deleted files" (regression for delete_plan.c counting at snapshot)."""
|
||||
source = os.path.join(TEST_DATA_DIR, "ddc_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "ddc_dst")
|
||||
clean_dir(source)
|
||||
clean_dir(dest)
|
||||
_write(os.path.join(source, "d", "keep.txt"), b"kept payload\n")
|
||||
_write(os.path.join(source, "d", "big.bin"), b"B" * BIG_BYTES)
|
||||
received = get_dest_received_dir(dest, source)
|
||||
extra_dir = os.path.join(received, "d", "extradir")
|
||||
os.makedirs(extra_dir, exist_ok=True)
|
||||
|
||||
def hook():
|
||||
# Runs while big.bin is in flight, after d's delete plan was processed.
|
||||
_write(os.path.join(extra_dir, "new.txt"), b"created mid-transfer\n")
|
||||
|
||||
with ServerManager() as server:
|
||||
server.start(extra_args=["--allow-delete"])
|
||||
proxy = _SlicingProxy(server.port, hook=hook, hook_after=MID_TRANSFER_BYTES,
|
||||
throttle=PROXY_THROTTLE, wait_for_reply=True)
|
||||
flags = ["--delete-delay", "--incremental", "--ignore-times", "--stats"]
|
||||
result, _ = run_client(source, dest, flags=flags, port=proxy.port)
|
||||
proxy.finish()
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:400]
|
||||
assert proxy.hook_called.is_set(), "hook never fired"
|
||||
assert os.path.exists(os.path.join(extra_dir, "new.txt")), "late file vanished"
|
||||
deleted = None
|
||||
for line in result.stdout.splitlines():
|
||||
if line.startswith("Number of deleted files:"):
|
||||
deleted = int(line.split(":", 1)[1].split()[0])
|
||||
assert deleted == 0, (deleted, result.stdout)
|
||||
|
||||
|
||||
class TestDeleteDelayMaxDeleteParity:
|
||||
"""--max-delete with --delete-delay: a partial deletion still reports the
|
||||
number of entries actually removed, matching rsync (the exact surviving set
|
||||
can differ; only the count is compared)."""
|
||||
|
||||
@requires_rsync
|
||||
def test_max_delete_count_matches_rsync(self):
|
||||
source = os.path.join(TEST_DATA_DIR, "ddm_src")
|
||||
rsync_dst = os.path.join(TEST_DATA_DIR, "ddm_rsync_dst")
|
||||
clean_dir(source)
|
||||
clean_dir(rsync_dst)
|
||||
_write(os.path.join(source, "d", "keep.txt"), b"keep\n")
|
||||
for i in range(1, 6):
|
||||
_write(os.path.join(rsync_dst, "d", f"e{i}.txt"), f"extra{i}\n".encode())
|
||||
|
||||
rsync_result = _rsync(["-a", "--delete-delay", "--max-delete=2", "--stats",
|
||||
source + "/", rsync_dst + "/"])
|
||||
# rsync exits 25 ("the --max-delete limit stopped deletions").
|
||||
assert rsync_result.returncode == 25, rsync_result.stderr
|
||||
rsync_count = _deleted_count(rsync_result.stdout)
|
||||
assert rsync_count == 2, rsync_result.stdout
|
||||
|
||||
dest = os.path.join(TEST_DATA_DIR, "ddm_dst")
|
||||
clean_dir(dest)
|
||||
received = get_dest_received_dir(dest, source)
|
||||
for i in range(1, 6):
|
||||
_write(os.path.join(received, "d", f"e{i}.txt"), f"extra{i}\n".encode())
|
||||
with ServerManager() as server:
|
||||
server.start(extra_args=["--allow-delete"])
|
||||
result, _ = run_client(
|
||||
source, dest,
|
||||
flags=["--delete-delay", "--max-delete=2", "--stats"],
|
||||
port=server.port,
|
||||
)
|
||||
# A capped --max-delete commit is a successful transfer that both tools
|
||||
# report with exit 25.
|
||||
assert result.returncode == 25, (result.stderr or result.stdout)[:300]
|
||||
assert _deleted_count(result.stdout) == rsync_count, result.stdout
|
||||
|
||||
|
||||
def _deleted_count(text):
|
||||
for line in text.splitlines():
|
||||
if line.startswith("Number of deleted files:"):
|
||||
return int(line.split(":", 1)[1].split()[0])
|
||||
return None
|
||||
|
||||
|
||||
class TestDeleteDelayVsAfterSnapshot:
|
||||
"""A destination entry created after its directory's scan survives under
|
||||
--delete-delay but is removed by --delete-after's fresh end scan."""
|
||||
|
||||
@pytest.mark.parametrize("mt", [False, True])
|
||||
def test_late_created_extra_survives_delay_not_after(self, mt):
|
||||
source, dest, received = _seed_pair("latecreate", big=True)
|
||||
source, dest, received = _seed_pair(f"latecreate_mt{int(mt)}", big=True)
|
||||
old_extra = os.path.join(received, "d", "old_extra")
|
||||
new_extra = os.path.join(received, "d", "new_extra")
|
||||
with ServerManager() as server:
|
||||
@@ -356,3 +445,68 @@ class TestDeleteDelayVsAfterSnapshot:
|
||||
f"{timing} (mt={mt}): new_extra present="
|
||||
f"{os.path.exists(new_extra)}, expected survives={new_survives}"
|
||||
)
|
||||
|
||||
|
||||
class TestDeleteAfterThreadsKeepSet:
|
||||
"""Regression: -m/--threads with the default delete-after timing (plain
|
||||
--delete) must still transmit the keep-set manifest and remove destination
|
||||
extras. PipelineContextSender.delete_suppressed was left uninitialized, so a
|
||||
garbage true silently skipped the manifest under --threads."""
|
||||
|
||||
@pytest.mark.parametrize("delete_flag", ["--delete", "--delete-after"])
|
||||
def test_threads_delete_after_sends_keep_set(self, delete_flag):
|
||||
source, dest, received = _seed_pair("mtkeep")
|
||||
extra = os.path.join(received, "d", "old_extra")
|
||||
assert os.path.exists(extra)
|
||||
with ServerManager() as server:
|
||||
server.start(extra_args=["--allow-delete"])
|
||||
result, _ = run_client(source, dest, flags=["--threads", delete_flag],
|
||||
port=server.port)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||
assert not os.path.exists(extra), (
|
||||
f"{delete_flag} --threads did not remove an extra: keep-set manifest was suppressed"
|
||||
)
|
||||
|
||||
class TestDeleteDelayMaxDeleteRefilledDir:
|
||||
"""--delete-delay charges the --max-delete budget at plan/snapshot time, so a
|
||||
refilled snapshotted directory that survives ENOTEMPTY still spends its slot
|
||||
and a later extra is skipped, while the reported count stays at actual
|
||||
removals.
|
||||
|
||||
The refilled directory is at the destination ROOT (its plan is always sent
|
||||
first) and the skipped extra is under a separate source directory, so the
|
||||
ordering that decides the budget charge is deterministic -- not readdir
|
||||
order. The refill is injected through the byte-barrier proxy so it is
|
||||
causally after the plan frame."""
|
||||
|
||||
def test_budget_charged_at_plan_time(self):
|
||||
source = os.path.join(TEST_DATA_DIR, "ddmb_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "ddmb_dst")
|
||||
clean_dir(source)
|
||||
clean_dir(dest)
|
||||
_write(os.path.join(source, "a", "keep.bin"), b"B" * BIG_BYTES)
|
||||
_write(os.path.join(source, "b", "keep.txt"), b"keep\n")
|
||||
received = get_dest_received_dir(dest, source)
|
||||
refilled_dir = os.path.join(received, "xdir")
|
||||
os.makedirs(refilled_dir, exist_ok=True)
|
||||
later_dir = os.path.join(received, "b", "ydir")
|
||||
os.makedirs(later_dir, exist_ok=True)
|
||||
|
||||
def hook():
|
||||
_write(os.path.join(refilled_dir, "new.txt"), b"created mid-transfer\n")
|
||||
|
||||
with ServerManager() as server:
|
||||
server.start(extra_args=["--allow-delete"])
|
||||
proxy = _SlicingProxy(server.port, hook=hook, hook_after=MID_TRANSFER_BYTES,
|
||||
throttle=PROXY_THROTTLE, wait_for_reply=True)
|
||||
flags = ["--delete-delay", "--max-delete=1", "--incremental", "--ignore-times", "--stats"]
|
||||
result, _ = run_client(source, dest, flags=flags, port=proxy.port)
|
||||
proxy.finish()
|
||||
assert result.returncode == 25, (result.stderr or result.stdout)[:400]
|
||||
assert proxy.hook_called.is_set(), "hook never fired"
|
||||
# The refilled directory still consumes the plan-time budget, so the
|
||||
# later extra is skipped...
|
||||
assert os.path.exists(os.path.join(refilled_dir, "new.txt")), "late file vanished"
|
||||
assert os.path.isdir(later_dir), "later extra was not skipped by the plan-time budget"
|
||||
# ...while the reported count reflects only actual removals (none here).
|
||||
assert _deleted_count(result.stdout) == 0, result.stdout
|
||||
|
||||
@@ -218,7 +218,8 @@ _CASES = [
|
||||
H.Case("files_from", "relative", ["--dirs", "-R"],
|
||||
files_from=("dir1", "sub/x.txt"), layout=H.RELATIVE, ci=True,
|
||||
ref="-d/--dirs + --files-from"),
|
||||
H.Case("dirs_plain", "basic", ["-d"], ref="-d/--dirs (plain)"),
|
||||
H.Case("dirs_plain", "basic", ["-d"], fs_src_suffix="/",
|
||||
ref="-d/--dirs (plain)"),
|
||||
H.Case("empty_dirs_recursive", "empty_dir", ["-a"],
|
||||
ref="recursive empty-directory residual"),
|
||||
H.Case("empty_dirs_files_from", "empty_dir", ["--dirs", "-R"],
|
||||
@@ -232,6 +233,11 @@ _CASES = [
|
||||
["-a", "--iconv=ISO-8859-1,UTF-8"],
|
||||
server_args=("--allow-super", "--iconv=UTF-8"),
|
||||
ref="--iconv conversion (receiver declares its own charset)"),
|
||||
# rsync's spec is LOCAL,REMOTE and the destination end's charset is REMOTE
|
||||
# on a push, so a default server writes the wire (UTF-8) names verbatim.
|
||||
H.Case("iconv_default_server", "iconv",
|
||||
["-a", "--iconv=ISO-8859-1,UTF-8"],
|
||||
ref="--iconv push direction (default receiver charset = REMOTE)"),
|
||||
|
||||
# --- partial ----------------------------------------------------------
|
||||
H.Case("partial_complete", "basic", ["-a", "--partial"], ref="--partial"),
|
||||
|
||||
@@ -36,7 +36,7 @@ from common import ( # noqa: E402
|
||||
verify_transfer,
|
||||
)
|
||||
|
||||
PROTOCOL_VERSION = b"2.26.0"
|
||||
PROTOCOL_VERSION = b"2.27.0"
|
||||
STATUS_MANIFEST = 5
|
||||
STATUS_OK = 0
|
||||
|
||||
|
||||
@@ -583,6 +583,55 @@ class TestRemoteDryRun:
|
||||
assert os.path.exists(extra), f"{flags} deleted an extra in dry-run"
|
||||
assert _snapshot_tree(received) == before, f"{flags} mutated the destination"
|
||||
|
||||
@pytest.mark.skipif(shutil.which("rsync") is None, reason="rsync not installed")
|
||||
def test_dry_run_delete_lines_over_report_residual(self):
|
||||
"""Documented residual (RSYNC_COMPAT.md `-n/--dry-run` row): FastSync's
|
||||
dry-run would-delete report includes the file that is merely being
|
||||
updated (derived from the receiver's STATUS_STATS extras) and, unlike
|
||||
rsync, also reports an excluded-but-protected extra. rsync `-n -i
|
||||
--delete` lists only genuine extras. Pins the residual that keeps the
|
||||
row Divergent."""
|
||||
source = os.path.join(TEST_DATA_DIR, "dryrep_src")
|
||||
rdst = os.path.join(TEST_DATA_DIR, "dryrep_rdst")
|
||||
fdst = os.path.join(TEST_DATA_DIR, "dryrep_fdst")
|
||||
clean_dir(source)
|
||||
clean_dir(rdst)
|
||||
clean_dir(fdst)
|
||||
with open(os.path.join(source, "a.txt"), "wb") as fh:
|
||||
fh.write(b"new content\n")
|
||||
os.utime(os.path.join(source, "a.txt"), (1_700_000_000, 1_700_000_000))
|
||||
for root in (rdst, fdst):
|
||||
with open(os.path.join(root, "a.txt"), "wb") as fh:
|
||||
fh.write(b"old\n")
|
||||
for name, data in (("extra.log", b"log\n"), ("extra.txt", b"extra\n")):
|
||||
with open(os.path.join(root, name), "wb") as fh:
|
||||
fh.write(data)
|
||||
for p in (os.path.join(root, "a.txt"), os.path.join(root, "extra.log"),
|
||||
os.path.join(root, "extra.txt")):
|
||||
os.utime(p, (1_500_000_000, 1_500_000_000))
|
||||
|
||||
r = subprocess.run(["rsync", "-an", "-i", "--delete", "--exclude=*.log",
|
||||
source + "/", rdst + "/"],
|
||||
capture_output=True, text=True,
|
||||
env=dict(os.environ, LC_ALL="C"))
|
||||
assert r.returncode == 0, r.stderr
|
||||
rsync_del = {l.split(None, 1)[1] for l in r.stdout.splitlines()
|
||||
if l.startswith("*deleting")}
|
||||
assert rsync_del == {"extra.txt"}, f"unexpected rsync deleting set: {rsync_del}"
|
||||
|
||||
with ServerManager() as server:
|
||||
server.start(extra_args=["--allow-delete"])
|
||||
result, _ = run_client(source, fdst,
|
||||
flags=["-a", "-n", "-i", "--delete", "--exclude=*.log"],
|
||||
port=server.port)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||
fs_del = {l.split(None, 1)[1] for l in (result.stdout or "").splitlines()
|
||||
if l.startswith("*deleting")}
|
||||
# Documented over-report: the transferred/updated file and the excluded
|
||||
# extra appear in FastSync's would-delete set.
|
||||
assert "a.txt" in fs_del, "residual changed: FastSync no longer over-reports the update"
|
||||
assert "extra.log" in fs_del, "residual changed: FastSync no longer reports excluded extra"
|
||||
|
||||
@pytest.mark.ci
|
||||
def test_remote_dry_run_quiet_is_silent(self, shared_server):
|
||||
source = os.path.join(TEST_DATA_DIR, "remote_dry_quiet_src")
|
||||
@@ -2376,6 +2425,61 @@ class TestTempDir:
|
||||
assert not mismatches, f"Mismatch: {mismatches}"
|
||||
self._assert_clean_scratch(os.path.join(dest, "scratch"))
|
||||
|
||||
@pytest.mark.skipif(shutil.which("rsync") is None, reason="rsync not installed")
|
||||
def test_relative_temp_dir_matches_rsync_absolute_rejected(self):
|
||||
"""Differential: a relative --temp-dir is resolved under the destination
|
||||
by both (rsync 3.4.1 and FastSync), producing identical trees. An
|
||||
absolute --temp-dir is used verbatim by rsync standalone, but the
|
||||
receiver deliberately confines it to the receive root (security
|
||||
invariant), so FastSync rejects it without writing outside the root.
|
||||
"""
|
||||
source = self._make_source("tempdir_diff_src")
|
||||
rdst = os.path.join(TEST_DATA_DIR, "tempdir_diff_rdst")
|
||||
fdst = os.path.join(TEST_DATA_DIR, "tempdir_diff_fdst")
|
||||
clean_dir(rdst)
|
||||
clean_dir(fdst)
|
||||
os.makedirs(os.path.join(rdst, "scratch"), exist_ok=True)
|
||||
os.makedirs(os.path.join(fdst, "scratch"), exist_ok=True)
|
||||
r = subprocess.run(["rsync", "-a", "--temp-dir=scratch", source + "/", rdst + "/"],
|
||||
capture_output=True, text=True,
|
||||
env=dict(os.environ, LC_ALL="C"))
|
||||
assert r.returncode == 0, r.stderr
|
||||
with ServerManager() as server:
|
||||
server.start()
|
||||
result, _ = run_client(source, fdst, flags=["--temp-dir=scratch"],
|
||||
port=server.port)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||
# rsync lays the source contents directly in rdst; FastSync mirrors the
|
||||
# absolute source path below fdst. Compare the mirrored content trees
|
||||
# (the scratch dir lives at each destination root).
|
||||
rtree = sorted(os.path.relpath(os.path.join(dp, n), rdst)
|
||||
for dp, dn, fn in os.walk(rdst)
|
||||
for n in dn + fn if os.path.join(dp, n) != os.path.join(rdst, "scratch"))
|
||||
mirror = get_dest_received_dir(fdst, source)
|
||||
ftree = sorted(os.path.relpath(os.path.join(dp, n), mirror)
|
||||
for dp, dn, fn in os.walk(mirror) for n in dn + fn)
|
||||
assert rtree == ftree, f"relative temp-dir tree mismatch: {rtree} != {ftree}"
|
||||
assert _walk_tmp_files(os.path.join(rdst, "scratch")) == []
|
||||
assert _walk_tmp_files(os.path.join(fdst, "scratch")) == []
|
||||
|
||||
# Absolute temp dir: rsync accepts it; FastSync rejects it safely.
|
||||
abs_scratch = os.path.join(TEST_DATA_DIR, "tempdir_diff_abs")
|
||||
clean_dir(abs_scratch)
|
||||
rdst2 = os.path.join(TEST_DATA_DIR, "tempdir_diff_rdst2")
|
||||
clean_dir(rdst2)
|
||||
r2 = subprocess.run(["rsync", "-a", "--temp-dir=" + abs_scratch, source + "/", rdst2 + "/"],
|
||||
capture_output=True, text=True,
|
||||
env=dict(os.environ, LC_ALL="C"))
|
||||
assert r2.returncode == 0, r2.stderr
|
||||
fdst2 = os.path.join(TEST_DATA_DIR, "tempdir_diff_fdst2")
|
||||
clean_dir(fdst2)
|
||||
with ServerManager() as server:
|
||||
server.start()
|
||||
result2, _ = run_client(source, fdst2, flags=["--temp-dir", abs_scratch],
|
||||
port=server.port)
|
||||
assert result2.returncode != 0, "an absolute --temp-dir must be rejected (confined)"
|
||||
assert os.listdir(abs_scratch) == [], "receiver wrote into an unconfined temp dir"
|
||||
|
||||
def test_default_behavior_has_no_scratch_dir(self, shared_server):
|
||||
source = self._make_source("tempdir_default_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "tempdir_default_dst")
|
||||
@@ -2837,6 +2941,41 @@ class TestDelayUpdates:
|
||||
assert not os.path.isdir(os.path.join(delay_dest, self.STAGING)), \
|
||||
"staging directory left behind after a successful delayed transfer"
|
||||
|
||||
@pytest.mark.skipif(shutil.which("rsync") is None, reason="rsync not installed")
|
||||
def test_delay_updates_staging_name_collision_residual(self):
|
||||
"""Documented residual (RSYNC_COMPAT.md `--delay-updates` row): FastSync
|
||||
uses a fixed `.fastsync-stage` staging name and wipes a pre-existing tree
|
||||
of that name at the start of a delayed run (crash-leftover cleanup),
|
||||
even without `--delete`; rsync leaves a genuine destination entry of that
|
||||
name untouched. Pins the divergence that keeps the row Divergent."""
|
||||
source = self._make_source("delay_collide_src")
|
||||
rdst = os.path.join(TEST_DATA_DIR, "delay_collide_rdst")
|
||||
fdst = os.path.join(TEST_DATA_DIR, "delay_collide_fdst")
|
||||
clean_dir(rdst)
|
||||
clean_dir(fdst)
|
||||
for root in (rdst, fdst):
|
||||
with open(os.path.join(root, "top.txt"), "wb") as fh:
|
||||
fh.write(b"old\n")
|
||||
stage = os.path.join(root, self.STAGING)
|
||||
os.makedirs(stage, exist_ok=True)
|
||||
with open(os.path.join(stage, "keepme.txt"), "wb") as fh:
|
||||
fh.write(b"genuine user data\n")
|
||||
|
||||
r = subprocess.run(["rsync", "-a", "--delay-updates", source + "/", rdst + "/"],
|
||||
capture_output=True, text=True,
|
||||
env=dict(os.environ, LC_ALL="C"))
|
||||
assert r.returncode == 0, r.stderr
|
||||
assert os.path.exists(os.path.join(rdst, self.STAGING, "keepme.txt")), \
|
||||
"rsync removed an unrelated destination entry named like the staging dir"
|
||||
|
||||
with ServerManager() as server:
|
||||
server.start()
|
||||
result, _ = run_client(source, fdst, flags=["--delay-updates"],
|
||||
port=server.port)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||
assert not os.path.exists(os.path.join(fdst, self.STAGING)), \
|
||||
"FastSync did not wipe the reserved staging name (residual changed)"
|
||||
|
||||
@pytest.mark.parametrize("mt", [False, True])
|
||||
def test_delay_updates_incremental_rerun_no_leftovers(self, shared_server, mt):
|
||||
source = self._make_source("delay_rerun_src")
|
||||
@@ -3523,23 +3662,25 @@ class TestMissingArgs:
|
||||
|
||||
|
||||
class TestNoImpliedDirs:
|
||||
"""--no-implied-dirs (only meaningful with -R + --files-from) refuses to
|
||||
place a listed file whose parent directory is not itself listed."""
|
||||
"""--no-implied-dirs (meaningful with -R) omits the source metadata of a
|
||||
listed path's implied parent directories but still creates those parents
|
||||
with default attributes, matching rsync 3.4.1."""
|
||||
|
||||
def _make(self):
|
||||
return _make_relative_source("noimplied_src")
|
||||
|
||||
@pytest.mark.parametrize("mt", [False, True])
|
||||
def test_implied_dir_only_fails_entry(self, shared_server, mt):
|
||||
def test_implied_dir_created_with_default_attrs(self, shared_server, mt):
|
||||
source = self._make()
|
||||
dest = os.path.join(TEST_DATA_DIR, "noimplied_dst")
|
||||
clean_dir(dest)
|
||||
lst = _write_rel_list(b"a/b.txt\n") # "a" itself is not listed
|
||||
flags = ["--files-from", lst, "-R", "--no-implied-dirs"] + (["--threads"] if mt else [])
|
||||
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
|
||||
assert result.returncode != 0, "implied parent directory was not rejected"
|
||||
assert "--no-implied-dirs" in (result.stderr or result.stdout)
|
||||
assert not os.path.exists(os.path.join(dest, "a", "b.txt"))
|
||||
assert result.returncode == 0, \
|
||||
f"implied parent directory was not created: {result.stderr[:200]}"
|
||||
assert os.path.isdir(os.path.join(dest, "a")), "implied parent 'a' was not created"
|
||||
assert _read_file(os.path.join(dest, "a", "b.txt")) == b"nested\n"
|
||||
|
||||
@pytest.mark.parametrize("mt", [False, True])
|
||||
def test_listed_dir_allows_file(self, shared_server, mt):
|
||||
@@ -3666,9 +3807,10 @@ class TestDirs:
|
||||
files.extend(os.path.relpath(os.path.join(root, n), mirror) for n in names)
|
||||
assert files == [], f"--dirs descended into contents: {files}"
|
||||
|
||||
def test_dirs_listed_dir_colliding_with_file_fails(self, shared_server):
|
||||
"""A listed directory that already exists as a regular file at the
|
||||
destination fails the transfer cleanly instead of clobbering the file."""
|
||||
def test_dirs_listed_dir_replaces_blocking_file(self, shared_server):
|
||||
"""rsync parity: a listed directory replaces a regular file already at
|
||||
its destination path (rsync removes the non-directory and creates the
|
||||
directory)."""
|
||||
source = self._make()
|
||||
dest = os.path.join(TEST_DATA_DIR, "dirs_coll_dst")
|
||||
clean_dir(dest)
|
||||
@@ -3678,8 +3820,10 @@ class TestDirs:
|
||||
lst = _write_rel_list(b"dir1\n")
|
||||
result, _ = run_client(source, dest, flags=["--files-from", lst, "--dirs", "-R"],
|
||||
port=shared_server.port)
|
||||
assert result.returncode != 0, "dir entry over an existing file did not fail"
|
||||
assert os.path.isfile(blocker), "blocking regular file was clobbered"
|
||||
assert result.returncode == 0, \
|
||||
f"dir entry over an existing file failed: {(result.stderr or result.stdout)[:300]}"
|
||||
assert os.path.isdir(blocker) and not os.path.islink(blocker), \
|
||||
"blocking regular file was not replaced by the incoming directory"
|
||||
|
||||
|
||||
class TestMkpath:
|
||||
@@ -4463,11 +4607,11 @@ class TestDeletePolicy:
|
||||
@pytest.mark.parametrize("mt", [False, True])
|
||||
@pytest.mark.setpriv
|
||||
def test_ignore_errors_keeps_deletion_active_on_scan_error(self, mt):
|
||||
"""A source I/O error (unreadable subdirectory) aborts the run so no
|
||||
deletion happens by default; --ignore-errors continues, still transfers
|
||||
the readable tree and still deletes, single-threaded and under -m. Run
|
||||
as an unprivileged user so the mode-000 directory is genuinely
|
||||
unreadable."""
|
||||
"""rsync's --ignore-errors semantics: a source I/O error (unreadable
|
||||
subdirectory) makes the run continue and transfer the readable tree, but
|
||||
the default suppresses deletion ("IO error encountered -- skipping file
|
||||
deletion"); --ignore-errors lets deletion proceed. Both exit 23. Run as
|
||||
an unprivileged user so the mode-000 directory is genuinely unreadable."""
|
||||
if os.geteuid() != 0 or shutil.which("setpriv") is None:
|
||||
pytest.skip("requires root + setpriv to drop privileges for the client")
|
||||
tag = f"ioerr_{os.getpid()}_{mt}"
|
||||
@@ -4487,11 +4631,15 @@ class TestDeletePolicy:
|
||||
try:
|
||||
os.chmod(os.path.join(source, "locked"), 0)
|
||||
|
||||
# Default: scan error aborts the run; nothing is deleted.
|
||||
# Default: the scan continues past the unreadable dir and the
|
||||
# readable tree transfers, but deletion is skipped (exit 23).
|
||||
self._write(os.path.join(received, "extra.txt"), b"extra\n")
|
||||
flags = ["--delete"] + (["--threads"] if mt else [])
|
||||
result = self._run_client_as_nobody(source, dest, server.port, flags)
|
||||
assert result.returncode != 0, "unreadable source dir did not fail the run"
|
||||
assert result.returncode == 23, \
|
||||
f"unreadable source dir should exit 23 (got {result.returncode})"
|
||||
assert os.path.exists(os.path.join(received, "top.txt")), \
|
||||
"readable tree did not transfer past the I/O error"
|
||||
assert os.path.exists(os.path.join(received, "extra.txt")), \
|
||||
"default run deleted although the scan hit an I/O error"
|
||||
|
||||
@@ -4499,6 +4647,8 @@ class TestDeletePolicy:
|
||||
self._write(os.path.join(received, "extra.txt"), b"extra\n")
|
||||
flags = ["--delete", "--ignore-errors"] + (["--threads"] if mt else [])
|
||||
result = self._run_client_as_nobody(source, dest, server.port, flags)
|
||||
assert result.returncode == 23, \
|
||||
f"--ignore-errors run should still exit 23 (got {result.returncode})"
|
||||
assert not os.path.exists(os.path.join(received, "extra.txt")), \
|
||||
f"--ignore-errors did not keep deletion active: {result.stderr[:300]}"
|
||||
assert not os.path.exists(os.path.join(received, "locked")), \
|
||||
@@ -6690,11 +6840,10 @@ class TestDirectoryAndSymlinkTimes:
|
||||
|
||||
@pytest.mark.ci
|
||||
@pytest.mark.parametrize("mt", [False, True])
|
||||
def test_preserve_does_not_create_empty_source_dir(self, shared_server, mt):
|
||||
"""P7 Wave D #1: a captured-but-EMPTY source directory is never created
|
||||
at the destination. The scanner records its time (it is transmitted via
|
||||
STATUS_DIR_TIMES), but the receiver treats that entry as record-only, so
|
||||
`-a` keeps the documented "empty dirs are never transferred" behavior."""
|
||||
def test_preserve_creates_empty_source_dir(self, shared_server, mt):
|
||||
"""rsync parity: a recursive `-a` transfer recreates an empty source
|
||||
directory at the destination (the scanner emits it as an explicit
|
||||
directory entry)."""
|
||||
source = os.path.join(TEST_DATA_DIR, f"empty_dir_{'m' if mt else 's'}_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, f"empty_dir_{'m' if mt else 's'}_dst")
|
||||
clean_dir(source)
|
||||
@@ -6705,8 +6854,8 @@ class TestDirectoryAndSymlinkTimes:
|
||||
flags = ["-a"] + (["--threads"] if mt else [])
|
||||
received = self._run(source, dest, flags, shared_server)
|
||||
assert os.path.isfile(os.path.join(received, "keep.txt")), "regular file missing"
|
||||
assert not os.path.lexists(os.path.join(received, "empty_sub")), \
|
||||
f"-a created an empty source directory at {received}/empty_sub"
|
||||
assert os.path.isdir(os.path.join(received, "empty_sub")), \
|
||||
f"-a did not recreate the empty source directory at {received}/empty_sub"
|
||||
|
||||
@pytest.mark.ci
|
||||
@pytest.mark.parametrize("mt", [False, True])
|
||||
@@ -6729,10 +6878,10 @@ class TestDirectoryAndSymlinkTimes:
|
||||
|
||||
@pytest.mark.ci
|
||||
@pytest.mark.parametrize("mt", [False, True])
|
||||
def test_collision_at_dir_time_path_does_not_abort(self, shared_server, mt):
|
||||
"""P7 Wave D #1: a pre-existing regular file at a source-empty-dir's
|
||||
mirror path must not abort the transfer (the old mkdir failed and failed
|
||||
the run) and must not be clobbered."""
|
||||
def test_collision_at_empty_dir_path_replaces_blocker(self, shared_server, mt):
|
||||
"""rsync parity: a pre-existing regular file at a source empty-dir's
|
||||
mirror path is replaced by the incoming directory (rsync removes the
|
||||
non-directory and creates the directory); the run succeeds."""
|
||||
source = os.path.join(TEST_DATA_DIR, f"dirtime_collide_{'m' if mt else 's'}_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, f"dirtime_collide_{'m' if mt else 's'}_dst")
|
||||
clean_dir(source)
|
||||
@@ -6751,10 +6900,8 @@ class TestDirectoryAndSymlinkTimes:
|
||||
assert result.returncode == 0, \
|
||||
f"-a aborted on a pre-existing file at an empty-dir path: " \
|
||||
f"{(result.stderr or result.stdout)[:400]}"
|
||||
assert os.path.isfile(blocker) and not os.path.islink(blocker), \
|
||||
"the pre-existing blocker was replaced by a directory"
|
||||
with open(blocker, "rb") as fh:
|
||||
assert fh.read() == b"pre-existing blocker\n", "the blocker file was clobbered"
|
||||
assert os.path.isdir(blocker) and not os.path.islink(blocker), \
|
||||
"the pre-existing blocker was not replaced by the incoming directory"
|
||||
assert os.path.isfile(os.path.join(received, "keep.txt")), "regular file missing"
|
||||
|
||||
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
"""--iconv=CONVERT_SPEC file-NAME charset conversion integration tests.
|
||||
|
||||
The client converts every source file name from LOCAL to REMOTE before it goes
|
||||
on the wire, and the receiver converts it back from REMOTE to LOCAL, so a
|
||||
source tree using one charset can be written into a destination tree using
|
||||
another (rsync compatibility; content bytes are never touched).
|
||||
rsync's spec is ``--iconv=LOCAL,REMOTE`` (the order is the same push or pull).
|
||||
The sender converts each source name from LOCAL to REMOTE for the wire, and on
|
||||
a PUSH the receiver's charset is the spec's REMOTE half, so it writes the wire
|
||||
bytes verbatim (only a server with its own ``--iconv`` declares a different
|
||||
destination charset and re-converts). Content bytes are never touched.
|
||||
"""
|
||||
import codecs
|
||||
import os
|
||||
import shutil
|
||||
|
||||
@@ -16,6 +18,11 @@ LATIN1_NAME = b"caf\xe9.txt"
|
||||
UTF8_NAME = "caf\u00e9.txt".encode("utf-8")
|
||||
|
||||
|
||||
def _to_utf8(name_bytes):
|
||||
"""The UTF-8 encoding of a name that is stored as ISO-8859-1 bytes."""
|
||||
return codecs.encode(codecs.decode(name_bytes, "iso-8859-1"), "utf-8")
|
||||
|
||||
|
||||
def _make(tag):
|
||||
source = os.path.join(TEST_DATA_DIR, f"iconv_{tag}_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, f"iconv_{tag}_dst")
|
||||
@@ -41,10 +48,10 @@ def _dest_file(source, dest, name):
|
||||
|
||||
|
||||
@pytest.mark.ci
|
||||
def test_iconv_latin1_roundtrip(shared_server):
|
||||
"""A source file whose name is ISO-8859-1 bytes is transferred with
|
||||
--iconv=iso-8859-1,utf-8 and lands on the destination with the ORIGINAL
|
||||
latin1 name (the wire carried it as UTF-8)."""
|
||||
def test_iconv_latin1_to_utf8_dest(shared_server):
|
||||
"""rsync push parity: --iconv=iso-8859-1,utf-8 converts a latin1 source name
|
||||
to the spec's REMOTE (UTF-8) on the wire and the default receiver writes it
|
||||
verbatim, so the destination name is UTF-8 (not the source's latin1)."""
|
||||
source, dest = _make("latin1")
|
||||
_place_bytes(source, LATIN1_NAME)
|
||||
|
||||
@@ -53,8 +60,10 @@ def test_iconv_latin1_roundtrip(shared_server):
|
||||
)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:400]
|
||||
|
||||
dst = _dest_file(source, dest, LATIN1_NAME)
|
||||
assert os.path.exists(dst), f"dest latin1-named file not found under {dest}"
|
||||
dst = _dest_file(source, dest, UTF8_NAME)
|
||||
assert os.path.exists(dst), f"dest UTF-8-named file not found under {dest}"
|
||||
assert not os.path.exists(_dest_file(source, dest, LATIN1_NAME)), \
|
||||
"destination kept the latin1 name instead of the wire (UTF-8) charset"
|
||||
|
||||
|
||||
@pytest.mark.ci
|
||||
@@ -153,7 +162,7 @@ def test_iconv_expanding_name_growth(shared_server):
|
||||
)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:400]
|
||||
|
||||
assert os.path.exists(_dest_file(source, dest, name_bytes))
|
||||
assert os.path.exists(_dest_file(source, dest, _to_utf8(name_bytes)))
|
||||
|
||||
|
||||
def test_iconv_symlink_path_and_target(shared_server):
|
||||
@@ -170,11 +179,13 @@ def test_iconv_symlink_path_and_target(shared_server):
|
||||
)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:400]
|
||||
|
||||
dst_target = _dest_file(source, dest, target)
|
||||
dst_link = _dest_file(source, dest, b"link\xe9")
|
||||
assert os.path.exists(dst_target), "dest latin1 target file missing"
|
||||
assert os.path.islink(dst_link), "dest latin1 symlink missing"
|
||||
assert os.readlink(dst_link) == target, "symlink target not preserved/decoded"
|
||||
utf8_target = _to_utf8(target)
|
||||
utf8_link = _to_utf8(b"link\xe9")
|
||||
dst_target = _dest_file(source, dest, utf8_target)
|
||||
dst_link = _dest_file(source, dest, utf8_link)
|
||||
assert os.path.exists(dst_target), "dest UTF-8 target file missing"
|
||||
assert os.path.islink(dst_link), "dest UTF-8 symlink missing"
|
||||
assert os.readlink(dst_link) == utf8_target, "symlink target not wire-converted"
|
||||
with open(dst_link, "rb") as fh:
|
||||
assert fh.read() == b"t\n"
|
||||
|
||||
@@ -198,8 +209,8 @@ def test_iconv_hardlink_path_and_target(shared_server):
|
||||
)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:400]
|
||||
|
||||
dst_a = _dest_file(source, dest, a)
|
||||
dst_b = _dest_file(source, dest, b)
|
||||
dst_a = _dest_file(source, dest, _to_utf8(a))
|
||||
dst_b = _dest_file(source, dest, _to_utf8(b))
|
||||
assert os.path.exists(dst_a) and os.path.exists(dst_b)
|
||||
assert os.stat(dst_a).st_ino == os.stat(dst_b).st_ino, \
|
||||
"hard-link relationship not preserved across the transfer"
|
||||
@@ -221,16 +232,16 @@ def test_iconv_delete_manifest_consistent(shared_server):
|
||||
flags = ["--iconv=iso-8859-1,utf-8"]
|
||||
result, _ = run_client(source, dest, flags=flags, port=server.port)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:400]
|
||||
assert os.path.exists(_dest_file(source, dest, keep))
|
||||
assert os.path.exists(_dest_file(source, dest, gone))
|
||||
assert os.path.exists(_dest_file(source, dest, _to_utf8(keep)))
|
||||
assert os.path.exists(_dest_file(source, dest, _to_utf8(gone)))
|
||||
|
||||
os.remove(os.path.join(os.fsencode(source), gone))
|
||||
result, _ = run_client(
|
||||
source, dest, flags=flags + ["--delete"], port=server.port
|
||||
)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:400]
|
||||
assert os.path.exists(_dest_file(source, dest, keep)), "kept file deleted"
|
||||
assert not os.path.exists(_dest_file(source, dest, gone)), \
|
||||
assert os.path.exists(_dest_file(source, dest, _to_utf8(keep))), "kept file deleted"
|
||||
assert not os.path.exists(_dest_file(source, dest, _to_utf8(gone))), \
|
||||
"missing file was not deleted"
|
||||
|
||||
|
||||
@@ -247,4 +258,4 @@ def test_iconv_chunk_serialization_blob(shared_server):
|
||||
)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:400]
|
||||
|
||||
assert os.path.exists(_dest_file(source, dest, name))
|
||||
assert os.path.exists(_dest_file(source, dest, _to_utf8(name)))
|
||||
@@ -0,0 +1,459 @@
|
||||
"""Differential parity tests for the option wave (bwlimit, --info=*, -M,
|
||||
--ignore-errors, --filter protect).
|
||||
|
||||
Every differential here runs the SAME scenario with real ``rsync 3.4.1`` and
|
||||
with fastsync and compares the observable result, so the modules are skipped
|
||||
when rsync is unavailable. The privilege-dependent --ignore-errors differential
|
||||
drops the client to an unprivileged uid so a mode-000 source directory is
|
||||
genuinely unreadable; it is marked ``setpriv`` (run as root locally, excluded
|
||||
from the root PR gate exactly like the other privilege tests).
|
||||
"""
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
from common import ( # noqa: E402
|
||||
CLIENT_CMD,
|
||||
TEST_DATA_DIR,
|
||||
ServerManager,
|
||||
clean_dir,
|
||||
get_dest_received_dir,
|
||||
run_client,
|
||||
)
|
||||
|
||||
RSYNC = shutil.which("rsync")
|
||||
requires_rsync = pytest.mark.skipif(RSYNC is None, reason="rsync 3.4.1 not installed")
|
||||
|
||||
|
||||
def _rsync(args, timeout=120, as_nobody=False):
|
||||
env = dict(os.environ, LC_ALL="C")
|
||||
cmd = [RSYNC] + args
|
||||
if as_nobody:
|
||||
cmd = ["setpriv", "--reuid=65534", "--regid=65534", "--clear-groups"] + cmd
|
||||
return subprocess.run(cmd, capture_output=True, text=True, env=env, timeout=timeout)
|
||||
|
||||
|
||||
def _write(path, content):
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
with open(path, "wb") as fh:
|
||||
fh.write(content)
|
||||
|
||||
|
||||
class TestBwlimitParity:
|
||||
"""--bwlimit must accept rsync 3.4.1's spellings and pace like it."""
|
||||
|
||||
ACCEPTED = ["100", "0", "1.5", "100K", "100KB", "100KiB", "1M", "1MB", "1m", "1G", "512"]
|
||||
REJECTED = ["-1", "abc", "1x", "1 000"]
|
||||
|
||||
@requires_rsync
|
||||
@pytest.mark.ci
|
||||
def test_parse_acceptance_matches_rsync(self, shared_server):
|
||||
source = os.path.join(TEST_DATA_DIR, "bwp_src")
|
||||
clean_dir(source)
|
||||
_write(os.path.join(source, "f.txt"), b"payload\n")
|
||||
|
||||
for value in self.ACCEPTED + self.REJECTED:
|
||||
rdst = os.path.join(TEST_DATA_DIR, "bwp_rdst")
|
||||
clean_dir(rdst)
|
||||
rsync_result = _rsync(["-a", "--bwlimit=" + value, source + "/", rdst + "/"])
|
||||
dest = os.path.join(TEST_DATA_DIR, "bwp_dst")
|
||||
clean_dir(dest)
|
||||
result, _ = run_client(source, dest, flags=["-a", "--bwlimit=" + value],
|
||||
port=shared_server.port)
|
||||
assert (result.returncode == 0) == (rsync_result.returncode == 0), (
|
||||
f"--bwlimit={value}: fastsync rc={result.returncode} "
|
||||
f"({(result.stderr or result.stdout)[:120]!r}) "
|
||||
f"rsync rc={rsync_result.returncode} ({rsync_result.stderr[:120]!r})"
|
||||
)
|
||||
|
||||
@requires_rsync
|
||||
@pytest.mark.ci
|
||||
def test_throttle_rate_matches_rsync(self, shared_server):
|
||||
"""A 4 MiB transfer at --bwlimit=2048 (2 MiB/s) must take about the same
|
||||
wall-clock time for both tools (~2 s with rsync's leaky bucket)."""
|
||||
source = os.path.join(TEST_DATA_DIR, "bwt_src")
|
||||
clean_dir(source)
|
||||
_write(os.path.join(source, "big.bin"), os.urandom(4 * 1024 * 1024))
|
||||
dest = os.path.join(TEST_DATA_DIR, "bwt_dst")
|
||||
rdst = os.path.join(TEST_DATA_DIR, "bwt_rdst")
|
||||
|
||||
clean_dir(rdst)
|
||||
start = time.monotonic()
|
||||
rsync_result = _rsync(["-a", "--bwlimit=2048", source + "/", rdst + "/"])
|
||||
rsync_secs = time.monotonic() - start
|
||||
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||
|
||||
clean_dir(dest)
|
||||
result, fast_secs = run_client(source, dest, flags=["-a", "--bwlimit=2048"],
|
||||
port=shared_server.port)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:200]
|
||||
# 4 MiB at 2 MiB/s rendezvous near 2 s. Use a coarse band on each side
|
||||
# (an unthrottled transfer finishes well under 1.5 s) plus a generous
|
||||
# cross-tolerance so a loaded CI runner cannot flake the parity assert.
|
||||
lo, hi = 1.5, 4.5
|
||||
assert lo <= fast_secs <= hi, f"fastsync throttle out of band: {fast_secs:.2f}s"
|
||||
assert lo <= rsync_secs <= hi, f"rsync throttle out of band: {rsync_secs:.2f}s"
|
||||
assert abs(fast_secs - rsync_secs) < 2.0, (
|
||||
f"fastsync {fast_secs:.2f}s vs rsync {rsync_secs:.2f}s"
|
||||
)
|
||||
|
||||
|
||||
def _output_tree(root):
|
||||
clean_dir(root)
|
||||
os.makedirs(os.path.join(root, "sub"))
|
||||
_write(os.path.join(root, "a.txt"), b"top\n")
|
||||
_write(os.path.join(root, "sub", "b.txt"), b"nested\n")
|
||||
os.symlink("a.txt", os.path.join(root, "link"))
|
||||
|
||||
|
||||
class TestInfoParity:
|
||||
"""The --info categories that map to a FastSync event must print rsync's
|
||||
line format."""
|
||||
|
||||
@requires_rsync
|
||||
@pytest.mark.ci
|
||||
def test_info_flist_matches_rsync(self, shared_server):
|
||||
source = os.path.join(TEST_DATA_DIR, "inf_fl_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "inf_fl_dst")
|
||||
rdst = os.path.join(TEST_DATA_DIR, "inf_fl_rdst")
|
||||
_output_tree(source)
|
||||
clean_dir(dest)
|
||||
clean_dir(rdst)
|
||||
rsync_result = _rsync(["-a", "--info=flist", source + "/", rdst + "/"])
|
||||
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||
result, _ = run_client(source, dest, flags=["-a", "--info=flist"],
|
||||
port=shared_server.port)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:200]
|
||||
assert "sending incremental file list" in result.stdout
|
||||
assert "sending incremental file list" in rsync_result.stdout
|
||||
|
||||
@requires_rsync
|
||||
@pytest.mark.ci
|
||||
def test_info_name_matches_rsync(self, shared_server):
|
||||
source = os.path.join(TEST_DATA_DIR, "inf_nm_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "inf_nm_dst")
|
||||
rdst = os.path.join(TEST_DATA_DIR, "inf_nm_rdst")
|
||||
_output_tree(source)
|
||||
clean_dir(dest)
|
||||
clean_dir(rdst)
|
||||
rsync_result = _rsync(["-a", "--info=name", source + "/", rdst + "/"])
|
||||
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||
result, _ = run_client(source, dest, flags=["-a", "--info=name"],
|
||||
port=shared_server.port)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:200]
|
||||
|
||||
def entries(text):
|
||||
# Compare the transferred entries only: rsync also prints the
|
||||
# transfer-root `./` and every directory (FastSync records dirs),
|
||||
# which are a separate documented divergence.
|
||||
out = []
|
||||
for line in text.splitlines():
|
||||
if not line or line.startswith("sending ") or line.startswith("created "):
|
||||
continue
|
||||
if line == "./" or line.endswith("/"):
|
||||
continue
|
||||
out.append(line)
|
||||
return sorted(out)
|
||||
|
||||
assert entries(result.stdout) == entries(rsync_result.stdout), (
|
||||
f"rsync={entries(rsync_result.stdout)} fastsync={entries(result.stdout)}"
|
||||
)
|
||||
|
||||
@requires_rsync
|
||||
@pytest.mark.ci
|
||||
def test_info_nonreg_matches_rsync(self, shared_server):
|
||||
source = os.path.join(TEST_DATA_DIR, "inf_nr_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "inf_nr_dst")
|
||||
rdst = os.path.join(TEST_DATA_DIR, "inf_nr_rdst")
|
||||
clean_dir(source)
|
||||
os.mkfifo(os.path.join(source, "fifo"))
|
||||
_write(os.path.join(source, "a.txt"), b"a\n")
|
||||
clean_dir(dest)
|
||||
clean_dir(rdst)
|
||||
rsync_result = _rsync(["-rlt", "--info=nonreg", source + "/", rdst + "/"])
|
||||
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||
result, _ = run_client(source, dest, flags=["-rlt", "--info=nonreg"],
|
||||
port=shared_server.port)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:200]
|
||||
rsync_lines = sorted(l for l in rsync_result.stdout.splitlines()
|
||||
if l.startswith("skipping non-regular"))
|
||||
fast_lines = sorted(l for l in result.stdout.splitlines()
|
||||
if l.startswith("skipping non-regular"))
|
||||
assert fast_lines == rsync_lines, (rsync_lines, fast_lines)
|
||||
assert fast_lines, "no non-regular skip line emitted"
|
||||
|
||||
@requires_rsync
|
||||
@pytest.mark.ci
|
||||
def test_info_del_real_matches_rsync(self, shared_server):
|
||||
source = os.path.join(TEST_DATA_DIR, "inf_dl_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "inf_dl_dst")
|
||||
rdst = os.path.join(TEST_DATA_DIR, "inf_dl_rdst")
|
||||
clean_dir(source)
|
||||
_write(os.path.join(source, "keep.txt"), b"keep\n")
|
||||
clean_dir(rdst)
|
||||
_write(os.path.join(rdst, "extra.txt"), b"x\n")
|
||||
_write(os.path.join(rdst, "extra2.txt"), b"y\n")
|
||||
rsync_result = _rsync(["-a", "--delete", "--info=del", source + "/", rdst + "/"])
|
||||
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||
rsync_lines = sorted(l for l in rsync_result.stdout.splitlines()
|
||||
if l.startswith("deleting "))
|
||||
|
||||
clean_dir(dest)
|
||||
received = get_dest_received_dir(dest, source)
|
||||
_write(os.path.join(received, "extra.txt"), b"x\n")
|
||||
_write(os.path.join(received, "extra2.txt"), b"y\n")
|
||||
with ServerManager() as server:
|
||||
server.start(extra_args=["--allow-delete"])
|
||||
result, _ = run_client(source, dest, flags=["-a", "--delete", "--info=del"],
|
||||
port=server.port)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:200]
|
||||
fast_lines = sorted(l for l in result.stdout.splitlines()
|
||||
if l.startswith("deleting "))
|
||||
assert fast_lines == rsync_lines, (rsync_lines, fast_lines)
|
||||
assert fast_lines, "no deletion lines emitted"
|
||||
|
||||
@requires_rsync
|
||||
@pytest.mark.ci
|
||||
def test_info_del_itemize_real_matches_rsync(self, shared_server):
|
||||
source = os.path.join(TEST_DATA_DIR, "inf_di_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "inf_di_dst")
|
||||
rdst = os.path.join(TEST_DATA_DIR, "inf_di_rdst")
|
||||
clean_dir(source)
|
||||
_write(os.path.join(source, "keep.txt"), b"keep\n")
|
||||
clean_dir(rdst)
|
||||
_write(os.path.join(rdst, "extra.txt"), b"x\n")
|
||||
rsync_result = _rsync(["-a", "-i", "--delete", source + "/", rdst + "/"])
|
||||
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||
rsync_lines = sorted(l for l in rsync_result.stdout.splitlines()
|
||||
if l.startswith("*deleting"))
|
||||
|
||||
clean_dir(dest)
|
||||
received = get_dest_received_dir(dest, source)
|
||||
_write(os.path.join(received, "extra.txt"), b"x\n")
|
||||
with ServerManager() as server:
|
||||
server.start(extra_args=["--allow-delete"])
|
||||
result, _ = run_client(source, dest, flags=["-a", "-i", "--delete"],
|
||||
port=server.port)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:200]
|
||||
fast_lines = sorted(l for l in result.stdout.splitlines()
|
||||
if l.startswith("*deleting"))
|
||||
assert fast_lines == rsync_lines, (rsync_lines, fast_lines)
|
||||
|
||||
@requires_rsync
|
||||
@pytest.mark.ci
|
||||
def test_info_del_dry_run_matches_rsync(self, shared_server):
|
||||
source = os.path.join(TEST_DATA_DIR, "inf_dd_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "inf_dd_dst")
|
||||
rdst = os.path.join(TEST_DATA_DIR, "inf_dd_rdst")
|
||||
clean_dir(source)
|
||||
_write(os.path.join(source, "keep.txt"), b"keep\n")
|
||||
clean_dir(rdst)
|
||||
_write(os.path.join(rdst, "extra.txt"), b"x\n")
|
||||
rsync_result = _rsync(["-a", "-n", "--delete", "--info=del", source + "/", rdst + "/"])
|
||||
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||
rsync_lines = sorted(l for l in rsync_result.stdout.splitlines()
|
||||
if l.startswith("deleting "))
|
||||
|
||||
clean_dir(dest)
|
||||
received = get_dest_received_dir(dest, source)
|
||||
_write(os.path.join(received, "extra.txt"), b"x\n")
|
||||
with ServerManager() as server:
|
||||
server.start(extra_args=["--allow-delete"])
|
||||
result, _ = run_client(source, dest, flags=["-a", "-n", "--delete", "--info=del"],
|
||||
port=server.port)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:200]
|
||||
fast_lines = sorted(l for l in result.stdout.splitlines()
|
||||
if l.startswith("deleting "))
|
||||
assert fast_lines == rsync_lines, (rsync_lines, fast_lines)
|
||||
|
||||
@requires_rsync
|
||||
@pytest.mark.ci
|
||||
def test_info_remove_matches_rsync(self, shared_server):
|
||||
tag = "inf_rm"
|
||||
rsync_src = os.path.join(TEST_DATA_DIR, f"{tag}_rsrc")
|
||||
rsync_dst = os.path.join(TEST_DATA_DIR, f"{tag}_rdst")
|
||||
fast_src = os.path.join(TEST_DATA_DIR, f"{tag}_fsrc")
|
||||
fast_dst = os.path.join(TEST_DATA_DIR, f"{tag}_fdst")
|
||||
for root in (rsync_src, rsync_dst, fast_src, fast_dst):
|
||||
clean_dir(root)
|
||||
_write(os.path.join(rsync_src, "a.txt"), b"a\n")
|
||||
_write(os.path.join(rsync_src, "sub", "b.txt"), b"b\n")
|
||||
_write(os.path.join(fast_src, "a.txt"), b"a\n")
|
||||
_write(os.path.join(fast_src, "sub", "b.txt"), b"b\n")
|
||||
|
||||
rsync_result = _rsync(["-a", "--remove-source-files", "--info=remove",
|
||||
rsync_src + "/", rsync_dst + "/"])
|
||||
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||
rsync_lines = sorted(l for l in rsync_result.stdout.splitlines()
|
||||
if l.startswith("sender removed "))
|
||||
|
||||
result, _ = run_client(fast_src, fast_dst,
|
||||
flags=["-a", "--remove-source-files", "--info=remove"],
|
||||
port=shared_server.port)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:200]
|
||||
fast_lines = sorted(l for l in result.stdout.splitlines()
|
||||
if l.startswith("sender removed "))
|
||||
assert fast_lines == rsync_lines, (rsync_lines, fast_lines)
|
||||
assert fast_lines, "no source-removal lines emitted"
|
||||
|
||||
|
||||
class TestIgnoreErrorsParity:
|
||||
"""--ignore-errors: a source I/O error skips deletion by default; the flag
|
||||
lets deletion proceed. Both exit 23. Run the client as an unprivileged user
|
||||
so the mode-000 directory is genuinely unreadable."""
|
||||
|
||||
@pytest.mark.setpriv
|
||||
def test_delete_after_io_error_matches_rsync(self):
|
||||
if os.geteuid() != 0 or shutil.which("setpriv") is None:
|
||||
pytest.skip("requires root + setpriv to drop privileges for the client")
|
||||
tag = f"ie_{os.getpid()}"
|
||||
source = os.path.join(TEST_DATA_DIR, f"{tag}_src")
|
||||
rsync_dst = os.path.join(TEST_DATA_DIR, f"{tag}_rdst")
|
||||
dest = os.path.join(TEST_DATA_DIR, f"{tag}_dst")
|
||||
clean_dir(source)
|
||||
clean_dir(rsync_dst)
|
||||
clean_dir(dest)
|
||||
_write(os.path.join(source, "top.txt"), b"top\n")
|
||||
_write(os.path.join(source, "locked", "blocked.txt"), b"blocked\n")
|
||||
os.chmod(os.path.join(source, "locked"), 0)
|
||||
os.chmod(TEST_DATA_DIR, 0o777)
|
||||
os.chmod(source, 0o755)
|
||||
os.chmod(rsync_dst, 0o777)
|
||||
os.chmod(dest, 0o777)
|
||||
try:
|
||||
for ignore in (False, True):
|
||||
flags = ["-a", "--delete-after"] + (["--ignore-errors"] if ignore else [])
|
||||
# rsync side
|
||||
_write(os.path.join(rsync_dst, "extra.txt"), b"x\n")
|
||||
os.chmod(os.path.join(rsync_dst, "extra.txt"), 0o666)
|
||||
rres = _rsync(flags + [source + "/", rsync_dst + "/"], as_nobody=True)
|
||||
rsync_extra = os.path.exists(os.path.join(rsync_dst, "extra.txt"))
|
||||
|
||||
# fastsync side
|
||||
received = get_dest_received_dir(dest, source)
|
||||
_write(os.path.join(received, "extra.txt"), b"x\n")
|
||||
os.chmod(os.path.join(received, "extra.txt"), 0o666)
|
||||
with ServerManager() as server:
|
||||
server.start(extra_args=["--allow-delete"])
|
||||
fflags = (["--delete", "--ignore-errors"] if ignore else ["--delete"])
|
||||
cmd = CLIENT_CMD + ["--source-dir", source, "--dest-dir", dest,
|
||||
"--save-to-disk", "--server-port", str(server.port)] + fflags
|
||||
fres = subprocess.run(
|
||||
["setpriv", "--reuid=65534", "--regid=65534", "--clear-groups"] + cmd,
|
||||
text=True, capture_output=True)
|
||||
fast_extra = os.path.exists(os.path.join(received, "extra.txt"))
|
||||
|
||||
assert rres.returncode == 23, (ignore, rres.returncode, rres.stderr[:200])
|
||||
assert fres.returncode == 23, (ignore, fres.returncode, fres.stderr[:200])
|
||||
assert rsync_extra == fast_extra, (
|
||||
f"ignore_errors={ignore}: rsync extra={rsync_extra} fastsync extra={fast_extra}"
|
||||
)
|
||||
assert fast_extra is (not ignore), (ignore, fast_extra)
|
||||
finally:
|
||||
os.chmod(os.path.join(source, "locked"), 0o755)
|
||||
|
||||
|
||||
class TestRemoteOptionDaemon:
|
||||
"""rsync forwards -M/--remote-option to its remote process over a daemon
|
||||
connection; FastSync's daemon has no per-connection argv channel and rejects
|
||||
it. This pins the documented divergence with evidence."""
|
||||
|
||||
@requires_rsync
|
||||
def test_rsync_forwards_M_over_daemon_and_fastsync_rejects(self, tmp_path):
|
||||
import socket
|
||||
|
||||
with socket.socket() as probe:
|
||||
probe.bind(("127.0.0.1", 0))
|
||||
port = probe.getsockname()[1]
|
||||
|
||||
module_root = tmp_path / "mod"
|
||||
module_root.mkdir()
|
||||
os.chmod(module_root, 0o777)
|
||||
source = tmp_path / "src"
|
||||
source.mkdir()
|
||||
(source / "a.txt").write_bytes(b"hello\n")
|
||||
conf = tmp_path / "rsyncd.conf"
|
||||
conf.write_text(
|
||||
f"port = {port}\nuse chroot = no\n[m]\npath = {module_root}\nread only = no\n"
|
||||
)
|
||||
daemon = subprocess.Popen(
|
||||
[RSYNC, "--daemon", "--no-detach", "--port", str(port), "--config", str(conf)],
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
try:
|
||||
deadline = time.monotonic() + 5
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with socket.create_connection(("127.0.0.1", port), timeout=0.3):
|
||||
break
|
||||
except OSError:
|
||||
time.sleep(0.05)
|
||||
else:
|
||||
pytest.skip("rsync daemon did not start")
|
||||
|
||||
# A well-formed -M option is forwarded and accepted by the daemon...
|
||||
ok = _rsync(["-a", "-M--safe-links", source.as_posix() + "/",
|
||||
f"rsync://127.0.0.1:{port}/m/"])
|
||||
# ...and a bogus one is rejected ON THE REMOTE with "unknown option",
|
||||
# which proves the option reached the daemon's parser.
|
||||
bogus = _rsync(["-a", "-M--totally-bogus", source.as_posix() + "/",
|
||||
f"rsync://127.0.0.1:{port}/m/"])
|
||||
assert bogus.returncode != 0
|
||||
assert "unknown option" in (bogus.stderr + bogus.stdout), bogus.stderr
|
||||
del ok
|
||||
finally:
|
||||
daemon.terminate()
|
||||
try:
|
||||
daemon.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
daemon.kill()
|
||||
|
||||
# FastSync rejects -M for a non-SSH transport up front.
|
||||
dest = os.path.join(TEST_DATA_DIR, "ro_dst")
|
||||
clean_dir(dest)
|
||||
result, _ = run_client(source.as_posix(), dest, flags=["-a", "-M--safe-links"])
|
||||
assert result.returncode != 0
|
||||
assert "remote-option" in (result.stderr + result.stdout)
|
||||
|
||||
|
||||
class TestFilterProtectDivergence:
|
||||
"""Documented residual: a protect rule that matches only a destination-only
|
||||
entry is not re-derived on the receiver (FastSync derives delete protection
|
||||
from the source scan), so rsync protects the extra but FastSync removes it."""
|
||||
|
||||
@requires_rsync
|
||||
@pytest.mark.ci
|
||||
def test_protect_dest_only_divergence(self, shared_server):
|
||||
source = os.path.join(TEST_DATA_DIR, "fpd_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "fpd_dst")
|
||||
rdst = os.path.join(TEST_DATA_DIR, "fpd_rdst")
|
||||
clean_dir(source)
|
||||
_write(os.path.join(source, "keep.txt"), b"keep\n")
|
||||
clean_dir(rdst)
|
||||
_write(os.path.join(rdst, "extra.log"), b"extra\n")
|
||||
_write(os.path.join(rdst, "other.txt"), b"other\n")
|
||||
|
||||
rsync_result = _rsync(["-a", "--delete", "--filter=P *.log", source + "/", rdst + "/"])
|
||||
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||
assert os.path.exists(os.path.join(rdst, "extra.log")), "rsync did not protect extra.log"
|
||||
|
||||
clean_dir(dest)
|
||||
received = get_dest_received_dir(dest, source)
|
||||
_write(os.path.join(received, "extra.log"), b"extra\n")
|
||||
_write(os.path.join(received, "other.txt"), b"other\n")
|
||||
with ServerManager() as server:
|
||||
server.start(extra_args=["--allow-delete"])
|
||||
result, _ = run_client(source, dest,
|
||||
flags=["-a", "--delete", "--filter=P *.log"],
|
||||
port=server.port)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:200]
|
||||
# Pin the known divergence: FastSync deletes the destination-only file.
|
||||
assert not os.path.exists(os.path.join(received, "extra.log")), (
|
||||
"FastSync now protects destination-only P matches; the --filter row may be "
|
||||
"upgradable to full parity"
|
||||
)
|
||||
assert not os.path.exists(os.path.join(received, "other.txt"))
|
||||
@@ -447,6 +447,43 @@ class TestWireStatsParity:
|
||||
assert fast_frames[0] == rsync_frames[0], (rsync_frames[0], fast_frames[0])
|
||||
assert "(xfr#1," in fast_frames[-1], fast_frames[-1]
|
||||
|
||||
@requires_rsync
|
||||
@pytest.mark.ci
|
||||
def test_progress_leading_root_line_and_to_chk_match_rsync(self, shared_server):
|
||||
"""A single-file transfer: rsync emits the transfer-root `./` name line
|
||||
and a `to-chk=0/2` denominator that counts that root entry. Both must
|
||||
match FastSync byte-for-byte for the deterministic frames."""
|
||||
source = os.path.join(TEST_DATA_DIR, "wire_pgroot_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "wire_pgroot_dst")
|
||||
rdst = os.path.join(TEST_DATA_DIR, "wire_pgroot_rdst")
|
||||
_make_one_file(source, "f.bin", 100)
|
||||
clean_dir(dest)
|
||||
# rsync prints the `./` root line only when the transfer root itself is
|
||||
# created, so make the rsync destination absent. The "created directory"
|
||||
# line it then emits has no FastSync counterpart (different mirror
|
||||
# layout), so only the name/frame lines are compared.
|
||||
shutil.rmtree(rdst, ignore_errors=True)
|
||||
rsync_result = _rsync(["-a", "--progress", source + "/", rdst + "/"])
|
||||
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||
result, _ = run_client(source, dest, flags=["-a", "--progress"],
|
||||
port=shared_server.port)
|
||||
assert result.returncode == 0, result.stderr[:300]
|
||||
|
||||
# subprocess text mode normalizes \r to \n (universal newlines).
|
||||
def lines_of(text):
|
||||
return [ln for ln in text.splitlines() if ln and not ln.startswith("created directory")]
|
||||
|
||||
rsync_lines = lines_of(rsync_result.stdout)
|
||||
fast_lines = lines_of(result.stdout)
|
||||
rsync_names = [ln for ln in rsync_lines if "%" not in ln]
|
||||
fast_names = [ln for ln in fast_lines if "%" not in ln]
|
||||
|
||||
assert rsync_names == ["sending incremental file list", "./", "f.bin"], rsync_names
|
||||
assert fast_names == rsync_names, (rsync_names, fast_names)
|
||||
# The final frame's to-chk denominator must include the source-root entry.
|
||||
assert "to-chk=0/2" in fast_lines[-1], fast_lines[-1]
|
||||
assert fast_lines[-1] == rsync_lines[-1], (rsync_lines[-1], fast_lines[-1])
|
||||
|
||||
@requires_rsync
|
||||
@pytest.mark.ci
|
||||
@pytest.mark.parametrize("mt", [False, True])
|
||||
@@ -488,18 +525,12 @@ class TestWireStatsParity:
|
||||
|
||||
@requires_rsync
|
||||
@pytest.mark.ci
|
||||
def test_stats_file_count_breakdown_residual(self, shared_server):
|
||||
"""Residual (row #3): rsync prints the `Number of files` and
|
||||
`Number of created files` lines with a per-type breakdown
|
||||
(`(reg: X, dir: Y, link: Z)`).
|
||||
|
||||
FastSync cannot reproduce it from what the sender currently knows: the
|
||||
scanner does not put directory entries in the transfer list (directories
|
||||
are created implicitly), and without a per-entry destination-probe the
|
||||
sender cannot tell which entries the receiver newly created. So FastSync
|
||||
prints the bare transferred-entry count. This test pins the divergence
|
||||
explicitly -- the row must not be marked ✅.
|
||||
"""
|
||||
def test_stats_file_count_breakdown_matches_rsync(self, shared_server):
|
||||
"""`Number of files` now carries rsync's per-type breakdown: the scanner
|
||||
accounts directory entries (captured for -a/-t/-p) plus reg/link/special
|
||||
from the transfer list. `Number of created files` still lacks the type
|
||||
breakdown (FastSync cannot tell which entries the receiver newly
|
||||
created), so that residual is pinned separately."""
|
||||
source = os.path.join(TEST_DATA_DIR, "wire_stc_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "wire_stc_dst")
|
||||
rdst = os.path.join(TEST_DATA_DIR, "wire_stc_rdst")
|
||||
@@ -523,15 +554,44 @@ class TestWireStatsParity:
|
||||
f_files = stats_line(result.stdout, "Number of files")
|
||||
f_created = stats_line(result.stdout, "Number of created files")
|
||||
|
||||
# rsync always carries the type breakdown (the source root counts as a
|
||||
# directory; the single regular file as reg).
|
||||
assert re.match(r"Number of files: 2 \(reg: 1, dir: 1\)$", r_files), r_files
|
||||
assert r_files == f_files, (r_files, f_files)
|
||||
# rsync always carries the created type breakdown; FastSync prints the
|
||||
# bare transferred-regular count (documented residual).
|
||||
assert re.match(r"Number of created files: 1 \(reg: 1\)$", r_created), r_created
|
||||
# FastSync prints only the bare count: no directory accounting and no
|
||||
# per-entry "created" knowledge.
|
||||
assert re.fullmatch(r"Number of files: 1", f_files), f_files
|
||||
assert re.fullmatch(r"Number of created files: 1", f_created), f_created
|
||||
|
||||
@requires_rsync
|
||||
@pytest.mark.ci
|
||||
@pytest.mark.parametrize("choice", ["xxh128", "xxh64", "xxh3", "md5", "md4", "sha1", "none"])
|
||||
def test_out_format_C_selected_algorithm_matches_rsync(self, shared_server, choice):
|
||||
"""`%C` must use the algorithm selected by --checksum-choice, not always
|
||||
xxh128, and render it exactly like rsync (big-endian for the 64-bit
|
||||
hashes, high-then-low for xxh128, standard hex for md5/md4/sha1)."""
|
||||
source = os.path.join(TEST_DATA_DIR, f"wire_cc_{choice}_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, f"wire_cc_{choice}_dst")
|
||||
rdst = os.path.join(TEST_DATA_DIR, f"wire_cc_{choice}_rdst")
|
||||
_make_one_file(source, "f.bin", 200000)
|
||||
clean_dir(dest)
|
||||
clean_dir(rdst)
|
||||
fmt = "%C %l %n"
|
||||
rsync_result = _rsync(["-a", "--checksum-choice=" + choice,
|
||||
"--out-format=" + fmt, source + "/", rdst + "/"])
|
||||
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||
result, _ = run_client(source, dest,
|
||||
flags=["-a", "--checksum-choice=" + choice,
|
||||
"--out-format=" + fmt],
|
||||
port=shared_server.port)
|
||||
assert result.returncode == 0, result.stderr[:300]
|
||||
|
||||
def file_lines(text):
|
||||
return [line for line in text.splitlines()
|
||||
if line and not line.rsplit(" ", 1)[-1].endswith("/")]
|
||||
|
||||
assert file_lines(result.stdout) == file_lines(rsync_result.stdout), (
|
||||
f"choice={choice}: rsync={rsync_result.stdout!r} fastsync={result.stdout!r}"
|
||||
)
|
||||
|
||||
@requires_rsync
|
||||
@pytest.mark.ci
|
||||
@pytest.mark.parametrize("mt", [False, True])
|
||||
|
||||
@@ -769,6 +769,47 @@ class TestVerifyAndFlip:
|
||||
assert os.stat(dest_file).st_ino == os.stat(basis_file).st_ino, \
|
||||
"--link-dest must hard-link to the basis file"
|
||||
|
||||
@requires_rsync
|
||||
def test_basis_dir_size_only_content_residual(self, shared_server):
|
||||
"""Documented residual (RSYNC_COMPAT.md basis-dir rows): FastSync
|
||||
xxHash-verifies a basis hit, while rsync's `--size-only` quick check
|
||||
trusts the size alone. With a same-size, different-content basis,
|
||||
rsync links/copies the wrong basis content while FastSync transfers the
|
||||
source. This test pins both observed behaviors (FastSync is stricter,
|
||||
so the rows are reclassified Divergent)."""
|
||||
source = self._src("basissz")
|
||||
rdest = self._dst("basissz_r")
|
||||
fdest = self._dst("basissz_f")
|
||||
with open(os.path.join(source, "f.txt"), "wb") as fh:
|
||||
fh.write(b"AAAA\n")
|
||||
OLD = 1_400_000_000
|
||||
# rsync basis at the transfer-relative path (relative to the dest dir).
|
||||
os.makedirs(os.path.join(rdest, "basis"), exist_ok=True)
|
||||
with open(os.path.join(rdest, "basis", "f.txt"), "wb") as fh:
|
||||
fh.write(b"BBBB\n")
|
||||
os.utime(os.path.join(rdest, "basis", "f.txt"), (OLD, OLD))
|
||||
rs = _rsync(["-a", "--size-only", "--link-dest=basis", source + "/", rdest + "/"])
|
||||
assert rs.returncode == 0, rs.stderr
|
||||
with open(os.path.join(rdest, "f.txt"), "rb") as fh:
|
||||
assert fh.read() == b"BBBB\n", "rsync --size-only did not trust the basis size"
|
||||
|
||||
# FastSync basis is relative to the receive root; the file mirrors the
|
||||
# source path.
|
||||
rel = os.path.abspath(source).lstrip(os.sep)
|
||||
basis = os.path.join(fdest, "basis", rel)
|
||||
os.makedirs(basis, exist_ok=True)
|
||||
with open(os.path.join(basis, "f.txt"), "wb") as fh:
|
||||
fh.write(b"BBBB\n")
|
||||
os.utime(os.path.join(basis, "f.txt"), (OLD, OLD))
|
||||
received = get_dest_received_dir(fdest, source)
|
||||
result, _ = run_client(source, fdest,
|
||||
flags=["-a", "--size-only", "--link-dest=basis", "--incremental"],
|
||||
port=shared_server.port)
|
||||
assert result.returncode == 0, result.stderr[:300]
|
||||
with open(os.path.join(received, "f.txt"), "rb") as fh:
|
||||
assert fh.read() == b"AAAA\n", \
|
||||
"FastSync must verify the basis content and transfer the source"
|
||||
|
||||
|
||||
class TestIgnoreExistingShortCircuit:
|
||||
"""#9: --ignore-existing is decided by the receiver during the per-file
|
||||
|
||||
@@ -111,6 +111,45 @@ class TestRelativeGeneral:
|
||||
assert int(rs.st_mtime) == int(fs.st_mtime), \
|
||||
f"mtime mismatch for {rel} with {extra}"
|
||||
|
||||
@requires_rsync
|
||||
@pytest.mark.ci
|
||||
def test_no_implied_dirs_files_from_matches_rsync(self, shared_server):
|
||||
"""-R --no-implied-dirs --files-from: a listed file whose parent is not
|
||||
itself listed still transfers; the implied parent is created with
|
||||
default attributes (rsync 3.4.1 parity)."""
|
||||
source = _make_tree(os.path.join(TEST_DATA_DIR, "sel_nidff_src"))
|
||||
# Make the implied parent unmistakably non-default on the source so a
|
||||
# wrongly-applied attribute would be observable.
|
||||
os.chmod(os.path.join(source, "foo"), 0o700)
|
||||
os.chmod(os.path.join(source, "foo", "bar"), 0o711)
|
||||
os.utime(os.path.join(source, "foo"), (978307200, 978307200))
|
||||
os.utime(os.path.join(source, "foo", "bar"), (978307200, 978307200))
|
||||
lst = os.path.join(TEST_DATA_DIR, "sel_nidff_list")
|
||||
with open(lst, "w") as fh:
|
||||
fh.write("foo/bar/baz/f.txt\n")
|
||||
dest = os.path.join(TEST_DATA_DIR, "sel_nidff_dst")
|
||||
rdst = os.path.join(TEST_DATA_DIR, "sel_nidff_rdst")
|
||||
clean_dir(dest)
|
||||
clean_dir(rdst)
|
||||
r = _rsync(["-rlpt", "-R", "--no-implied-dirs", "--files-from=" + lst,
|
||||
source + "/", rdst + "/"])
|
||||
assert r.returncode == 0, r.stderr
|
||||
result, _ = run_client(source, dest, flags=[
|
||||
"-rlpt", "-R", "--no-implied-dirs", "--files-from", lst],
|
||||
port=shared_server.port)
|
||||
assert result.returncode == 0, result.stderr[:300]
|
||||
assert _tree(rdst) == _tree(dest), "implied-parent layout mismatch"
|
||||
# The implied parents exist on both sides and carry the run-time default
|
||||
# attributes, not the source's (non-default) ones.
|
||||
for rel in ("foo", "foo/bar", "foo/bar/baz"):
|
||||
rs = os.stat(os.path.join(rdst, rel))
|
||||
fs = os.stat(os.path.join(dest, rel))
|
||||
assert (rs.st_mode & 0o7777) == (fs.st_mode & 0o7777), \
|
||||
f"mode mismatch for implied {rel}"
|
||||
# The listed file is transferred with its content.
|
||||
with open(os.path.join(dest, "foo", "bar", "baz", "f.txt"), "rb") as fh:
|
||||
assert fh.read() == b"deep\n"
|
||||
|
||||
|
||||
class TestDirsOneLevel:
|
||||
"""#13: -d with a trailing slash (or '.') lists the source's immediate
|
||||
|
||||
@@ -94,14 +94,14 @@ def _seed_protocol_source(source):
|
||||
class TestProtocol:
|
||||
@pytest.mark.ci
|
||||
def test_protocol_current_version_accepted(self, shared_server):
|
||||
"""--protocol=2.26.0 (the current PROTOCOL_VERSION) is accepted and the
|
||||
"""--protocol=2.27.0 (the current PROTOCOL_VERSION) is accepted and the
|
||||
transfer completes normally."""
|
||||
source = os.path.join(TEST_DATA_DIR, "proto_ok_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "proto_ok_dst")
|
||||
shutil.rmtree(dest, ignore_errors=True)
|
||||
os.makedirs(dest)
|
||||
_seed_protocol_source(source)
|
||||
result, _ = run_client(source, dest, flags=["--protocol=2.26.0"],
|
||||
result, _ = run_client(source, dest, flags=["--protocol=2.27.0"],
|
||||
port=shared_server.port)
|
||||
assert result.returncode == 0, \
|
||||
f"--protocol current run failed: {(result.stderr or result.stdout)[:400]}"
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
#include "test_daemon_conf.h"
|
||||
#include "test_daemon_limits.h"
|
||||
#include "test_delay_updates.h"
|
||||
#include "test_delete_plan.h"
|
||||
#include "test_delta.h"
|
||||
#include "test_file.h"
|
||||
#include "test_file_list.h"
|
||||
@@ -66,6 +67,7 @@ int main() {
|
||||
RUN_TEST(test_scanner);
|
||||
RUN_TEST(test_checksum);
|
||||
RUN_TEST(test_delta);
|
||||
RUN_TEST(test_delete_plan);
|
||||
RUN_TEST(test_data);
|
||||
RUN_TEST(test_protocol);
|
||||
RUN_TEST(test_protocol_error);
|
||||
|
||||
@@ -177,6 +177,40 @@ static void test_change_list_enabled() {
|
||||
config_delete(config); /* closes config->log_file */
|
||||
}
|
||||
|
||||
/* %C uses the negotiated TRANSFER checksum's column width (not the pre-transfer
|
||||
* whole-file digest), and `none` renders as a blank 2-char column, matching
|
||||
* rsync. A not-yet-filled checksum renders as spaces. */
|
||||
static void test_format_C_padding_uses_transfer_algo() {
|
||||
ChangeEvent event = sample_event();
|
||||
Config* config = config_create();
|
||||
EXPECT_NOT_NULL(config);
|
||||
/* Deliberately different pre-transfer algorithm: the transfer one must win. */
|
||||
config->checksum_algo = (int)CHECKSUM_ALGO_MD4;
|
||||
struct {
|
||||
int algo;
|
||||
int width;
|
||||
} cases[] = {
|
||||
{CHECKSUM_ALGO_XXH128, 32}, {CHECKSUM_ALGO_XXH64, 16}, {CHECKSUM_ALGO_XXH3, 16},
|
||||
{CHECKSUM_ALGO_MD5, 32}, {CHECKSUM_ALGO_MD4, 32}, {CHECKSUM_ALGO_SHA1, 40},
|
||||
{CHECKSUM_ALGO_NONE, 2},
|
||||
};
|
||||
for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
|
||||
config->checksum_transfer_algo = cases[i].algo;
|
||||
char expected[64];
|
||||
size_t n = 0;
|
||||
expected[n++] = '[';
|
||||
for (int j = 0; j < cases[i].width; j++)
|
||||
expected[n++] = ' ';
|
||||
expected[n++] = ']';
|
||||
expected[n] = '\0';
|
||||
char* line = change_render_format("[%C]", config, &event);
|
||||
EXPECT_NOT_NULL(line);
|
||||
EXPECT_EQ_STR(line, expected);
|
||||
free(line);
|
||||
}
|
||||
config_delete(config);
|
||||
}
|
||||
|
||||
void test_change_list() {
|
||||
test_format_tokens();
|
||||
test_format_unknown_tokens_preserved();
|
||||
@@ -188,4 +222,5 @@ void test_change_list() {
|
||||
test_render_itemize_up_to_date_is_empty();
|
||||
test_render_list_line();
|
||||
test_change_list_enabled();
|
||||
test_format_C_padding_uses_transfer_algo();
|
||||
}
|
||||
|
||||
+37
-1
@@ -1,7 +1,9 @@
|
||||
#include "test_checksum.h"
|
||||
#include "checksum.h"
|
||||
#include "test_utils.h"
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <unistd.h>
|
||||
|
||||
/* Known xxHash64 vector (seed 0) for the empty string and a literal.
|
||||
* The md5 vectors are the standard NIST/RFC1321 test strings. These pin the
|
||||
@@ -230,6 +232,39 @@ static void test_checksum_null_empty_digest() {
|
||||
EXPECT_TRUE(memcmp(a, b, alen) == 0);
|
||||
}
|
||||
|
||||
/* Every algorithm checksum_digest_file() claims to support must produce the
|
||||
* SAME digest as the in-memory one-shot, including the newly added md4/sha1/
|
||||
* none. A 200000-byte payload forces several 64 KiB streaming reads. */
|
||||
static void test_checksum_digest_file_matches_oneshot(void) {
|
||||
static const ChecksumAlgo algos[] = {
|
||||
CHECKSUM_ALGO_XXH64, CHECKSUM_ALGO_XXH3, CHECKSUM_ALGO_XXH128, CHECKSUM_ALGO_MD5,
|
||||
CHECKSUM_ALGO_MD4, CHECKSUM_ALGO_SHA1, CHECKSUM_ALGO_NONE,
|
||||
};
|
||||
enum { SIZE = 200000 };
|
||||
uint8_t* data = malloc(SIZE);
|
||||
EXPECT_NOT_NULL(data);
|
||||
for (int i = 0; i < SIZE; i++)
|
||||
data[i] = (uint8_t)((i * 7 + 3) & 0xff);
|
||||
char path[] = "/tmp/fastsync_ck_XXXXXX";
|
||||
int fd = mkstemp(path);
|
||||
EXPECT_TRUE(fd >= 0);
|
||||
ssize_t written = write(fd, data, SIZE);
|
||||
close(fd);
|
||||
EXPECT_EQ_INT((int)written, SIZE);
|
||||
for (size_t a = 0; a < sizeof(algos) / sizeof(algos[0]); a++) {
|
||||
ChecksumAlgo algo = algos[a];
|
||||
uint8_t one[CHECKSUM_MAX_DIGEST_LEN];
|
||||
uint8_t file[CHECKSUM_MAX_DIGEST_LEN];
|
||||
size_t one_len = 0, file_len = 0;
|
||||
EXPECT_TRUE(checksum_digest(algo, 0, data, SIZE, one, sizeof(one), &one_len));
|
||||
EXPECT_TRUE(checksum_digest_file(algo, 0, path, file, sizeof(file), &file_len));
|
||||
EXPECT_EQ_INT((int)file_len, (int)one_len);
|
||||
EXPECT_TRUE(memcmp(one, file, one_len) == 0);
|
||||
}
|
||||
unlink(path);
|
||||
free(data);
|
||||
}
|
||||
|
||||
void test_checksum(void) {
|
||||
test_checksum_xxh64_seed0();
|
||||
test_checksum_xxh64_empty();
|
||||
@@ -245,4 +280,5 @@ void test_checksum(void) {
|
||||
test_checksum_xxh3_xxh128();
|
||||
test_checksum_truncated_buffer_rejected();
|
||||
test_checksum_null_empty_digest();
|
||||
}
|
||||
test_checksum_digest_file_matches_oneshot();
|
||||
}
|
||||
|
||||
+141
-7
@@ -7,6 +7,7 @@
|
||||
#include "delta.h"
|
||||
#include "file_list.h"
|
||||
#include "log.h"
|
||||
#include "protocol.h"
|
||||
#include "test_utils.h"
|
||||
#include "utils.h"
|
||||
#include <pwd.h>
|
||||
@@ -317,7 +318,7 @@ static void test_parse_args_protocol_accept_current() {
|
||||
Config* cfg = valid_client_config();
|
||||
EXPECT_NOT_NULL(cfg);
|
||||
char* argv_equals[] = {"fastsync", "--source-dir", "/src",
|
||||
"--dest-dir", "/dst", "--protocol=2.26.0"};
|
||||
"--dest-dir", "/dst", "--protocol=2.27.0"};
|
||||
int positional_args[2];
|
||||
int positional_count = 0;
|
||||
EXPECT_EQ_INT(parse_args(cfg, 6, argv_equals, positional_args, &positional_count), 0);
|
||||
@@ -327,7 +328,7 @@ static void test_parse_args_protocol_accept_current() {
|
||||
cfg = valid_client_config();
|
||||
EXPECT_NOT_NULL(cfg);
|
||||
char* argv_space[] = {"fastsync", "--source-dir", "/src", "--dest-dir",
|
||||
"/dst", "--protocol", "2.26.0"};
|
||||
"/dst", "--protocol", "2.27.0"};
|
||||
positional_count = 0;
|
||||
EXPECT_EQ_INT(parse_args(cfg, 7, argv_space, positional_args, &positional_count), 0);
|
||||
EXPECT_EQ_STR(cfg->version, PROTOCOL_VERSION);
|
||||
@@ -1327,7 +1328,7 @@ static void test_parse_args_rejects_invalid_info_flag() {
|
||||
config_delete(cfg);
|
||||
}
|
||||
|
||||
/* rsync's info "name" category maps to fastsync's per-file name logging, and
|
||||
/* rsync's info "name" category maps to fastsync's per-file name output, and
|
||||
* --info=help prints the flag list and exits without error. */
|
||||
static void test_parse_args_info_name_and_help() {
|
||||
Config* cfg = config_create();
|
||||
@@ -1335,7 +1336,7 @@ static void test_parse_args_info_name_and_help() {
|
||||
int positional_args[2];
|
||||
int positional_count = 0;
|
||||
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
|
||||
EXPECT_EQ_INT(cfg->info_level, LOG_INFO_COPY);
|
||||
EXPECT_EQ_INT(cfg->info_level, LOG_INFO_NAME);
|
||||
config_delete(cfg);
|
||||
|
||||
cfg = config_create();
|
||||
@@ -1345,8 +1346,10 @@ static void test_parse_args_info_name_and_help() {
|
||||
config_delete(cfg);
|
||||
}
|
||||
|
||||
/* rsync 3.4.1's remaining --info/--debug categories parse successfully but
|
||||
* have no FastSync output wired to them, so they must not set any log flag. */
|
||||
/* rsync 3.4.1's full --info/--debug vocabulary parses. The info categories
|
||||
* with a FastSync event set their flag; the remaining rsync-only categories
|
||||
* (backup/mount/symsafe/syms) parse but stay silent. Every --debug category
|
||||
* listed here is FastSync-silent, so debug_level stays 0. */
|
||||
static void test_parse_args_rsync_flag_vocabulary_accepted() {
|
||||
Config* cfg = config_create();
|
||||
char* argv[] = {"fastsync", "--info=backup,del,flist,mount,nonreg,progress,remove,symsafe,syms",
|
||||
@@ -1358,7 +1361,8 @@ static void test_parse_args_rsync_flag_vocabulary_accepted() {
|
||||
int positional_count = 0;
|
||||
|
||||
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
|
||||
EXPECT_EQ_INT(cfg->info_level, 0);
|
||||
EXPECT_EQ_INT(cfg->info_level, LOG_INFO_DEL | LOG_INFO_FLIST | LOG_INFO_NONREG |
|
||||
LOG_INFO_PROGRESS | LOG_INFO_REMOVE);
|
||||
EXPECT_EQ_INT(cfg->debug_level, 0);
|
||||
config_delete(cfg);
|
||||
}
|
||||
@@ -3870,6 +3874,113 @@ static void test_parse_args_unsigned_options_reject_sign() {
|
||||
config_delete(cfg);
|
||||
}
|
||||
|
||||
/* --bwlimit must parse with rsync 3.4.1's units and quantization: a bare value
|
||||
* is KiB/s, K/M/G/T/P are binary multipliers, KB/MB are decimal, KiB/MiB are
|
||||
* binary, decimals are rounded to whole KiB like rsync's (size + 512) / 1024,
|
||||
* and 0 (or an empty value) means "no limit". */
|
||||
static void test_parse_args_bwlimit_rsync_units() {
|
||||
struct {
|
||||
const char* value;
|
||||
unsigned long long expected; /* bytes/sec */
|
||||
int ok;
|
||||
} cases[] = {
|
||||
{"100", 100ULL * 1024, 1},
|
||||
{"0", 0, 1},
|
||||
{"", 0, 1},
|
||||
{"1.5", 2ULL * 1024, 1},
|
||||
{"100K", 100ULL * 1024, 1},
|
||||
{"100KiB", 100ULL * 1024, 1},
|
||||
{"100KB", (100000ULL + 512) / 1024 * 1024, 1},
|
||||
{"1M", 1024ULL * 1024, 1},
|
||||
{"1MB", (1000000ULL + 512) / 1024 * 1024, 1},
|
||||
{"1.5m", 1536ULL * 1024, 1},
|
||||
{"1G", 1024ULL * 1024 * 1024, 1},
|
||||
{"1000B", (1000ULL + 512) / 1024 * 1024, 1},
|
||||
{"100B", 0, 0}, /* below the 512-byte floor (not 0) */
|
||||
{"0.4", 0, 0}, /* 409 bytes, below the floor */
|
||||
{"511", 511ULL * 1024, 1},
|
||||
{"-1", 0, 0},
|
||||
{"abc", 0, 0},
|
||||
{"1x", 0, 0},
|
||||
};
|
||||
for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
|
||||
Config* cfg = config_create();
|
||||
EXPECT_NOT_NULL(cfg);
|
||||
int positional_args[2];
|
||||
int positional_count = 0;
|
||||
char option[32];
|
||||
snprintf(option, sizeof(option), "--bwlimit=%s", cases[i].value);
|
||||
char* argv[] = {"fastsync", option, "/src", "/dst"};
|
||||
int rc = parse_args(cfg, 4, argv, positional_args, &positional_count);
|
||||
if (cases[i].ok) {
|
||||
EXPECT_EQ_INT(rc, 0);
|
||||
EXPECT_TRUE(io_get_bwlimit() == cases[i].expected);
|
||||
} else {
|
||||
EXPECT_EQ_INT(rc, -1);
|
||||
}
|
||||
config_delete(cfg);
|
||||
}
|
||||
|
||||
Config* cfg = config_create();
|
||||
EXPECT_NOT_NULL(cfg);
|
||||
int positional_args[2];
|
||||
int positional_count = 0;
|
||||
char* argv[] = {"fastsync", "--bwlimit", "512", "/src", "/dst"};
|
||||
EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0);
|
||||
EXPECT_TRUE(io_get_bwlimit() == 512ULL * 1024);
|
||||
config_delete(cfg);
|
||||
io_set_bwlimit(0);
|
||||
}
|
||||
|
||||
/* Huge/malformed --bwlimit values must be rejected (not accepted or UB) and
|
||||
* oversized-but-representable ones must still be accepted: the scaling used to
|
||||
* be done with an unchecked signed double->long long cast, which is undefined
|
||||
* when the product leaves long long's range. */
|
||||
static void test_parse_args_bwlimit_huge_and_boundary() {
|
||||
struct {
|
||||
const char* value;
|
||||
unsigned long long expected; /* bytes/sec, ignored when !ok */
|
||||
int ok;
|
||||
} cases[] = {
|
||||
/* Malformed / non-numeric prefixes. */
|
||||
{"1e300", 0, 0},
|
||||
{"99999999999999999999999999e3", 0, 0},
|
||||
/* Products that exceed LLONG_MAX at every suffix. */
|
||||
{"99999999999999999999999999", 0, 0},
|
||||
{"99999999999999999999999999K", 0, 0},
|
||||
{"100000000000000000000P", 0, 0},
|
||||
{"99999999999999999999999999999999999999999999999999B", 0, 0},
|
||||
/* `strtod` overflow to +inf must be caught by the isfinite() guard. */
|
||||
{"9999999999999999999999999999999999999999999999999999999999999999999999"
|
||||
"9999999999999999999999999999999999999999999999999999999999999999999999"
|
||||
"99999999999999999999999999999999999999999999999999999999999999999999999",
|
||||
0, 0},
|
||||
/* 2^52 KiB/s: the largest power-of-two scaling that still fits. */
|
||||
{"4503599627370496", 4611686018427387904ULL, 1},
|
||||
/* The +/-1 suffix forms accepted by rsync. */
|
||||
{"1+1", 1024ULL, 1},
|
||||
{"1-1", 1024ULL, 1},
|
||||
};
|
||||
for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
|
||||
Config* cfg = config_create();
|
||||
EXPECT_NOT_NULL(cfg);
|
||||
int positional_args[2];
|
||||
int positional_count = 0;
|
||||
char option[1024];
|
||||
snprintf(option, sizeof(option), "--bwlimit=%s", cases[i].value);
|
||||
char* argv[] = {"fastsync", option, "/src", "/dst"};
|
||||
int rc = parse_args(cfg, 4, argv, positional_args, &positional_count);
|
||||
if (cases[i].ok) {
|
||||
EXPECT_EQ_INT(rc, 0);
|
||||
EXPECT_TRUE(io_get_bwlimit() == cases[i].expected);
|
||||
} else {
|
||||
EXPECT_EQ_INT(rc, -1);
|
||||
}
|
||||
config_delete(cfg);
|
||||
io_set_bwlimit(0);
|
||||
}
|
||||
}
|
||||
|
||||
/* --dry-run must not emit a batch file, so it is rejected alongside
|
||||
* --read-batch/--only-write-batch. */
|
||||
static void test_validate_config_dry_run_rejects_write_batch() {
|
||||
@@ -4394,6 +4505,26 @@ static void test_parse_args_rejects_unsupported_short() {
|
||||
}
|
||||
}
|
||||
|
||||
/* The --ignore-errors deletion gate, unit-tested without a privileged source
|
||||
* directory: a clean scan always deletes; an I/O error suppresses deletion
|
||||
* unless --ignore-errors is set. (The end-to-end mode-000 differential lives in
|
||||
* the setpriv integration test; this pins the decision itself in the PR gate.) */
|
||||
static void test_ignore_errors_allows_delete(void) {
|
||||
Config* cfg = config_create();
|
||||
EXPECT_NOT_NULL(cfg);
|
||||
EXPECT_TRUE(ignore_errors_allows_delete(cfg, false));
|
||||
EXPECT_FALSE(ignore_errors_allows_delete(cfg, true));
|
||||
|
||||
cfg->ignore_errors = true;
|
||||
EXPECT_TRUE(ignore_errors_allows_delete(cfg, false));
|
||||
EXPECT_TRUE(ignore_errors_allows_delete(cfg, true));
|
||||
|
||||
/* A NULL config cannot opt into --ignore-errors. */
|
||||
EXPECT_TRUE(ignore_errors_allows_delete(NULL, false));
|
||||
EXPECT_FALSE(ignore_errors_allows_delete(NULL, true));
|
||||
config_delete(cfg);
|
||||
}
|
||||
|
||||
void test_client_cli() {
|
||||
test_validate_config_required_paths();
|
||||
test_parse_args_numeric_ids();
|
||||
@@ -4579,6 +4710,8 @@ void test_client_cli() {
|
||||
test_parse_args_password_file();
|
||||
test_parse_args_pattern_file_oversized_rejected();
|
||||
test_parse_args_unsigned_options_reject_sign();
|
||||
test_parse_args_bwlimit_rsync_units();
|
||||
test_parse_args_bwlimit_huge_and_boundary();
|
||||
test_validate_config_dry_run_rejects_write_batch();
|
||||
test_parse_args_short_clustering();
|
||||
test_parse_args_attached_short_values();
|
||||
@@ -4587,4 +4720,5 @@ void test_client_cli() {
|
||||
test_parse_args_noop_does_not_consume_argv();
|
||||
test_parse_args_backup_copy_links_shorts();
|
||||
test_parse_args_rejects_unsupported_short();
|
||||
test_ignore_errors_allows_delete();
|
||||
}
|
||||
|
||||
+7
-6
@@ -2831,14 +2831,15 @@ static void golden_config_populate(Config* c) {
|
||||
c->copy_as_gid = 222;
|
||||
}
|
||||
|
||||
/* The pinned golden frame (protocol 2.26.0). The values below are the only
|
||||
/* The pinned golden frame (protocol 2.27.0). The values below are the only
|
||||
* thing that ties the generated table to the historical wire format; update
|
||||
* them ONLY with a PROTOCOL_VERSION bump and a documented reason. The 2.24.0
|
||||
* delete-plan wave changed only the version string; 2.25.0 appended the
|
||||
* report_stats bool and 2.26.0 appended the compression_algo int. The
|
||||
* byte-exact values are recomputed for the merged layout. */
|
||||
#define GOLDEN_WIRE_LEN 705
|
||||
#define GOLDEN_WIRE_HASH 4673424031554175633ULL
|
||||
* report_stats bool, 2.26.0 appended the compression_algo int, and 2.27.0
|
||||
* appended the report_deletes bool. The byte-exact values are recomputed for
|
||||
* the merged layout. */
|
||||
#define GOLDEN_WIRE_LEN 709
|
||||
#define GOLDEN_WIRE_HASH 14423869696887880000ULL
|
||||
|
||||
static unsigned long long fnv1a_64(const unsigned char* buf, size_t len) {
|
||||
unsigned long long h = 1469598103934665603ULL;
|
||||
@@ -2920,7 +2921,7 @@ static unsigned long long capture_wire_hash(const Config* cfg, size_t* out_len)
|
||||
return h;
|
||||
}
|
||||
|
||||
/* Byte-for-byte wire compatibility guard (protocol 2.26.0). The expected hash
|
||||
/* Byte-for-byte wire compatibility guard (protocol 2.27.0). The expected hash
|
||||
* pins the pre-X-macro byte stream; the refactor MUST NOT change it. */
|
||||
static void test_config_wire_golden() {
|
||||
if (is_running_under_valgrind())
|
||||
|
||||
@@ -0,0 +1,225 @@
|
||||
#include "test_delete_plan.h"
|
||||
#include "charset.h"
|
||||
#include "config.h"
|
||||
#include "delete_plan.h"
|
||||
#include "protocol.h"
|
||||
#include "test_utils.h"
|
||||
#include "utils.h"
|
||||
#include <fcntl.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/socket.h>
|
||||
#include <sys/stat.h>
|
||||
#include <unistd.h>
|
||||
|
||||
/* Send one STATUS_DELETE_PLAN body (the leading status is consumed by the
|
||||
* caller/receiver entry point) describing `dir` with no kept children. */
|
||||
static void send_plan_frame(int fd, const char* dir) {
|
||||
EXPECT_TRUE(send_int(fd, 0)); /* has_config */
|
||||
EXPECT_TRUE(send_wire_str(fd, dir));
|
||||
EXPECT_TRUE(send_int(fd, 0)); /* kept child dirs */
|
||||
EXPECT_TRUE(send_int(fd, 0)); /* kept child files */
|
||||
}
|
||||
|
||||
/* --delete-delay: a directory snapshotted into the plan that is refilled before
|
||||
* the commit must NOT be counted as deleted once its unlink fails ENOTEMPTY.
|
||||
* Regression for delete_plan.c counting at snapshot (defer_add) instead of at
|
||||
* the actual removal. */
|
||||
static void test_delete_delay_refilled_dir_not_counted(void) {
|
||||
char root[] = "/tmp/fastsync_dp_refill_XXXXXX";
|
||||
EXPECT_TRUE(mkdtemp(root) != NULL);
|
||||
char extra[1024];
|
||||
snprintf(extra, sizeof(extra), "%s/extra", root);
|
||||
EXPECT_EQ_INT(mkdir(extra, 0700), 0);
|
||||
|
||||
Config* config = config_create();
|
||||
EXPECT_NOT_NULL(config);
|
||||
config->receive_root_directory = str_dup(root);
|
||||
config->use_delete = true;
|
||||
config->delete_delay = true;
|
||||
|
||||
int p[2];
|
||||
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
|
||||
|
||||
DeletePlanSession* session = delete_plan_session_create(config);
|
||||
EXPECT_NOT_NULL(session);
|
||||
send_plan_frame(p[1], ".");
|
||||
EXPECT_EQ_INT(delete_plan_session_receive(session, config, p[0]), 0);
|
||||
/* The empty extra directory was snapshotted, not removed yet. */
|
||||
EXPECT_EQ_INT((int)delete_plan_session_deleted(session), 0);
|
||||
|
||||
/* Refill the directory while the deferred commit is pending. */
|
||||
char refill[1200];
|
||||
snprintf(refill, sizeof(refill), "%s/new.txt", extra);
|
||||
int fd = open(refill, O_WRONLY | O_CREAT | O_TRUNC, 0600);
|
||||
EXPECT_TRUE(fd >= 0);
|
||||
close(fd);
|
||||
|
||||
EXPECT_EQ_INT(delete_plan_session_commit(session, config), DELETE_COMMIT_OK);
|
||||
/* ENOTEMPTY: the directory survives, so it must not be reported as deleted. */
|
||||
EXPECT_EQ_INT((int)delete_plan_session_deleted(session), 0);
|
||||
struct stat st;
|
||||
EXPECT_EQ_INT(lstat(extra, &st), 0);
|
||||
|
||||
delete_plan_session_destroy(session);
|
||||
close(p[0]);
|
||||
close(p[1]);
|
||||
unlink(refill);
|
||||
rmdir(extra);
|
||||
rmdir(root);
|
||||
config_delete(config);
|
||||
}
|
||||
|
||||
/* The complement: a deferred regular extra that DOES get removed is counted. */
|
||||
static void test_delete_delay_removed_file_counted(void) {
|
||||
char root[] = "/tmp/fastsync_dp_file_XXXXXX";
|
||||
EXPECT_TRUE(mkdtemp(root) != NULL);
|
||||
char extra[1024];
|
||||
snprintf(extra, sizeof(extra), "%s/extra.txt", root);
|
||||
int fd = open(extra, O_WRONLY | O_CREAT | O_TRUNC, 0600);
|
||||
EXPECT_TRUE(fd >= 0);
|
||||
close(fd);
|
||||
|
||||
Config* config = config_create();
|
||||
EXPECT_NOT_NULL(config);
|
||||
config->receive_root_directory = str_dup(root);
|
||||
config->use_delete = true;
|
||||
config->delete_delay = true;
|
||||
|
||||
int p[2];
|
||||
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
|
||||
|
||||
DeletePlanSession* session = delete_plan_session_create(config);
|
||||
EXPECT_NOT_NULL(session);
|
||||
send_plan_frame(p[1], ".");
|
||||
EXPECT_EQ_INT(delete_plan_session_receive(session, config, p[0]), 0);
|
||||
EXPECT_EQ_INT((int)delete_plan_session_deleted(session), 0);
|
||||
EXPECT_EQ_INT(delete_plan_session_commit(session, config), DELETE_COMMIT_OK);
|
||||
EXPECT_EQ_INT((int)delete_plan_session_deleted(session), 1);
|
||||
EXPECT_TRUE(lstat(extra, &(struct stat){0}) != 0);
|
||||
|
||||
delete_plan_session_destroy(session);
|
||||
close(p[0]);
|
||||
close(p[1]);
|
||||
rmdir(root);
|
||||
config_delete(config);
|
||||
}
|
||||
|
||||
/* --max-delete still bounds the deferred plan; the actual (removed) count must
|
||||
* not exceed the limit even though more extras existed. */
|
||||
static void test_delete_delay_max_delete_bounds_actual(void) {
|
||||
char root[] = "/tmp/fastsync_dp_max_XXXXXX";
|
||||
EXPECT_TRUE(mkdtemp(root) != NULL);
|
||||
for (int i = 0; i < 3; i++) {
|
||||
char path[1024];
|
||||
snprintf(path, sizeof(path), "%s/e%d.txt", root, i);
|
||||
int fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0600);
|
||||
EXPECT_TRUE(fd >= 0);
|
||||
close(fd);
|
||||
}
|
||||
|
||||
Config* config = config_create();
|
||||
EXPECT_NOT_NULL(config);
|
||||
config->receive_root_directory = str_dup(root);
|
||||
config->use_delete = true;
|
||||
config->delete_delay = true;
|
||||
config->max_delete = 1;
|
||||
|
||||
int p[2];
|
||||
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
|
||||
|
||||
DeletePlanSession* session = delete_plan_session_create(config);
|
||||
EXPECT_NOT_NULL(session);
|
||||
send_plan_frame(p[1], ".");
|
||||
EXPECT_EQ_INT(delete_plan_session_receive(session, config, p[0]), 0);
|
||||
EXPECT_TRUE(delete_plan_session_limit_reached(session));
|
||||
EXPECT_EQ_INT(delete_plan_session_commit(session, config), DELETE_COMMIT_LIMIT_REACHED);
|
||||
EXPECT_EQ_INT((int)delete_plan_session_deleted(session), 1);
|
||||
|
||||
delete_plan_session_destroy(session);
|
||||
close(p[0]);
|
||||
close(p[1]);
|
||||
for (int i = 0; i < 3; i++) {
|
||||
char path[1024];
|
||||
snprintf(path, sizeof(path), "%s/e%d.txt", root, i);
|
||||
unlink(path);
|
||||
}
|
||||
rmdir(root);
|
||||
config_delete(config);
|
||||
}
|
||||
|
||||
/* --max-delete is charged at plan/snapshot time, not at actual removal: a
|
||||
* deferred entry that survives ENOTEMPTY still consumes its budget slot, so a
|
||||
* later directory's extra is skipped even though nothing was actually removed.
|
||||
* The reported count stays 0 (actual removals) while the run is partial. The
|
||||
* two plans are sent as separate frames for "a" then "b", so the ordering that
|
||||
* decides which entry gets the budget is deterministic (unlike readdir order). */
|
||||
static void test_delete_delay_refilled_dir_charges_budget_at_plan(void) {
|
||||
char root[] = "/tmp/fastsync_dp_planbudget_XXXXXX";
|
||||
EXPECT_TRUE(mkdtemp(root) != NULL);
|
||||
char adir[1024], bdir[1024], xdir[1024], ydir[1024];
|
||||
snprintf(adir, sizeof(adir), "%s/a", root);
|
||||
snprintf(bdir, sizeof(bdir), "%s/b", root);
|
||||
snprintf(xdir, sizeof(xdir), "%s/a/x", root);
|
||||
snprintf(ydir, sizeof(ydir), "%s/b/y", root);
|
||||
EXPECT_EQ_INT(mkdir(adir, 0700), 0);
|
||||
EXPECT_EQ_INT(mkdir(bdir, 0700), 0);
|
||||
EXPECT_EQ_INT(mkdir(xdir, 0700), 0);
|
||||
EXPECT_EQ_INT(mkdir(ydir, 0700), 0);
|
||||
|
||||
Config* config = config_create();
|
||||
EXPECT_NOT_NULL(config);
|
||||
config->receive_root_directory = str_dup(root);
|
||||
config->use_delete = true;
|
||||
config->delete_delay = true;
|
||||
config->max_delete = 1;
|
||||
|
||||
int p[2];
|
||||
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
|
||||
|
||||
DeletePlanSession* session = delete_plan_session_create(config);
|
||||
EXPECT_NOT_NULL(session);
|
||||
/* Plan "a" first: its empty extra dir snapshots and charges the budget. */
|
||||
send_plan_frame(p[1], "a");
|
||||
EXPECT_EQ_INT(delete_plan_session_receive(session, config, p[0]), 0);
|
||||
EXPECT_FALSE(delete_plan_session_limit_reached(session));
|
||||
/* Plan "b": the budget is already spent at snapshot time, so b/y is skipped
|
||||
even though a/x has not (and will not) be removed. */
|
||||
send_plan_frame(p[1], "b");
|
||||
EXPECT_EQ_INT(delete_plan_session_receive(session, config, p[0]), 0);
|
||||
EXPECT_TRUE(delete_plan_session_limit_reached(session));
|
||||
EXPECT_EQ_INT((int)delete_plan_session_deleted(session), 0);
|
||||
|
||||
/* Refill a/x so its deferred rmdir fails ENOTEMPTY. */
|
||||
char refill[1200];
|
||||
snprintf(refill, sizeof(refill), "%s/new.txt", xdir);
|
||||
int fd = open(refill, O_WRONLY | O_CREAT | O_TRUNC, 0600);
|
||||
EXPECT_TRUE(fd >= 0);
|
||||
close(fd);
|
||||
|
||||
EXPECT_EQ_INT(delete_plan_session_commit(session, config), DELETE_COMMIT_LIMIT_REACHED);
|
||||
/* Nothing was actually removed, and the plan-time budget still stopped b/y. */
|
||||
EXPECT_EQ_INT((int)delete_plan_session_deleted(session), 0);
|
||||
struct stat st;
|
||||
EXPECT_EQ_INT(lstat(xdir, &st), 0);
|
||||
EXPECT_EQ_INT(lstat(ydir, &st), 0);
|
||||
|
||||
delete_plan_session_destroy(session);
|
||||
close(p[0]);
|
||||
close(p[1]);
|
||||
unlink(refill);
|
||||
rmdir(xdir);
|
||||
rmdir(ydir);
|
||||
rmdir(adir);
|
||||
rmdir(bdir);
|
||||
rmdir(root);
|
||||
config_delete(config);
|
||||
}
|
||||
|
||||
void test_delete_plan(void) {
|
||||
test_delete_delay_refilled_dir_not_counted();
|
||||
test_delete_delay_removed_file_counted();
|
||||
test_delete_delay_max_delete_bounds_actual();
|
||||
test_delete_delay_refilled_dir_charges_budget_at_plan();
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
#ifndef TEST_DELETE_PLAN_H
|
||||
#define TEST_DELETE_PLAN_H
|
||||
|
||||
void test_delete_plan(void);
|
||||
|
||||
#endif
|
||||
+17
-3
@@ -148,10 +148,23 @@ static void test_iconv_wire_sender_converts_local_to_remote() {
|
||||
charset_wire_free();
|
||||
}
|
||||
|
||||
static void test_iconv_wire_receiver_converts_remote_to_local() {
|
||||
/* rsync push parity: with no server --iconv the destination charset is the
|
||||
* client spec's REMOTE half, so the receiver writes the wire bytes verbatim. */
|
||||
static void test_iconv_wire_receiver_default_writes_remote() {
|
||||
EXPECT_TRUE(charset_wire_init_receiver("utf-8,iso-8859-1", NULL));
|
||||
char* local = charset_wire_apply("caf\xe9");
|
||||
EXPECT_NOT_NULL(local);
|
||||
EXPECT_EQ_INT(strcmp(local, "caf\xe9"), 0);
|
||||
free(local);
|
||||
charset_wire_free();
|
||||
}
|
||||
|
||||
/* A server that declares its own --iconv LOCAL converts wire(REMOTE) into that
|
||||
* declared charset (the daemon "charset" analog). */
|
||||
static void test_iconv_wire_receiver_server_local_override() {
|
||||
EXPECT_TRUE(charset_wire_init_receiver("utf-8,iso-8859-1", "utf-8"));
|
||||
char* local = charset_wire_apply("caf\xe9");
|
||||
EXPECT_NOT_NULL(local);
|
||||
EXPECT_EQ_INT(strcmp(local, "caf\xc3\xa9"), 0);
|
||||
free(local);
|
||||
charset_wire_free();
|
||||
@@ -179,7 +192,7 @@ static void test_iconv_wire_str_roundtrip() {
|
||||
close(p[1]);
|
||||
io_set_fds(p[0], p[0]);
|
||||
charset_wire_free();
|
||||
charset_wire_init_receiver("utf-8,iso-8859-1", NULL);
|
||||
charset_wire_init_receiver("utf-8,iso-8859-1", "utf-8");
|
||||
char* got = receive_wire_str(p[0]);
|
||||
bool ok = got != NULL && strcmp(got, "caf\xc3\xa9") == 0;
|
||||
free(got);
|
||||
@@ -211,7 +224,8 @@ void test_iconv() {
|
||||
test_iconv_exact_fill_no_overflow();
|
||||
test_iconv_growth_expanding_name();
|
||||
test_iconv_wire_sender_converts_local_to_remote();
|
||||
test_iconv_wire_receiver_converts_remote_to_local();
|
||||
test_iconv_wire_receiver_default_writes_remote();
|
||||
test_iconv_wire_receiver_server_local_override();
|
||||
test_iconv_wire_disabled_passthrough();
|
||||
// This subtest forks to exercise the wire string handshake; the instrumented
|
||||
// parent is too slow under valgrind for the child's blocking reads.
|
||||
|
||||
@@ -103,6 +103,62 @@ static void test_sender_zero_capacity() {
|
||||
config_delete(cfg);
|
||||
}
|
||||
|
||||
/* Regression (blocker): PipelineContextSender.delete_suppressed must be
|
||||
initialized false. A garbage true silently suppresses the --delete keep-set
|
||||
manifest under -m/--threads, so destination extras would never be removed. */
|
||||
static void test_sender_delete_suppressed_initialized() {
|
||||
Config* cfg = config_create();
|
||||
EXPECT_NOT_NULL(cfg);
|
||||
free(cfg->version);
|
||||
cfg->version = str_dup(PROTOCOL_VERSION);
|
||||
cfg->send_directory = str_dup("/src");
|
||||
cfg->receive_root_directory = str_dup("/dst");
|
||||
|
||||
Queue* q1 = queue_create(5, NULL);
|
||||
Queue* q2 = queue_create(5, NULL);
|
||||
EXPECT_NOT_NULL(q1);
|
||||
EXPECT_NOT_NULL(q2);
|
||||
PipelineContextSender* ctx = pipeline_context_sender_create(cfg, q1, q2);
|
||||
EXPECT_NOT_NULL(ctx);
|
||||
EXPECT_FALSE(ctx->delete_suppressed);
|
||||
pipeline_context_sender_destroy(ctx);
|
||||
config_delete(cfg);
|
||||
}
|
||||
|
||||
/* --info=del (report_deletes) is the only reason the receiver retains the
|
||||
actually-removed paths: a plain --delete receiver must not allocate the list,
|
||||
and with report_deletes armed the observer records into it. */
|
||||
static void test_receiver_deleted_paths_gated_by_report_deletes() {
|
||||
Config* plain = config_create();
|
||||
EXPECT_NOT_NULL(plain);
|
||||
free(plain->version);
|
||||
plain->version = str_dup(PROTOCOL_VERSION);
|
||||
plain->receive_root_directory = str_dup("/dst");
|
||||
plain->report_deletes = false;
|
||||
Queue* q_plain = queue_create(5, file_destroy);
|
||||
EXPECT_NOT_NULL(q_plain);
|
||||
PipelineContextReceiver* ctx_plain = pipeline_context_receiver_create(plain, q_plain, -1, NULL);
|
||||
EXPECT_NOT_NULL(ctx_plain);
|
||||
EXPECT_NULL(ctx_plain->deleted_paths);
|
||||
pipeline_context_receiver_destroy(ctx_plain);
|
||||
|
||||
Config* info = config_create();
|
||||
EXPECT_NOT_NULL(info);
|
||||
free(info->version);
|
||||
info->version = str_dup(PROTOCOL_VERSION);
|
||||
info->receive_root_directory = str_dup("/dst");
|
||||
info->report_deletes = true;
|
||||
Queue* q_info = queue_create(5, file_destroy);
|
||||
EXPECT_NOT_NULL(q_info);
|
||||
PipelineContextReceiver* ctx_info = pipeline_context_receiver_create(info, q_info, -1, NULL);
|
||||
EXPECT_NOT_NULL(ctx_info);
|
||||
EXPECT_NOT_NULL(ctx_info->deleted_paths);
|
||||
receiver_record_deleted_path(ctx_info->deleted_paths, "d/old_extra");
|
||||
EXPECT_EQ_INT(ctx_info->deleted_paths->size, 1);
|
||||
EXPECT_EQ_STR((const char*)ctx_info->deleted_paths->items[0], "d/old_extra");
|
||||
pipeline_context_receiver_destroy(ctx_info);
|
||||
}
|
||||
|
||||
/* Test receiver with zero file_descriptor */
|
||||
static void test_receiver_fd_zero() {
|
||||
Config* cfg = config_create();
|
||||
@@ -244,6 +300,7 @@ static void test_write_thread_done() {
|
||||
* and queue_destroy which would double-free since we created them
|
||||
* in this test. Let me just free the context directly. */
|
||||
array_list_delete(ctx->would_delete);
|
||||
array_list_delete(ctx->deleted_paths);
|
||||
mtx_destroy(&ctx->mutex);
|
||||
cnd_destroy(&ctx->condition_not_full);
|
||||
cnd_destroy(&ctx->condition_not_empty);
|
||||
@@ -329,6 +386,7 @@ static void test_receiver_enqueue_byte_budget() {
|
||||
|
||||
/* Tear down: the second file is still queued and is freed by queue_destroy. */
|
||||
array_list_delete(ctx->would_delete);
|
||||
array_list_delete(ctx->deleted_paths);
|
||||
mtx_destroy(&ctx->mutex);
|
||||
cnd_destroy(&ctx->condition_not_full);
|
||||
cnd_destroy(&ctx->condition_not_empty);
|
||||
@@ -460,6 +518,8 @@ void test_multiprocessing() {
|
||||
test_receiver_create_destroy();
|
||||
test_sender_queue_capacities();
|
||||
test_sender_zero_capacity();
|
||||
test_sender_delete_suppressed_initialized();
|
||||
test_receiver_deleted_paths_gated_by_report_deletes();
|
||||
test_receiver_fd_zero();
|
||||
if (!is_running_under_valgrind()) {
|
||||
test_receive_thread_finished();
|
||||
|
||||
Reference in New Issue
Block a user