Release v2.28.0 #304
@@ -221,6 +221,7 @@ set(TEST_SRCS
|
||||
tests/test_daemon_limits.c
|
||||
tests/test_data.c
|
||||
tests/test_delay_updates.c
|
||||
tests/test_delete_plan.c
|
||||
tests/test_delta.c
|
||||
tests/test_file.c
|
||||
tests/test_file_list.c
|
||||
|
||||
+2
-1
@@ -39,7 +39,8 @@
|
||||
docs state push-only / remote-source unsupported.
|
||||
5. **Preserve-attribute split (protocol 2.22.0)** landed on `feat/preserve-attr-split`: per-attribute `-p/-t/-o/-g` + `--no-*` negations, `-a` = `-rlptgoD`, and the 2.21.0 → 2.22.0 wire bump.
|
||||
6. **Rsync-parity wave (protocol 2.23.0)** on `feat/rsync-parity`: rsync short options/clustering/attached values (`-r`/`-b`/`-L`/`-B`, `-av`, `-aAX`, `-B1000`, `-essh`, `-MOPT`), `-c` checksum quick-check, `--checksum-choice`/`--compress-choice` validation and seed randomization, rsync timeout/max-alloc defaults, temp-dir confinement + `EXDEV` fallback, ownership/mapping parity (numeric-ids modifier, map ranges/`*`/empty-FROM, `--chown`+map conflicts, fake-super resolved-owner record), verbatim symlink storage with rsync `--safe-links`/`--munge-links`, socket recreation under `--specials`, `--chmod` 3.4.1 semantics, and delete scoping + `--max-delete` partial/exit-25. Wire: appended delete-manifest synchronized-directory section and `STATUS_DELETE_LIMIT`.
|
||||
7. **Parity-completion wave (protocol 2.24.0 → 2.26.0)** on `feat/parity-completion`: per-directory delete plans (`STATUS_DELETE_PLAN`) for `--delete-during`/`--delete-delay`; receiver `STATUS_STATS` counters feeding `--stats`/`--progress` and `--out-format %b/%c/%C`, plus `-n --delete` lines; `lz4`/`zlib`/`zlibx` compression and `md4`/`sha1`/`none` checksums with `auto` negotiation (default `xxh128`/`zstd`); general `-R`/`--no-implied-dirs`/`-d`; the full filter grammar (`merge`/`dir-merge`/`hide`/`show`/`protect`/`risk`/`clear` + modifiers) and corrected `-F`/`-FF`; receiver-side `--chown`/map TO-name resolution; absolute basis dirs + `--link-dest` relink; receiver-side `--ignore-existing` short-circuit; `--preallocate` over `--sparse` via `fallocate(2)`; `--iconv=.`/`-`/`--no-iconv`; lone `-h` help; aliases `--ignore-non-existing`/`--protect-args`/`--msgs2stderr`; and the full `--info`/`--debug` vocabulary. `RSYNC_COMPAT.md` reclassifies the matrix to 106 ✅ / 27 ⚠️ / 23 ❌.
|
||||
7. **Parity-completion wave (protocol 2.24.0 → 2.26.0)** on `feat/parity-completion`: per-directory delete plans (`STATUS_DELETE_PLAN`) for `--delete-during`/`--delete-delay`; receiver `STATUS_STATS` counters feeding `--stats`/`--progress` and `--out-format %b/%c/%C`, plus `-n --delete` lines; `lz4`/`zlib`/`zlibx` compression and `md4`/`sha1`/`none` checksums with `auto` negotiation (default `xxh128`/`zstd`); general `-R`/`--no-implied-dirs`/`-d`; the full filter grammar (`merge`/`dir-merge`/`hide`/`show`/`protect`/`risk`/`clear` + modifiers) and corrected `-F`/`-FF`; receiver-side `--chown`/map TO-name resolution; absolute basis dirs + `--link-dest` relink; receiver-side `--ignore-existing` short-circuit; `--preallocate` over `--sparse` via `fallocate(2)`; `--iconv=.`/`-`/`--no-iconv`; lone `-h` help; aliases `--ignore-non-existing`/`--protect-args`/`--msgs2stderr`; and the full `--info`/`--debug` vocabulary. `RSYNC_COMPAT.md` reclassifies the matrix to 106 ✅ / 27 ⚠️ / 23 ❌; the later rsync-parity-stats pass (`fix/parity-stats`) moves it to 107 ✅ / 25 ⚠️ / 24 ❌ (see item 8).
|
||||
8. **rsync-parity-stats pass** on `fix/parity-stats` (no wire change, `PROTOCOL_VERSION` stays `2.26.0`): `--delete-delay` now counts/budgets only entries actually removed (a refilled deferred directory that survives `ENOTEMPTY` is not counted; `--max-delete` partial-delete count matches rsync); `--stats` gained the `(reg/dir/link/special)` `Number of files` breakdown and now counts only regular files actually stored for `Number of regular files transferred`/transferred size/literal data (up-to-date re-runs report 0); `Total file size` includes symlink target lengths; `--progress` prints the leading `./` root line and counts it in `to-chk` so a single-file transfer matches rsync; and `%C` uses the selected transfer checksum with `checksum_digest_file` supporting md4/sha1/none, byte-identical to rsync for every algorithm. `--out-format` reclassified ❌ (`%b`/delta-`%c` are protocol-specific). Differential + regression tests added; full suite + ASan + clang-format + cppcheck clean.
|
||||
|
||||
## Next steps
|
||||
1. **Merge PR #284** (`dev` -> `main`) once reviewed (protected branch).
|
||||
|
||||
+17
-13
@@ -6,9 +6,9 @@ This document maps rsync's full feature set to FastSync's current implementation
|
||||
|
||||
| Status | Count | Description |
|
||||
|--------|-------|-------------|
|
||||
| ✅ Parity | 106 | Reproduces rsync's semantics for this option's scope |
|
||||
| ⚠️ Caveat | 27 | Wired and tested, but carries a documented behavioral difference from rsync (named in the row and/or the wave notes) |
|
||||
| ❌ Divergent | 23 | Rejected, an accepted no-op, deliberately non-rsync (native config/auth/batch, privileged namespaces, safe-subset privilege), or impossible on any portable filesystem call |
|
||||
| ✅ Parity | 107 | Reproduces rsync's semantics for this option's scope |
|
||||
| ⚠️ Caveat | 25 | Wired and tested, but carries a documented behavioral difference from rsync (named in the row and/or the wave notes) |
|
||||
| ❌ Divergent | 24 | Rejected, an accepted no-op, deliberately non-rsync (native config/auth/batch, privileged namespaces, safe-subset privilege), or impossible on any portable filesystem call |
|
||||
| **Total** | **156** | One row per rsync option/feature group; a row may name several spellings |
|
||||
|
||||
This matrix reports honest rsync parity, not "implemented" as a synonym for
|
||||
@@ -64,12 +64,12 @@ Every one of those has an entry below with its remaining caveats.
|
||||
|
||||
| Flag | Rsync Description | FastSync Status | Notes |
|
||||
|------|-------------------|-----------------|-------|
|
||||
| `--stats` | Give transfer stats | ⚠️ Caveat | Prints transfer statistics. Protocol 2.25.0 populates the receiver-only counters the sender cannot observe: `Matched data` (a delta basis's reused bytes) and `Number of deleted files` come from the receiver's `STATUS_STATS` report, and the protocol-independent lines (regular files transferred, total/transferred file size, literal data, matched data, deleted files, file-list size) match rsync exactly in both the sequential and `--threads` paths. **Remaining divergence:** rsync prints `Number of files` and `Number of created files` with a per-type breakdown (`(reg: X, dir: Y, link: Z)`); FastSync prints the bare transferred-entry count because its scanner does not put directory entries in the transfer list and the sender cannot tell which entries the receiver newly created. `Total bytes sent`/`received` are FastSync wire bytes and are not numerically comparable to rsync's |
|
||||
| `--stats` | Give transfer stats | ⚠️ Caveat | Prints transfer statistics. Protocol 2.25.0 populates the receiver-only counters the sender cannot observe: `Matched data` (a delta basis's reused bytes) and `Number of deleted files` come from the receiver's `STATUS_STATS` report. The sender now tracks the scanned file list per type and only counts regular files the receiver actually stored, so `Number of files` carries rsync's `(reg: X, dir: Y, link: Z, special: W)` breakdown (directories come from the scanner's captured directory list, present for `-a`/`-t`/`-p`), `Number of regular files transferred` excludes symlinks/specials and up-to-date files, `Total file size` includes symlink target lengths, and `Total transferred file size`/`Literal data` count only transferred files — all differential-tested in the sequential and `--threads` paths. **Remaining divergences:** `Number of created files` is the transferred-regular count (FastSync cannot tell which entries the receiver newly created, so on an update where rsync reports 0 created FastSync can report the transferred file) and lacks the type breakdown; a recursive scan that preserves no directory attribute (`-r` without `-t`/`-p`) captures no directory entries, so the `dir:` category is then omitted; rsync's per-type breakdown on `Number of deleted files` is not reproduced; and `Total bytes sent`/`received` are FastSync wire bytes framed differently from rsync's, so they are not numerically comparable |
|
||||
| `-h`, `--human-readable` | Human-readable numbers | ✅ Parity | Formats transfer byte and rate counts using rsync's **decimal** (base-1000) units, matching rsync `-h` (e.g. `1.23M`), not binary units. **A lone `-h` with no transfer arguments prints help instead** (protocol 2.26.0), matching the rsync idiom; `-h` alongside a transfer remains human-readable |
|
||||
| `-i`, `--itemize-changes` | Per-file change summary | ✅ Parity | Prints rsync-style `>f+++++++++` lines to stdout only for files actually sent (also under `-j`/`--threads`); unchanged files print nothing, matching single-`-i` behavior |
|
||||
| `--progress` | Show progress | ⚠️ Caveat | Protocol 2.25.0 prints rsync-style per-file progress blocks (percent, transferred/total bytes, rate, elapsed, `(xfr#N, to-chk=M/T)`) fed by the receiver's `STATUS_STATS`, in both the sequential and `--threads` send paths; the first frame for a sub-32 KiB file is byte-identical to rsync. **Remaining divergences:** FastSync does not print rsync's leading `./` whole-transfer line, its `to-chk` total differs by the source-root entry (the scanner does not emit the root directory as a transfer entry), and the rate/ETA are wall-clock dependent, so only the first frame is pinned against rsync |
|
||||
| `--progress` | Show progress | ⚠️ Caveat | Protocol 2.25.0 prints rsync-style per-file progress blocks (percent, transferred/total bytes, rate, elapsed, `(xfr#N, to-chk=M/T)`) fed by the receiver's `STATUS_STATS`, in both the sequential and `--threads` send paths. FastSync now also prints rsync's leading `./` transfer-root line and counts that root entry in the `to-chk` denominator, so a **single-file transfer's name lines and deterministic frames are byte-identical to rsync** (differential test). **Remaining divergences:** for a multi-directory tree rsync prints a per-directory name line and its `to-chk` denominator includes every directory/symlink/special entry; FastSync's streaming scan emits only file-name lines and counts just the root plus transferred files (a full flist pre-count would be needed), and the rate/ETA are wall-clock dependent |
|
||||
| `-P` | Same as --partial --progress | ✅ Parity | Parses to `--partial` + `--progress`. The independent `--partial` retention semantics are rsync parity: an interrupted write retains the already-written temp at the destination (best-effort) so a later `--append`/`--append-verify` can resume. Progress presentation is owned by the `--progress` row; there is no separate `-P` divergence |
|
||||
| `--out-format=FORMAT` | Custom output format | ⚠️ Caveat | Per-transfer template on stdout; tokens `%f` `%n` `%l` `%b` `%c` `%C` `%i` `%M` `%o` `%U` `%G` `%t` `%%`. Protocol 2.25.0 adds the wire counters: `%C` is the whole-file digest (default `xxh128`, seed 0), so `%C %l %n` matches rsync byte-for-byte for a whole-file transfer. **Remaining divergences:** `%b` counts FastSync's own wire bytes (framing and checksum trailer), not rsync's protocol-specific count, so the two are not numerically equal; `%c` matches rsync's 16-byte block-sum header for whole-file transfers but differs in delta mode (each counts its own handshake bytes). **Also:** when `--checksum-choice=xxh64` is selected explicitly, `%C` still prints an xxh128 digest rather than the selected xxh64 (`change_list.c:208-220`) |
|
||||
| `--out-format=FORMAT` | Custom output format | ❌ Divergent | Per-transfer template on stdout; tokens `%f` `%n` `%l` `%b` `%c` `%C` `%i` `%M` `%o` `%U` `%G` `%t` `%%`. `%C` now uses the negotiated transfer algorithm (`--checksum-choice`, default `xxh128`, seed 0) and renders every algorithm exactly like rsync — xxh128 high-then-low, xxh64/xxh3 big-endian, md5/md4/sha1 standard hex, `none` a blank 2-char column — differential-tested across all algorithms. `%f`/`%n`/`%l`/`%i`/`%M`/`%U`/`%G`/`%B` also match. **Reclassified because `%b`/`%c` are protocol-specific and cannot match:** a differential against rsync 3.4.1 shows whole-file `%c = 16` for both, but rsync whole-file `%b = filesize + 27 + transfer-digest-bytes` (39 for a 0-byte file; 43/35/47 for xxh128/xxh64/sha1 on a 12-byte file) while FastSync `%b` counts its own framing; in delta mode rsync `%c = 16 + 6·ceil(filesize/block_size)` (verified at block sizes 512/700/1024/2048) while FastSync counts its own signature handshake, and rsync `%b` is its token stream. FastSync's wire bytes are a different quantity, so exact `%b`/delta-`%c` equality is impossible |
|
||||
| `--log-file=FILE` | Log to file | ✅ Parity | `log_file` config field |
|
||||
| `--log-file-format=FMT` | Log format | ✅ Parity | Requires `--log-file`; writes one template line per transferred file using the same token set as `--out-format` (including `%b` as the wire byte count) |
|
||||
| `--8-bit-output`, `-8` | Leave high-bit chars unescaped | ✅ Parity | Applies to displayed paths and protocol debug output |
|
||||
@@ -138,7 +138,7 @@ Every one of those has an entry below with its remaining caveats.
|
||||
| `--delete` | Delete extraneous files from dest | ⚠️ Caveat | `use_delete` config field. Deletion is always derived from the transmitted keep-set manifest of the paths the sender sent/keeps (never from unchecked input), runs through the symlink-safe walker bounded by `MAX_SERVER_DELETE_COUNT`, and skips the `.fastsync-stage` staging dir under `--delay-updates`. FastSync's default timing when no timing flag is given is **delete-after** (extras are removed only once the whole transfer succeeded) — intentionally NOT rsync's `--del`/delete-during default, to preserve FastSync's commit-style safety. By default the destination mirror of a path the source scan pruned (filter/exclude/size rules) is **protected** from deletion — matching rsync, which does not delete excluded files under `--delete`; `--delete-excluded` opts back into deleting them (see below). Deletion is scoped to the **synchronized directories** sent in the manifest (protocol 2.23.0), so a `--files-from` subset no longer deletes untransmitted paths outside the listed directory subtrees. The walk is bounded: a client `--max-delete=NUM` (or the 100000-entry server bound) makes it **partial** — entries up to the bound are removed, the rest are skipped, and the client exits **25** (`RERR_PARTIAL`), matching rsync, rather than failing the transfer. Extraneous destination symlinks are unlinked by name (never followed); a directory still holding a kept/protected entry is left behind rather than failing |
|
||||
| `--delete-before` | Delete before transfer | ⚠️ Caveat | Implies `--delete`. The sender runs a full source pre-scan (paths only) and transmits the keep-set manifest BEFORE any file data; the receiver validates it, removes every destination entry not listed (bounded walk, staging-dir skip, protected prefixes honored), then acks `STATUS_OK`. The sender only starts streaming after the deletion committed, or aborts if the receiver reported a deletion error. By definition the deletions already happened when a later transfer phase fails — rsync's delete-before is destructive the same way; a subsequent failure does not restore the removed files. Divergence: the keep-set is the pre-scan snapshot, so a file that appears on the source between the pre-scan and the data pass is still transferred but was not protected from deletion |
|
||||
| `--del`, `--delete-during` | Delete during transfer | ⚠️ Caveat | Both spellings accepted; imply `--delete`. **Protocol 2.24.0 implements per-directory delete plans:** as the sender finishes each source directory it streams a `STATUS_DELETE_PLAN` for that directory and the receiver removes that directory's extras before applying the next directory's data, so a mid-transfer failure has already removed the extras of the directories reached (verified with a byte-slicing proxy). **Remaining divergence:** the exact abort boundary and the progressive ordering of removals versus rsync's generator can differ, and `-d`/`--dirs` (no descent) falls back to the end-of-transfer commit. `-R` plans are scoped to the transferred prefix subtree |
|
||||
| `--delete-delay` | Find deletions during, delete after | ⚠️ Caveat | Implies `--delete`. **Protocol 2.24.0 implements rsync's delete-delay timing:** the sender records each directory's delete plan while scanning and the receiver commits those removals only after the whole transfer succeeds (per plan), so an extra created in the destination after its directory's plan survives while `--delete-after` re-scans and removes it, and a failed transfer removes nothing. **Remaining divergence:** exact ordering/abort boundaries can differ from rsync's generator, and `-d` falls back to the end commit. **Also:** the reported "Number of deleted files" can be inflated because a directory snapshotted into the delete plan that later fails to delete (ENOTEMPTY) is still counted (`delete_plan.c:583-593`, `866`, `891`) |
|
||||
| `--delete-delay` | Find deletions during, delete after | ✅ Parity | Implies `--delete`. **Protocol 2.24.0 implements rsync's delete-delay timing:** the sender records each directory's delete plan while scanning and the receiver commits those removals only after the whole transfer succeeds (per plan), so an extra created in the destination after its directory's plan survives while `--delete-after` re-scans and removes it, and a failed transfer removes nothing. The deleted count and `--max-delete` budget now advance only on an actual removal: a directory snapshotted into the plan that is refilled before the commit and survives `ENOTEMPTY` is **not** reported, and a `--max-delete=2` partial delete reports exactly 2 (differential test vs rsync 3.4.1, exit 25 both). Unit tests cover the refilled-directory and removed-file accounting. Exact ordering of which extras are removed first can still differ from rsync's generator |
|
||||
| `--delete-after` | Delete after transfer | ✅ Parity | Implies `--delete`. The delete-after timing is also what plain `--delete` does: the keep-set manifest closes the data stream and the receiver commits the bounded deletion only after the terminal `STATUS_FINISHED` proves the whole transfer (every data frame received and stored) succeeded. A failed or aborted transfer removes nothing |
|
||||
| `--delete-excluded` | Also delete excluded files | ⚠️ Caveat | `delete_excluded` config field. Under `--delete` FastSync protects (rsync's default) the destination mirror of paths the sender's source scan pruned by the user-selection rules — the `--filter`/`-F`/`-C` layer and the legacy `--exclude`/`--include` layer. The sender transmits those concrete pruned paths as **protected prefixes** in the delete-manifest frame (see the Phase-3 notes below); the walker never descends into or removes them. `--delete-excluded` opts back in: the sender sends an empty protected list, so the excluded destination mirrors become ordinary extras and are removed. **`--max-size`/`--min-size` pruned mirrors are a separate, always-on protection** (protocol 2.23.0, rsync parity): size-pruned source mirrors survive `--delete` even with `--delete-excluded`. Divergences (documented): protection is derived only from what the source scan actually pruned — a stray destination-only file that happens to match an exclude rule is not protected (FastSync never re-applies rules to the destination, keeping deletion sender-derived) |
|
||||
| `--max-delete=NUM` | Max files to delete | ✅ Parity | `max_delete` config field (default -1 = no client limit; 0 = delete nothing). **Protocol 2.23.0 matches rsync's partial semantics:** the receiver deletes up to NUM entries (regular files, symlinks and empty directories; each directory removal counts as one) and then **stops deleting, skips the rest, and reports the run as partial**. The client prints a "deletions stopped due to `--max-delete` limit" message and exits **25** (rsync's `RERR_PARTIAL`), not a hard failure — the transfer itself succeeded. NUM only applies together with `--delete` (it is inert otherwise, matching rsync). A client NUM below the server hard bound `MAX_SERVER_DELETE_COUNT` (100000) replaces it; a NUM above it never raises that cap. Deleting an entire destination with no limit is still bounded by the server's 100000-entry ceiling. `--delete-missing-args` exact-path deletions and the ordinary extras walk draw from the same budget, matching rsync |
|
||||
@@ -629,7 +629,7 @@ targets verbatim, matching rsync.
|
||||
| Flag | Rsync Description | FastSync Status | Notes |
|
||||
|------|-------------------|-----------------|-------|
|
||||
| `--checksum` | Skip based on checksum | ✅ Parity | `-c`/`--checksum` compares per-file whole-file content digests to skip unchanged files. **As of protocol 2.23.0 the short `-c` implies the checksum quick-check**, so a plain `-c` run verifies content rather than only affecting the `--incremental` handshake. The digest algorithm is `xxh128` by default (protocol 2.26.0's negotiated default) and is selectable via `--checksum-choice`/`--cc` (`xxh128`/`xxh3`/`xxh64`/`xxhash`/`md5`/`md4`/`sha1`/`none`/`auto`, plus rsync's two-name form) and `--checksum-seed=NUM` (see those rows) |
|
||||
| `--checksum-choice=STR`, `--cc=STR` | Choose checksum algorithm | ⚠️ Caveat | Real algorithm selection for the per-file whole-file digest used by the `--incremental`/`--checksum` handshake and basis-dir verification. **Protocol 2.26.0 accepts rsync 3.4.1's full set** — `xxh128` (the negotiated default), `xxh3`, `xxh64`, `xxhash`, `md5`, `md4`, `sha1`, `none`, `auto`, and the two-name `transfer,pre-transfer` form — with rsync's exit-4 rejection of an unknown name and of `none` on the transfer side when `--checksum` is on. `--cc=ALG` and space forms both parse. The algorithm id and seed cross the wire; the receiver hashes its old file with the same algorithm+seed and the per-file `STATUS_CHECK` handshake carries a bounded digest pinned to the negotiated length. **Remaining divergences:** rsync uses this choice for the transfer checksum on the wire as well, while FastSync selects only the whole-file comparison digest and keeps the delta BLOCK strong checksum at xxHash32; the `RSYNC_CHECKSUM_LIST` environment variable is not consulted; and `auto` always resolves deterministically to the first supported entry in rsync's preference order rather than probing the peer |
|
||||
| `--checksum-choice=STR`, `--cc=STR` | Choose checksum algorithm | ⚠️ Caveat | Real algorithm selection for the per-file whole-file digest used by the `--incremental`/`--checksum` handshake and basis-dir verification. **Protocol 2.26.0 accepts rsync 3.4.1's full set** — `xxh128` (the negotiated default), `xxh3`, `xxh64`, `xxhash`, `md5`, `md4`, `sha1`, `none`, `auto`, and the two-name `transfer,pre-transfer` form — with rsync's exit-4 rejection of an unknown name and of `none` on the transfer side when `--checksum` is on. `--cc=ALG` and space forms both parse. The algorithm id and seed cross the wire; the receiver hashes its old file with the same algorithm+seed and the per-file `STATUS_CHECK` handshake carries a bounded digest pinned to the negotiated length. `checksum_digest_file` now streams **every** supported algorithm (md4 via the self-contained RFC 1320 code, sha1/md5 via EVP, none as an empty digest), so the streaming path matches its contract, and `--out-format %C` uses the selected **transfer** half of a two-name choice and renders each algorithm byte-for-byte like rsync (xxh128 high-then-low, xxh64/xxh3 big-endian, md5/md4/sha1 standard hex, none a blank 2-char column) — differential-tested across all algorithms. **Remaining divergences:** rsync uses this choice for the block checksum on the wire too, while FastSync selects only the whole-file comparison digest and keeps the delta BLOCK strong checksum at xxHash32; the `RSYNC_CHECKSUM_LIST` environment variable is not consulted; and `auto` always resolves deterministically to the first supported entry in rsync's preference order rather than probing the peer |
|
||||
| `--compare-dest=DIR` | Compare dest files relative to DIR | ⚠️ Caveat | DIR is a receiver-side basis; protocol 2.26.0 uses an absolute path verbatim (rsync semantics) and resolves a relative path below the destination root (`..` components are rejected, `//` collapsed and trailing `/` dropped). On the receiver's per-file check (implies `--incremental`) an exact match = same size + mtime (unless `--size-only`; `-I` disables matching) **and** equal xxHash64 of the sender's file; a match suppresses the data transfer. compare-dest never copies: it only skips a file the destination does **not** already hold (sparse destination, rsync parity), and is consulted before the normal delta/full paths. Repeatable; searched in command-line order, first match wins. Divergences: when the destination already holds a *different* version rsync deletes it but FastSync instead transfers the data (keeps the mirror complete; never deletes without `--delete`); attribute-only differences on a match are not re-applied (data is skipped so the sender never sends metadata); content is verified by xxHash64, stricter than rsync's default quick check. Sizing: FastSync's whole-file payload limit is 256 MiB on **every** transfer path (not basis-specific); rsync applies basis dirs to arbitrary sizes, so FastSync refuses a basis run whose source contains a larger file up front with a clear error before any transfer. Wire: a basis-count field is always present on the config frame (protocol 2.9.0, so clients and servers must both be 2.9.0) |
|
||||
| `--copy-dest=DIR` | Include copies of unchanged files | ⚠️ Caveat | Same basis rules as `--compare-dest`, but an exact match materializes a **local copy** of the DIR file into the destination (via the normal atomic temp+rename store path, so `--existing`/`--ignore-existing`/`--update`/`--backup`/`--delay-updates` all still apply) instead of transferring data. Repeatable; command-line order = priority. Content is xxHash64-verified before the copy. Divergences: a basis-hit destination keeps the basis file's own mode/uid/gid and mtime (the sender sends no metadata on a skip), so with `--size-only` its mtime can differ from the source and attribute-only differences are copied with the basis attributes rather than rsync's "copy + fix attributes". Requires `--incremental` (implied); incompatible with `-s`. Wire: protocol 2.9.0 |
|
||||
| `--link-dest=DIR` | Hardlink to files when unchanged | ⚠️ Caveat | Same basis rules as `--copy-dest`, but an exact match installs an atomic **hard link** to the DIR file (temp hard link + rename) so no data or disk space is used; where the link is impossible (basis on another filesystem, filesystem refuses links) it falls back cleanly to a byte-identical local copy, never a corrupt/partial file. `--delay-updates` stages the link and publishes by rename, so the final entry stays a real hard link. Repeatable (searched in command-line order, first match wins). Content is xxHash64-verified before linking. Divergences and caveats: protocol 2.26.0 re-links an already up-to-date destination file to the basis (the relink path installs the hard link when the content matches); a link keeps the basis inode's own mode/uid/gid and mtime — metadata is never written through the shared inode (that would mutate the basis file), so a later `--inplace` run that rewrites such a destination path **will mutate the basis snapshot** through the shared inode (use `--copy-dest` when the destination must stay independently writable); with `--size-only` the linked mtime can differ from the source; a `--remove-source-files` source satisfied by a basis dir is treated as skipped and therefore **retained** (never removed); basis dirs are excluded from `--delete`. Requires `--incremental` (implied); incompatible with `-s`. Wire: protocol 2.9.0 |
|
||||
@@ -893,7 +893,7 @@ These are the last compatibility items and the closing phase toward rsync flag p
|
||||
|
||||
**Wire:** two trailing config-frame blocks after the `--iconv` spec, in fixed order — `send_privilege_options`/`receive_privilege_options` (one `super_mode` int, validated `0..2`), then `send_copy_as_options`/`receive_copy_as_options` (presence int + two int32 ids, validated `>= 0`, with `copy_as_set ⇒ use_metadata`). `PROTOCOL_VERSION` bumped **2.17.0 → 2.18.0**. **Divergences from rsync:** rsync's `--super` elevates the receiver and `--copy-as` actually switches its credentials; FastSync never elevates and only permits/forwards confined attempts, and `--copy-as` forces ownership rather than switching identity.
|
||||
|
||||
**Honest status after the parity-completion wave (protocol 2.26.0).** ✅ Parity 106 / ⚠️ Caveat 27 / ❌ Divergent 23 = 156 rows. Earlier revisions of this document reported "143 ✅ / 0 divergence / 0 partial"; that conflated "parsed and tested" with "rsync parity", because many rows carried documented behavioral differences and some short options were not parsed at all. This reclassification makes every difference explicit. The completion wave closed 23 previously-caveated rows (9 that triage showed were already parity, plus 14 genuine fixes) and turned the 17 inherently non-rsync rows — native daemon config/auth, the FastSync batch container, the safe-subset device/privilege flags, `-X`'s privileged namespaces, `--fake-super`'s native xattr format, and the `--old-args` no-op — into explicit ❌ divergences. The remaining ⚠️ rows are the ones with a documented residual (see the row notes and the **Parity Completion Wave (protocol 2.26.0)** section below).
|
||||
**Honest status after the parity-completion wave (protocol 2.26.0), updated by the rsync-parity-stats pass.** ✅ Parity 107 / ⚠️ Caveat 25 / ❌ Divergent 24 = 156 rows. Earlier revisions of this document reported "143 ✅ / 0 divergence / 0 partial"; that conflated "parsed and tested" with "rsync parity", because many rows carried documented behavioral differences and some short options were not parsed at all. This reclassification makes every difference explicit. The completion wave closed 23 previously-caveated rows (9 that triage showed were already parity, plus 14 genuine fixes) and turned the 17 inherently non-rsync rows — native daemon config/auth, the FastSync batch container, the safe-subset device/privilege flags, `-X`'s privileged namespaces, `--fake-super`'s native xattr format, and the `--old-args` no-op — into explicit ❌ divergences. The stats pass flipped `--delete-delay` to ✅ (actual-removal accounting) and reclassified `--out-format` to ❌ (protocol-specific `%b`/delta-`%c`), and sharpened the `--stats`/`--progress`/`--checksum-choice` residuals. The remaining ⚠️ rows are the ones with a documented residual (see the row notes and the **Parity Completion Wave (protocol 2.26.0)** section below).
|
||||
|
||||
**Preserve-attribute split (protocol 2.21.0 → 2.22.0) — ✅ implemented.** FastSync splits the former single metadata bundle into four independent, rsync-compatible per-attribute flags — `-p/--perms`, `-t/--times`, `-o/--owner`, `-g/--group` — each with a negation (`--no-perms`/`--no-times`/`--no-owner`/`--no-group`, short `--no-p`/`--no-t`/`--no-o`/`--no-g`), plus `--no-preserve` clearing all four. `-a/--archive` is now full rsync `-rlptgoD` (owner and group included, though their application stays privilege-gated), `-A/--acls` implies `-p`, `-X/--xattrs` does not, `-E/--executability` sets only executability, and `-U`/`-N` do not imply `-t`. `--incremental`/`--delta` still auto-preserve perms+times unless the user explicitly negated them. Wire: the binary config frame gains four appended booleans (`preserve_perms`/`preserve_times`/`preserve_owner`/`preserve_group`) after `omit_link_times`, so `PROTOCOL_VERSION` is bumped **2.21.0 → 2.22.0**; the fixed-width `FileMetadata` layout is unchanged and the receiver gates the metadata frame on a derived `use_metadata`. Receiver behavior: each attribute is applied independently, directory modes are applied under `-p` (at the end of the transfer, alongside dir times), symlink mode under `-p`, and `-O/--omit-dir-times` suppresses directory times only. Documented divergences as of 2.22.0, **all but (d)/(e) removed by the rsync-parity wave (protocol 2.23.0)**: (a) the mode-masking divergence is **gone** — under `-p` the source mode is now copied exactly, including `S_IWGRP`/`S_IWOTH` and setuid/setgid/sticky; (b) a brand-new file without `-p` still gets `source_mode & ~umask` when metadata is present (else the historical fixed `0644`), and a new *directory* without `-p` still uses FastSync's `0755` default; (c) the `--chmod`-implies-`-p` divergence is **gone** — `--chmod` no longer implies `-p` (rsync parity); (d) `-o`/`-g` map by name on the receiver with a raw-numeric fallback (only numeric ids cross the wire); (e) a daemon module without `client owner = yes` does not refuse a plain `-a`/`-o`/`-g` — it forces super off, applies no ownership, and logs a warning, while explicit `--chown`/`--usermap`/`--groupmap`/`--numeric-ids`/`--copy-as`/`--super` are still refused.
|
||||
|
||||
@@ -1123,10 +1123,14 @@ wire protocol three times (full rationale in `src/shared/config.h`):
|
||||
|
||||
These remain after the wave; the individual rows carry the precise wording.
|
||||
|
||||
- **`--stats`** lacks rsync's `(reg/dir/link)` breakdown on `Number of files`
|
||||
and `Number of created files`; **`--progress`** omits the leading `./` line
|
||||
and its `to-chk` total differs by the root entry; **`--out-format`** `%b`/`%c`
|
||||
count FastSync wire bytes.
|
||||
- **`--stats`** now reproduces rsync's `(reg/dir/link/special)` breakdown on
|
||||
`Number of files`, the regular-transferred count and the size totals, but
|
||||
`Number of created files` is the transferred-regular count without a type
|
||||
breakdown and wire-byte totals differ; **`--progress`** now prints the leading
|
||||
`./` line and includes the root in `to-chk` (single-file output is
|
||||
byte-identical), but a multi-directory `to-chk` denominator and per-directory
|
||||
name lines still differ; **`--out-format`** `%C` matches for every algorithm,
|
||||
but `%b`/`%c` count FastSync wire bytes (protocol-specific, hence ❌).
|
||||
- **`-n --delete`** ordering can differ from rsync's delete-during walk and a
|
||||
filtered dry-run can over-report.
|
||||
- **Delete timing:** the default `--delete` remains delete-after rather than
|
||||
|
||||
+21
-18
@@ -194,25 +194,18 @@ char* change_render_itemize(const Config* config, const ChangeEvent* event) {
|
||||
|
||||
/* ---- --out-format / --log-file-format ---- */
|
||||
|
||||
/* rsync 3.4.1's `%C` uses the negotiated transfer checksum; with the default
|
||||
* "auto" choice on both ends that is xxh128. FastSync's internal XXH64 default
|
||||
* is not an rsync algorithm, so map it to xxh128 for parity. */
|
||||
/* rsync 3.4.1's `%C` uses the negotiated TRANSFER checksum (the first name of a
|
||||
* two-name "transfer,pre-transfer" --checksum-choice), not the pre-transfer
|
||||
* whole-file digest FastSync compares against on the wire. The default "auto"
|
||||
* resolves to xxh128, so an explicit selection and the default both render the
|
||||
* selected algorithm's digest. */
|
||||
static ChecksumAlgo out_format_checksum_algo(const Config* config) {
|
||||
switch ((ChecksumAlgo)config->checksum_algo) {
|
||||
case CHECKSUM_ALGO_MD5:
|
||||
return CHECKSUM_ALGO_MD5;
|
||||
case CHECKSUM_ALGO_XXH3:
|
||||
return CHECKSUM_ALGO_XXH3;
|
||||
case CHECKSUM_ALGO_XXH128:
|
||||
return CHECKSUM_ALGO_XXH128;
|
||||
case CHECKSUM_ALGO_XXH64:
|
||||
default:
|
||||
return CHECKSUM_ALGO_XXH128;
|
||||
}
|
||||
return (ChecksumAlgo)config->checksum_transfer_algo;
|
||||
}
|
||||
|
||||
/* Render a digest as rsync's sum_as_hex: for xxh128 the HIGH 64-bit half is
|
||||
* printed before the low half; every other algorithm prints its bytes in order. */
|
||||
/* Render a digest as rsync's sum_as_hex: xxh128 prints the HIGH 64-bit half
|
||||
* before the low half, and xxh64/xxh3 print their 64-bit value big-endian; every
|
||||
* other algorithm prints its bytes in order. */
|
||||
static void digest_to_hex(ChecksumAlgo algo, const uint8_t* digest, size_t len, char* out) {
|
||||
if (algo == CHECKSUM_ALGO_XXH128 && len == 16) {
|
||||
uint64_t low = 0;
|
||||
@@ -222,6 +215,12 @@ static void digest_to_hex(ChecksumAlgo algo, const uint8_t* digest, size_t len,
|
||||
snprintf(out, len * 2 + 1, "%016llx%016llx", (unsigned long long)high, (unsigned long long)low);
|
||||
return;
|
||||
}
|
||||
if ((algo == CHECKSUM_ALGO_XXH64 || algo == CHECKSUM_ALGO_XXH3) && len == 8) {
|
||||
uint64_t value = 0;
|
||||
memcpy(&value, digest, sizeof(value));
|
||||
snprintf(out, len * 2 + 1, "%016llx", (unsigned long long)value);
|
||||
return;
|
||||
}
|
||||
static const char hex[] = "0123456789abcdef";
|
||||
for (size_t i = 0; i < len; i++) {
|
||||
out[i * 2] = hex[(digest[i] >> 4) & 0xf];
|
||||
@@ -260,6 +259,9 @@ static void fill_event_checksum(const Config* config, const File* file, ChangeEv
|
||||
if (file->path == NULL)
|
||||
return;
|
||||
ChecksumAlgo algo = out_format_checksum_algo(config);
|
||||
/* rsync renders `--checksum-choice=none` as a blank 2-character column. */
|
||||
if (algo == CHECKSUM_ALGO_NONE)
|
||||
return;
|
||||
uint8_t digest[CHECKSUM_MAX_DIGEST_LEN];
|
||||
size_t len = 0;
|
||||
/* rsync's %C is the transfer checksum, which is always seeded with 0 (it is
|
||||
@@ -329,9 +331,10 @@ char* change_render_format(const char* format, const Config* config, const Chang
|
||||
if (event->checksum_known) {
|
||||
ok = strbuf_append(&line, event->checksum);
|
||||
} else {
|
||||
/* rsync pads a non-regular / untransferred entry with spaces. */
|
||||
/* rsync pads a non-regular / untransferred / `none` entry with spaces;
|
||||
`none` renders as a blank 2-character column. */
|
||||
ChecksumAlgo algo = out_format_checksum_algo(config);
|
||||
int width = checksum_digest_len(algo) * 2;
|
||||
int width = algo == CHECKSUM_ALGO_NONE ? 2 : checksum_digest_len(algo) * 2;
|
||||
for (int i = 0; i < width && ok; i++)
|
||||
ok = strbuf_append_char(&line, ' ');
|
||||
}
|
||||
|
||||
+154
-36
@@ -83,15 +83,53 @@ static const char* stats_bytes(const Config* config, unsigned long long bytes, c
|
||||
return buffer;
|
||||
}
|
||||
|
||||
/* Print the rsync `--stats` block on stdout. Byte totals use the process-wide
|
||||
wire counters and the receiver-only counters come from the STATUS_STATS frame;
|
||||
the labels, layout and rate/speedup formulas match rsync 3.4.1. Shared by the
|
||||
single-threaded and multithreaded send paths. */
|
||||
static void report_transfer_stats(const Config* config, int total_files,
|
||||
unsigned long long total_bytes, time_t start,
|
||||
/* Build rsync's `Number of files` parenthetical: each non-zero category, in
|
||||
reg/dir/link/special order. Empty when the flist counted nothing. */
|
||||
static void stats_type_breakdown(const TransferStats* stats, char* out, size_t out_size) {
|
||||
unsigned long long total =
|
||||
stats->flist_reg + stats->flist_dir + stats->flist_link + stats->flist_special;
|
||||
if (total == 0) {
|
||||
out[0] = '\0';
|
||||
return;
|
||||
}
|
||||
out[0] = '\0';
|
||||
size_t used = 0;
|
||||
const struct {
|
||||
const char* name;
|
||||
unsigned long long count;
|
||||
} parts[4] = {{"reg", stats->flist_reg},
|
||||
{"dir", stats->flist_dir},
|
||||
{"link", stats->flist_link},
|
||||
{"special", stats->flist_special}};
|
||||
bool first = true;
|
||||
for (size_t i = 0; i < 4; i++) {
|
||||
if (parts[i].count == 0)
|
||||
continue;
|
||||
int written = snprintf(out + used, out_size - used, "%s%s: %llu", first ? "(" : ", ",
|
||||
parts[i].name, parts[i].count);
|
||||
if (written < 0 || (size_t)written >= out_size - used)
|
||||
break;
|
||||
used += (size_t)written;
|
||||
first = false;
|
||||
}
|
||||
if (!first && used + 1 < out_size)
|
||||
out[used++] = ')';
|
||||
out[used] = '\0';
|
||||
}
|
||||
|
||||
/* Print the rsync `--stats` block on stdout. The source-side flist and
|
||||
transferred counters come from `stats` (filled while scanning/sending), the
|
||||
receiver-only counters from the STATUS_STATS frame, and the wire byte totals
|
||||
from the process-wide protocol counters. The labels, layout and
|
||||
rate/speedup formulas match rsync 3.4.1. Shared by the single-threaded and
|
||||
multithreaded send paths. */
|
||||
static void report_transfer_stats(const Config* config, const TransferStats* stats, time_t start,
|
||||
const ReceiverStats* recv) {
|
||||
if (!config->stats || config->quiet)
|
||||
return;
|
||||
TransferStats empty = {0};
|
||||
if (stats == NULL)
|
||||
stats = ∅
|
||||
ReceiverStats none = {0};
|
||||
if (recv == NULL)
|
||||
recv = &none;
|
||||
@@ -101,11 +139,18 @@ static void report_transfer_stats(const Config* config, int total_files,
|
||||
double elapsed = difftime(time(NULL), start);
|
||||
double rate = (double)(sent + received) / (0.5 + elapsed);
|
||||
char total_buffer[32];
|
||||
char transferred_buffer[32];
|
||||
char literal_buffer[32];
|
||||
char sent_buffer[32];
|
||||
char recv_buffer[32];
|
||||
char rate_buffer[32] = {0};
|
||||
char human_rate[32] = {0};
|
||||
const char* total = stats_bytes(config, total_bytes, total_buffer, sizeof(total_buffer));
|
||||
const char* total =
|
||||
stats_bytes(config, stats->total_file_size, total_buffer, sizeof(total_buffer));
|
||||
const char* transferred = stats_bytes(config, stats->transferred_file_size, transferred_buffer,
|
||||
sizeof(transferred_buffer));
|
||||
const char* literal =
|
||||
stats_bytes(config, stats->literal_data, literal_buffer, sizeof(literal_buffer));
|
||||
const char* sent_s = stats_bytes(config, sent, sent_buffer, sizeof(sent_buffer));
|
||||
const char* recv_s = stats_bytes(config, received, recv_buffer, sizeof(recv_buffer));
|
||||
const char* rate_str = rate_buffer;
|
||||
@@ -116,15 +161,26 @@ static void report_transfer_stats(const Config* config, int total_files,
|
||||
} else {
|
||||
snprintf(rate_buffer, sizeof(rate_buffer), "%.2f", rate);
|
||||
}
|
||||
double speedup = (sent + received) > 0 ? (double)total_bytes / (double)(sent + received) : 0.0;
|
||||
double speedup =
|
||||
(sent + received) > 0 ? (double)stats->total_file_size / (double)(sent + received) : 0.0;
|
||||
char breakdown[128];
|
||||
stats_type_breakdown(stats, breakdown, sizeof(breakdown));
|
||||
unsigned long long flist_total =
|
||||
stats->flist_reg + stats->flist_dir + stats->flist_link + stats->flist_special;
|
||||
printf("\n");
|
||||
printf("Number of files: %d\n", total_files);
|
||||
printf("Number of created files: %d\n", total_files);
|
||||
if (breakdown[0] != '\0')
|
||||
printf("Number of files: %llu %s\n", flist_total, breakdown);
|
||||
else
|
||||
printf("Number of files: %llu\n", flist_total);
|
||||
/* FastSync cannot tell which entries the receiver newly created, so it
|
||||
reports the transferred regular files (which are created on a fresh
|
||||
destination). See RSYNC_COMPAT.md for the documented residual. */
|
||||
printf("Number of created files: %llu\n", stats->transferred_regular);
|
||||
printf("Number of deleted files: %llu\n", recv->deleted_files);
|
||||
printf("Number of regular files transferred: %d\n", total_files);
|
||||
printf("Number of regular files transferred: %llu\n", stats->transferred_regular);
|
||||
printf("Total file size: %s bytes\n", total);
|
||||
printf("Total transferred file size: %s bytes\n", total);
|
||||
printf("Literal data: %s bytes\n", total);
|
||||
printf("Total transferred file size: %s bytes\n", transferred);
|
||||
printf("Literal data: %s bytes\n", literal);
|
||||
printf("Matched data: %llu bytes\n", recv->matched_data);
|
||||
printf("File list size: 0\n");
|
||||
printf("File list generation time: 0.000 seconds\n");
|
||||
@@ -138,6 +194,45 @@ static void report_transfer_stats(const Config* config, int total_files,
|
||||
fflush(stdout);
|
||||
}
|
||||
|
||||
/* Classify one scanned source entry into the rsync flist counters. Called for
|
||||
every entry the sender walks, transferred or skipped. Directory entries are
|
||||
counted here only for the explicit -d/--dirs generator; a recursive scan's
|
||||
directories are accounted from the scanner's dir_entries list at report time. */
|
||||
static void transfer_stats_note_entry(TransferStats* stats, const File* file) {
|
||||
if (stats == NULL || file == NULL)
|
||||
return;
|
||||
if (file->is_dir) {
|
||||
stats->flist_dir++;
|
||||
return;
|
||||
}
|
||||
if (file->is_symlink) {
|
||||
stats->flist_link++;
|
||||
stats->total_file_size += file->symlink_target ? strlen(file->symlink_target) : 0;
|
||||
return;
|
||||
}
|
||||
if (file->is_special) {
|
||||
stats->flist_special++;
|
||||
return;
|
||||
}
|
||||
stats->flist_reg++;
|
||||
stats->total_file_size += file->data ? file->data->size : 0;
|
||||
}
|
||||
|
||||
/* Account for a regular file (or a whole-file append) the receiver actually
|
||||
stored: rsync's transferred-file count and transferred/literal byte totals. */
|
||||
static void transfer_stats_note_transferred(TransferStats* stats, const File* file) {
|
||||
if (stats == NULL || file == NULL)
|
||||
return;
|
||||
if (file->is_dir || file->is_symlink || file->is_special)
|
||||
return;
|
||||
if (file->link_group != 0 && !file->link_first)
|
||||
return;
|
||||
unsigned long long size = file->data ? file->data->size : 0;
|
||||
stats->transferred_regular++;
|
||||
stats->transferred_file_size += size;
|
||||
stats->literal_data += size;
|
||||
}
|
||||
|
||||
/* ---- rsync-style per-file --progress ------------------------------------
|
||||
* rsync prints, for each transferred regular file, the file name followed by a
|
||||
* two-frame progress line: the first at the initial 32 KiB read window (always
|
||||
@@ -188,10 +283,14 @@ static void progress_final_frame(unsigned long long size, char* out, size_t out_
|
||||
unsigned long long remain = (unsigned long long)(diff_ms / 1000);
|
||||
snprintf(rembuf, sizeof(rembuf), "%4u:%02u:%02u", (unsigned)(remain / 3600),
|
||||
(unsigned)((remain / 60) % 60), (unsigned)(remain % 60));
|
||||
/* rsync's `to-chk` denominator is the whole file list, which includes the
|
||||
transfer-root directory FastSync never emits as a transfer entry. Count
|
||||
that root entry so a single-file transfer matches rsync exactly. */
|
||||
unsigned long long total = g_progress_seen + 1;
|
||||
unsigned long long to_chk =
|
||||
g_progress_seen > g_progress_xferred ? g_progress_seen - g_progress_xferred : 0;
|
||||
snprintf(out, out_size, "\r%15s %3d%% %7.2f%s %s (xfr#%llu, to-chk=%llu/%llu)\n", ofs_buf, 100,
|
||||
rate, units, rembuf, g_progress_xferred, to_chk, g_progress_seen);
|
||||
rate, units, rembuf, g_progress_xferred, to_chk, total);
|
||||
}
|
||||
|
||||
static void client_progress_begin(const Config* config) {
|
||||
@@ -201,6 +300,10 @@ static void client_progress_begin(const Config* config) {
|
||||
if (!g_progress_active)
|
||||
return;
|
||||
printf("sending incremental file list\n");
|
||||
/* rsync prints the transfer-root directory's name before the first file when
|
||||
that directory is created; FastSync mirrors the source root below the
|
||||
receive root and creates it on a fresh destination, so emit it here. */
|
||||
printf("./\n");
|
||||
fflush(stdout);
|
||||
}
|
||||
|
||||
@@ -1986,7 +2089,16 @@ static int send_dry_run_remote(Config* config) {
|
||||
else
|
||||
printf("Total: %d files, %.1f MB\n", file_count, (double)total_bytes / (double)BYTES_PER_MIB);
|
||||
}
|
||||
report_transfer_stats(config, file_count, total_bytes, dry_start, &dry_stats);
|
||||
{
|
||||
TransferStats dry_transfer;
|
||||
memset(&dry_transfer, 0, sizeof(dry_transfer));
|
||||
dry_transfer.flist_reg = (unsigned long long)file_count;
|
||||
dry_transfer.total_file_size = total_bytes;
|
||||
dry_transfer.transferred_regular = (unsigned long long)file_count;
|
||||
dry_transfer.transferred_file_size = total_bytes;
|
||||
dry_transfer.literal_data = total_bytes;
|
||||
report_transfer_stats(config, &dry_transfer, dry_start, &dry_stats);
|
||||
}
|
||||
ret = io_error ? 1 : 0;
|
||||
|
||||
dry_fail:
|
||||
@@ -2236,7 +2348,7 @@ static bool source_is_regular_file(const File* file) {
|
||||
}
|
||||
|
||||
static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config,
|
||||
ArrayList* remove_sources) {
|
||||
ArrayList* remove_sources, TransferStats* stats) {
|
||||
if (config->use_chunk_serialization) {
|
||||
if (remove_sources) {
|
||||
for (int i = 0; i < chunk->element_count; i++) {
|
||||
@@ -2263,10 +2375,13 @@ static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config,
|
||||
for (int i = 0; i < chunk->element_count; i++) {
|
||||
if (chunk->items[i] == NULL)
|
||||
continue;
|
||||
transfer_stats_note_entry(stats, chunk->items[i]);
|
||||
if (chunk->items[i]->is_dir)
|
||||
change_emit_dir_sent(config, chunk->items[i]);
|
||||
else
|
||||
change_emit_file_sent(config, chunk->items[i]);
|
||||
if (!chunk->items[i]->is_dir)
|
||||
transfer_stats_note_transferred(stats, chunk->items[i]);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
@@ -2275,6 +2390,7 @@ static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config,
|
||||
File* f = chunk->items[i];
|
||||
if (f == NULL)
|
||||
continue;
|
||||
transfer_stats_note_entry(stats, f);
|
||||
if (f->is_dir) {
|
||||
/* Explicit directory entry (--dirs): a MKDIR frame carrying the
|
||||
destination path (and metadata when negotiated). Directories have no
|
||||
@@ -2328,6 +2444,7 @@ static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config,
|
||||
source_file_destroy(source);
|
||||
return -1;
|
||||
}
|
||||
transfer_stats_note_transferred(stats, f);
|
||||
change_emit_file_sent_bytes(config, f, protocol_bytes_written() - bytes_before,
|
||||
protocol_bytes_read() - read_before);
|
||||
client_progress_file(config, f);
|
||||
@@ -2436,7 +2553,7 @@ static int send_chunks_multithreaded(void* pipeline_context) {
|
||||
return thrd_error;
|
||||
}
|
||||
if (send_chunk_with_removal(client, current_chunk, context->config,
|
||||
context->remove_source_files) != 0) {
|
||||
context->remove_source_files, &context->stats) != 0) {
|
||||
log_message(LOG_LEVEL_ERROR, "unexpected error while sending chunk");
|
||||
chunk_destroy(current_chunk);
|
||||
pipeline_cancel(context);
|
||||
@@ -2533,13 +2650,12 @@ static int send_chunks_multithreaded(void* pipeline_context) {
|
||||
"server reported a deletion failure (--delete); see the server log for the reason");
|
||||
if (ok)
|
||||
remove_transferred_sources(context->config, context->remove_source_files);
|
||||
mtx_lock(&context->mutex_progress);
|
||||
int total_files = context->total_files;
|
||||
unsigned long long total_bytes = context->total_bytes;
|
||||
mtx_unlock(&context->mutex_progress);
|
||||
report_transfer_stats(context->config, total_files, total_bytes, start, &recv_stats);
|
||||
log_info_message(LOG_INFO_STATS, "Transfer summary: %d files, %.1f MB", total_files,
|
||||
(double)total_bytes / (double)BYTES_PER_MIB);
|
||||
if (context->dir_entries)
|
||||
context->stats.flist_dir += (unsigned long long)context->dir_entries->size;
|
||||
report_transfer_stats(context->config, &context->stats, start, &recv_stats);
|
||||
log_info_message(LOG_INFO_STATS, "Transfer summary: %llu files, %.1f MB",
|
||||
context->stats.transferred_regular,
|
||||
(double)context->stats.transferred_file_size / (double)BYTES_PER_MIB);
|
||||
disconnect_transfer_client(client);
|
||||
mark_sender_done(context);
|
||||
protocol_session_unbind();
|
||||
@@ -3003,8 +3119,8 @@ int send_files(Config* config) {
|
||||
goto send_fail;
|
||||
|
||||
Chunk* current_chunk;
|
||||
unsigned long long total_bytes = 0;
|
||||
int total_files = 0;
|
||||
TransferStats transfer_stats;
|
||||
memset(&transfer_stats, 0, sizeof(transfer_stats));
|
||||
time_t start = time(NULL);
|
||||
client_progress_begin(config);
|
||||
/* True when the stop deadline cut the scan short so the keep-set manifest is
|
||||
@@ -3030,11 +3146,6 @@ int send_files(Config* config) {
|
||||
scan_stopped_early = true;
|
||||
break;
|
||||
}
|
||||
unsigned long long chunk_bytes = 0;
|
||||
for (int i = 0; i < current_chunk->element_count; i++) {
|
||||
chunk_bytes += current_chunk->items[i]->data->size;
|
||||
total_files++;
|
||||
}
|
||||
if (manifest && !add_chunk_to_manifest(manifest, current_chunk)) {
|
||||
chunk_destroy(current_chunk);
|
||||
goto send_fail;
|
||||
@@ -3061,13 +3172,13 @@ int send_files(Config* config) {
|
||||
send_failed = true;
|
||||
break;
|
||||
}
|
||||
if (send_chunk_with_removal(client, current_chunk, config, remove_sources) != 0) {
|
||||
if (send_chunk_with_removal(client, current_chunk, config, remove_sources, &transfer_stats) !=
|
||||
0) {
|
||||
log_message(LOG_LEVEL_ERROR, "Failed to send chunk");
|
||||
chunk_destroy(current_chunk);
|
||||
send_failed = true;
|
||||
break;
|
||||
}
|
||||
total_bytes += chunk_bytes;
|
||||
chunk_destroy(current_chunk);
|
||||
}
|
||||
if (send_failed) {
|
||||
@@ -3152,9 +3263,16 @@ int send_files(Config* config) {
|
||||
"server reported a deletion failure (--delete); see the server log for the reason");
|
||||
if (ok)
|
||||
remove_transferred_sources(config, remove_sources);
|
||||
report_transfer_stats(config, total_files, total_bytes, start, &recv_stats);
|
||||
log_info_message(LOG_INFO_STATS, "Transfer summary: %d files, %.1f MB", total_files,
|
||||
(double)total_bytes / (double)BYTES_PER_MIB);
|
||||
/* A recursive -a scan has no directory entries in its chunks; account them
|
||||
from the scanner's captured directory list (present whenever a directory
|
||||
attribute is preserved, e.g. -a/-t/-p). The -d generator counts its
|
||||
explicit directory entries inline instead. */
|
||||
if (dir_entries)
|
||||
transfer_stats.flist_dir += (unsigned long long)dir_entries->size;
|
||||
report_transfer_stats(config, &transfer_stats, start, &recv_stats);
|
||||
log_info_message(LOG_INFO_STATS, "Transfer summary: %llu files, %.1f MB",
|
||||
transfer_stats.transferred_regular,
|
||||
(double)transfer_stats.transferred_file_size / (double)BYTES_PER_MIB);
|
||||
/* A skipped source entry (--ignore-errors past an unreadable directory, or a
|
||||
dereferenced symlink with no referent) makes rsync report a partial
|
||||
transfer (exit 23) even though the rest of the run succeeded. A
|
||||
|
||||
+27
-2
@@ -223,17 +223,25 @@ bool checksum_digest_file(ChecksumAlgo algo, uint64_t seed, const char* path, ui
|
||||
if (fd < 0)
|
||||
return false;
|
||||
|
||||
if (algo == CHECKSUM_ALGO_NONE) {
|
||||
/* No checksum requested: nothing to read; an empty digest succeeds. */
|
||||
close(fd);
|
||||
*out_len = 0;
|
||||
return true;
|
||||
}
|
||||
|
||||
uint8_t buffer[64 * 1024];
|
||||
bool ok = false;
|
||||
|
||||
if (algo == CHECKSUM_ALGO_MD5) {
|
||||
if (algo == CHECKSUM_ALGO_MD5 || algo == CHECKSUM_ALGO_SHA1) {
|
||||
const EVP_MD* md = algo == CHECKSUM_ALGO_MD5 ? EVP_md5() : EVP_sha1();
|
||||
EVP_MD_CTX* ctx = EVP_MD_CTX_new();
|
||||
if (!ctx) {
|
||||
close(fd);
|
||||
return false;
|
||||
}
|
||||
unsigned int digest_len = 0;
|
||||
if (EVP_DigestInit_ex(ctx, EVP_md5(), NULL) == 1) {
|
||||
if (EVP_DigestInit_ex(ctx, md, NULL) == 1) {
|
||||
ok = true;
|
||||
ssize_t got;
|
||||
while ((got = read(fd, buffer, sizeof(buffer))) > 0) {
|
||||
@@ -254,6 +262,23 @@ bool checksum_digest_file(ChecksumAlgo algo, uint64_t seed, const char* path, ui
|
||||
return ok;
|
||||
}
|
||||
|
||||
if (algo == CHECKSUM_ALGO_MD4) {
|
||||
Md4Ctx ctx;
|
||||
md4_init(&ctx);
|
||||
ok = true;
|
||||
ssize_t got;
|
||||
while ((got = read(fd, buffer, sizeof(buffer))) > 0)
|
||||
md4_update(&ctx, buffer, (size_t)got);
|
||||
if (got < 0)
|
||||
ok = false;
|
||||
if (ok) {
|
||||
md4_final(&ctx, out);
|
||||
*out_len = 16;
|
||||
}
|
||||
close(fd);
|
||||
return ok;
|
||||
}
|
||||
|
||||
XXH64_state_t xxh64;
|
||||
XXH3_state_t* xxh3 = NULL;
|
||||
if (algo == CHECKSUM_ALGO_XXH64) {
|
||||
|
||||
@@ -412,6 +412,11 @@ struct DeletePlanSession {
|
||||
bool dry_run;
|
||||
size_t max_delete;
|
||||
size_t deleted;
|
||||
/* Removals charged against --max-delete. In --delete-delay mode a path is
|
||||
planned (and the budget consumed) while scanning, but `deleted` advances
|
||||
only when the commit actually unlinks it, so an entry that survives the
|
||||
commit (a directory refilled mid-transfer -> ENOTEMPTY) is not reported. */
|
||||
size_t planned;
|
||||
size_t skipped;
|
||||
bool limit_hit;
|
||||
bool limit_logged;
|
||||
@@ -565,7 +570,7 @@ static bool build_plan_skips(const Config* config, const DeletePlanSession* sess
|
||||
}
|
||||
|
||||
static bool budget_available(const DeletePlanSession* session) {
|
||||
return session->deleted < session->max_delete;
|
||||
return session->planned < session->max_delete;
|
||||
}
|
||||
|
||||
static void note_skipped(DeletePlanSession* session) {
|
||||
@@ -579,7 +584,9 @@ static void log_deleted(const char* rel) {
|
||||
free(escaped);
|
||||
}
|
||||
|
||||
/* Append a snapshot path for --delete-delay. */
|
||||
/* Append a snapshot path for --delete-delay. The budget is charged here, but
|
||||
* `deleted` is not: the path counts only once apply_deferred_path truly
|
||||
* unlinks it. */
|
||||
static bool defer_add(DeletePlanSession* session, const char* rel) {
|
||||
char* copy = str_dup(rel);
|
||||
if (!copy)
|
||||
@@ -588,7 +595,7 @@ static bool defer_add(DeletePlanSession* session, const char* rel) {
|
||||
free(copy);
|
||||
return false;
|
||||
}
|
||||
session->deleted++;
|
||||
session->planned++;
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -633,6 +640,7 @@ static bool process_extra_dir(int dirfd, const char* name, const char* child_rel
|
||||
}
|
||||
if (unlinkat(dirfd, name, AT_REMOVEDIR) == 0) {
|
||||
session->deleted++;
|
||||
session->planned++;
|
||||
log_deleted(child_rel);
|
||||
*removed = true;
|
||||
return true;
|
||||
@@ -657,6 +665,7 @@ static bool process_extra_file(int dirfd, const char* name, const char* child_re
|
||||
}
|
||||
if (unlinkat(dirfd, name, 0) == 0) {
|
||||
session->deleted++;
|
||||
session->planned++;
|
||||
log_deleted(child_rel);
|
||||
} else if (errno != ENOENT) {
|
||||
return false;
|
||||
@@ -768,13 +777,14 @@ static bool apply_missing(DeletePlanSession* session, const Config* config) {
|
||||
return true;
|
||||
DeleteManifest manifest = {
|
||||
.keeps = NULL, .protected = NULL, .missing = session->missing, .dirs = NULL};
|
||||
size_t remaining = budget_available(session) ? session->max_delete - session->deleted : 0;
|
||||
size_t remaining = budget_available(session) ? session->max_delete - session->planned : 0;
|
||||
size_t deleted = 0;
|
||||
size_t skipped = 0;
|
||||
bool limit = false;
|
||||
bool ok = manifest_delete_missing_args_limited(config, &manifest, remaining, &deleted, &skipped,
|
||||
&limit);
|
||||
session->deleted += deleted;
|
||||
session->planned += deleted;
|
||||
session->skipped += skipped;
|
||||
if (limit)
|
||||
session->limit_hit = true;
|
||||
@@ -839,7 +849,6 @@ int delete_plan_session_receive(DeletePlanSession* session, const Config* config
|
||||
/* Apply one snapshotted --delete-delay path (post-order: children precede their
|
||||
* parent directory). */
|
||||
static bool apply_deferred_path(DeletePlanSession* session, const Config* config, const char* rel) {
|
||||
(void)session;
|
||||
char* full = path_cat(config->receive_root_directory, rel);
|
||||
if (!full)
|
||||
return false;
|
||||
@@ -863,8 +872,10 @@ static bool apply_deferred_path(DeletePlanSession* session, const Config* config
|
||||
else
|
||||
rc = unlinkat(parent_fd, leaf, 0);
|
||||
bool ok = rc == 0 || errno == ENOENT || errno == ENOTEMPTY || errno == EEXIST;
|
||||
if (rc == 0)
|
||||
if (rc == 0) {
|
||||
session->deleted++;
|
||||
log_deleted(rel);
|
||||
}
|
||||
close(parent_fd);
|
||||
free(leaf);
|
||||
return ok;
|
||||
|
||||
@@ -73,4 +73,22 @@ typedef struct {
|
||||
bool format_stats_send(int fd, const ReceiverStats* stats);
|
||||
bool format_stats_receive(int fd, ReceiverStats* stats);
|
||||
|
||||
/* Sender-side file-list accounting for rsync's `--stats` block. Filled while
|
||||
* the scan/send loops walk each entry: the flist counters describe every
|
||||
* scanned source entry (transferred or skipped), while the transferred/literal
|
||||
* counters describe only the regular files the receiver actually stored. The
|
||||
* type split lets the client print rsync's `Number of files` breakdown; the
|
||||
* receiver-only counters (matched data, deleted, created) come from
|
||||
* STATUS_STATS. */
|
||||
typedef struct {
|
||||
unsigned long long flist_reg;
|
||||
unsigned long long flist_dir;
|
||||
unsigned long long flist_link;
|
||||
unsigned long long flist_special;
|
||||
unsigned long long total_file_size; /* sum of entry sizes (link target len) */
|
||||
unsigned long long transferred_regular; /* regular files actually stored */
|
||||
unsigned long long transferred_file_size; /* source size of those files */
|
||||
unsigned long long literal_data; /* literal bytes sent for them */
|
||||
} TransferStats;
|
||||
|
||||
#endif
|
||||
|
||||
@@ -42,6 +42,7 @@ PipelineContextSender* pipeline_context_sender_create(Config* config, Queue* que
|
||||
context->total_files = 0;
|
||||
context->progress_bytes = 0;
|
||||
context->total_bytes = 0;
|
||||
memset(&context->stats, 0, sizeof(context->stats));
|
||||
context->sender_done = false;
|
||||
atomic_init(&context->cancelled, false);
|
||||
protocol_session_init(&context->allocation_session, -1, -1);
|
||||
|
||||
@@ -9,6 +9,7 @@
|
||||
#include "config.h"
|
||||
#include "delete_plan.h"
|
||||
#include "file.h"
|
||||
#include "format.h"
|
||||
#include "protocol.h"
|
||||
#include "queue.h"
|
||||
#include "stop_condition.h"
|
||||
@@ -86,6 +87,9 @@ typedef struct {
|
||||
int total_files;
|
||||
unsigned long long progress_bytes;
|
||||
unsigned long long total_bytes;
|
||||
/* Per-type flist / transferred accounting for the rsync --stats breakdown and
|
||||
the progress `to-chk` denominator. Guarded by mutex_progress. */
|
||||
TransferStats stats;
|
||||
bool sender_done;
|
||||
atomic_bool cancelled;
|
||||
ProtocolSession allocation_session;
|
||||
|
||||
@@ -313,6 +313,91 @@ class TestDeleteTimingFailure:
|
||||
)
|
||||
|
||||
|
||||
class TestDeleteDelayDeletedCount:
|
||||
"""The reported deleted count must reflect entries actually removed."""
|
||||
|
||||
def test_refilled_deferred_dir_is_not_counted(self):
|
||||
"""A directory snapshotted into a --delete-delay plan that is refilled
|
||||
before the commit survives ENOTEMPTY and must NOT inflate "Number of
|
||||
deleted files" (regression for delete_plan.c counting at snapshot)."""
|
||||
source = os.path.join(TEST_DATA_DIR, "ddc_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "ddc_dst")
|
||||
clean_dir(source)
|
||||
clean_dir(dest)
|
||||
_write(os.path.join(source, "d", "keep.txt"), b"kept payload\n")
|
||||
_write(os.path.join(source, "d", "big.bin"), b"B" * BIG_BYTES)
|
||||
received = get_dest_received_dir(dest, source)
|
||||
extra_dir = os.path.join(received, "d", "extradir")
|
||||
os.makedirs(extra_dir, exist_ok=True)
|
||||
|
||||
def hook():
|
||||
# Runs while big.bin is in flight, after d's delete plan was processed.
|
||||
_write(os.path.join(extra_dir, "new.txt"), b"created mid-transfer\n")
|
||||
|
||||
with ServerManager() as server:
|
||||
server.start(extra_args=["--allow-delete"])
|
||||
proxy = _SlicingProxy(server.port, hook=hook, hook_after=MID_TRANSFER_BYTES,
|
||||
throttle=PROXY_THROTTLE, wait_for_reply=True)
|
||||
flags = ["--delete-delay", "--incremental", "--ignore-times", "--stats"]
|
||||
result, _ = run_client(source, dest, flags=flags, port=proxy.port)
|
||||
proxy.finish()
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:400]
|
||||
assert proxy.hook_called.is_set(), "hook never fired"
|
||||
assert os.path.exists(os.path.join(extra_dir, "new.txt")), "late file vanished"
|
||||
deleted = None
|
||||
for line in result.stdout.splitlines():
|
||||
if line.startswith("Number of deleted files:"):
|
||||
deleted = int(line.split(":", 1)[1].split()[0])
|
||||
assert deleted == 0, (deleted, result.stdout)
|
||||
|
||||
|
||||
class TestDeleteDelayMaxDeleteParity:
|
||||
"""--max-delete with --delete-delay: a partial deletion still reports the
|
||||
number of entries actually removed, matching rsync (the exact surviving set
|
||||
can differ; only the count is compared)."""
|
||||
|
||||
@requires_rsync
|
||||
def test_max_delete_count_matches_rsync(self):
|
||||
source = os.path.join(TEST_DATA_DIR, "ddm_src")
|
||||
rsync_dst = os.path.join(TEST_DATA_DIR, "ddm_rsync_dst")
|
||||
clean_dir(source)
|
||||
clean_dir(rsync_dst)
|
||||
_write(os.path.join(source, "d", "keep.txt"), b"keep\n")
|
||||
for i in range(1, 6):
|
||||
_write(os.path.join(rsync_dst, "d", f"e{i}.txt"), f"extra{i}\n".encode())
|
||||
|
||||
rsync_result = _rsync(["-a", "--delete-delay", "--max-delete=2", "--stats",
|
||||
source + "/", rsync_dst + "/"])
|
||||
# rsync exits 25 ("the --max-delete limit stopped deletions").
|
||||
assert rsync_result.returncode == 25, rsync_result.stderr
|
||||
rsync_count = _deleted_count(rsync_result.stdout)
|
||||
assert rsync_count == 2, rsync_result.stdout
|
||||
|
||||
dest = os.path.join(TEST_DATA_DIR, "ddm_dst")
|
||||
clean_dir(dest)
|
||||
received = get_dest_received_dir(dest, source)
|
||||
for i in range(1, 6):
|
||||
_write(os.path.join(received, "d", f"e{i}.txt"), f"extra{i}\n".encode())
|
||||
with ServerManager() as server:
|
||||
server.start(extra_args=["--allow-delete"])
|
||||
result, _ = run_client(
|
||||
source, dest,
|
||||
flags=["--delete-delay", "--max-delete=2", "--stats"],
|
||||
port=server.port,
|
||||
)
|
||||
# A capped --max-delete commit is a successful transfer that both tools
|
||||
# report with exit 25.
|
||||
assert result.returncode == 25, (result.stderr or result.stdout)[:300]
|
||||
assert _deleted_count(result.stdout) == rsync_count, result.stdout
|
||||
|
||||
|
||||
def _deleted_count(text):
|
||||
for line in text.splitlines():
|
||||
if line.startswith("Number of deleted files:"):
|
||||
return int(line.split(":", 1)[1].split()[0])
|
||||
return None
|
||||
|
||||
|
||||
class TestDeleteDelayVsAfterSnapshot:
|
||||
"""A destination entry created after its directory's scan survives under
|
||||
--delete-delay but is removed by --delete-after's fresh end scan."""
|
||||
|
||||
@@ -447,6 +447,43 @@ class TestWireStatsParity:
|
||||
assert fast_frames[0] == rsync_frames[0], (rsync_frames[0], fast_frames[0])
|
||||
assert "(xfr#1," in fast_frames[-1], fast_frames[-1]
|
||||
|
||||
@requires_rsync
|
||||
@pytest.mark.ci
|
||||
def test_progress_leading_root_line_and_to_chk_match_rsync(self, shared_server):
|
||||
"""A single-file transfer: rsync emits the transfer-root `./` name line
|
||||
and a `to-chk=0/2` denominator that counts that root entry. Both must
|
||||
match FastSync byte-for-byte for the deterministic frames."""
|
||||
source = os.path.join(TEST_DATA_DIR, "wire_pgroot_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "wire_pgroot_dst")
|
||||
rdst = os.path.join(TEST_DATA_DIR, "wire_pgroot_rdst")
|
||||
_make_one_file(source, "f.bin", 100)
|
||||
clean_dir(dest)
|
||||
# rsync prints the `./` root line only when the transfer root itself is
|
||||
# created, so make the rsync destination absent. The "created directory"
|
||||
# line it then emits has no FastSync counterpart (different mirror
|
||||
# layout), so only the name/frame lines are compared.
|
||||
shutil.rmtree(rdst, ignore_errors=True)
|
||||
rsync_result = _rsync(["-a", "--progress", source + "/", rdst + "/"])
|
||||
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||
result, _ = run_client(source, dest, flags=["-a", "--progress"],
|
||||
port=shared_server.port)
|
||||
assert result.returncode == 0, result.stderr[:300]
|
||||
|
||||
# subprocess text mode normalizes \r to \n (universal newlines).
|
||||
def lines_of(text):
|
||||
return [ln for ln in text.splitlines() if ln and not ln.startswith("created directory")]
|
||||
|
||||
rsync_lines = lines_of(rsync_result.stdout)
|
||||
fast_lines = lines_of(result.stdout)
|
||||
rsync_names = [ln for ln in rsync_lines if "%" not in ln]
|
||||
fast_names = [ln for ln in fast_lines if "%" not in ln]
|
||||
|
||||
assert rsync_names == ["sending incremental file list", "./", "f.bin"], rsync_names
|
||||
assert fast_names == rsync_names, (rsync_names, fast_names)
|
||||
# The final frame's to-chk denominator must include the source-root entry.
|
||||
assert "to-chk=0/2" in fast_lines[-1], fast_lines[-1]
|
||||
assert fast_lines[-1] == rsync_lines[-1], (rsync_lines[-1], fast_lines[-1])
|
||||
|
||||
@requires_rsync
|
||||
@pytest.mark.ci
|
||||
@pytest.mark.parametrize("mt", [False, True])
|
||||
@@ -488,18 +525,12 @@ class TestWireStatsParity:
|
||||
|
||||
@requires_rsync
|
||||
@pytest.mark.ci
|
||||
def test_stats_file_count_breakdown_residual(self, shared_server):
|
||||
"""Residual (row #3): rsync prints the `Number of files` and
|
||||
`Number of created files` lines with a per-type breakdown
|
||||
(`(reg: X, dir: Y, link: Z)`).
|
||||
|
||||
FastSync cannot reproduce it from what the sender currently knows: the
|
||||
scanner does not put directory entries in the transfer list (directories
|
||||
are created implicitly), and without a per-entry destination-probe the
|
||||
sender cannot tell which entries the receiver newly created. So FastSync
|
||||
prints the bare transferred-entry count. This test pins the divergence
|
||||
explicitly -- the row must not be marked ✅.
|
||||
"""
|
||||
def test_stats_file_count_breakdown_matches_rsync(self, shared_server):
|
||||
"""`Number of files` now carries rsync's per-type breakdown: the scanner
|
||||
accounts directory entries (captured for -a/-t/-p) plus reg/link/special
|
||||
from the transfer list. `Number of created files` still lacks the type
|
||||
breakdown (FastSync cannot tell which entries the receiver newly
|
||||
created), so that residual is pinned separately."""
|
||||
source = os.path.join(TEST_DATA_DIR, "wire_stc_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "wire_stc_dst")
|
||||
rdst = os.path.join(TEST_DATA_DIR, "wire_stc_rdst")
|
||||
@@ -523,15 +554,44 @@ class TestWireStatsParity:
|
||||
f_files = stats_line(result.stdout, "Number of files")
|
||||
f_created = stats_line(result.stdout, "Number of created files")
|
||||
|
||||
# rsync always carries the type breakdown (the source root counts as a
|
||||
# directory; the single regular file as reg).
|
||||
assert re.match(r"Number of files: 2 \(reg: 1, dir: 1\)$", r_files), r_files
|
||||
assert r_files == f_files, (r_files, f_files)
|
||||
# rsync always carries the created type breakdown; FastSync prints the
|
||||
# bare transferred-regular count (documented residual).
|
||||
assert re.match(r"Number of created files: 1 \(reg: 1\)$", r_created), r_created
|
||||
# FastSync prints only the bare count: no directory accounting and no
|
||||
# per-entry "created" knowledge.
|
||||
assert re.fullmatch(r"Number of files: 1", f_files), f_files
|
||||
assert re.fullmatch(r"Number of created files: 1", f_created), f_created
|
||||
|
||||
@requires_rsync
|
||||
@pytest.mark.ci
|
||||
@pytest.mark.parametrize("choice", ["xxh128", "xxh64", "xxh3", "md5", "md4", "sha1", "none"])
|
||||
def test_out_format_C_selected_algorithm_matches_rsync(self, shared_server, choice):
|
||||
"""`%C` must use the algorithm selected by --checksum-choice, not always
|
||||
xxh128, and render it exactly like rsync (big-endian for the 64-bit
|
||||
hashes, high-then-low for xxh128, standard hex for md5/md4/sha1)."""
|
||||
source = os.path.join(TEST_DATA_DIR, f"wire_cc_{choice}_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, f"wire_cc_{choice}_dst")
|
||||
rdst = os.path.join(TEST_DATA_DIR, f"wire_cc_{choice}_rdst")
|
||||
_make_one_file(source, "f.bin", 200000)
|
||||
clean_dir(dest)
|
||||
clean_dir(rdst)
|
||||
fmt = "%C %l %n"
|
||||
rsync_result = _rsync(["-a", "--checksum-choice=" + choice,
|
||||
"--out-format=" + fmt, source + "/", rdst + "/"])
|
||||
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||
result, _ = run_client(source, dest,
|
||||
flags=["-a", "--checksum-choice=" + choice,
|
||||
"--out-format=" + fmt],
|
||||
port=shared_server.port)
|
||||
assert result.returncode == 0, result.stderr[:300]
|
||||
|
||||
def file_lines(text):
|
||||
return [line for line in text.splitlines()
|
||||
if line and not line.rsplit(" ", 1)[-1].endswith("/")]
|
||||
|
||||
assert file_lines(result.stdout) == file_lines(rsync_result.stdout), (
|
||||
f"choice={choice}: rsync={rsync_result.stdout!r} fastsync={result.stdout!r}"
|
||||
)
|
||||
|
||||
@requires_rsync
|
||||
@pytest.mark.ci
|
||||
@pytest.mark.parametrize("mt", [False, True])
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
#include "test_daemon_conf.h"
|
||||
#include "test_daemon_limits.h"
|
||||
#include "test_delay_updates.h"
|
||||
#include "test_delete_plan.h"
|
||||
#include "test_delta.h"
|
||||
#include "test_file.h"
|
||||
#include "test_file_list.h"
|
||||
@@ -66,6 +67,7 @@ int main() {
|
||||
RUN_TEST(test_scanner);
|
||||
RUN_TEST(test_checksum);
|
||||
RUN_TEST(test_delta);
|
||||
RUN_TEST(test_delete_plan);
|
||||
RUN_TEST(test_data);
|
||||
RUN_TEST(test_protocol);
|
||||
RUN_TEST(test_protocol_error);
|
||||
|
||||
@@ -177,6 +177,40 @@ static void test_change_list_enabled() {
|
||||
config_delete(config); /* closes config->log_file */
|
||||
}
|
||||
|
||||
/* %C uses the negotiated TRANSFER checksum's column width (not the pre-transfer
|
||||
* whole-file digest), and `none` renders as a blank 2-char column, matching
|
||||
* rsync. A not-yet-filled checksum renders as spaces. */
|
||||
static void test_format_C_padding_uses_transfer_algo() {
|
||||
ChangeEvent event = sample_event();
|
||||
Config* config = config_create();
|
||||
EXPECT_NOT_NULL(config);
|
||||
/* Deliberately different pre-transfer algorithm: the transfer one must win. */
|
||||
config->checksum_algo = (int)CHECKSUM_ALGO_MD4;
|
||||
struct {
|
||||
int algo;
|
||||
int width;
|
||||
} cases[] = {
|
||||
{CHECKSUM_ALGO_XXH128, 32}, {CHECKSUM_ALGO_XXH64, 16}, {CHECKSUM_ALGO_XXH3, 16},
|
||||
{CHECKSUM_ALGO_MD5, 32}, {CHECKSUM_ALGO_MD4, 32}, {CHECKSUM_ALGO_SHA1, 40},
|
||||
{CHECKSUM_ALGO_NONE, 2},
|
||||
};
|
||||
for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
|
||||
config->checksum_transfer_algo = cases[i].algo;
|
||||
char expected[64];
|
||||
size_t n = 0;
|
||||
expected[n++] = '[';
|
||||
for (int j = 0; j < cases[i].width; j++)
|
||||
expected[n++] = ' ';
|
||||
expected[n++] = ']';
|
||||
expected[n] = '\0';
|
||||
char* line = change_render_format("[%C]", config, &event);
|
||||
EXPECT_NOT_NULL(line);
|
||||
EXPECT_EQ_STR(line, expected);
|
||||
free(line);
|
||||
}
|
||||
config_delete(config);
|
||||
}
|
||||
|
||||
void test_change_list() {
|
||||
test_format_tokens();
|
||||
test_format_unknown_tokens_preserved();
|
||||
@@ -188,4 +222,5 @@ void test_change_list() {
|
||||
test_render_itemize_up_to_date_is_empty();
|
||||
test_render_list_line();
|
||||
test_change_list_enabled();
|
||||
test_format_C_padding_uses_transfer_algo();
|
||||
}
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
#include "test_checksum.h"
|
||||
#include "checksum.h"
|
||||
#include "test_utils.h"
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <unistd.h>
|
||||
|
||||
/* Known xxHash64 vector (seed 0) for the empty string and a literal.
|
||||
* The md5 vectors are the standard NIST/RFC1321 test strings. These pin the
|
||||
@@ -230,6 +232,39 @@ static void test_checksum_null_empty_digest() {
|
||||
EXPECT_TRUE(memcmp(a, b, alen) == 0);
|
||||
}
|
||||
|
||||
/* Every algorithm checksum_digest_file() claims to support must produce the
|
||||
* SAME digest as the in-memory one-shot, including the newly added md4/sha1/
|
||||
* none. A 200000-byte payload forces several 64 KiB streaming reads. */
|
||||
static void test_checksum_digest_file_matches_oneshot(void) {
|
||||
static const ChecksumAlgo algos[] = {
|
||||
CHECKSUM_ALGO_XXH64, CHECKSUM_ALGO_XXH3, CHECKSUM_ALGO_XXH128, CHECKSUM_ALGO_MD5,
|
||||
CHECKSUM_ALGO_MD4, CHECKSUM_ALGO_SHA1, CHECKSUM_ALGO_NONE,
|
||||
};
|
||||
enum { SIZE = 200000 };
|
||||
uint8_t* data = malloc(SIZE);
|
||||
EXPECT_NOT_NULL(data);
|
||||
for (int i = 0; i < SIZE; i++)
|
||||
data[i] = (uint8_t)((i * 7 + 3) & 0xff);
|
||||
char path[] = "/tmp/fastsync_ck_XXXXXX";
|
||||
int fd = mkstemp(path);
|
||||
EXPECT_TRUE(fd >= 0);
|
||||
ssize_t written = write(fd, data, SIZE);
|
||||
close(fd);
|
||||
EXPECT_EQ_INT((int)written, SIZE);
|
||||
for (size_t a = 0; a < sizeof(algos) / sizeof(algos[0]); a++) {
|
||||
ChecksumAlgo algo = algos[a];
|
||||
uint8_t one[CHECKSUM_MAX_DIGEST_LEN];
|
||||
uint8_t file[CHECKSUM_MAX_DIGEST_LEN];
|
||||
size_t one_len = 0, file_len = 0;
|
||||
EXPECT_TRUE(checksum_digest(algo, 0, data, SIZE, one, sizeof(one), &one_len));
|
||||
EXPECT_TRUE(checksum_digest_file(algo, 0, path, file, sizeof(file), &file_len));
|
||||
EXPECT_EQ_INT((int)file_len, (int)one_len);
|
||||
EXPECT_TRUE(memcmp(one, file, one_len) == 0);
|
||||
}
|
||||
unlink(path);
|
||||
free(data);
|
||||
}
|
||||
|
||||
void test_checksum(void) {
|
||||
test_checksum_xxh64_seed0();
|
||||
test_checksum_xxh64_empty();
|
||||
@@ -245,4 +280,5 @@ void test_checksum(void) {
|
||||
test_checksum_xxh3_xxh128();
|
||||
test_checksum_truncated_buffer_rejected();
|
||||
test_checksum_null_empty_digest();
|
||||
test_checksum_digest_file_matches_oneshot();
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
#include "test_delete_plan.h"
|
||||
#include "charset.h"
|
||||
#include "config.h"
|
||||
#include "delete_plan.h"
|
||||
#include "protocol.h"
|
||||
#include "test_utils.h"
|
||||
#include "utils.h"
|
||||
#include <fcntl.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/socket.h>
|
||||
#include <sys/stat.h>
|
||||
#include <unistd.h>
|
||||
|
||||
/* Send one STATUS_DELETE_PLAN body (the leading status is consumed by the
|
||||
* caller/receiver entry point) describing `dir` with no kept children. */
|
||||
static void send_plan_frame(int fd, const char* dir) {
|
||||
EXPECT_TRUE(send_int(fd, 0)); /* has_config */
|
||||
EXPECT_TRUE(send_wire_str(fd, dir));
|
||||
EXPECT_TRUE(send_int(fd, 0)); /* kept child dirs */
|
||||
EXPECT_TRUE(send_int(fd, 0)); /* kept child files */
|
||||
}
|
||||
|
||||
/* --delete-delay: a directory snapshotted into the plan that is refilled before
|
||||
* the commit must NOT be counted as deleted once its unlink fails ENOTEMPTY.
|
||||
* Regression for delete_plan.c counting at snapshot (defer_add) instead of at
|
||||
* the actual removal. */
|
||||
static void test_delete_delay_refilled_dir_not_counted(void) {
|
||||
char root[] = "/tmp/fastsync_dp_refill_XXXXXX";
|
||||
EXPECT_TRUE(mkdtemp(root) != NULL);
|
||||
char extra[1024];
|
||||
snprintf(extra, sizeof(extra), "%s/extra", root);
|
||||
EXPECT_EQ_INT(mkdir(extra, 0700), 0);
|
||||
|
||||
Config* config = config_create();
|
||||
EXPECT_NOT_NULL(config);
|
||||
config->receive_root_directory = str_dup(root);
|
||||
config->use_delete = true;
|
||||
config->delete_delay = true;
|
||||
|
||||
int p[2];
|
||||
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
|
||||
|
||||
DeletePlanSession* session = delete_plan_session_create(config);
|
||||
EXPECT_NOT_NULL(session);
|
||||
send_plan_frame(p[1], ".");
|
||||
EXPECT_EQ_INT(delete_plan_session_receive(session, config, p[0]), 0);
|
||||
/* The empty extra directory was snapshotted, not removed yet. */
|
||||
EXPECT_EQ_INT((int)delete_plan_session_deleted(session), 0);
|
||||
|
||||
/* Refill the directory while the deferred commit is pending. */
|
||||
char refill[1200];
|
||||
snprintf(refill, sizeof(refill), "%s/new.txt", extra);
|
||||
int fd = open(refill, O_WRONLY | O_CREAT | O_TRUNC, 0600);
|
||||
EXPECT_TRUE(fd >= 0);
|
||||
close(fd);
|
||||
|
||||
EXPECT_EQ_INT(delete_plan_session_commit(session, config), DELETE_COMMIT_OK);
|
||||
/* ENOTEMPTY: the directory survives, so it must not be reported as deleted. */
|
||||
EXPECT_EQ_INT((int)delete_plan_session_deleted(session), 0);
|
||||
struct stat st;
|
||||
EXPECT_EQ_INT(lstat(extra, &st), 0);
|
||||
|
||||
delete_plan_session_destroy(session);
|
||||
close(p[0]);
|
||||
close(p[1]);
|
||||
unlink(refill);
|
||||
rmdir(extra);
|
||||
rmdir(root);
|
||||
config_delete(config);
|
||||
}
|
||||
|
||||
/* The complement: a deferred regular extra that DOES get removed is counted. */
|
||||
static void test_delete_delay_removed_file_counted(void) {
|
||||
char root[] = "/tmp/fastsync_dp_file_XXXXXX";
|
||||
EXPECT_TRUE(mkdtemp(root) != NULL);
|
||||
char extra[1024];
|
||||
snprintf(extra, sizeof(extra), "%s/extra.txt", root);
|
||||
int fd = open(extra, O_WRONLY | O_CREAT | O_TRUNC, 0600);
|
||||
EXPECT_TRUE(fd >= 0);
|
||||
close(fd);
|
||||
|
||||
Config* config = config_create();
|
||||
EXPECT_NOT_NULL(config);
|
||||
config->receive_root_directory = str_dup(root);
|
||||
config->use_delete = true;
|
||||
config->delete_delay = true;
|
||||
|
||||
int p[2];
|
||||
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
|
||||
|
||||
DeletePlanSession* session = delete_plan_session_create(config);
|
||||
EXPECT_NOT_NULL(session);
|
||||
send_plan_frame(p[1], ".");
|
||||
EXPECT_EQ_INT(delete_plan_session_receive(session, config, p[0]), 0);
|
||||
EXPECT_EQ_INT((int)delete_plan_session_deleted(session), 0);
|
||||
EXPECT_EQ_INT(delete_plan_session_commit(session, config), DELETE_COMMIT_OK);
|
||||
EXPECT_EQ_INT((int)delete_plan_session_deleted(session), 1);
|
||||
EXPECT_TRUE(lstat(extra, &(struct stat){0}) != 0);
|
||||
|
||||
delete_plan_session_destroy(session);
|
||||
close(p[0]);
|
||||
close(p[1]);
|
||||
rmdir(root);
|
||||
config_delete(config);
|
||||
}
|
||||
|
||||
/* --max-delete still bounds the deferred plan; the actual (removed) count must
|
||||
* not exceed the limit even though more extras existed. */
|
||||
static void test_delete_delay_max_delete_bounds_actual(void) {
|
||||
char root[] = "/tmp/fastsync_dp_max_XXXXXX";
|
||||
EXPECT_TRUE(mkdtemp(root) != NULL);
|
||||
for (int i = 0; i < 3; i++) {
|
||||
char path[1024];
|
||||
snprintf(path, sizeof(path), "%s/e%d.txt", root, i);
|
||||
int fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0600);
|
||||
EXPECT_TRUE(fd >= 0);
|
||||
close(fd);
|
||||
}
|
||||
|
||||
Config* config = config_create();
|
||||
EXPECT_NOT_NULL(config);
|
||||
config->receive_root_directory = str_dup(root);
|
||||
config->use_delete = true;
|
||||
config->delete_delay = true;
|
||||
config->max_delete = 1;
|
||||
|
||||
int p[2];
|
||||
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
|
||||
|
||||
DeletePlanSession* session = delete_plan_session_create(config);
|
||||
EXPECT_NOT_NULL(session);
|
||||
send_plan_frame(p[1], ".");
|
||||
EXPECT_EQ_INT(delete_plan_session_receive(session, config, p[0]), 0);
|
||||
EXPECT_TRUE(delete_plan_session_limit_reached(session));
|
||||
EXPECT_EQ_INT(delete_plan_session_commit(session, config), DELETE_COMMIT_LIMIT_REACHED);
|
||||
EXPECT_EQ_INT((int)delete_plan_session_deleted(session), 1);
|
||||
|
||||
delete_plan_session_destroy(session);
|
||||
close(p[0]);
|
||||
close(p[1]);
|
||||
for (int i = 0; i < 3; i++) {
|
||||
char path[1024];
|
||||
snprintf(path, sizeof(path), "%s/e%d.txt", root, i);
|
||||
unlink(path);
|
||||
}
|
||||
rmdir(root);
|
||||
config_delete(config);
|
||||
}
|
||||
|
||||
void test_delete_plan(void) {
|
||||
test_delete_delay_refilled_dir_not_counted();
|
||||
test_delete_delay_removed_file_counted();
|
||||
test_delete_delay_max_delete_bounds_actual();
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
#ifndef TEST_DELETE_PLAN_H
|
||||
#define TEST_DELETE_PLAN_H
|
||||
|
||||
void test_delete_plan(void);
|
||||
|
||||
#endif
|
||||
Reference in New Issue
Block a user