Compare commits
18
Commits
6701c103cb
...
829e760086
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
829e760086 | ||
|
|
4c98744014 | ||
|
|
f99e6e1edf | ||
|
|
681d7a8532 | ||
|
|
c6758531f6 | ||
|
|
e754f0d4eb | ||
|
|
4b74f6a41d | ||
|
|
56981a0d9d | ||
|
|
a2ce0a8e41 | ||
|
|
df250ec9c9 | ||
|
|
0afda6b094 | ||
|
|
bfb3a531e9 | ||
|
|
ffdbb6568f | ||
|
|
6ad3887aa1 | ||
|
|
0feb545008 | ||
|
|
be752b9bfd | ||
|
|
97f2dc468b | ||
|
|
c9bd76e633 |
No files matched your search
+54
-10
@@ -6,11 +6,11 @@ This document maps rsync's full feature set to FastSync's current implementation
|
|||||||
|
|
||||||
| Status | Count | Description |
|
| Status | Count | Description |
|
||||||
|--------|-------|-------------|
|
|--------|-------|-------------|
|
||||||
| ✅ Implemented | 80 | Feature works end-to-end |
|
| ✅ Implemented | 86 | Feature works end-to-end |
|
||||||
| 🔀 Alt Arg | 3 | Functionality exists but under different flag/semantics |
|
| 🔀 Alt Arg | 3 | Functionality exists but under different flag/semantics |
|
||||||
| ⚠️ Partial | 5 | Flag parsed/stored but behavior incomplete |
|
| ⚠️ Partial | 5 | Flag parsed/stored but behavior incomplete |
|
||||||
| 🔄 Compatibility No-op | 1 | Flag is accepted for CLI compatibility but has no effect |
|
| 🔄 Compatibility No-op | 1 | Flag is accepted for CLI compatibility but has no effect |
|
||||||
| ❌ Not Implemented | 58 | Flag not recognized or no behavior |
|
| ❌ Not Implemented | 52 | Flag not recognized or no behavior |
|
||||||
| **Total** | **147** | |
|
| **Total** | **147** | |
|
||||||
|
|
||||||
---
|
---
|
||||||
@@ -54,7 +54,7 @@ This document maps rsync's full feature set to FastSync's current implementation
|
|||||||
| `--exclude-from=FILE` | Read exclude patterns from file | ✅ Implemented | Reads patterns from file |
|
| `--exclude-from=FILE` | Read exclude patterns from file | ✅ Implemented | Reads patterns from file |
|
||||||
| `--include-from=FILE` | Read include patterns from file | ✅ Implemented | Reads patterns from file |
|
| `--include-from=FILE` | Read include patterns from file | ✅ Implemented | Reads patterns from file |
|
||||||
| `--filter=RULE` | Add file-filtering rule | ✅ Implemented | Long option only: rsync's short `-f` conflicts with FastSync sendfile (see FastSync-specific list), so `-f` is not reassigned. Supported subset: `+`/`-` include/exclude, implicit-exclude patterns, `include`/`exclude` word forms, a leading `/` anchor (to the transfer root, or to a `.rsync-filter` file's directory), and a trailing `/` for dir-only rules; first match wins with a default of include inside the filter layer. Filters are an independent layer from `--exclude`/`--include` (an entry must pass both). Rejected with a clear error (no silent no-ops): `merge`/`dir-merge`/`hide`/`show`/`protect`/`risk`/`clear` words, rules that begin with `:`/`.`/`!` (merge/dir-merge/list-clear shorthands), and include/exclude modifiers other than `/` (`! C s r p x`) |
|
| `--filter=RULE` | Add file-filtering rule | ✅ Implemented | Long option only: rsync's short `-f` conflicts with FastSync sendfile (see FastSync-specific list), so `-f` is not reassigned. Supported subset: `+`/`-` include/exclude, implicit-exclude patterns, `include`/`exclude` word forms, a leading `/` anchor (to the transfer root, or to a `.rsync-filter` file's directory), and a trailing `/` for dir-only rules; first match wins with a default of include inside the filter layer. Filters are an independent layer from `--exclude`/`--include` (an entry must pass both). Rejected with a clear error (no silent no-ops): `merge`/`dir-merge`/`hide`/`show`/`protect`/`risk`/`clear` words, rules that begin with `:`/`.`/`!` (merge/dir-merge/list-clear shorthands), and include/exclude modifiers other than `/` (`! C s r p x`) |
|
||||||
| `--files-from=FILE` | Read source file list from file | ✅ Implemented | Entries are paths relative to the source root (leading `./` stripped, `..`/absolute entries rejected at parse time, blank lines ignored; NUL-delimited with `-0`). A listed regular file is transferred; a listed directory transfers its whole subtree (FastSync recursion is always on, unlike rsync's non-recursive default). Non-listed paths and their subtrees are pruned by the scanner. A listed entry that does not exist under the source (and an empty list) is a hard error reported before any transfer; listing `.` (whole tree) and empty listed directories are fine. Scalability note: `file_list_affects` is O(list size) per scanned entry, so a very large `--files-from` list against a huge tree is quadratic; lists are typically small enough that this is acceptable, but it is the documented bound. The delete manifest still derives from what was actually sent, so `--delete` stays consistent with the subset |
|
| `--files-from=FILE` | Read source file list from file | ✅ Implemented | Entries are paths relative to the source root (leading `./` stripped, `..`/absolute entries rejected at parse time, blank lines ignored; NUL-delimited with `-0`). A listed regular file is transferred; a listed directory transfers its whole subtree (FastSync recursion is always on, unlike rsync's non-recursive default). Non-listed paths and their subtrees are pruned by the scanner. A listed entry that does not exist under the source (and an empty list) is a hard error reported before any transfer, unless `--ignore-missing-args` / `--delete-missing-args` is given (see the Safety & Security rows): those flags downgrade the listed-but-missing case to a skip and, for `--delete-missing-args`, a destination deletion; an empty list stays a hard error in every mode. Listing `.` (whole tree) and empty listed directories are fine. Scalability note: `file_list_affects` is O(list size) per scanned entry, so a very large `--files-from` list against a huge tree is quadratic; lists are typically small enough that this is acceptable, but it is the documented bound. The delete manifest still derives from what was actually sent, so `--delete` stays consistent with the subset |
|
||||||
| `-0`, `--from0` | Delimit *-from files with NULs | ✅ Implemented | `--files-from` entries become NUL-delimited; the flag may appear before or after `--files-from` on the command line. NUL mode preserves entry bytes exactly (trailing CR/LF are part of the name; only newline mode trims them) |
|
| `-0`, `--from0` | Delimit *-from files with NULs | ✅ Implemented | `--files-from` entries become NUL-delimited; the flag may appear before or after `--files-from` on the command line. NUL mode preserves entry bytes exactly (trailing CR/LF are part of the name; only newline mode trims them) |
|
||||||
| `--max-size=SIZE` | Skip files larger than SIZE | ✅ Implemented | `max_size` in scanner |
|
| `--max-size=SIZE` | Skip files larger than SIZE | ✅ Implemented | `max_size` in scanner |
|
||||||
| `--min-size=SIZE` | Skip files smaller than SIZE | ✅ Implemented | `min_size` in scanner |
|
| `--min-size=SIZE` | Skip files smaller than SIZE | ✅ Implemented | `min_size` in scanner |
|
||||||
@@ -83,8 +83,8 @@ This document maps rsync's full feature set to FastSync's current implementation
|
|||||||
|------|-------------------|-----------------|-------|
|
|------|-------------------|-----------------|-------|
|
||||||
| `-u`, `--update` | Skip files newer on receiver | ❌ Not Implemented | Removed because it had no effect |
|
| `-u`, `--update` | Skip files newer on receiver | ❌ Not Implemented | Removed because it had no effect |
|
||||||
| `--inplace` | Update files in-place | ✅ Implemented | Direct write mode |
|
| `--inplace` | Update files in-place | ✅ Implemented | Direct write mode |
|
||||||
| `--append` | Append data to shorter files | ❌ Not Implemented | Removed because it had no effect |
|
| `--append` | Append data to shorter files | ✅ Implemented | Tail-only resume. When an existing destination file is SHORTER than the source, the receiver negotiates a resume offset with the sender and only the tail is transferred; the receiver rebuilds the full file (retained prefix + tail) and installs it through the normal atomic store path, so the result is byte-identical to the source whenever the retained prefix matches. Plain `--append` does NOT content-verify that prefix (rsync parity): a destination whose prefix differs from the source is resumed anyway, so the result (wrong prefix + correct tail) is NOT byte-identical and the file is effectively left corrupt — the documented rsync-parity risk (use `--append-verify` when the prefix cannot be trusted). Non-content attributes (permissions/ownership/mtime, via `-M`) are still applied. Requires the per-file `STATUS_CHECK` handshake, so it implies `--incremental`; it takes precedence over block delta for a growing file and falls back to delta/full when the destination is not shorter. Incompatible with `-s` (chunk serialization) and `--whole-file` (both rejected up front so the mode never silently degrades to a full transfer). Combines with `--inplace`, `--partial`/`--partial-dir`, and `--delay-updates` (the reconstructed full file flows through those paths unchanged). Divergence: rsync appends in place; FastSync reconstructs and atomically installs, so an interrupted or failed resume never leaves a half-written file at the destination (no corruption window), and `--append` is thus safe to use with the normal atomic path — not only with in-place writes |
|
||||||
| `--append-verify` | Append with old-data checksum | ❌ Not Implemented | Removed because it had no effect |
|
| `--append-verify` | Append with old-data checksum | ✅ Implemented | Like `--append`, but the retained prefix IS verified before resuming: the sender transmits the source prefix checksum and the receiver compares it to the xxHash64 of the retained destination prefix; on a match only the tail is transferred, on a MISMATCH the run falls back to a clean full transfer so the result is always a byte-identical source copy (never a corrupt prefix+tail blend). Wire/protocol: the append handshake adds `STATUS_APPEND` / `STATUS_APPEND_SIG` / `STATUS_APPEND_OK` / `STATUS_APPEND_DATA` frames and `PROTOCOL_VERSION` was bumped **2.9.0 → 2.10.0** (peers must match, and both must be 2.10.0 or the run fails the version check). Same implications/incompatibilities as `--append`; when both spellings are given `--append-verify` wins (the safer semantics). See the Phase-3 append notes below |
|
||||||
| `-W`, `--whole-file` | Copy whole file (no delta) | ❌ Not Implemented | |
|
| `-W`, `--whole-file` | Copy whole file (no delta) | ❌ Not Implemented | |
|
||||||
| `--block-size=SIZE` | Force checksum block-size | ⚠️ Partial | Parsed as `--delta-block`; controls delta transfer block size |
|
| `--block-size=SIZE` | Force checksum block-size | ⚠️ Partial | Parsed as `--delta-block`; controls delta transfer block size |
|
||||||
|
|
||||||
@@ -144,6 +144,24 @@ limit"). These wire/layout changes bumped `PROTOCOL_VERSION` **2.8.0 → 2.9.0**
|
|||||||
`delete_excluded`, `prune_empty_dirs`, `max_delete` — round-trip unchanged and
|
`delete_excluded`, `prune_empty_dirs`, `max_delete` — round-trip unchanged and
|
||||||
are validated on receive.
|
are validated on receive.
|
||||||
|
|
||||||
|
**Missing-args note (Phase 3, missing-args wave):** `--ignore-missing-args` and
|
||||||
|
`--delete-missing-args` are implemented as described in the Safety & Security
|
||||||
|
rows. Wire impact: the `STATUS_MANIFEST` frame now carries a **third section** —
|
||||||
|
a list of destination-relative **exact-delete paths** (the missing entries'
|
||||||
|
mirrors) — and the config frame gained a `delete_missing_args` boolean
|
||||||
|
(`ignore_missing_args` stays client-only, exactly like `ignore_errors`). These
|
||||||
|
wire/layout changes bumped `PROTOCOL_VERSION` **2.9.0 → 2.10.0** (peers must
|
||||||
|
match). The receiver validates the third section identically to the keep-set
|
||||||
|
(non-empty, relative, traversal-free; `MAX_MANIFEST_ENTRIES` per section, a
|
||||||
|
single `MAX_MANIFEST_BYTES` budget shared across all three). On commit the
|
||||||
|
receiver runs the exact-path deletions FIRST (`manifest_delete_missing_args`:
|
||||||
|
confined per-path unlink/rmdir, deep removal only under `--force`/`--delete`,
|
||||||
|
staging/basis protected, never blocked by the protected-prefix list) and then
|
||||||
|
the ordinary extras walk when `--delete` is active (`manifest_delete_all`). A
|
||||||
|
client may request the exact-path deletions without `--delete`; the server's
|
||||||
|
`--allow-delete` policy gates them exactly like `--delete`, so an unauthorized
|
||||||
|
server ignores the request while the missing entries are still skipped.
|
||||||
|
|
||||||
The deletion walker is now **all-or-nothing**: before any unlink it rehearses
|
The deletion walker is now **all-or-nothing**: before any unlink it rehearses
|
||||||
the deletion (an fd-relative walk identical to the delete pass, counting every
|
the deletion (an fd-relative walk identical to the delete pass, counting every
|
||||||
regular file it would unlink and every directory it would remove) and refuses to
|
regular file it would unlink and every directory it would remove) and refuses to
|
||||||
@@ -189,6 +207,32 @@ order-independent because it runs over the fully parsed config. The deletion
|
|||||||
POLICY flags (`--delete-excluded`, `--max-delete`, `--ignore-errors`, `--force`)
|
POLICY flags (`--delete-excluded`, `--max-delete`, `--ignore-errors`, `--force`)
|
||||||
do NOT imply `--delete`; without `--delete` they are inert (matching rsync).
|
do NOT imply `--delete`; without `--delete` they are inert (matching rsync).
|
||||||
|
|
||||||
|
**Append-resume notes (Phase 3, append wave):** `--append` and `--append-verify`
|
||||||
|
are real. Both are negotiated when an existing destination file is found to be
|
||||||
|
**shorter** than the source during the per-file `STATUS_CHECK`; the receiver
|
||||||
|
replies with a new `STATUS_APPEND` frame carrying the resume offset (the prefix
|
||||||
|
length it already holds) instead of `STATUS_NEXT`/`STATUS_DELTA_SIGNATURE`.
|
||||||
|
The sender transmits ONLY the tail. For `--append-verify` it first sends the
|
||||||
|
source's prefix xxHash64 in a `STATUS_APPEND_SIG` frame; the receiver compares
|
||||||
|
it to the retained prefix and answers `STATUS_APPEND_OK` (transfer the tail) or
|
||||||
|
`STATUS_NEXT` (prefix mismatch → the sender falls back to a byte-exact full
|
||||||
|
transfer). The tail arrives in a `STATUS_APPEND_DATA` frame (compression and
|
||||||
|
metadata still apply). The receiver then rebuilds the full file in memory
|
||||||
|
(prefix + tail) and routes it through the existing atomic store engine, so all
|
||||||
|
of `--inplace`, `--partial`/`--partial-dir`, `--delay-updates`, `--backup`,
|
||||||
|
`--existing`/`--ignore-existing`/`--update` and delete-manifest behaviour is
|
||||||
|
unchanged and the result is a byte-identical source copy (given a matching
|
||||||
|
prefix). These new frames changed the wire, so `PROTOCOL_VERSION` was bumped
|
||||||
|
**2.9.0 → 2.10.0** (peers must match; the pre-existing `append`/`append_verify`
|
||||||
|
config booleans already crossed the wire). CLI: both flags imply `--incremental`
|
||||||
|
(the handshake needs it); they are incompatible with `-s` (chunk serialization)
|
||||||
|
and `--whole-file` (both rejected up front, never a silent full transfer); when
|
||||||
|
both spellings are given `--append-verify` wins. The FastSync divergence from
|
||||||
|
rsync is intentional and safer: rsync appends in place, whereas FastSync
|
||||||
|
reconstructs the whole file and atomically installs it, so an interrupted or
|
||||||
|
failed resume never leaves a partial/corrupt file at the destination — this is
|
||||||
|
why plain `--append` works on the normal atomic path, not only with `--inplace`.
|
||||||
|
|
||||||
## 8. Metadata Preservation
|
## 8. Metadata Preservation
|
||||||
|
|
||||||
| Flag | Rsync Description | FastSync Status | Notes |
|
| Flag | Rsync Description | FastSync Status | Notes |
|
||||||
@@ -244,8 +288,8 @@ do NOT imply `--delete`; without `--delete` they are inert (matching rsync).
|
|||||||
|
|
||||||
| Flag | Rsync Description | FastSync Status | Notes |
|
| Flag | Rsync Description | FastSync Status | Notes |
|
||||||
|------|-------------------|-----------------|-------|
|
|------|-------------------|-----------------|-------|
|
||||||
| `--checksum` | Skip based on checksum | ✅ Implemented | With `--incremental`, compares xxHash64 content checksums; `-c` remains compression |
|
| `--checksum` | Skip based on checksum | ✅ Implemented | With `--incremental`, compares per-file whole-file content digests to skip unchanged files. The digest algorithm is `xxh64` with seed 0 by default and is selectable via `--checksum-choice`/`--cc` (xxh64/xxhash or md5) and `--checksum-seed=NUM` (see those rows); `-c` remains compression |
|
||||||
| `--checksum-choice=STR` | Choose checksum algorithm | ❌ Not Implemented | xxHash used internally |
|
| `--checksum-choice=STR`, `--cc=STR` | Choose checksum algorithm | ✅ Implemented | Real algorithm selection for the per-file whole-file digest used by the `--incremental`/`--checksum` handshake and by the basis-dir content verification. FastSync genuinely supports `xxh64` (the default, exact xxHash64, seeded by `--checksum-seed`) and `md5` (via OpenSSL EVP); `xxhash` is accepted as rsync's spelling of xxHash64. Any other name (md4/sha1/sha256/crc32/none/…) is rejected with a clear error at parse time — never a silent no-op. `--cc` is the alias (`--cc=ALG` and space forms both parse). The algorithm id and seed cross the wire with the config frame, so the receiver hashes its on-disk old file with the SAME algorithm+seed the sender used and both agree on a match; the sender's digest and the receiver's comparison live in the per-file `STATUS_CHECK` handshake, which now carries a length-prefixed, bounded (1..16 byte) digest instead of a fixed 64-bit value, and the receiver pins the received length to the negotiated algorithm's digest length (defense-in-depth: a mismatched/malicious length only forces a safe re-transfer). Note: `md5` is a FIPS-non-approved algorithm, so under an OpenSSL build with FIPS mode enabled `--checksum-choice=md5` fails loudly rather than silently falling back. Protocol/layout: `PROTOCOL_VERSION` bumped **2.9.0 → 2.10.0** (peers must match). Defaults preserve the pre-existing behavior byte-for-byte (xxh64, seed 0). Like rsync, the choice only takes effect where a whole-file digest is actually computed (`--checksum` on, or a basis-dir flag); it does not itself enable `--checksum`. Closely-related divergence: the delta BLOCK strong checksum (§11 delta) stays xxHash32 — `--checksum-choice` selects only the whole-file digest, matching rsync where the per-block checksum is independent of the whole-file checksum choice |
|
||||||
| `--compare-dest=DIR` | Compare dest files relative to DIR | ✅ Implemented | DIR is a receiver-side basis relative to the destination root (confined below it; absolute/`..`/`.` rejected, `//` collapsed and trailing `/` dropped). On the receiver's per-file check (implies `--incremental`) an exact match = same size + mtime (unless `--size-only`; `-I` disables matching) **and** equal xxHash64 of the sender's file; a match suppresses the data transfer. compare-dest never copies: it only skips a file the destination does **not** already hold (sparse destination, rsync parity), and is consulted before the normal delta/full paths. Repeatable; searched in command-line order, first match wins. Divergences: when the destination already holds a *different* version rsync deletes it but FastSync instead transfers the data (keeps the mirror complete; never deletes without `--delete`); attribute-only differences on a match are not re-applied (data is skipped so the sender never sends metadata); content is verified by xxHash64, stricter than rsync's default quick check. Sizing: FastSync's whole-file payload limit is 256 MiB on **every** transfer path (not basis-specific); rsync applies basis dirs to arbitrary sizes, so FastSync refuses a basis run whose source contains a larger file up front with a clear error before any transfer. Wire: a basis-count field is always present on the config frame (protocol 2.9.0, so clients and servers must both be 2.9.0) |
|
| `--compare-dest=DIR` | Compare dest files relative to DIR | ✅ Implemented | DIR is a receiver-side basis relative to the destination root (confined below it; absolute/`..`/`.` rejected, `//` collapsed and trailing `/` dropped). On the receiver's per-file check (implies `--incremental`) an exact match = same size + mtime (unless `--size-only`; `-I` disables matching) **and** equal xxHash64 of the sender's file; a match suppresses the data transfer. compare-dest never copies: it only skips a file the destination does **not** already hold (sparse destination, rsync parity), and is consulted before the normal delta/full paths. Repeatable; searched in command-line order, first match wins. Divergences: when the destination already holds a *different* version rsync deletes it but FastSync instead transfers the data (keeps the mirror complete; never deletes without `--delete`); attribute-only differences on a match are not re-applied (data is skipped so the sender never sends metadata); content is verified by xxHash64, stricter than rsync's default quick check. Sizing: FastSync's whole-file payload limit is 256 MiB on **every** transfer path (not basis-specific); rsync applies basis dirs to arbitrary sizes, so FastSync refuses a basis run whose source contains a larger file up front with a clear error before any transfer. Wire: a basis-count field is always present on the config frame (protocol 2.9.0, so clients and servers must both be 2.9.0) |
|
||||||
| `--copy-dest=DIR` | Include copies of unchanged files | ✅ Implemented | Same basis rules as `--compare-dest`, but an exact match materializes a **local copy** of the DIR file into the destination (via the normal atomic temp+rename store path, so `--existing`/`--ignore-existing`/`--update`/`--backup`/`--delay-updates` all still apply) instead of transferring data. Repeatable; command-line order = priority. Content is xxHash64-verified before the copy. Divergences: a basis-hit destination keeps the basis file's own mode/uid/gid and mtime (the sender sends no metadata on a skip), so with `--size-only` its mtime can differ from the source and attribute-only differences are copied with the basis attributes rather than rsync's "copy + fix attributes". Requires `--incremental` (implied); incompatible with `-s`. Wire: protocol 2.9.0 |
|
| `--copy-dest=DIR` | Include copies of unchanged files | ✅ Implemented | Same basis rules as `--compare-dest`, but an exact match materializes a **local copy** of the DIR file into the destination (via the normal atomic temp+rename store path, so `--existing`/`--ignore-existing`/`--update`/`--backup`/`--delay-updates` all still apply) instead of transferring data. Repeatable; command-line order = priority. Content is xxHash64-verified before the copy. Divergences: a basis-hit destination keeps the basis file's own mode/uid/gid and mtime (the sender sends no metadata on a skip), so with `--size-only` its mtime can differ from the source and attribute-only differences are copied with the basis attributes rather than rsync's "copy + fix attributes". Requires `--incremental` (implied); incompatible with `-s`. Wire: protocol 2.9.0 |
|
||||||
| `--link-dest=DIR` | Hardlink to files when unchanged | ✅ Implemented | Same basis rules as `--copy-dest`, but an exact match installs an atomic **hard link** to the DIR file (temp hard link + rename) so no data or disk space is used; where the link is impossible (basis on another filesystem, filesystem refuses links) it falls back cleanly to a byte-identical local copy, never a corrupt/partial file. `--delay-updates` stages the link and publishes by rename, so the final entry stays a real hard link. Repeatable (searched in command-line order, first match wins). Content is xxHash64-verified before linking. Divergences and caveats: an already up-to-date destination file is not re-linked to a basis file (only files that would otherwise be written are linked); a link keeps the basis inode's own mode/uid/gid and mtime — metadata is never written through the shared inode (that would mutate the basis file), so a later `--inplace` run that rewrites such a destination path **will mutate the basis snapshot** through the shared inode (use `--copy-dest` when the destination must stay independently writable); with `--size-only` the linked mtime can differ from the source; a `--remove-source-files` source satisfied by a basis dir is treated as skipped and therefore **retained** (never removed); basis dirs are excluded from `--delete`. Requires `--incremental` (implied); incompatible with `-s`. Wire: protocol 2.9.0 |
|
| `--link-dest=DIR` | Hardlink to files when unchanged | ✅ Implemented | Same basis rules as `--copy-dest`, but an exact match installs an atomic **hard link** to the DIR file (temp hard link + rename) so no data or disk space is used; where the link is impossible (basis on another filesystem, filesystem refuses links) it falls back cleanly to a byte-identical local copy, never a corrupt/partial file. `--delay-updates` stages the link and publishes by rename, so the final entry stays a real hard link. Repeatable (searched in command-line order, first match wins). Content is xxHash64-verified before linking. Divergences and caveats: an already up-to-date destination file is not re-linked to a basis file (only files that would otherwise be written are linked); a link keeps the basis inode's own mode/uid/gid and mtime — metadata is never written through the shared inode (that would mutate the basis file), so a later `--inplace` run that rewrites such a destination path **will mutate the basis snapshot** through the shared inode (use `--copy-dest` when the destination must stay independently writable); with `--size-only` the linked mtime can differ from the source; a `--remove-source-files` source satisfied by a basis dir is treated as skipped and therefore **retained** (never removed); basis dirs are excluded from `--delete`. Requires `--incremental` (implied); incompatible with `-s`. Wire: protocol 2.9.0 |
|
||||||
@@ -299,8 +343,8 @@ do NOT imply `--delete`; without `--delete` they are inert (matching rsync).
|
|||||||
| `--max-alloc=SIZE` | Limit a single memory allocation | ✅ Implemented | Caps the largest single allocation; binary units, default 1G |
|
| `--max-alloc=SIZE` | Limit a single memory allocation | ✅ Implemented | Caps the largest single allocation; binary units, default 1G |
|
||||||
| `--trust-sender` | Trust remote sender's file list | ❌ Not Implemented | |
|
| `--trust-sender` | Trust remote sender's file list | ❌ Not Implemented | |
|
||||||
| `--old-args` | Disable modern arg protection | ✅ Implemented | SSH-only legacy mode; restores raw remote command construction and permits shell interpretation of the configured server path |
|
| `--old-args` | Disable modern arg protection | ✅ Implemented | SSH-only legacy mode; restores raw remote command construction and permits shell interpretation of the configured server path |
|
||||||
| `--ignore-missing-args` | Ignore missing source args | ❌ Not Implemented | |
|
| `--ignore-missing-args` | Ignore missing source args | ✅ Implemented | FastSync has a single source-root argument (which always exists), so the "explicitly requested source arguments" are the `--files-from` entries and the flags only ever apply there (inert without `--files-from`, like `-R`). Without the flag a listed-but-missing entry stays a hard pre-transfer error (nothing is transferred). With it each missing entry is skipped: nothing is sent for it, it never enters the keep-set, and the run succeeds for the rest — an all-missing non-empty list succeeds transferring nothing, matching rsync. `--dirs` + `--files-from` missing entries are skipped the same way. Every skipped entry is logged and a per-run warning names the count, so the handling is never a silent no-op. Divergences: an EMPTY `--files-from` file stays a hard error in every mode (no argument was requested at all; rsync likewise reports "no source files specified"); missing-arg skipping only applies to the pre-transfer list validation, so an entry that is present at preflight and vanishes mid-transfer still fails (matching rsync, whose flag "does not affect subsequent vanished-file errors"); `--no-ignore-missing-args` is not a supported negation |
|
||||||
| `--delete-missing-args` | Delete missing source args | ❌ Not Implemented | |
|
| `--delete-missing-args` | Delete missing source args | ✅ Implemented | Implies `--ignore-missing-args` (order-independent) and additionally removes each missing entry's destination mirror receiver-side. The mirror is computed exactly like a present sibling's wire path: the bare relative entry under `-R`, otherwise the full source-mirror path below the destination root. rsync parity, verified against the man page: it does **not** imply `--delete` generally and is "independent of any other type of delete processing" — unrelated destination extras are untouched unless `--delete` is also present. Composition with `--delete` + timing: the exact-path deletions commit with the manifest, early for `--delete-before`/`--delete-during`, else only after a fully-successful transfer (delete-after/commit). A non-empty directory mirror is removed only when `--force` or `--delete` is in effect (otherwise it is left with a warning and the run continues, like rsync); an absent mirror is a no-op. An explicitly listed missing arg is a user request, not an excluded file: its deletion is never blocked by the filter-exclusion protection of excluded destination mirrors (a mirror sitting inside a filter-excluded directory is still removed). Safety/policy: gated by the server `--allow-delete` policy like `--delete`; the request paths cross the wire only in the delete-manifest frame and are confined by the same receiver validation as the keep-set (non-empty, relative, traversal-free, bounded by the per-section/per-frame manifest caps); the `--delay-updates` staging directory and basis snapshots are protected exactly as in the extras walker. Divergence: the missing-args deletions are not counted toward `--max-delete` (they are explicit per-path requests, not discovered extras). See the Phase-3 wire note below for the `PROTOCOL_VERSION` bump |
|
||||||
|
|
||||||
## 16. Batch Operations
|
## 16. Batch Operations
|
||||||
|
|
||||||
@@ -319,7 +363,7 @@ do NOT imply `--delete`; without `--delete` they are inert (matching rsync).
|
|||||||
| `--fsync` | Fsync every written file before publication | ✅ Implemented | |
|
| `--fsync` | Fsync every written file before publication | ✅ Implemented | |
|
||||||
| `--protocol=NUM` | Force older protocol version | ❌ Not Implemented | |
|
| `--protocol=NUM` | Force older protocol version | ❌ Not Implemented | |
|
||||||
| `--iconv=CONVERT_SPEC` | Charset conversion | ❌ Not Implemented | |
|
| `--iconv=CONVERT_SPEC` | Charset conversion | ❌ Not Implemented | |
|
||||||
| `--checksum-seed=NUM` | Set checksum seed | ❌ Not Implemented | |
|
| `--checksum-seed=NUM` | Set checksum seed | ✅ Implemented | Sets the seed for FastSync's whole-file xxHash64 digest (full 64-bit seed) and for the delta path's per-block xxHash32 strong checksum (low 32 bits of the seed). An explicit seed deterministically changes every computed digest on BOTH endpoints (sender and receiver share the seed via the config frame, protocol 2.10.0), so identical runs with the same seed skip the same files and a changed seed changes the digests — the explicit-seed path that makes xxHash comparisons deterministic. `--checksum-choice=md5` has no seed and ignores it (documented). The value is a strict decimal 0..2⁶⁴-1 (blank, signed, or non-numeric values are rejected). Like rsync, a seed only matters where a digest is actually computed (`--checksum` or a basis-dir run, or a delta transfer); it does not by itself enable `--checksum`/`--delta`. Divergence from rsync: the default is seed 0, and FastSync never randomizes the seed (rsync uses a random per-transfer seed when `--checksum-seed` is unset); FastSync's unset default therefore reproduces its historical byte-for-byte behavior |
|
||||||
| `--secluded-args` | Use protocol to send args | 🔄 Compatibility No-op | Accepted for CLI compatibility; it does not change FastSync transport or protocol behavior. `-s` remains chunk serialization. |
|
| `--secluded-args` | Use protocol to send args | 🔄 Compatibility No-op | Accepted for CLI compatibility; it does not change FastSync transport or protocol behavior. `-s` remains chunk serialization. |
|
||||||
| `--no-OPTION` | Turn off implied option | ✅ Supported | Supported boolean FastSync options and archive-implied options; unsafe or value-taking options are rejected. |
|
| `--no-OPTION` | Turn off implied option | ✅ Supported | Supported boolean FastSync options and archive-implied options; unsafe or value-taking options are rejected. |
|
||||||
|
|
||||||
|
|||||||
+79
-1
@@ -94,6 +94,47 @@ static int set_compression_choice(Config* config, const char* value) {
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Validate and store the --checksum-choice/--cc algorithm. Only the algorithms
|
||||||
|
* the engine genuinely supports are accepted (xxHash64 and md5); anything else
|
||||||
|
* is a clear error, never a silent no-op. "xxhash" is accepted as rsync's
|
||||||
|
* spelling of xxHash64. */
|
||||||
|
static int set_checksum_choice(Config* config, const char* value) {
|
||||||
|
int algo = checksum_algo_from_name(value);
|
||||||
|
if (algo < 0) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "--checksum-choice must be xxh64 (or xxhash) or md5 (got '%s')",
|
||||||
|
value);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
config->checksum_algo = algo;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* parse_ull_arg is defined later in this file; declared here for the seed
|
||||||
|
parser below. */
|
||||||
|
static int parse_ull_arg(const char* val, unsigned long long* out, const char* optname);
|
||||||
|
|
||||||
|
/* Parse --checksum-seed=NUM as a strict decimal 0..UINT64_MAX. A blank value,
|
||||||
|
* a sign, or any non-digit (which parse_ull_arg's strtoull would silently
|
||||||
|
* coerce) is rejected: an explicit seed must be an exact unsigned integer or
|
||||||
|
* the run fails with a clear error rather than quietly ignoring the value. */
|
||||||
|
static int set_checksum_seed(Config* config, const char* value) {
|
||||||
|
if (!value || *value == '\0') {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "--checksum-seed must be a non-negative integer");
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
for (const char* p = value; *p; p++) {
|
||||||
|
if (*p < '0' || *p > '9') {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "--checksum-seed must be a non-negative integer");
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
unsigned long long seed;
|
||||||
|
if (parse_ull_arg(value, &seed, "--checksum-seed") != 0)
|
||||||
|
return -1;
|
||||||
|
config->checksum_seed = seed;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
static int set_compression_threads_option(int* dest, const char* value) {
|
static int set_compression_threads_option(int* dest, const char* value) {
|
||||||
if (set_positive_int_option(dest, value, "--compress-threads") != 0)
|
if (set_positive_int_option(dest, value, "--compress-threads") != 0)
|
||||||
return -1;
|
return -1;
|
||||||
@@ -435,6 +476,8 @@ static const OptionEntry OPTION_TABLE[] = {
|
|||||||
{"--copy-unsafe-links", NULL, OPT_FLAG, offsetof(Config, copy_unsafe_links)},
|
{"--copy-unsafe-links", NULL, OPT_FLAG, offsetof(Config, copy_unsafe_links)},
|
||||||
{"--sparse", "-S", OPT_FLAG, offsetof(Config, preserve_sparse)},
|
{"--sparse", "-S", OPT_FLAG, offsetof(Config, preserve_sparse)},
|
||||||
{"--inplace", NULL, OPT_FLAG, offsetof(Config, inplace)},
|
{"--inplace", NULL, OPT_FLAG, offsetof(Config, inplace)},
|
||||||
|
{"--append", NULL, OPT_FLAG, offsetof(Config, append)},
|
||||||
|
{"--append-verify", NULL, OPT_FLAG, offsetof(Config, append_verify)},
|
||||||
{"--fsync", NULL, OPT_FLAG, offsetof(Config, use_fsync)},
|
{"--fsync", NULL, OPT_FLAG, offsetof(Config, use_fsync)},
|
||||||
{"--checksum", NULL, OPT_FLAG, offsetof(Config, checksum)},
|
{"--checksum", NULL, OPT_FLAG, offsetof(Config, checksum)},
|
||||||
{"--8-bit-output", "-8", OPT_FLAG, offsetof(Config, eight_bit_output)},
|
{"--8-bit-output", "-8", OPT_FLAG, offsetof(Config, eight_bit_output)},
|
||||||
@@ -460,6 +503,8 @@ static const OptionEntry OPTION_TABLE[] = {
|
|||||||
{"--ignore-errors", NULL, OPT_FLAG, offsetof(Config, ignore_errors)},
|
{"--ignore-errors", NULL, OPT_FLAG, offsetof(Config, ignore_errors)},
|
||||||
{"--force", NULL, OPT_FLAG, offsetof(Config, force_delete)},
|
{"--force", NULL, OPT_FLAG, offsetof(Config, force_delete)},
|
||||||
{"--prune-empty-dirs", NULL, OPT_FLAG, offsetof(Config, prune_empty_dirs)},
|
{"--prune-empty-dirs", NULL, OPT_FLAG, offsetof(Config, prune_empty_dirs)},
|
||||||
|
{"--ignore-missing-args", NULL, OPT_FLAG, offsetof(Config, ignore_missing_args)},
|
||||||
|
{"--delete-missing-args", NULL, OPT_FLAG, offsetof(Config, delete_missing_args)},
|
||||||
|
|
||||||
{"--source-dir", NULL, OPT_STRING, offsetof(Config, send_directory)},
|
{"--source-dir", NULL, OPT_STRING, offsetof(Config, send_directory)},
|
||||||
{"--dest-dir", NULL, OPT_STRING, offsetof(Config, receive_root_directory)},
|
{"--dest-dir", NULL, OPT_STRING, offsetof(Config, receive_root_directory)},
|
||||||
@@ -739,6 +784,12 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args,
|
|||||||
entry->offset == offsetof(Config, delete_delay) ||
|
entry->offset == offsetof(Config, delete_delay) ||
|
||||||
entry->offset == offsetof(Config, delete_after))
|
entry->offset == offsetof(Config, delete_after))
|
||||||
config->use_delete = true;
|
config->use_delete = true;
|
||||||
|
/* --delete-missing-args implies --ignore-missing-args (missing entries
|
||||||
|
are skipped for deletion instead of failing the run). The implication
|
||||||
|
is order-independent because it is applied over the final parsed
|
||||||
|
config. */
|
||||||
|
if (entry->offset == offsetof(Config, delete_missing_args))
|
||||||
|
config->ignore_missing_args = true;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1009,7 +1060,23 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args,
|
|||||||
log_message(LOG_LEVEL_ERROR, "missing argument for %s", argv[i]);
|
log_message(LOG_LEVEL_ERROR, "missing argument for %s", argv[i]);
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
log_message(LOG_LEVEL_ERROR, "%s is not supported yet (xxHash64 is used)", argv[i]);
|
if (set_checksum_choice(config, argv[++i]) != 0)
|
||||||
|
return -1;
|
||||||
|
} else if (strncmp(argv[i], "--checksum-choice=", 18) == 0) {
|
||||||
|
if (set_checksum_choice(config, argv[i] + 18) != 0)
|
||||||
|
return -1;
|
||||||
|
} else if (strncmp(argv[i], "--cc=", 5) == 0) {
|
||||||
|
if (set_checksum_choice(config, argv[i] + 5) != 0)
|
||||||
|
return -1;
|
||||||
|
} else if (strncmp(argv[i], "--checksum-seed=", 16) == 0) {
|
||||||
|
if (set_checksum_seed(config, argv[i] + 16) != 0)
|
||||||
|
return -1;
|
||||||
|
} else if (opt_is(argv[i], "--checksum-seed", NULL)) {
|
||||||
|
if (i + 1 >= argc) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "missing argument for --checksum-seed");
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
if (set_checksum_seed(config, argv[++i]) != 0)
|
||||||
return -1;
|
return -1;
|
||||||
} else if (strncmp(argv[i], "--compare-dest=", 15) == 0) {
|
} else if (strncmp(argv[i], "--compare-dest=", 15) == 0) {
|
||||||
if (set_basis_dest_option(config, BASIS_DEST_COMPARE, argv[i] + 15, "--compare-dest") != 0)
|
if (set_basis_dest_option(config, BASIS_DEST_COMPARE, argv[i] + 15, "--compare-dest") != 0)
|
||||||
@@ -1102,6 +1169,17 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args,
|
|||||||
config->use_delta = true;
|
config->use_delta = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* --append / --append-verify resume a shorter existing destination file.
|
||||||
|
* The receiver must run the per-file STATUS_CHECK handshake to learn the
|
||||||
|
* destination length and reply STATUS_APPEND, so an append mode forces
|
||||||
|
* --incremental on (exactly like the basis-dir options: the handshake is
|
||||||
|
* required, not optional). The resume itself is a dedicated tail-only
|
||||||
|
* exchange, not the block delta, so no delta implication is made. When both
|
||||||
|
* spelling are given the safer --append-verify semantics win. */
|
||||||
|
if (config->append || config->append_verify) {
|
||||||
|
config->use_incremental = true;
|
||||||
|
}
|
||||||
|
|
||||||
/* Incremental and delta transfers need metadata unless the user disabled it. */
|
/* Incremental and delta transfers need metadata unless the user disabled it. */
|
||||||
if ((config->use_incremental || config->use_delta) && !config->use_metadata &&
|
if ((config->use_incremental || config->use_delta) && !config->use_metadata &&
|
||||||
!config->metadata_explicitly_disabled) {
|
!config->metadata_explicitly_disabled) {
|
||||||
|
|||||||
+324
-42
@@ -104,6 +104,7 @@ static bool prepare_scanner(const Config* config, int num_threads, PreparedScann
|
|||||||
options->relative = config->relative;
|
options->relative = config->relative;
|
||||||
options->prune_empty_dirs = config->prune_empty_dirs;
|
options->prune_empty_dirs = config->prune_empty_dirs;
|
||||||
options->ignore_io_errors = config->ignore_errors;
|
options->ignore_io_errors = config->ignore_errors;
|
||||||
|
options->ignore_missing_args = config->ignore_missing_args || config->delete_missing_args;
|
||||||
options->excluded_paths = NULL;
|
options->excluded_paths = NULL;
|
||||||
options->excluded_mutex = NULL;
|
options->excluded_mutex = NULL;
|
||||||
return true;
|
return true;
|
||||||
@@ -182,13 +183,40 @@ static bool no_implied_dirs_files_from_valid(const Config* config) {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* The destination-relative mirror path for a missing --files-from entry: where
|
||||||
|
a PRESENT entry with the same name would have been written. With -R that is
|
||||||
|
the entry's bare relative path (the bare wire path the receiver uses);
|
||||||
|
otherwise it is the full source mirror below the destination root
|
||||||
|
(`send_directory` joined to the entry, leading '/' stripped), exactly the
|
||||||
|
path the manifest records for a present sibling. Returns an owned string, or
|
||||||
|
NULL on allocation failure. */
|
||||||
|
static char* files_from_missing_dest_path(const Config* config, const char* entry) {
|
||||||
|
if (config->relative)
|
||||||
|
return str_dup(entry);
|
||||||
|
char* joined = path_cat(config->send_directory, entry);
|
||||||
|
if (!joined)
|
||||||
|
return NULL;
|
||||||
|
const char* rel = *joined == '/' ? joined + 1 : joined;
|
||||||
|
char* dup = str_dup(rel);
|
||||||
|
free(joined);
|
||||||
|
return dup;
|
||||||
|
}
|
||||||
|
|
||||||
/* --files-from semantics: every listed entry must resolve under the source
|
/* --files-from semantics: every listed entry must resolve under the source
|
||||||
* root, otherwise rsync reports a hard error instead of silently transferring
|
* root, otherwise rsync reports a hard error instead of silently transferring
|
||||||
* nothing. An empty list is also an error. An entry of "." (the whole tree)
|
* nothing. An empty list is also an error. An entry of "." (the whole tree)
|
||||||
* and listed-but-empty directories are valid. Runs before any transfer so the
|
* and listed-but-empty directories are valid. With --ignore-missing-args
|
||||||
* failure is surfaced uniformly in the single-threaded, -m, dry-run and
|
* (implied by --delete-missing-args) a listed-but-missing entry is instead
|
||||||
* --list-only paths. */
|
* skipped: nothing is transferred for it, it never enters the keep-set and the
|
||||||
static bool files_from_list_valid(const Config* config) {
|
* run succeeds for the rest (an all-missing non-empty list succeeds
|
||||||
|
* transferring nothing, matching rsync). With --delete-missing-args
|
||||||
|
* `missing_dest` (when non-NULL) collects the entry's destination-relative
|
||||||
|
* mirror for the receiver's exact-deletion request. An empty list stays a
|
||||||
|
* hard error in every mode (nothing was requested at all). Runs before any
|
||||||
|
* transfer so the failure/skip is surfaced uniformly in the single-threaded,
|
||||||
|
* -m, dry-run and --list-only paths. */
|
||||||
|
static bool files_from_list_check(const Config* config, ArrayList* missing_dest, int* skipped_out) {
|
||||||
|
*skipped_out = 0;
|
||||||
const FileListSet* set = (const FileListSet*)config->files_from_set;
|
const FileListSet* set = (const FileListSet*)config->files_from_set;
|
||||||
if (!set)
|
if (!set)
|
||||||
return true;
|
return true;
|
||||||
@@ -201,6 +229,7 @@ static bool files_from_list_valid(const Config* config) {
|
|||||||
config->files_from ? config->files_from : "");
|
config->files_from ? config->files_from : "");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
bool ignore = config->ignore_missing_args || config->delete_missing_args;
|
||||||
for (int i = 0; i < set->count; i++) {
|
for (int i = 0; i < set->count; i++) {
|
||||||
const char* entry = set->entries[i];
|
const char* entry = set->entries[i];
|
||||||
if (entry[0] == '\0')
|
if (entry[0] == '\0')
|
||||||
@@ -212,13 +241,51 @@ static bool files_from_list_valid(const Config* config) {
|
|||||||
}
|
}
|
||||||
struct stat st;
|
struct stat st;
|
||||||
if (lstat(full, &st) != 0) {
|
if (lstat(full, &st) != 0) {
|
||||||
|
free(full);
|
||||||
|
if (ignore) {
|
||||||
|
(*skipped_out)++;
|
||||||
|
log_info_message(LOG_INFO_MISC, "skipping missing --files-from entry '%s'", entry);
|
||||||
|
if (config->delete_missing_args && missing_dest) {
|
||||||
|
char* mirror = files_from_missing_dest_path(config, entry);
|
||||||
|
if (!mirror || !array_list_add(missing_dest, mirror)) {
|
||||||
|
free(mirror);
|
||||||
|
log_message(LOG_LEVEL_ERROR, "memory allocation failed while validating --files-from");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
log_message(LOG_LEVEL_ERROR, "--files-from entry '%s' not found in source '%s'", entry,
|
log_message(LOG_LEVEL_ERROR, "--files-from entry '%s' not found in source '%s'", entry,
|
||||||
config->send_directory);
|
config->send_directory);
|
||||||
free(full);
|
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
free(full);
|
free(full);
|
||||||
}
|
}
|
||||||
|
if (*skipped_out > 0) {
|
||||||
|
if (config->delete_missing_args) {
|
||||||
|
/* --list-only never deletes and a --dry-run only shows intent, so the
|
||||||
|
summary must not claim a real deletion happened in those modes. */
|
||||||
|
if (config->list_only)
|
||||||
|
log_message(LOG_LEVEL_WARNING,
|
||||||
|
"--delete-missing-args: %d missing --files-from entr%s skipped (--list-only "
|
||||||
|
"never deletes)",
|
||||||
|
*skipped_out, *skipped_out == 1 ? "y" : "ies");
|
||||||
|
else if (config->dry_run)
|
||||||
|
log_message(LOG_LEVEL_WARNING,
|
||||||
|
"--delete-missing-args: %d missing --files-from entr%s would be deleted from "
|
||||||
|
"the destination (dry run)",
|
||||||
|
*skipped_out, *skipped_out == 1 ? "y" : "ies");
|
||||||
|
else
|
||||||
|
log_message(
|
||||||
|
LOG_LEVEL_WARNING,
|
||||||
|
"--delete-missing-args: %d missing --files-from entr%s will be deleted from the "
|
||||||
|
"destination",
|
||||||
|
*skipped_out, *skipped_out == 1 ? "y" : "ies");
|
||||||
|
} else if (config->ignore_missing_args)
|
||||||
|
log_message(LOG_LEVEL_WARNING,
|
||||||
|
"--ignore-missing-args: ignored %d missing --files-from entr%s", *skipped_out,
|
||||||
|
*skipped_out == 1 ? "y" : "ies");
|
||||||
|
}
|
||||||
return no_implied_dirs_files_from_valid(config);
|
return no_implied_dirs_files_from_valid(config);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -458,15 +525,30 @@ static void pipeline_cancel(PipelineContextSender* context) {
|
|||||||
|
|
||||||
/* Print dry-run manifest showing files that would be transferred. Returns 0 on success. */
|
/* Print dry-run manifest showing files that would be transferred. Returns 0 on success. */
|
||||||
static int send_dry_run_manifest(const Config* config) {
|
static int send_dry_run_manifest(const Config* config) {
|
||||||
if (!files_from_list_valid(config))
|
int skipped = 0;
|
||||||
|
ArrayList* missing_dest = NULL;
|
||||||
|
if (config->delete_missing_args) {
|
||||||
|
missing_dest = array_list_create(free);
|
||||||
|
if (!missing_dest)
|
||||||
return -1;
|
return -1;
|
||||||
|
}
|
||||||
|
if (!files_from_list_check(config, missing_dest, &skipped)) {
|
||||||
|
if (missing_dest)
|
||||||
|
array_list_delete(missing_dest);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
PreparedScanner prepared;
|
PreparedScanner prepared;
|
||||||
if (!prepare_scanner(config, 0, &prepared))
|
if (!prepare_scanner(config, 0, &prepared)) {
|
||||||
|
if (missing_dest)
|
||||||
|
array_list_delete(missing_dest);
|
||||||
return -1;
|
return -1;
|
||||||
|
}
|
||||||
DirectoryScanner* scanner =
|
DirectoryScanner* scanner =
|
||||||
directory_scanner_create_with_options(config->send_directory, &prepared.options);
|
directory_scanner_create_with_options(config->send_directory, &prepared.options);
|
||||||
if (!scanner) {
|
if (!scanner) {
|
||||||
prepared_scanner_destroy(&prepared);
|
prepared_scanner_destroy(&prepared);
|
||||||
|
if (missing_dest)
|
||||||
|
array_list_delete(missing_dest);
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
Chunk* chunk;
|
Chunk* chunk;
|
||||||
@@ -484,6 +566,8 @@ static int send_dry_run_manifest(const Config* config) {
|
|||||||
chunk_destroy(chunk);
|
chunk_destroy(chunk);
|
||||||
directory_scanner_destroy(scanner);
|
directory_scanner_destroy(scanner);
|
||||||
prepared_scanner_destroy(&prepared);
|
prepared_scanner_destroy(&prepared);
|
||||||
|
if (missing_dest)
|
||||||
|
array_list_delete(missing_dest);
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
if (config->human_readable)
|
if (config->human_readable)
|
||||||
@@ -501,6 +585,17 @@ static int send_dry_run_manifest(const Config* config) {
|
|||||||
}
|
}
|
||||||
directory_scanner_destroy(scanner);
|
directory_scanner_destroy(scanner);
|
||||||
prepared_scanner_destroy(&prepared);
|
prepared_scanner_destroy(&prepared);
|
||||||
|
/* --delete-missing-args: the missing entries' destination mirrors render as
|
||||||
|
would-be deletions (rsync's dry-run also lists its *deleting lines). */
|
||||||
|
if (missing_dest && !config->quiet) {
|
||||||
|
for (int i = 0; i < missing_dest->size; i++) {
|
||||||
|
char* escaped = output_escape((char*)missing_dest->items[i], config->eight_bit_output);
|
||||||
|
printf(" %s (missing; would be deleted)\n", escaped ? escaped : "<allocation failed>");
|
||||||
|
free(escaped);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (missing_dest)
|
||||||
|
array_list_delete(missing_dest);
|
||||||
if (!config->quiet) {
|
if (!config->quiet) {
|
||||||
if (config->human_readable)
|
if (config->human_readable)
|
||||||
printf("Total: %d files, %s\n", file_count,
|
printf("Total: %d files, %s\n", file_count,
|
||||||
@@ -537,7 +632,8 @@ static int compare_list_entries(const void* left, const void* right) {
|
|||||||
* Directory lines are not printed because the scanner only yields regular
|
* Directory lines are not printed because the scanner only yields regular
|
||||||
* transfer candidates. Returns 0 on success, 1 on error. */
|
* transfer candidates. Returns 0 on success, 1 on error. */
|
||||||
static int send_list_only(const Config* config) {
|
static int send_list_only(const Config* config) {
|
||||||
if (!files_from_list_valid(config))
|
int skipped = 0;
|
||||||
|
if (!files_from_list_check(config, NULL, &skipped))
|
||||||
return 1;
|
return 1;
|
||||||
PreparedScanner prepared;
|
PreparedScanner prepared;
|
||||||
if (!prepare_scanner(config, 0, &prepared))
|
if (!prepare_scanner(config, 0, &prepared))
|
||||||
@@ -626,22 +722,26 @@ static int send_list_only(const Config* config) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/* Send the delete manifest (keep-set paths plus the protected excluded
|
/* Send the delete manifest (keep-set paths plus the protected excluded
|
||||||
prefixes) to the server. Returns 0 on success, -1 on failure. When
|
prefixes and the --delete-missing-args exact-delete paths) to the server.
|
||||||
--delete-excluded is given `protected` is empty: excluded destination
|
Returns 0 on success, -1 on failure. When --delete-excluded is given
|
||||||
mirrors are then ordinary extras and are removed. Both sections are
|
`protected` is empty: excluded destination mirrors are then ordinary extras
|
||||||
unbounded on the sender; the receiver enforces MAX_MANIFEST_ENTRIES per
|
and are removed. When --delete-missing-args is active `missing_args` holds
|
||||||
section and a single MAX_MANIFEST_BYTES budget shared across the two
|
the destination mirrors of missing --files-from entries: each is an explicit
|
||||||
sections, rejecting (with STATUS_ERROR) an over-budget frame. A heavily
|
receiver-side deletion request, independent of the extras walk. A NULL
|
||||||
filtered source whose exclusion list is large therefore fails the run
|
keep-set / protected / missing list transmits an empty section. All three
|
||||||
cleanly on the receiver rather than being truncated. */
|
sections are unbounded on the sender; the receiver enforces
|
||||||
static int send_delete_manifest(int fd, ArrayList* manifest, ArrayList* protected_prefixes) {
|
MAX_MANIFEST_ENTRIES per section and a single MAX_MANIFEST_BYTES budget
|
||||||
if (!manifest)
|
shared across the sections, rejecting (with STATUS_ERROR) an over-budget
|
||||||
return -1;
|
frame. A heavily filtered source whose exclusion list is large therefore
|
||||||
|
fails the run cleanly on the receiver rather than being truncated. */
|
||||||
|
static int send_delete_manifest(int fd, ArrayList* manifest, ArrayList* protected_prefixes,
|
||||||
|
ArrayList* missing_args) {
|
||||||
if (!send_status(fd, STATUS_MANIFEST))
|
if (!send_status(fd, STATUS_MANIFEST))
|
||||||
return -1;
|
return -1;
|
||||||
if (!send_int(fd, manifest->size))
|
int keep_count = manifest ? manifest->size : 0;
|
||||||
|
if (!send_int(fd, keep_count))
|
||||||
return -1;
|
return -1;
|
||||||
for (int i = 0; i < manifest->size; i++) {
|
for (int i = 0; i < keep_count; i++) {
|
||||||
if (!send_str(fd, (char*)manifest->items[i]))
|
if (!send_str(fd, (char*)manifest->items[i]))
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
@@ -652,6 +752,13 @@ static int send_delete_manifest(int fd, ArrayList* manifest, ArrayList* protecte
|
|||||||
if (!send_str(fd, (char*)protected_prefixes->items[i]))
|
if (!send_str(fd, (char*)protected_prefixes->items[i]))
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
int missing_count = missing_args ? missing_args->size : 0;
|
||||||
|
if (!send_int(fd, missing_count))
|
||||||
|
return -1;
|
||||||
|
for (int i = 0; i < missing_count; i++) {
|
||||||
|
if (!send_str(fd, (char*)missing_args->items[i]))
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -667,10 +774,11 @@ static int send_delete_manifest(int fd, ArrayList* manifest, ArrayList* protecte
|
|||||||
#define DELETE_ACK_TIMEOUT_SEC 3600
|
#define DELETE_ACK_TIMEOUT_SEC 3600
|
||||||
|
|
||||||
static bool send_delete_manifest_early(Client* client, ArrayList* manifest,
|
static bool send_delete_manifest_early(Client* client, ArrayList* manifest,
|
||||||
ArrayList* protected_prefixes) {
|
ArrayList* protected_prefixes, ArrayList* missing_args) {
|
||||||
if (!client || !manifest)
|
if (!client || !manifest)
|
||||||
return false;
|
return false;
|
||||||
if (send_delete_manifest(client->file_descriptor, manifest, protected_prefixes) != 0)
|
if (send_delete_manifest(client->file_descriptor, manifest, protected_prefixes, missing_args) !=
|
||||||
|
0)
|
||||||
return false;
|
return false;
|
||||||
Status ack;
|
Status ack;
|
||||||
if (!receive_status_timed(client->file_descriptor, &ack, DELETE_ACK_TIMEOUT_SEC))
|
if (!receive_status_timed(client->file_descriptor, &ack, DELETE_ACK_TIMEOUT_SEC))
|
||||||
@@ -717,8 +825,10 @@ static bool scan_paths_only(const Config* config, const ScannerOptions* options,
|
|||||||
}
|
}
|
||||||
|
|
||||||
static int incremental_check(Client* client, File* file, const Config* config,
|
static int incremental_check(Client* client, File* file, const Config* config,
|
||||||
DeltaSignature** out_sig) {
|
DeltaSignature** out_sig, unsigned long long* resume_offset) {
|
||||||
*out_sig = NULL;
|
*out_sig = NULL;
|
||||||
|
if (resume_offset)
|
||||||
|
*resume_offset = 0;
|
||||||
if (!send_status(client->file_descriptor, STATUS_CHECK))
|
if (!send_status(client->file_descriptor, STATUS_CHECK))
|
||||||
return -1;
|
return -1;
|
||||||
if (!send_str(client->file_descriptor, file_wire_path(file)))
|
if (!send_str(client->file_descriptor, file_wire_path(file)))
|
||||||
@@ -733,12 +843,18 @@ static int incremental_check(Client* client, File* file, const Config* config,
|
|||||||
if (!send_n_data(client->file_descriptor, &mtime_nsec, sizeof(mtime_nsec)))
|
if (!send_n_data(client->file_descriptor, &mtime_nsec, sizeof(mtime_nsec)))
|
||||||
return -1;
|
return -1;
|
||||||
/* With alternate basis directories the receiver must be able to verify the
|
/* With alternate basis directories the receiver must be able to verify the
|
||||||
* content of every candidate basis file, so the sender supplies its xxHash64
|
* content of every candidate basis file, so the sender supplies its whole-file
|
||||||
* for every file even when --checksum was not requested. */
|
* digest (computed with the negotiated --checksum-choice algorithm and
|
||||||
|
* --checksum-seed) for every file even when --checksum was not requested. */
|
||||||
if (config->checksum || config_has_basis(config)) {
|
if (config->checksum || config_has_basis(config)) {
|
||||||
uint64_t checksum;
|
uint8_t digest[CHECKSUM_MAX_DIGEST_LEN];
|
||||||
if (!file_checksum(file, &checksum) ||
|
size_t digest_len = 0;
|
||||||
!send_n_data(client->file_descriptor, &checksum, sizeof(checksum)))
|
if (!file_checksum(file, (ChecksumAlgo)config->checksum_algo, config->checksum_seed, digest,
|
||||||
|
sizeof(digest), &digest_len))
|
||||||
|
return -1;
|
||||||
|
uint8_t wire_len = (uint8_t)digest_len;
|
||||||
|
if (!send_n_data(client->file_descriptor, &wire_len, sizeof(wire_len)) ||
|
||||||
|
!send_n_data(client->file_descriptor, digest, wire_len))
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
Status s;
|
Status s;
|
||||||
@@ -765,6 +881,19 @@ static int incremental_check(Client* client, File* file, const Config* config,
|
|||||||
*out_sig = sig;
|
*out_sig = sig;
|
||||||
return 2;
|
return 2;
|
||||||
}
|
}
|
||||||
|
if (s == STATUS_APPEND) {
|
||||||
|
/* --append / --append-verify tail resume: the receiver found an existing
|
||||||
|
destination SHORTER than the source and wants only the tail from this
|
||||||
|
offset (the bytes it already holds). */
|
||||||
|
unsigned long long offset;
|
||||||
|
if (!receive_n_data(client->file_descriptor, &offset, sizeof(offset))) {
|
||||||
|
send_status(client->file_descriptor, STATUS_ERROR);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
if (resume_offset)
|
||||||
|
*resume_offset = offset;
|
||||||
|
return 3;
|
||||||
|
}
|
||||||
if (s != STATUS_NEXT) {
|
if (s != STATUS_NEXT) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Unexpected server status");
|
log_message(LOG_LEVEL_ERROR, "Unexpected server status");
|
||||||
send_status(client->file_descriptor, STATUS_ERROR);
|
send_status(client->file_descriptor, STATUS_ERROR);
|
||||||
@@ -774,7 +903,8 @@ static int incremental_check(Client* client, File* file, const Config* config,
|
|||||||
}
|
}
|
||||||
|
|
||||||
static int send_delta(Client* client, File* file, DeltaSignature* sig, Config* config) {
|
static int send_delta(Client* client, File* file, DeltaSignature* sig, Config* config) {
|
||||||
Delta* delta = delta_compute(file->data->data, file->data->size, sig, config->delta_block_size);
|
Delta* delta = delta_compute_seeded(file->data->data, file->data->size, sig,
|
||||||
|
config->delta_block_size, (uint32_t)config->checksum_seed);
|
||||||
/* The receiver is blocked after sending the signature. Every local
|
/* The receiver is blocked after sending the signature. Every local
|
||||||
fallback therefore needs the explicit NEXT response before full data. */
|
fallback therefore needs the explicit NEXT response before full data. */
|
||||||
if (!delta)
|
if (!delta)
|
||||||
@@ -813,6 +943,89 @@ static int send_delta(Client* client, File* file, DeltaSignature* sig, Config* c
|
|||||||
return ok ? 0 : -1;
|
return ok ? 0 : -1;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* --append / --append-verify tail resume. The receiver learned the existing
|
||||||
|
* destination is SHORTER than the source and replied STATUS_APPEND with the
|
||||||
|
* resume offset (prefix bytes it already holds). For plain --append we send
|
||||||
|
* the tail immediately (the prefix is not content-verified, matching rsync).
|
||||||
|
* For --append-verify we first send the source prefix xxHash64; the receiver
|
||||||
|
* compares it to the retained prefix and replies STATUS_APPEND_OK (send the
|
||||||
|
* tail) or STATUS_NEXT (prefix mismatch -> full transfer, never corrupt).
|
||||||
|
* Returns 0 on success, 1 when a full transfer was done instead, -1 on error. */
|
||||||
|
static int send_append(const Client* client, File* file, Config* config,
|
||||||
|
unsigned long long offset) {
|
||||||
|
int fd = client->file_descriptor;
|
||||||
|
const unsigned long long fsize = file->data->size;
|
||||||
|
if (offset >= fsize) {
|
||||||
|
send_status(fd, STATUS_ERROR);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
size_t off = (size_t)offset;
|
||||||
|
size_t tail_len = (size_t)(fsize - off);
|
||||||
|
int compression_level = config->use_compression ? config->compression_level : 0;
|
||||||
|
int skip_count = config->skip_compress_set ? config->skip_compress_count : -1;
|
||||||
|
bool compress = compression_level > 0 &&
|
||||||
|
!compression_should_skip_with_suffixes(file->path, config->skip_compress_suffixes,
|
||||||
|
skip_count);
|
||||||
|
|
||||||
|
/* --append-verify: exchange the source prefix checksum and await the verdict. */
|
||||||
|
if (config->append_verify) {
|
||||||
|
uint64_t prefix_hash = delta_xxhash64(file->data->data, off);
|
||||||
|
if (!send_status(fd, STATUS_APPEND_SIG) || !send_n_data(fd, &prefix_hash, sizeof(prefix_hash)))
|
||||||
|
return -1;
|
||||||
|
Status resp;
|
||||||
|
if (!receive_status(fd, &resp))
|
||||||
|
return -1;
|
||||||
|
if (resp == STATUS_NEXT) {
|
||||||
|
/* Retained prefix does not match the source: fall back to the atomic full
|
||||||
|
transfer (byte-identical, never a corrupt prefix+tail blend). */
|
||||||
|
int rc = file_send_single_calls_with_skip(file, fd, config->use_metadata, compression_level,
|
||||||
|
false, config->skip_compress_suffixes, skip_count,
|
||||||
|
config->compression_threads)
|
||||||
|
? 1
|
||||||
|
: -1;
|
||||||
|
return rc;
|
||||||
|
}
|
||||||
|
if (resp != STATUS_APPEND_OK) {
|
||||||
|
send_status(fd, STATUS_ERROR);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!send_status(fd, STATUS_APPEND_DATA)) {
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
if (config->use_metadata && !metadata_send(fd, file->metadata)) {
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
bool ok;
|
||||||
|
if (compress) {
|
||||||
|
/* Compression needs an owned copy of the tail to compress. */
|
||||||
|
Data* tail = data_create_empty(tail_len);
|
||||||
|
if (!tail) {
|
||||||
|
send_status(fd, STATUS_ERROR);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
memcpy(tail->data, (const char*)file->data->data + off, tail_len);
|
||||||
|
Data* comp = data_compress_with_threads(tail, compression_level, config->compression_threads);
|
||||||
|
data_destroy(tail);
|
||||||
|
if (!comp) {
|
||||||
|
send_status(fd, STATUS_ERROR);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
ok = send_data(fd, comp);
|
||||||
|
data_destroy(comp);
|
||||||
|
} else {
|
||||||
|
/* Uncompressed: send directly from the source buffer (no per-file copy;
|
||||||
|
send_data is synchronous, so the view outlives the call). */
|
||||||
|
Data tail_view;
|
||||||
|
tail_view.data = (char*)file->data->data + off;
|
||||||
|
tail_view.size = tail_len;
|
||||||
|
tail_view.protocol_charge = 0;
|
||||||
|
ok = send_data(fd, &tail_view);
|
||||||
|
}
|
||||||
|
return ok ? 0 : -1;
|
||||||
|
}
|
||||||
|
|
||||||
// Send a single file directly (non-incremental path).
|
// Send a single file directly (non-incremental path).
|
||||||
static bool send_file_direct(File* file, int fd, bool use_metadata, int compression_level,
|
static bool send_file_direct(File* file, int fd, bool use_metadata, int compression_level,
|
||||||
const Config* config) {
|
const Config* config) {
|
||||||
@@ -867,7 +1080,8 @@ static int send_single_file(Client* client, File* file, Config* config, bool use
|
|||||||
// Incremental path: use sendfile for the actual data if enabled and no compression
|
// Incremental path: use sendfile for the actual data if enabled and no compression
|
||||||
if (use_sendfile) {
|
if (use_sendfile) {
|
||||||
DeltaSignature* sig = NULL;
|
DeltaSignature* sig = NULL;
|
||||||
int rc = incremental_check(client, file, config, &sig);
|
unsigned long long resume_offset = 0;
|
||||||
|
int rc = incremental_check(client, file, config, &sig, &resume_offset);
|
||||||
if (rc == 1) {
|
if (rc == 1) {
|
||||||
log_info_message(LOG_INFO_SKIP, "Skipping unchanged %s", file->path);
|
log_info_message(LOG_INFO_SKIP, "Skipping unchanged %s", file->path);
|
||||||
delta_signature_destroy(sig);
|
delta_signature_destroy(sig);
|
||||||
@@ -877,6 +1091,16 @@ static int send_single_file(Client* client, File* file, Config* config, bool use
|
|||||||
delta_signature_destroy(sig);
|
delta_signature_destroy(sig);
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
// rc == 3: append resume (tail-only) -- send_append uses the data path.
|
||||||
|
if (rc == 3) {
|
||||||
|
delta_signature_destroy(sig);
|
||||||
|
int arc = send_append(client, file, config, resume_offset);
|
||||||
|
if (arc == 1) {
|
||||||
|
log_info_message(LOG_INFO_COPY, "Append prefix mismatch; full transfer of %s", file->path);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
return arc == 0 ? 0 : -1;
|
||||||
|
}
|
||||||
// rc == 0: unchanged file, skip
|
// rc == 0: unchanged file, skip
|
||||||
// rc == 2: server sent delta signature but sendfile doesn't support delta
|
// rc == 2: server sent delta signature but sendfile doesn't support delta
|
||||||
delta_signature_destroy(sig);
|
delta_signature_destroy(sig);
|
||||||
@@ -896,7 +1120,8 @@ static int send_single_file(Client* client, File* file, Config* config, bool use
|
|||||||
|
|
||||||
// Incremental path with single_calls (supports compression and delta)
|
// Incremental path with single_calls (supports compression and delta)
|
||||||
DeltaSignature* sig = NULL;
|
DeltaSignature* sig = NULL;
|
||||||
int rc = incremental_check(client, file, config, &sig);
|
unsigned long long resume_offset = 0;
|
||||||
|
int rc = incremental_check(client, file, config, &sig, &resume_offset);
|
||||||
if (rc < 0) {
|
if (rc < 0) {
|
||||||
delta_signature_destroy(sig);
|
delta_signature_destroy(sig);
|
||||||
return -1;
|
return -1;
|
||||||
@@ -906,6 +1131,17 @@ static int send_single_file(Client* client, File* file, Config* config, bool use
|
|||||||
delta_signature_destroy(sig);
|
delta_signature_destroy(sig);
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
|
if (rc == 3) {
|
||||||
|
/* --append / --append-verify tail resume. send_append reports 1 when the
|
||||||
|
verified prefix mismatched and a full transfer was sent instead. */
|
||||||
|
delta_signature_destroy(sig);
|
||||||
|
int arc = send_append(client, file, config, resume_offset);
|
||||||
|
if (arc == 1) {
|
||||||
|
log_info_message(LOG_INFO_COPY, "Append prefix mismatch; full transfer of %s", file->path);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
return arc == 0 ? 0 : -1;
|
||||||
|
}
|
||||||
if (rc == 2 && config->use_delta && !config->whole_file) {
|
if (rc == 2 && config->use_delta && !config->whole_file) {
|
||||||
int drc = send_delta(client, file, sig, config);
|
int drc = send_delta(client, file, sig, config);
|
||||||
delta_signature_destroy(sig);
|
delta_signature_destroy(sig);
|
||||||
@@ -1031,7 +1267,8 @@ static int send_chunks_multithreaded(void* pipeline_context) {
|
|||||||
if (context->early_delete) {
|
if (context->early_delete) {
|
||||||
/* The keep-set manifest was prebuilt by a path-only pre-scan. Transmit it
|
/* The keep-set manifest was prebuilt by a path-only pre-scan. Transmit it
|
||||||
and wait for the receiver to delete extras before streaming any data. */
|
and wait for the receiver to delete extras before streaming any data. */
|
||||||
if (!send_delete_manifest_early(client, context->manifest, context->excluded_paths)) {
|
if (!send_delete_manifest_early(client, context->manifest, context->excluded_paths,
|
||||||
|
context->missing_args)) {
|
||||||
pipeline_cancel(context);
|
pipeline_cancel(context);
|
||||||
disconnect_transfer_client(client);
|
disconnect_transfer_client(client);
|
||||||
mark_sender_done(context);
|
mark_sender_done(context);
|
||||||
@@ -1066,7 +1303,13 @@ static int send_chunks_multithreaded(void* pipeline_context) {
|
|||||||
goto send_fail;
|
goto send_fail;
|
||||||
}
|
}
|
||||||
if (send_delete_manifest(client->file_descriptor, context->manifest,
|
if (send_delete_manifest(client->file_descriptor, context->manifest,
|
||||||
context->excluded_paths) != 0)
|
context->excluded_paths, context->missing_args) != 0)
|
||||||
|
goto send_fail;
|
||||||
|
} else if (context->config->delete_missing_args && !context->early_delete) {
|
||||||
|
/* --delete-missing-args without --delete: no keep-set is built, but the
|
||||||
|
exact-delete paths still ride the same manifest frame (commit once the
|
||||||
|
transfer succeeded). */
|
||||||
|
if (send_delete_manifest(client->file_descriptor, NULL, NULL, context->missing_args) != 0)
|
||||||
goto send_fail;
|
goto send_fail;
|
||||||
}
|
}
|
||||||
bool ok = finalize_transfer(client, context->config, context->remove_source_files);
|
bool ok = finalize_transfer(client, context->config, context->remove_source_files);
|
||||||
@@ -1322,10 +1565,23 @@ int send_files(Config* config) {
|
|||||||
return send_list_only(config);
|
return send_list_only(config);
|
||||||
if (config->dry_run)
|
if (config->dry_run)
|
||||||
return send_dry_run_manifest(config);
|
return send_dry_run_manifest(config);
|
||||||
if (!files_from_list_valid(config))
|
ArrayList* missing_args = NULL;
|
||||||
|
int skipped = 0;
|
||||||
|
if (config->delete_missing_args) {
|
||||||
|
missing_args = array_list_create(free);
|
||||||
|
if (!missing_args)
|
||||||
return 1;
|
return 1;
|
||||||
if (config_has_basis(config) && !basis_oversize_preflight(config))
|
}
|
||||||
|
if (!files_from_list_check(config, missing_args, &skipped)) {
|
||||||
|
if (missing_args)
|
||||||
|
array_list_delete(missing_args);
|
||||||
return 1;
|
return 1;
|
||||||
|
}
|
||||||
|
if (config_has_basis(config) && !basis_oversize_preflight(config)) {
|
||||||
|
if (missing_args)
|
||||||
|
array_list_delete(missing_args);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
Client* client = connect_transfer_client(config);
|
Client* client = connect_transfer_client(config);
|
||||||
if (!client) {
|
if (!client) {
|
||||||
@@ -1392,7 +1648,7 @@ int send_files(Config* config) {
|
|||||||
"with an empty keep-set (--delete)");
|
"with an empty keep-set (--delete)");
|
||||||
prescan_ok = false;
|
prescan_ok = false;
|
||||||
} else {
|
} else {
|
||||||
early_ok = send_delete_manifest_early(client, early_manifest, excluded);
|
early_ok = send_delete_manifest_early(client, early_manifest, excluded, missing_args);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
array_list_delete(early_manifest);
|
array_list_delete(early_manifest);
|
||||||
@@ -1478,17 +1734,24 @@ int send_files(Config* config) {
|
|||||||
"an empty keep-set (--delete)");
|
"an empty keep-set (--delete)");
|
||||||
goto send_fail;
|
goto send_fail;
|
||||||
}
|
}
|
||||||
|
if ((manifest || config->delete_missing_args) && !delete_early) {
|
||||||
|
/* Late (commit) ordering: all file data is out; transmit the manifest so
|
||||||
|
the receiver commits the extras walk (--delete) and/or the
|
||||||
|
--delete-missing-args exact-path deletions only after the transfer
|
||||||
|
succeeds. In the early modes (--delete-before/--delete-during) the
|
||||||
|
manifest already went out up front, so nothing is re-sent here. */
|
||||||
|
if (send_delete_manifest(client->file_descriptor, manifest, excluded, missing_args) != 0) {
|
||||||
if (manifest) {
|
if (manifest) {
|
||||||
/* Late (commit) ordering: all file data is out; transmit the keep-set
|
|
||||||
manifest so the receiver deletes only after the transfer succeeds. */
|
|
||||||
if (send_delete_manifest(client->file_descriptor, manifest, excluded) != 0) {
|
|
||||||
array_list_delete(manifest);
|
array_list_delete(manifest);
|
||||||
manifest = NULL;
|
manifest = NULL;
|
||||||
|
}
|
||||||
goto send_fail;
|
goto send_fail;
|
||||||
}
|
}
|
||||||
|
if (manifest) {
|
||||||
array_list_delete(manifest);
|
array_list_delete(manifest);
|
||||||
manifest = NULL;
|
manifest = NULL;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
bool ok = finalize_transfer(client, config, remove_sources);
|
bool ok = finalize_transfer(client, config, remove_sources);
|
||||||
if (!ok && config->use_delete)
|
if (!ok && config->use_delete)
|
||||||
log_message(LOG_LEVEL_ERROR,
|
log_message(LOG_LEVEL_ERROR,
|
||||||
@@ -1526,6 +1789,8 @@ send_fail:
|
|||||||
array_list_delete(manifest);
|
array_list_delete(manifest);
|
||||||
if (excluded)
|
if (excluded)
|
||||||
array_list_delete(excluded);
|
array_list_delete(excluded);
|
||||||
|
if (missing_args)
|
||||||
|
array_list_delete(missing_args);
|
||||||
if (remove_sources)
|
if (remove_sources)
|
||||||
array_list_delete(remove_sources);
|
array_list_delete(remove_sources);
|
||||||
if (scanner)
|
if (scanner)
|
||||||
@@ -1544,10 +1809,23 @@ int send_files_multithreaded(Config** config_ptr) {
|
|||||||
return send_list_only(config);
|
return send_list_only(config);
|
||||||
if (config->dry_run)
|
if (config->dry_run)
|
||||||
return send_dry_run_manifest(config);
|
return send_dry_run_manifest(config);
|
||||||
if (!files_from_list_valid(config))
|
ArrayList* missing_args = NULL;
|
||||||
|
int skipped = 0;
|
||||||
|
if (config->delete_missing_args) {
|
||||||
|
missing_args = array_list_create(free);
|
||||||
|
if (!missing_args)
|
||||||
return 1;
|
return 1;
|
||||||
if (config_has_basis(config) && !basis_oversize_preflight(config))
|
}
|
||||||
|
if (!files_from_list_check(config, missing_args, &skipped)) {
|
||||||
|
if (missing_args)
|
||||||
|
array_list_delete(missing_args);
|
||||||
return 1;
|
return 1;
|
||||||
|
}
|
||||||
|
if (config_has_basis(config) && !basis_oversize_preflight(config)) {
|
||||||
|
if (missing_args)
|
||||||
|
array_list_delete(missing_args);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
long pages = sysconf(_SC_AVPHYS_PAGES);
|
long pages = sysconf(_SC_AVPHYS_PAGES);
|
||||||
long page_size = sysconf(_SC_PAGE_SIZE);
|
long page_size = sysconf(_SC_PAGE_SIZE);
|
||||||
@@ -1574,8 +1852,12 @@ int send_files_multithreaded(Config** config_ptr) {
|
|||||||
if (!context) {
|
if (!context) {
|
||||||
queue_destroy(q1);
|
queue_destroy(q1);
|
||||||
queue_destroy(q2);
|
queue_destroy(q2);
|
||||||
|
if (missing_args)
|
||||||
|
array_list_delete(missing_args);
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
|
context->missing_args = missing_args;
|
||||||
|
missing_args = NULL; /* owned by the context from here on */
|
||||||
*config_ptr = NULL; /* context now owns config through all remaining paths */
|
*config_ptr = NULL; /* context now owns config through all remaining paths */
|
||||||
bool collect_excluded = config->use_delete && !config->delete_excluded;
|
bool collect_excluded = config->use_delete && !config->delete_excluded;
|
||||||
if (config->use_delete) {
|
if (config->use_delete) {
|
||||||
|
|||||||
@@ -51,14 +51,26 @@ bool validate_config(const Config* config) {
|
|||||||
log_message(LOG_LEVEL_ERROR, "--delta cannot be combined with -f (sendfile)");
|
log_message(LOG_LEVEL_ERROR, "--delta cannot be combined with -f (sendfile)");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
if (config->log_file_format && !config->log_file) {
|
/* --append / --append-verify resume a shorter existing destination by
|
||||||
log_message(LOG_LEVEL_ERROR, "--log-file-format requires --log-file");
|
transmitting only the tail. The resume needs the per-file STATUS_CHECK
|
||||||
|
handshake (so the dest length is learned), which chunk serialization -s
|
||||||
|
disables; and whole-file is the opposite intent (send everything), so the
|
||||||
|
two would silently make the resume pointless. Both are rejected up front
|
||||||
|
rather than silently degrading to a full transfer. */
|
||||||
|
if ((config->append || config->append_verify) && config->use_chunk_serialization) {
|
||||||
|
log_message(LOG_LEVEL_ERROR,
|
||||||
|
"--append/--append-verify require the per-file incremental check and cannot be "
|
||||||
|
"combined with -s (chunk serialization)");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
if (config->append || config->append_verify) {
|
if ((config->append || config->append_verify) && config->whole_file) {
|
||||||
fprintf(
|
log_message(LOG_LEVEL_ERROR,
|
||||||
stderr,
|
"--append/--append-verify are incompatible with --whole-file (which forces a "
|
||||||
"Error: --append and --append-verify are not supported yet; refusing to ignore option\n");
|
"full transfer)");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (config->log_file_format && !config->log_file) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "--log-file-format requires --log-file");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
if (config->use_tls) {
|
if (config->use_tls) {
|
||||||
|
|||||||
@@ -352,6 +352,7 @@ DirectoryScanner* directory_scanner_create_with_options(const char* root_directo
|
|||||||
scanner->excluded_paths = options->excluded_paths;
|
scanner->excluded_paths = options->excluded_paths;
|
||||||
scanner->excluded_mutex = options->excluded_mutex;
|
scanner->excluded_mutex = options->excluded_mutex;
|
||||||
scanner->ignore_io_errors = options->ignore_io_errors;
|
scanner->ignore_io_errors = options->ignore_io_errors;
|
||||||
|
scanner->ignore_missing_args = options->ignore_missing_args;
|
||||||
scanner->io_error = false;
|
scanner->io_error = false;
|
||||||
scanner->dirs_mode = options->dirs;
|
scanner->dirs_mode = options->dirs;
|
||||||
scanner->relative_mode = options->relative && options->file_list != NULL;
|
scanner->relative_mode = options->relative && options->file_list != NULL;
|
||||||
@@ -591,6 +592,16 @@ static File* dirs_file_for_entry(DirectoryScanner* scanner, const char* entry) {
|
|||||||
}
|
}
|
||||||
struct stat link_stats;
|
struct stat link_stats;
|
||||||
if (lstat(abs_path, &link_stats) != 0) {
|
if (lstat(abs_path, &link_stats) != 0) {
|
||||||
|
/* --ignore-missing-args (implied by --delete-missing-args): an explicitly
|
||||||
|
listed entry that does not exist under the source is a preflight-detected
|
||||||
|
missing argument and is skipped here, exactly as the recursive scan skips
|
||||||
|
nothing (missing entries never appear there). Without the flags it stays
|
||||||
|
a hard pre-transfer error. */
|
||||||
|
if (scanner->ignore_missing_args) {
|
||||||
|
log_info_message(LOG_INFO_MISC, "skipping missing --files-from entry '%s'", entry);
|
||||||
|
free(abs_path);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
log_message(LOG_LEVEL_ERROR, "--dirs listed entry is not present under the source: %s", entry);
|
log_message(LOG_LEVEL_ERROR, "--dirs listed entry is not present under the source: %s", entry);
|
||||||
free(abs_path);
|
free(abs_path);
|
||||||
scanner->failed = true;
|
scanner->failed = true;
|
||||||
|
|||||||
@@ -59,6 +59,11 @@ typedef struct {
|
|||||||
/* --ignore-errors: an unreadable directory during the scan is recorded as an
|
/* --ignore-errors: an unreadable directory during the scan is recorded as an
|
||||||
* I/O error and skipped instead of aborting the scan. Client-only. */
|
* I/O error and skipped instead of aborting the scan. Client-only. */
|
||||||
bool ignore_io_errors;
|
bool ignore_io_errors;
|
||||||
|
/* --ignore-missing-args (implied by --delete-missing-args): an explicitly
|
||||||
|
* --files-from-listed entry that does not exist under the source is skipped
|
||||||
|
* instead of failing (the --dirs generator is the only scanner path that
|
||||||
|
* observes a listed-but-missing entry). */
|
||||||
|
bool ignore_missing_args;
|
||||||
} ScannerOptions;
|
} ScannerOptions;
|
||||||
|
|
||||||
/* Internal per-scanner filter state. FilterNode chains represent the ordered
|
/* Internal per-scanner filter state. FilterNode chains represent the ordered
|
||||||
@@ -112,6 +117,9 @@ typedef struct {
|
|||||||
mtx_t* excluded_mutex;
|
mtx_t* excluded_mutex;
|
||||||
/* --ignore-errors: continue past unreadable directories (records io_error). */
|
/* --ignore-errors: continue past unreadable directories (records io_error). */
|
||||||
bool ignore_io_errors;
|
bool ignore_io_errors;
|
||||||
|
/* --ignore-missing-args: --dirs listed-but-missing entries are skipped, not
|
||||||
|
fatal (see ScannerOptions.ignore_missing_args). */
|
||||||
|
bool ignore_missing_args;
|
||||||
/* A directory could not be opened (I/O error, e.g. EACCES). With
|
/* A directory could not be opened (I/O error, e.g. EACCES). With
|
||||||
--ignore-errors the scan continues past it and the caller decides what to
|
--ignore-errors the scan continues past it and the caller decides what to
|
||||||
do; `failed` is reserved for fatal errors that always abort the scan. */
|
do; `failed` is reserved for fatal errors that always abort the scan. */
|
||||||
|
|||||||
+16
-2
@@ -46,6 +46,12 @@ void print_usage(void) {
|
|||||||
printf(" deletion\n");
|
printf(" deletion\n");
|
||||||
printf(" --force A file may replace a destination directory by removing\n");
|
printf(" --force A file may replace a destination directory by removing\n");
|
||||||
printf(" that (non-empty) directory first\n");
|
printf(" that (non-empty) directory first\n");
|
||||||
|
printf(" --ignore-missing-args A --files-from entry that does not exist under the\n");
|
||||||
|
printf(" source is silently skipped instead of failing the run\n");
|
||||||
|
printf(" --delete-missing-args Implies --ignore-missing-args; also deletes each missing\n");
|
||||||
|
printf(" entry's destination mirror receiver-side. Independent of\n");
|
||||||
|
printf(" --delete (it does not imply --delete; a non-empty directory\n");
|
||||||
|
printf(" mirror is removed only with --force or --delete)\n");
|
||||||
printf(" --prune-empty-dirs Do not transfer empty directory entries (--dirs mode);\n");
|
printf(" --prune-empty-dirs Do not transfer empty directory entries (--dirs mode);\n");
|
||||||
printf(" recursive transfers never send empty dirs. rsync's -m\n");
|
printf(" recursive transfers never send empty dirs. rsync's -m\n");
|
||||||
printf(" short form stays FastSync multithreading\n");
|
printf(" short form stays FastSync multithreading\n");
|
||||||
@@ -91,8 +97,12 @@ void print_usage(void) {
|
|||||||
printf(" into the destination instead of transferring its data\n");
|
printf(" into the destination instead of transferring its data\n");
|
||||||
printf(" --link-dest <dir> Like --copy-dest, but hard-links the unchanged file from DIR\n");
|
printf(" --link-dest <dir> Like --copy-dest, but hard-links the unchanged file from DIR\n");
|
||||||
printf(" into the destination (repeatable; earlier DIRs win)\n");
|
printf(" into the destination (repeatable; earlier DIRs win)\n");
|
||||||
printf(" --checksum-choice, --cc <alg> Checksum algorithm (not supported yet; xxHash64 is "
|
printf(" --checksum-choice, --cc <alg> Whole-file checksum algorithm for --incremental/\n");
|
||||||
"used)\n");
|
printf(" --checksum compares (xxh64/xxhash or md5; default xxh64 with\n");
|
||||||
|
printf(" seed 0). The seed comes from --checksum-seed\n");
|
||||||
|
printf(" --checksum-seed <num> Seed for the whole-file xxHash64 digest (and the delta\n");
|
||||||
|
printf(" block strong hash, low 32 bits); md5 ignores the seed. The\n");
|
||||||
|
printf(" digest algorithm and seed must match on sender and receiver\n");
|
||||||
printf(" --delta Delta transfer for changed files (requires --incremental)\n");
|
printf(" --delta Delta transfer for changed files (requires --incremental)\n");
|
||||||
printf(" -W, --whole-file Transfer changed files without delta processing\n");
|
printf(" -W, --whole-file Transfer changed files without delta processing\n");
|
||||||
printf(" -y, --fuzzy Use a similar-named file already in the destination\n");
|
printf(" -y, --fuzzy Use a similar-named file already in the destination\n");
|
||||||
@@ -159,6 +169,10 @@ void print_usage(void) {
|
|||||||
printf(" --copy-unsafe-links Only transform unsafe symlinks into referent files\n");
|
printf(" --copy-unsafe-links Only transform unsafe symlinks into referent files\n");
|
||||||
printf(" -S, --sparse Handle sparse files efficiently\n");
|
printf(" -S, --sparse Handle sparse files efficiently\n");
|
||||||
printf(" --inplace Update files in-place (no temp+rename)\n");
|
printf(" --inplace Update files in-place (no temp+rename)\n");
|
||||||
|
printf(" --append Resume a shorter destination by appending only its tail\n");
|
||||||
|
printf(" (prefix is not verified; requires --incremental)\n");
|
||||||
|
printf(" --append-verify Like --append, but verifies the retained prefix checksum\n");
|
||||||
|
printf(" before appending (falls back to a full transfer on mismatch)\n");
|
||||||
printf(" --fsync Fsync every written file before publication\n");
|
printf(" --fsync Fsync every written file before publication\n");
|
||||||
printf(" --compress-level <n> Compression level (default: 5)\n");
|
printf(" --compress-level <n> Compression level (default: 5)\n");
|
||||||
printf(" --zl <n> Alias for --compress-level\n");
|
printf(" --zl <n> Alias for --compress-level\n");
|
||||||
|
|||||||
@@ -191,7 +191,9 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver
|
|||||||
so the sender only starts streaming once the deletion committed (or
|
so the sender only starts streaming once the deletion committed (or
|
||||||
failed). This is the rsync delete-before/delete-during window: a
|
failed). This is the rsync delete-before/delete-during window: a
|
||||||
later transfer failure does not restore these deletions. */
|
later transfer failure does not restore these deletions. */
|
||||||
bool deletion_ok = config->use_delete ? manifest_delete_extras(config, manifest) : true;
|
bool deletion_ok = (config->use_delete || config->delete_missing_args)
|
||||||
|
? manifest_delete_all(config, manifest)
|
||||||
|
: true;
|
||||||
delete_manifest_free(manifest);
|
delete_manifest_free(manifest);
|
||||||
if (!deletion_ok) {
|
if (!deletion_ok) {
|
||||||
send_status(file_descriptor, STATUS_ERROR);
|
send_status(file_descriptor, STATUS_ERROR);
|
||||||
@@ -199,9 +201,10 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver
|
|||||||
}
|
}
|
||||||
if (!send_status(file_descriptor, STATUS_OK))
|
if (!send_status(file_descriptor, STATUS_OK))
|
||||||
goto fail;
|
goto fail;
|
||||||
} else if (config->use_delete) {
|
} else if (config->use_delete || config->delete_missing_args) {
|
||||||
/* Plain --delete / --delete-after / --delete-delay: hold the keep-set
|
/* Plain --delete / --delete-after / --delete-delay and the
|
||||||
and commit the deletion only after STATUS_FINISHED. */
|
--delete-missing-args exact-path deletions: hold the manifest and
|
||||||
|
commit it only after STATUS_FINISHED. */
|
||||||
if (deferred_manifest) {
|
if (deferred_manifest) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Received a second delete manifest");
|
log_message(LOG_LEVEL_ERROR, "Received a second delete manifest");
|
||||||
delete_manifest_free(deferred_manifest);
|
delete_manifest_free(deferred_manifest);
|
||||||
@@ -246,7 +249,7 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver
|
|||||||
*pending_manifest = deferred_manifest;
|
*pending_manifest = deferred_manifest;
|
||||||
deferred_manifest = NULL;
|
deferred_manifest = NULL;
|
||||||
} else {
|
} else {
|
||||||
bool deletion_ok = manifest_delete_extras(config, deferred_manifest);
|
bool deletion_ok = manifest_delete_all(config, deferred_manifest);
|
||||||
delete_manifest_free(deferred_manifest);
|
delete_manifest_free(deferred_manifest);
|
||||||
deferred_manifest = NULL;
|
deferred_manifest = NULL;
|
||||||
if (!deletion_ok) {
|
if (!deletion_ok) {
|
||||||
|
|||||||
+6
-1
@@ -180,6 +180,11 @@ void handler(int file_descriptor) {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
config->use_delete = config->use_delete && allow_delete;
|
config->use_delete = config->use_delete && allow_delete;
|
||||||
|
/* --delete-missing-args deletes destination mirrors receiver-side, so it is
|
||||||
|
deletion and stays gated by the same --allow-delete server policy. When
|
||||||
|
the server policy is off the flag is inert (the missing entries are still
|
||||||
|
skipped via its implied --ignore-missing-args, but nothing is deleted). */
|
||||||
|
config->delete_missing_args = config->delete_missing_args && allow_delete;
|
||||||
/* --mkpath: create the destination root (and its missing leading components)
|
/* --mkpath: create the destination root (and its missing leading components)
|
||||||
before anything else; without it the root must pre-exist. A failure here
|
before anything else; without it the root must pre-exist. A failure here
|
||||||
aborts the connection cleanly before any file data is exchanged. */
|
aborts the connection cleanly before any file data is exchanged. */
|
||||||
@@ -262,7 +267,7 @@ void handler(int file_descriptor) {
|
|||||||
known to have succeeded. Remove the extras before publishing a
|
known to have succeeded. Remove the extras before publishing a
|
||||||
--delay-updates run; the walker skips the staging directory. */
|
--delay-updates run; the walker skips the staging directory. */
|
||||||
if (context->deferred_manifest) {
|
if (context->deferred_manifest) {
|
||||||
if (!manifest_delete_extras(config, context->deferred_manifest)) {
|
if (!manifest_delete_all(config, context->deferred_manifest)) {
|
||||||
transfer_ok = false;
|
transfer_ok = false;
|
||||||
}
|
}
|
||||||
delete_manifest_free(context->deferred_manifest);
|
delete_manifest_free(context->deferred_manifest);
|
||||||
|
|||||||
@@ -0,0 +1,75 @@
|
|||||||
|
#include "checksum.h"
|
||||||
|
#include <openssl/evp.h>
|
||||||
|
#include <string.h>
|
||||||
|
#include <strings.h>
|
||||||
|
|
||||||
|
/* delta.c owns the single XXH_IMPLEMENTATION that provides the xxHash symbols
|
||||||
|
* for the whole binary; this TU only needs the declarations. */
|
||||||
|
#include <xxhash.h>
|
||||||
|
|
||||||
|
bool checksum_digest(ChecksumAlgo algo, uint64_t seed, const void* data, size_t size, uint8_t* out,
|
||||||
|
size_t out_capacity, size_t* out_len) {
|
||||||
|
if (!out || !out_len || out_capacity < CHECKSUM_MAX_DIGEST_LEN)
|
||||||
|
return false;
|
||||||
|
if (data == NULL && size != 0)
|
||||||
|
return false;
|
||||||
|
|
||||||
|
if (algo == CHECKSUM_ALGO_XXH64) {
|
||||||
|
uint64_t digest = XXH64(data, size, seed);
|
||||||
|
memcpy(out, &digest, sizeof(digest));
|
||||||
|
*out_len = sizeof(digest);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (algo == CHECKSUM_ALGO_MD5) {
|
||||||
|
/* md5 takes no seed; the caller's seed is deliberately ignored (documented
|
||||||
|
* in RSYNC_COMPAT.md). OpenSSL's one-shot EVP_Digest needs a non-NULL
|
||||||
|
* buffer even for an empty input, so map a NULL data + size==0 to an empty
|
||||||
|
* buffer. */
|
||||||
|
static const uint8_t empty = 0;
|
||||||
|
const void* input = data ? data : ∅
|
||||||
|
unsigned int digest_len = 0;
|
||||||
|
if (EVP_Digest(input, size, out, &digest_len, EVP_md5(), NULL) != 1)
|
||||||
|
return false;
|
||||||
|
if (digest_len > out_capacity)
|
||||||
|
return false;
|
||||||
|
*out_len = digest_len;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
int checksum_algo_from_name(const char* name) {
|
||||||
|
if (!name)
|
||||||
|
return -1;
|
||||||
|
if (strcasecmp(name, "xxh64") == 0 || strcasecmp(name, "xxhash") == 0)
|
||||||
|
return (int)CHECKSUM_ALGO_XXH64;
|
||||||
|
if (strcasecmp(name, "md5") == 0)
|
||||||
|
return (int)CHECKSUM_ALGO_MD5;
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
const char* checksum_algo_name(ChecksumAlgo algo) {
|
||||||
|
switch (algo) {
|
||||||
|
case CHECKSUM_ALGO_XXH64:
|
||||||
|
return "xxh64";
|
||||||
|
case CHECKSUM_ALGO_MD5:
|
||||||
|
return "md5";
|
||||||
|
}
|
||||||
|
return "<unknown>";
|
||||||
|
}
|
||||||
|
|
||||||
|
bool checksum_algo_valid(int algo) {
|
||||||
|
return algo == (int)CHECKSUM_ALGO_XXH64 || algo == (int)CHECKSUM_ALGO_MD5;
|
||||||
|
}
|
||||||
|
|
||||||
|
uint8_t checksum_digest_len(ChecksumAlgo algo) {
|
||||||
|
switch (algo) {
|
||||||
|
case CHECKSUM_ALGO_XXH64:
|
||||||
|
return 8;
|
||||||
|
case CHECKSUM_ALGO_MD5:
|
||||||
|
return 16;
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
@@ -0,0 +1,45 @@
|
|||||||
|
#ifndef CHECKSUM_H
|
||||||
|
#define CHECKSUM_H
|
||||||
|
|
||||||
|
#include <stdbool.h>
|
||||||
|
#include <stddef.h>
|
||||||
|
#include <stdint.h>
|
||||||
|
|
||||||
|
/* Whole-file content-digest algorithms selectable with --checksum-choice and
|
||||||
|
* seeded with --checksum-seed. The ids are the values actually placed on the
|
||||||
|
* wire (config frame), so they must be kept stable and validated on receive.
|
||||||
|
* CHECKSUM_ALGO_XXH64 == 0 is the default and is byte-for-byte what FastSync
|
||||||
|
* computed before these options existed (xxHash64 with seed 0). */
|
||||||
|
typedef enum { CHECKSUM_ALGO_XXH64 = 0, CHECKSUM_ALGO_MD5 = 1 } ChecksumAlgo;
|
||||||
|
|
||||||
|
/* md5 digest is 16 bytes, the longest supported. */
|
||||||
|
#define CHECKSUM_MAX_DIGEST_LEN 16
|
||||||
|
|
||||||
|
/* Compute the whole-file digest of the first `size` bytes of `data`.
|
||||||
|
*
|
||||||
|
* - CHECKSUM_ALGO_XXH64: xxHash64(data, size, seed) (full 64-bit seed).
|
||||||
|
* - CHECKSUM_ALGO_MD5: md5(data, size) via OpenSSL EVP.
|
||||||
|
* md5 has no seed, so `seed` is ignored (documented).
|
||||||
|
* - `size == 0` hashes the empty input (plus its seed), not a NULL input.
|
||||||
|
*
|
||||||
|
* Writes up to `out_capacity` bytes into `out`, storing the digest length in
|
||||||
|
* *out_len. Returns false on NULL out* or when the digest would not fit.
|
||||||
|
* Never writes more than CHECKSUM_MAX_DIGEST_LEN bytes. */
|
||||||
|
bool checksum_digest(ChecksumAlgo algo, uint64_t seed, const void* data, size_t size, uint8_t* out,
|
||||||
|
size_t out_capacity, size_t* out_len);
|
||||||
|
|
||||||
|
/* Resolve a --checksum-choice string (case-insensitive) to an algorithm id.
|
||||||
|
* Accepts "xxh64" and "xxhash" (both map to CHECKSUM_ALGO_XXH64, rsync's
|
||||||
|
* xxhash spelling) and "md5". Returns -1 for any unsupported name. */
|
||||||
|
int checksum_algo_from_name(const char* name);
|
||||||
|
|
||||||
|
/* Canonical name of an algorithm (used in CLI error messages). */
|
||||||
|
const char* checksum_algo_name(ChecksumAlgo algo);
|
||||||
|
|
||||||
|
/* True when `algo` is a supported id (used by config receive validation). */
|
||||||
|
bool checksum_algo_valid(int algo);
|
||||||
|
|
||||||
|
/* Digest length in bytes for an algorithm (xxx64 = 8, md5 = 16). */
|
||||||
|
uint8_t checksum_digest_len(ChecksumAlgo algo);
|
||||||
|
|
||||||
|
#endif /* CHECKSUM_H */
|
||||||
+51
-13
@@ -96,6 +96,8 @@ static void config_set_defaults(Config* config) {
|
|||||||
config->max_delete = -1;
|
config->max_delete = -1;
|
||||||
config->ignore_errors = false;
|
config->ignore_errors = false;
|
||||||
config->force_delete = false;
|
config->force_delete = false;
|
||||||
|
config->ignore_missing_args = false;
|
||||||
|
config->delete_missing_args = false;
|
||||||
config->filters = NULL;
|
config->filters = NULL;
|
||||||
config->files_from = NULL;
|
config->files_from = NULL;
|
||||||
config->files_from_set = NULL;
|
config->files_from_set = NULL;
|
||||||
@@ -127,6 +129,8 @@ static void config_set_defaults(Config* config) {
|
|||||||
config->daemon_config = NULL;
|
config->daemon_config = NULL;
|
||||||
config->server_mode = false;
|
config->server_mode = false;
|
||||||
config->checksum = false;
|
config->checksum = false;
|
||||||
|
config->checksum_algo = CHECKSUM_ALGO_XXH64;
|
||||||
|
config->checksum_seed = 0;
|
||||||
config->compress_choice = NULL;
|
config->compress_choice = NULL;
|
||||||
config->chmod_spec = NULL;
|
config->chmod_spec = NULL;
|
||||||
config->skip_compress_suffixes = NULL;
|
config->skip_compress_suffixes = NULL;
|
||||||
@@ -165,15 +169,20 @@ static bool validate_received_config(const Config* config) {
|
|||||||
valid_wire_bool(config->inplace) && valid_wire_bool(config->append) &&
|
valid_wire_bool(config->inplace) && valid_wire_bool(config->append) &&
|
||||||
valid_wire_bool(config->use_fsync) && valid_wire_bool(config->append_verify) &&
|
valid_wire_bool(config->use_fsync) && valid_wire_bool(config->append_verify) &&
|
||||||
valid_wire_bool(config->delete_excluded) && valid_wire_bool(config->force_delete) &&
|
valid_wire_bool(config->delete_excluded) && valid_wire_bool(config->force_delete) &&
|
||||||
valid_wire_bool(config->delete_after) && valid_wire_bool(config->delete_delay) &&
|
valid_wire_bool(config->delete_missing_args) && valid_wire_bool(config->delete_after) &&
|
||||||
valid_wire_bool(config->delete_during) && valid_wire_bool(config->relative) &&
|
valid_wire_bool(config->delete_delay) && valid_wire_bool(config->delete_during) &&
|
||||||
valid_wire_bool(config->prune_empty_dirs) && valid_wire_bool(config->delay_updates) &&
|
valid_wire_bool(config->relative) && valid_wire_bool(config->prune_empty_dirs) &&
|
||||||
valid_wire_bool(config->mkpath) && !(config->delay_updates && config->inplace) &&
|
valid_wire_bool(config->delay_updates) && valid_wire_bool(config->mkpath) &&
|
||||||
|
!(config->delay_updates && config->inplace) &&
|
||||||
!(config->delay_updates && delay_updates_staging_name_conflict(config->backup_dir)) &&
|
!(config->delay_updates && delay_updates_staging_name_conflict(config->backup_dir)) &&
|
||||||
valid_wire_bool(config->partial) && valid_wire_bool(config->delete_before) &&
|
valid_wire_bool(config->partial) && valid_wire_bool(config->delete_before) &&
|
||||||
valid_wire_bool(config->checksum) && valid_wire_bool(config->eight_bit_output) &&
|
valid_wire_bool(config->checksum) && valid_wire_bool(config->eight_bit_output) &&
|
||||||
config_has_valid_delete_timing(config) &&
|
checksum_algo_valid(config->checksum_algo) && config_has_valid_delete_timing(config) &&
|
||||||
!(config->skip_compress_set && config->use_chunk_serialization) &&
|
!(config->skip_compress_set && config->use_chunk_serialization) &&
|
||||||
|
/* --append / --append-verify tail resume needs the per-file check,
|
||||||
|
which chunk serialization -s disables: reject on the receiver too
|
||||||
|
so a -s sender cannot negotiate an inert append mode. */
|
||||||
|
!((config->append || config->append_verify) && config->use_chunk_serialization) &&
|
||||||
(!config->use_compression ||
|
(!config->use_compression ||
|
||||||
(config->compression_level >= 1 && config->compression_level <= 22)) &&
|
(config->compression_level >= 1 && config->compression_level <= 22)) &&
|
||||||
config->chunk_size > 0 && config->chunk_size <= MAX_CHUNK_SIZE &&
|
config->chunk_size > 0 && config->chunk_size <= MAX_CHUNK_SIZE &&
|
||||||
@@ -421,9 +430,10 @@ static bool send_selection_options(int fd, const Config* c) {
|
|||||||
send_int(fd, c->inplace) && send_int(fd, c->delay_updates) && send_int(fd, c->append) &&
|
send_int(fd, c->inplace) && send_int(fd, c->delay_updates) && send_int(fd, c->append) &&
|
||||||
send_int(fd, c->use_fsync) && send_int(fd, c->append_verify) &&
|
send_int(fd, c->use_fsync) && send_int(fd, c->append_verify) &&
|
||||||
send_int(fd, c->delete_excluded) && send_int(fd, c->force_delete) &&
|
send_int(fd, c->delete_excluded) && send_int(fd, c->force_delete) &&
|
||||||
send_int(fd, c->delete_after) && send_n_data(fd, &c->max_delete, sizeof(c->max_delete)) &&
|
send_int(fd, c->delete_missing_args) && send_int(fd, c->delete_after) &&
|
||||||
send_int(fd, c->relative) && send_int(fd, c->prune_empty_dirs) &&
|
send_n_data(fd, &c->max_delete, sizeof(c->max_delete)) && send_int(fd, c->relative) &&
|
||||||
send_int(fd, c->mkpath) && send_int(fd, c->delete_during) && send_int(fd, c->delete_delay);
|
send_int(fd, c->prune_empty_dirs) && send_int(fd, c->mkpath) &&
|
||||||
|
send_int(fd, c->delete_during) && send_int(fd, c->delete_delay);
|
||||||
}
|
}
|
||||||
|
|
||||||
static bool send_skip_compress_options(int fd, const Config* c) {
|
static bool send_skip_compress_options(int fd, const Config* c) {
|
||||||
@@ -462,6 +472,15 @@ static bool send_fuzzy_option(int fd, const Config* c) {
|
|||||||
return send_int(fd, c->fuzzy);
|
return send_int(fd, c->fuzzy);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* --checksum-choice/--cc + --checksum-seed. The algorithm id and seed travel
|
||||||
|
* with the config so the receiver hashes the on-disk old file with the same
|
||||||
|
* parameters the sender used for its digest (see checksum.h). Trailing fields
|
||||||
|
* on the config frame; protocol 2.10.0. */
|
||||||
|
static bool send_checksum_options(int fd, const Config* c) {
|
||||||
|
return send_int(fd, c->checksum_algo) &&
|
||||||
|
send_n_data(fd, &c->checksum_seed, sizeof(c->checksum_seed));
|
||||||
|
}
|
||||||
|
|
||||||
static bool receive_core_fields(int fd, Config* c) {
|
static bool receive_core_fields(int fd, Config* c) {
|
||||||
int value;
|
int value;
|
||||||
if (!receive_wire_bool(fd, &c->eight_bit_output))
|
if (!receive_wire_bool(fd, &c->eight_bit_output))
|
||||||
@@ -532,9 +551,18 @@ static bool receive_file_options(int fd, Config* c) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
static bool receive_selection_options(int fd, Config* c) {
|
static bool receive_selection_options(int fd, Config* c) {
|
||||||
bool* flags[] = {&c->ignore_existing, &c->existing, &c->update, &c->inplace,
|
bool* flags[] = {&c->ignore_existing,
|
||||||
&c->delay_updates, &c->append, &c->use_fsync, &c->append_verify,
|
&c->existing,
|
||||||
&c->delete_excluded, &c->force_delete, &c->delete_after};
|
&c->update,
|
||||||
|
&c->inplace,
|
||||||
|
&c->delay_updates,
|
||||||
|
&c->append,
|
||||||
|
&c->use_fsync,
|
||||||
|
&c->append_verify,
|
||||||
|
&c->delete_excluded,
|
||||||
|
&c->force_delete,
|
||||||
|
&c->delete_missing_args,
|
||||||
|
&c->delete_after};
|
||||||
for (size_t i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) {
|
for (size_t i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) {
|
||||||
if (!receive_wire_bool(fd, flags[i]))
|
if (!receive_wire_bool(fd, flags[i]))
|
||||||
return false;
|
return false;
|
||||||
@@ -637,13 +665,22 @@ static bool receive_fuzzy_option(int fd, Config* c) {
|
|||||||
return receive_wire_bool(fd, &c->fuzzy);
|
return receive_wire_bool(fd, &c->fuzzy);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static bool receive_checksum_options(int fd, Config* c) {
|
||||||
|
int algo;
|
||||||
|
if (!receive_int(fd, &algo) || !checksum_algo_valid(algo))
|
||||||
|
return false;
|
||||||
|
c->checksum_algo = algo;
|
||||||
|
return receive_n_data(fd, &c->checksum_seed, sizeof(c->checksum_seed));
|
||||||
|
}
|
||||||
|
|
||||||
bool config_send(int file_descriptor, const Config* config) {
|
bool config_send(int file_descriptor, const Config* config) {
|
||||||
protocol_session_set_max_alloc(NULL, config->max_alloc);
|
protocol_session_set_max_alloc(NULL, config->max_alloc);
|
||||||
if (!send_core_fields(file_descriptor, config) || !send_delta_fields(file_descriptor, config) ||
|
if (!send_core_fields(file_descriptor, config) || !send_delta_fields(file_descriptor, config) ||
|
||||||
!send_file_options(file_descriptor, config) ||
|
!send_file_options(file_descriptor, config) ||
|
||||||
!send_selection_options(file_descriptor, config) ||
|
!send_selection_options(file_descriptor, config) ||
|
||||||
!send_resume_options(file_descriptor, config) ||
|
!send_resume_options(file_descriptor, config) ||
|
||||||
!send_basis_options(file_descriptor, config) || !send_fuzzy_option(file_descriptor, config))
|
!send_basis_options(file_descriptor, config) || !send_fuzzy_option(file_descriptor, config) ||
|
||||||
|
!send_checksum_options(file_descriptor, config))
|
||||||
return false;
|
return false;
|
||||||
Status status;
|
Status status;
|
||||||
if (!receive_status(file_descriptor, &status))
|
if (!receive_status(file_descriptor, &status))
|
||||||
@@ -677,7 +714,8 @@ Config* config_receive(int file_descriptor) {
|
|||||||
!receive_selection_options(file_descriptor, config) ||
|
!receive_selection_options(file_descriptor, config) ||
|
||||||
!receive_resume_options(file_descriptor, config) ||
|
!receive_resume_options(file_descriptor, config) ||
|
||||||
!receive_basis_options(file_descriptor, config) ||
|
!receive_basis_options(file_descriptor, config) ||
|
||||||
!receive_fuzzy_option(file_descriptor, config))
|
!receive_fuzzy_option(file_descriptor, config) ||
|
||||||
|
!receive_checksum_options(file_descriptor, config))
|
||||||
goto error;
|
goto error;
|
||||||
if (config->compress_choice[0] != '\0' && strcmp(config->compress_choice, "zstd") != 0 &&
|
if (config->compress_choice[0] != '\0' && strcmp(config->compress_choice, "zstd") != 0 &&
|
||||||
strcmp(config->compress_choice, "none") != 0) {
|
strcmp(config->compress_choice, "none") != 0) {
|
||||||
|
|||||||
+27
-1
@@ -2,6 +2,7 @@
|
|||||||
#define CONFIG_H
|
#define CONFIG_H
|
||||||
|
|
||||||
#include "array_list.h"
|
#include "array_list.h"
|
||||||
|
#include "checksum.h"
|
||||||
#include <stdbool.h>
|
#include <stdbool.h>
|
||||||
#include <stdint.h>
|
#include <stdint.h>
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
@@ -145,6 +146,19 @@ typedef struct Config {
|
|||||||
* directory by removing that (possibly non-empty, symlink-safe) directory
|
* directory by removing that (possibly non-empty, symlink-safe) directory
|
||||||
* tree first, instead of failing the write. Crosses the wire. */
|
* tree first, instead of failing the write. Crosses the wire. */
|
||||||
bool force_delete;
|
bool force_delete;
|
||||||
|
/* --ignore-missing-args (client-only, never serialized): a --files-from
|
||||||
|
* entry that does not exist under the source is silently skipped instead of
|
||||||
|
* failing the run. Sender-side only: nothing is sent for it and it never
|
||||||
|
* enters the keep-set. Implied by --delete-missing-args. */
|
||||||
|
bool ignore_missing_args;
|
||||||
|
/* --delete-missing-args: implies --ignore-missing-args; additionally each
|
||||||
|
* missing entry's destination mirror (computed like a present entry's wire
|
||||||
|
* path) is deleted receiver-side. Crosses the wire and is gated by the
|
||||||
|
* server's --allow-delete policy like --delete. rsync-parity: independent
|
||||||
|
* of ordinary --delete processing (it does not imply --delete); a non-empty
|
||||||
|
* directory mirror is only removed with --force or --delete in effect, and
|
||||||
|
* the missing-args deletions are not counted toward --max-delete. */
|
||||||
|
bool delete_missing_args;
|
||||||
|
|
||||||
// Issue #129: Advanced file selection. These fields are CLIENT-ONLY: they are
|
// Issue #129: Advanced file selection. These fields are CLIENT-ONLY: they are
|
||||||
// never serialized to the wire (the receiver must not learn them).
|
// never serialized to the wire (the receiver must not learn them).
|
||||||
@@ -222,6 +236,18 @@ typedef struct Config {
|
|||||||
char* compress_choice;
|
char* compress_choice;
|
||||||
char* chmod_spec;
|
char* chmod_spec;
|
||||||
|
|
||||||
|
/* --checksum-choice / --cc and --checksum-seed. checksum_algo is the id of
|
||||||
|
* the whole-file content-digest algorithm used by the per-file --incremental
|
||||||
|
* handshake (sender computes it, receiver compares it to skip unchanged
|
||||||
|
* files) and by the basis-dir content verification. checksum_seed is passed
|
||||||
|
* to xxHash64 (and to the delta block strong hash, low 32 bits); md5 has no
|
||||||
|
* seed so it is ignored there. Both cross the wire: the receiver MUST hash
|
||||||
|
* the on-disk old file with the same algorithm and seed to reach a matching
|
||||||
|
* digest. Defaults (XXH64 / seed 0) reproduce the pre-existing behavior
|
||||||
|
* byte-for-byte. */
|
||||||
|
int checksum_algo; /* ChecksumAlgo, default CHECKSUM_ALGO_XXH64 */
|
||||||
|
uint64_t checksum_seed; /* default 0 */
|
||||||
|
|
||||||
char** skip_compress_suffixes;
|
char** skip_compress_suffixes;
|
||||||
int skip_compress_count;
|
int skip_compress_count;
|
||||||
bool skip_compress_set;
|
bool skip_compress_set;
|
||||||
@@ -231,7 +257,7 @@ typedef struct Config {
|
|||||||
DelayUpdatesContext* delay_context;
|
DelayUpdatesContext* delay_context;
|
||||||
} Config;
|
} Config;
|
||||||
|
|
||||||
#define PROTOCOL_VERSION "2.9.0"
|
#define PROTOCOL_VERSION "2.10.0"
|
||||||
#define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024)
|
#define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024)
|
||||||
/* Upper bound on total basis-dir entries (rsync caps --link-dest at 20). */
|
/* Upper bound on total basis-dir entries (rsync caps --link-dest at 20). */
|
||||||
#define MAX_BASIS_DIRS 64
|
#define MAX_BASIS_DIRS 64
|
||||||
|
|||||||
+19
-5
@@ -29,12 +29,21 @@ uint32_t delta_xxhash32(const void* data, uint32_t len) {
|
|||||||
return XXH32(data, len, 0);
|
return XXH32(data, len, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
uint32_t delta_xxhash32_seeded(const void* data, uint32_t len, uint32_t seed) {
|
||||||
|
return XXH32(data, len, seed);
|
||||||
|
}
|
||||||
|
|
||||||
uint64_t delta_xxhash64(const void* data, size_t len) {
|
uint64_t delta_xxhash64(const void* data, size_t len) {
|
||||||
return XXH64(data, len, 0);
|
return XXH64(data, len, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
DeltaSignature* delta_signature_create(const void* old_file_data, uint64_t old_file_size,
|
DeltaSignature* delta_signature_create(const void* old_file_data, uint64_t old_file_size,
|
||||||
uint32_t block_size) {
|
uint32_t block_size) {
|
||||||
|
return delta_signature_create_seeded(old_file_data, old_file_size, block_size, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
DeltaSignature* delta_signature_create_seeded(const void* old_file_data, uint64_t old_file_size,
|
||||||
|
uint32_t block_size, uint32_t seed) {
|
||||||
if (old_file_data == NULL || old_file_size == 0 || block_size == 0)
|
if (old_file_data == NULL || old_file_size == 0 || block_size == 0)
|
||||||
return NULL;
|
return NULL;
|
||||||
|
|
||||||
@@ -67,7 +76,7 @@ DeltaSignature* delta_signature_create(const void* old_file_data, uint64_t old_f
|
|||||||
uint32_t len =
|
uint32_t len =
|
||||||
(uint32_t)((old_file_size - offset < block_size) ? (old_file_size - offset) : block_size);
|
(uint32_t)((old_file_size - offset < block_size) ? (old_file_size - offset) : block_size);
|
||||||
sig->blocks[i].adler32 = delta_adler32(data + offset, len);
|
sig->blocks[i].adler32 = delta_adler32(data + offset, len);
|
||||||
sig->blocks[i].xxhash = delta_xxhash32(data + offset, len);
|
sig->blocks[i].xxhash = delta_xxhash32_seeded(data + offset, len, seed);
|
||||||
}
|
}
|
||||||
|
|
||||||
return sig;
|
return sig;
|
||||||
@@ -297,7 +306,7 @@ static uint32_t* delta_build_index(const DeltaSignature* sig, uint32_t bucket_co
|
|||||||
* O(1) per window); otherwise an exact linear scan is used. */
|
* O(1) per window); otherwise an exact linear scan is used. */
|
||||||
static uint32_t delta_find_match(const uint8_t* window, uint32_t window_len, uint32_t adler,
|
static uint32_t delta_find_match(const uint8_t* window, uint32_t window_len, uint32_t adler,
|
||||||
bool full_window, const DeltaSignature* sig, const uint32_t* heads,
|
bool full_window, const DeltaSignature* sig, const uint32_t* heads,
|
||||||
const uint32_t* next, uint32_t mask) {
|
const uint32_t* next, uint32_t mask, uint32_t seed) {
|
||||||
if (!full_window || sig->block_count == 0)
|
if (!full_window || sig->block_count == 0)
|
||||||
return DELTA_NO_BLOCK;
|
return DELTA_NO_BLOCK;
|
||||||
|
|
||||||
@@ -309,7 +318,7 @@ static uint32_t delta_find_match(const uint8_t* window, uint32_t window_len, uin
|
|||||||
if (sig->blocks[j].adler32 != adler)
|
if (sig->blocks[j].adler32 != adler)
|
||||||
continue;
|
continue;
|
||||||
if (!have_xxh) {
|
if (!have_xxh) {
|
||||||
window_xxh = delta_xxhash32(window, window_len);
|
window_xxh = delta_xxhash32_seeded(window, window_len, seed);
|
||||||
have_xxh = true;
|
have_xxh = true;
|
||||||
}
|
}
|
||||||
if (window_xxh == sig->blocks[j].xxhash)
|
if (window_xxh == sig->blocks[j].xxhash)
|
||||||
@@ -321,7 +330,7 @@ static uint32_t delta_find_match(const uint8_t* window, uint32_t window_len, uin
|
|||||||
/* Fallback used when the index could not be allocated. */
|
/* Fallback used when the index could not be allocated. */
|
||||||
for (uint32_t j = 0; j < sig->block_count; j++) {
|
for (uint32_t j = 0; j < sig->block_count; j++) {
|
||||||
if (sig->blocks[j].adler32 == adler) {
|
if (sig->blocks[j].adler32 == adler) {
|
||||||
uint32_t window_xxh = delta_xxhash32(window, window_len);
|
uint32_t window_xxh = delta_xxhash32_seeded(window, window_len, seed);
|
||||||
if (window_xxh == sig->blocks[j].xxhash)
|
if (window_xxh == sig->blocks[j].xxhash)
|
||||||
return j;
|
return j;
|
||||||
}
|
}
|
||||||
@@ -331,6 +340,11 @@ static uint32_t delta_find_match(const uint8_t* window, uint32_t window_len, uin
|
|||||||
|
|
||||||
Delta* delta_compute(const void* new_file_data, uint64_t new_file_size, const DeltaSignature* sig,
|
Delta* delta_compute(const void* new_file_data, uint64_t new_file_size, const DeltaSignature* sig,
|
||||||
uint32_t block_size) {
|
uint32_t block_size) {
|
||||||
|
return delta_compute_seeded(new_file_data, new_file_size, sig, block_size, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
Delta* delta_compute_seeded(const void* new_file_data, uint64_t new_file_size,
|
||||||
|
const DeltaSignature* sig, uint32_t block_size, uint32_t seed) {
|
||||||
if (!new_file_data || !sig || !sig->blocks || new_file_size == 0 || block_size == 0 ||
|
if (!new_file_data || !sig || !sig->blocks || new_file_size == 0 || block_size == 0 ||
|
||||||
block_size > DELTA_BLOCK_SIZE_MAX || sig->block_size != block_size)
|
block_size > DELTA_BLOCK_SIZE_MAX || sig->block_size != block_size)
|
||||||
return NULL;
|
return NULL;
|
||||||
@@ -397,7 +411,7 @@ Delta* delta_compute(const void* new_file_data, uint64_t new_file_size, const De
|
|||||||
|
|
||||||
bool matched = false;
|
bool matched = false;
|
||||||
uint32_t match_block = delta_find_match(new_data + i, window_len, adler, full_window, sig,
|
uint32_t match_block = delta_find_match(new_data + i, window_len, adler, full_window, sig,
|
||||||
index, chain_next, mask);
|
index, chain_next, mask, seed);
|
||||||
if (match_block != DELTA_NO_BLOCK) {
|
if (match_block != DELTA_NO_BLOCK) {
|
||||||
if (has_literal) {
|
if (has_literal) {
|
||||||
if (!flush_literal(&instrs, &capacity, &count, new_data, literal_start, i)) {
|
if (!flush_literal(&instrs, &capacity, &count, new_data, literal_start, i)) {
|
||||||
|
|||||||
@@ -56,12 +56,22 @@ typedef struct {
|
|||||||
|
|
||||||
DeltaSignature* delta_signature_create(const void* old_file_data, uint64_t old_file_size,
|
DeltaSignature* delta_signature_create(const void* old_file_data, uint64_t old_file_size,
|
||||||
uint32_t block_size);
|
uint32_t block_size);
|
||||||
|
/* Seeded equivalent of delta_signature_create: the per-block strong (xxHash32)
|
||||||
|
* checksum uses `seed` (the low 32 bits of --checksum-seed). Passing seed 0 is
|
||||||
|
* identical to the unseeded function. */
|
||||||
|
DeltaSignature* delta_signature_create_seeded(const void* old_file_data, uint64_t old_file_size,
|
||||||
|
uint32_t block_size, uint32_t seed);
|
||||||
Data* delta_signature_serialize(const DeltaSignature* sig);
|
Data* delta_signature_serialize(const DeltaSignature* sig);
|
||||||
DeltaSignature* delta_signature_deserialize(const Data* data);
|
DeltaSignature* delta_signature_deserialize(const Data* data);
|
||||||
void delta_signature_destroy(DeltaSignature* sig);
|
void delta_signature_destroy(DeltaSignature* sig);
|
||||||
|
|
||||||
Delta* delta_compute(const void* new_file_data, uint64_t new_file_size, const DeltaSignature* sig,
|
Delta* delta_compute(const void* new_file_data, uint64_t new_file_size, const DeltaSignature* sig,
|
||||||
uint32_t block_size);
|
uint32_t block_size);
|
||||||
|
/* Seeded equivalent of delta_compute: the per-window strong (xxHash32) check
|
||||||
|
* uses `seed` (the low 32 bits of --checksum-seed). The receiver's signature
|
||||||
|
* must have been built with the same seed for matching. */
|
||||||
|
Delta* delta_compute_seeded(const void* new_file_data, uint64_t new_file_size,
|
||||||
|
const DeltaSignature* sig, uint32_t block_size, uint32_t seed);
|
||||||
Data* delta_serialize(const Delta* delta);
|
Data* delta_serialize(const Delta* delta);
|
||||||
Delta* delta_deserialize(const Data* data);
|
Delta* delta_deserialize(const Data* data);
|
||||||
void* delta_apply(const void* old_data, uint64_t old_size, const Delta* delta, uint32_t block_size);
|
void* delta_apply(const void* old_data, uint64_t old_size, const Delta* delta, uint32_t block_size);
|
||||||
@@ -72,6 +82,7 @@ bool delta_is_worthwhile(const Delta* delta, uint64_t new_file_size);
|
|||||||
|
|
||||||
uint32_t delta_adler32(const void* data, uint32_t len);
|
uint32_t delta_adler32(const void* data, uint32_t len);
|
||||||
uint32_t delta_xxhash32(const void* data, uint32_t len);
|
uint32_t delta_xxhash32(const void* data, uint32_t len);
|
||||||
|
uint32_t delta_xxhash32_seeded(const void* data, uint32_t len, uint32_t seed);
|
||||||
uint64_t delta_xxhash64(const void* data, size_t len);
|
uint64_t delta_xxhash64(const void* data, size_t len);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
+6
-6
@@ -43,17 +43,17 @@ static unsigned long long next_temp_sequence(void) {
|
|||||||
return atomic_fetch_add_explicit(&sequence, 1, memory_order_relaxed);
|
return atomic_fetch_add_explicit(&sequence, 1, memory_order_relaxed);
|
||||||
}
|
}
|
||||||
|
|
||||||
bool file_checksum(File* file, uint64_t* checksum) {
|
bool file_checksum(File* file, ChecksumAlgo algo, uint64_t seed, uint8_t* out, size_t out_capacity,
|
||||||
if (!file || !checksum || !file->data)
|
size_t* out_len) {
|
||||||
|
if (!file || !out || !out_len || !file->data)
|
||||||
return false;
|
return false;
|
||||||
if (file->data->size == 0) {
|
if (file->data->size == 0) {
|
||||||
*checksum = delta_xxhash64("", 0);
|
return checksum_digest(algo, seed, "", 0, out, out_capacity, out_len);
|
||||||
return true;
|
|
||||||
}
|
}
|
||||||
if (!file->data->data && !file_load_data(file))
|
if (!file->data->data && !file_load_data(file))
|
||||||
return false;
|
return false;
|
||||||
*checksum = delta_xxhash64(file->data->data, file->data->size);
|
return checksum_digest(algo, seed, file->data->data, file->data->size, out, out_capacity,
|
||||||
return true;
|
out_len);
|
||||||
}
|
}
|
||||||
|
|
||||||
File* file_create(const char* path) {
|
File* file_create(const char* path) {
|
||||||
|
|||||||
+7
-1
@@ -4,6 +4,7 @@
|
|||||||
#include "file_send.h"
|
#include "file_send.h"
|
||||||
#include "file_receive.h"
|
#include "file_receive.h"
|
||||||
#include "file_types.h"
|
#include "file_types.h"
|
||||||
|
#include "checksum.h"
|
||||||
#include <stdbool.h>
|
#include <stdbool.h>
|
||||||
#include <stdint.h>
|
#include <stdint.h>
|
||||||
#include <sys/stat.h>
|
#include <sys/stat.h>
|
||||||
@@ -14,7 +15,12 @@
|
|||||||
File* file_create(const char* path);
|
File* file_create(const char* path);
|
||||||
void file_destroy(void* item);
|
void file_destroy(void* item);
|
||||||
bool file_load_data(File* file);
|
bool file_load_data(File* file);
|
||||||
bool file_checksum(File* file, uint64_t* checksum);
|
/* Compute the whole-file content digest of `file` with the negotiated
|
||||||
|
* --checksum-choice algorithm and --checksum-seed. Writes the digest into
|
||||||
|
* `out` (capacity `out_capacity`) and its length into `*out_len`. Returns
|
||||||
|
* false on read/allocation failure or when the digest would not fit. */
|
||||||
|
bool file_checksum(File* file, ChecksumAlgo algo, uint64_t seed, uint8_t* out, size_t out_capacity,
|
||||||
|
size_t* out_len);
|
||||||
size_t file_content_to_buffer(File* file);
|
size_t file_content_to_buffer(File* file);
|
||||||
FileMetadata* file_metadata_create(const struct stat* stats);
|
FileMetadata* file_metadata_create(const struct stat* stats);
|
||||||
void file_metadata_destroy(void* metadata);
|
void file_metadata_destroy(void* metadata);
|
||||||
|
|||||||
+462
-63
@@ -355,7 +355,8 @@ static File* receive_delta_file(int fd, const Config* config, const char* check_
|
|||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
DeltaSignature* sig = delta_signature_create(old_data, old_size, config->delta_block_size);
|
DeltaSignature* sig = delta_signature_create_seeded(old_data, old_size, config->delta_block_size,
|
||||||
|
(uint32_t)config->checksum_seed);
|
||||||
if (!sig) {
|
if (!sig) {
|
||||||
free(old_data);
|
free(old_data);
|
||||||
*failed = true;
|
*failed = true;
|
||||||
@@ -630,8 +631,8 @@ static bool basis_quick_matches(const Config* config, const struct stat* st, tim
|
|||||||
so the caller can materialize the file without re-reading it. */
|
so the caller can materialize the file without re-reading it. */
|
||||||
static bool basis_match_find(const Config* config, const char* check_path,
|
static bool basis_match_find(const Config* config, const char* check_path,
|
||||||
unsigned long long check_size, time_t check_mtime,
|
unsigned long long check_size, time_t check_mtime,
|
||||||
long check_mtime_nsec, uint64_t check_checksum, bool load_content,
|
long check_mtime_nsec, const uint8_t* check_digest,
|
||||||
BasisMatch* out) {
|
size_t check_digest_len, bool load_content, BasisMatch* out) {
|
||||||
memset(out, 0, sizeof(*out));
|
memset(out, 0, sizeof(*out));
|
||||||
if (!config || !config_has_basis(config) || config->ignore_times)
|
if (!config || !config_has_basis(config) || config->ignore_times)
|
||||||
return false;
|
return false;
|
||||||
@@ -651,10 +652,13 @@ static bool basis_match_find(const Config* config, const char* check_path,
|
|||||||
if (basis_quick_matches(config, &st, check_mtime, check_mtime_nsec)) {
|
if (basis_quick_matches(config, &st, check_mtime, check_mtime_nsec)) {
|
||||||
Data* content = basis_read_content(fd, check_size);
|
Data* content = basis_read_content(fd, check_size);
|
||||||
if (content) {
|
if (content) {
|
||||||
uint64_t basis_hash = check_size == 0 ? delta_xxhash64("", 0)
|
uint8_t basis_digest[CHECKSUM_MAX_DIGEST_LEN];
|
||||||
: content->data ? delta_xxhash64(content->data, content->size)
|
size_t basis_len = 0;
|
||||||
: 0;
|
bool hashed = checksum_digest((ChecksumAlgo)config->checksum_algo, config->checksum_seed,
|
||||||
if (basis_hash == check_checksum) {
|
content->data, content->size, basis_digest,
|
||||||
|
sizeof(basis_digest), &basis_len);
|
||||||
|
if (hashed && basis_len == check_digest_len && check_digest_len > 0 &&
|
||||||
|
memcmp(basis_digest, check_digest, check_digest_len) == 0) {
|
||||||
out->hit = true;
|
out->hit = true;
|
||||||
out->type = entry->type;
|
out->type = entry->type;
|
||||||
out->basis_path = candidate;
|
out->basis_path = candidate;
|
||||||
@@ -983,6 +987,50 @@ static void* fuzzy_basis_find_and_load(const Config* config, const char* check_p
|
|||||||
return basis;
|
return basis;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Read the remainder of a full-file transfer after the receiver has already
|
||||||
|
* sent STATUS_NEXT: receive the metadata frame (when enabled) followed by the
|
||||||
|
* data frame, and return an owned File. Shared by the plain full-transfer path
|
||||||
|
* and the --append-verify prefix-mismatch fallback (a clean full transfer
|
||||||
|
* instead of a corrupt prefix+tail blend). */
|
||||||
|
static File* receive_full_file(int fd, const Config* config, const char* path) {
|
||||||
|
File* file = file_create(path);
|
||||||
|
if (!file)
|
||||||
|
return NULL;
|
||||||
|
if (config->use_metadata) {
|
||||||
|
int meta_ok = 1;
|
||||||
|
file->metadata = metadata_receive(fd, &meta_ok);
|
||||||
|
if (!meta_ok) {
|
||||||
|
file_destroy(file);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Data* file_data = receive_data_limited(fd, MAX_RECEIVE_WHOLE_FILE_SIZE);
|
||||||
|
if (file_data == NULL) {
|
||||||
|
file_destroy(file);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
if (config->use_compression &&
|
||||||
|
!compression_should_skip_with_suffixes(file->path, config->skip_compress_suffixes,
|
||||||
|
config->skip_compress_set ? config->skip_compress_count
|
||||||
|
: -1)) {
|
||||||
|
Data* uncompressed = data_decompress_limited(file_data, MAX_RECEIVE_WHOLE_FILE_SIZE);
|
||||||
|
data_destroy(file_data);
|
||||||
|
if (uncompressed == NULL) {
|
||||||
|
file_destroy(file);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
if (uncompressed->size > MAX_FILE_DATA_SIZE) {
|
||||||
|
data_destroy(uncompressed);
|
||||||
|
file_destroy(file);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
file_data = uncompressed;
|
||||||
|
}
|
||||||
|
data_destroy(file->data);
|
||||||
|
file->data = file_data;
|
||||||
|
return file;
|
||||||
|
}
|
||||||
|
|
||||||
File* receive_incremental_check(int fd, const Config* config, bool* skipped) {
|
File* receive_incremental_check(int fd, const Config* config, bool* skipped) {
|
||||||
if (!config || !skipped) {
|
if (!config || !skipped) {
|
||||||
send_status(fd, STATUS_ERROR);
|
send_status(fd, STATUS_ERROR);
|
||||||
@@ -997,7 +1045,8 @@ File* receive_incremental_check(int fd, const Config* config, bool* skipped) {
|
|||||||
unsigned long long check_size;
|
unsigned long long check_size;
|
||||||
long long check_mtime;
|
long long check_mtime;
|
||||||
long long check_mtime_nsec;
|
long long check_mtime_nsec;
|
||||||
uint64_t check_checksum = 0;
|
uint8_t check_digest[CHECKSUM_MAX_DIGEST_LEN];
|
||||||
|
size_t check_digest_len = 0;
|
||||||
if (!receive_n_data(fd, &check_size, sizeof(check_size)) ||
|
if (!receive_n_data(fd, &check_size, sizeof(check_size)) ||
|
||||||
!receive_n_data(fd, &check_mtime, sizeof(check_mtime))) {
|
!receive_n_data(fd, &check_mtime, sizeof(check_mtime))) {
|
||||||
free(check_path);
|
free(check_path);
|
||||||
@@ -1009,11 +1058,21 @@ File* receive_incremental_check(int fd, const Config* config, bool* skipped) {
|
|||||||
send_status(fd, STATUS_ERROR);
|
send_status(fd, STATUS_ERROR);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
if ((config->checksum || config_has_basis(config)) &&
|
if ((config->checksum || config_has_basis(config))) {
|
||||||
!receive_n_data(fd, &check_checksum, sizeof(check_checksum))) {
|
uint8_t wire_len;
|
||||||
|
if (!receive_n_data(fd, &wire_len, sizeof(wire_len)) || wire_len == 0 ||
|
||||||
|
wire_len > CHECKSUM_MAX_DIGEST_LEN ||
|
||||||
|
wire_len != checksum_digest_len((ChecksumAlgo)config->checksum_algo)) {
|
||||||
|
free(check_path);
|
||||||
|
send_status(fd, STATUS_ERROR);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
check_digest_len = wire_len;
|
||||||
|
if (!receive_n_data(fd, check_digest, check_digest_len)) {
|
||||||
free(check_path);
|
free(check_path);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (check_size > MAX_RECEIVE_WHOLE_FILE_SIZE) {
|
if (check_size > MAX_RECEIVE_WHOLE_FILE_SIZE) {
|
||||||
free(check_path);
|
free(check_path);
|
||||||
@@ -1098,10 +1157,13 @@ File* receive_incremental_check(int fd, const Config* config, bool* skipped) {
|
|||||||
skipped and the transfer proceeds with the full new contents. */
|
skipped and the transfer proceeds with the full new contents. */
|
||||||
bool match = false;
|
bool match = false;
|
||||||
if (checksum_needs_read) {
|
if (checksum_needs_read) {
|
||||||
if (old_size == 0)
|
uint8_t old_digest[CHECKSUM_MAX_DIGEST_LEN];
|
||||||
match = delta_xxhash64("", 0) == check_checksum;
|
size_t old_len = 0;
|
||||||
else
|
bool hashed = checksum_digest((ChecksumAlgo)config->checksum_algo, config->checksum_seed,
|
||||||
match = old_data != NULL && delta_xxhash64(old_data, (size_t)old_size) == check_checksum;
|
old_size == 0 ? "" : old_data, (size_t)old_size, old_digest,
|
||||||
|
sizeof(old_digest), &old_len);
|
||||||
|
match = hashed && old_len == check_digest_len && check_digest_len > 0 &&
|
||||||
|
memcmp(old_digest, check_digest, check_digest_len) == 0;
|
||||||
} else if (size_equal && !config->ignore_times) {
|
} else if (size_equal && !config->ignore_times) {
|
||||||
match = config->size_only || match_by_metadata;
|
match = config->size_only || match_by_metadata;
|
||||||
}
|
}
|
||||||
@@ -1125,7 +1187,7 @@ File* receive_incremental_check(int fd, const Config* config, bool* skipped) {
|
|||||||
if (config_has_basis(config)) {
|
if (config_has_basis(config)) {
|
||||||
BasisMatch basis;
|
BasisMatch basis;
|
||||||
basis_match_find(config, check_path, check_size, (time_t)check_mtime, (long)check_mtime_nsec,
|
basis_match_find(config, check_path, check_size, (time_t)check_mtime, (long)check_mtime_nsec,
|
||||||
check_checksum, true, &basis);
|
check_digest, check_digest_len, true, &basis);
|
||||||
if (basis.hit) {
|
if (basis.hit) {
|
||||||
if (basis.type == BASIS_DEST_COMPARE) {
|
if (basis.type == BASIS_DEST_COMPARE) {
|
||||||
/* compare-dest never copies: an exact match only suppresses the data
|
/* compare-dest never copies: an exact match only suppresses the data
|
||||||
@@ -1193,6 +1255,224 @@ File* receive_incremental_check(int fd, const Config* config, bool* skipped) {
|
|||||||
basis_match_free(&basis);
|
basis_match_free(&basis);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* ---- --append / --append-verify tail resume ----
|
||||||
|
* When the existing destination file is SHORTER than the source, an append
|
||||||
|
* mode resumes it by negotiating a resume offset (the prefix length already
|
||||||
|
* present) from the receiver and transferring ONLY the tail. The receiver
|
||||||
|
* then reconstructs the full file (prefix + tail) and installs it through the
|
||||||
|
* normal atomic store path, so the result is byte-identical to the source.
|
||||||
|
* This takes precedence over block delta (a growing file is cheapest as a
|
||||||
|
* pure tail), and falls through to delta/full only when no shorter old file
|
||||||
|
* makes a resume possible. */
|
||||||
|
bool append_resume = (config->append || config->append_verify) && has_old_file &&
|
||||||
|
append_resume_eligible(old_size, check_size);
|
||||||
|
if (append_resume) {
|
||||||
|
/* Ensure the retained prefix (== the whole, shorter destination file) is
|
||||||
|
in memory; it is needed both to rebuild the full file and, for
|
||||||
|
--append-verify, to checksum it. A load failure is not fatal: the
|
||||||
|
resume is simply not possible and we fall through to the other paths. */
|
||||||
|
if (old_data == NULL && old_size > 0 && old_size <= MAX_RECEIVE_WHOLE_FILE_SIZE &&
|
||||||
|
old_size <= SIZE_MAX) {
|
||||||
|
old_data = protocol_alloc((size_t)old_size);
|
||||||
|
if (old_data) {
|
||||||
|
size_t got = 0;
|
||||||
|
while (got < (size_t)old_size) {
|
||||||
|
ssize_t n = read(old_fd, (char*)old_data + got, (size_t)old_size - got);
|
||||||
|
if (n <= 0) {
|
||||||
|
free(old_data);
|
||||||
|
old_data = NULL;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
got += (size_t)n;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (old_data != NULL || old_size == 0) {
|
||||||
|
if (!send_status(fd, STATUS_APPEND) || !send_n_data(fd, &old_size, sizeof(old_size))) {
|
||||||
|
close(old_fd);
|
||||||
|
free(full_path);
|
||||||
|
free(check_path);
|
||||||
|
free(old_data);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
bool verify = config->append_verify;
|
||||||
|
bool full_fallback = false;
|
||||||
|
if (verify) {
|
||||||
|
Status sig_status;
|
||||||
|
if (!receive_status(fd, &sig_status)) {
|
||||||
|
close(old_fd);
|
||||||
|
free(full_path);
|
||||||
|
free(check_path);
|
||||||
|
free(old_data);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
if (sig_status != STATUS_APPEND_SIG) {
|
||||||
|
send_status(fd, STATUS_ERROR);
|
||||||
|
close(old_fd);
|
||||||
|
free(full_path);
|
||||||
|
free(check_path);
|
||||||
|
free(old_data);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
uint64_t src_prefix_hash;
|
||||||
|
if (!receive_n_data(fd, &src_prefix_hash, sizeof(src_prefix_hash))) {
|
||||||
|
close(old_fd);
|
||||||
|
free(full_path);
|
||||||
|
free(check_path);
|
||||||
|
free(old_data);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
/* Compare the retained prefix against the source prefix. A mismatch
|
||||||
|
must never be silently appended to: fall back to a full transfer so
|
||||||
|
the result is a byte-identical source copy. */
|
||||||
|
uint64_t dst_prefix_hash =
|
||||||
|
old_size == 0 ? delta_xxhash64("", 0) : delta_xxhash64(old_data, (size_t)old_size);
|
||||||
|
if (dst_prefix_hash == src_prefix_hash) {
|
||||||
|
if (!send_status(fd, STATUS_APPEND_OK)) {
|
||||||
|
close(old_fd);
|
||||||
|
free(full_path);
|
||||||
|
free(check_path);
|
||||||
|
free(old_data);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if (!send_status(fd, STATUS_NEXT)) {
|
||||||
|
close(old_fd);
|
||||||
|
free(full_path);
|
||||||
|
free(check_path);
|
||||||
|
free(old_data);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
full_fallback = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (full_fallback) {
|
||||||
|
/* Retained prefix differed: receive the sender's full transfer. */
|
||||||
|
free(old_data);
|
||||||
|
old_data = NULL;
|
||||||
|
close(old_fd);
|
||||||
|
File* file = receive_full_file(fd, config, check_path);
|
||||||
|
free(check_path);
|
||||||
|
free(full_path);
|
||||||
|
return file;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Receive the tail (STATUS_APPEND_DATA + metadata + tail bytes). */
|
||||||
|
Status tail_status;
|
||||||
|
if (!receive_status(fd, &tail_status)) {
|
||||||
|
close(old_fd);
|
||||||
|
free(full_path);
|
||||||
|
free(check_path);
|
||||||
|
free(old_data);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
if (tail_status != STATUS_APPEND_DATA) {
|
||||||
|
send_status(fd, STATUS_ERROR);
|
||||||
|
close(old_fd);
|
||||||
|
free(full_path);
|
||||||
|
free(check_path);
|
||||||
|
free(old_data);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
FileMetadata* meta = NULL;
|
||||||
|
if (config->use_metadata) {
|
||||||
|
int meta_ok = 1;
|
||||||
|
meta = metadata_receive(fd, &meta_ok);
|
||||||
|
if (!meta_ok) {
|
||||||
|
close(old_fd);
|
||||||
|
free(full_path);
|
||||||
|
free(check_path);
|
||||||
|
free(old_data);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Data* tail = receive_data_limited(fd, MAX_RECEIVE_WHOLE_FILE_SIZE);
|
||||||
|
if (tail == NULL) {
|
||||||
|
close(old_fd);
|
||||||
|
free(full_path);
|
||||||
|
free(check_path);
|
||||||
|
free(old_data);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
if (config->use_compression &&
|
||||||
|
!compression_should_skip_with_suffixes(
|
||||||
|
check_path, config->skip_compress_suffixes,
|
||||||
|
config->skip_compress_set ? config->skip_compress_count : -1)) {
|
||||||
|
Data* uncompressed = data_decompress_limited(tail, MAX_RECEIVE_WHOLE_FILE_SIZE);
|
||||||
|
data_destroy(tail);
|
||||||
|
if (uncompressed == NULL) {
|
||||||
|
close(old_fd);
|
||||||
|
free(full_path);
|
||||||
|
free(check_path);
|
||||||
|
free(old_data);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
if (uncompressed->size > MAX_FILE_DATA_SIZE) {
|
||||||
|
data_destroy(uncompressed);
|
||||||
|
close(old_fd);
|
||||||
|
free(full_path);
|
||||||
|
free(check_path);
|
||||||
|
free(old_data);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
tail = uncompressed;
|
||||||
|
}
|
||||||
|
/* The tail must complete the file exactly; anything else is a protocol
|
||||||
|
violation (never a truncated or overrun file). */
|
||||||
|
unsigned long long expected_tail;
|
||||||
|
if (!append_tail_length(old_size, check_size, &expected_tail) ||
|
||||||
|
tail->size != (size_t)expected_tail) {
|
||||||
|
send_status(fd, STATUS_ERROR);
|
||||||
|
data_destroy(tail);
|
||||||
|
close(old_fd);
|
||||||
|
free(full_path);
|
||||||
|
free(check_path);
|
||||||
|
free(old_data);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
size_t full_size = (size_t)check_size;
|
||||||
|
void* full = protocol_alloc(full_size ? full_size : 1);
|
||||||
|
if (!full) {
|
||||||
|
data_destroy(tail);
|
||||||
|
close(old_fd);
|
||||||
|
free(full_path);
|
||||||
|
free(check_path);
|
||||||
|
free(old_data);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
if (old_size > 0 && old_data)
|
||||||
|
memcpy(full, old_data, (size_t)old_size);
|
||||||
|
if (tail->size > 0)
|
||||||
|
memcpy((char*)full + old_size, tail->data, tail->size);
|
||||||
|
data_destroy(tail);
|
||||||
|
free(old_data);
|
||||||
|
old_data = NULL;
|
||||||
|
|
||||||
|
File* file = file_create(check_path);
|
||||||
|
if (!file) {
|
||||||
|
free(full);
|
||||||
|
close(old_fd);
|
||||||
|
free(full_path);
|
||||||
|
free(check_path);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
file->metadata = meta;
|
||||||
|
file->data = data_create(full, full_size);
|
||||||
|
if (!file->data) { /* data_create already freed full on failure */
|
||||||
|
file_destroy(file);
|
||||||
|
close(old_fd);
|
||||||
|
free(full_path);
|
||||||
|
free(check_path);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
close(old_fd);
|
||||||
|
free(full_path);
|
||||||
|
free(check_path);
|
||||||
|
return file;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (try_delta && old_data != NULL) {
|
if (try_delta && old_data != NULL) {
|
||||||
bool delta_failed = false;
|
bool delta_failed = false;
|
||||||
File* delta_file =
|
File* delta_file =
|
||||||
@@ -1256,48 +1536,9 @@ File* receive_incremental_check(int fd, const Config* config, bool* skipped) {
|
|||||||
}
|
}
|
||||||
close(old_fd);
|
close(old_fd);
|
||||||
|
|
||||||
File* file = file_create(check_path);
|
File* file = receive_full_file(fd, config, check_path);
|
||||||
free(check_path);
|
free(check_path);
|
||||||
free(full_path);
|
free(full_path);
|
||||||
if (file == NULL) {
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (config->use_metadata) {
|
|
||||||
int meta_ok = 1;
|
|
||||||
file->metadata = metadata_receive(fd, &meta_ok);
|
|
||||||
if (!meta_ok) {
|
|
||||||
file_destroy(file);
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Data* file_data = receive_data_limited(fd, MAX_RECEIVE_WHOLE_FILE_SIZE);
|
|
||||||
if (file_data == NULL) {
|
|
||||||
file_destroy(file);
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (config->use_compression &&
|
|
||||||
!compression_should_skip_with_suffixes(file->path, config->skip_compress_suffixes,
|
|
||||||
config->skip_compress_set ? config->skip_compress_count
|
|
||||||
: -1)) {
|
|
||||||
Data* uncompressed = data_decompress_limited(file_data, MAX_RECEIVE_WHOLE_FILE_SIZE);
|
|
||||||
data_destroy(file_data);
|
|
||||||
if (uncompressed == NULL) {
|
|
||||||
file_destroy(file);
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
if (uncompressed->size > MAX_FILE_DATA_SIZE) {
|
|
||||||
data_destroy(uncompressed);
|
|
||||||
file_destroy(file);
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
file_data = uncompressed;
|
|
||||||
}
|
|
||||||
|
|
||||||
data_destroy(file->data);
|
|
||||||
file->data = file_data;
|
|
||||||
return file;
|
return file;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1380,12 +1621,16 @@ File* file_receive_directory(int file_descriptor) {
|
|||||||
/* Read a delete-manifest frame (the STATUS_MANIFEST leading code has already
|
/* Read a delete-manifest frame (the STATUS_MANIFEST leading code has already
|
||||||
been consumed): a keep-set entry count followed by that many
|
been consumed): a keep-set entry count followed by that many
|
||||||
destination-relative paths, then a protected-prefix count followed by that
|
destination-relative paths, then a protected-prefix count followed by that
|
||||||
many destination-relative prefixes. The frame is self-delimiting (the counts
|
many destination-relative prefixes, then (protocol 2.10.0+) a missing-args
|
||||||
are authoritative), so the caller decides what to do next and continues
|
count followed by that many destination-relative delete paths. The frame is
|
||||||
reading the following STATUS_* frame. Returns an owned DeleteManifest, or
|
self-delimiting (the counts are authoritative), so the caller decides what to
|
||||||
NULL after sending STATUS_ERROR when the frame is malformed (bad count,
|
do next and continues reading the following STATUS_* frame. Every section is
|
||||||
empty/absolute path, path traversal, or an aggregate size beyond
|
validated identically: an entry must be non-empty, relative and traversal-free
|
||||||
MAX_MANIFEST_BYTES). */
|
and the aggregate length across ALL sections is capped by MAX_MANIFEST_BYTES
|
||||||
|
(so the missing-args deletion requests are confined like the rest of the
|
||||||
|
manifest). Returns an owned DeleteManifest, or NULL after sending STATUS_ERROR
|
||||||
|
when the frame is malformed (bad count, empty/absolute path, path traversal,
|
||||||
|
or an aggregate size beyond MAX_MANIFEST_BYTES). */
|
||||||
static bool receive_manifest_section(int fd, ArrayList* list, size_t* manifest_bytes) {
|
static bool receive_manifest_section(int fd, ArrayList* list, size_t* manifest_bytes) {
|
||||||
int count;
|
int count;
|
||||||
if (!receive_int(fd, &count)) {
|
if (!receive_int(fd, &count)) {
|
||||||
@@ -1418,14 +1663,16 @@ DeleteManifest* receive_manifest_entries(int fd) {
|
|||||||
}
|
}
|
||||||
manifest->keeps = array_list_create(free);
|
manifest->keeps = array_list_create(free);
|
||||||
manifest->protected = array_list_create(free);
|
manifest->protected = array_list_create(free);
|
||||||
if (!manifest->keeps || !manifest->protected) {
|
manifest->missing = array_list_create(free);
|
||||||
|
if (!manifest->keeps || !manifest->protected || !manifest->missing) {
|
||||||
delete_manifest_free(manifest);
|
delete_manifest_free(manifest);
|
||||||
send_status(fd, STATUS_ERROR);
|
send_status(fd, STATUS_ERROR);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
size_t manifest_bytes = 0;
|
size_t manifest_bytes = 0;
|
||||||
if (!receive_manifest_section(fd, manifest->keeps, &manifest_bytes) ||
|
if (!receive_manifest_section(fd, manifest->keeps, &manifest_bytes) ||
|
||||||
!receive_manifest_section(fd, manifest->protected, &manifest_bytes)) {
|
!receive_manifest_section(fd, manifest->protected, &manifest_bytes) ||
|
||||||
|
!receive_manifest_section(fd, manifest->missing, &manifest_bytes)) {
|
||||||
delete_manifest_free(manifest);
|
delete_manifest_free(manifest);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
@@ -1437,6 +1684,7 @@ void delete_manifest_free(DeleteManifest* manifest) {
|
|||||||
return;
|
return;
|
||||||
array_list_delete(manifest->keeps);
|
array_list_delete(manifest->keeps);
|
||||||
array_list_delete(manifest->protected);
|
array_list_delete(manifest->protected);
|
||||||
|
array_list_delete(manifest->missing);
|
||||||
free(manifest);
|
free(manifest);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1518,3 +1766,154 @@ bool manifest_delete_extras(const Config* config, DeleteManifest* manifest) {
|
|||||||
}
|
}
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* --delete-missing-args exact-path deletions: each destination mirror in
|
||||||
|
manifest->missing is an explicit user request, so it is removed even when the
|
||||||
|
ordinary extras walk (with its protected prefixes) would leave it alone. The
|
||||||
|
--delay-updates staging directory and basis snapshots are receiver artifacts
|
||||||
|
and stay protected exactly as in the extras walker. A regular file or
|
||||||
|
symlink is unlinked, an empty directory removed, and a NON-empty directory is
|
||||||
|
removed recursively only when --delete or --force is in effect (rsync parity:
|
||||||
|
the man page says a non-empty directory mirror is only deleted with --force
|
||||||
|
or --delete); otherwise it is left with a warning and the run continues. A
|
||||||
|
mirror that does not exist is a no-op. Returns false only on a genuine error
|
||||||
|
(a confinement failure on a validated path or an I/O error), which fails the
|
||||||
|
run. */
|
||||||
|
bool manifest_delete_missing_args(const Config* config, DeleteManifest* manifest) {
|
||||||
|
if (!config || !manifest)
|
||||||
|
return false;
|
||||||
|
if (!manifest->missing || manifest->missing->size == 0)
|
||||||
|
return true;
|
||||||
|
fprintf(stderr, "Deleting destination mirrors of missing source arguments...\n");
|
||||||
|
int skip_count = (config->delay_updates ? 1 : 0) + config->basis_count;
|
||||||
|
DeleteSkipEntry* skips = NULL;
|
||||||
|
if (skip_count > 0) {
|
||||||
|
skips = calloc((size_t)skip_count, sizeof(DeleteSkipEntry));
|
||||||
|
if (!skips)
|
||||||
|
return false;
|
||||||
|
int idx = 0;
|
||||||
|
if (config->delay_updates) {
|
||||||
|
skips[idx].prefix = DELAY_UPDATES_STAGING_DIR;
|
||||||
|
skips[idx].top_level_only = true;
|
||||||
|
idx++;
|
||||||
|
}
|
||||||
|
for (int i = 0; i < config->basis_count; i++) {
|
||||||
|
skips[idx].prefix = config->basis_dirs[i].path;
|
||||||
|
skips[idx].top_level_only = false;
|
||||||
|
idx++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
bool ok = true;
|
||||||
|
for (int i = 0; i < manifest->missing->size; i++) {
|
||||||
|
const char* rel = (const char*)manifest->missing->items[i];
|
||||||
|
if (!rel || *rel == '\0' || *rel == '/' || has_path_traversal(rel)) {
|
||||||
|
/* Defensive only: receive_manifest_entries already validated every
|
||||||
|
section identically, so a controlled peer never reaches this branch. */
|
||||||
|
log_message(LOG_LEVEL_ERROR, "invalid missing-args delete path");
|
||||||
|
ok = false;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
bool at_root = strchr(rel, '/') == NULL;
|
||||||
|
if (path_under_skip_prefix(rel, at_root, skips, skip_count)) {
|
||||||
|
char* escaped = output_escape(rel, log_get_8_bit_output());
|
||||||
|
log_message(LOG_LEVEL_WARNING,
|
||||||
|
"missing-args path '%s' is protected (staging directory or basis snapshot); "
|
||||||
|
"not deleting",
|
||||||
|
escaped ? escaped : "<allocation failed>");
|
||||||
|
free(escaped);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
char* full = path_cat(config->receive_root_directory, rel);
|
||||||
|
if (!full) {
|
||||||
|
ok = false;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
char* leaf = NULL;
|
||||||
|
int parent_fd = file_open_secure_parent(full, &leaf, false);
|
||||||
|
if (parent_fd < 0) {
|
||||||
|
/* The mirror's parent directory may itself not exist on the destination
|
||||||
|
(a deeper missing entry whose leading directories were never created).
|
||||||
|
That is a no-op -- there is nothing to delete -- matching
|
||||||
|
file_remove_tree_secure's absent-path handling; only a genuine I/O
|
||||||
|
error (EACCES, a symlink loop, ...) fails the run. */
|
||||||
|
bool absent = errno == ENOENT || errno == ENOTDIR;
|
||||||
|
free(full);
|
||||||
|
free(leaf);
|
||||||
|
if (!absent)
|
||||||
|
ok = false;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
struct stat st;
|
||||||
|
if (fstatat(parent_fd, leaf, &st, AT_SYMLINK_NOFOLLOW) != 0) {
|
||||||
|
/* Already absent: nothing to delete (a no-op, not a deletion). */
|
||||||
|
if (errno != ENOENT)
|
||||||
|
ok = false;
|
||||||
|
close(parent_fd);
|
||||||
|
free(leaf);
|
||||||
|
free(full);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
bool removed = false;
|
||||||
|
if (S_ISDIR(st.st_mode)) {
|
||||||
|
if (unlinkat(parent_fd, leaf, AT_REMOVEDIR) == 0) {
|
||||||
|
removed = true;
|
||||||
|
} else if (errno == ENOTEMPTY || errno == EEXIST) {
|
||||||
|
close(parent_fd);
|
||||||
|
parent_fd = -1;
|
||||||
|
free(leaf);
|
||||||
|
leaf = NULL;
|
||||||
|
if (config->use_delete || config->force_delete) {
|
||||||
|
if (!file_remove_tree_secure(full))
|
||||||
|
ok = false;
|
||||||
|
else
|
||||||
|
removed = true;
|
||||||
|
} else {
|
||||||
|
char* escaped = output_escape(rel, log_get_8_bit_output());
|
||||||
|
log_message(LOG_LEVEL_WARNING,
|
||||||
|
"missing-args destination '%s' is a non-empty directory; use --force or "
|
||||||
|
"--delete to remove it",
|
||||||
|
escaped ? escaped : "<allocation failed>");
|
||||||
|
free(escaped);
|
||||||
|
}
|
||||||
|
} else if (errno != ENOENT) {
|
||||||
|
ok = false;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if (unlinkat(parent_fd, leaf, 0) == 0) {
|
||||||
|
removed = true;
|
||||||
|
} else if (errno != ENOENT) {
|
||||||
|
ok = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (removed) {
|
||||||
|
char* escaped = output_escape(rel, log_get_8_bit_output());
|
||||||
|
fprintf(stderr, " Deleted: %s\n", escaped ? escaped : "<allocation failed>");
|
||||||
|
free(escaped);
|
||||||
|
}
|
||||||
|
if (parent_fd >= 0)
|
||||||
|
close(parent_fd);
|
||||||
|
free(leaf);
|
||||||
|
free(full);
|
||||||
|
if (!ok)
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
free(skips);
|
||||||
|
return ok;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Commit every deletion family the manifest carries. The --delete-missing-args
|
||||||
|
exact-path deletions run FIRST: they are explicit user requests and must not
|
||||||
|
be blocked by the extras walker's filter-exclusion protection (a protected
|
||||||
|
leftover inside a missing-argument directory must not make that user-requested
|
||||||
|
removal fail). The ordinary extras walk then runs when --delete is active.
|
||||||
|
Returns true when there was nothing to do or every requested deletion
|
||||||
|
committed. */
|
||||||
|
bool manifest_delete_all(const Config* config, DeleteManifest* manifest) {
|
||||||
|
if (!config || !manifest)
|
||||||
|
return false;
|
||||||
|
if (config->delete_missing_args && !manifest_delete_missing_args(config, manifest))
|
||||||
|
return false;
|
||||||
|
if (config->use_delete && !manifest_delete_extras(config, manifest))
|
||||||
|
return false;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
@@ -16,17 +16,22 @@ File* receive_incremental_check(int fd, const Config* config, bool* skipped);
|
|||||||
prefixes the sender asks the receiver never to delete (paths excluded on the
|
prefixes the sender asks the receiver never to delete (paths excluded on the
|
||||||
source, protected at any depth). When --delete-excluded is given the sender
|
source, protected at any depth). When --delete-excluded is given the sender
|
||||||
transmits an empty protected list so excluded destination mirrors are treated
|
transmits an empty protected list so excluded destination mirrors are treated
|
||||||
as ordinary extras. */
|
as ordinary extras. With --delete-missing-args a third section (`missing`)
|
||||||
|
carries the destination mirrors of explicitly-listed source entries that do
|
||||||
|
not exist: each is an exact deletion request, independent of the ordinary
|
||||||
|
extras walk (never blocked by the protected prefixes) and processed when the
|
||||||
|
manifest is committed. */
|
||||||
typedef struct DeleteManifest {
|
typedef struct DeleteManifest {
|
||||||
ArrayList* keeps;
|
ArrayList* keeps;
|
||||||
ArrayList* protected;
|
ArrayList* protected;
|
||||||
|
ArrayList* missing;
|
||||||
} DeleteManifest;
|
} DeleteManifest;
|
||||||
|
|
||||||
void delete_manifest_free(DeleteManifest* manifest);
|
void delete_manifest_free(DeleteManifest* manifest);
|
||||||
/* Read a delete-manifest frame: keep count + keeps, then protected count +
|
/* Read a delete-manifest frame: keep count + keeps, then protected count +
|
||||||
protected prefixes (self-delimiting; the leading STATUS_MANIFEST code has been
|
protected prefixes, then missing count + missing paths (self-delimiting; the
|
||||||
consumed). Returns an owned DeleteManifest, or NULL after signalling
|
leading STATUS_MANIFEST code has been consumed). Returns an owned
|
||||||
STATUS_ERROR on a malformed frame. */
|
DeleteManifest, or NULL after signalling STATUS_ERROR on a malformed frame. */
|
||||||
DeleteManifest* receive_manifest_entries(int fd);
|
DeleteManifest* receive_manifest_entries(int fd);
|
||||||
/* Remove destination entries under config->receive_root_directory that are not
|
/* Remove destination entries under config->receive_root_directory that are not
|
||||||
in `manifest` (bounded, all-or-nothing walk; staging-dir, basis-dir and
|
in `manifest` (bounded, all-or-nothing walk; staging-dir, basis-dir and
|
||||||
@@ -34,6 +39,20 @@ DeleteManifest* receive_manifest_entries(int fd);
|
|||||||
caller decides WHEN to run it based on the negotiated delete timing. Returns
|
caller decides WHEN to run it based on the negotiated delete timing. Returns
|
||||||
false (and the transfer fails) when the deletion cannot be committed. */
|
false (and the transfer fails) when the deletion cannot be committed. */
|
||||||
bool manifest_delete_extras(const Config* config, DeleteManifest* manifest);
|
bool manifest_delete_extras(const Config* config, DeleteManifest* manifest);
|
||||||
|
/* --delete-missing-args exact-path deletions: remove each destination mirror
|
||||||
|
in `manifest->missing` (never blocked by the protected prefixes, staging dir
|
||||||
|
and basis dirs excluded). A regular file/symlink is unlinked; an empty
|
||||||
|
directory is removed; a NON-empty directory is removed recursively only when
|
||||||
|
--delete or --force is in effect, otherwise it is left with a warning (rsync
|
||||||
|
parity). A missing path is a no-op. Returns false only on a genuine
|
||||||
|
confinement or I/O error (the run then fails); tolerated per-path cases are
|
||||||
|
reported and skipped. */
|
||||||
|
bool manifest_delete_missing_args(const Config* config, DeleteManifest* manifest);
|
||||||
|
/* Run every deletion family the manifest carries: the --delete-missing-args
|
||||||
|
exact-path deletions first (user requests are not blocked by exclusion
|
||||||
|
protection), then the ordinary extras walk when --delete is active. Returns
|
||||||
|
true when nothing to do or everything committed. */
|
||||||
|
bool manifest_delete_all(const Config* config, DeleteManifest* manifest);
|
||||||
|
|
||||||
/* Outcome of a single file_save_to_disk operation. The receiver needs to
|
/* Outcome of a single file_save_to_disk operation. The receiver needs to
|
||||||
distinguish "written" from "skipped" so --remove-source-files can be told
|
distinguish "written" from "skipped" so --remove-source-files can be told
|
||||||
|
|||||||
@@ -27,6 +27,7 @@ PipelineContextSender* pipeline_context_sender_create(Config* config, Queue* que
|
|||||||
context->loader_done = false;
|
context->loader_done = false;
|
||||||
context->manifest = NULL;
|
context->manifest = NULL;
|
||||||
context->excluded_paths = NULL;
|
context->excluded_paths = NULL;
|
||||||
|
context->missing_args = NULL;
|
||||||
context->scan_had_io_error = false;
|
context->scan_had_io_error = false;
|
||||||
context->remove_source_files = NULL;
|
context->remove_source_files = NULL;
|
||||||
context->early_delete = false;
|
context->early_delete = false;
|
||||||
@@ -86,6 +87,8 @@ void pipeline_context_sender_destroy(PipelineContextSender* context) {
|
|||||||
}
|
}
|
||||||
if (context->excluded_paths)
|
if (context->excluded_paths)
|
||||||
array_list_delete(context->excluded_paths);
|
array_list_delete(context->excluded_paths);
|
||||||
|
if (context->missing_args)
|
||||||
|
array_list_delete(context->missing_args);
|
||||||
if (context->remove_source_files)
|
if (context->remove_source_files)
|
||||||
array_list_delete(context->remove_source_files);
|
array_list_delete(context->remove_source_files);
|
||||||
config_delete(context->config);
|
config_delete(context->config);
|
||||||
|
|||||||
@@ -32,6 +32,12 @@ typedef struct {
|
|||||||
scanner's exclusion sink) or, in the early modes, by the path-only pre-scan
|
scanner's exclusion sink) or, in the early modes, by the path-only pre-scan
|
||||||
on the calling thread before the pipeline starts. */
|
on the calling thread before the pipeline starts. */
|
||||||
ArrayList* excluded_paths;
|
ArrayList* excluded_paths;
|
||||||
|
/* --delete-missing-args: the destination-relative mirrors of the --files-from
|
||||||
|
entries that are missing under the source. Computed by the preflight on
|
||||||
|
the calling thread before the pipeline starts; the sender thread transmits
|
||||||
|
them in the manifest frame's third section and the receiver deletes each as
|
||||||
|
an explicit request. */
|
||||||
|
ArrayList* missing_args;
|
||||||
/* A source I/O error (unreadable directory) was recorded during the scan.
|
/* A source I/O error (unreadable directory) was recorded during the scan.
|
||||||
Set by the pre-scan (before the threads start) or by the scanner thread
|
Set by the pre-scan (before the threads start) or by the scanner thread
|
||||||
under mutex_scanner; the caller turns it into a non-zero exit when
|
under mutex_scanner; the caller turns it into a non-zero exit when
|
||||||
|
|||||||
@@ -397,6 +397,14 @@ static const char* status_to_string(Status status) {
|
|||||||
return "CHECK_BATCH";
|
return "CHECK_BATCH";
|
||||||
case STATUS_MKDIR:
|
case STATUS_MKDIR:
|
||||||
return "MKDIR";
|
return "MKDIR";
|
||||||
|
case STATUS_APPEND:
|
||||||
|
return "APPEND";
|
||||||
|
case STATUS_APPEND_SIG:
|
||||||
|
return "APPEND_SIG";
|
||||||
|
case STATUS_APPEND_OK:
|
||||||
|
return "APPEND_OK";
|
||||||
|
case STATUS_APPEND_DATA:
|
||||||
|
return "APPEND_DATA";
|
||||||
default:
|
default:
|
||||||
return "UNKNOWN";
|
return "UNKNOWN";
|
||||||
}
|
}
|
||||||
|
|||||||
+15
-1
@@ -70,7 +70,21 @@ enum NET_STATUS {
|
|||||||
STATUS_CHECK_BATCH,
|
STATUS_CHECK_BATCH,
|
||||||
/* An explicit directory entry (--dirs): the sender transmits only the path;
|
/* An explicit directory entry (--dirs): the sender transmits only the path;
|
||||||
* the receiver creates the directory below the receive root. */
|
* the receiver creates the directory below the receive root. */
|
||||||
STATUS_MKDIR
|
STATUS_MKDIR,
|
||||||
|
/* --append / --append-verify tail resume. STATUS_APPEND is sent by the
|
||||||
|
* receiver after a per-file STATUS_CHECK when the existing destination file
|
||||||
|
* is SHORTER than the source and an append mode is negotiated: its payload is
|
||||||
|
* the resume offset (the number of prefix bytes already present), after which
|
||||||
|
* the sender answers either directly with STATUS_APPEND_DATA (plain --append,
|
||||||
|
* prefix not verified) or, for --append-verify, first with STATUS_APPEND_SIG
|
||||||
|
* carrying the xxHash64 of the source prefix; the receiver then replies
|
||||||
|
* STATUS_APPEND_OK (prefix matched -> sender transmits the tail) or
|
||||||
|
* STATUS_NEXT (prefix mismatch -> sender falls back to a full transfer).
|
||||||
|
* STATUS_APPEND_DATA carries the tail bytes (compressed data frame). */
|
||||||
|
STATUS_APPEND,
|
||||||
|
STATUS_APPEND_SIG,
|
||||||
|
STATUS_APPEND_OK,
|
||||||
|
STATUS_APPEND_DATA
|
||||||
};
|
};
|
||||||
|
|
||||||
void io_set_fds(int read_fd, int write_fd);
|
void io_set_fds(int read_fd, int write_fd);
|
||||||
|
|||||||
+14
-2
@@ -208,8 +208,8 @@ static bool is_dir_in_manifest(const char* rel_path, ArrayList* manifest) {
|
|||||||
therefore protects "a" and "a/b/c" but not "ab". Entries with top_level_only
|
therefore protects "a" and "a/b/c" but not "ab". Entries with top_level_only
|
||||||
set only protect DIRECT children of the receive root (at_root); nested
|
set only protect DIRECT children of the receive root (at_root); nested
|
||||||
directories that share such a name stay ordinary destination content. */
|
directories that share such a name stay ordinary destination content. */
|
||||||
static bool path_under_skip_prefix(const char* child_rel, bool at_root,
|
bool path_under_skip_prefix(const char* child_rel, bool at_root, const DeleteSkipEntry* skips,
|
||||||
const DeleteSkipEntry* skips, int skip_count) {
|
int skip_count) {
|
||||||
for (int i = 0; i < skip_count; i++) {
|
for (int i = 0; i < skip_count; i++) {
|
||||||
if (skips[i].top_level_only && !at_root)
|
if (skips[i].top_level_only && !at_root)
|
||||||
continue;
|
continue;
|
||||||
@@ -531,3 +531,15 @@ char* path_cat(const char* path1, const char* path2) {
|
|||||||
new_path[path1_len + path2_len + 1] = '\0';
|
new_path[path1_len + path2_len + 1] = '\0';
|
||||||
return new_path;
|
return new_path;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool append_resume_eligible(unsigned long long old_size, unsigned long long check_size) {
|
||||||
|
return old_size < check_size;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool append_tail_length(unsigned long long old_size, unsigned long long check_size,
|
||||||
|
unsigned long long* tail_out) {
|
||||||
|
if (!tail_out || !append_resume_eligible(old_size, check_size))
|
||||||
|
return false;
|
||||||
|
*tail_out = check_size - old_size;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
@@ -32,6 +32,11 @@ typedef struct {
|
|||||||
const char* prefix;
|
const char* prefix;
|
||||||
bool top_level_only;
|
bool top_level_only;
|
||||||
} DeleteSkipEntry;
|
} DeleteSkipEntry;
|
||||||
|
/* True when child_rel is, or lies below, one of the protected entries (a prefix
|
||||||
|
"a" protects "a" and "a/b/c" but not "ab"; top_level_only entries protect
|
||||||
|
only DIRECT children of the destination root, i.e. child_rel has no '/'). */
|
||||||
|
bool path_under_skip_prefix(const char* child_rel, bool at_root, const DeleteSkipEntry* skips,
|
||||||
|
int skip_count);
|
||||||
/* Remove files/dirs under dest_root that are not listed in manifest without
|
/* Remove files/dirs under dest_root that are not listed in manifest without
|
||||||
ever descending into a protected prefix (see DeleteSkipEntry). When
|
ever descending into a protected prefix (see DeleteSkipEntry). When
|
||||||
max_delete is not SIZE_MAX the run is all-or-nothing: extras are counted
|
max_delete is not SIZE_MAX the run is all-or-nothing: extras are counted
|
||||||
@@ -53,5 +58,13 @@ void utils_set_authorized_root_fd(int fd);
|
|||||||
bool has_path_traversal(const char* path);
|
bool has_path_traversal(const char* path);
|
||||||
bool utils_valid_batch_path(const char* path);
|
bool utils_valid_batch_path(const char* path);
|
||||||
bool format_human_bytes(unsigned long long bytes, char* buffer, size_t buffer_size);
|
bool format_human_bytes(unsigned long long bytes, char* buffer, size_t buffer_size);
|
||||||
|
/* --append / --append-verify tail-resume math (pure). A resume is eligible only
|
||||||
|
when an existing destination file is SHORTER than the source; the tail length
|
||||||
|
is then the difference. append_resume_eligible answers whether the shorter
|
||||||
|
file makes a resume possible; append_tail_length additionally returns that
|
||||||
|
tail length, refusing (false) the degenerate old_size >= check_size case. */
|
||||||
|
bool append_resume_eligible(unsigned long long old_size, unsigned long long check_size);
|
||||||
|
bool append_tail_length(unsigned long long old_size, unsigned long long check_size,
|
||||||
|
unsigned long long* tail_out);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
@@ -0,0 +1,141 @@
|
|||||||
|
"""--append / --append-verify tail-resume integration tests.
|
||||||
|
|
||||||
|
A shorter existing destination file is resumed by transferring only the tail:
|
||||||
|
--append sends it without verifying the retained prefix (rsync parity: a wrong
|
||||||
|
prefix is kept, so the result can differ from the source), while --append-verify
|
||||||
|
checksums the retained prefix against the source and, on a mismatch, falls back
|
||||||
|
to a clean full transfer so the result is always a byte-identical source copy.
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
import random
|
||||||
|
import shutil
|
||||||
|
|
||||||
|
from common import (
|
||||||
|
TEST_DATA_DIR,
|
||||||
|
run_client, CountingProxy, clean_dir,
|
||||||
|
get_dest_received_dir, CLIENT_CMD,
|
||||||
|
)
|
||||||
|
|
||||||
|
REL = "sub/grow.dat"
|
||||||
|
|
||||||
|
|
||||||
|
def _grow_payload(prefix_size, added_size, seed=99):
|
||||||
|
r = random.Random(seed)
|
||||||
|
return bytes(r.randbytes(prefix_size)), bytes(r.randbytes(added_size))
|
||||||
|
|
||||||
|
|
||||||
|
class TestAppend:
|
||||||
|
def _make(self, tag):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, f"append_{tag}_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, f"append_{tag}_dst")
|
||||||
|
clean_dir(source)
|
||||||
|
shutil.rmtree(dest, ignore_errors=True)
|
||||||
|
return source, dest
|
||||||
|
|
||||||
|
def _place(self, root, rel, data):
|
||||||
|
p = os.path.join(root, rel)
|
||||||
|
os.makedirs(os.path.dirname(p), exist_ok=True)
|
||||||
|
with open(p, "wb") as fh:
|
||||||
|
fh.write(data)
|
||||||
|
return p
|
||||||
|
|
||||||
|
def _read(self, root, rel):
|
||||||
|
with open(os.path.join(root, rel), "rb") as fh:
|
||||||
|
return fh.read()
|
||||||
|
|
||||||
|
def _dest_file(self, source, dest, rel):
|
||||||
|
return os.path.join(get_dest_received_dir(dest, source), rel)
|
||||||
|
|
||||||
|
def test_append_resumes_short_dest_atomically(self, shared_server):
|
||||||
|
"""A shorter dest with a MATCHING prefix is resumed; the reconstructed
|
||||||
|
file is byte-identical to the source."""
|
||||||
|
source, dest = self._make("atomic")
|
||||||
|
prefix, added = _grow_payload(1 * 1024 * 1024, 64 * 1024)
|
||||||
|
self._place(source, REL, prefix + added)
|
||||||
|
self._place(self._dest_file(source, dest, ""), REL, prefix)
|
||||||
|
|
||||||
|
result, _ = run_client(source, dest, flags=["--append"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, \
|
||||||
|
f"--append failed: {(result.stderr or result.stdout)[:400]}"
|
||||||
|
assert self._read(self._dest_file(source, dest, ""), REL) == prefix + added
|
||||||
|
|
||||||
|
def test_append_verify_matching_prefix_succeeds(self, shared_server):
|
||||||
|
source, dest = self._make("verify_ok")
|
||||||
|
prefix, added = _grow_payload(512 * 1024, 32 * 1024)
|
||||||
|
self._place(source, REL, prefix + added)
|
||||||
|
self._place(self._dest_file(source, dest, ""), REL, prefix)
|
||||||
|
|
||||||
|
result, _ = run_client(source, dest, flags=["--append-verify"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, \
|
||||||
|
f"--append-verify failed: {(result.stderr or result.stdout)[:400]}"
|
||||||
|
assert self._read(self._dest_file(source, dest, ""), REL) == prefix + added
|
||||||
|
|
||||||
|
def test_append_sends_only_tail(self, shared_server):
|
||||||
|
"""Sorted transfer moves only the tail: wire bytes stay well below the
|
||||||
|
full source size (incompressible payload, no -c)."""
|
||||||
|
source, dest = self._make("tail")
|
||||||
|
prefix, added = _grow_payload(4 * 1024 * 1024, 8 * 1024, seed=7)
|
||||||
|
full = prefix + added
|
||||||
|
self._place(source, REL, full)
|
||||||
|
self._place(self._dest_file(source, dest, ""), REL, prefix)
|
||||||
|
|
||||||
|
proxy = CountingProxy(shared_server.port)
|
||||||
|
cmd = (CLIENT_CMD + ["--source-dir", source, "--dest-dir", dest,
|
||||||
|
"--save-to-disk", "--server-port", str(proxy.port), "--append"])
|
||||||
|
result = proxy.run(cmd)
|
||||||
|
assert result.returncode == 0, \
|
||||||
|
f"--append failed: {(result.stderr or result.stdout)[:400]}"
|
||||||
|
assert self._read(self._dest_file(source, dest, ""), REL) == full
|
||||||
|
assert proxy.client_to_server < full.__len__() // 2, \
|
||||||
|
f"expected a tail-only transfer, sent {proxy.client_to_server}B for {full.__len__()}B"
|
||||||
|
|
||||||
|
def test_plain_append_wrong_prefix_is_rsync_parity(self, shared_server):
|
||||||
|
"""--append does NOT verify the retained prefix: a wrong prefix is kept,
|
||||||
|
so the result is prefix+tail (differs from the source). This is the
|
||||||
|
documented rsync-parity risk of plain --append."""
|
||||||
|
source, dest = self._make("plain_wrong")
|
||||||
|
correct_prefix, added = _grow_payload(256 * 1024, 32 * 1024, seed=1)
|
||||||
|
wrong_prefix = bytes(b ^ 0xFF for b in correct_prefix)
|
||||||
|
self._place(source, REL, correct_prefix + added)
|
||||||
|
self._place(self._dest_file(source, dest, ""), REL, wrong_prefix)
|
||||||
|
|
||||||
|
result, _ = run_client(source, dest, flags=["--append"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0
|
||||||
|
assert self._read(self._dest_file(source, dest, ""), REL) == wrong_prefix + added
|
||||||
|
|
||||||
|
def test_append_verify_wrong_prefix_never_corrupts(self, shared_server):
|
||||||
|
"""--append-verify detects the retained prefix mismatch and falls back to
|
||||||
|
a full transfer, so the result is a byte-identical source copy."""
|
||||||
|
source, dest = self._make("verify_wrong")
|
||||||
|
correct_prefix, added = _grow_payload(256 * 1024, 32 * 1024, seed=2)
|
||||||
|
wrong_prefix = bytes(b ^ 0xFF for b in correct_prefix)
|
||||||
|
self._place(source, REL, correct_prefix + added)
|
||||||
|
self._place(self._dest_file(source, dest, ""), REL, wrong_prefix)
|
||||||
|
|
||||||
|
result, _ = run_client(source, dest, flags=["--append-verify"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, \
|
||||||
|
f"--append-verify mismatch fallback failed: {(result.stderr or result.stdout)[:400]}"
|
||||||
|
assert self._read(self._dest_file(source, dest, ""), REL) == correct_prefix + added
|
||||||
|
|
||||||
|
def test_append_with_inplace(self, shared_server):
|
||||||
|
source, dest = self._make("inplace")
|
||||||
|
prefix, added = _grow_payload(128 * 1024, 16 * 1024, seed=3)
|
||||||
|
self._place(source, REL, prefix + added)
|
||||||
|
self._place(self._dest_file(source, dest, ""), REL, prefix)
|
||||||
|
|
||||||
|
result, _ = run_client(source, dest, flags=["--append", "--inplace"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, \
|
||||||
|
f"--append --inplace failed: {(result.stderr or result.stdout)[:400]}"
|
||||||
|
assert self._read(self._dest_file(source, dest, ""), REL) == prefix + added
|
||||||
|
|
||||||
|
def test_append_multithreaded(self, shared_server):
|
||||||
|
source, dest = self._make("mthread")
|
||||||
|
prefix, added = _grow_payload(512 * 1024, 32 * 1024, seed=4)
|
||||||
|
self._place(source, REL, prefix + added)
|
||||||
|
self._place(self._dest_file(source, dest, ""), REL, prefix)
|
||||||
|
|
||||||
|
result, _ = run_client(source, dest, flags=["--append", "-m"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, \
|
||||||
|
f"--append -m failed: {(result.stderr or result.stdout)[:400]}"
|
||||||
|
assert self._read(self._dest_file(source, dest, ""), REL) == prefix + added
|
||||||
@@ -550,6 +550,147 @@ class TestIncremental:
|
|||||||
assert not mismatches, f"Mismatch: {mismatches}"
|
assert not mismatches, f"Mismatch: {mismatches}"
|
||||||
|
|
||||||
|
|
||||||
|
class TestChecksumChoice:
|
||||||
|
"""--checksum-choice/--cc and --checksum-seed: the whole-file digest used by
|
||||||
|
the --incremental/--checksum handshake is selectable and seedable. The
|
||||||
|
receiver hashes the on-disk old file with the SAME algorithm+seed, so an
|
||||||
|
unchanged file is skipped and a changed file (even with identical size and
|
||||||
|
mtime) is transferred -- and the transfer always lands byte-exact.
|
||||||
|
FastSync accepts xxh64 (default, seed-aware) and md5; names it does not
|
||||||
|
implement are rejected, never silently ignored."""
|
||||||
|
|
||||||
|
def test_unsupported_algorithm_is_rejected(self, shared_server):
|
||||||
|
result, _ = run_client(
|
||||||
|
SOURCE_DIR, DEST_DIR,
|
||||||
|
flags=["--checksum", "--checksum-choice=sha256"],
|
||||||
|
port=shared_server.port,
|
||||||
|
)
|
||||||
|
assert result.returncode != 0, "sha256 must be rejected, not silently ignored"
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("algo", ["xxh64", "md5"])
|
||||||
|
@pytest.mark.parametrize("mt", [False, True])
|
||||||
|
def test_unchanged_skipped_and_bytes_preserved(self, shared_server, algo, mt):
|
||||||
|
clean_dir(DEST_DIR)
|
||||||
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}"
|
||||||
|
|
||||||
|
flags = (["-M", "--incremental", "--checksum", f"--checksum-choice={algo}"] +
|
||||||
|
(["-m"] if mt else []))
|
||||||
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=flags, port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"checksum {algo} run failed: {result.stderr[:200]}"
|
||||||
|
|
||||||
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
||||||
|
mismatches, missing = verify_transfer(SOURCE_DIR, received)
|
||||||
|
assert not missing, f"Missing: {missing}"
|
||||||
|
assert not mismatches, f"Mismatch: {mismatches}"
|
||||||
|
|
||||||
|
# A changed source file with the SAME size and mtime must still be
|
||||||
|
# detected (and re-transferred byte-exactly) because the whole-file digest
|
||||||
|
# differs -- the explicit reason --checksum exists. This exercises the
|
||||||
|
# sender/receiver digest agreement for a non-default algorithm.
|
||||||
|
@pytest.mark.parametrize("algo", ["xxh64", "md5"])
|
||||||
|
@pytest.mark.parametrize("mt", [False, True])
|
||||||
|
def test_changed_same_size_mtime_redetected(self, shared_server, algo, mt):
|
||||||
|
clean_dir(DEST_DIR)
|
||||||
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0
|
||||||
|
|
||||||
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
||||||
|
source_file = os.path.join(SOURCE_DIR, "small.txt") # "hello world\n" (12 bytes)
|
||||||
|
received_file = os.path.join(received, "small.txt")
|
||||||
|
source_stat = os.stat(source_file)
|
||||||
|
with open(received_file, "wb") as f:
|
||||||
|
f.write(b"DDDDDDDDDDDD") # same size, different content
|
||||||
|
os.utime(received_file, (source_stat.st_atime, source_stat.st_mtime))
|
||||||
|
|
||||||
|
flags = (["-M", "--incremental", "--checksum", f"--checksum-choice={algo}"] +
|
||||||
|
(["-m"] if mt else []))
|
||||||
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=flags, port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"checksum {algo} redetect failed: {result.stderr[:200]}"
|
||||||
|
with open(received_file, "rb") as f:
|
||||||
|
assert f.read() == b"hello world\n"
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("algo", ["xxh64", "md5"])
|
||||||
|
def test_unchanged_run_transfers_almost_no_data(self, shared_server, algo):
|
||||||
|
# A fully-unchanged --checksum run skips every file: only the config + a
|
||||||
|
# small handshake travels, not the payloads. Proxy byte counts are not
|
||||||
|
# available for -m (multithreaded connections), so single-thread only.
|
||||||
|
clean_dir(DEST_DIR)
|
||||||
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0
|
||||||
|
|
||||||
|
flags = ["-M", "--incremental", "--checksum", f"--checksum-choice={algo}"]
|
||||||
|
proxy = CountingProxy(shared_server.port)
|
||||||
|
cmd = (CLIENT_CMD + ["--source-dir", SOURCE_DIR, "--dest-dir", DEST_DIR,
|
||||||
|
"--save-to-disk", "--server-port", str(proxy.port)] + flags)
|
||||||
|
result = proxy.run(cmd)
|
||||||
|
assert result.returncode == 0, f"checksum {algo} skip run failed: {result.stderr[:200]}"
|
||||||
|
assert proxy.client_to_server < 100000, \
|
||||||
|
f"unchanged --checksum run sent {proxy.client_to_server} bytes; expected a skip"
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("mt", [False, True])
|
||||||
|
def test_seed_is_deterministic_and_preserves_content(self, shared_server, mt):
|
||||||
|
clean_dir(DEST_DIR)
|
||||||
|
flags = ["-M", "--incremental", "--checksum",
|
||||||
|
"--checksum-choice=xxh64", "--checksum-seed=987654"] + (["-m"] if mt else [])
|
||||||
|
first, _ = run_client(SOURCE_DIR, DEST_DIR, flags=flags, port=shared_server.port)
|
||||||
|
assert first.returncode == 0, f"seeded run failed: {first.stderr[:200]}"
|
||||||
|
|
||||||
|
# A second run with the SAME seed and unchanged content skips everything
|
||||||
|
# deterministically (same digests both sides).
|
||||||
|
second, _ = run_client(SOURCE_DIR, DEST_DIR, flags=flags, port=shared_server.port)
|
||||||
|
assert second.returncode == 0, f"deterministic rerun failed: {second.stderr[:200]}"
|
||||||
|
|
||||||
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
||||||
|
mismatches, missing = verify_transfer(SOURCE_DIR, received)
|
||||||
|
assert not missing and not mismatches, f"missing={missing} mismatches={mismatches}"
|
||||||
|
|
||||||
|
# A changed file with the same size and mtime is still caught and fixed
|
||||||
|
# (a non-zero seed does not weaken the comparison).
|
||||||
|
source_file = os.path.join(SOURCE_DIR, "medium.txt")
|
||||||
|
received_file = os.path.join(received, "medium.txt")
|
||||||
|
source_stat = os.stat(source_file)
|
||||||
|
with open(received_file, "wb") as f:
|
||||||
|
f.write(b"z" * os.path.getsize(source_file))
|
||||||
|
os.utime(received_file, (source_stat.st_atime, source_stat.st_mtime))
|
||||||
|
third, _ = run_client(SOURCE_DIR, DEST_DIR, flags=flags, port=shared_server.port)
|
||||||
|
assert third.returncode == 0, f"seeded redetect failed: {third.stderr[:200]}"
|
||||||
|
with open(received_file, "rb") as f:
|
||||||
|
assert f.read() == open(source_file, "rb").read()
|
||||||
|
|
||||||
|
# --checksum-seed also feeds the delta path's per-block strong checksum on
|
||||||
|
# both ends (receiver signature and sender window hash use the same seed),
|
||||||
|
# so a seeded delta transfer still lands byte-exact.
|
||||||
|
@pytest.mark.parametrize("mt", [False, True])
|
||||||
|
def test_seed_delta_block_hash_transfers_byte_exact(self, shared_server, mt):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, f"ccseed_{'m' if mt else 's'}_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, f"ccseed_{'m' if mt else 's'}_dst")
|
||||||
|
clean_dir(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
big = os.path.join(source, "big.bin")
|
||||||
|
with open(big, "wb") as f:
|
||||||
|
f.write(bytes(range(256)) * 200) # 51200 bytes > delta 16K floor
|
||||||
|
result, _ = run_client(source, dest, flags=["-M"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"seed delta seed failed: {result.stderr[:200]}"
|
||||||
|
|
||||||
|
# Edit a region so the receiver must match a changed block with the seed.
|
||||||
|
with open(big, "r+b") as f:
|
||||||
|
f.seek(1000)
|
||||||
|
f.write(b"\x00" * 64)
|
||||||
|
# Force an mtime mismatch: the incremental quick-check skips files whose
|
||||||
|
# stored mtime second equals the source's, which can collide when the
|
||||||
|
# edit and the prior sync share a second. Setting an old dest mtime
|
||||||
|
# guarantees the delta path is exercised deterministically.
|
||||||
|
os.utime(os.path.join(get_dest_received_dir(dest, source), "big.bin"), (0, 0))
|
||||||
|
flags = (["-M", "--incremental", "--delta", "--checksum-seed=314159"] +
|
||||||
|
(["-m"] if mt else []))
|
||||||
|
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"seed delta run failed: {result.stderr[:200]}"
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
assert _read_file(os.path.join(received, "big.bin")) == _read_file(big), \
|
||||||
|
"seeded delta transfer is not byte-exact"
|
||||||
|
|
||||||
|
|
||||||
class TestUpdate:
|
class TestUpdate:
|
||||||
def test_update_skips_older_destination_and_allows_equal_or_newer_source(self, shared_server):
|
def test_update_skips_older_destination_and_allows_equal_or_newer_source(self, shared_server):
|
||||||
clean_dir(DEST_DIR)
|
clean_dir(DEST_DIR)
|
||||||
@@ -1742,6 +1883,278 @@ class TestRelativeFilesFrom:
|
|||||||
"unlisted relative file was not deleted"
|
"unlisted relative file was not deleted"
|
||||||
|
|
||||||
|
|
||||||
|
class TestMissingArgs:
|
||||||
|
"""--ignore-missing-args / --delete-missing-args: a --files-from entry that
|
||||||
|
does not exist under the source is skipped instead of failing the run, and
|
||||||
|
(delete-missing) its destination mirror is removed receiver-side. Following
|
||||||
|
rsync, --delete-missing-args implies --ignore-missing-args but is
|
||||||
|
independent of --delete: unrelated extras stay unless --delete is also
|
||||||
|
given, and the missing-args deletion (an explicit user request) is never
|
||||||
|
blocked by filter-exclusion protection."""
|
||||||
|
|
||||||
|
def _make_source(self, name):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, name)
|
||||||
|
clean_dir(source)
|
||||||
|
for rel, content in {
|
||||||
|
"a.txt": b"a\n",
|
||||||
|
"sub/b.txt": b"b\n",
|
||||||
|
"keep.txt": b"keep\n",
|
||||||
|
"prot/kept.txt": b"kept\n",
|
||||||
|
}.items():
|
||||||
|
full = os.path.join(source, rel)
|
||||||
|
os.makedirs(os.path.dirname(full), exist_ok=True)
|
||||||
|
with open(full, "wb") as fh:
|
||||||
|
fh.write(content)
|
||||||
|
return source
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("mt", [False, True])
|
||||||
|
def test_missing_entry_is_hard_error_before_transfer(self, shared_server, mt):
|
||||||
|
source = self._make_source("mg_default_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "mg_default_dst")
|
||||||
|
clean_dir(dest)
|
||||||
|
lst = _write_rel_list(b"a.txt\ngone.txt\nsub/b.txt\n")
|
||||||
|
flags = ["--files-from", lst] + (["-m"] if mt else [])
|
||||||
|
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
|
||||||
|
assert result.returncode != 0, "a listed-but-missing entry did not fail the run"
|
||||||
|
assert "gone.txt" in (result.stderr or result.stdout)
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
assert not os.path.isfile(os.path.join(received, "a.txt")), \
|
||||||
|
"the transfer started despite the missing-entry hard error"
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("mt", [False, True])
|
||||||
|
def test_ignore_missing_args_transfers_the_rest(self, shared_server, mt):
|
||||||
|
source = self._make_source("mg_ignore_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "mg_ignore_dst")
|
||||||
|
clean_dir(dest)
|
||||||
|
lst = _write_rel_list(b"a.txt\ngone.txt\nsub/b.txt\n")
|
||||||
|
flags = ["--files-from", lst, "--ignore-missing-args"] + (["-m"] if mt else [])
|
||||||
|
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"ignore-missing-args sync failed: {result.stderr[:300]}"
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
assert _read_file(os.path.join(received, "a.txt")) == b"a\n"
|
||||||
|
assert _read_file(os.path.join(received, "sub", "b.txt")) == b"b\n"
|
||||||
|
assert not os.path.exists(os.path.join(received, "gone.txt")), \
|
||||||
|
"nothing was transferred for the missing entry"
|
||||||
|
assert "--ignore-missing-args" in (result.stderr or result.stdout), \
|
||||||
|
"the skipped entry must be observable (not a silent no-op)"
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("mt", [False, True])
|
||||||
|
def test_all_missing_entries_succeed_transferring_nothing(self, shared_server, mt):
|
||||||
|
source = self._make_source("mg_all_missing_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "mg_all_missing_dst")
|
||||||
|
clean_dir(dest)
|
||||||
|
lst = _write_rel_list(b"gone1.txt\ngone2.txt\n")
|
||||||
|
flags = ["--files-from", lst, "--ignore-missing-args"] + (["-m"] if mt else [])
|
||||||
|
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
|
||||||
|
assert result.returncode == 0, \
|
||||||
|
f"all-missing run should succeed (rsync parity): {result.stderr[:300]}"
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
assert not os.path.exists(os.path.join(received, "gone1.txt"))
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("mt", [False, True])
|
||||||
|
def test_empty_list_stays_a_hard_error(self, shared_server, mt):
|
||||||
|
source = self._make_source("mg_empty_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "mg_empty_dst")
|
||||||
|
clean_dir(dest)
|
||||||
|
lst = _write_rel_list(b"")
|
||||||
|
flags = ["--files-from", lst, "--ignore-missing-args"] + (["-m"] if mt else [])
|
||||||
|
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
|
||||||
|
assert result.returncode != 0, "an empty --files-from list must stay a hard error"
|
||||||
|
assert "contains no entries" in (result.stderr or result.stdout)
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("mt", [False, True])
|
||||||
|
def test_delete_missing_removes_mirror_not_unrelated(self, mt):
|
||||||
|
"""-R layout: --delete-missing-args deletes exactly the missing entry's
|
||||||
|
destination mirror (bare relative path) and leaves unrelated extras
|
||||||
|
untouched; with --delete also present the unrelated extras go too."""
|
||||||
|
source = self._make_source("mg_del_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "mg_del_dst")
|
||||||
|
clean_dir(dest)
|
||||||
|
with ServerManager() as server:
|
||||||
|
server.start(extra_args=["--allow-delete"])
|
||||||
|
seed = _write_rel_list(b"a.txt\nsub/b.txt\n")
|
||||||
|
result, _ = run_client(source, dest,
|
||||||
|
flags=["--files-from", seed, "-R"] + (["-m"] if mt else []),
|
||||||
|
port=server.port)
|
||||||
|
assert result.returncode == 0, f"seed -R sync failed: {result.stderr[:200]}"
|
||||||
|
assert os.path.isfile(os.path.join(dest, "a.txt"))
|
||||||
|
assert os.path.isfile(os.path.join(dest, "sub", "b.txt"))
|
||||||
|
|
||||||
|
# Plant the missing entry's destination mirror and an unrelated extra.
|
||||||
|
with open(os.path.join(dest, "gone.txt"), "w") as fh:
|
||||||
|
fh.write("stale mirror")
|
||||||
|
with open(os.path.join(dest, "unrelated.txt"), "w") as fh:
|
||||||
|
fh.write("unrelated")
|
||||||
|
|
||||||
|
lst = _write_rel_list(b"a.txt\ngone.txt\nsub/b.txt\n")
|
||||||
|
flags = ["--files-from", lst, "-R", "--delete-missing-args"] + (["-m"] if mt else [])
|
||||||
|
result, _ = run_client(source, dest, flags=flags, port=server.port)
|
||||||
|
assert result.returncode == 0, f"delete-missing sync failed: {result.stderr[:300]}"
|
||||||
|
assert not os.path.exists(os.path.join(dest, "gone.txt")), \
|
||||||
|
"the missing entry's destination mirror was not deleted"
|
||||||
|
assert os.path.isfile(os.path.join(dest, "unrelated.txt")), \
|
||||||
|
"--delete-missing-args removed an unrelated extra (only --delete may)"
|
||||||
|
assert os.path.isfile(os.path.join(dest, "a.txt"))
|
||||||
|
assert os.path.isfile(os.path.join(dest, "sub", "b.txt"))
|
||||||
|
|
||||||
|
# Now with --delete the unrelated extra is an ordinary extra and must go.
|
||||||
|
lst2 = _write_rel_list(b"a.txt\ngone.txt\nsub/b.txt\n")
|
||||||
|
flags2 = ["--files-from", lst2, "-R", "--delete-missing-args", "--delete"] + \
|
||||||
|
(["-m"] if mt else [])
|
||||||
|
result, _ = run_client(source, dest, flags=flags2, port=server.port)
|
||||||
|
assert result.returncode == 0, f"delete-missing + delete sync failed: {result.stderr[:300]}"
|
||||||
|
assert not os.path.exists(os.path.join(dest, "unrelated.txt")), \
|
||||||
|
"--delete did not remove the unrelated extra"
|
||||||
|
assert not os.path.exists(os.path.join(dest, "gone.txt"))
|
||||||
|
assert os.path.isfile(os.path.join(dest, "a.txt"))
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("mt", [False, True])
|
||||||
|
def test_delete_missing_mirror_outside_relative_layout(self, mt):
|
||||||
|
"""Without -R the missing entry's mirror mirrors the full source path
|
||||||
|
below the destination root, exactly like a present sibling's."""
|
||||||
|
source = self._make_source("mg_del_nor_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "mg_del_nor_dst")
|
||||||
|
clean_dir(dest)
|
||||||
|
with ServerManager() as server:
|
||||||
|
server.start(extra_args=["--allow-delete"])
|
||||||
|
# Full-tree seed places every current source file in the mirrored layout.
|
||||||
|
result, _ = run_client(source, dest, flags=["--delete"], port=server.port)
|
||||||
|
assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}"
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
assert os.path.isfile(os.path.join(received, "a.txt"))
|
||||||
|
|
||||||
|
# Plant a stale mirror for an entry not (yet) on the source.
|
||||||
|
with open(os.path.join(received, "gone.txt"), "w") as fh:
|
||||||
|
fh.write("stale")
|
||||||
|
lst = _write_rel_list(b"a.txt\ngone.txt\n")
|
||||||
|
result, _ = run_client(source, dest,
|
||||||
|
flags=["--files-from", lst, "--delete-missing-args"],
|
||||||
|
port=server.port)
|
||||||
|
assert result.returncode == 0, f"delete-missing no-R sync failed: {result.stderr[:300]}"
|
||||||
|
assert not os.path.exists(os.path.join(received, "gone.txt")), \
|
||||||
|
"the full-source-mirror path of the missing entry was not deleted"
|
||||||
|
assert os.path.isfile(os.path.join(received, "a.txt"))
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("mt", [False, True])
|
||||||
|
def test_delete_missing_args_not_blocked_by_exclude_protection(self, mt):
|
||||||
|
"""A missing-arg mirror that sits under a filter-excluded directory is an
|
||||||
|
explicit user request, so --delete-missing-args removes it even though an
|
||||||
|
ordinary --delete honours the exclusion protection (rsync parity). Uses
|
||||||
|
the non-relative layout: exclusion protection is only recorded there."""
|
||||||
|
source = self._make_source("mg_excl_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "mg_excl_dst")
|
||||||
|
clean_dir(dest)
|
||||||
|
with ServerManager() as server:
|
||||||
|
server.start(extra_args=["--allow-delete"])
|
||||||
|
# Full-tree seed mirrors the whole source below the destination root.
|
||||||
|
result, _ = run_client(source, dest, port=server.port)
|
||||||
|
assert result.returncode == 0, f"seed failed: {result.stderr[:200]}"
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
assert os.path.isfile(os.path.join(received, "prot", "kept.txt"))
|
||||||
|
|
||||||
|
# A stale mirror under the (now excluded) prot/ directory, plus an extra.
|
||||||
|
with open(os.path.join(received, "prot", "gone.txt"), "w") as fh:
|
||||||
|
fh.write("stale")
|
||||||
|
with open(os.path.join(received, "extra.txt"), "w") as fh:
|
||||||
|
fh.write("extra")
|
||||||
|
|
||||||
|
lst = _write_rel_list(b"a.txt\nprot/gone.txt\n")
|
||||||
|
flags = ["--files-from", lst, "--filter=- prot/", "--delete-missing-args",
|
||||||
|
"--delete"] + (["-m"] if mt else [])
|
||||||
|
result, _ = run_client(source, dest, flags=flags, port=server.port)
|
||||||
|
assert result.returncode == 0, f"delete-missing exclude sync failed: {result.stderr[:300]}"
|
||||||
|
assert not os.path.exists(os.path.join(received, "prot", "gone.txt")), \
|
||||||
|
"the explicit missing-arg deletion was blocked by exclusion protection"
|
||||||
|
assert os.path.isfile(os.path.join(received, "prot", "kept.txt")), \
|
||||||
|
"the excluded-but-present destination file must stay (default protection)"
|
||||||
|
assert not os.path.exists(os.path.join(received, "extra.txt")), \
|
||||||
|
"--delete did not remove the unrelated extra"
|
||||||
|
assert os.path.isfile(os.path.join(received, "a.txt"))
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("mt", [False, True])
|
||||||
|
def test_delete_missing_args_with_delete_before(self, mt):
|
||||||
|
"""--delete-before (early delete timing) composes with --delete-missing-args:
|
||||||
|
the exact-path deletions commit with the early manifest, before data, and
|
||||||
|
--delete-before implies --delete (so unrelated extras go too)."""
|
||||||
|
source = self._make_source("mg_early_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "mg_early_dst")
|
||||||
|
clean_dir(dest)
|
||||||
|
with open(os.path.join(dest, "gone.txt"), "w") as fh:
|
||||||
|
fh.write("stale")
|
||||||
|
with open(os.path.join(dest, "extra.txt"), "w") as fh:
|
||||||
|
fh.write("extra")
|
||||||
|
with ServerManager() as server:
|
||||||
|
server.start(extra_args=["--allow-delete"])
|
||||||
|
lst = _write_rel_list(b"a.txt\ngone.txt\ngone2.txt\n")
|
||||||
|
flags = ["--files-from", lst, "-R", "--delete-missing-args", "--delete-before"] + \
|
||||||
|
(["-m"] if mt else [])
|
||||||
|
result, _ = run_client(source, dest, flags=flags, port=server.port)
|
||||||
|
assert result.returncode == 0, f"early delete-missing sync failed: {result.stderr[:300]}"
|
||||||
|
assert not os.path.exists(os.path.join(dest, "gone.txt")), \
|
||||||
|
"early timing did not remove the missing-arg mirror"
|
||||||
|
assert os.path.isfile(os.path.join(dest, "a.txt")), "a.txt was not transferred"
|
||||||
|
assert not os.path.exists(os.path.join(dest, "extra.txt")), \
|
||||||
|
"--delete-before implies --delete: unrelated extras must go"
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("mt", [False, True])
|
||||||
|
@pytest.mark.parametrize("relative", [False, True])
|
||||||
|
def test_delete_missing_deep_entry_with_absent_parent(self, mt, relative):
|
||||||
|
"""A missing entry whose destination mirror's parent directory does not
|
||||||
|
exist is a no-op (nothing to delete), never a run failure: the
|
||||||
|
exact-path deletions must not abort the --delete extras walk. Covers
|
||||||
|
the -R bare-relative layout and the full source-mirror layout."""
|
||||||
|
source = self._make_source("mg_deep_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "mg_deep_dst")
|
||||||
|
clean_dir(dest)
|
||||||
|
rel_flags = ["-R"] if relative else []
|
||||||
|
with ServerManager() as server:
|
||||||
|
server.start(extra_args=["--allow-delete"])
|
||||||
|
if relative:
|
||||||
|
target_root = dest
|
||||||
|
else:
|
||||||
|
# Non-relative layout: seed a.txt so the receive-root mirror
|
||||||
|
# tree exists (its sub/ sibling deliberately does not).
|
||||||
|
seed = _write_rel_list(b"a.txt\n")
|
||||||
|
result, _ = run_client(source, dest,
|
||||||
|
flags=["--files-from", seed] + rel_flags,
|
||||||
|
port=server.port)
|
||||||
|
assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}"
|
||||||
|
target_root = get_dest_received_dir(dest, source)
|
||||||
|
assert os.path.isfile(os.path.join(target_root, "a.txt"))
|
||||||
|
with open(os.path.join(target_root, "extra.txt"), "w") as fh:
|
||||||
|
fh.write("extra")
|
||||||
|
|
||||||
|
lst = _write_rel_list(b"a.txt\nsub/gone.txt\n")
|
||||||
|
flags = ["--files-from", lst, "--delete-missing-args", "--delete"] + rel_flags + \
|
||||||
|
(["-m"] if mt else [])
|
||||||
|
result, _ = run_client(source, dest, flags=flags, port=server.port)
|
||||||
|
assert result.returncode == 0, \
|
||||||
|
f"deep missing-entry sync failed: {result.stderr[:300]}"
|
||||||
|
assert _read_file(os.path.join(target_root, "a.txt")) == b"a\n"
|
||||||
|
assert not os.path.exists(os.path.join(target_root, "extra.txt")), \
|
||||||
|
"--delete extras walk was aborted by the absent-parent missing entry"
|
||||||
|
assert not os.path.exists(os.path.join(target_root, "sub")), \
|
||||||
|
"the absent parent directory of the missing entry was created"
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("mt", [False, True])
|
||||||
|
def test_dirs_missing_entry_skipped_in_scanner(self, shared_server, mt):
|
||||||
|
"""--dirs + --files-from: a listed-but-missing entry is skipped in the
|
||||||
|
--dirs generator (which would otherwise hard-fail), transferring the
|
||||||
|
rest of the list."""
|
||||||
|
source = self._make_source("mg_dirs_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "mg_dirs_dst")
|
||||||
|
clean_dir(dest)
|
||||||
|
lst = _write_rel_list(b"a.txt\ngone.txt\n")
|
||||||
|
flags = ["--files-from", lst, "--dirs", "-R", "--ignore-missing-args"] + \
|
||||||
|
(["-m"] if mt else [])
|
||||||
|
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"--dirs ignore-missing sync failed: {result.stderr[:300]}"
|
||||||
|
assert _read_file(os.path.join(dest, "a.txt")) == b"a\n", \
|
||||||
|
"the listed present file was not transferred"
|
||||||
|
assert not os.path.exists(os.path.join(dest, "gone.txt")), \
|
||||||
|
"a directory/file was created for the missing --dirs entry"
|
||||||
|
|
||||||
|
|
||||||
class TestNoImpliedDirs:
|
class TestNoImpliedDirs:
|
||||||
"""--no-implied-dirs (only meaningful with -R + --files-from) refuses to
|
"""--no-implied-dirs (only meaningful with -R + --files-from) refuses to
|
||||||
place a listed file whose parent directory is not itself listed."""
|
place a listed file whose parent directory is not itself listed."""
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
#include "test_array_list.h"
|
#include "test_array_list.h"
|
||||||
#include "test_chunk.h"
|
#include "test_chunk.h"
|
||||||
#include "test_change_list.h"
|
#include "test_change_list.h"
|
||||||
|
#include "test_checksum.h"
|
||||||
#include "test_client_cli.h"
|
#include "test_client_cli.h"
|
||||||
#include "test_compression.h"
|
#include "test_compression.h"
|
||||||
#include "test_config.h"
|
#include "test_config.h"
|
||||||
@@ -46,6 +47,7 @@ int main() {
|
|||||||
RUN_TEST(test_config);
|
RUN_TEST(test_config);
|
||||||
RUN_TEST(test_compression);
|
RUN_TEST(test_compression);
|
||||||
RUN_TEST(test_scanner);
|
RUN_TEST(test_scanner);
|
||||||
|
RUN_TEST(test_checksum);
|
||||||
RUN_TEST(test_delta);
|
RUN_TEST(test_delta);
|
||||||
RUN_TEST(test_data);
|
RUN_TEST(test_data);
|
||||||
RUN_TEST(test_protocol);
|
RUN_TEST(test_protocol);
|
||||||
|
|||||||
@@ -0,0 +1,147 @@
|
|||||||
|
#include "test_checksum.h"
|
||||||
|
#include "checksum.h"
|
||||||
|
#include "test_utils.h"
|
||||||
|
#include <string.h>
|
||||||
|
|
||||||
|
/* Known xxHash64 vector (seed 0) for the empty string and a literal.
|
||||||
|
* The md5 vectors are the standard NIST/RFC1321 test strings. These pin the
|
||||||
|
* digest selection to genuinely distinct algorithm outputs so a --checksum-
|
||||||
|
* choice change is observable, not a silent no-op. */
|
||||||
|
|
||||||
|
static void test_checksum_xxh64_seed0() {
|
||||||
|
uint8_t out[CHECKSUM_MAX_DIGEST_LEN];
|
||||||
|
size_t len = 0;
|
||||||
|
EXPECT_TRUE(checksum_digest(CHECKSUM_ALGO_XXH64, 0, "hello", 5, out, sizeof(out), &len));
|
||||||
|
EXPECT_TRUE(len == (size_t)8);
|
||||||
|
/* Hard-coded: XXH64("hello", 5, 0). */
|
||||||
|
const uint8_t expect[8] = {0xa3, 0x6d, 0x9f, 0x88, 0x7d, 0x82, 0xc7, 0x26};
|
||||||
|
for (int i = 0; i < 8; i++)
|
||||||
|
EXPECT_EQ_INT(out[i], expect[i]);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_checksum_xxh64_empty() {
|
||||||
|
uint8_t out[CHECKSUM_MAX_DIGEST_LEN];
|
||||||
|
size_t len = 0;
|
||||||
|
EXPECT_TRUE(checksum_digest(CHECKSUM_ALGO_XXH64, 0, "", 0, out, sizeof(out), &len));
|
||||||
|
EXPECT_TRUE(len == (size_t)8);
|
||||||
|
/* XXH64("", 0, 0). */
|
||||||
|
const uint8_t expect[8] = {0x99, 0xe9, 0xd8, 0x51, 0x37, 0xdb, 0x46, 0xef};
|
||||||
|
for (int i = 0; i < 8; i++)
|
||||||
|
EXPECT_EQ_INT(out[i], expect[i]);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* A nonzero seed must change the xxh64 digest: the algorithm is genuinely
|
||||||
|
* seed-aware, deterministic, and distinct from seed 0. */
|
||||||
|
static void test_checksum_xxh64_seed_changes_digest() {
|
||||||
|
uint8_t a[CHECKSUM_MAX_DIGEST_LEN], b[CHECKSUM_MAX_DIGEST_LEN];
|
||||||
|
size_t alen = 0, blen = 0;
|
||||||
|
EXPECT_TRUE(checksum_digest(CHECKSUM_ALGO_XXH64, 7, "payload", 7, a, sizeof(a), &alen));
|
||||||
|
EXPECT_TRUE(checksum_digest(CHECKSUM_ALGO_XXH64, 0, "payload", 7, b, sizeof(b), &blen));
|
||||||
|
EXPECT_TRUE(alen == blen);
|
||||||
|
EXPECT_TRUE(memcmp(a, b, alen) != 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_checksum_xxh64_seed_deterministic() {
|
||||||
|
uint8_t a[CHECKSUM_MAX_DIGEST_LEN], b[CHECKSUM_MAX_DIGEST_LEN];
|
||||||
|
size_t alen = 0, blen = 0;
|
||||||
|
EXPECT_TRUE(checksum_digest(CHECKSUM_ALGO_XXH64, 12345, "same", 4, a, sizeof(a), &alen));
|
||||||
|
EXPECT_TRUE(checksum_digest(CHECKSUM_ALGO_XXH64, 12345, "same", 4, b, sizeof(b), &blen));
|
||||||
|
EXPECT_TRUE(alen == blen);
|
||||||
|
EXPECT_TRUE(memcmp(a, b, alen) == 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_checksum_md5_vectors() {
|
||||||
|
uint8_t out[CHECKSUM_MAX_DIGEST_LEN];
|
||||||
|
size_t len = 0;
|
||||||
|
EXPECT_TRUE(checksum_digest(CHECKSUM_ALGO_MD5, 0, "", 0, out, sizeof(out), &len));
|
||||||
|
EXPECT_TRUE(len == (size_t)16);
|
||||||
|
const uint8_t expect_empty[16] = {0xd4, 0x1d, 0x8c, 0xd9, 0x8f, 0x00, 0xb2, 0x04,
|
||||||
|
0xe9, 0x80, 0x09, 0x98, 0xec, 0xf8, 0x42, 0x7e};
|
||||||
|
EXPECT_TRUE(memcmp(out, expect_empty, 16) == 0);
|
||||||
|
|
||||||
|
/* MD5("abc") */
|
||||||
|
EXPECT_TRUE(checksum_digest(CHECKSUM_ALGO_MD5, 0, "abc", 3, out, sizeof(out), &len));
|
||||||
|
const uint8_t expect_abc[16] = {0x90, 0x01, 0x50, 0x98, 0x3c, 0xd2, 0x4f, 0xb0,
|
||||||
|
0xd6, 0x96, 0x3f, 0x7d, 0x28, 0xe1, 0x7f, 0x72};
|
||||||
|
EXPECT_TRUE(memcmp(out, expect_abc, 16) == 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* md5 is 16 bytes and differs from the 8-byte xxh64 for the same input, so the
|
||||||
|
* choice is observably different both in length and in content. */
|
||||||
|
static void test_checksum_algo_lengths_distinct() {
|
||||||
|
EXPECT_EQ_INT((int)checksum_digest_len(CHECKSUM_ALGO_XXH64), 8);
|
||||||
|
EXPECT_EQ_INT((int)checksum_digest_len(CHECKSUM_ALGO_MD5), 16);
|
||||||
|
|
||||||
|
uint8_t x[CHECKSUM_MAX_DIGEST_LEN], m[CHECKSUM_MAX_DIGEST_LEN];
|
||||||
|
size_t xl = 0, ml = 0;
|
||||||
|
EXPECT_TRUE(checksum_digest(CHECKSUM_ALGO_XXH64, 0, "same content", 12, x, sizeof(x), &xl));
|
||||||
|
EXPECT_TRUE(checksum_digest(CHECKSUM_ALGO_MD5, 0, "same content", 12, m, sizeof(m), &ml));
|
||||||
|
EXPECT_TRUE(xl == (size_t)8);
|
||||||
|
EXPECT_TRUE(ml == (size_t)16);
|
||||||
|
EXPECT_TRUE(memcmp(x, m, 8) != 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* md5 has no seed: two distinct seeds give the same md5 digest (documented);
|
||||||
|
* the seed is only honored by xxh64 and the delta block hash (low 32 bits). */
|
||||||
|
static void test_checksum_md5_seed_ignored() {
|
||||||
|
uint8_t a[CHECKSUM_MAX_DIGEST_LEN], b[CHECKSUM_MAX_DIGEST_LEN];
|
||||||
|
size_t alen = 0, blen = 0;
|
||||||
|
EXPECT_TRUE(checksum_digest(CHECKSUM_ALGO_MD5, 0, "data", 4, a, sizeof(a), &alen));
|
||||||
|
EXPECT_TRUE(checksum_digest(CHECKSUM_ALGO_MD5, 99, "data", 4, b, sizeof(b), &blen));
|
||||||
|
EXPECT_TRUE(memcmp(a, b, alen) == 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_checksum_algo_name_mapping() {
|
||||||
|
EXPECT_EQ_INT(checksum_algo_from_name("xxh64"), (int)CHECKSUM_ALGO_XXH64);
|
||||||
|
EXPECT_EQ_INT(checksum_algo_from_name("XXH64"), (int)CHECKSUM_ALGO_XXH64);
|
||||||
|
EXPECT_EQ_INT(checksum_algo_from_name("xxhash"), (int)CHECKSUM_ALGO_XXH64);
|
||||||
|
EXPECT_EQ_INT(checksum_algo_from_name("XXHASH"), (int)CHECKSUM_ALGO_XXH64);
|
||||||
|
EXPECT_EQ_INT(checksum_algo_from_name("md5"), (int)CHECKSUM_ALGO_MD5);
|
||||||
|
EXPECT_EQ_INT(checksum_algo_from_name("MD5"), (int)CHECKSUM_ALGO_MD5);
|
||||||
|
EXPECT_TRUE(checksum_algo_from_name("sha256") < 0);
|
||||||
|
EXPECT_TRUE(checksum_algo_from_name("crc32") < 0);
|
||||||
|
EXPECT_TRUE(checksum_algo_from_name("none") < 0);
|
||||||
|
EXPECT_TRUE(checksum_algo_from_name("xxh3") < 0);
|
||||||
|
EXPECT_TRUE(checksum_algo_from_name("") < 0);
|
||||||
|
EXPECT_TRUE(checksum_algo_from_name(NULL) < 0);
|
||||||
|
|
||||||
|
EXPECT_TRUE(checksum_algo_valid((int)CHECKSUM_ALGO_XXH64));
|
||||||
|
EXPECT_TRUE(checksum_algo_valid((int)CHECKSUM_ALGO_MD5));
|
||||||
|
EXPECT_FALSE(checksum_algo_valid(99));
|
||||||
|
EXPECT_EQ_STR(checksum_algo_name(CHECKSUM_ALGO_XXH64), "xxh64");
|
||||||
|
EXPECT_EQ_STR(checksum_algo_name(CHECKSUM_ALGO_MD5), "md5");
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_checksum_truncated_buffer_rejected() {
|
||||||
|
uint8_t small[4];
|
||||||
|
size_t len = 0;
|
||||||
|
/* The digest cannot fit in a 4-byte buffer. */
|
||||||
|
EXPECT_FALSE(checksum_digest(CHECKSUM_ALGO_XXH64, 0, "x", 1, small, sizeof(small), &len));
|
||||||
|
EXPECT_FALSE(checksum_digest(CHECKSUM_ALGO_MD5, 0, "x", 1, small, sizeof(small), &len));
|
||||||
|
EXPECT_FALSE(checksum_digest(CHECKSUM_ALGO_XXH64, 0, NULL, 5, small, sizeof(small), &len));
|
||||||
|
EXPECT_FALSE(checksum_digest(CHECKSUM_ALGO_XXH64, 0, "x", 1, NULL, 0, &len));
|
||||||
|
EXPECT_FALSE(checksum_digest(CHECKSUM_ALGO_XXH64, 0, "x", 1, small, sizeof(small), NULL));
|
||||||
|
}
|
||||||
|
|
||||||
|
/* A NULL data pointer with size 0 is the empty input, not an error. */
|
||||||
|
static void test_checksum_null_empty_digest() {
|
||||||
|
uint8_t a[CHECKSUM_MAX_DIGEST_LEN], b[CHECKSUM_MAX_DIGEST_LEN];
|
||||||
|
size_t alen = 0, blen = 0;
|
||||||
|
EXPECT_TRUE(checksum_digest(CHECKSUM_ALGO_XXH64, 0, NULL, 0, a, sizeof(a), &alen));
|
||||||
|
EXPECT_TRUE(checksum_digest(CHECKSUM_ALGO_XXH64, 0, "", 0, b, sizeof(b), &blen));
|
||||||
|
EXPECT_TRUE(alen == blen);
|
||||||
|
EXPECT_TRUE(memcmp(a, b, alen) == 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
void test_checksum(void) {
|
||||||
|
test_checksum_xxh64_seed0();
|
||||||
|
test_checksum_xxh64_empty();
|
||||||
|
test_checksum_xxh64_seed_changes_digest();
|
||||||
|
test_checksum_xxh64_seed_deterministic();
|
||||||
|
test_checksum_md5_vectors();
|
||||||
|
test_checksum_algo_lengths_distinct();
|
||||||
|
test_checksum_md5_seed_ignored();
|
||||||
|
test_checksum_algo_name_mapping();
|
||||||
|
test_checksum_truncated_buffer_rejected();
|
||||||
|
test_checksum_null_empty_digest();
|
||||||
|
}
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
#ifndef TEST_CHECKSUM_H
|
||||||
|
#define TEST_CHECKSUM_H
|
||||||
|
|
||||||
|
void test_checksum(void);
|
||||||
|
|
||||||
|
#endif
|
||||||
+244
-5
@@ -1,4 +1,5 @@
|
|||||||
#include "test_client_cli.h"
|
#include "test_client_cli.h"
|
||||||
|
#include "checksum.h"
|
||||||
#include "client_validation.h"
|
#include "client_validation.h"
|
||||||
#include "chmod.h"
|
#include "chmod.h"
|
||||||
#include "config.h"
|
#include "config.h"
|
||||||
@@ -773,8 +774,6 @@ static void test_parse_args_rejects_unimplemented_options() {
|
|||||||
"--xattrs",
|
"--xattrs",
|
||||||
"-D",
|
"-D",
|
||||||
"--devices",
|
"--devices",
|
||||||
"--append",
|
|
||||||
"--append-verify",
|
|
||||||
"--delete-excluded",
|
"--delete-excluded",
|
||||||
"--max-delete",
|
"--max-delete",
|
||||||
"--prune-empty-dirs",
|
"--prune-empty-dirs",
|
||||||
@@ -787,8 +786,7 @@ static void test_parse_args_rejects_unimplemented_options() {
|
|||||||
"--ipv4",
|
"--ipv4",
|
||||||
"--daemon",
|
"--daemon",
|
||||||
"--config",
|
"--config",
|
||||||
"--server",
|
"--server"};
|
||||||
"--checksum-choice"};
|
|
||||||
|
|
||||||
for (size_t i = 0; i < sizeof(options) / sizeof(options[0]); i++) {
|
for (size_t i = 0; i < sizeof(options) / sizeof(options[0]); i++) {
|
||||||
Config* cfg = config_create();
|
Config* cfg = config_create();
|
||||||
@@ -953,7 +951,8 @@ static void test_parse_args_no_preserve_blocks_implicit_metadata() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Checksum-choice spellings are recognized and rejected until algorithms are implemented. */
|
/* --checksum-choice and its --cc alias select the whole-file digest algorithm
|
||||||
|
(default xxh64; both "xxh64" and the rsync "xxhash" spelling accepted). */
|
||||||
static void test_parse_args_checksum_choice_aliases() {
|
static void test_parse_args_checksum_choice_aliases() {
|
||||||
static const char* const options[] = {"--checksum-choice", "--cc"};
|
static const char* const options[] = {"--checksum-choice", "--cc"};
|
||||||
|
|
||||||
@@ -962,12 +961,88 @@ static void test_parse_args_checksum_choice_aliases() {
|
|||||||
char* argv[] = {"fastsync", (char*)options[i], "xxh64", "/src", "/dst"};
|
char* argv[] = {"fastsync", (char*)options[i], "xxh64", "/src", "/dst"};
|
||||||
int positional_args[2];
|
int positional_args[2];
|
||||||
int positional_count = 0;
|
int positional_count = 0;
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0);
|
||||||
|
EXPECT_EQ_INT(cfg->checksum_algo, (int)CHECKSUM_ALGO_XXH64);
|
||||||
|
config_delete(cfg);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Both the "--checksum-choice=ALG" and "--cc=ALG" inline forms parse. */
|
||||||
|
static void test_parse_args_checksum_choice_equals_forms() {
|
||||||
|
Config* cfg = config_create();
|
||||||
|
char* argv[] = {"fastsync", "--checksum-choice=md5", "/src", "/dst"};
|
||||||
|
int positional_args[2];
|
||||||
|
int positional_count = 0;
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
|
||||||
|
EXPECT_EQ_INT(cfg->checksum_algo, (int)CHECKSUM_ALGO_MD5);
|
||||||
|
config_delete(cfg);
|
||||||
|
|
||||||
|
cfg = config_create();
|
||||||
|
char* argv2[] = {"fastsync", "--cc=xxhash", "/src", "/dst"};
|
||||||
|
positional_count = 0;
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 4, argv2, positional_args, &positional_count), 0);
|
||||||
|
EXPECT_EQ_INT(cfg->checksum_algo, (int)CHECKSUM_ALGO_XXH64);
|
||||||
|
config_delete(cfg);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* An algorithm FastSync does not support must be rejected, never a silent
|
||||||
|
no-op. */
|
||||||
|
static void test_parse_args_checksum_choice_rejects_unsupported() {
|
||||||
|
static const char* const bad[] = {"md4", "sha256", "crc32", "none", "bogus"};
|
||||||
|
for (size_t i = 0; i < sizeof(bad) / sizeof(bad[0]); i++) {
|
||||||
|
Config* cfg = config_create();
|
||||||
|
char* argv[] = {"fastsync", "--checksum-choice", (char*)bad[i], "/src", "/dst"};
|
||||||
|
int positional_args[2];
|
||||||
|
int positional_count = 0;
|
||||||
EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), -1);
|
EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), -1);
|
||||||
config_delete(cfg);
|
config_delete(cfg);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* --checksum-seed parses as a 64-bit non-negative integer (space and = forms);
|
||||||
|
invalid values are rejected. */
|
||||||
|
static void test_parse_args_checksum_seed() {
|
||||||
|
Config* cfg = config_create();
|
||||||
|
char* argv[] = {"fastsync", "--checksum-seed=42", "/src", "/dst"};
|
||||||
|
int positional_args[2];
|
||||||
|
int positional_count = 0;
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
|
||||||
|
EXPECT_TRUE(cfg->checksum_seed == 42ULL);
|
||||||
|
config_delete(cfg);
|
||||||
|
|
||||||
|
cfg = config_create();
|
||||||
|
char* argv2[] = {"fastsync", "--checksum-seed", "12345", "/src", "/dst"};
|
||||||
|
positional_count = 0;
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 5, argv2, positional_args, &positional_count), 0);
|
||||||
|
EXPECT_TRUE(cfg->checksum_seed == 12345ULL);
|
||||||
|
config_delete(cfg);
|
||||||
|
|
||||||
|
/* 0 is a valid (and default) seed. */
|
||||||
|
cfg = config_create();
|
||||||
|
char* argv3[] = {"fastsync", "--checksum-seed=0", "/src", "/dst"};
|
||||||
|
positional_count = 0;
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 4, argv3, positional_args, &positional_count), 0);
|
||||||
|
EXPECT_TRUE(cfg->checksum_seed == 0ULL);
|
||||||
|
config_delete(cfg);
|
||||||
|
|
||||||
|
/* Non-numeric and negative seeds are rejected. */
|
||||||
|
static const char* const bad[] = {"abc", "-5", "1.5", ""};
|
||||||
|
for (size_t i = 0; i < sizeof(bad) / sizeof(bad[0]); i++) {
|
||||||
|
cfg = config_create();
|
||||||
|
char* argv4[] = {"fastsync", "--checksum-seed", (char*)bad[i], "/src", "/dst"};
|
||||||
|
positional_count = 0;
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 5, argv4, positional_args, &positional_count), -1);
|
||||||
|
config_delete(cfg);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Missing value is rejected. */
|
||||||
|
cfg = config_create();
|
||||||
|
char* argv5[] = {"fastsync", "--checksum-seed"};
|
||||||
|
positional_count = 0;
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 2, argv5, positional_args, &positional_count), -1);
|
||||||
|
config_delete(cfg);
|
||||||
|
}
|
||||||
|
|
||||||
static void test_parse_args_rejects_unsafe_negation() {
|
static void test_parse_args_rejects_unsafe_negation() {
|
||||||
static const char* const options[] = {"--no-archive", "--no-timeout", "--no-unknown"};
|
static const char* const options[] = {"--no-archive", "--no-timeout", "--no-unknown"};
|
||||||
for (size_t i = 0; i < sizeof(options) / sizeof(options[0]); i++) {
|
for (size_t i = 0; i < sizeof(options) / sizeof(options[0]); i++) {
|
||||||
@@ -1855,8 +1930,168 @@ static void test_parse_args_max_delete_inert_without_delete() {
|
|||||||
config_delete(cfg);
|
config_delete(cfg);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* --ignore-missing-args / --delete-missing-args parse onto their config fields.
|
||||||
|
* --delete-missing-args implies --ignore-missing-args (order-independent),
|
||||||
|
* does NOT imply --delete (rsync: independent of other delete processing), and
|
||||||
|
* the config stays valid in every combination. */
|
||||||
|
static void test_parse_args_missing_args_flags() {
|
||||||
|
Config* cfg = config_create();
|
||||||
|
int positional_args[2];
|
||||||
|
int positional_count = 0;
|
||||||
|
char* argv[] = {"fastsync", "--ignore-missing-args", "/src", "/dst"};
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
|
||||||
|
EXPECT_TRUE(cfg->ignore_missing_args);
|
||||||
|
EXPECT_FALSE(cfg->delete_missing_args);
|
||||||
|
EXPECT_FALSE(cfg->use_delete);
|
||||||
|
cfg->send_directory = str_dup("/src");
|
||||||
|
cfg->receive_root_directory = str_dup("/dst");
|
||||||
|
EXPECT_TRUE(validate_config(cfg));
|
||||||
|
config_delete(cfg);
|
||||||
|
|
||||||
|
cfg = config_create();
|
||||||
|
positional_count = 0;
|
||||||
|
char* argv2[] = {"fastsync", "--delete-missing-args", "/src", "/dst"};
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 4, argv2, positional_args, &positional_count), 0);
|
||||||
|
EXPECT_TRUE(cfg->delete_missing_args);
|
||||||
|
EXPECT_TRUE(cfg->ignore_missing_args);
|
||||||
|
EXPECT_FALSE(cfg->use_delete);
|
||||||
|
cfg->send_directory = str_dup("/src");
|
||||||
|
cfg->receive_root_directory = str_dup("/dst");
|
||||||
|
EXPECT_TRUE(validate_config(cfg));
|
||||||
|
config_delete(cfg);
|
||||||
|
|
||||||
|
/* The implication is order-independent: even with the explicit flag first. */
|
||||||
|
cfg = config_create();
|
||||||
|
positional_count = 0;
|
||||||
|
char* argv3[] = {"fastsync", "--ignore-missing-args", "--delete-missing-args", "/src", "/dst"};
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 5, argv3, positional_args, &positional_count), 0);
|
||||||
|
EXPECT_TRUE(cfg->ignore_missing_args);
|
||||||
|
EXPECT_TRUE(cfg->delete_missing_args);
|
||||||
|
config_delete(cfg);
|
||||||
|
|
||||||
|
/* --delete-missing-args composes with --delete (both active) and with a
|
||||||
|
delete-timing flag (which implies --delete); timing stays valid. */
|
||||||
|
cfg = config_create();
|
||||||
|
positional_count = 0;
|
||||||
|
char* argv4[] = {"fastsync", "--delete", "--delete-missing-args", "/src", "/dst"};
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 5, argv4, positional_args, &positional_count), 0);
|
||||||
|
EXPECT_TRUE(cfg->use_delete);
|
||||||
|
EXPECT_TRUE(cfg->delete_missing_args);
|
||||||
|
EXPECT_TRUE(cfg->ignore_missing_args);
|
||||||
|
cfg->send_directory = str_dup("/src");
|
||||||
|
cfg->receive_root_directory = str_dup("/dst");
|
||||||
|
EXPECT_TRUE(validate_config(cfg));
|
||||||
|
config_delete(cfg);
|
||||||
|
|
||||||
|
cfg = config_create();
|
||||||
|
positional_count = 0;
|
||||||
|
char* argv5[] = {"fastsync", "--delete-before", "--delete-missing-args", "/src", "/dst"};
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 5, argv5, positional_args, &positional_count), 0);
|
||||||
|
EXPECT_TRUE(cfg->use_delete);
|
||||||
|
EXPECT_TRUE(cfg->delete_before);
|
||||||
|
EXPECT_TRUE(cfg->delete_missing_args);
|
||||||
|
cfg->send_directory = str_dup("/src");
|
||||||
|
cfg->receive_root_directory = str_dup("/dst");
|
||||||
|
EXPECT_TRUE(validate_config(cfg));
|
||||||
|
config_delete(cfg);
|
||||||
|
}
|
||||||
|
/* --append is accepted and implies the per-file incremental check a tail resume
|
||||||
|
* needs; it validates cleanly on its own. */
|
||||||
|
static void test_parse_args_append() {
|
||||||
|
Config* cfg = config_create();
|
||||||
|
cfg->send_directory = str_dup("/src");
|
||||||
|
cfg->receive_root_directory = str_dup("/dst");
|
||||||
|
char* argv[] = {"fastsync", "--append", "/src", "/dst"};
|
||||||
|
int positional_args[2];
|
||||||
|
int positional_count = 0;
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
|
||||||
|
EXPECT_TRUE(cfg->append);
|
||||||
|
EXPECT_FALSE(cfg->append_verify);
|
||||||
|
EXPECT_TRUE(cfg->use_incremental);
|
||||||
|
EXPECT_TRUE(validate_config(cfg));
|
||||||
|
config_delete(cfg);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_parse_args_append_verify() {
|
||||||
|
Config* cfg = config_create();
|
||||||
|
cfg->send_directory = str_dup("/src");
|
||||||
|
cfg->receive_root_directory = str_dup("/dst");
|
||||||
|
char* argv[] = {"fastsync", "--append-verify", "/src", "/dst"};
|
||||||
|
int positional_args[2];
|
||||||
|
int positional_count = 0;
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
|
||||||
|
EXPECT_TRUE(cfg->append_verify);
|
||||||
|
EXPECT_FALSE(cfg->append);
|
||||||
|
EXPECT_TRUE(cfg->use_incremental);
|
||||||
|
EXPECT_TRUE(validate_config(cfg));
|
||||||
|
config_delete(cfg);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Both spellings are accepted; the safer --append-verify semantics win on the
|
||||||
|
* wire (the sender checks append_verify first), so neither flag is silently
|
||||||
|
* dropped but the run is still valid. */
|
||||||
|
static void test_parse_args_append_both() {
|
||||||
|
Config* cfg = config_create();
|
||||||
|
cfg->send_directory = str_dup("/src");
|
||||||
|
cfg->receive_root_directory = str_dup("/dst");
|
||||||
|
char* argv[] = {"fastsync", "--append", "--append-verify", "/src", "/dst"};
|
||||||
|
int positional_args[2];
|
||||||
|
int positional_count = 0;
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0);
|
||||||
|
EXPECT_TRUE(cfg->append);
|
||||||
|
EXPECT_TRUE(cfg->append_verify);
|
||||||
|
EXPECT_TRUE(validate_config(cfg));
|
||||||
|
config_delete(cfg);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_validate_config_append_rejects_chunk_serialization() {
|
||||||
|
Config* cfg = config_create();
|
||||||
|
char* argv[] = {"fastsync", "--append", "-s", "/src", "/dst"};
|
||||||
|
int positional_args[2];
|
||||||
|
int positional_count = 0;
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0);
|
||||||
|
EXPECT_FALSE(validate_config(cfg));
|
||||||
|
config_delete(cfg);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_validate_config_append_verify_rejects_chunk_serialization() {
|
||||||
|
Config* cfg = config_create();
|
||||||
|
char* argv[] = {"fastsync", "--append-verify", "-s", "/src", "/dst"};
|
||||||
|
int positional_args[2];
|
||||||
|
int positional_count = 0;
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0);
|
||||||
|
EXPECT_FALSE(validate_config(cfg));
|
||||||
|
config_delete(cfg);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_validate_config_append_rejects_whole_file() {
|
||||||
|
Config* cfg = config_create();
|
||||||
|
char* argv[] = {"fastsync", "--append", "-W", "/src", "/dst"};
|
||||||
|
int positional_args[2];
|
||||||
|
int positional_count = 0;
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0);
|
||||||
|
EXPECT_FALSE(validate_config(cfg));
|
||||||
|
config_delete(cfg);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_validate_config_append_verify_rejects_whole_file() {
|
||||||
|
Config* cfg = config_create();
|
||||||
|
char* argv[] = {"fastsync", "--append-verify", "-W", "/src", "/dst"};
|
||||||
|
int positional_args[2];
|
||||||
|
int positional_count = 0;
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0);
|
||||||
|
EXPECT_FALSE(validate_config(cfg));
|
||||||
|
config_delete(cfg);
|
||||||
|
}
|
||||||
void test_client_cli() {
|
void test_client_cli() {
|
||||||
test_validate_config_required_paths();
|
test_validate_config_required_paths();
|
||||||
|
test_parse_args_append();
|
||||||
|
test_parse_args_append_verify();
|
||||||
|
test_parse_args_append_both();
|
||||||
|
test_validate_config_append_rejects_chunk_serialization();
|
||||||
|
test_validate_config_append_verify_rejects_chunk_serialization();
|
||||||
|
test_validate_config_append_rejects_whole_file();
|
||||||
|
test_validate_config_append_verify_rejects_whole_file();
|
||||||
test_validate_config_incompatible_options();
|
test_validate_config_incompatible_options();
|
||||||
test_validate_config_tls_requirements();
|
test_validate_config_tls_requirements();
|
||||||
test_validate_config_delta_sendfile_constraints();
|
test_validate_config_delta_sendfile_constraints();
|
||||||
@@ -1943,6 +2178,9 @@ void test_client_cli() {
|
|||||||
test_parse_args_log_file_format();
|
test_parse_args_log_file_format();
|
||||||
test_parse_args_checksum_choice_aliases();
|
test_parse_args_checksum_choice_aliases();
|
||||||
test_parse_args_checksum_choice_requires_value();
|
test_parse_args_checksum_choice_requires_value();
|
||||||
|
test_parse_args_checksum_choice_equals_forms();
|
||||||
|
test_parse_args_checksum_choice_rejects_unsupported();
|
||||||
|
test_parse_args_checksum_seed();
|
||||||
test_parse_args_temp_dir();
|
test_parse_args_temp_dir();
|
||||||
test_parse_args_delay_updates();
|
test_parse_args_delay_updates();
|
||||||
test_validate_config_delay_updates_rejects_inplace();
|
test_validate_config_delay_updates_rejects_inplace();
|
||||||
@@ -1956,4 +2194,5 @@ void test_client_cli() {
|
|||||||
test_parse_args_delete_policy_flags();
|
test_parse_args_delete_policy_flags();
|
||||||
test_parse_args_delete_policy_invalid_values();
|
test_parse_args_delete_policy_invalid_values();
|
||||||
test_parse_args_max_delete_inert_without_delete();
|
test_parse_args_max_delete_inert_without_delete();
|
||||||
|
test_parse_args_missing_args_flags();
|
||||||
}
|
}
|
||||||
@@ -619,6 +619,57 @@ static void test_config_delete_policy_wire_roundtrip() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* --delete-missing-args crosses the wire (the receiver executes the exact-path
|
||||||
|
deletions) while --ignore-missing-args is client-only: the receiver must
|
||||||
|
observe delete_missing_args unchanged and ignore_missing_args always false. */
|
||||||
|
static void test_config_delete_missing_args_wire_roundtrip() {
|
||||||
|
if (is_running_under_valgrind())
|
||||||
|
return;
|
||||||
|
|
||||||
|
struct {
|
||||||
|
bool delete_missing_args, ignore_missing_args;
|
||||||
|
} cases[] = {
|
||||||
|
{false, false},
|
||||||
|
{true, false},
|
||||||
|
{true, true},
|
||||||
|
};
|
||||||
|
for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
|
||||||
|
int p[2];
|
||||||
|
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
|
||||||
|
pid_t pid = fork();
|
||||||
|
if (pid == 0) {
|
||||||
|
close(p[1]);
|
||||||
|
io_set_fds(p[0], p[0]);
|
||||||
|
Config* recv = config_receive(p[0]);
|
||||||
|
bool ok = recv != NULL;
|
||||||
|
if (ok) {
|
||||||
|
ok = recv->delete_missing_args == cases[i].delete_missing_args &&
|
||||||
|
/* ignore_missing_args never crosses the wire. */
|
||||||
|
recv->ignore_missing_args == false;
|
||||||
|
}
|
||||||
|
config_delete(recv);
|
||||||
|
close(p[0]);
|
||||||
|
_exit(ok ? 0 : 1);
|
||||||
|
} else {
|
||||||
|
close(p[0]);
|
||||||
|
io_set_fds(p[1], p[1]);
|
||||||
|
Config* send_cfg = config_create();
|
||||||
|
EXPECT_NOT_NULL(send_cfg);
|
||||||
|
send_cfg->send_directory = str_dup("/src");
|
||||||
|
send_cfg->receive_root_directory = str_dup("/dst");
|
||||||
|
send_cfg->delete_missing_args = cases[i].delete_missing_args;
|
||||||
|
send_cfg->ignore_missing_args = cases[i].ignore_missing_args;
|
||||||
|
bool sent = config_send(p[1], send_cfg);
|
||||||
|
int status;
|
||||||
|
waitpid(pid, &status, 0);
|
||||||
|
close(p[1]);
|
||||||
|
config_delete(send_cfg);
|
||||||
|
EXPECT_TRUE(sent);
|
||||||
|
EXPECT_TRUE(WIFEXITED(status) && WEXITSTATUS(status) == 0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/* Basis-dir lists survive the config wire: each entry's type and path must
|
/* Basis-dir lists survive the config wire: each entry's type and path must
|
||||||
round-trip unchanged. */
|
round-trip unchanged. */
|
||||||
static void test_config_basis_roundtrip() {
|
static void test_config_basis_roundtrip() {
|
||||||
@@ -754,6 +805,109 @@ static void test_config_is_remote_dest() {
|
|||||||
EXPECT_TRUE(config_is_remote_dest("user@host:"));
|
EXPECT_TRUE(config_is_remote_dest("user@host:"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* The append-mode fields cross the wire unchanged: --append and --append-verify
|
||||||
|
are negotiated to the receiver so it knows to reply STATUS_APPEND on a
|
||||||
|
shorter destination. */
|
||||||
|
static void test_config_append_wire_roundtrip() {
|
||||||
|
struct {
|
||||||
|
bool append, append_verify;
|
||||||
|
} cases[] = {{true, false}, {false, true}, {true, true}, {false, false}};
|
||||||
|
if (is_running_under_valgrind())
|
||||||
|
return;
|
||||||
|
for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
|
||||||
|
int p[2];
|
||||||
|
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
|
||||||
|
pid_t pid = fork();
|
||||||
|
if (pid == 0) {
|
||||||
|
close(p[1]);
|
||||||
|
io_set_fds(p[0], p[0]);
|
||||||
|
Config* recv = config_receive(p[0]);
|
||||||
|
bool ok = recv != NULL;
|
||||||
|
if (ok)
|
||||||
|
ok = recv->append == cases[i].append && recv->append_verify == cases[i].append_verify;
|
||||||
|
config_delete(recv);
|
||||||
|
close(p[0]);
|
||||||
|
_exit(ok ? 0 : 1);
|
||||||
|
} else {
|
||||||
|
close(p[0]);
|
||||||
|
io_set_fds(p[1], p[1]);
|
||||||
|
Config* send_cfg = config_create();
|
||||||
|
EXPECT_NOT_NULL(send_cfg);
|
||||||
|
send_cfg->send_directory = str_dup("/src");
|
||||||
|
send_cfg->receive_root_directory = str_dup("/dst");
|
||||||
|
send_cfg->append = cases[i].append;
|
||||||
|
send_cfg->append_verify = cases[i].append_verify;
|
||||||
|
bool sent = config_send(p[1], send_cfg);
|
||||||
|
int status;
|
||||||
|
waitpid(pid, &status, 0);
|
||||||
|
close(p[1]);
|
||||||
|
config_delete(send_cfg);
|
||||||
|
EXPECT_TRUE(sent);
|
||||||
|
EXPECT_TRUE(WIFEXITED(status) && WEXITSTATUS(status) == 0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* --checksum-choice/--cc and --checksum-seed cross the wire intact so the
|
||||||
|
receiver hashes the on-disk old file with the same algorithm and seed. */
|
||||||
|
static void test_config_checksum_options_wire_roundtrip() {
|
||||||
|
struct {
|
||||||
|
int algo;
|
||||||
|
unsigned long long seed;
|
||||||
|
} cases[] = {
|
||||||
|
{CHECKSUM_ALGO_XXH64, 0},
|
||||||
|
{CHECKSUM_ALGO_XXH64, 42},
|
||||||
|
{CHECKSUM_ALGO_MD5, 7},
|
||||||
|
{CHECKSUM_ALGO_MD5, 0},
|
||||||
|
};
|
||||||
|
if (is_running_under_valgrind())
|
||||||
|
return;
|
||||||
|
for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
|
||||||
|
int p[2];
|
||||||
|
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
|
||||||
|
pid_t pid = fork();
|
||||||
|
if (pid == 0) {
|
||||||
|
close(p[1]);
|
||||||
|
io_set_fds(p[0], p[0]);
|
||||||
|
Config* recv = config_receive(p[0]);
|
||||||
|
bool ok = recv != NULL && recv->checksum_algo == cases[i].algo &&
|
||||||
|
recv->checksum_seed == cases[i].seed;
|
||||||
|
config_delete(recv);
|
||||||
|
close(p[0]);
|
||||||
|
_exit(ok ? 0 : 1);
|
||||||
|
} else {
|
||||||
|
close(p[0]);
|
||||||
|
io_set_fds(p[1], p[1]);
|
||||||
|
Config* send_cfg = config_create();
|
||||||
|
EXPECT_NOT_NULL(send_cfg);
|
||||||
|
send_cfg->send_directory = str_dup("/src");
|
||||||
|
send_cfg->receive_root_directory = str_dup("/dst");
|
||||||
|
send_cfg->checksum_algo = cases[i].algo;
|
||||||
|
send_cfg->checksum_seed = cases[i].seed;
|
||||||
|
bool sent = config_send(p[1], send_cfg);
|
||||||
|
int status;
|
||||||
|
waitpid(pid, &status, 0);
|
||||||
|
close(p[1]);
|
||||||
|
config_delete(send_cfg);
|
||||||
|
EXPECT_TRUE(sent);
|
||||||
|
EXPECT_TRUE(WIFEXITED(status) && WEXITSTATUS(status) == 0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* An out-of-range algorithm id on the wire must be rejected on receive, never
|
||||||
|
accepted as-is (prevents mixing unsupported digests on a path). */
|
||||||
|
static void test_config_receive_rejects_invalid_checksum_algo() {
|
||||||
|
if (is_running_under_valgrind())
|
||||||
|
return;
|
||||||
|
Config* c = config_create();
|
||||||
|
EXPECT_NOT_NULL(c);
|
||||||
|
c->send_directory = str_dup("/src");
|
||||||
|
c->receive_root_directory = str_dup("/dst");
|
||||||
|
c->checksum_algo = 99;
|
||||||
|
EXPECT_FALSE(roundtrip_config_ok(c));
|
||||||
|
config_delete(c);
|
||||||
|
}
|
||||||
void test_config() {
|
void test_config() {
|
||||||
test_config_lifecycle();
|
test_config_lifecycle();
|
||||||
test_config_ssh_dest();
|
test_config_ssh_dest();
|
||||||
@@ -771,9 +925,13 @@ void test_config() {
|
|||||||
test_config_delete_timing_wire_roundtrip();
|
test_config_delete_timing_wire_roundtrip();
|
||||||
test_config_delete_timing_conflict_rejected();
|
test_config_delete_timing_conflict_rejected();
|
||||||
test_config_delete_policy_wire_roundtrip();
|
test_config_delete_policy_wire_roundtrip();
|
||||||
|
test_config_delete_missing_args_wire_roundtrip();
|
||||||
|
test_config_append_wire_roundtrip();
|
||||||
test_config_basis_roundtrip();
|
test_config_basis_roundtrip();
|
||||||
test_config_basis_wire_rejects_escaping();
|
test_config_basis_wire_rejects_escaping();
|
||||||
test_config_basis_normalization();
|
test_config_basis_normalization();
|
||||||
|
test_config_checksum_options_wire_roundtrip();
|
||||||
|
test_config_receive_rejects_invalid_checksum_algo();
|
||||||
}
|
}
|
||||||
test_config_delete_timing_early_helper();
|
test_config_delete_timing_early_helper();
|
||||||
test_config_is_remote_dest();
|
test_config_is_remote_dest();
|
||||||
|
|||||||
@@ -634,12 +634,65 @@ static void test_delta_hash_index_large_mostly_matching() {
|
|||||||
free(new_data);
|
free(new_data);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* --checksum-seed: the delta strong (block) hash is genuinely seed-aware. A
|
||||||
|
* nonzero seed changes the per-block xxHash32, and a signature + delta computed
|
||||||
|
* with the same seed still reconstruct the file exactly (symmetric), while a
|
||||||
|
* mismatched seed produces a delta that does not match the signature blocks. */
|
||||||
|
static void test_delta_xxhash32_seeded() {
|
||||||
|
const char* data = "seedme";
|
||||||
|
uint32_t a = delta_xxhash32(data, 6);
|
||||||
|
uint32_t b = delta_xxhash32_seeded(data, 6, 42);
|
||||||
|
uint32_t c = delta_xxhash32_seeded(data, 6, 42);
|
||||||
|
EXPECT_TRUE(a != b);
|
||||||
|
EXPECT_EQ_INT((int)b, (int)c);
|
||||||
|
/* Unseeded == seeded with 0 (default reproduces today's behavior). */
|
||||||
|
EXPECT_EQ_INT((int)delta_xxhash32(data, 6), (int)delta_xxhash32_seeded(data, 6, 0));
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_delta_seeded_signature_compute_matches() {
|
||||||
|
uint32_t block_size = 1024;
|
||||||
|
/* Identical old/new data with a non-zero seed: the receiver builds a seeded
|
||||||
|
signature and the sender computes a seeded delta over the same bytes, so
|
||||||
|
every block matches and applying the delta rebuilds the file exactly. */
|
||||||
|
char data[4096];
|
||||||
|
for (int i = 0; i < 4096; i++)
|
||||||
|
data[i] = (char)(i % 256);
|
||||||
|
|
||||||
|
DeltaSignature* sig = delta_signature_create_seeded(data, 4096, block_size, 99);
|
||||||
|
EXPECT_NOT_NULL(sig);
|
||||||
|
Delta* delta = delta_compute_seeded(data, 4096, sig, block_size, 99);
|
||||||
|
EXPECT_NOT_NULL(delta);
|
||||||
|
void* rebuilt = delta_apply(data, 4096, delta, block_size);
|
||||||
|
EXPECT_NOT_NULL(rebuilt);
|
||||||
|
EXPECT_TRUE(memcmp(rebuilt, data, 4096) == 0);
|
||||||
|
free(rebuilt);
|
||||||
|
delta_destroy(delta);
|
||||||
|
delta_signature_destroy(sig);
|
||||||
|
|
||||||
|
/* A MISMATCHED seed means the sender's window xxHash32 never equals the
|
||||||
|
receiver's signature-block xxHash32: no block can match, so the delta is
|
||||||
|
not worthwhile / has no block matches. This proves the seed really gates
|
||||||
|
the block comparison rather than being an inert parameter. */
|
||||||
|
sig = delta_signature_create_seeded(data, 4096, block_size, 99);
|
||||||
|
EXPECT_NOT_NULL(sig);
|
||||||
|
delta = delta_compute_seeded(data, 4096, sig, block_size, 7);
|
||||||
|
EXPECT_NOT_NULL(delta);
|
||||||
|
bool any_match = false;
|
||||||
|
for (uint32_t i = 0; i < delta->instruction_count; i++)
|
||||||
|
if (delta->instructions[i].type == DELTA_INSTR_BLOCK_MATCH)
|
||||||
|
any_match = true;
|
||||||
|
EXPECT_FALSE(any_match);
|
||||||
|
delta_destroy(delta);
|
||||||
|
delta_signature_destroy(sig);
|
||||||
|
}
|
||||||
|
|
||||||
void test_delta() {
|
void test_delta() {
|
||||||
test_adler32_basic();
|
test_adler32_basic();
|
||||||
test_adler32_different_data();
|
test_adler32_different_data();
|
||||||
test_xxhash32_basic();
|
test_xxhash32_basic();
|
||||||
test_xxhash32_different_data();
|
test_xxhash32_different_data();
|
||||||
test_xxhash64_different_data();
|
test_xxhash64_different_data();
|
||||||
|
test_delta_xxhash32_seeded();
|
||||||
test_signature_roundtrip();
|
test_signature_roundtrip();
|
||||||
test_delta_identical_files();
|
test_delta_identical_files();
|
||||||
test_delta_small_edit();
|
test_delta_small_edit();
|
||||||
@@ -653,4 +706,5 @@ void test_delta() {
|
|||||||
test_delta_apply_rejects_output_overflow();
|
test_delta_apply_rejects_output_overflow();
|
||||||
test_delta_hash_index_matches_linear_reference();
|
test_delta_hash_index_matches_linear_reference();
|
||||||
test_delta_hash_index_large_mostly_matching();
|
test_delta_hash_index_large_mostly_matching();
|
||||||
|
test_delta_seeded_signature_compute_matches();
|
||||||
}
|
}
|
||||||
+175
-1
@@ -231,7 +231,10 @@ static char* make_check_root(const char* tag) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
static void write_check_file(const char* dir, const char* name, const char* content) {
|
static void write_check_file(const char* dir, const char* name, const char* content) {
|
||||||
char path[1024];
|
/* Sized so a caller that passes a PATH_MAX-bounded `dir` (e.g. one of the
|
||||||
|
test's own char[1024] stack buffers) still provably fits with the joined
|
||||||
|
name, keeping -Werror=format-truncation quiet. */
|
||||||
|
char path[4096];
|
||||||
snprintf(path, sizeof(path), "%s/%s", dir, name);
|
snprintf(path, sizeof(path), "%s/%s", dir, name);
|
||||||
int fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0644);
|
int fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0644);
|
||||||
if (fd >= 0) {
|
if (fd >= 0) {
|
||||||
@@ -493,6 +496,7 @@ static void test_late_manifest_abort_frees_keepset() {
|
|||||||
EXPECT_TRUE(send_int(p[1], 1));
|
EXPECT_TRUE(send_int(p[1], 1));
|
||||||
EXPECT_TRUE(send_str(p[1], "keep.txt"));
|
EXPECT_TRUE(send_str(p[1], "keep.txt"));
|
||||||
EXPECT_TRUE(send_int(p[1], 0)); /* protected-prefix section is empty */
|
EXPECT_TRUE(send_int(p[1], 0)); /* protected-prefix section is empty */
|
||||||
|
EXPECT_TRUE(send_int(p[1], 0)); /* missing-args section is empty */
|
||||||
EXPECT_TRUE(send_status(p[1], STATUS_ABORT));
|
EXPECT_TRUE(send_status(p[1], STATUS_ABORT));
|
||||||
|
|
||||||
DeleteManifest* pending = NULL;
|
DeleteManifest* pending = NULL;
|
||||||
@@ -516,6 +520,7 @@ static void test_late_manifest_eof_frees_keepset() {
|
|||||||
EXPECT_TRUE(send_int(p[1], 1));
|
EXPECT_TRUE(send_int(p[1], 1));
|
||||||
EXPECT_TRUE(send_str(p[1], "keep.txt"));
|
EXPECT_TRUE(send_str(p[1], "keep.txt"));
|
||||||
EXPECT_TRUE(send_int(p[1], 0)); /* protected-prefix section is empty */
|
EXPECT_TRUE(send_int(p[1], 0)); /* protected-prefix section is empty */
|
||||||
|
EXPECT_TRUE(send_int(p[1], 0)); /* missing-args section is empty */
|
||||||
shutdown(p[1], SHUT_WR);
|
shutdown(p[1], SHUT_WR);
|
||||||
|
|
||||||
DeleteManifest* pending = NULL;
|
DeleteManifest* pending = NULL;
|
||||||
@@ -539,10 +544,12 @@ static void test_late_second_manifest_frees_both() {
|
|||||||
EXPECT_TRUE(send_int(p[1], 1));
|
EXPECT_TRUE(send_int(p[1], 1));
|
||||||
EXPECT_TRUE(send_str(p[1], "first.txt"));
|
EXPECT_TRUE(send_str(p[1], "first.txt"));
|
||||||
EXPECT_TRUE(send_int(p[1], 0)); /* protected-prefix section is empty */
|
EXPECT_TRUE(send_int(p[1], 0)); /* protected-prefix section is empty */
|
||||||
|
EXPECT_TRUE(send_int(p[1], 0)); /* missing-args section is empty */
|
||||||
EXPECT_TRUE(send_status(p[1], STATUS_MANIFEST));
|
EXPECT_TRUE(send_status(p[1], STATUS_MANIFEST));
|
||||||
EXPECT_TRUE(send_int(p[1], 1));
|
EXPECT_TRUE(send_int(p[1], 1));
|
||||||
EXPECT_TRUE(send_str(p[1], "second.txt"));
|
EXPECT_TRUE(send_str(p[1], "second.txt"));
|
||||||
EXPECT_TRUE(send_int(p[1], 0)); /* protected-prefix section is empty */
|
EXPECT_TRUE(send_int(p[1], 0)); /* protected-prefix section is empty */
|
||||||
|
EXPECT_TRUE(send_int(p[1], 0)); /* missing-args section is empty */
|
||||||
|
|
||||||
DeleteManifest* pending = NULL;
|
DeleteManifest* pending = NULL;
|
||||||
EXPECT_EQ_INT(run_pending_receiver(cfg, p[0], &pending), -1);
|
EXPECT_EQ_INT(run_pending_receiver(cfg, p[0], &pending), -1);
|
||||||
@@ -553,6 +560,170 @@ static void test_late_second_manifest_frees_both() {
|
|||||||
config_delete(cfg);
|
config_delete(cfg);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* A delete-manifest frame with a third (missing-args) section round-trips: the
|
||||||
|
receiver keeps all three sections and the missing paths are confined exactly
|
||||||
|
like the keep-set (a traversal entry in the missing section is rejected).
|
||||||
|
receive_manifest_entries() reads the counts directly (the leading
|
||||||
|
STATUS_MANIFEST code is consumed by the caller, so these frames do not send
|
||||||
|
it). */
|
||||||
|
static void test_receive_manifest_three_sections() {
|
||||||
|
Config* cfg = config_create();
|
||||||
|
EXPECT_NOT_NULL(cfg);
|
||||||
|
cfg->receive_root_directory = str_dup("/tmp/dst");
|
||||||
|
int p[2];
|
||||||
|
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
|
||||||
|
io_set_fds(p[0], p[1]);
|
||||||
|
|
||||||
|
EXPECT_TRUE(send_int(p[1], 1));
|
||||||
|
EXPECT_TRUE(send_str(p[1], "keep.txt"));
|
||||||
|
EXPECT_TRUE(send_int(p[1], 1));
|
||||||
|
EXPECT_TRUE(send_str(p[1], "protected.txt"));
|
||||||
|
EXPECT_TRUE(send_int(p[1], 2));
|
||||||
|
EXPECT_TRUE(send_str(p[1], "gone.txt"));
|
||||||
|
EXPECT_TRUE(send_str(p[1], "dir/gone.bin"));
|
||||||
|
|
||||||
|
DeleteManifest* manifest = receive_manifest_entries(p[0]);
|
||||||
|
EXPECT_NOT_NULL(manifest);
|
||||||
|
EXPECT_EQ_INT(manifest->keeps->size, 1);
|
||||||
|
EXPECT_EQ_STR((char*)manifest->keeps->items[0], "keep.txt");
|
||||||
|
EXPECT_EQ_INT(manifest->protected->size, 1);
|
||||||
|
EXPECT_EQ_STR((char*)manifest->protected->items[0], "protected.txt");
|
||||||
|
EXPECT_EQ_INT(manifest->missing->size, 2);
|
||||||
|
EXPECT_EQ_STR((char*)manifest->missing->items[0], "gone.txt");
|
||||||
|
EXPECT_EQ_STR((char*)manifest->missing->items[1], "dir/gone.bin");
|
||||||
|
delete_manifest_free(manifest);
|
||||||
|
|
||||||
|
/* A traversal entry in the third section is rejected like every other. */
|
||||||
|
EXPECT_TRUE(send_int(p[1], 0));
|
||||||
|
EXPECT_TRUE(send_int(p[1], 0));
|
||||||
|
EXPECT_TRUE(send_int(p[1], 1));
|
||||||
|
EXPECT_TRUE(send_str(p[1], "../escape"));
|
||||||
|
EXPECT_NULL(receive_manifest_entries(p[0]));
|
||||||
|
Status status;
|
||||||
|
EXPECT_TRUE(receive_status(p[1], &status));
|
||||||
|
EXPECT_EQ_INT(status, STATUS_ERROR);
|
||||||
|
|
||||||
|
close(p[0]);
|
||||||
|
close(p[1]);
|
||||||
|
config_delete(cfg);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* --delete-missing-args exact-path deletions: regular files and empty
|
||||||
|
directories are removed, a non-empty directory survives without
|
||||||
|
--force/--delete and is recursively removed with --force or --delete, and a
|
||||||
|
missing mirror is a no-op. */
|
||||||
|
static void test_manifest_delete_missing_args() {
|
||||||
|
char* root = make_check_root("qmissing");
|
||||||
|
EXPECT_NOT_NULL(root);
|
||||||
|
write_check_file(root, "gone.txt", "stale");
|
||||||
|
char empty_dir[1024], full_dir[1024], inner[1024];
|
||||||
|
snprintf(empty_dir, sizeof(empty_dir), "%s/empty_dir", root);
|
||||||
|
snprintf(full_dir, sizeof(full_dir), "%s/full_dir", root);
|
||||||
|
snprintf(inner, sizeof(inner), "%s/full_dir/inner.txt", root);
|
||||||
|
EXPECT_EQ_INT(mkdir(empty_dir, 0755), 0);
|
||||||
|
EXPECT_EQ_INT(mkdir(full_dir, 0755), 0);
|
||||||
|
write_check_file(full_dir, "inner.txt", "content");
|
||||||
|
|
||||||
|
Config* cfg = config_create();
|
||||||
|
EXPECT_NOT_NULL(cfg);
|
||||||
|
cfg->receive_root_directory = str_dup(root);
|
||||||
|
cfg->delete_missing_args = true;
|
||||||
|
|
||||||
|
DeleteManifest* manifest = calloc(1, sizeof(DeleteManifest));
|
||||||
|
EXPECT_NOT_NULL(manifest);
|
||||||
|
manifest->keeps = array_list_create(free);
|
||||||
|
manifest->protected = array_list_create(free);
|
||||||
|
manifest->missing = array_list_create(free);
|
||||||
|
EXPECT_TRUE(array_list_add(manifest->missing, str_dup("gone.txt")));
|
||||||
|
EXPECT_TRUE(array_list_add(manifest->missing, str_dup("empty_dir")));
|
||||||
|
EXPECT_TRUE(array_list_add(manifest->missing, str_dup("full_dir")));
|
||||||
|
EXPECT_TRUE(array_list_add(manifest->missing, str_dup("never_here.txt")));
|
||||||
|
/* A deeper entry whose destination parent directory does not exist is a
|
||||||
|
no-op (nothing to delete), never a failure. */
|
||||||
|
EXPECT_TRUE(array_list_add(manifest->missing, str_dup("no_parent_here/gone.txt")));
|
||||||
|
|
||||||
|
/* Without --delete/--force the non-empty directory survives (rsync parity). */
|
||||||
|
EXPECT_TRUE(manifest_delete_missing_args(cfg, manifest));
|
||||||
|
char path[1024];
|
||||||
|
snprintf(path, sizeof(path), "%s/gone.txt", root);
|
||||||
|
EXPECT_EQ_INT(access(path, F_OK), -1);
|
||||||
|
snprintf(path, sizeof(path), "%s/empty_dir", root);
|
||||||
|
EXPECT_EQ_INT(access(path, F_OK), -1);
|
||||||
|
snprintf(path, sizeof(path), "%s/full_dir", root);
|
||||||
|
EXPECT_EQ_INT(access(path, F_OK), 0);
|
||||||
|
EXPECT_EQ_INT(access(inner, F_OK), 0);
|
||||||
|
|
||||||
|
/* With --force the non-empty directory mirror is removed recursively. */
|
||||||
|
cfg->force_delete = true;
|
||||||
|
EXPECT_TRUE(array_list_add(manifest->missing, str_dup("full_dir")));
|
||||||
|
EXPECT_TRUE(manifest_delete_missing_args(cfg, manifest));
|
||||||
|
EXPECT_EQ_INT(access(full_dir, F_OK), -1);
|
||||||
|
snprintf(path, sizeof(path), "%s/no_parent_here", root);
|
||||||
|
EXPECT_EQ_INT(access(path, F_OK), -1);
|
||||||
|
|
||||||
|
delete_manifest_free(manifest);
|
||||||
|
config_delete(cfg);
|
||||||
|
remove(full_dir);
|
||||||
|
rmdir(empty_dir);
|
||||||
|
rmdir(root);
|
||||||
|
free(root);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* A delete-missing-args manifest parked by the commit path is committed after
|
||||||
|
STATUS_FINISHED: the mirror that exists is removed, a missing mirror is a
|
||||||
|
no-op, and unrelated destination content is untouched (no --delete). */
|
||||||
|
static void test_receiver_pending_commits_missing_args() {
|
||||||
|
char* root = make_check_root("qmisscomm");
|
||||||
|
EXPECT_NOT_NULL(root);
|
||||||
|
write_check_file(root, "gone.txt", "stale");
|
||||||
|
write_check_file(root, "extra.txt", "unrelated");
|
||||||
|
|
||||||
|
Config* cfg = config_create();
|
||||||
|
EXPECT_NOT_NULL(cfg);
|
||||||
|
cfg->send_directory = str_dup("/src");
|
||||||
|
cfg->receive_root_directory = str_dup(root);
|
||||||
|
cfg->delete_missing_args = true;
|
||||||
|
|
||||||
|
int p[2];
|
||||||
|
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
|
||||||
|
io_set_fds(p[0], p[1]);
|
||||||
|
io_set_bwlimit(0);
|
||||||
|
|
||||||
|
EXPECT_TRUE(send_status(p[1], STATUS_MANIFEST));
|
||||||
|
EXPECT_TRUE(send_int(p[1], 0)); /* keep-set empty */
|
||||||
|
EXPECT_TRUE(send_int(p[1], 0)); /* protected empty */
|
||||||
|
EXPECT_TRUE(send_int(p[1], 2));
|
||||||
|
EXPECT_TRUE(send_str(p[1], "gone.txt"));
|
||||||
|
EXPECT_TRUE(send_str(p[1], "never_here.txt"));
|
||||||
|
EXPECT_TRUE(send_status(p[1], STATUS_FINISHED));
|
||||||
|
|
||||||
|
/* NULL pending: the single-threaded commit path deletes at FINISHED. The
|
||||||
|
sink sends the terminal STATUS_OK success frame. */
|
||||||
|
ReceiverSink sink = {.send_success = true};
|
||||||
|
EXPECT_EQ_INT(receiver_process_pending(cfg, p[0], &sink, NULL), 0);
|
||||||
|
Status ack;
|
||||||
|
EXPECT_TRUE(receive_status(p[1], &ack));
|
||||||
|
EXPECT_EQ_INT(ack, STATUS_OK);
|
||||||
|
|
||||||
|
char path[1024];
|
||||||
|
snprintf(path, sizeof(path), "%s/gone.txt", root);
|
||||||
|
EXPECT_EQ_INT(access(path, F_OK), -1);
|
||||||
|
snprintf(path, sizeof(path), "%s/extra.txt", root);
|
||||||
|
EXPECT_EQ_INT(access(path, F_OK), 0);
|
||||||
|
|
||||||
|
close(p[0]);
|
||||||
|
close(p[1]);
|
||||||
|
config_delete(cfg);
|
||||||
|
{
|
||||||
|
/* remove fixtures */
|
||||||
|
char pth[1024];
|
||||||
|
snprintf(pth, sizeof(pth), "%s/extra.txt", root);
|
||||||
|
remove(pth);
|
||||||
|
rmdir(root);
|
||||||
|
}
|
||||||
|
free(root);
|
||||||
|
}
|
||||||
|
|
||||||
void test_server() {
|
void test_server() {
|
||||||
if (!is_running_under_valgrind()) {
|
if (!is_running_under_valgrind()) {
|
||||||
test_receive_files_finished();
|
test_receive_files_finished();
|
||||||
@@ -566,5 +737,8 @@ void test_server() {
|
|||||||
test_late_manifest_abort_frees_keepset();
|
test_late_manifest_abort_frees_keepset();
|
||||||
test_late_manifest_eof_frees_keepset();
|
test_late_manifest_eof_frees_keepset();
|
||||||
test_late_second_manifest_frees_both();
|
test_late_second_manifest_frees_both();
|
||||||
|
test_receive_manifest_three_sections();
|
||||||
|
test_manifest_delete_missing_args();
|
||||||
|
test_receiver_pending_commits_missing_args();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -276,6 +276,22 @@ void test_shared_utils() {
|
|||||||
test_walker_unlimited_deletes_all();
|
test_walker_unlimited_deletes_all();
|
||||||
test_walker_hard_bound_all_or_nothing();
|
test_walker_hard_bound_all_or_nothing();
|
||||||
|
|
||||||
|
/* --append / --append-verify tail-resume math: a resume is eligible only for
|
||||||
|
a shorter existing destination, and the tail length is then the difference. */
|
||||||
|
EXPECT_TRUE(append_resume_eligible(0, 10));
|
||||||
|
EXPECT_TRUE(append_resume_eligible(7, 10));
|
||||||
|
EXPECT_FALSE(append_resume_eligible(10, 10));
|
||||||
|
EXPECT_FALSE(append_resume_eligible(11, 10));
|
||||||
|
|
||||||
|
unsigned long long tail;
|
||||||
|
EXPECT_TRUE(append_tail_length(0, 10, &tail));
|
||||||
|
EXPECT_EQ_INT((int)tail, 10);
|
||||||
|
EXPECT_TRUE(append_tail_length(7, 10, &tail));
|
||||||
|
EXPECT_EQ_INT((int)tail, 3);
|
||||||
|
EXPECT_FALSE(append_tail_length(10, 10, &tail));
|
||||||
|
EXPECT_FALSE(append_tail_length(11, 10, &tail));
|
||||||
|
EXPECT_FALSE(append_tail_length(7, 10, NULL));
|
||||||
|
|
||||||
char formatted[32];
|
char formatted[32];
|
||||||
EXPECT_TRUE(format_human_bytes(0, formatted, sizeof(formatted)));
|
EXPECT_TRUE(format_human_bytes(0, formatted, sizeof(formatted)));
|
||||||
EXPECT_EQ_STR(formatted, "0 B");
|
EXPECT_EQ_STR(formatted, "0 B");
|
||||||
|
|||||||
Reference in new issue
Block a user