Compare commits
56
Commits
v2.28.0
...
07dfec629f
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
07dfec629f | ||
|
|
c062a0762e | ||
|
|
283f9f0823 | ||
|
|
5754b9a952 | ||
|
|
b8a0efef7b | ||
|
|
2e77c09447 | ||
|
|
06c4026b74 | ||
|
|
9f47b13712 | ||
|
|
b1eddf0133 | ||
|
|
338c27db73 | ||
|
|
8d46a26c04 | ||
|
|
707ba272df | ||
|
|
bc71a3c0a5 | ||
|
|
cee9b7647c | ||
|
|
3adb6dddb5 | ||
|
|
d77849774e | ||
|
|
b5c6f8b60f | ||
|
|
134dcd74cc | ||
|
|
42f2f845ac | ||
|
|
0669335ca5 | ||
|
|
391f76cd56 | ||
|
|
2021afe613 | ||
|
|
ba914e8ab3 | ||
|
|
df8fe1ae4e | ||
|
|
2c490d58b7 | ||
|
|
d55dabff2e | ||
|
|
b478a59a81 | ||
|
|
865941f850 | ||
|
|
221cefa7cc | ||
|
|
bb9b59024a | ||
|
|
5baf120243 | ||
|
|
84b7e1fd2f | ||
|
|
800a978e15 | ||
|
|
0f40e747f0 | ||
|
|
b07306d5bc | ||
|
|
f2c89b6e7c | ||
|
|
379f127859 | ||
|
|
3799200f71 | ||
|
|
91c4a967e6 | ||
|
|
88c8968b4c | ||
|
|
082b31886a | ||
|
|
423a62e691 | ||
|
|
bc18ae205b | ||
|
|
10a61c6101 | ||
|
|
bc1e1191af | ||
|
|
0fbb9de915 | ||
|
|
9b05972375 | ||
|
|
d119f35066 | ||
|
|
00829fd265 | ||
|
|
ff261bc38a | ||
|
|
b235721f8b | ||
|
|
79a28cdb96 | ||
|
|
402cae80ad | ||
|
|
9691dba6f0 | ||
|
|
558782d339 | ||
|
|
5597e74f6a |
No files matched your search
@@ -12,3 +12,12 @@ build_docker2/
|
|||||||
# Test/run artifacts
|
# Test/run artifacts
|
||||||
root/
|
root/
|
||||||
test_partial_install_tmp/
|
test_partial_install_tmp/
|
||||||
|
|
||||||
|
# Editor/tooling + test caches/artifacts
|
||||||
|
.pytest_cache/
|
||||||
|
*.gcda
|
||||||
|
*.gcno
|
||||||
|
*.gcov
|
||||||
|
di/
|
||||||
|
test_data-manual/
|
||||||
|
*.log
|
||||||
@@ -4,9 +4,9 @@ FastSync is a high-performance file synchronization system written in C11. It su
|
|||||||
|
|
||||||
## Dependency installation
|
## Dependency installation
|
||||||
|
|
||||||
**CI rule:** never add `apt-get install` / `pip install` steps to CI workflows — use the custom Docker image instead. The image is built from the repo-root `Dockerfile` and is the same image CI uses: `gitea.tap-tap.win/taptap/fastsync-ci:v11`. It contains the full toolchain: gcc/g++, CMake, libzstd-dev, libssl-dev, make, git, cppcheck, clang-format, python3 + pytest + pytest-xdist, openssh-client, Node.js, plus `rsync` 3.4.1 (with zstd/xxhash/lz4), `acl` and `attr` (setfacl/getfacl, setfattr/getfattr) for drop-in parity tests.
|
**CI rule:** never add `apt-get install` / `pip install` steps to CI workflows — use the custom Docker image instead. The image is built from the repo-root `Dockerfile` and is the same image CI uses: `gitea.tap-tap.win/taptap/fastsync-ci:v11`. It contains the full toolchain: gcc/g++, CMake, libzstd-dev, zlib1g-dev, liblz4-dev, libxxhash-dev, libssl-dev, make, git, cppcheck, clang-format, python3 + pytest + pytest-xdist, openssh-client, Node.js, plus `rsync` 3.4.1 (with zstd/xxhash/lz4), `acl` and `attr` (setfacl/getfacl, setfattr/getfattr) for drop-in parity tests. (CMake hard-requires zstd, zlib, and lz4; xxHash is fetched via `FetchContent`.)
|
||||||
|
|
||||||
**Host rule:** for local development, use `nix-shell` (see `README.md`) which provides zstd, OpenSSL, CMake, and gcc. The Docker image can also be used locally for CI parity.
|
**Host rule:** for local development, use `nix-shell` (see `README.md`) which provides zstd, zlib, lz4, OpenSSL, CMake, and gcc. The Docker image can also be used locally for CI parity.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Use the prebuilt CI image directly (faster, guaranteed CI parity)
|
# Use the prebuilt CI image directly (faster, guaranteed CI parity)
|
||||||
@@ -38,11 +38,12 @@ If a dependency is missing from the CI image, add it to the `Dockerfile` (and re
|
|||||||
When configuring for CI parity, use:
|
When configuring for CI parity, use:
|
||||||
```bash
|
```bash
|
||||||
cmake -B build -S . -DSTRICT_WARNINGS=ON # -Wextra -Wpedantic -Werror
|
cmake -B build -S . -DSTRICT_WARNINGS=ON # -Wextra -Wpedantic -Werror
|
||||||
cmake -B build -S . -DSANITIZER=address # AddressSanitizer (ASan)
|
cmake -B build -S . -DSANITIZER=address # AddressSanitizer (ASan); in the CI matrix
|
||||||
cmake -B build -S . -DSANITIZER=thread # ThreadSanitizer (TSan)
|
cmake -B build -S . -DSANITIZER=undefined # UndefinedBehaviorSanitizer (UBSan); in the CI matrix
|
||||||
|
cmake -B build -S . -DSANITIZER=thread # ThreadSanitizer (TSan); local-only, NOT in CI
|
||||||
```
|
```
|
||||||
|
|
||||||
The CI workflow (`.gitea/workflows/ci.yaml`) runs lint (clang-format, cppcheck), then a **fast PR gate** — build + unit + a representative subset of integration tests marked `@pytest.mark.ci`, parallelized with pytest-xdist (`-n 4 --dist=load`). The full coverage jobs (full integration suite as `-m "not setpriv"`, sanitizer, fuzz, coverage, valgrind) run **only on push to `dev`/`main`**; pull requests skip them to keep PR CI under ~3 minutes. The two `setpriv` privilege tests are excluded from CI via a marker because their result depends on the runner/container uid and host mount permissions.
|
The CI workflow (`.gitea/workflows/ci.yaml`) runs lint (clang-format, cppcheck), then a **fast PR gate** — build + unit + a representative subset of integration tests marked `@pytest.mark.ci`, parallelized with pytest-xdist (`-n 4 --dist=load`). The full coverage jobs (full integration suite as `-m "not setpriv"`, the `address`+`undefined` sanitizer matrix, fuzz, coverage, valgrind) run **only on push to `dev`/`main`**; pull requests skip them to keep PR CI under ~3 minutes. TSan is not part of the CI matrix and is a local-only configuration. The `setpriv`-marked privilege tests (four decorated functions, collecting to eight instances because two are parametrized) are excluded from CI via a marker because their result depends on the runner/container uid and host mount permissions.
|
||||||
|
|
||||||
## Build
|
## Build
|
||||||
|
|
||||||
@@ -105,7 +106,7 @@ Two main branches: `dev` (integration) and `main` (stable releases).
|
|||||||
|
|
||||||
### Rules
|
### Rules
|
||||||
- **All PRs target `dev`** — never target `main` directly
|
- **All PRs target `dev`** — never target `main` directly
|
||||||
- **`dev` is the default branch** in Gitea repo settings
|
- **`dev` is intended to be the default branch** in Gitea repo settings — verify in the repo settings, since this clone's `origin/HEAD` still points at `main`
|
||||||
- **`main` is protected** — only merged from `dev` via PR with 2 approvals + full CI pass
|
- **`main` is protected** — only merged from `dev` via PR with 2 approvals + full CI pass
|
||||||
- **Feature/bug branches** branch from `dev`, PR back to `dev`
|
- **Feature/bug branches** branch from `dev`, PR back to `dev`
|
||||||
- **`dev` → `main` merges** happen on-demand or weekly, requiring full CI + review
|
- **`dev` → `main` merges** happen on-demand or weekly, requiring full CI + review
|
||||||
|
|||||||
+131
@@ -4,6 +4,137 @@ All notable changes to FastSync are documented here. Versions match
|
|||||||
`PROTOCOL_VERSION` (printed by `fastsync --version`); the client and server must
|
`PROTOCOL_VERSION` (printed by `fastsync --version`); the client and server must
|
||||||
run the same version because the handshake is strict.
|
run the same version because the handshake is strict.
|
||||||
|
|
||||||
|
## [Unreleased]
|
||||||
|
|
||||||
|
The rsync-parity cycle 2.29 (no wire change; `PROTOCOL_VERSION` stays 2.28.0).
|
||||||
|
`RSYNC_COMPAT.md` moves from **116 ✅ / 14 ⚠️ / 27 ❌** to
|
||||||
|
**120 ✅ / 10 ⚠️ / 27 ❌** of 157 rows.
|
||||||
|
|
||||||
|
An audit cycle follows on the same wire version (`PROTOCOL_VERSION` stays
|
||||||
|
2.28.0): a security-and-correctness pass over the parity-2.29 baseline, plus a
|
||||||
|
set of audit follow-ups (filter merge modifiers, the `--inplace`/`--partial-dir`
|
||||||
|
conflict, credential-file hardening, and small leak/log/test fixes). It fixes
|
||||||
|
a `--temp-dir` symlink escape, gates client-controlled special permission bits,
|
||||||
|
corrects `--partial-dir`/`--bwlimit`/`-z` behavior, handles unsupported filter
|
||||||
|
modifiers, and tightens client and wire validation. The only parity
|
||||||
|
reclassification is `--filter=RULE` moving ✅ → ⚠️, because its merge-only
|
||||||
|
`e`/`n`/`w`/`-` modifiers are now accepted and consumed but their semantics
|
||||||
|
remain unimplemented (accepted-but-ignored); the matrix is therefore **119 ✅ /
|
||||||
|
11 ⚠️ / 27 ❌** of 157 rows. The affected rows' notes and the summary tally in
|
||||||
|
`RSYNC_COMPAT.md` were updated.
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
- **rsync-exact traversal order.** The sequential scanner now walks each
|
||||||
|
directory's entries in rsync 3.4.1's flist order (non-directories ascending,
|
||||||
|
then directories ascending, depth-first), so `--info=name`, the
|
||||||
|
`--delete-during`/`--delete-delay`/`-n` would-delete order and the partial
|
||||||
|
`--max-delete` survivor set match rsync byte-for-byte. `--threads` has no
|
||||||
|
rsync analogue and stays unordered.
|
||||||
|
- **Delete timing.** The complete `--delete-during`/`--delete-delay`
|
||||||
|
per-directory plan set is transmitted before the first data frame, so a
|
||||||
|
mid-transfer abort has already removed every planned extra like rsync's
|
||||||
|
generator; `-d/--dirs` uses per-directory plans (shielded untraversed
|
||||||
|
subdirectories) instead of the end-of-transfer commit. `-n`, `--delete`,
|
||||||
|
`--del`/`--delete-during` and `--delete-delay` are now ✅ Parity.
|
||||||
|
- **Basis directories.** A relative `--compare-dest`/`--copy-dest`/`--link-dest`
|
||||||
|
DIR resolves against the destination directory with the transfer-relative
|
||||||
|
name appended, exactly like rsync 3.4.1.
|
||||||
|
- **`-y`/`--fuzzy`.** The candidate search no longer inherits the ordinary delta
|
||||||
|
engine's 16 KiB minimum or 10× size-ratio bound, so an oversized or
|
||||||
|
sub-16-KiB sibling is reused exactly as rsync reuses it.
|
||||||
|
- `--info=mount` prints rsync's mount-point skip line (repeated `-xx` drops the
|
||||||
|
mount-point directory); `--info=stats` enables the `--stats` block; `-x` is
|
||||||
|
repeatable. `--stats` counts traversed directories for the `Number of files`
|
||||||
|
breakdown under a plain `-r` scan. `--debug` emits real output for
|
||||||
|
`flist`/`del`/`hash`/`deltasum`/`recv`/`filter`/`send`.
|
||||||
|
|
||||||
|
### Known residuals
|
||||||
|
|
||||||
|
- `--progress` and `--info` still need a receiver→sender event channel for the
|
||||||
|
root `./` line, ancestor-directory suppression, receiver-side `skip`/`backup`
|
||||||
|
wording, and symlink/empty-directory quick-checks.
|
||||||
|
- `--delete-before`'s phase-0 late-file divergence remains (rsync's pre-scan
|
||||||
|
fixes the file list before the data pass).
|
||||||
|
- A single file larger than 256 MiB cannot be streamed in the default path
|
||||||
|
(a general whole-file limit, not basis-specific).
|
||||||
|
- `--stats` byte totals and `--msgs2stderr` stay documented divergences.
|
||||||
|
|
||||||
|
### Security
|
||||||
|
|
||||||
|
- **`--temp-dir` symlink escape fixed.** The receiver's scratch directory was
|
||||||
|
opened with a bare `open()`, so a symlink planted under the receive root could
|
||||||
|
redirect receiver scratch files outside the authorized root. The opened
|
||||||
|
directory is now judged by the real path of its fd (`/proc/self/fd` via
|
||||||
|
`realpath`) and an escaping target is refused (`EACCES`, logged); an in-root
|
||||||
|
link to another filesystem (the `EXDEV` fallback case) still works.
|
||||||
|
- **Client-controlled special bits masked when super-user activities are not
|
||||||
|
permitted.** Setuid/setgid/sticky bits (`--perms`, `--chmod`, the symlink and
|
||||||
|
special-node paths, and deferred directory modes) are now stripped when the
|
||||||
|
connection forbids super activities (`--no-super`, a non-opted daemon module,
|
||||||
|
a privileged listener without `--allow-super`); exact rsync semantics are
|
||||||
|
preserved wherever super activities are permitted.
|
||||||
|
- **Daemon umask no longer forced to `0`.** `daemonize()` now sets the
|
||||||
|
conventional `022`, so implied parent directories created without `-p` are no
|
||||||
|
longer world-writable `0777`.
|
||||||
|
- **Credentials and signal handling hardened.** Secret files are opened with
|
||||||
|
`O_NOFOLLOW|O_NONBLOCK` (while allowing fd-backed store paths and bound-waiting
|
||||||
|
a FIFO read for ~3 s so a slow process substitution works but a connected-but-
|
||||||
|
silent FIFO cannot hang), and signal handlers use `sigaction` with
|
||||||
|
async-signal-safe bodies.
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
|
||||||
|
- **`-z` on 100–256 MiB files.** The decompressor's internal ceiling was 100 MiB
|
||||||
|
while the receiver advertises and the sender compresses whole files up to
|
||||||
|
`MAX_RECEIVE_WHOLE_FILE_SIZE` (256 MiB), so `-z` on a 100–256 MiB regular file
|
||||||
|
failed with `Declared decompressed size exceeds 104857600 bytes`. The ceiling
|
||||||
|
is now defined in terms of the protocol whole-file bound (still an
|
||||||
|
allocation-clamped bomb guard).
|
||||||
|
- **`--bwlimit` now paces `--sendfile`.** The plaintext-TCP `--sendfile` fast
|
||||||
|
path bypassed the protocol's token bucket, so the limit was ignored there. It
|
||||||
|
now throttles through the same per-session leaky bucket as the TLS path.
|
||||||
|
- **`--partial-dir` implies `--partial`.** Matching rsync 3.4.1 (which sets
|
||||||
|
`keep_partial` after option parsing), `--partial-dir=DIR` alone retains an
|
||||||
|
interrupted transfer's partial and wins over an explicit `--no-partial`;
|
||||||
|
`--inplace` still bypasses the partial machinery, and combining `--inplace`
|
||||||
|
with `--partial-dir` is now rejected up front with rsync's message
|
||||||
|
(`--inplace cannot be used with --partial-dir`).
|
||||||
|
- **Filter modifiers handled.** The `x` xattr-name modifier is rejected with a
|
||||||
|
clear error everywhere. The merge-only `e`/`n`/`w` and `-` modifiers are now
|
||||||
|
accepted and consumed on `merge`/`dir-merge` rules (so they no longer leak
|
||||||
|
into the merge filename) while still being rejected on non-merge rules,
|
||||||
|
matching rsync; their semantics remain unimplemented (accepted-but-ignored).
|
||||||
|
Glued patterns (`-newfile`, `-e2e`) and mixed tokens (`H,!secret`) keep their
|
||||||
|
historical parsing.
|
||||||
|
- **Credential-file reads hardened.** Secret files (`--password-file`/
|
||||||
|
`--early-input`/`--hash-credentials` input) are opened with `O_NOFOLLOW`, so a
|
||||||
|
symlinked credential path now fails closed (`ELOOP`) instead of being followed
|
||||||
|
before the owner/mode gate; literal fd-backed paths (`/dev/fd/<digits>`,
|
||||||
|
`/proc/self/fd/<digits>`) are exempt so process substitution still works. A
|
||||||
|
FIFO/process-substitution read now waits under a bounded ~3 s deadline for its
|
||||||
|
writer, so a slow producer works while a connected-but-silent FIFO fails
|
||||||
|
instead of hanging.
|
||||||
|
- **Miscellaneous correctness fixes:** `--filter` rule count is checked
|
||||||
|
client-side against `MAX_FILTER_RULES` before any network I/O (the receiver
|
||||||
|
still re-checks the expanded count); unknown wire `Status` values are rejected
|
||||||
|
as protocol errors; a mutex leak on an init-failure path, an `errno` read
|
||||||
|
after `free()` in deferred delete application, `log_perror` misuse for
|
||||||
|
non-`errno` conditions, and a `NULL` `server_host`/`ssh_destination`
|
||||||
|
allocation path were fixed (the `config_create` failure now releases through
|
||||||
|
`config_delete`); the decompression-limit log now prints the effective bound
|
||||||
|
rather than the compile-time ceiling; the daemon umask and root test fixtures
|
||||||
|
were hardened; `SSL_read` length is clamped and `sendfile` `poll()` retries on
|
||||||
|
`EINTR`.
|
||||||
|
|
||||||
|
### Refactored / Docs
|
||||||
|
|
||||||
|
- Dropped dead `filter_rules_apply` and dead `--old-args` plumbing, unified
|
||||||
|
`set_error`, deduplicated `path_is_within` and shared constants, and added
|
||||||
|
printf format attributes (fixing format mismatches). `RSYNC_COMPAT.md`,
|
||||||
|
`CHANGELOG.md` and `HANDOFF.md` were updated for the audit cycle; the
|
||||||
|
`RSYNC_COMPAT.md` summary tally was corrected to match the rows.
|
||||||
|
|
||||||
## [2.28.0] - 2026-09-20
|
## [2.28.0] - 2026-09-20
|
||||||
|
|
||||||
The rsync-parity cycle. `PROTOCOL_VERSION` moves `2.26.0 → 2.27.0 → 2.28.0`;
|
The rsync-parity cycle. `PROTOCOL_VERSION` moves `2.26.0 → 2.27.0 → 2.28.0`;
|
||||||
|
|||||||
+3
-2
@@ -26,9 +26,9 @@ elseif(NOT SANITIZER STREQUAL "none")
|
|||||||
endif()
|
endif()
|
||||||
|
|
||||||
# --- Strict warnings option ---
|
# --- Strict warnings option ---
|
||||||
option(STRICT_WARNINGS "Enable strict warnings (Wextra, Wpedantic, Werror)" OFF)
|
option(STRICT_WARNINGS "Enable strict warnings (Wextra, Wpedantic, Wformat-signedness, Werror)" OFF)
|
||||||
if(STRICT_WARNINGS)
|
if(STRICT_WARNINGS)
|
||||||
add_compile_options(-Wextra -Wpedantic -Werror)
|
add_compile_options(-Wextra -Wpedantic -Wformat-signedness -Werror)
|
||||||
endif()
|
endif()
|
||||||
|
|
||||||
# --- Coverage option ---
|
# --- Coverage option ---
|
||||||
@@ -226,6 +226,7 @@ set(TEST_SRCS
|
|||||||
tests/test_file.c
|
tests/test_file.c
|
||||||
tests/test_file_list.c
|
tests/test_file_list.c
|
||||||
tests/test_file_sendfile.c
|
tests/test_file_sendfile.c
|
||||||
|
tests/test_filter.c
|
||||||
tests/test_format.c
|
tests/test_format.c
|
||||||
tests/test_fuzz_smoke.c
|
tests/test_fuzz_smoke.c
|
||||||
tests/test_glob.c
|
tests/test_glob.c
|
||||||
|
|||||||
+62
-23
@@ -1,18 +1,30 @@
|
|||||||
# FastSync — Session Handoff (2026-09-19)
|
# FastSync — Session Handoff (2026-09-21)
|
||||||
|
|
||||||
## Current status
|
## Current status
|
||||||
- **rsync-parity tracks 1-6 landed on `dev`** via **PR #303** (`10159dc`,
|
- **Release `v2.28.0`** is tagged and merged to `main`: tag `v2.28.0` points at
|
||||||
"feat(parity): rsync parity tracks 1-6 (protocol 2.28.0)"). Dev push CI run
|
`ee6523a`, and the PR #304 merge commit `b4d54504` is on `main`.
|
||||||
**581** fully green: lint, build-and-test, parity-full, ASan, UBSan,
|
- **`dev` is at `0fbb9de`** — the merge of parity cycle 2.29 (PR #305). The old
|
||||||
fuzz-build, coverage, valgrind.
|
`558782d` (incremental-check flake fix) is an ancestor.
|
||||||
- **`PROTOCOL_VERSION` = `"2.28.0"`** (`src/shared/config.h`); CMake
|
- **`PROTOCOL_VERSION` = `"2.28.0"`** (`src/shared/config.h`); CMake
|
||||||
`project(FastFileTransfer VERSION 2.28.0)`. The cycle batched all wire
|
`project(FastFileTransfer VERSION 2.28.0)`.
|
||||||
changes (stats counters, filter-rule block, `--verify-basis`) under the one
|
- **Parity cycle 2.29 is merged to `dev`** (PR #305), no wire change. It closed
|
||||||
bump.
|
the scanner-order, delete-timing, relative-basis and fuzzy-eligibility
|
||||||
- **`main` = `ef76c90`** (tag `v2.26.0`); the 2.27.0/2.28.0 work is on `dev`
|
residuals and improved the `--info`/`--stats`/`--debug` partials. Parity
|
||||||
and not yet released. A `dev -> main` v2.28.0 release PR is the next step.
|
matrix: **120 ✅ / 10 ⚠️ / 27 ❌ = 157**. Remaining ⚠️ rows: `--info`,
|
||||||
- Parity matrix: **116 ✅ / 14 ⚠️ / 27 ❌ = 157** (was 111/13/33 at cycle start).
|
`--debug`, `--msgs2stderr`, `--stats`, `--progress`, `--delete-before`, the
|
||||||
- Working tree clean; feature branch deleted; no scratch trees or worktrees.
|
three basis-dir options, and `-y`/`--fuzzy`.
|
||||||
|
- **Audit cycle complete on branch `fix/audit-cycle`** (branched from `dev` @
|
||||||
|
`0fbb9de`), integration PR to `dev` pending. No wire change
|
||||||
|
(`PROTOCOL_VERSION` stays 2.28.0). It lands the receiver/client security and
|
||||||
|
correctness fixes — `--temp-dir` symlink-escape confinement, special-bit
|
||||||
|
masking under a super-off policy, daemon `umask(022)`, the `-z` decompression
|
||||||
|
ceiling raised to the 256 MiB whole-file bound, `--bwlimit` pacing the
|
||||||
|
plaintext `--sendfile` path, `--partial-dir` implying `--partial`, rejection
|
||||||
|
of unsupported filter modifiers (`x`/`e`/`n`/`w`), client-side
|
||||||
|
`MAX_FILTER_RULES` enforcement, unknown wire `Status` rejection, and the
|
||||||
|
accompanying refactors/docs. The parity matrix is unchanged at
|
||||||
|
**120 ✅ / 10 ⚠️ / 27 ❌ = 157**; this docs pass (worktree `fix/audit-docs2`)
|
||||||
|
corrects the `RSYNC_COMPAT.md` summary tally to match the rows.
|
||||||
|
|
||||||
|
|
||||||
## What landed this session
|
## What landed this session
|
||||||
@@ -98,7 +110,8 @@
|
|||||||
uptodate` plus the leading `./` root name line for `--info=name` (only the
|
uptodate` plus the leading `./` root name line for `--info=name` (only the
|
||||||
root-line trigger condition and receiver-side `skip` wording remain). Matrix
|
root-line trigger condition and receiver-side `skip` wording remain). Matrix
|
||||||
now **111 ✅ / 14 ⚠️ / 32 ❌ = 157**; differential + unit tests added in
|
now **111 ✅ / 14 ⚠️ / 32 ❌ = 157**; differential + unit tests added in
|
||||||
`test_features.py`, `test_option_parity.py`, `test_delete_plan.c`,
|
`test_features.py`, `test_option_parity.py`, the unit test
|
||||||
|
`tests/test_delete_plan.c`,
|
||||||
`test_delete_delay_budget_parity.py`, `test_delete_timing_parity.py`.
|
`test_delete_delay_budget_parity.py`, `test_delete_timing_parity.py`.
|
||||||
12. **No-wire parity track 2b** on `feat/parity-2.28` (no protocol change):
|
12. **No-wire parity track 2b** on `feat/parity-2.28` (no protocol change):
|
||||||
`--progress`/`-P`/`--info=progress` (when not `--quiet`) now run an opt-in
|
`--progress`/`-P`/`--info=progress` (when not `--quiet`) now run an opt-in
|
||||||
@@ -195,17 +208,43 @@
|
|||||||
**116 ✅ / 14 ⚠️ / 27 ❌ = 157** (the `--delete`/`--delete-during` rows stay
|
**116 ✅ / 14 ⚠️ / 27 ❌ = 157** (the `--delete`/`--delete-during` rows stay
|
||||||
⚠️ for the abort boundary; `--delete-after` stays ✅).
|
⚠️ for the abort boundary; `--delete-after` stays ✅).
|
||||||
|
|
||||||
|
17. **Audit cycle** on `fix/audit-cycle` (from `dev` @ `0fbb9de`;
|
||||||
|
`PROTOCOL_VERSION` stays `2.28.0`): a security/correctness pass over the
|
||||||
|
parity-2.29 baseline. It raises the decompression ceiling to the 256 MiB
|
||||||
|
protocol whole-file bound (`-z` on 100–256 MiB files now works), paces the
|
||||||
|
plaintext-TCP `--sendfile` path with `--bwlimit`, confines the `--temp-dir`
|
||||||
|
scratch dir by the fd's real path (symlink escape refused), masks
|
||||||
|
client-controlled setuid/setgid/sticky bits when super activities are not
|
||||||
|
permitted, sets the daemon umask to `022`, makes `--partial-dir` imply
|
||||||
|
`--partial`, rejects the unsupported filter modifiers (`x`/`e`/`n`/`w`),
|
||||||
|
enforces `MAX_FILTER_RULES` client-side, rejects unknown wire `Status`
|
||||||
|
values, and hardens credentials/signal handling (with the accompanying
|
||||||
|
refactors and docs). No row changes classification, so the matrix stays
|
||||||
|
**120 ✅ / 10 ⚠️ / 27 ❌ = 157**. This docs pass is on `fix/audit-docs2`.
|
||||||
|
|
||||||
## Next steps
|
## Next steps
|
||||||
1. **Merge PR #284** (`dev` -> `main`) once reviewed (protected branch).
|
1. **Open and merge the audit-cycle PR** (`fix/audit-cycle`, including this
|
||||||
2. **Deferred security items** (documented, not implemented):
|
`fix/audit-docs2` docs pass) into `dev` once reviewed. `dev` is the default
|
||||||
- Pre-auth config/daemon-auth handshake has no aggregate wall-clock deadline
|
branch; all PRs target `dev`, never `main` directly.
|
||||||
(per-message timeout only) — slowloris holds connection slots.
|
2. **Remaining deferred items:**
|
||||||
- Per-source registry fails open when the shared table is full (per-module/global
|
- **Large structural refactors:** delete-engine consolidation
|
||||||
caps and host ACLs still apply); consider fail-closed or larger/evicting table.
|
(`delete_extras_fd`/`manifest_delete_extras`/the delete-plan path),
|
||||||
- SCRAM-like daemon auth has no TLS channel binding (and is not RFC 5802).
|
god-function splits, and translation-unit splits.
|
||||||
- `cleanup()` signal handler calls non-async-signal-safe teardown; daemon `umask(0)`.
|
- **`--progress`/`--info` receiver→sender event channel:** the root `./`
|
||||||
- Wire protocol assumes homogeneous word size/endianness (lengths are native
|
line, ancestor-directory suppression, receiver-side `skip`/`backup` echo,
|
||||||
`size_t`) — document or move to fixed-width framing.
|
and symlink/empty-dir quick-check feedback.
|
||||||
|
- **`--delete-before` phase-0 keep-set** (rsync fixes the file list before
|
||||||
|
the data pass; FastSync keeps its pre-scan snapshot race).
|
||||||
|
- **>256 MiB single-file streaming** (B4, the general whole-file limit).
|
||||||
|
- **Wire native-size framing:** lengths are native `size_t` and the protocol
|
||||||
|
assumes homogeneous word size/endianness — document or move to fixed-width
|
||||||
|
framing.
|
||||||
|
- **SCRAM-like daemon auth channel binding:** no TLS channel binding today
|
||||||
|
(and it is not RFC 5802).
|
||||||
|
- Still-open security nits: the pre-auth config/daemon-auth handshake has no
|
||||||
|
aggregate wall-clock deadline (per-message timeout only — slowloris holds
|
||||||
|
connection slots); the per-source registry fails open when the shared table
|
||||||
|
is full (per-module/global caps and host ACLs still apply).
|
||||||
3. **Out of scope / intentional:** pull (remote source) mode is **not** planned —
|
3. **Out of scope / intentional:** pull (remote source) mode is **not** planned —
|
||||||
FastSync is push-only; see `RSYNC_COMPAT.md#direction`.
|
FastSync is push-only; see `RSYNC_COMPAT.md#direction`.
|
||||||
|
|
||||||
|
|||||||
@@ -75,8 +75,9 @@ matrix is classified as parity, caveat, or divergent in
|
|||||||
owner, group, devices, and special files — and does not imply compression or
|
owner, group, devices, and special files — and does not imply compression or
|
||||||
multithreading (see [Client](#client)). Ownership application is still
|
multithreading (see [Client](#client)). Ownership application is still
|
||||||
privilege-gated: a receiver that cannot `chown` logs a warning and skips it.
|
privilege-gated: a receiver that cannot `chown` logs a warning and skips it.
|
||||||
Under `-p` the source mode is copied exactly, including setuid/setgid/sticky
|
Under `-p` the source mode is copied exactly, including group/other-write
|
||||||
and group/other-write bits (strict rsync parity; see
|
bits; setuid/setgid/sticky bits are copied only when super-user activities are
|
||||||
|
permitted, and are masked under `SUPER_MODE_OFF`/`--no-super` (see
|
||||||
[`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)).
|
[`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)).
|
||||||
- Symlink transfer stores targets **verbatim** (`-l`/`--links`), including
|
- Symlink transfer stores targets **verbatim** (`-l`/`--links`), including
|
||||||
absolute and `..`-bearing targets, matching rsync. The receiver does not
|
absolute and `..`-bearing targets, matching rsync. The receiver does not
|
||||||
@@ -172,7 +173,7 @@ This produces `./build/client` and `./build/server`. `compile_commands.json` is
|
|||||||
| `--preserve` | Preserve mode and mtime (`-p` + `-t`; add `-o`/`-g` for owner/group or `-U`/`--atimes` for atime; `-N`/`--crtimes` captures birth time but cannot apply it) |
|
| `--preserve` | Preserve mode and mtime (`-p` + `-t`; add `-o`/`-g` for owner/group or `-U`/`--atimes` for atime; `-N`/`--crtimes` captures birth time but cannot apply it) |
|
||||||
| `-U, --atimes` | Preserve access times. Captured with the metadata payload; does not enable ownership. |
|
| `-U, --atimes` | Preserve access times. Captured with the metadata payload; does not enable ownership. |
|
||||||
| `-N, --crtimes` | Capture birth time; cannot be applied (documented divergence) |
|
| `-N, --crtimes` | Capture birth time; cannot be applied (documented divergence) |
|
||||||
| `-p, --perms` | Preserve permission bits. Strict rsync parity: the source mode is copied exactly, including setuid/setgid/sticky and group/other-write bits |
|
| `-p, --perms` | Preserve permission bits. The source mode is copied exactly, including group/other-write bits; setuid/setgid/sticky are copied only when super-user activities are permitted (`SUPER_MODE_OFF`/`--no-super` masks them) |
|
||||||
| `-t, --times` | Preserve modification times |
|
| `-t, --times` | Preserve modification times |
|
||||||
| `-o, --owner` | Preserve the source owner (privilege-gated; mapped by name on the receiver with a numeric fallback) |
|
| `-o, --owner` | Preserve the source owner (privilege-gated; mapped by name on the receiver with a numeric fallback) |
|
||||||
| `-g, --group` | Preserve the source group (privilege-gated; mapped by name on the receiver with a numeric fallback) |
|
| `-g, --group` | Preserve the source group (privilege-gated; mapped by name on the receiver with a numeric fallback) |
|
||||||
@@ -204,9 +205,10 @@ This produces `./build/client` and `./build/server`. `compile_commands.json` is
|
|||||||
| `-u, --update` | Skip files newer than the source on the receiver |
|
| `-u, --update` | Skip files newer than the source on the receiver |
|
||||||
| `--incremental` | Skip files unchanged since last transfer (size + mtime). Auto-enables `--preserve`. Incompatible with `--chunk-serialization`. |
|
| `--incremental` | Skip files unchanged since last transfer (size + mtime). Auto-enables `--preserve`. Incompatible with `--chunk-serialization`. |
|
||||||
| `--existing` | Skip files not already present at the destination; update existing files normally. |
|
| `--existing` | Skip files not already present at the destination; update existing files normally. |
|
||||||
| `--compare-dest <dir>` | Extra comparison basis: unchanged files are not transferred (requires/implies `--incremental`) |
|
| `--ignore-existing` | Skip files that already exist on the receiver; like rsync it does not apply to directories or symlinks. |
|
||||||
| `--copy-dest <dir>` | Like `--compare-dest`, but copies the unchanged file from DIR into the destination |
|
| `--compare-dest <dir>` | Extra comparison basis: unchanged files are not transferred (requires/implies `--incremental`; a basis MISS above the 256 MiB whole-file payload bound is refused — see [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)) |
|
||||||
| `--link-dest <dir>` | Like `--copy-dest`, but hard-links the unchanged file from DIR (repeatable; earlier DIRs win) |
|
| `--copy-dest <dir>` | Like `--compare-dest`, but copies the unchanged file from DIR into the destination (same basis-size caveat; see [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)) |
|
||||||
|
| `--link-dest <dir>` | Like `--copy-dest`, but hard-links the unchanged file from DIR (repeatable; earlier DIRs win; same basis-size caveat; see [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)) |
|
||||||
| `--verify-basis` | FastSync-only: require a basis hit (`--compare-dest`/`--copy-dest`/`--link-dest`) to match the source by whole-file digest instead of trusting the size+mtime quick-check (default matches rsync) |
|
| `--verify-basis` | FastSync-only: require a basis hit (`--compare-dest`/`--copy-dest`/`--link-dest`) to match the source by whole-file digest instead of trusting the size+mtime quick-check (default matches rsync) |
|
||||||
| `--delete` | Delete files on receiver not present in source (default timing: delete-during, matching rsync, so destination space is freed progressively). Scoped to the synchronized directories, so `--files-from` subsets are safe |
|
| `--delete` | Delete files on receiver not present in source (default timing: delete-during, matching rsync, so destination space is freed progressively). Scoped to the synchronized directories, so `--files-from` subsets are safe |
|
||||||
| `--delete-before` | Delete extras before the transfer starts (implies `--delete`) |
|
| `--delete-before` | Delete extras before the transfer starts (implies `--delete`) |
|
||||||
@@ -216,16 +218,16 @@ This produces `./build/client` and `./build/server`. `compile_commands.json` is
|
|||||||
| `--delete-commit` | FastSync-only: keep the pre-2.28 atomic timing — delete only after the whole transfer succeeded (identical timing to `--delete-after`) |
|
| `--delete-commit` | FastSync-only: keep the pre-2.28 atomic timing — delete only after the whole transfer succeeded (identical timing to `--delete-after`) |
|
||||||
| `--delete-excluded` | Also delete filter-excluded destination mirrors (size-pruned mirrors stay protected) |
|
| `--delete-excluded` | Also delete filter-excluded destination mirrors (size-pruned mirrors stay protected) |
|
||||||
| `--max-delete <n>` | Delete at most n destination entries; the rest are skipped and the run exits 25 (partial), matching rsync |
|
| `--max-delete <n>` | Delete at most n destination entries; the rest are skipped and the run exits 25 (partial), matching rsync |
|
||||||
| `--delay-updates` | Put updated files into place only at the end of the transfer (`--force` is honored at publication) |
|
| `--delay-updates` | Put updated files into place only at the end of the transfer (`--force` is honored at publication; the fixed `.fastsync-stage` staging name diverges from rsync — see [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)) |
|
||||||
| `-T, --temp-dir <dir>` | Scratch directory for temp files before the atomic install; confined to the receive root (relative only), with an `EXDEV` non-atomic copy fallback |
|
| `-T, --temp-dir <dir>` | Scratch directory for temp files before the atomic install; confined to the receive root (relative only), with an `EXDEV` non-atomic copy fallback |
|
||||||
| `-n, --dry-run` | Report what would be transferred without mutating the destination. Since protocol 2.21.0 a server-routed target contacts the receiver and reports would-transfer based on receiver state; a plain local destination keeps the client-side scan. Never mutates or deletes. |
|
| `-n, --dry-run` | Report what would be transferred without mutating the destination. Since protocol 2.21.0 a server-routed target contacts the receiver and reports would-transfer based on receiver state; a plain local destination keeps the client-side scan. Never mutates or deletes. |
|
||||||
| `-v, --verbose` | Enable debug logging |
|
| `-v, --verbose` | Enable debug logging |
|
||||||
| `-q, --quiet` | Suppress non-error output |
|
| `-q, --quiet` | Suppress non-error output |
|
||||||
| `--progress` | Show rsync-style per-file progress blocks from the receiver's wire counters (FastSync does not print rsync's leading `./` line) |
|
| `--progress` | Show rsync-style per-file progress blocks from the receiver's wire counters; the root `./` line is printed whenever progress is active (rsync prints it only when the transfer root is created) |
|
||||||
| `-P` | Enables partial-transfer mode + progress output; interrupted writes retain the already-written temp for resumption |
|
| `-P` | Enables partial-transfer mode + progress output; interrupted writes retain the already-written temp for resumption |
|
||||||
| `--stats` | Print transfer statistics at end (bytes, files, timing), including the receiver-only counters reported over the wire; rsync's per-type `Number of files` breakdown is not reproduced |
|
| `--stats` | Print transfer statistics at end (bytes, files, timing), including the receiver-only counters reported over the wire; `Number of files` and `Number of created files` carry rsync's per-type breakdown (deleted files are reported as a single total) |
|
||||||
| `-i, --itemize-changes` | Print an rsync-style per-file change line |
|
| `-i, --itemize-changes` | Print an rsync-style per-file change line |
|
||||||
| `--out-format=FORMAT` | Output format for changed files (`%f %n %l %b %M %%`) |
|
| `--out-format=FORMAT` | Output format for changed files (`%f %n %l %b %c %C %i %M %%`) |
|
||||||
| `--list-only` | List source files instead of transferring |
|
| `--list-only` | List source files instead of transferring |
|
||||||
| `--fsync` | Fsync every written file before publication |
|
| `--fsync` | Fsync every written file before publication |
|
||||||
| `-h, --human-readable` | Format transfer byte/rate counts with rsync's decimal (base-1000) units |
|
| `-h, --human-readable` | Format transfer byte/rate counts with rsync's decimal (base-1000) units |
|
||||||
@@ -246,7 +248,7 @@ This produces `./build/client` and `./build/server`. `compile_commands.json` is
|
|||||||
| `-4, --ipv4` | Force IPv4 for destination resolution |
|
| `-4, --ipv4` | Force IPv4 for destination resolution |
|
||||||
| `-6, --ipv6` | Force IPv6 for destination resolution |
|
| `-6, --ipv6` | Force IPv6 for destination resolution |
|
||||||
| `--sockopts=OPTS` | Comma-separated OPT=VAL socket options applied before connect (`TCP_NODELAY`, `SO_KEEPALIVE`, `SO_RCVBUF`, `SO_SNDBUF`, `SO_REUSEADDR`) |
|
| `--sockopts=OPTS` | Comma-separated OPT=VAL socket options applied before connect (`TCP_NODELAY`, `SO_KEEPALIVE`, `SO_RCVBUF`, `SO_SNDBUF`, `SO_REUSEADDR`) |
|
||||||
| `--bwlimit <KB/s>` | Bandwidth limit in kilobytes per second |
|
| `--bwlimit <KB/s>` | Bandwidth limit in kilobytes per second; also paces `--sendfile` transfers |
|
||||||
| `--chunk-size <n>` | Chunk size in bytes (default: 10485760) |
|
| `--chunk-size <n>` | Chunk size in bytes (default: 10485760) |
|
||||||
| `--timeout <sec>` | I/O timeout in seconds, applied to both the socket (`SO_RCVTIMEO`/`SO_SNDTIMEO`) and the per-message protocol poll deadline. Default `0` = disabled (matching rsync); `0` disables it. `--no-timeout` is the negation. The value is not sent on the wire; the server side keeps its own safe floor. |
|
| `--timeout <sec>` | I/O timeout in seconds, applied to both the socket (`SO_RCVTIMEO`/`SO_SNDTIMEO`) and the per-message protocol poll deadline. Default `0` = disabled (matching rsync); `0` disables it. `--no-timeout` is the negation. The value is not sent on the wire; the server side keeps its own safe floor. |
|
||||||
| `--contimeout <sec>` | Connection timeout in seconds (default: 60, matching rsync); `0` disables it (`--no-contimeout` is the negation) |
|
| `--contimeout <sec>` | Connection timeout in seconds (default: 60, matching rsync); `0` disables it (`--no-contimeout` is the negation) |
|
||||||
@@ -314,17 +316,22 @@ transfer is never aborted.
|
|||||||
`timeout`, `contimeout`, `quiet`, `stats`, `max_depth`, and `log_file` are
|
`timeout`, `contimeout`, `quiet`, `stats`, `max_depth`, and `log_file` are
|
||||||
client-only.
|
client-only.
|
||||||
5. **Queue** — thread-safe bounded queue with condition variables.
|
5. **Queue** — thread-safe bounded queue with condition variables.
|
||||||
6. **DirectoryScanner** — recursive BFS traversal with exclude and include
|
6. **DirectoryScanner** — recursive traversal that buffers and sorts each
|
||||||
pattern support, max-depth enforcement.
|
directory (non-directories ascending, then directories ascending) and walks
|
||||||
|
depth-first in rsync flist order, with exclude and include pattern support and
|
||||||
|
max-depth enforcement.
|
||||||
|
|
||||||
### Key Algorithms
|
### Key Algorithms
|
||||||
|
|
||||||
1. **File scanning** — BFS directory traversal; entries matched against exclude
|
1. **File scanning** — sorted depth-first traversal in rsync flist order (each
|
||||||
and include patterns, with max-depth enforced.
|
directory's non-directories ascending, then its directories ascending);
|
||||||
|
entries matched against exclude and include patterns, with max-depth
|
||||||
|
enforced. The `--threads` parallel scanner remains unordered.
|
||||||
2. **Chunking** — files accumulated until the `chunk_size` threshold (default
|
2. **Chunking** — files accumulated until the `chunk_size` threshold (default
|
||||||
10 MiB) is reached, then flushed.
|
10 MiB) is reached, then flushed.
|
||||||
3. **Compression** — streaming zstd via `ZSTD_compressStream2()` /
|
3. **Compression** — streaming zstd via `ZSTD_compressStream2()` /
|
||||||
`ZSTD_decompressStream()`.
|
`ZSTD_decompressStream()`, with lz4 and zlib/zlibx codecs also supported
|
||||||
|
(selectable with `--compress-choice`).
|
||||||
4. **Network protocol** — status-code-driven exchange with metadata packing,
|
4. **Network protocol** — status-code-driven exchange with metadata packing,
|
||||||
keep-alive, and abort support.
|
keep-alive, and abort support.
|
||||||
5. **Incremental check** — the client sends `STATUS_CHECK` + path + size +
|
5. **Incremental check** — the client sends `STATUS_CHECK` + path + size +
|
||||||
@@ -379,6 +386,8 @@ Received files are written to a temporary path (suffixed with `.tmp`) and then a
|
|||||||
- C11 compiler
|
- C11 compiler
|
||||||
- CMake >= 3.22
|
- CMake >= 3.22
|
||||||
- zstd library
|
- zstd library
|
||||||
|
- zlib library
|
||||||
|
- lz4 library
|
||||||
- OpenSSL (development headers and libraries)
|
- OpenSSL (development headers and libraries)
|
||||||
- pthreads
|
- pthreads
|
||||||
- SSH client (for SSH transport mode only)
|
- SSH client (for SSH transport mode only)
|
||||||
@@ -387,12 +396,12 @@ Received files are written to a temporary path (suffixed with `.tmp`) and then a
|
|||||||
|
|
||||||
**Ubuntu/Debian:**
|
**Ubuntu/Debian:**
|
||||||
```bash
|
```bash
|
||||||
sudo apt install cmake build-essential libzstd-dev libssl-dev openssh-client
|
sudo apt install cmake build-essential libzstd-dev zlib1g-dev liblz4-dev libssl-dev openssh-client
|
||||||
```
|
```
|
||||||
|
|
||||||
**Nix:**
|
**Nix:**
|
||||||
```bash
|
```bash
|
||||||
nix-shell # provides zstd, openssl, cmake, gcc
|
nix-shell # provides zstd, zlib, lz4, openssl, cmake, gcc
|
||||||
```
|
```
|
||||||
|
|
||||||
## Building
|
## Building
|
||||||
@@ -526,9 +535,9 @@ features without changing the meaning of ordinary compatibility options.
|
|||||||
| `--server-host <host>` | Select the TCP server host. |
|
| `--server-host <host>` | Select the TCP server host. |
|
||||||
| `--server-port <port>` | Select the TCP server port (`--port <port>` and `--port=<port>` are rsync-friendly aliases). |
|
| `--server-port <port>` | Select the TCP server port (`--port <port>` and `--port=<port>` are rsync-friendly aliases). |
|
||||||
| `--tls` | Enable TLS for TCP transport. |
|
| `--tls` | Enable TLS for TCP transport. |
|
||||||
| `--bwlimit <KB/s>` | Apply token-bucket bandwidth limiting. |
|
| `--bwlimit <KB/s>` | Apply token-bucket bandwidth limiting (also paces `--sendfile` transfers). |
|
||||||
| `--progress` | Show rsync-style per-file progress blocks from the receiver's wire counters (FastSync omits rsync's leading `./` line). |
|
| `--progress` | Show rsync-style per-file progress blocks from the receiver's wire counters; the root `./` line is printed whenever progress is active (rsync prints it only when the transfer root is created). |
|
||||||
| `--stats` | Print transfer statistics, including the receiver-only counters reported over the wire; rsync's per-type `Number of files` breakdown is not reproduced. |
|
| `--stats` | Print transfer statistics, including the receiver-only counters reported over the wire; `Number of files`/`Number of created files` carry rsync's per-type breakdown (deleted files are a single total). |
|
||||||
| `--timeout <seconds>` | Set the socket **and** per-message protocol I/O timeout. Default `0` = disabled (matching rsync); `0` disables it. |
|
| `--timeout <seconds>` | Set the socket **and** per-message protocol I/O timeout. Default `0` = disabled (matching rsync); `0` disables it. |
|
||||||
| `--contimeout <seconds>` | Connection timeout (default 60, matching rsync); `0` disables it. |
|
| `--contimeout <seconds>` | Connection timeout (default 60, matching rsync); `0` disables it. |
|
||||||
|
|
||||||
@@ -562,23 +571,26 @@ remote SSH argv is already built injection-safe.
|
|||||||
| `--size-only` | Skip incremental files matching in size, ignoring mtime. |
|
| `--size-only` | Skip incremental files matching in size, ignoring mtime. |
|
||||||
| `-I, --ignore-times` | Transfer files even when size and mtime match. |
|
| `-I, --ignore-times` | Transfer files even when size and mtime match. |
|
||||||
| `-u, --update` | Skip files newer than the source on the receiver. |
|
| `-u, --update` | Skip files newer than the source on the receiver. |
|
||||||
|
| `--ignore-existing` | Skip files that already exist on the receiver; like rsync it does not apply to directories or symlinks. |
|
||||||
|
| `-@, --modify-window <sec>` | Modification-time tolerance (seconds) for the incremental/basis quick-check; `0` requires an exact mtime match. |
|
||||||
| `-W, --whole-file` | Transfer changed files without delta processing (`--no-whole-file` clears it). |
|
| `-W, --whole-file` | Transfer changed files without delta processing (`--no-whole-file` clears it). |
|
||||||
| `-B <n>, --block-size <n>` | Delta block size in bytes (alias `--delta-block`). |
|
| `-B <n>, --block-size <n>` | Delta block size in bytes (alias `--delta-block`). |
|
||||||
| `-d, --dirs` | Transfer the named directory entries without recursing into their contents (aliases `--old-dirs`/`--old-d`). |
|
| `-d, --dirs` | Transfer the named directory entries without recursing into their contents (aliases `--old-dirs`/`--old-d`). |
|
||||||
| `-R, --relative` | Use rsync's relative path semantics (including the `/./` cut); with `--files-from`, preserve each listed entry's relative path below the destination root. |
|
| `-R, --relative` | Use rsync's relative path semantics (including the `/./` cut); with `--files-from`, preserve each listed entry's relative path below the destination root. |
|
||||||
| `--files-from <file>` | Read the source file list from FILE (paths relative to the source root). |
|
| `--files-from <file>` | Read the source file list from FILE (paths relative to the source root). |
|
||||||
| `--delay-updates` | Put updated files into place only at the end of the transfer. |
|
| `-0, --from0` | Treat entries in `--files-from` files as NUL-delimited instead of newline-delimited. |
|
||||||
| `--compare-dest <dir>` | Extra comparison basis: unchanged files are not transferred (requires/implies `--incremental`). |
|
| `--delay-updates` | Put updated files into place only at the end of the transfer (the fixed `.fastsync-stage` staging name diverges from rsync; see [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)). |
|
||||||
| `--copy-dest <dir>` | Like `--compare-dest`, but copies the unchanged file from DIR into the destination. |
|
| `--compare-dest <dir>` | Extra comparison basis: unchanged files are not transferred (requires/implies `--incremental`; a basis MISS above the 256 MiB whole-file payload bound is refused — see [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)). |
|
||||||
| `--link-dest <dir>` | Like `--copy-dest`, but hard-links the unchanged file from DIR (repeatable; earlier DIRs win). |
|
| `--copy-dest <dir>` | Like `--compare-dest`, but copies the unchanged file from DIR into the destination (same basis-size caveat; see [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)). |
|
||||||
|
| `--link-dest <dir>` | Like `--copy-dest`, but hard-links the unchanged file from DIR (repeatable; earlier DIRs win; same basis-size caveat; see [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)). |
|
||||||
| `--verify-basis` | FastSync-only: require a basis hit to match the source by whole-file digest instead of trusting the size+mtime quick-check (default matches rsync). |
|
| `--verify-basis` | FastSync-only: require a basis hit to match the source by whole-file digest instead of trusting the size+mtime quick-check (default matches rsync). |
|
||||||
| `--preallocate` | Allocate destination file space up front (fail-fast on a full disk). |
|
| `--preallocate` | Allocate destination file space up front (fail-fast on a full disk). |
|
||||||
| `--append` | Resume a shorter destination by appending only its tail (prefix not verified; requires `--incremental`). |
|
| `--append` | Resume a shorter destination by appending only its tail (prefix not verified; requires `--incremental`). |
|
||||||
| `--append-verify` | Like `--append`, but verifies the retained prefix checksum first (falls back to a full transfer on mismatch). |
|
| `--append-verify` | Like `--append`, but verifies the retained prefix checksum first (falls back to a full transfer on mismatch). |
|
||||||
| `--delete` | Request removal of destination entries absent from the source. The server must allow deletion. Default timing is delete-after: extras are removed only after the whole transfer succeeded. Scoped to the synchronized directories, so `--files-from` subsets are safe. |
|
| `--delete` | Request removal of destination entries absent from the source. The server must allow deletion. Default timing is delete-during (matching rsync's `--del`): extras are removed per directory as the transfer proceeds, so destination space is freed progressively. Scoped to the synchronized directories, so `--files-from` subsets are safe. |
|
||||||
| `--delete-before` | Delete extras before the transfer starts (implies `--delete`). |
|
| `--delete-before` | Delete extras before the transfer starts (implies `--delete`). |
|
||||||
| `--delete-during`, `--del` | Delete extras once the keep-set manifest is known, before data is applied (implies `--delete`; early mode, same engine behaviour as `--delete-before`). |
|
| `--delete-during`, `--del` | Delete each directory's extras as that directory is processed (implies `--delete`). Since protocol 2.24.0 the sender streams a per-directory `STATUS_DELETE_PLAN` frame as it reaches each source directory; this is also the default timing of a plain `--delete`. |
|
||||||
| `--delete-delay` | Delete extras only after a successful transfer (implies `--delete`; commit mode, same behaviour as `--delete-after`). |
|
| `--delete-delay` | Record extras per directory during the scan but remove them only after a successful transfer (implies `--delete`). Uses the same per-directory `STATUS_DELETE_PLAN` frames as `--delete-during`, applied late. |
|
||||||
| `--delete-commit` | FastSync-only: atomic delete-after timing (only after the whole transfer succeeded). |
|
| `--delete-commit` | FastSync-only: atomic delete-after timing (only after the whole transfer succeeded). |
|
||||||
| `--delete-after` | Explicit delete-after timing: delete only after the transfer succeeded (implies `--delete`). |
|
| `--delete-after` | Explicit delete-after timing: delete only after the transfer succeeded (implies `--delete`). |
|
||||||
| `--delete-excluded` | Also delete filter-excluded destination mirrors (size-pruned mirrors stay protected). |
|
| `--delete-excluded` | Also delete filter-excluded destination mirrors (size-pruned mirrors stay protected). |
|
||||||
@@ -598,8 +610,8 @@ remote SSH argv is already built injection-safe.
|
|||||||
| `--backup-dir <dir>` | Store backups under a separate directory (requires `--backup`). |
|
| `--backup-dir <dir>` | Store backups under a separate directory (requires `--backup`). |
|
||||||
| `--suffix <suffix>` | Set the backup filename suffix (default: `~`). |
|
| `--suffix <suffix>` | Set the backup filename suffix (default: `~`). |
|
||||||
| `--partial` | Select partial-transfer handling. On failed/interrupted writes the already-written temp file is retained (best-effort) for resumption. With `--partial --partial-dir <dir>`, completed files are written under the partial directory and installed atomically. |
|
| `--partial` | Select partial-transfer handling. On failed/interrupted writes the already-written temp file is retained (best-effort) for resumption. With `--partial --partial-dir <dir>`, completed files are written under the partial directory and installed atomically. |
|
||||||
| `--partial-dir <dir>` | Set a relative partial-transfer directory below the server destination root. Use with `--partial`. |
|
| `--partial-dir <dir>` | Set a relative partial-transfer directory below the server destination root. Implies `--partial`. Rejected together with `--inplace` (`--inplace cannot be used with --partial-dir`, matching rsync), because the inplace path bypasses partial/temp staging. |
|
||||||
| `--inplace` | Write directly to the destination instead of using a temporary file. |
|
| `--inplace` | Write directly to the destination instead of using a temporary file. Cannot be combined with `--partial-dir`. |
|
||||||
| `--fsync` | Fsync every written file before publication. |
|
| `--fsync` | Fsync every written file before publication. |
|
||||||
| `--write-batch=FILE` | Run the normal live transfer and also emit a self-contained batch file of the source tree. |
|
| `--write-batch=FILE` | Run the normal live transfer and also emit a self-contained batch file of the source tree. |
|
||||||
| `--only-write-batch=FILE` | Emit the batch file only (no destination, no server). |
|
| `--only-write-batch=FILE` | Emit the batch file only (no destination, no server). |
|
||||||
@@ -614,8 +626,11 @@ remote SSH argv is already built injection-safe.
|
|||||||
| `--preserve` | Preserve mode and mtime (long form only; equivalent to `-p` + `-t`). Add `-o`/`-g` for owner/group, `-U`/`--atimes` for atime, or an identity flag (`--chown`/`--usermap`/`--groupmap`/`--numeric-ids`/`--copy-as`) for mapped ownership. |
|
| `--preserve` | Preserve mode and mtime (long form only; equivalent to `-p` + `-t`). Add `-o`/`-g` for owner/group, `-U`/`--atimes` for atime, or an identity flag (`--chown`/`--usermap`/`--groupmap`/`--numeric-ids`/`--copy-as`) for mapped ownership. |
|
||||||
| `-U`, `--atimes` | Preserve access times. Captured with the metadata payload; does not enable ownership. |
|
| `-U`, `--atimes` | Preserve access times. Captured with the metadata payload; does not enable ownership. |
|
||||||
| `-N`, `--crtimes` | Capture birth time and transmit it; it cannot be applied because no portable filesystem call can set a birth time (documented divergence). |
|
| `-N`, `--crtimes` | Capture birth time and transmit it; it cannot be applied because no portable filesystem call can set a birth time (documented divergence). |
|
||||||
| `-p`, `--perms` | Preserve permission bits. One of the four per-attribute preserve flags (with `-t`/`-o`/`-g`); under `-p` the source mode is copied exactly (setuid/setgid/sticky and group/other-write included), matching rsync. |
|
| `-p`, `--perms` | Preserve permission bits. One of the four per-attribute preserve flags (with `-t`/`-o`/`-g`); under `-p` the source mode is copied exactly (group/other-write included; setuid/setgid/sticky included only when super-user activities are permitted, masked under `SUPER_MODE_OFF`/`--no-super`), matching rsync otherwise. |
|
||||||
| `-t`, `--times` | Preserve modification times. Independent of the other attributes; `-O`/`--omit-dir-times` suppresses directories only. |
|
| `-t`, `--times` | Preserve modification times. Independent of the other attributes; `-O`/`--omit-dir-times` suppresses directories only. |
|
||||||
|
| `-O`, `--omit-dir-times` | Do not apply modification times to directories. |
|
||||||
|
| `-J`, `--omit-link-times` | Do not apply times to symlinks. |
|
||||||
|
| `--open-noatime` | Open source files with `O_NOATIME` so reading for a transfer does not update their access time (client-only). |
|
||||||
| `-o`, `--owner` | Preserve the source owner (uid). Mapped by name on the receiver with a raw-numeric fallback (only numeric ids cross the wire); application is privilege-gated. |
|
| `-o`, `--owner` | Preserve the source owner (uid). Mapped by name on the receiver with a raw-numeric fallback (only numeric ids cross the wire); application is privilege-gated. |
|
||||||
| `-g`, `--group` | Preserve the source group (gid). Same name-mapping/numeric-fallback and privilege gating as `-o`. |
|
| `-g`, `--group` | Preserve the source group (gid). Same name-mapping/numeric-fallback and privilege gating as `-o`. |
|
||||||
| `--no-perms`, `--no-times`, `--no-owner`, `--no-group` | Negate each per-attribute flag (also `--no-p`/`--no-t`/`--no-o`/`--no-g`); `--no-preserve` clears all four. |
|
| `--no-perms`, `--no-times`, `--no-owner`, `--no-group` | Negate each per-attribute flag (also `--no-p`/`--no-t`/`--no-o`/`--no-g`); `--no-preserve` clears all four. |
|
||||||
@@ -642,6 +657,7 @@ remote SSH argv is already built injection-safe.
|
|||||||
| `-D` | Preserve device and special files (implies `--devices --specials`). |
|
| `-D` | Preserve device and special files (implies `--devices --specials`). |
|
||||||
| `--devices` | Recreate device nodes on the destination (privileged; skipped without `CAP_MKNOD`). |
|
| `--devices` | Recreate device nodes on the destination (privileged; skipped without `CAP_MKNOD`). |
|
||||||
| `--specials` | Recreate special files: FIFOs and unix sockets. |
|
| `--specials` | Recreate special files: FIFOs and unix sockets. |
|
||||||
|
| `--copy-devices` | Copy a source device's content as an ordinary regular file on the destination (rsync's non-privileged safe mode) instead of recreating the device node. |
|
||||||
| `-S`, `--sparse` | Sparse-file handling: receiver preserves holes (zero runs are written as holes; no wire change). |
|
| `-S`, `--sparse` | Sparse-file handling: receiver preserves holes (zero runs are written as holes; no wire change). |
|
||||||
|
|
||||||
### Output and logging
|
### Output and logging
|
||||||
@@ -650,12 +666,17 @@ remote SSH argv is already built injection-safe.
|
|||||||
|---|---|
|
|---|---|
|
||||||
| `-v`, `--verbose` | Enable debug logging. |
|
| `-v`, `--verbose` | Enable debug logging. |
|
||||||
| `-q`, `--quiet` | Suppress non-error output. |
|
| `-q`, `--quiet` | Suppress non-error output. |
|
||||||
| `--progress` | Show rsync-style per-file progress blocks (not rsync's leading `./` line). |
|
| `--progress` | Show rsync-style per-file progress blocks; the root `./` line is printed whenever progress is active (rsync prints it only when the transfer root is created). |
|
||||||
| `--stats` | Print transfer statistics, including the receiver-only counters reported over the wire. |
|
| `--stats` | Print transfer statistics, including the receiver-only counters reported over the wire; `Number of files`/`Number of created files` carry rsync's per-type breakdown (deleted files are a single total). |
|
||||||
| `-i`, `--itemize-changes` | Print an rsync-style per-file change line. |
|
| `-i, --itemize-changes` | Print an rsync-style per-file change line. |
|
||||||
| `--out-format=FORMAT` | Output format for changed files (`%f %n %l %b %M %%`). |
|
| `--out-format=FORMAT` | Output format for changed files (`%f %n %l %b %c %C %i %M %%`). |
|
||||||
| `--list-only` | List source files instead of transferring. |
|
| `--list-only` | List source files instead of transferring. |
|
||||||
|
| `--outbuf=MODE` | stdout/stderr buffering: `N` (none/unbuffered), `L` (line-buffered), or `B` (block-buffered, default). |
|
||||||
| `--log-file <path>` | Write log output to a file. |
|
| `--log-file <path>` | Write log output to a file. |
|
||||||
|
| `--log-file-format=FORMAT` | Per-file log-line format (requires `--log-file`). |
|
||||||
|
| `--stderr=MODE` | Route logging to stderr: `errors` or `all`. |
|
||||||
|
| `--msgs2stderr` | Route all messages to stderr (deprecated spelling of `--stderr=all`). |
|
||||||
|
| `--no-msgs2stderr` | Select errors-only stderr (deprecated spelling; the default). |
|
||||||
| `-V`, `--version` | Print the FastSync protocol version. |
|
| `-V`, `--version` | Print the FastSync protocol version. |
|
||||||
| `--help` | Print command usage. |
|
| `--help` | Print command usage. |
|
||||||
|
|
||||||
@@ -680,6 +701,11 @@ remote SSH argv is already built injection-safe.
|
|||||||
| `-4`, `--ipv4` | Force IPv4 for destination resolution. |
|
| `-4`, `--ipv4` | Force IPv4 for destination resolution. |
|
||||||
| `-6`, `--ipv6` | Force IPv6 for destination resolution. |
|
| `-6`, `--ipv6` | Force IPv6 for destination resolution. |
|
||||||
| `--sockopts=OPTS` | Comma-separated OPT=VAL socket options applied before connect. |
|
| `--sockopts=OPTS` | Comma-separated OPT=VAL socket options applied before connect. |
|
||||||
|
| `--blocking-io` | SSH transport only: leave the socket without read/write timeouts so it blocks naturally (no effect on TCP). |
|
||||||
|
| `--protocol=NUM` | Force the wire protocol version; must equal the current `PROTOCOL_VERSION` (FastSync cannot speak older/virtual wire formats). |
|
||||||
|
| `--old-args` | Accepted for rsync CLI compatibility; no effect (the remote server path is always safely quoted). |
|
||||||
|
| `--iconv=LOCAL[,REMOTE]` | Convert file-name charsets at the wire boundary (`LOCAL` is our names' charset, `REMOTE` the peer's, defaulting to `LOCAL`). |
|
||||||
|
| `--no-iconv` | Disable `--iconv` charset conversion (same as `--iconv=-`). |
|
||||||
| `--tls` | Enable TLS. Requires `--cert`, `--key`, and `--ca`. |
|
| `--tls` | Enable TLS. Requires `--cert`, `--key`, and `--ca`. |
|
||||||
| `--cert <path>` | TLS certificate file. |
|
| `--cert <path>` | TLS certificate file. |
|
||||||
| `--key <path>` | TLS private key file. |
|
| `--key <path>` | TLS private key file. |
|
||||||
|
|||||||
+57
-28
@@ -6,8 +6,8 @@ This document maps rsync's full feature set to FastSync's current implementation
|
|||||||
|
|
||||||
| Status | Count | Description |
|
| Status | Count | Description |
|
||||||
|--------|-------|-------------|
|
|--------|-------|-------------|
|
||||||
| ✅ Parity | 116 | Reproduces rsync's semantics for this option's scope |
|
| ✅ Parity | 119 | Reproduces rsync's semantics for this option's scope |
|
||||||
| ⚠️ Caveat | 14 | Wired and tested, but carries a documented behavioral difference from rsync (named in the row and/or the wave notes) |
|
| ⚠️ Caveat | 11 | Wired and tested, but carries a documented behavioral difference from rsync (named in the row and/or the wave notes) |
|
||||||
| ❌ Divergent | 27 | Rejected, an accepted no-op, deliberately non-rsync (native config/auth/batch, privileged namespaces, safe-subset privilege), or impossible on any portable filesystem call |
|
| ❌ Divergent | 27 | Rejected, an accepted no-op, deliberately non-rsync (native config/auth/batch, privileged namespaces, safe-subset privilege), or impossible on any portable filesystem call |
|
||||||
| **Total** | **157** | One row per rsync option/feature group; a row may name several spellings |
|
| **Total** | **157** | One row per rsync option/feature group; a row may name several spellings |
|
||||||
|
|
||||||
@@ -55,9 +55,30 @@ matrix is **111 ✅ / 13 ⚠️ / 33 ❌ = 157**.
|
|||||||
|
|
||||||
**Lockstep track 6 (delete default; `PROTOCOL_VERSION` stays 2.28.0).** Plain `--delete` with no explicit timing flag now defaults to rsync's delete-during (`--del`) timing: the client normalizes it onto the existing `delete_during` wire bool in `cli_finalize_config`, so no config-frame field was added, and a tight destination no longer has to hold the whole old+new tree at once (the old atomic commit could hit `ENOSPC`). The old late whole-tree commit is opt-in via `--delete-after` or the FastSync-only long spelling `--delete-commit`, which selects the identical `delete_after` timing (documented equivalence). Precedence is unchanged and order-independent: each timing flag implies `--delete`, at most one timing flag may be given, and a timing flag with `--no-delete` is rejected. `-d/--dirs` still falls back to the end commit; `--delay-updates` still deletes genuine extras before publication (the per-directory skip list protects the staging dir); `--files-from`/`-R` scope is unchanged. The per-directory `STATUS_DELETE_PLAN` frame gained a one-int `apply` flag (still 2.28.0): the one-shot per-run config block (protected prefixes, size-pruned mirrors, `--delete-missing-args` exact paths) is now always sent first on a config-only carrier with `apply=false`, fixing a latent bug where a `--delete-missing-args` run whose `--files-from` list synchronized no directory never transmitted its exact deletions. Differential evidence: `delete` (plain, vs rsync's default), `delete_commit` (FastSync `--delete-commit` vs rsync `--delete-after`), and `filter_protect_after` (whole-tree protect) cases; `TestDeleteTimingFinalStateParity` compares plain `--delete`/`--delete-commit` against rsync on completed runs, and `TestDeleteTimingFailure` proves plain `--delete` removes reached extras on a mid-transfer abort while `--delete-commit` removes nothing. The matrix is unchanged at **116 ✅ / 14 ⚠️ / 27 ❌ = 157** (the `--delete`/`--delete-during` rows stay ⚠️ for the abort boundary; `--delete-after` stays ✅).
|
**Lockstep track 6 (delete default; `PROTOCOL_VERSION` stays 2.28.0).** Plain `--delete` with no explicit timing flag now defaults to rsync's delete-during (`--del`) timing: the client normalizes it onto the existing `delete_during` wire bool in `cli_finalize_config`, so no config-frame field was added, and a tight destination no longer has to hold the whole old+new tree at once (the old atomic commit could hit `ENOSPC`). The old late whole-tree commit is opt-in via `--delete-after` or the FastSync-only long spelling `--delete-commit`, which selects the identical `delete_after` timing (documented equivalence). Precedence is unchanged and order-independent: each timing flag implies `--delete`, at most one timing flag may be given, and a timing flag with `--no-delete` is rejected. `-d/--dirs` still falls back to the end commit; `--delay-updates` still deletes genuine extras before publication (the per-directory skip list protects the staging dir); `--files-from`/`-R` scope is unchanged. The per-directory `STATUS_DELETE_PLAN` frame gained a one-int `apply` flag (still 2.28.0): the one-shot per-run config block (protected prefixes, size-pruned mirrors, `--delete-missing-args` exact paths) is now always sent first on a config-only carrier with `apply=false`, fixing a latent bug where a `--delete-missing-args` run whose `--files-from` list synchronized no directory never transmitted its exact deletions. Differential evidence: `delete` (plain, vs rsync's default), `delete_commit` (FastSync `--delete-commit` vs rsync `--delete-after`), and `filter_protect_after` (whole-tree protect) cases; `TestDeleteTimingFinalStateParity` compares plain `--delete`/`--delete-commit` against rsync on completed runs, and `TestDeleteTimingFailure` proves plain `--delete` removes reached extras on a mid-transfer abort while `--delete-commit` removes nothing. The matrix is unchanged at **116 ✅ / 14 ⚠️ / 27 ❌ = 157** (the `--delete`/`--delete-during` rows stay ⚠️ for the abort boundary; `--delete-after` stays ✅).
|
||||||
|
|
||||||
|
**Parity cycle 2.29 (on `feat/parity-2.29`; `PROTOCOL_VERSION` stays 2.28.0 — no wire change was needed).** Five independent residuals were closed and four rows moved to ✅:
|
||||||
|
- **Scanner order.** The sequential scanner now buffers and sorts each directory's inspected entries (non-directories ascending, then directories ascending) and walks them depth-first, reproducing rsync 3.4.1's flist order. This makes the `--info=name` transfer order, the `--delete-during`/`--delete-delay`/`-n` would-delete order, and the partial-`--max-delete` survivor set byte-identical to rsync (`test_parity_order.py`). `--threads` has no rsync analogue and stays unordered.
|
||||||
|
- **Delete timing.** The complete per-directory plan set is transmitted before the first data frame, so a mid-transfer abort has already removed every planned extra like rsync's generator; `-d/--dirs` uses the same per-directory plans (shielded untraversed subdirectories) instead of the end-of-transfer commit (`test_delete_boundary_parity.py`). `-n`/`--delete`/`--del`/`--delete-delay` move ⚠️ → ✅.
|
||||||
|
- **Basis relative-DIR.** A relative `--compare-dest`/`--copy-dest`/`--link-dest` DIR resolves against the destination directory with the transfer-relative name appended, exactly like rsync (`test_parity_basis_fuzzy.py`); the >256 MiB basis-MISS limit remains (a general whole-file limit, not basis-specific).
|
||||||
|
- **Fuzzy eligibility.** The `-y/--fuzzy` candidate search no longer inherits the ordinary delta engine's 16 KiB minimum or 10× ratio bound, so an oversized or sub-16-KiB sibling is reused as rsync reuses it (`test_parity_basis_fuzzy.py`).
|
||||||
|
- **Output partials.** `--info=mount`/`--info=stats`, the `--stats` `dir:` breakdown under `-r`, and real `--debug` output for `flist`/`del`/`hash`/`deltasum`/`recv`/`filter`/`send` were added (`test_parity_info_mount_stats.py`, `test_output_parity.py`, `test_parity_debug.py`); those rows stay ⚠️ for their remaining documented residuals. `--delete-before`'s phase-0 late-file divergence and the `--progress` root/ancestor/symlink feedback remain open (they need a receiver→sender event channel), and the >256 MiB single-file streaming limit (B4) was not addressed. The matrix is now **120 ✅ / 10 ⚠️ / 27 ❌ = 157**.
|
||||||
|
|
||||||
|
**Audit cycle (no wire change; `PROTOCOL_VERSION` stays 2.28.0).** A security-and-correctness audit pass ran against the parity-2.29 baseline, followed by a set of audit follow-ups (filter merge modifiers, the `--inplace`/`--partial-dir` conflict, credential-file hardening, and small leak/log/test fixes). The only classification change is `--filter=RULE` moving ✅ → ⚠️, because its merge-only `e`/`n`/`w`/`-` modifiers are now accepted and consumed but their semantics remain unimplemented (accepted-but-ignored); the matrix is therefore **119 ✅ / 11 ⚠️ / 27 ❌ = 157**. The affected rows (`-z`/`--compress`, `--bwlimit`, `-T`/`--temp-dir`, `-p`/`--chmod`, `--partial-dir`, `--filter`) had their notes updated in place:
|
||||||
|
|
||||||
|
- **Decompression ceiling.** `MAX_DECOMPRESSED_SIZE` was 100 MiB while the receiver advertises and the sender compresses whole files up to `MAX_RECEIVE_WHOLE_FILE_SIZE` (256 MiB), so `-z` on a 100–256 MiB regular file failed with `Declared decompressed size exceeds 104857600 bytes`. The ceiling is now defined in terms of the protocol whole-file bound (still a real allocation-clamped bomb guard), so the two cannot drift; `-z` on 100–256 MiB files now works.
|
||||||
|
- **`--bwlimit` with `--sendfile`.** The plaintext-TCP `--sendfile` fast path wrote through `sendfile(2)` without passing through the protocol's token bucket, so `--bwlimit` was ignored on that path. It is now paced through the same per-session leaky bucket, so TLS and plaintext transports share identical `--bwlimit` semantics.
|
||||||
|
- **`--temp-dir` confinement.** The receiver's scratch dir was opened with a bare `open()`, so a client-planted symlink under the receive root could redirect receiver scratch files outside the authorized root. The opened directory is now judged by the real path of its fd (`/proc/self/fd` via `realpath`), and an escaping target is refused (`EACCES`, logged); an in-root link to another filesystem (the `EXDEV` fallback case) still works.
|
||||||
|
- **Special-bit masking and daemon umask.** Setuid/setgid/sticky bits from the client (`--perms`, `--chmod`, symlink and special-node paths, deferred directory modes) were applied even when the connection forbade super-user activities. They are now stripped when the super policy is off (`FileAttrPolicy.super_permitted`), and exact rsync semantics are preserved when permitted. The daemon's forced `umask(0)` is now `umask(022)`, so implied parent directories are no longer world-writable `0777`.
|
||||||
|
- **`--partial-dir` implies `--partial`.** Matching rsync 3.4.1 (which sets `keep_partial` after option parsing), `--partial-dir=DIR` alone now retains an interrupted transfer's partial and wins over an explicit `--no-partial`; `--inplace` still bypasses the partial machinery, and combining `--inplace` with `--partial-dir` is now rejected up front with rsync's message (`--inplace cannot be used with --partial-dir`) instead of silently ignoring the partial dir.
|
||||||
|
- **Filter modifiers.** The `x` xattr-name modifier is rejected everywhere with a clear error. The merge-only `e`/`n`/`w` and `-` modifiers are now accepted and consumed on `merge`/`dir-merge` rules (so they no longer leak into the merge filename) while still being rejected on non-merge rules, matching rsync; their semantics remain unimplemented (accepted-but-ignored). Glued patterns (`-newfile`, `-e2e`) and mixed tokens (`H,!secret`) keep their historical parsing.
|
||||||
|
- **Bounds and wire validation.** `--filter` rule count is now checked client-side against `MAX_FILTER_RULES` (with an actionable message before any network I/O) rather than surfacing as an opaque receiver protocol error; `send_protect_entries()` still re-checks the expanded count. Unknown wire `Status` values are rejected as protocol errors (`status_is_valid()`), and the audit also fixed a mutex leak on an init-failure path, an `errno`-after-`free()` in deferred delete application, `log_perror` misuse for non-`errno` conditions, `SSL_read` length clamping, `sendfile` `poll` `EINTR` retry, and printf-format/attribute issues.
|
||||||
|
- **Credential-file hardening follow-up.** `secret_file_open()` now opens `--password-file`/`--early-input`/`--hash-credentials` inputs with `O_NOFOLLOW`, so a symlinked credential path fails closed (`ELOOP`) instead of being followed before the owner/mode gate; literal fd-backed paths (`/dev/fd/<digits>`, `/proc/self/fd/<digits>`, which is what a bash process substitution passes) are exempt, so process substitution still works. A FIFO/process-substitution read now waits under a bounded ~3 s deadline for its writer, so a slow producer works while a connected-but-silent FIFO fails instead of hanging. The follow-up also fixed a `config_create` allocation leak on its `server_host` failure path (`config_delete` now releases it), corrected the decompression-limit log message to print the effective bound rather than the compile-time ceiling, and hardened the daemon umask/root test fixtures.
|
||||||
|
|
||||||
**Parity completion wave (protocol 2.23.0 → 2.26.0).** This wave closed the
|
**Parity completion wave (protocol 2.23.0 → 2.26.0).** This wave closed the
|
||||||
remaining gaps the rsync-parity wave left open (delete timing, wire counters and
|
remaining gaps the rsync-parity wave left open (delete timing, wire counters and
|
||||||
output, codec breadth, general `-R`/`-d`, the full filter grammar, receiver-side
|
output, codec breadth, general `-R`/`-d`, the filter grammar (the unsupported
|
||||||
|
`x` xattr-name modifier is explicitly rejected everywhere, while the merge-only
|
||||||
|
`e`/`n`/`w`/`-` modifiers are accepted and consumed on merge/dir-merge rules and
|
||||||
|
rejected elsewhere — see the audit-cycle follow-up note above), receiver-side
|
||||||
name resolution, absolute basis dirs, and the remaining client quick wins) and
|
name resolution, absolute basis dirs, and the remaining client quick wins) and
|
||||||
reclassified the inherently non-rsync rows as **divergent** (native daemon
|
reclassified the inherently non-rsync rows as **divergent** (native daemon
|
||||||
config/auth, the non-interoperable batch container, `--fake-super`'s xattr
|
config/auth, the non-interoperable batch container, `--fake-super`'s xattr
|
||||||
@@ -85,8 +106,8 @@ Every one of those has an entry below with its remaining caveats.
|
|||||||
| `-q`, `--quiet` | Suppress non-error messages | ✅ Parity | Suppresses client output while preserving errors |
|
| `-q`, `--quiet` | Suppress non-error messages | ✅ Parity | Suppresses client output while preserving errors |
|
||||||
| `--help` | Show help | ✅ Parity | Prints usage and exits. A lone `-h` with no other transfer arguments also prints help (protocol 2.26.0), matching the rsync idiom; `-h` alongside a transfer keeps its rsync meaning of `--human-readable` (see that row) |
|
| `--help` | Show help | ✅ Parity | Prints usage and exits. A lone `-h` with no other transfer arguments also prints help (protocol 2.26.0), matching the rsync idiom; `-h` alongside a transfer keeps its rsync meaning of `--human-readable` (see that row) |
|
||||||
| `-V`, `--version` | Print version | ✅ Parity | |
|
| `-V`, `--version` | Print version | ✅ Parity | |
|
||||||
| `--info=FLAGS` | Fine-grained info verbosity | ⚠️ Caveat | Accepts rsync 3.4.1's full `--info` vocabulary — `backup`, `copy`, `del`, `flist`, `misc`, `mount`, `name`, `nonreg`, `progress`, `remove`, `skip`, `stats`, `symsafe`, `all`, `none` — with optional level suffixes (`--info=stats2`), so a valid rsync invocation is never rejected up front. Protocol 2.27.0 wires the categories that map to a real FastSync event, matching rsync's line format: `name` prints the updated entry names (with the ` -> target` link suffix), `flist` prints `sending incremental file list`, `del` prints `deleting PATH` (or `*deleting PATH` under `-i`/`--out-format`) for both dry-run would-delete and real deletions (real runs carry the removed paths over the new `report_deletes` wire bool), `remove` prints `sender removed PATH`, `nonreg` prints `skipping non-regular file "NAME"`, `progress` drives the per-file progress output, and `copy`/`misc`/`skip`/`stats` keep their existing channels. `none` suppresses info output, explicit flags override `--verbose`, and a genuinely unknown name is still rejected by name (matching rsync). **Fixed (no-wire):** `--info=name2` (and higher) also prints rsync's `NAME is uptodate` lines for entries the receiver already has, and `--info=name` emits the leading transfer-root `./` name line before the first transferred entry (the marker rides in the existing `info_level` bitset; differential tests vs rsync 3.4.1). **Caveat:** the root `./` line is emitted before the first transferred name rather than keyed off rsync's root-attribute-change decision, so a pre-existing root that rsync leaves untouched can differ; the categories with no client-observable event stay accepted-but-silent — `symsafe`, `mount`, and `backup` (the backup happens on the receiver, which FastSync's protocol does not echo back); and `skip` maps to FastSync's sender-side skip logging rather than rsync's receiver-side "not creating new file" lines |
|
| `--info=FLAGS` | Fine-grained info verbosity | ⚠️ Caveat | Accepts rsync 3.4.1's full `--info` vocabulary — `backup`, `copy`, `del`, `flist`, `misc`, `mount`, `name`, `nonreg`, `progress`, `remove`, `skip`, `stats`, `symsafe`, `all`, `none` — with optional level suffixes (`--info=stats2`), so a valid rsync invocation is never rejected up front. Protocol 2.27.0 wires the categories that map to a real FastSync event, matching rsync's line format: `name` prints the updated entry names (with the ` -> target` link suffix), `flist` prints `sending incremental file list`, `del` prints `deleting PATH` (or `*deleting PATH` under `-i`/`--out-format`) for both dry-run would-delete and real deletions (real runs carry the removed paths over the new `report_deletes` wire bool), `remove` prints `sender removed PATH`, `nonreg` prints `skipping non-regular file "NAME"`, `progress` drives the per-file progress output, `copy`/`misc`/`skip` keep their existing channels, `stats` enables the same transfer-statistics block as `--stats`, and `mount` prints rsync's `[sender] skipping mount-point dir NAME` when `-xx` drops a mount-point directory (plain `-x` keeps the empty directory entry and stays silent, matching rsync; both differential-tested). `none` suppresses info output, explicit flags override `--verbose`, and a genuinely unknown name is still rejected by name (matching rsync). **Fixed (no-wire):** `--info=name2` (and higher) also prints rsync's `NAME is uptodate` lines for entries the receiver already has, and `--info=name` emits the leading transfer-root `./` name line before the first transferred entry (the marker rides in the existing `info_level` bitset; differential tests vs rsync 3.4.1). **Caveat:** the root `./` line is emitted before the first transferred name rather than keyed off rsync's root-attribute-change decision, so a pre-existing root that rsync leaves untouched can differ; the categories with no client-observable event stay accepted-but-silent — `symsafe` and `backup` (the backup happens on the receiver, which FastSync's protocol does not echo back); and `skip` maps to FastSync's sender-side skip logging rather than rsync's receiver-side "not creating new file" lines |
|
||||||
| `--debug=FLAGS` | Fine-grained debug verbosity | ⚠️ Caveat | Protocol 2.26.0 accepts rsync 3.4.1's full `--debug` vocabulary with optional level suffixes. FastSync emits for its own channels (`io`, `proto`, `pack`, `util`, plus the aliases `hl`/`owner`); the rsync-only categories (`acl`, `filter`, `send`, ...) are accepted silently. `--debug=help` lists the flags; a genuinely unknown name is rejected by name. **Caveat:** most accepted rsync categories produce no output (e.g. `acl`, `filter`, `send`, `flist`, `del`, `deltasum`, `hash`, `recv`, `time`), so they are accepted for CLI compatibility only |
|
| `--debug=FLAGS` | Fine-grained debug verbosity | ⚠️ Caveat | Protocol 2.26.0 accepts rsync 3.4.1's full `--debug` vocabulary with optional level suffixes. FastSync emits for its own channels (`io`, `proto`, `pack`, `util`, plus the aliases `hl`/`owner`) and maps the remaining categories that have a natural FastSync event onto real debug output: `flist` (per-directory scan progress), `del` (receiver-removed paths, riding the existing `report_deletes` wire bool), `hash`/`deltasum` (whole-file hashing and delta-sum generation), `recv` (receiver verdicts/signatures), `filter` (selection/exclusion decisions) and `send` (files handed to the sender). A normal run prints none of it; `--debug=help` lists the flags and a genuinely unknown name is rejected by name. **Caveat:** the output is FastSync's own timestamped debug format (it does not reproduce rsync's exact per-category lines), and the synthetic/rsync-internal categories (`acl`, `backup`, `bind`, `time`, ...) stay accepted-but-silent, so the row remains ⚠️ |
|
||||||
| `--stderr=MODE` | Change stderr output mode | ❌ Divergent | `errors` (default) and `all` are supported; `client` is rejected with a clear error (`--stderr=client is not supported`) because FastSync has no rsync client-message channel — the rejection itself is the documented behavior (Phase 7 Wave B decision). The modes that exist work; the missing rsync channel cannot be emulated without a wire change |
|
| `--stderr=MODE` | Change stderr output mode | ❌ Divergent | `errors` (default) and `all` are supported; `client` is rejected with a clear error (`--stderr=client is not supported`) because FastSync has no rsync client-message channel — the rejection itself is the documented behavior (Phase 7 Wave B decision). The modes that exist work; the missing rsync channel cannot be emulated without a wire change |
|
||||||
| `--msgs2stderr`, `--no-msgs2stderr` | Deprecated `--stderr` aliases | ⚠️ Caveat | `--msgs2stderr` maps to `--stderr=all` (supported, matching rsync). `--no-msgs2stderr` is rsync's spelling of `--stderr=client`, which FastSync has no client-message channel for, so it maps to the errors-only default instead of reproducing rsync's client mode. See `--stderr=MODE` |
|
| `--msgs2stderr`, `--no-msgs2stderr` | Deprecated `--stderr` aliases | ⚠️ Caveat | `--msgs2stderr` maps to `--stderr=all` (supported, matching rsync). `--no-msgs2stderr` is rsync's spelling of `--stderr=client`, which FastSync has no client-message channel for, so it maps to the errors-only default instead of reproducing rsync's client mode. See `--stderr=MODE` |
|
||||||
| `--no-motd` | Suppress daemon MOTD | ✅ Parity | Client-only display switch (Wave C): the daemon still sends the configured `motd file` on a `host::module/path` connection; the client reads and discards the frame without showing it. Without the flag the MOTD is printed to stdout after the config/auth handshake and escaped so control bytes cannot inject terminal sequences |
|
| `--no-motd` | Suppress daemon MOTD | ✅ Parity | Client-only display switch (Wave C): the daemon still sends the configured `motd file` on a `host::module/path` connection; the client reads and discards the frame without showing it. Without the flag the MOTD is printed to stdout after the config/auth handshake and escaped so control bytes cannot inject terminal sequences |
|
||||||
@@ -98,10 +119,10 @@ Every one of those has an entry below with its remaining caveats.
|
|||||||
|
|
||||||
| Flag | Rsync Description | FastSync Status | Notes |
|
| Flag | Rsync Description | FastSync Status | Notes |
|
||||||
|------|-------------------|-----------------|-------|
|
|------|-------------------|-----------------|-------|
|
||||||
| `--stats` | Give transfer stats | ⚠️ Caveat | Prints transfer statistics. Protocol 2.25.0 populates the receiver-only counters the sender cannot observe (`Matched data`, `Number of deleted files`) from the receiver's `STATUS_STATS` report; the sender tracks the scanned file list per type so `Number of files` carries rsync's `(reg: X, dir: Y, link: Z, special: W)` breakdown (directories come from the scanner's captured directory list, present for `-a`/`-t`/`-p`), `Number of regular files transferred` excludes symlinks/specials and up-to-date files, `Total file size` includes symlink target lengths, and `Total transferred file size` counts only transferred files. **Protocol 2.28.0 extends `STATUS_STATS`** with receiver-observed `literal_bytes` and the four `created_*` counters: `Number of created files` now carries rsync's `(reg/dir/link/special)` breakdown (the receiver reports which destination entries it newly created, including implicitly-created parent directories below the transfer root) and `Literal data` is exact for a delta transfer (the receiver counts the literal fragments it stored, not the whole source size) — all differential-tested in the sequential and `--threads` paths against rsync 3.4.1 for fresh-create, update and delta shapes. **Remaining divergences:** a recursive scan that preserves no directory attribute (`-r` without `-t`/`-p`) captures no directory entries, so the `dir:` category is then omitted from `Number of files`; rsync's per-type breakdown on `Number of deleted files` is not reproduced; and `Total bytes sent`/`received` are FastSync wire bytes framed differently from rsync's, so they are not numerically comparable |
|
| `--stats` | Give transfer stats | ⚠️ Caveat | Prints transfer statistics. Protocol 2.25.0 populates the receiver-only counters the sender cannot observe (`Matched data`, `Number of deleted files`) from the receiver's `STATUS_STATS` report; the sender tracks the scanned file list per type so `Number of files` carries rsync's `(reg: X, dir: Y, link: Z, special: W)` breakdown (directories come from the scanner's captured directory list for `-a`/`-t`/`-p`, or from a lightweight traversed-directory counter on a plain `-r` run so the `dir:` category is present there too), `Number of regular files transferred` excludes symlinks/specials and up-to-date files, `Total file size` includes symlink target lengths, and `Total transferred file size` counts only transferred files. **Protocol 2.28.0 extends `STATUS_STATS`** with receiver-observed `literal_bytes` and the four `created_*` counters: `Number of created files` now carries rsync's `(reg/dir/link/special)` breakdown (the receiver reports which destination entries it newly created, including implicitly-created parent directories below the transfer root) and `Literal data` is exact for a delta transfer (the receiver counts the literal fragments it stored, not the whole source size) — all differential-tested in the sequential and `--threads` paths against rsync 3.4.1 for fresh-create, update and delta shapes. **Remaining divergences:** rsync's per-type breakdown on `Number of deleted files` is not reproduced; and `Total bytes sent`/`received` are FastSync wire bytes framed differently from rsync's, so they are not numerically comparable |
|
||||||
| `-h`, `--human-readable` | Human-readable numbers | ✅ Parity | Formats transfer byte and rate counts using rsync's **decimal** (base-1000) units, matching rsync `-h` (e.g. `1.23M`), not binary units. **A lone `-h` with no transfer arguments prints help instead** (protocol 2.26.0), matching the rsync idiom; `-h` alongside a transfer remains human-readable |
|
| `-h`, `--human-readable` | Human-readable numbers | ✅ Parity | Formats transfer byte and rate counts using rsync's **decimal** (base-1000) units, matching rsync `-h` (e.g. `1.23M`), not binary units. **A lone `-h` with no transfer arguments prints help instead** (protocol 2.26.0), matching the rsync idiom; `-h` alongside a transfer remains human-readable |
|
||||||
| `-i`, `--itemize-changes` | Per-file change summary | ✅ Parity | Prints rsync-style `>f+++++++++` lines to stdout only for files actually sent (also under `-j`/`--threads`); unchanged files print nothing, matching single-`-i` behavior |
|
| `-i`, `--itemize-changes` | Per-file change summary | ✅ Parity | Prints rsync-style `>f+++++++++` lines to stdout only for files actually sent (also under `-j`/`--threads`); unchanged files print nothing, matching single-`-i` behavior |
|
||||||
| `--progress` | Show progress | ⚠️ Caveat | Protocol 2.25.0 prints rsync-style per-file progress blocks (percent, transferred/total bytes, rate, elapsed, `(xfr#N, to-chk=M/T)`) fed by the receiver's `STATUS_STATS`, in both the sequential and `--threads` send paths. FastSync also prints rsync's leading `./` transfer-root line and, when progress is requested (`--progress`/`-P`/`--info=progress`) and not `--quiet`, runs a **paths-only metadata pre-scan** (no file reads, no hashing) that supplies rsync's file-list total `T` for the `to-chk` denominator and the directory names; `--delete-during`/`--delete-delay` reuse their existing keep-set pre-scan instead of walking twice, and non-progress runs are untouched. Per-directory name lines are emitted (trailing `/`), and symlink (` -> target`) and special entries are named too, so a **fresh multi-directory tree's name set and `to-chk` denominator match rsync 3.4.1** (differential test, sequential and `--threads`) and a **single-file transfer's name lines and deterministic frames remain byte-identical** to rsync. **Remaining divergences:** rsync emits entries in sorted depth-first order while FastSync streams them in the scanner's readdir/BFS order, so the interleaving and the `to-chk` numerator differ (the denominator matches); the leading `./` root line is emitted unconditionally rather than keyed off rsync's root-attribute-change decision, and an ancestor directory line is emitted whenever a child transfers (rsync suppresses it when the directory itself is unchanged); on a re-run, entries without a quick-check (symlinks, empty directories) are still named where rsync stays silent; and the rate/ETA are wall-clock dependent |
|
| `--progress` | Show progress | ⚠️ Caveat | Protocol 2.25.0 prints rsync-style per-file progress blocks (percent, transferred/total bytes, rate, elapsed, `(xfr#N, to-chk=M/T)`) fed by the receiver's `STATUS_STATS`, in both the sequential and `--threads` send paths. FastSync also prints rsync's leading `./` transfer-root line and, when progress is requested (`--progress`/`-P`/`--info=progress`) and not `--quiet`, runs a **paths-only metadata pre-scan** (no file reads, no hashing) that supplies rsync's file-list total `T` for the `to-chk` denominator and the directory names; `--delete-during`/`--delete-delay` reuse their existing keep-set pre-scan instead of walking twice, and non-progress runs are untouched. Per-directory name lines are emitted (trailing `/`), and symlink (` -> target`) and special entries are named too, so a **fresh multi-directory tree's name set and `to-chk` denominator match rsync 3.4.1** (differential test, sequential and `--threads`) and a **single-file transfer's name lines and deterministic frames remain byte-identical** to rsync. **Order parity (parity-2.29):** the sequential scanner now emits entries in rsync's sorted depth-first flist order (non-directories ascending, then directories ascending), so the interleaving and the `to-chk` numerator match rsync for the default single-threaded transfer (differential `test_parity_order.py`; `--threads` has no rsync analogue and stays unordered). **Remaining divergences:** the leading `./` root line is emitted unconditionally rather than keyed off rsync's root-attribute-change decision, and an ancestor directory line is emitted whenever a child transfers (rsync suppresses it when the directory itself is unchanged); on a re-run, entries without a quick-check (symlinks, empty directories) are still named where rsync stays silent; and the rate/ETA are wall-clock dependent |
|
||||||
| `-P` | Same as --partial --progress | ✅ Parity | Parses to `--partial` + `--progress`. The independent `--partial` retention semantics are rsync parity: an interrupted write retains the already-written temp at the destination (best-effort) so a later `--append`/`--append-verify` can resume. Progress presentation is owned by the `--progress` row; there is no separate `-P` divergence |
|
| `-P` | Same as --partial --progress | ✅ Parity | Parses to `--partial` + `--progress`. The independent `--partial` retention semantics are rsync parity: an interrupted write retains the already-written temp at the destination (best-effort) so a later `--append`/`--append-verify` can resume. Progress presentation is owned by the `--progress` row; there is no separate `-P` divergence |
|
||||||
| `--out-format=FORMAT` | Custom output format | ❌ Divergent | Per-transfer template on stdout; tokens `%f` `%n` `%l` `%b` `%c` `%C` `%i` `%M` `%o` `%U` `%G` `%t` `%%`. `%C` now uses the negotiated transfer algorithm (`--checksum-choice`, default `xxh128`, seed 0) and renders every algorithm exactly like rsync — xxh128 high-then-low, xxh64/xxh3 big-endian, md5/md4/sha1 standard hex, `none` a blank 2-char column — differential-tested across all algorithms. `%f`/`%n`/`%l`/`%i`/`%M`/`%U`/`%G`/`%B` also match. **Reclassified because `%b`/`%c` are protocol-specific and cannot match:** a differential against rsync 3.4.1 shows whole-file `%c = 16` for both, but rsync whole-file `%b = filesize + 27 + transfer-digest-bytes` (39 for a 0-byte file; 43/35/47 for xxh128/xxh64/sha1 on a 12-byte file) while FastSync `%b` counts its own framing; in delta mode rsync `%c = 16 + 6·ceil(filesize/block_size)` (verified at block sizes 512/700/1024/2048) while FastSync counts its own signature handshake, and rsync `%b` is its token stream. FastSync's wire bytes are a different quantity, so exact `%b`/delta-`%c` equality is impossible |
|
| `--out-format=FORMAT` | Custom output format | ❌ Divergent | Per-transfer template on stdout; tokens `%f` `%n` `%l` `%b` `%c` `%C` `%i` `%M` `%o` `%U` `%G` `%t` `%%`. `%C` now uses the negotiated transfer algorithm (`--checksum-choice`, default `xxh128`, seed 0) and renders every algorithm exactly like rsync — xxh128 high-then-low, xxh64/xxh3 big-endian, md5/md4/sha1 standard hex, `none` a blank 2-char column — differential-tested across all algorithms. `%f`/`%n`/`%l`/`%i`/`%M`/`%U`/`%G`/`%B` also match. **Reclassified because `%b`/`%c` are protocol-specific and cannot match:** a differential against rsync 3.4.1 shows whole-file `%c = 16` for both, but rsync whole-file `%b = filesize + 27 + transfer-digest-bytes` (39 for a 0-byte file; 43/35/47 for xxh128/xxh64/sha1 on a 12-byte file) while FastSync `%b` counts its own framing; in delta mode rsync `%c = 16 + 6·ceil(filesize/block_size)` (verified at block sizes 512/700/1024/2048) while FastSync counts its own signature handshake, and rsync `%b` is its token stream. FastSync's wire bytes are a different quantity, so exact `%b`/delta-`%c` equality is impossible |
|
||||||
| `--log-file=FILE` | Log to file | ✅ Parity | `log_file` config field |
|
| `--log-file=FILE` | Log to file | ✅ Parity | `log_file` config field |
|
||||||
@@ -115,7 +136,7 @@ Every one of those has an entry below with its remaining caveats.
|
|||||||
|------|-------------------|-----------------|-------|
|
|------|-------------------|-----------------|-------|
|
||||||
| `--exclude-from=FILE` | Read exclude patterns from file | ✅ Parity | Reads patterns from file |
|
| `--exclude-from=FILE` | Read exclude patterns from file | ✅ Parity | Reads patterns from file |
|
||||||
| `--include-from=FILE` | Read include patterns from file | ✅ Parity | Reads patterns from file |
|
| `--include-from=FILE` | Read include patterns from file | ✅ Parity | Reads patterns from file |
|
||||||
| `--filter=RULE` | Add file-filtering rule | ✅ Parity | The short `-f` **is** bound to `--filter` (the old FastSync sendfile conflict is gone; sendfile is long-only `--sendfile`), and `-f RULE`, `-f=RULE`, `--filter=RULE` and the two-argument form all parse. Protocol 2.26.0 implements rsync's filter grammar: `+`/`-`, `include`/`exclude`, a leading `/` anchor (to the transfer root or a `.rsync-filter` file's directory), a trailing `/` dir-only rule, and the `merge`/`.`, `dir-merge`/`:`, `hide`/`H`, `show`/`S`, `protect`/`P`, `risk`/`R` and `clear`/`!` words, including the `:`/`.` modifiers. First match wins; the filter layer is independent of `--exclude`/`--include`. **Track 4a (protocol 2.28.0) adds the receiver filter engine:** the sender compiles its root-level rules exactly as the scanner does (`filter_base_build`) and streams them as one bounded, self-describing config-frame block; the receiver reconstructs them and re-applies first-match-wins to every extraneous destination path during deletion, so a `P *.log` rule protects a destination-only `extra.log` (differential `filter_protect`/`filter_protect_during`/`filter_protect_delay` vs rsync 3.4.1, plus the `-n` would-delete enumeration) — matching rsync's dual-sided engine for the command-line rule set. **Remaining residual:** per-directory merge (`:`/`.`, and therefore `-F`) is not yet re-derived on the receiver; a destination-only entry that matches ONLY a per-directory merge rule is still protected only through the sender-derived source-mirror prefixes, not by the received base rule list |
|
| `--filter=RULE` | Add file-filtering rule | ⚠️ Caveat | The short `-f` **is** bound to `--filter` (the old FastSync sendfile conflict is gone; sendfile is long-only `--sendfile`), and `-f RULE`, `-f=RULE`, `--filter=RULE` and the two-argument form all parse. Protocol 2.26.0 implements rsync's filter grammar: `+`/`-`, `include`/`exclude`, a leading `/` anchor (to the transfer root or a `.rsync-filter` file's directory), a trailing `/` dir-only rule, and the `merge`/`.`, `dir-merge`/`:`, `hide`/`H`, `show`/`S`, `protect`/`P`, `risk`/`R` and `clear`/`!` words, including the `:`/`.` modifiers. The xattr-name `x` modifier is **explicitly rejected everywhere with a clear error**. The merge-only `e`/`n`/`w` and `-` modifiers are **accepted and consumed on `merge`/`dir-merge` rules** (so they no longer leak into the merge filename) while still being **rejected on non-merge rules**, matching rsync; their semantics remain unimplemented, so they are accepted-but-ignored (the reason this row is a caveat rather than parity). A token made up solely of modifier characters that names an unsupported modifier is rejected on non-merge rules, while glued patterns (`-newfile`, `-e2e`) and mixed tokens (`H,!secret`) keep their historical parsing. First match wins; the filter layer is independent of `--exclude`/`--include`. **Track 4a (protocol 2.28.0) adds the receiver filter engine:** the sender compiles its root-level rules exactly as the scanner does (`filter_base_build`) and streams them as one bounded, self-describing config-frame block; the receiver reconstructs them and re-applies first-match-wins to every extraneous destination path during deletion, so a `P *.log` rule protects a destination-only `extra.log` (differential `filter_protect`/`filter_protect_during`/`filter_protect_delay` vs rsync 3.4.1, plus the `-n` would-delete enumeration) — matching rsync's dual-sided engine for the command-line rule set. **Remaining residual:** per-directory merge (`:`/`.`, and therefore `-F`) is not yet re-derived on the receiver; a destination-only entry that matches ONLY a per-directory merge rule is still protected only through the sender-derived source-mirror prefixes, not by the received base rule list |
|
||||||
| `--files-from=FILE` | Read source file list from file | ✅ Parity | Entries are paths relative to the source root (leading `./` stripped, `..`/absolute rejected at parse time, blank lines ignored; NUL-delimited with `-0`). A listed regular file is transferred; a listed directory transfers its whole subtree (FastSync recursion is always on). Non-listed paths are pruned by the scanner; the delete manifest is scoped to the listed directory subtrees. A listed entry that does not exist is a hard error unless `--ignore-missing-args`/`--delete-missing-args` is given. **An empty list is a zero-transfer success (exit 0), matching rsync 3.4.1** — the earlier claim that rsync reports "no source files specified" was wrong. Scalability note: `file_list_affects` is O(list size) per scanned entry, so a very large list against a huge tree is quadratic (the documented bound) |
|
| `--files-from=FILE` | Read source file list from file | ✅ Parity | Entries are paths relative to the source root (leading `./` stripped, `..`/absolute rejected at parse time, blank lines ignored; NUL-delimited with `-0`). A listed regular file is transferred; a listed directory transfers its whole subtree (FastSync recursion is always on). Non-listed paths are pruned by the scanner; the delete manifest is scoped to the listed directory subtrees. A listed entry that does not exist is a hard error unless `--ignore-missing-args`/`--delete-missing-args` is given. **An empty list is a zero-transfer success (exit 0), matching rsync 3.4.1** — the earlier claim that rsync reports "no source files specified" was wrong. Scalability note: `file_list_affects` is O(list size) per scanned entry, so a very large list against a huge tree is quadratic (the documented bound) |
|
||||||
| `-0`, `--from0` | Delimit *-from files with NULs | ✅ Parity | `--files-from` entries become NUL-delimited; the flag may appear before or after `--files-from` on the command line. NUL mode preserves entry bytes exactly (trailing CR/LF are part of the name; only newline mode trims them) |
|
| `-0`, `--from0` | Delimit *-from files with NULs | ✅ Parity | `--files-from` entries become NUL-delimited; the flag may appear before or after `--files-from` on the command line. NUL mode preserves entry bytes exactly (trailing CR/LF are part of the name; only newline mode trims them) |
|
||||||
| `--max-size=SIZE` | Skip files larger than SIZE | ✅ Parity | `max_size` in scanner |
|
| `--max-size=SIZE` | Skip files larger than SIZE | ✅ Parity | `max_size` in scanner |
|
||||||
@@ -156,23 +177,23 @@ Every one of those has an entry below with its remaining caveats.
|
|||||||
|
|
||||||
| Flag | Rsync Description | FastSync Status | Notes |
|
| Flag | Rsync Description | FastSync Status | Notes |
|
||||||
|------|-------------------|-----------------|-------|
|
|------|-------------------|-----------------|-------|
|
||||||
| `-n`, `--dry-run` | Trial run with no changes | ⚠️ Caveat | Server-contacting since protocol 2.21.0. The routing predicate `dry_run_targets_server()` selects the server-contacting path for any target a real run would reach over the wire (SSH, daemon `host::module`, explicit `--server-host`/`--server-port`, TLS, source-bind `--address`); the client handshakes with the receiver, which runs the normal read-only per-file check and answers `STATUS_DRY_RUN_TRANSFER`/`STATUS_OK` without mutating anything. Protocol 2.25.0 also reports would-delete lines: with `--delete` the receiver's `STATUS_STATS` carries the extras it would have removed and the client prints rsync-style `*deleting` lines (sequential and `--threads`; control bytes escaped). Dry-run never deletes. **Fixed (no-wire):** the dry-run keep-set manifest now carries the same filter-excluded and size-pruned protected prefixes and synchronized-directory scope a real run sends, and the would-be-transferred entries are in the keep-set, so `-n --delete` lists exactly rsync's extras for source-derived protections — the file merely being updated is kept and an excluded/pruned source entry is protected (`test_dry_run_delete_lines_match_rsync`, differential vs rsync 3.4.1). **Track 4a (protocol 2.28.0):** the received receiver-side `protect`/`risk` rules are also applied to the would-delete enumeration, so a destination-only entry matching a `P` rule is no longer reported (or removed in a real run) — matching rsync (`TestFilterProtect::test_protect_dest_only_dry_run_enumeration`). **Remaining caveat:** the would-delete line ordering follows the destination readdir order rather than rsync's reverse-sorted walk (the differential compares the sorted set) |
|
| `-n`, `--dry-run` | Trial run with no changes | ✅ Parity | Server-contacting since protocol 2.21.0. The routing predicate `dry_run_targets_server()` selects the server-contacting path for any target a real run would reach over the wire (SSH, daemon `host::module`, explicit `--server-host`/`--server-port`, TLS, source-bind `--address`); the client handshakes with the receiver, which runs the normal read-only per-file check and answers `STATUS_DRY_RUN_TRANSFER`/`STATUS_OK` without mutating anything. Protocol 2.25.0 also reports would-delete lines: with `--delete` the receiver's `STATUS_STATS` carries the extras it would have removed and the client prints rsync-style `*deleting` lines (sequential and `--threads`; control bytes escaped). Dry-run never deletes. **Fixed (no-wire):** the dry-run keep-set manifest now carries the same filter-excluded and size-pruned protected prefixes and synchronized-directory scope a real run sends, and the would-be-transferred entries are in the keep-set, so `-n --delete` lists exactly rsync's extras for source-derived protections — the file merely being updated is kept and an excluded/pruned source entry is protected (`test_dry_run_delete_lines_match_rsync`, differential vs rsync 3.4.1). **Track 4a (protocol 2.28.0):** the received receiver-side `protect`/`risk` rules are also applied to the would-delete enumeration, so a destination-only entry matching a `P` rule is no longer reported (or removed in a real run) — matching rsync (`TestFilterProtect::test_protect_dest_only_dry_run_enumeration`). **Order parity (parity-2.29):** the sequential scanner now walks the tree in rsync's sorted depth-first flist order (non-directories ascending, then directories ascending) and the delete walkers sort each directory the same way, so the would-delete lines are emitted in rsync's exact reverse-sorted order — differential `test_parity_order.py::test_dry_run_delete_order_matches_rsync` compares the ordered sequence, not a sorted set |
|
||||||
| `-b`, `--backup` | Make backups of overwritten files | ✅ Parity | Backup before overwrite |
|
| `-b`, `--backup` | Make backups of overwritten files | ✅ Parity | Backup before overwrite |
|
||||||
| `--backup-dir=DIR` | Backup directory hierarchy | ✅ Parity | `backup_dir` config field |
|
| `--backup-dir=DIR` | Backup directory hierarchy | ✅ Parity | `backup_dir` config field |
|
||||||
| `--suffix=SUFFIX` | Backup suffix (default ~) | ✅ Parity | `suffix` config field |
|
| `--suffix=SUFFIX` | Backup suffix (default ~) | ✅ Parity | `suffix` config field |
|
||||||
| `--delay-updates` | Put updated files in place at end | ❌ Divergent | Successfully received files are staged under a private 0700 `.fastsync-stage` dir inside the receive root and atomically renamed into their final destinations only after the whole transfer (manifest/delete handling included) succeeds, just before the success/outcome frame is sent. The delete walker deliberately skips the staging dir at the receive root, so `--delete` removes genuine extras but never the staged files (deletion runs before publication; rsync's delete-after ordering is not implemented). `--existing`/`--ignore-existing`/`--update` decide against the final destination path at stage time; `--backup` moves the old file aside at publication, and **`--force` is honored at publication** (protocol 2.23.0): a staged regular file or symlink may replace a destination directory that blocks it. Incompatible with `--inplace` and with `--backup-dir=.fastsync-stage` (the internal staging name is reserved; both are rejected). The staging dir name is fixed, so two simultaneous delayed transfers to the same destination root are serialized with an exclusive advisory lock held for the whole transfer: the second session fails cleanly instead of corrupting the first. Aborting or failing before publication installs nothing and removes the staging tree; a crash between stage and publish leaves staged leftovers that the next delayed run wipes at start (process death releases the lock). A stage→publish failure aborts the transfer (best-effort cleanup of the not-yet-published staged files; already-published files are not rolled back). **Reclassified Divergent (differential evidence):** the staging name is fixed and a delayed run wipes a pre-existing destination tree of that name at start even without `--delete`, whereas rsync uses its own internal temp name and leaves a genuine destination entry named `.fastsync-stage` untouched (`test_delay_updates_staging_name_collision_residual`); deletion also runs before publication while rsync's `--delay-updates` implies `--delete-after`. Works in single-threaded and `-j`/`--threads` modes |
|
| `--delay-updates` | Put updated files in place at end | ❌ Divergent | Successfully received files are staged under a private 0700 `.fastsync-stage` dir inside the receive root and atomically renamed into their final destinations only after the whole transfer (manifest/delete handling included) succeeds, just before the success/outcome frame is sent. The delete walker deliberately skips the staging dir at the receive root, so `--delete` removes genuine extras but never the staged files (deletion runs before publication; rsync's delete-after ordering is not implemented). `--existing`/`--ignore-existing`/`--update` decide against the final destination path at stage time; `--backup` moves the old file aside at publication, and **`--force` is honored at publication** (protocol 2.23.0): a staged regular file or symlink may replace a destination directory that blocks it. Incompatible with `--inplace` and with `--backup-dir=.fastsync-stage` (the internal staging name is reserved; both are rejected). The staging dir name is fixed, so two simultaneous delayed transfers to the same destination root are serialized with an exclusive advisory lock held for the whole transfer: the second session fails cleanly instead of corrupting the first. Aborting or failing before publication installs nothing and removes the staging tree; a crash between stage and publish leaves staged leftovers that the next delayed run wipes at start (process death releases the lock). A stage→publish failure aborts the transfer (best-effort cleanup of the not-yet-published staged files; already-published files are not rolled back). **Reclassified Divergent (differential evidence):** the staging name is fixed and a delayed run wipes a pre-existing destination tree of that name at start even without `--delete`, whereas rsync uses its own internal temp name and leaves a genuine destination entry named `.fastsync-stage` untouched (`test_delay_updates_staging_name_collision_residual`); deletion also runs before publication while rsync's `--delay-updates` implies `--delete-after`. Works in single-threaded and `-j`/`--threads` modes |
|
||||||
| `-T`, `--temp-dir=DIR` | Create temporary files in DIR | ❌ Divergent | `--temp-dir` with the rsync short `-T` (the timeout alias moved to long-only `--timeout`). A **relative** dir matches rsync exactly: it is resolved below the receive/destination root and must already exist (differentially verified: `rsync -a --temp-dir=scratch src/ dst/` and FastSync produce identical trees and an empty scratch dir). **Reclassified as a deliberate divergence because an absolute `--temp-dir` is rejected by the receiver** — it is resolved verbatim by rsync standalone (which will use `/tmp` or any other absolute directory, including one outside the destination), but FastSync's security-reviewed receiver confines the scratch dir to the authorized receive root and rejects any absolute path or one containing `..`. A differential test confirms rsync exits 0 using an absolute scratch dir while FastSync refuses before writing anything into it (the scratch dir stays empty). Its daemon mode also confines relative to the module, but standalone rsync's absolute-temp-dir behavior is not reproduced because it would let a client place receiver scratch files outside the sandbox. Temp copies use a unique name in the scratch dir and are atomically renamed into place; **on `EXDEV` (scratch dir and destination on different filesystems, reachable via a confined relative symlink) the receiver falls back to a non-atomic copy instead of aborting**, matching rsync. `--inplace` and `--partial-dir` writes bypass the scratch dir |
|
| `-T`, `--temp-dir=DIR` | Create temporary files in DIR | ❌ Divergent | `--temp-dir` with the rsync short `-T` (the timeout alias moved to long-only `--timeout`). A **relative** dir matches rsync exactly: it is resolved below the receive/destination root and must already exist (differentially verified: `rsync -a --temp-dir=scratch src/ dst/` and FastSync produce identical trees and an empty scratch dir). **Reclassified as a deliberate divergence because an absolute `--temp-dir` is rejected by the receiver** — it is resolved verbatim by rsync standalone (which will use `/tmp` or any other absolute directory, including one outside the destination), but FastSync's security-reviewed receiver confines the scratch dir to the authorized receive root and rejects any absolute path or one containing `..`. **Audit-cycle hardening:** the opened dir is additionally judged by the real path of its fd (`/proc/self/fd`), so a client-planted symlink under the receive root cannot redirect receiver scratch files outside the authorized root (an escaping target is refused with `EACCES`), while an in-root symlink to another filesystem — the `EXDEV` fallback case — still works. A differential test confirms rsync exits 0 using an absolute scratch dir while FastSync refuses before writing anything into it (the scratch dir stays empty). Its daemon mode also confines relative to the module, but standalone rsync's absolute-temp-dir behavior is not reproduced because it would let a client place receiver scratch files outside the sandbox. Temp copies use a unique name in the scratch dir and are atomically renamed into place; **on `EXDEV` (scratch dir and destination on different filesystems, reachable via a confined relative symlink) the receiver falls back to a non-atomic copy instead of aborting**, matching rsync. `--inplace` and `--partial-dir` writes bypass the scratch dir |
|
||||||
| `--partial` | Keep partially transferred files | ✅ Parity | On a failed/interrupted write the already-written temp file is retained at the destination path (best-effort rename instead of unlink) so a later `--append`/`--append-verify` run can resume it. Retention never runs when no data was actually written or under `--ignore-existing`/`--existing` (the destination is not ours to overwrite), and it only ever renames the already-written temp. A failed rename falls back to the normal unlink |
|
| `--partial` | Keep partially transferred files | ✅ Parity | On a failed/interrupted write the already-written temp file is retained at the destination path (best-effort rename instead of unlink) so a later `--append`/`--append-verify` run can resume it. Retention never runs when no data was actually written or under `--ignore-existing`/`--existing` (the destination is not ours to overwrite), and it only ever renames the already-written temp. A failed rename falls back to the normal unlink |
|
||||||
| `--partial-dir=DIR` | Keep partial files in DIR | ✅ Parity | With `--partial`, the working file is written under the confined partial directory (a relative dir below the receive root) and atomically renamed into place once complete, so an interrupted transfer leaves a resumable copy there and completed transfers do not linger under it. `--inplace` bypasses the partial dir (rsync parity). Requires `--partial` |
|
| `--partial-dir=DIR` | Keep partial files in DIR | ✅ Parity | The working file is written under the confined partial directory (a relative dir below the receive root) and atomically renamed into place once complete, so an interrupted transfer leaves a resumable copy there and completed transfers do not linger under it. `--inplace` bypasses the partial dir (rsync parity), and combining `--inplace` with `--partial-dir` is now **rejected up front** with rsync's message (`--inplace cannot be used with --partial-dir`) instead of silently ignoring the partial dir. **Implies `--partial`** (audit-cycle fix, matching rsync 3.4.1, which sets `keep_partial` after option parsing): `--partial-dir=DIR` alone retains an interrupted transfer's partial, and the implication wins over an explicit `--no-partial` regardless of order |
|
||||||
|
|
||||||
## 7. Deletion
|
## 7. Deletion
|
||||||
|
|
||||||
| Flag | Rsync Description | FastSync Status | Notes |
|
| Flag | Rsync Description | FastSync Status | Notes |
|
||||||
|------|-------------------|-----------------|-------|
|
|------|-------------------|-----------------|-------|
|
||||||
| `--delete` | Delete extraneous files from dest | ⚠️ Caveat | `use_delete` config field. Deletion is always derived from the keep-set the sender actually transmitted (the per-directory `STATUS_DELETE_PLAN` set by default, or the whole-tree manifest for the late timings — never from unchecked input), runs through the symlink-safe walker bounded by `MAX_SERVER_DELETE_COUNT`, and skips the `.fastsync-stage` staging dir under `--delay-updates`. **Lockstep track 6 (protocol 2.28.0): plain `--delete` with no explicit timing flag now defaults to `--delete-during`**, exactly like rsync's `--del` (the client normalizes it to the existing `delete_during` wire bool; no new wire field). This frees destination space progressively during the transfer and avoids the whole-old+new-tree peak that could `ENOSPC` a tight destination. The old late whole-tree commit is opt-in via `--delete-after` or the FastSync-only long spelling `--delete-commit`. **Caveat (shared with `--delete-during`):** the exact mid-transfer abort boundary can differ from rsync's generator (rsync removes all extras ahead of its throttled sender; FastSync removes only the directories it has reached), `-d/--dirs` falls back to the end-of-transfer commit, and the surviving-set ordering under a partial `--max-delete` can differ — on a completed run the trees agree. By default the destination mirror of a path the source scan pruned (filter/exclude/size rules) is **protected** from deletion — matching rsync, which does not delete excluded files under `--delete`; `--delete-excluded` opts back into deleting them (see below). Deletion is scoped to the **synchronized directories** sent on the wire (protocol 2.23.0), so a `--files-from` subset no longer deletes untransmitted paths outside the listed directory subtrees. The walk is bounded: a client `--max-delete=NUM` (or the 100000-entry server bound) makes it **partial** — entries up to the bound are removed, the rest are skipped, and the client exits **25** (`RERR_PARTIAL`), matching rsync, rather than failing the transfer. Extraneous destination symlinks are unlinked by name (never followed); a directory still holding a kept/protected entry is left behind rather than failing |
|
| `--delete` | Delete extraneous files from dest | ✅ Parity | `use_delete` config field. Deletion is always derived from the keep-set the sender actually transmitted (the per-directory `STATUS_DELETE_PLAN` set by default, or the whole-tree manifest for the late timings — never from unchecked input), runs through the symlink-safe walker bounded by `MAX_SERVER_DELETE_COUNT`, and skips the `.fastsync-stage` staging dir under `--delay-updates`. **Lockstep track 6 (protocol 2.28.0): plain `--delete` with no explicit timing flag now defaults to `--delete-during`**, exactly like rsync's `--del` (the client normalizes it to the existing `delete_during` wire bool; no new wire field). This frees destination space progressively during the transfer and avoids the whole-old+new-tree peak that could `ENOSPC` a tight destination. The old late whole-tree commit is opt-in via `--delete-after` or the FastSync-only long spelling `--delete-commit`. **Abort/ordering parity (parity-2.29):** the complete per-directory plan set is transmitted before the first data frame, so a mid-transfer abort has already applied every planned removal exactly like rsync's generator (which runs ahead of its throttled sender); `-d/--dirs` uses the same per-directory plans (the generator records only the directories whose direct children it enumerated, so an untraversed subdirectory's mirror is shielded); and the sorted depth-first traversal makes the removal order — and therefore the survivor set under a partial `--max-delete` — match rsync exactly (`test_delete_boundary_parity.py`, `test_parity_order.py`). By default the destination mirror of a path the source scan pruned (filter/exclude/size rules) is **protected** from deletion — matching rsync, which does not delete excluded files under `--delete`; `--delete-excluded` opts back into deleting them (see below). Deletion is scoped to the **synchronized directories** sent on the wire (protocol 2.23.0), so a `--files-from` subset no longer deletes untransmitted paths outside the listed directory subtrees. The walk is bounded: a client `--max-delete=NUM` (or the 100000-entry server bound) makes it **partial** — entries up to the bound are removed, the rest are skipped, and the client exits **25** (`RERR_PARTIAL`), matching rsync, rather than failing the transfer. Extraneous destination symlinks are unlinked by name (never followed); a directory still holding a kept/protected entry is left behind rather than failing |
|
||||||
| `--delete-before` | Delete before transfer | ⚠️ Caveat | Implies `--delete`. The sender runs a full source pre-scan (paths only) and transmits the keep-set manifest BEFORE any file data; the receiver validates it, removes every destination entry not listed (bounded walk, staging-dir skip, protected prefixes honored), then acks `STATUS_OK`. The sender only starts streaming after the deletion committed, or aborts if the receiver reported a deletion error. By definition the deletions already happened when a later transfer phase fails — rsync's delete-before is destructive the same way; a subsequent failure does not restore the removed files. **Phase-0 divergence (sharpened):** rsync builds the full file list first, so a source file created after that scan is NOT transferred and its destination extra is deleted; FastSync's single-threaded data pass re-scans the source, so the late file IS transferred (a safe superset), while FastSync `--threads` pipelines the scan and matches rsync |
|
| `--delete-before` | Delete before transfer | ⚠️ Caveat | Implies `--delete`. The sender runs a full source pre-scan (paths only) and transmits the keep-set manifest BEFORE any file data; the receiver validates it, removes every destination entry not listed (bounded walk, staging-dir skip, protected prefixes honored), then acks `STATUS_OK`. The sender only starts streaming after the deletion committed, or aborts if the receiver reported a deletion error. By definition the deletions already happened when a later transfer phase fails — rsync's delete-before is destructive the same way; a subsequent failure does not restore the removed files. **Phase-0 divergence (sharpened):** rsync builds the full file list first, so a source file created after that scan is NOT transferred and its destination extra is deleted; FastSync's single-threaded data pass re-scans the source, so the late file IS transferred (a safe superset), while FastSync `--threads` pipelines the scan and matches rsync |
|
||||||
| `--del`, `--delete-during` | Delete during transfer | ⚠️ Caveat | Both spellings accepted; imply `--delete`, and since lockstep track 6 this is also the default timing of a plain `--delete`. **Protocol 2.24.0 implements per-directory delete plans:** as the sender reaches each source directory it streams a `STATUS_DELETE_PLAN` for that directory and the receiver removes that directory's extras (verified with a byte-slicing proxy). The one-shot per-run config block (protected prefixes, size-pruned mirrors, `--delete-missing-args` exact paths) rides a dedicated config-only carrier frame with an `apply=false` flag, so it reaches the receiver even when the scope allows no directory plan at all (a `--files-from` list of bare files synchronizes no directory). **Phase-0 divergence (sharpened):** on a mid-transfer abort rsync's generator (which runs ahead of its throttled sender) has already removed ALL the extras it planned, whereas FastSync has removed only the directories it actually reached; on a completed run both agree. Also `-d`/`--dirs` (no descent) falls back to the end-of-transfer whole-tree commit, and the surviving-set ordering under a partial `--max-delete` can differ. `-R` plans are scoped to the transferred prefix subtree |
|
| `--del`, `--delete-during` | Delete during transfer | ✅ Parity | Both spellings accepted; imply `--delete`, and since lockstep track 6 this is also the default timing of a plain `--delete`. **Protocol 2.24.0 implements per-directory delete plans:** as the sender reaches each source directory it streams a `STATUS_DELETE_PLAN` for that directory and the receiver removes that directory's extras (verified with a byte-slicing proxy). The one-shot per-run config block (protected prefixes, size-pruned mirrors, `--delete-missing-args` exact paths) rides a dedicated config-only carrier frame with an `apply=false` flag, so it reaches the receiver even when the scope allows no directory plan at all (a `--files-from` list of bare files synchronizes no directory). **Abort/ordering parity (parity-2.29):** the complete plan set is transmitted before the first data frame, so on a mid-transfer abort every planned extra has already been removed exactly like rsync's generator (which runs ahead of its throttled sender); `-d/--dirs` no longer falls back to the end-of-transfer commit but records only the directories whose direct children it enumerated; and the sorted depth-first traversal makes the removal order — and the partial-`--max-delete` survivor set — identical to rsync (`test_delete_boundary_parity.py`, `test_parity_order.py`). `-R` plans are scoped to the transferred prefix subtree |
|
||||||
| `--delete-delay` | Find deletions during, delete after | ⚠️ Caveat | Implies `--delete`. **Protocol 2.24.0 implements rsync's delete-delay timing:** the sender records each directory's delete plan while scanning and the receiver commits those removals only after the whole transfer succeeds (per plan), so an extra created in the destination after its directory's plan survives while `--delete-after` re-scans and removes it, and a failed transfer removes nothing. The **reported** deleted count advances only on an actual removal. **Fixed (no-wire):** the `--max-delete` budget is now charged on ACTUAL removals (an unlink/rmdir that succeeded), not at plan/snapshot time, and a queued directory is re-scanned at commit and removed recursively (content created after the plan included), matching rsync: a snapshotted entry that fails or is skipped consumes no budget, so a later extra rsync would delete is still deleted. The deferred snapshot list keeps an independent hard cap (`DELETE_PLAN_SERVER_LIMIT`) so it cannot grow without bound now that the budget is no longer charged while scanning. A `--max-delete=2` partial delete reports exactly 2 and exits 25 in both tools, and the refilled-directory differential (late content removed, directory removed, budget shared) now matches rsync 3.4.1 on both sides (`test_delete_delay_budget_parity.py`, `test_delete_timing_parity.py`). Unit tests cover recursive removal, actual-removal charging, and the bounded deferred list. **Caveat:** the exact ordering of which extras are removed first under a partial `--max-delete` can still differ from rsync's generator (the survivor set is compared by count, not identity) |
|
| `--delete-delay` | Find deletions during, delete after | ✅ Parity | Implies `--delete`. **Protocol 2.24.0 implements rsync's delete-delay timing:** the sender records each directory's delete plan while scanning and the receiver commits those removals only after the whole transfer succeeds (per plan), so an extra created in the destination after its directory's plan survives while `--delete-after` re-scans and removes it, and a failed transfer removes nothing. The **reported** deleted count advances only on an actual removal. **Fixed (no-wire):** the `--max-delete` budget is now charged on ACTUAL removals (an unlink/rmdir that succeeded), not at plan/snapshot time, and a queued directory is re-scanned at commit and removed recursively (content created after the plan included), matching rsync: a snapshotted entry that fails or is skipped consumes no budget, so a later extra rsync would delete is still deleted. The deferred snapshot list keeps an independent hard cap (`DELETE_PLAN_SERVER_LIMIT`) so it cannot grow without bound now that the budget is no longer charged while scanning. A `--max-delete=2` partial delete reports exactly 2 and exits 25 in both tools, and the refilled-directory differential (late content removed, directory removed, budget shared) now matches rsync 3.4.1 on both sides (`test_delete_delay_budget_parity.py`, `test_delete_timing_parity.py`). Unit tests cover recursive removal, actual-removal charging, and the bounded deferred list. **Ordering parity (parity-2.29):** the sorted depth-first traversal plus the up-front plan set make the order in which extras are removed — and therefore the survivor set under a partial `--max-delete` — match rsync exactly (differential `test_parity_order.py::test_delete_delay_deletion_order_matches_rsync` and `::test_partial_max_delete_survivor_order_matches_rsync`) |
|
||||||
| `--delete-after` | Delete after transfer | ✅ Parity | Implies `--delete`. Selects the late whole-tree commit: the keep-set manifest closes the data stream and the receiver commits the bounded deletion only after the terminal `STATUS_FINISHED` proves the whole transfer (every data frame received and stored) succeeded. A failed or aborted transfer removes nothing. Since lockstep track 6 a plain `--delete` defaults to delete-during (rsync's `--del`); `--delete-after` — or the FastSync-only `--delete-commit` spelling, which selects the identical timing — is the explicit way to keep the old commit-style behavior |
|
| `--delete-after` | Delete after transfer | ✅ Parity | Implies `--delete`. Selects the late whole-tree commit: the keep-set manifest closes the data stream and the receiver commits the bounded deletion only after the terminal `STATUS_FINISHED` proves the whole transfer (every data frame received and stored) succeeded. A failed or aborted transfer removes nothing. Since lockstep track 6 a plain `--delete` defaults to delete-during (rsync's `--del`); `--delete-after` — or the FastSync-only `--delete-commit` spelling, which selects the identical timing — is the explicit way to keep the old commit-style behavior |
|
||||||
| `--delete-excluded` | Also delete excluded files | ✅ Parity | `delete_excluded` config field. Under `--delete` FastSync protects (rsync's default) the destination mirror of paths the sender's source scan pruned by the user-selection rules — the `--filter`/`-F`/`-C` layer and the legacy `--exclude`/`--include` layer. The sender transmits those concrete pruned paths as **protected prefixes** in the delete-manifest frame (see the Phase-3 notes below); the walker never descends into or removes them. `--delete-excluded` opts back in: the sender sends an empty protected list, so the excluded destination mirrors become ordinary extras and are removed. **`--max-size`/`--min-size` pruned mirrors are a separate, always-on protection** (protocol 2.23.0, rsync parity): size-pruned source mirrors survive `--delete` even with `--delete-excluded`. Track 4a (protocol 2.28.0) additionally re-applies the received `protect`/`risk` rules on the receiver, so a destination-only entry matching an exclude rule is protected (or left at risk) exactly like rsync; the remaining sender-derived `--delete-excluded` behavior (an unqualified rule becomes sender-only, so its source mirror and matching destination-only extras are deleted) is unchanged |
|
| `--delete-excluded` | Also delete excluded files | ✅ Parity | `delete_excluded` config field. Under `--delete` FastSync protects (rsync's default) the destination mirror of paths the sender's source scan pruned by the user-selection rules — the `--filter`/`-F`/`-C` layer and the legacy `--exclude`/`--include` layer. The sender transmits those concrete pruned paths as **protected prefixes** in the delete-manifest frame (see the Phase-3 notes below); the walker never descends into or removes them. `--delete-excluded` opts back in: the sender sends an empty protected list, so the excluded destination mirrors become ordinary extras and are removed. **`--max-size`/`--min-size` pruned mirrors are a separate, always-on protection** (protocol 2.23.0, rsync parity): size-pruned source mirrors survive `--delete` even with `--delete-excluded`. Track 4a (protocol 2.28.0) additionally re-applies the received `protect`/`risk` rules on the receiver, so a destination-only entry matching an exclude rule is protected (or left at risk) exactly like rsync; the remaining sender-derived `--delete-excluded` behavior (an unqualified rule becomes sender-only, so its source mirror and matching destination-only extras are deleted) is unchanged |
|
||||||
| `--max-delete=NUM` | Max files to delete | ✅ Parity | `max_delete` config field (default -1 = no client limit; 0 = delete nothing). **Protocol 2.23.0 matches rsync's partial semantics:** the receiver deletes up to NUM entries (regular files, symlinks and empty directories; each directory removal counts as one) and then **stops deleting, skips the rest, and reports the run as partial**. The client prints a "deletions stopped due to `--max-delete` limit" message and exits **25** (rsync's `RERR_PARTIAL`), not a hard failure — the transfer itself succeeded. NUM only applies together with `--delete` (it is inert otherwise, matching rsync). A client NUM below the server hard bound `MAX_SERVER_DELETE_COUNT` (100000) replaces it; a NUM above it never raises that cap. Deleting an entire destination with no limit is still bounded by the server's 100000-entry ceiling. `--delete-missing-args` exact-path deletions and the ordinary extras walk draw from the same budget, matching rsync |
|
| `--max-delete=NUM` | Max files to delete | ✅ Parity | `max_delete` config field (default -1 = no client limit; 0 = delete nothing). **Protocol 2.23.0 matches rsync's partial semantics:** the receiver deletes up to NUM entries (regular files, symlinks and empty directories; each directory removal counts as one) and then **stops deleting, skips the rest, and reports the run as partial**. The client prints a "deletions stopped due to `--max-delete` limit" message and exits **25** (rsync's `RERR_PARTIAL`), not a hard failure — the transfer itself succeeded. NUM only applies together with `--delete` (it is inert otherwise, matching rsync). A client NUM below the server hard bound `MAX_SERVER_DELETE_COUNT` (100000) replaces it; a NUM above it never raises that cap. Deleting an entire destination with no limit is still bounded by the server's 100000-entry ceiling. `--delete-missing-args` exact-path deletions and the ordinary extras walk draw from the same budget, matching rsync |
|
||||||
@@ -309,12 +330,12 @@ why plain `--append` works on the normal atomic path, not only with `--inplace`.
|
|||||||
| Flag | Rsync Description | FastSync Status | Notes |
|
| Flag | Rsync Description | FastSync Status | Notes |
|
||||||
|------|-------------------|-----------------|-------|
|
|------|-------------------|-----------------|-------|
|
||||||
| `--preserve` | (FastSync alias, not an rsync flag) | ✅ Parity | **FastSync-only alias** for `-p` + `-t` (mode + mtime), long-form only. It is not rsync's `--preserve` (rsync has no such option); the short `-M` that used to spell it is now rsync's `--remote-option`. The wire metadata also carries uid/gid for `-o`/`-g`/`-a`, and ownership is applied via `-o`/`-g`, `-a`, or an explicit identity flag (`--numeric-ids`/`--usermap`/`--groupmap`/`--chown`/`--copy-as`) |
|
| `--preserve` | (FastSync alias, not an rsync flag) | ✅ Parity | **FastSync-only alias** for `-p` + `-t` (mode + mtime), long-form only. It is not rsync's `--preserve` (rsync has no such option); the short `-M` that used to spell it is now rsync's `--remote-option`. The wire metadata also carries uid/gid for `-o`/`-g`/`-a`, and ownership is applied via `-o`/`-g`, `-a`, or an explicit identity flag (`--numeric-ids`/`--usermap`/`--groupmap`/`--chown`/`--copy-as`) |
|
||||||
| `-p`, `--perms` | Preserve permissions | ✅ Parity | Real per-attribute flag (protocol 2.22.0): `preserve_perms` applies the source mode independently of times/owner/group. **Strict rsync parity (protocol 2.23.0): the source mode is copied exactly, including setuid/setgid/sticky and group/other-write bits — there is no masking.** Without `-p`, a new file gets `source_mode & ~umask` when metadata is present (else the historical fixed `0644`); new directories without `-p` still use FastSync's `0755` creation default, because directory metadata is only applied when a directory attribute is requested. `-A/--acls` implies `-p`; `--chmod` does **not** imply `-p` (rsync parity) and applies its own unsanitized changes to the new mode. `-X/--xattrs` does not imply `-p`. The SSH port moved to `--ssh-port`. rsync-parity short form |
|
| `-p`, `--perms` | Preserve permissions | ✅ Parity | Real per-attribute flag (protocol 2.22.0): `preserve_perms` applies the source mode independently of times/owner/group. **Strict rsync parity when super-user activities are permitted (protocol 2.23.0): the source mode is copied exactly, including setuid/setgid/sticky and group/other-write bits.** **Audit-cycle fix:** when the connection forbids super-user activities (`--no-super`, a non-opted daemon module, or a privileged standalone listener without `--allow-super`), the setuid/setgid/sticky bits are masked from the applied mode (the other bits are unaffected); exact rsync semantics are preserved wherever super activities are permitted. Without `-p`, a new file gets `source_mode & ~umask` when metadata is present (else the historical fixed `0644`); new directories without `-p` still use FastSync's `0755` creation default, because directory metadata is only applied when a directory attribute is requested. **Audit-cycle fix:** the daemon no longer forces `umask(0)` (which made implied parent directories world-writable `0777`); it uses the conventional `022`, and `-p`/`-a` still restore the exact source mode via `fchmod`. `-A/--acls` implies `-p`; `--chmod` does **not** imply `-p` (rsync parity) and applies its own unsanitized changes to the new mode. `-X/--xattrs` does not imply `-p`. The SSH port moved to `--ssh-port`. rsync-parity short form |
|
||||||
| `-o`, `--owner` | Preserve owner | ✅ Parity | Real per-attribute flag (`preserve_owner`): preserve the source uid, resolved on the receiver by name against its own user database with a raw-numeric fallback (only numeric ids cross the wire). `--usermap`/`--chown=USER` imply it. Application follows the `--super`/`--no-super` policy; a non-opted daemon module applies no ownership (see the Daemon Mode notes) |
|
| `-o`, `--owner` | Preserve owner | ✅ Parity | Real per-attribute flag (`preserve_owner`): preserve the source uid, resolved on the receiver by name against its own user database with a raw-numeric fallback (only numeric ids cross the wire). `--usermap`/`--chown=USER` imply it. Application follows the `--super`/`--no-super` policy; a non-opted daemon module applies no ownership (see the Daemon Mode notes) |
|
||||||
| `-g`, `--group` | Preserve group | ✅ Parity | Real per-attribute flag (`preserve_group`): preserve the source gid, resolved by name on the receiver with a raw-numeric fallback. `--groupmap`/`--chown=:GROUP` imply it. Same privilege/super-policy gating as `-o` |
|
| `-g`, `--group` | Preserve group | ✅ Parity | Real per-attribute flag (`preserve_group`): preserve the source gid, resolved by name on the receiver with a raw-numeric fallback. `--groupmap`/`--chown=:GROUP` imply it. Same privilege/super-policy gating as `-o` |
|
||||||
| `-t`, `--times` | Preserve modification times | ✅ Parity | Real per-attribute flag (`preserve_times`): apply the source mtime independently of the other attributes. `-O/--omit-dir-times` suppresses directories only and `-J/--omit-link-times` suppresses symlinks only; `-U`/`-N` do not imply it. `--preserve`/`-a` imply it, and `--incremental`/`--delta` auto-enable it unless `--no-times`/`--no-preserve` |
|
| `-t`, `--times` | Preserve modification times | ✅ Parity | Real per-attribute flag (`preserve_times`): apply the source mtime independently of the other attributes. `-O/--omit-dir-times` suppresses directories only and `-J/--omit-link-times` suppresses symlinks only; `-U`/`-N` do not imply it. `--preserve`/`-a` imply it, and `--incremental`/`--delta` auto-enable it unless `--no-times`/`--no-preserve` |
|
||||||
| `-E`, `--executability` | Preserve executability | ✅ Parity | Preserves executable permission bits (implies metadata preservation) |
|
| `-E`, `--executability` | Preserve executability | ✅ Parity | Preserves executable permission bits (implies metadata preservation) |
|
||||||
| `--chmod=CHMOD` | Affect file permissions | ✅ Parity | Faithful port of rsync 3.4.1's `parse_chmod`/`tweak_mode`: numeric octal and symbolic `ugo`/`rwx` changes, `D`/`F` directory/file selectors, `X` (execute only on directories or already-executable files), `s`/`t` setuid/setgid/sticky, and append semantics — repeated clauses and repeated `--chmod` options accumulate in order (joined with commas). The changes are applied to the new mode **without sanitization** (matching rsync) and `--chmod` does **not** imply `-p` (rsync parity). Applied to files and directories on the receiver |
|
| `--chmod=CHMOD` | Affect file permissions | ✅ Parity | Faithful port of rsync 3.4.1's `parse_chmod`/`tweak_mode`: numeric octal and symbolic `ugo`/`rwx` changes, `D`/`F` directory/file selectors, `X` (execute only on directories or already-executable files), `s`/`t` setuid/setgid/sticky, and append semantics — repeated clauses and repeated `--chmod` options accumulate in order (joined with commas). The changes are applied to the new mode **without sanitization** (matching rsync), except that setuid/setgid/sticky are masked when the connection forbids super-user activities (audit-cycle fix, see `-p`), and `--chmod` does **not** imply `-p` (rsync parity). Applied to files and directories on the receiver |
|
||||||
| `-A`, `--acls` | Preserve ACLs | ✅ Parity | Implemented on Linux via the POSIX-ACL xattr representation: the sender captures the `system.posix_acl_access` / `system.posix_acl_default` xattrs and the receiver re-applies them fd-relative. A differential test with `setfacl` confirms the complete access and default ACL sets (including `mask`) are identical to rsync's on a directory. libacl is not required; a `fsetxattr` an unprivileged receiver may not perform is logged and skipped, never fatal. Only the `system.posix_acl_*` namespaces plus `user.*` are ever applied; privileged namespaces are never applied. Implies metadata transmission |
|
| `-A`, `--acls` | Preserve ACLs | ✅ Parity | Implemented on Linux via the POSIX-ACL xattr representation: the sender captures the `system.posix_acl_access` / `system.posix_acl_default` xattrs and the receiver re-applies them fd-relative. A differential test with `setfacl` confirms the complete access and default ACL sets (including `mask`) are identical to rsync's on a directory. libacl is not required; a `fsetxattr` an unprivileged receiver may not perform is logged and skipped, never fatal. Only the `system.posix_acl_*` namespaces plus `user.*` are ever applied; privileged namespaces are never applied. Implies metadata transmission |
|
||||||
| `-X`, `--xattrs` | Preserve extended attributes | ❌ Divergent | Deliberately restricted to unprivileged `user.*` extended attributes plus the two POSIX ACL xattrs; `security.*` (SELinux, capabilities, ...) and `trusted.*` are **never** captured or applied — a client can never force a privileged attribute onto the destination, and the receiver independently re-validates every incoming name against the whitelist. This is a security-policy divergence from rsync, which can preserve the privileged namespaces with the needed privilege; implementing them would defeat FastSync's privilege-escalation guard. `user.*` capture/apply matches rsync in a differential test. Payloads are bounded on both ends. Incompatible with `-s` |
|
| `-X`, `--xattrs` | Preserve extended attributes | ❌ Divergent | Deliberately restricted to unprivileged `user.*` extended attributes plus the two POSIX ACL xattrs; `security.*` (SELinux, capabilities, ...) and `trusted.*` are **never** captured or applied — a client can never force a privileged attribute onto the destination, and the receiver independently re-validates every incoming name against the whitelist. This is a security-policy divergence from rsync, which can preserve the privileged namespaces with the needed privilege; implementing them would defeat FastSync's privilege-escalation guard. `user.*` capture/apply matches rsync in a differential test. Payloads are bounded on both ends. Incompatible with `-s` |
|
||||||
| `-H`, `--hard-links` | Preserve hard links | ✅ Parity | Files on the source that share an inode (`st_dev`+`st_ino`, e.g. a `cp -al` tree) are re-created as hard links to one another on the destination, so duplicate links stay deduplicated and only the first member's data is sent (later members are transmitted as payload-less `STATUS_HARDLINK` frames). The receiver links each sibling to the first member's installed file with an atomic link + rename; on `link()` failure it falls back to a byte-identical local copy of the first member, never a partial/corrupt file. Requires the sequential scan for ordering (the first member is always emitted and installed before any sibling is linked). Works single-threaded and under `-j`/`--threads`, `--inplace`, `--delay-updates` (links staged and published by rename) and `--partial`. Crosses the wire (`preserve_hard_links` bool; `PROTOCOL_VERSION` bumped **2.11.0 → 2.12.0**, peers must match). Incompatible with `-s` (chunk serialization) and `--append`/`--append-verify`, rejected up front with a distinct error. See the Phase-4 hard-links notes below |
|
| `-H`, `--hard-links` | Preserve hard links | ✅ Parity | Files on the source that share an inode (`st_dev`+`st_ino`, e.g. a `cp -al` tree) are re-created as hard links to one another on the destination, so duplicate links stay deduplicated and only the first member's data is sent (later members are transmitted as payload-less `STATUS_HARDLINK` frames). The receiver links each sibling to the first member's installed file with an atomic link + rename; on `link()` failure it falls back to a byte-identical local copy of the first member, never a partial/corrupt file. Requires the sequential scan for ordering (the first member is always emitted and installed before any sibling is linked). Works single-threaded and under `-j`/`--threads`, `--inplace`, `--delay-updates` (links staged and published by rename) and `--partial`. Crosses the wire (`preserve_hard_links` bool; `PROTOCOL_VERSION` bumped **2.11.0 → 2.12.0**, peers must match). Incompatible with `-s` (chunk serialization) and `--append`/`--append-verify`, rejected up front with a distinct error. See the Phase-4 hard-links notes below |
|
||||||
@@ -667,16 +688,16 @@ targets verbatim, matching rsync.
|
|||||||
|------|-------------------|-----------------|-------|
|
|------|-------------------|-----------------|-------|
|
||||||
| `--checksum` | Skip based on checksum | ✅ Parity | `-c`/`--checksum` compares per-file whole-file content digests to skip unchanged files. **As of protocol 2.23.0 the short `-c` implies the checksum quick-check**, so a plain `-c` run verifies content rather than only affecting the `--incremental` handshake. The digest algorithm is `xxh128` by default (protocol 2.26.0's negotiated default) and is selectable via `--checksum-choice`/`--cc` (`xxh128`/`xxh3`/`xxh64`/`xxhash`/`md5`/`md4`/`sha1`/`none`/`auto`, plus rsync's two-name form) and `--checksum-seed=NUM` (see those rows) |
|
| `--checksum` | Skip based on checksum | ✅ Parity | `-c`/`--checksum` compares per-file whole-file content digests to skip unchanged files. **As of protocol 2.23.0 the short `-c` implies the checksum quick-check**, so a plain `-c` run verifies content rather than only affecting the `--incremental` handshake. The digest algorithm is `xxh128` by default (protocol 2.26.0's negotiated default) and is selectable via `--checksum-choice`/`--cc` (`xxh128`/`xxh3`/`xxh64`/`xxhash`/`md5`/`md4`/`sha1`/`none`/`auto`, plus rsync's two-name form) and `--checksum-seed=NUM` (see those rows) |
|
||||||
| `--checksum-choice=STR`, `--cc=STR` | Choose checksum algorithm | ✅ Parity | Real algorithm selection for the per-file whole-file digest used by the `--incremental`/`--checksum` handshake and basis-dir verification. **Protocol 2.26.0 accepts rsync 3.4.1's full set** — `xxh128` (the negotiated default), `xxh3`, `xxh64`, `xxhash`, `md5`, `md4`, `sha1`, `none`, `auto`, and the two-name `transfer,pre-transfer` form — with rsync's exit-4 rejection of an unknown name and of `none` on the transfer side when `--checksum` is on. `--cc=ALG` and space forms both parse. The algorithm id and seed cross the wire; the receiver hashes its old file with the same algorithm+seed and the per-file `STATUS_CHECK` handshake carries a bounded digest pinned to the negotiated length. `checksum_digest_file` now streams **every** supported algorithm (md4 via the self-contained RFC 1320 code, sha1/md5 via EVP, none as an empty digest), so the streaming path matches its contract, and `--out-format %C` uses the selected **transfer** half of a two-name choice and renders each algorithm byte-for-byte like rsync (xxh128 high-then-low, xxh64/xxh3 big-endian, md5/md4/sha1 standard hex, none a blank 2-char column) — differential-tested across all algorithms. **Track-3b finding — the block-checksum residual is not observable.** rsync applies the choice to the block checksum on its wire too, while FastSync selects only the whole-file comparison digest and keeps the delta BLOCK strong checksum fixed at xxHash32 (`DeltaBlockSig`, `delta_signature_create_seeded`). Because FastSync does not interoperate with rsync on the wire, only the compared surface matters, and a pre-seeded delta differential against rsync 3.4.1 (`--no-whole-file -B8192 --stats --out-format=%c|%C %n` vs `--incremental --delta`) shows the choice does not move it: across `xxh64`, `xxh128`, `xxh3`, `md5`, `md4`, `sha1` and both two-name orders the destination tree is byte-identical, `Matched data`/`Literal data`/`Total transferred file size` are unchanged (and equal to rsync's with the block size pinned), the `%c` block-checksum token is invariant (rsync `16 + 6·ceil(size/block)`, already documented under `--out-format`; FastSync its own basis-read counter), and the exit code is 0. The negotiated algorithm is visible only in `%C`, which applies it to the whole-file transfer digest and matches rsync byte-for-byte. A false block match would require the 4-byte adler32 AND the 4-byte xxHash32 to collide; at the 256 MiB maximum with the 1 KiB minimum block size the expected false matches are ≤2⁻¹⁸, and the choice cannot change this because FastSync's block strong sum is fixed. `auto` now consults `RSYNC_CHECKSUM_LIST` (rsync's whitespace-separated preference list; unknown names skipped, first supported wins, all-unknown is exit 4) before the compiled-in order; because both peers run the identical build this deterministic resolution needs no rsync peer probe, and an explicit `--cc` still wins. The list is differential-tested through `--out-format %C` (byte-identical digests to rsync for md5/sha1/xxh3) |
|
| `--checksum-choice=STR`, `--cc=STR` | Choose checksum algorithm | ✅ Parity | Real algorithm selection for the per-file whole-file digest used by the `--incremental`/`--checksum` handshake and basis-dir verification. **Protocol 2.26.0 accepts rsync 3.4.1's full set** — `xxh128` (the negotiated default), `xxh3`, `xxh64`, `xxhash`, `md5`, `md4`, `sha1`, `none`, `auto`, and the two-name `transfer,pre-transfer` form — with rsync's exit-4 rejection of an unknown name and of `none` on the transfer side when `--checksum` is on. `--cc=ALG` and space forms both parse. The algorithm id and seed cross the wire; the receiver hashes its old file with the same algorithm+seed and the per-file `STATUS_CHECK` handshake carries a bounded digest pinned to the negotiated length. `checksum_digest_file` now streams **every** supported algorithm (md4 via the self-contained RFC 1320 code, sha1/md5 via EVP, none as an empty digest), so the streaming path matches its contract, and `--out-format %C` uses the selected **transfer** half of a two-name choice and renders each algorithm byte-for-byte like rsync (xxh128 high-then-low, xxh64/xxh3 big-endian, md5/md4/sha1 standard hex, none a blank 2-char column) — differential-tested across all algorithms. **Track-3b finding — the block-checksum residual is not observable.** rsync applies the choice to the block checksum on its wire too, while FastSync selects only the whole-file comparison digest and keeps the delta BLOCK strong checksum fixed at xxHash32 (`DeltaBlockSig`, `delta_signature_create_seeded`). Because FastSync does not interoperate with rsync on the wire, only the compared surface matters, and a pre-seeded delta differential against rsync 3.4.1 (`--no-whole-file -B8192 --stats --out-format=%c|%C %n` vs `--incremental --delta`) shows the choice does not move it: across `xxh64`, `xxh128`, `xxh3`, `md5`, `md4`, `sha1` and both two-name orders the destination tree is byte-identical, `Matched data`/`Literal data`/`Total transferred file size` are unchanged (and equal to rsync's with the block size pinned), the `%c` block-checksum token is invariant (rsync `16 + 6·ceil(size/block)`, already documented under `--out-format`; FastSync its own basis-read counter), and the exit code is 0. The negotiated algorithm is visible only in `%C`, which applies it to the whole-file transfer digest and matches rsync byte-for-byte. A false block match would require the 4-byte adler32 AND the 4-byte xxHash32 to collide; at the 256 MiB maximum with the 1 KiB minimum block size the expected false matches are ≤2⁻¹⁸, and the choice cannot change this because FastSync's block strong sum is fixed. `auto` now consults `RSYNC_CHECKSUM_LIST` (rsync's whitespace-separated preference list; unknown names skipped, first supported wins, all-unknown is exit 4) before the compiled-in order; because both peers run the identical build this deterministic resolution needs no rsync peer probe, and an explicit `--cc` still wins. The list is differential-tested through `--out-format %C` (byte-identical digests to rsync for md5/sha1/xxh3) |
|
||||||
| `--compare-dest=DIR` | Compare dest files relative to DIR | ⚠️ Caveat | DIR is a receiver-side basis; protocol 2.26.0 uses an absolute path verbatim (rsync semantics) and resolves a relative path below the destination root (`..` components are rejected, `//` collapsed and trailing `/` dropped) — note rsync resolves a relative DIR against the destination directory while FastSync resolves it below the receive root and appends the mirrored source path, so the same relative spelling addresses a different tree (use an absolute DIR for exact parity). On the receiver's per-file check (implies `--incremental`) an exact match is rsync's metadata quick-check: same size and mtime (unless `--size-only`; `-I` disables matching), with NO content digest required by default (track 5a). A match suppresses the data transfer. The FastSync-only `--verify-basis` restores the stricter whole-file content equality. compare-dest never copies: it only skips a file the destination does **not** already hold (sparse destination, rsync parity), and is consulted before the normal delta/full paths. Repeatable; searched in command-line order, first match wins. Differential-tested against rsync 3.4.1 (`compare_dest`, and `test_verify_basis_restores_strict_content`). Residual: a basis MISS above the 256 MiB whole-file payload bound is refused up front (FastSync's general whole-file limit, not basis-specific); rsync applies basis dirs to arbitrary sizes. Wire: a basis-count field plus the `verify_basis` bool are present on the config frame (protocol 2.9.0/2.28.0) |
|
| `--compare-dest=DIR` | Compare dest files relative to DIR | ⚠️ Caveat | DIR is a receiver-side basis; protocol 2.26.0 uses an absolute path verbatim (rsync semantics) and resolves a relative path below the destination root (`..` components are rejected, `//` collapsed and trailing `/` dropped) — note rsync resolves a relative DIR against the destination directory while FastSync resolves it below the receive root and appends the mirrored source path, so the same relative spelling addresses a different tree (use an absolute DIR for exact parity). On the receiver's per-file check (implies `--incremental`) an exact match is rsync's metadata quick-check: same size and mtime (unless `--size-only`; `-I` disables matching), with NO content digest required by default (track 5a). A match suppresses the data transfer. The FastSync-only `--verify-basis` restores the stricter whole-file content equality. compare-dest never copies: it only skips a file the destination does **not** already hold (sparse destination, rsync parity), and is consulted before the normal delta/full paths. Repeatable; searched in command-line order, first match wins. Differential-tested against rsync 3.4.1 (`compare_dest`, and `test_verify_basis_restores_strict_content`). **Relative-DIR parity (parity-2.29):** a relative DIR now resolves against the destination directory with the file's transfer-relative name appended, exactly like rsync 3.4.1, instead of FastSync's source-mirrored wire path (the historical spelling stays as a fallback; differential `test_parity_basis_fuzzy.py`). Residual: a basis MISS above the 256 MiB whole-file payload bound is refused up front (FastSync's general whole-file limit, not basis-specific); rsync applies basis dirs to arbitrary sizes. Wire: a basis-count field plus the `verify_basis` bool are present on the config frame (protocol 2.9.0/2.28.0) |
|
||||||
| `--copy-dest=DIR` | Include copies of unchanged files | ⚠️ Caveat | Same basis rules as `--compare-dest`, but an exact match materializes a **local copy** of the DIR file into the destination (via the atomic temp+rename store path, so `--existing`/`--ignore-existing`/`--update`/`--backup`/`--delay-updates` all still apply) instead of transferring data. Track 5a re-applies the SOURCE attributes on the copy (rsync's "copy then fix attributes"): the sender transmits the source metadata with the basis check frame, so the copy's mode/uid/gid/mtime match the source rather than the basis inode (differential `copy_dest` compares modes). The copy streams the basis file through a bounded buffer, so a basis larger than the whole-file payload bound still materializes. Repeatable; command-line order = priority. Requires `--incremental` (implied); incompatible with `-s`. Wire: protocol 2.9.0 |
|
| `--copy-dest=DIR` | Include copies of unchanged files | ⚠️ Caveat | Same basis rules as `--compare-dest`, but an exact match materializes a **local copy** of the DIR file into the destination (via the atomic temp+rename store path, so `--existing`/`--ignore-existing`/`--update`/`--backup`/`--delay-updates` all still apply) instead of transferring data. Track 5a re-applies the SOURCE attributes on the copy (rsync's "copy then fix attributes"): the sender transmits the source metadata with the basis check frame, so the copy's mode/uid/gid/mtime match the source rather than the basis inode (differential `copy_dest` compares modes). The copy streams the basis file through a bounded buffer, so a basis larger than the whole-file payload bound still materializes. Repeatable; command-line order = priority. Requires `--incremental` (implied); incompatible with `-s`. Wire: protocol 2.9.0 |
|
||||||
| `--link-dest=DIR` | Hardlink to files when unchanged | ⚠️ Caveat | Same basis rules as `--copy-dest`, but an exact match installs an atomic **hard link** to the DIR file (temp hard link + rename) so no data or disk space is used; where the link is impossible (basis on another filesystem, filesystem refuses links) it falls back cleanly to a byte-identical local copy (streamed from the basis, so an over-limit basis still works), never a corrupt/partial file. `--delay-updates` stages the link and publishes by rename, so the final entry stays a real hard link. Repeatable (searched in command-line order, first match wins). Differential-tested against rsync 3.4.1 (`link_dest`). Inherent shared-inode semantics (identical to rsync): a link keeps the basis inode's own mode/uid/gid and mtime — metadata is never written through the shared inode (that would mutate the basis file), so a later `--inplace` run that rewrites such a destination path **will mutate the basis snapshot** through the shared inode (use `--copy-dest` when the destination must stay independently writable); protocol 2.26.0 re-links an already up-to-date destination file to the basis; a `--remove-source-files` source satisfied by a basis dir is treated as skipped and therefore **retained** (never removed); basis dirs are excluded from `--delete`. Residual: a basis MISS above the 256 MiB whole-file payload bound is refused (FastSync's general whole-file limit). Requires `--incremental` (implied); incompatible with `-s`. Wire: protocol 2.9.0 |
|
| `--link-dest=DIR` | Hardlink to files when unchanged | ⚠️ Caveat | Same basis rules as `--copy-dest`, but an exact match installs an atomic **hard link** to the DIR file (temp hard link + rename) so no data or disk space is used; where the link is impossible (basis on another filesystem, filesystem refuses links) it falls back cleanly to a byte-identical local copy (streamed from the basis, so an over-limit basis still works), never a corrupt/partial file. `--delay-updates` stages the link and publishes by rename, so the final entry stays a real hard link. Repeatable (searched in command-line order, first match wins). Differential-tested against rsync 3.4.1 (`link_dest`). Inherent shared-inode semantics (identical to rsync): a link keeps the basis inode's own mode/uid/gid and mtime — metadata is never written through the shared inode (that would mutate the basis file), so a later `--inplace` run that rewrites such a destination path **will mutate the basis snapshot** through the shared inode (use `--copy-dest` when the destination must stay independently writable); protocol 2.26.0 re-links an already up-to-date destination file to the basis; a `--remove-source-files` source satisfied by a basis dir is treated as skipped and therefore **retained** (never removed); basis dirs are excluded from `--delete`. **Relative-DIR parity (parity-2.29):** a relative DIR resolves against the destination directory with the transfer-relative name appended, exactly like rsync 3.4.1 (differential `test_parity_basis_fuzzy.py`). Residual: a basis MISS above the 256 MiB whole-file payload bound is refused (FastSync's general whole-file limit). Requires `--incremental` (implied); incompatible with `-s`. Wire: protocol 2.9.0 |
|
||||||
| `-y`, `--fuzzy`, `--no-fuzzy` | Find similar file for basis | ⚠️ Caveat | `-y/--fuzzy` is a pure bandwidth optimization on the existing receiver-driven delta path: when a file must be transferred and the destination holds no usable content at the exact path (file absent, or the destination file is outside the delta engine's size bounds), the receiver searches the SAME destination directory for an existing regular file whose basename is similar to the incoming name and uses it as the delta basis, so the sender transmits only the differences instead of the whole file. The output is always byte-exact regardless of which (or whether any) basis is chosen. Decision location: the receiver performs the candidate search inside `receive_incremental_check` and sends the normal `STATUS_DELTA_SIGNATURE`; the sender never learns the basis was a different file, so no new frame type or sender logic was needed — only the config frame grew a `fuzzy` boolean, so `PROTOCOL_VERSION` was bumped **2.8.0 → 2.9.0** (peers must match). Similarity heuristic (a deterministic port of rsync 3.4.1's matcher — `util1.c` `fuzzy_distance`/`find_filename_suffix` plus `generator.c find_fuzzy`'s exact size+mtime pass — documented precisely): candidates are the target's sibling entries in its destination directory, opened `O_NOFOLLOW`/`AT_SYMLINK_NOFOLLOW` under the confined root (symlinks never followed; nothing outside the destination root is ever read or hashed); dotfiles, directories, the target's own name, and the `.fastsync-stage`/temp scratch names are excluded; like the ordinary delta path, the block signature the receiver transmits is derived from on-disk content it may not otherwise send, so a negotiated `--fuzzy` run exposes the destination's sibling files (at block granularity) to the sender as a known-plaintext oracle — the same information class as the normal delta handshake over the file being replaced; the size gate is the delta engine's own bounds (both files ≥ 16 KiB, ≤ `--delta-max`, ratio ≤ 10×); rsync's fuzzy matcher is not tied to a delta size bound and empirically reuses a basis well outside FastSync's window (a 64 KiB source against a repeated-content sibling from 0.25× to 10000×, and files as small as 300 B), so candidate ELIGIBILITY — and hence the chosen basis — can differ even though the name heuristic is the same; protocol 2.26.0 uses rsync's weighted-Levenshtein name/suffix distance plus an exact size+mtime pass and reads a single best candidate; the tie-break (smallest size gap, then lexical name) is deterministic where rsync leaves equal distances to its file-list order; the directory scan is capped at 4096 entries so a pathological directory cannot stall a transfer. When fuzzy applies: only to files the receiver would otherwise send whole — the destination's own file is always preferred as the delta basis when it exists and fits the delta size bounds, so fuzzy does NOT replace an existing-but-different destination basis; FastSync's 10× delta size-ratio bound means an existing destination file that is too far away in size still lets the fuzzy search run. When no similar candidate exists the transfer falls back to the normal whole-file transfer. rsync-divergence note: the name matching is rsync's own rule; the residual is eligibility bounded by FastSync's delta engine, so no name-matcher port can widen it. Because FastSync's delta machinery is off by default (rsync's is on), `--fuzzy` implies `--incremental` + `--delta` (unless `--whole-file`/`-W` or an explicit `--no-delta` switched delta off, in which case fuzzy is inert — matching rsync where `--whole-file` makes fuzzy irrelevant). Unlike the basis-dir options, `--fuzzy` honors an explicit `--no-incremental` (it does not force the handshake back on); an explicit `--no-incremental` also suppresses the delta implication so no invalid `--delta requires --incremental` config results. `--no-fuzzy` negates it. All surrounding semantics are untouched: a fuzzy-reconstructed file is stored as a normal file, so `--remove-source-files`, itemize/`-i`, `--stats`, `--backup`, `--delay-updates`, `--existing`/`--ignore-existing`/`--update` behave exactly as for a whole-file transfer (the fuzzy delta does not skip the file). **Reclassified Caveat (track 5b):** the output is always byte-exact regardless of the basis, and the name rule is rsync's, so the residual is candidate ELIGIBILITY: FastSync's 10× ratio / 16 KiB delta gates make its size window strictly narrower than rsync's, and when eligibility differs the chosen basis — and therefore the `--stats` `Matched data`/`Literal data`/`Total transferred file size` counters — can differ even though the tree cannot. Where the two tools' choices coincide and the block size is pinned, both the tree and the counters match rsync (differential `fuzzy_basis`); `TestFuzzy` pins the window boundary on both sides (a >10× sibling and a <16 KiB sibling are declined, with a byte-exact whole-file fallbaLine truncated
|
| `-y`, `--fuzzy`, `--no-fuzzy` | Find similar file for basis | ⚠️ Caveat | `-y/--fuzzy` is a pure bandwidth optimization on the existing receiver-driven delta path: when a file must be transferred and the destination holds no usable content at the exact path (file absent, or the destination file is outside the delta engine's size bounds), the receiver searches the SAME destination directory for an existing regular file whose basename is similar to the incoming name and uses it as the delta basis, so the sender transmits only the differences instead of the whole file. The output is always byte-exact regardless of which (or whether any) basis is chosen. Decision location: the receiver performs the candidate search inside `receive_incremental_check` and sends the normal `STATUS_DELTA_SIGNATURE`; the sender never learns the basis was a different file, so no new frame type or sender logic was needed — only the config frame grew a `fuzzy` boolean, so `PROTOCOL_VERSION` was bumped **2.8.0 → 2.9.0** (peers must match). Similarity heuristic (a deterministic port of rsync 3.4.1's matcher — `util1.c` `fuzzy_distance`/`find_filename_suffix` plus `generator.c find_fuzzy`'s exact size+mtime pass — documented precisely): candidates are the target's sibling entries in its destination directory, opened `O_NOFOLLOW`/`AT_SYMLINK_NOFOLLOW` under the confined root (symlinks never followed; nothing outside the destination root is ever read or hashed); dotfiles, directories, the target's own name, and the `.fastsync-stage`/temp scratch names are excluded; like the ordinary delta path, the block signature the receiver transmits is derived from on-disk content it may not otherwise send, so a negotiated `--fuzzy` run exposes the destination's sibling files (at block granularity) to the sender as a known-plaintext oracle — the same information class as the normal delta handshake over the file being replaced; the size gate is the delta engine's own bounds (both files ≥ 16 KiB, ≤ `--delta-max`, ratio ≤ 10×); rsync's fuzzy matcher is not tied to a delta size bound and empirically reuses a basis well outside FastSync's window (a 64 KiB source against a repeated-content sibling from 0.25× to 10000×, and files as small as 300 B), so candidate ELIGIBILITY — and hence the chosen basis — can differ even though the name heuristic is the same; protocol 2.26.0 uses rsync's weighted-Levenshtein name/suffix distance plus an exact size+mtime pass and reads a single best candidate; the tie-break (smallest size gap, then lexical name) is deterministic where rsync leaves equal distances to its file-list order; the directory scan is capped at 4096 entries so a pathological directory cannot stall a transfer. When fuzzy applies: only to files the receiver would otherwise send whole — the destination's own file is always preferred as the delta basis when it exists and fits the delta size bounds, so fuzzy does NOT replace an existing-but-different destination basis; FastSync's 10× delta size-ratio bound means an existing destination file that is too far away in size still lets the fuzzy search run. When no similar candidate exists the transfer falls back to the normal whole-file transfer. rsync-divergence note: the name matching is rsync's own rule; the residual is eligibility bounded by FastSync's delta engine, so no name-matcher port can widen it. Because FastSync's delta machinery is off by default (rsync's is on), `--fuzzy` implies `--incremental` + `--delta` (unless `--whole-file`/`-W` or an explicit `--no-delta` switched delta off, in which case fuzzy is inert — matching rsync where `--whole-file` makes fuzzy irrelevant). Unlike the basis-dir options, `--fuzzy` honors an explicit `--no-incremental` (it does not force the handshake back on); an explicit `--no-incremental` also suppresses the delta implication so no invalid `--delta requires --incremental` config results. `--no-fuzzy` negates it. All surrounding semantics are untouched: a fuzzy-reconstructed file is stored as a normal file, so `--remove-source-files`, itemize/`-i`, `--stats`, `--backup`, `--delay-updates`, `--existing`/`--ignore-existing`/`--update` behave exactly as for a whole-file transfer (the fuzzy delta does not skip the file). **Reclassified Caveat (track 5b):** the output is always byte-exact regardless of the basis, and the name rule is rsync's, **Eligibility parity (parity-2.29):** the fuzzy candidate search no longer inherits the ordinary delta engine's 16 KiB minimum or 10× size-ratio bound, so an oversized or sub-16-KiB sibling is now reused exactly as rsync reuses it — the chosen basis (and therefore `Matched data`/`Literal data`/`Total transferred file size`) matches rsync where the block size is pinned (differential `test_parity_basis_fuzzy.py`; the old boundary tests were flipped to assert both tools reuse the basis). Residual: the whole-basis buffer cap (`MAX_RECEIVE_WHOLE_FILE_SIZE`, 256 MiB) and the deterministic tie-break where Line truncated
|
||||||
|
|
||||||
## 12. Compression
|
## 12. Compression
|
||||||
|
|
||||||
| Flag | Rsync Description | FastSync Status | Notes |
|
| Flag | Rsync Description | FastSync Status | Notes |
|
||||||
|------|-------------------|-----------------|-------|
|
|------|-------------------|-----------------|-------|
|
||||||
| `-z`, `--compress` | Compress file data | ✅ Parity | Streaming compression. **Protocol 2.26.0 implements rsync 3.4.1's codec set** (`zstd` default, `lz4`, `zlib`, `zlibx`, `none`), selectable via `--compress-choice`/`--zc` and negotiated with `auto`. `-z` is the compression short form; `-c` is rsync's `--checksum`. `--skip-compress` applies rsync 3.4.1's default suffix list when no list is given. **Track 3a closes the codec caveats:** `zlibx` is no longer a divergence — FastSync's zlib stream already carries only the delta/token (literal) bytes, which is exactly rsync's zlibx semantics, so `--zc=zlib` and `--zc=zlibx` land the same tree/stdout/exit (differential `test_compress_codec_matches_rsync_bytes`) and the zlib/zlibx aliasing is only an implementation detail. Each codec now uses rsync's own default `--compress-level` (zstd 3, zlib/zlibx 6, lz4 ignored) and `auto` consults `RSYNC_COMPRESS_LIST` before the compiled-in order; the deterministic same-build resolution needs no peer probe |
|
| `-z`, `--compress` | Compress file data | ✅ Parity | Streaming compression. **Protocol 2.26.0 implements rsync 3.4.1's codec set** (`zstd` default, `lz4`, `zlib`, `zlibx`, `none`), selectable via `--compress-choice`/`--zc` and negotiated with `auto`. `-z` is the compression short form; `-c` is rsync's `--checksum`. `--skip-compress` applies rsync 3.4.1's default suffix list when no list is given. **Track 3a closes the codec caveats:** `zlibx` is no longer a divergence — FastSync's zlib stream already carries only the delta/token (literal) bytes, which is exactly rsync's zlibx semantics, so `--zc=zlib` and `--zc=zlibx` land the same tree/stdout/exit (differential `test_compress_codec_matches_rsync_bytes`) and the zlib/zlibx aliasing is only an implementation detail. Each codec now uses rsync's own default `--compress-level` (zstd 3, zlib/zlibx 6, lz4 ignored) and `auto` consults `RSYNC_COMPRESS_LIST` before the compiled-in order; the deterministic same-build resolution needs no peer probe. **Audit-cycle fix:** the decompressor's internal ceiling is now defined by the protocol whole-file bound (`MAX_RECEIVE_WHOLE_FILE_SIZE`, 256 MiB) instead of a separate 100 MiB constant, so `-z` on a 100–256 MiB regular file no longer fails with `Declared decompressed size exceeds 104857600 bytes` |
|
||||||
| `--compress-choice=STR`, `--zc=STR` | Choose compression algorithm | ✅ Parity | Protocol 2.26.0 accepts rsync 3.4.1's compiled-in choices — `zstd` (default), `lz4`, `zlib`, `zlibx`, `none`, `auto` — and rejects an unknown name with exit 4 like rsync. The negotiated codec id crosses the wire (`compression_algo`), so the receiver decodes with the sender's codec. `--zc` is the alias. `auto` now resolves through `RSYNC_COMPRESS_LIST` (whitespace-separated; unknown names skipped, first supported wins, all-unknown is exit 4) and then the compiled-in order, and an explicit `--zc` wins; the deterministic same-build resolution needs no peer probe. `zlib`/`zlibx` share FastSync's literal-only zlib path, which is rsync's zlibx behavior and is observably identical for both, so `zlibx` is not a divergence (the aliasing is an implementation detail) |
|
| `--compress-choice=STR`, `--zc=STR` | Choose compression algorithm | ✅ Parity | Protocol 2.26.0 accepts rsync 3.4.1's compiled-in choices — `zstd` (default), `lz4`, `zlib`, `zlibx`, `none`, `auto` — and rejects an unknown name with exit 4 like rsync. The negotiated codec id crosses the wire (`compression_algo`), so the receiver decodes with the sender's codec. `--zc` is the alias. `auto` now resolves through `RSYNC_COMPRESS_LIST` (whitespace-separated; unknown names skipped, first supported wins, all-unknown is exit 4) and then the compiled-in order, and an explicit `--zc` wins; the deterministic same-build resolution needs no peer probe. `zlib`/`zlibx` share FastSync's literal-only zlib path, which is rsync's zlibx behavior and is observably identical for both, so `zlibx` is not a divergence (the aliasing is an implementation detail) |
|
||||||
| `--compress-level=NUM`, `--zl=NUM` | Set compression level | ✅ Parity | Accepted range 1-22. When omitted, rsync 3.4.1's **per-codec default** applies: zstd 3 (`ZSTD_CLEVEL_DEFAULT`), zlib/zlibx 6 (`Z_DEFAULT_COMPRESSION` resolved), lz4 ignored (no tunable level; FastSync keeps a positive gate value and `lz4_compress` ignores it, so the bytes match rsync). An explicit level is clamped per codec like rsync's `init_compression_level()`: zstd 1-22, zlib/zlibx 1-9, lz4 ignored. Verified against `rsync --debug=NSTR1`, which reports the same effective level per codec |
|
| `--compress-level=NUM`, `--zl=NUM` | Set compression level | ✅ Parity | Accepted range 1-22. When omitted, rsync 3.4.1's **per-codec default** applies: zstd 3 (`ZSTD_CLEVEL_DEFAULT`), zlib/zlibx 6 (`Z_DEFAULT_COMPRESSION` resolved), lz4 ignored (no tunable level; FastSync keeps a positive gate value and `lz4_compress` ignores it, so the bytes match rsync). An explicit level is clamped per codec like rsync's `init_compression_level()`: zstd 1-22, zlib/zlibx 1-9, lz4 ignored. Verified against `rsync --debug=NSTR1`, which reports the same effective level per codec |
|
||||||
| `--compress-threads=NUM` | Set compression threads | ✅ Parity | `compression_threads` config field (client-only; does not cross the wire). Sets the number of worker threads used by the zstd compression pool to NUM (1..64; 0/garbage/oversized rejected up front). Accepted in both `--compress-threads=NUM` and two-argument `--compress-threads NUM` forms. Composes with `-z`/compression; under the `-j`/`--threads` multithreaded pipeline it parallelizes compressed chunk encoding. See test_tcp.py `-z --compress-threads=2` and test_client_cli.c |
|
| `--compress-threads=NUM` | Set compression threads | ✅ Parity | `compression_threads` config field (client-only; does not cross the wire). Sets the number of worker threads used by the zstd compression pool to NUM (1..64; 0/garbage/oversized rejected up front). Accepted in both `--compress-threads=NUM` and two-argument `--compress-threads NUM` forms. Composes with `-z`/compression; under the `-j`/`--threads` multithreaded pipeline it parallelizes compressed chunk encoding. See test_tcp.py `-z --compress-threads=2` and test_client_cli.c |
|
||||||
@@ -697,7 +718,7 @@ targets verbatim, matching rsync.
|
|||||||
| `-4`, `--ipv4` | Prefer IPv4 | ✅ Parity | Forces `AF_INET` in the `getaddrinfo` hints for client destination/source resolution and the server bind (see the Phase 5, Wave B note). Mutually exclusive with `-6` |
|
| `-4`, `--ipv4` | Prefer IPv4 | ✅ Parity | Forces `AF_INET` in the `getaddrinfo` hints for client destination/source resolution and the server bind (see the Phase 5, Wave B note). Mutually exclusive with `-6` |
|
||||||
| `-6`, `--ipv6` | Prefer IPv6 | ✅ Parity | Forces `AF_INET6` in the `getaddrinfo` hints for client destination/source resolution and the server bind. Mutually exclusive with `-4` |
|
| `-6`, `--ipv6` | Prefer IPv6 | ✅ Parity | Forces `AF_INET6` in the `getaddrinfo` hints for client destination/source resolution and the server bind. Mutually exclusive with `-4` |
|
||||||
| `--remote-option=OPT`, `-M` | Send an option only to the remote side | ❌ Divergent | Each value is appended to the remote server invocation over SSH as an individually single-quote-escaped shell word in `ssh_build_remote_command()`. Values are validated (non-empty, no control characters) and shell metacharacters cannot break out of the quoting (`;`, `&`, `\|`, <code>`</code>, `$`, `(`, `)`, quotes are neutralized), so a value cannot inject an arbitrary remote command and a subsequent `--` on the client line cannot be turned into one. The short `-M` form (`-M OPT`, `-M=OPT`, and rsync-style attached `-MOPT`) is available, matching rsync; metadata mode moved to long-only `--preserve`. **Reclassified because the daemon/TCP case cannot be reproduced:** `-M` is only meaningful for the SSH transport (`user@host:path`); a daemon (`host::module/path`) or local TCP destination **rejects** it, whereas rsync forwards it to its own remote process on every transport. A differential test starts a real rsync daemon and shows `-M--totally-bogus` reaching the remote parser (`unknown option`) while a valid `-M--safe-links` is accepted. FastSync's daemon handshake is a fixed binary config frame with no per-connection argv channel; adding one would let a client set arbitrary server-side options (the same class of divergence as the native daemon config/auth), so the safe subset stays SSH-only |
|
| `--remote-option=OPT`, `-M` | Send an option only to the remote side | ❌ Divergent | Each value is appended to the remote server invocation over SSH as an individually single-quote-escaped shell word in `ssh_build_remote_command()`. Values are validated (non-empty, no control characters) and shell metacharacters cannot break out of the quoting (`;`, `&`, `\|`, <code>`</code>, `$`, `(`, `)`, quotes are neutralized), so a value cannot inject an arbitrary remote command and a subsequent `--` on the client line cannot be turned into one. The short `-M` form (`-M OPT`, `-M=OPT`, and rsync-style attached `-MOPT`) is available, matching rsync; metadata mode moved to long-only `--preserve`. **Reclassified because the daemon/TCP case cannot be reproduced:** `-M` is only meaningful for the SSH transport (`user@host:path`); a daemon (`host::module/path`) or local TCP destination **rejects** it, whereas rsync forwards it to its own remote process on every transport. A differential test starts a real rsync daemon and shows `-M--totally-bogus` reaching the remote parser (`unknown option`) while a valid `-M--safe-links` is accepted. FastSync's daemon handshake is a fixed binary config frame with no per-connection argv channel; adding one would let a client set arbitrary server-side options (the same class of divergence as the native daemon config/auth), so the safe subset stays SSH-only |
|
||||||
| `--bwlimit=RATE` | Limit I/O bandwidth | ✅ Parity | A faithful port of rsync 3.4.1's `parse_size_arg(bwlimit_arg, 'K', "bwlimit", 512, -1, True)`: a bare value is KiB/s, `K`/`M`/`G`/`T`/`P` are binary suffixes, `KB`/`MB` are decimal, `KiB`/`MiB` are binary, decimals are accepted and quantized to whole KiB exactly like rsync's `(size + 512) / 1024`, `0` (or an empty value) means "no limit", and any other value below the 512-byte floor is rejected. The token bucket's burst capacity is ~100 ms of bandwidth, matching the point at which rsync's leaky bucket starts sleeping, so a throttled transfer paces like rsync (4 MiB at `--bwlimit=1024`/`2048` matches rsync within ~4%). Differential-tested: the accept/reject matrix and the wall-clock rate both match rsync 3.4.1. The limit is a local I/O concern and is not negotiated on the wire |
|
| `--bwlimit=RATE` | Limit I/O bandwidth | ✅ Parity | A faithful port of rsync 3.4.1's `parse_size_arg(bwlimit_arg, 'K', "bwlimit", 512, -1, True)`: a bare value is KiB/s, `K`/`M`/`G`/`T`/`P` are binary suffixes, `KB`/`MB` are decimal, `KiB`/`MiB` are binary, decimals are accepted and quantized to whole KiB exactly like rsync's `(size + 512) / 1024`, `0` (or an empty value) means "no limit", and any other value below the 512-byte floor is rejected. The token bucket's burst capacity is ~100 ms of bandwidth, matching the point at which rsync's leaky bucket starts sleeping, so a throttled transfer paces like rsync (4 MiB at `--bwlimit=1024`/`2048` matches rsync within ~4%). Differential-tested: the accept/reject matrix and the wall-clock rate both match rsync 3.4.1. **Audit-cycle fix:** the plaintext-TCP `--sendfile` fast path now passes its writes through the same token bucket, so `--bwlimit` also paces it (previously the `sendfile(2)` path bypassed the limiter entirely); the TLS and plaintext transports therefore share identical throttling. The limit is a local I/O concern and is not negotiated on the wire |
|
||||||
|
|
||||||
## 14. Daemon Mode
|
## 14. Daemon Mode
|
||||||
|
|
||||||
@@ -707,8 +728,8 @@ targets verbatim, matching rsync.
|
|||||||
| `--config=FILE` | Alternate rsyncd.conf file | ❌ Divergent | Wave A: selects the daemon config file. Default when omitted (in `--daemon` mode): `~/.config/fastsync/fastsyncd.conf` if it exists, else `/etc/fastsyncd.conf`. The grammar is FastSync-native (documented in the Daemon Mode notes below) and strictly rejects unknown keys so a typo can never silently change what a module serves; requires `--daemon` |
|
| `--config=FILE` | Alternate rsyncd.conf file | ❌ Divergent | Wave A: selects the daemon config file. Default when omitted (in `--daemon` mode): `~/.config/fastsync/fastsyncd.conf` if it exists, else `/etc/fastsyncd.conf`. The grammar is FastSync-native (documented in the Daemon Mode notes below) and strictly rejects unknown keys so a typo can never silently change what a module serves; requires `--daemon` |
|
||||||
| `--dparam=OVERRIDE` | Override global daemon config | ❌ Divergent | Wave A: overrides one global scalar from the command line (`--dparam port=8734` and `--dparam=KEY=VALUE` both work). Limited to the global keys the grammar defines (`port`, `motd file`, `address`, `max connections`, `max connections per host`, `auth failure delay`, `auth lockout threshold`, `auth lockout duration`, `hosts allow`, `hosts deny`); keys are case-insensitive and unknown keys/invalid values are rejected. Requires `--daemon` |
|
| `--dparam=OVERRIDE` | Override global daemon config | ❌ Divergent | Wave A: overrides one global scalar from the command line (`--dparam port=8734` and `--dparam=KEY=VALUE` both work). Limited to the global keys the grammar defines (`port`, `motd file`, `address`, `max connections`, `max connections per host`, `auth failure delay`, `auth lockout threshold`, `auth lockout duration`, `hosts allow`, `hosts deny`); keys are case-insensitive and unknown keys/invalid values are rejected. Requires `--daemon` |
|
||||||
| `--no-detach` | Don't detach from parent | ✅ Parity | Wave A: with `--daemon`, keeps the listener in the foreground (what integration tests use). Without it the daemonizes (fork/setsid, stdio redirected to /dev/null) after the listening socket is bound. Requires `--daemon` |
|
| `--no-detach` | Don't detach from parent | ✅ Parity | Wave A: with `--daemon`, keeps the listener in the foreground (what integration tests use). Without it the daemonizes (fork/setsid, stdio redirected to /dev/null) after the listening socket is bound. Requires `--daemon` |
|
||||||
| `--password-file=FILE` | Read daemon password from file | ❌ Divergent | A7 daemon auth. Client: `--password-file` supplies `user:password` for a `host::module/path` destination (the username is taken from this file, so `user@host::module` stays rejected); the literal password is held client-side only for the SCRAM handshake and wiped at teardown. Server (`fastsync-server --daemon --password-file FILE`): the salted-PBKDF2 verifier store that modules with `auth users` are verified against. **Neither the password nor any replayable bearer value crosses the wire or is stored server-side** — the store holds a per-user salt plus derived keys, and the daemon proves the secret with a per-connection nonce challenge. The file must be private to its owner: both the client and server verify the exact inode they read (open-then-`fstat`, so the check cannot be raced) and refuse a `--password-file`/`--early-input` that is not owned by the current user or grants any group/other permission bit (mode 0600), mirroring the TLS private-key check. A process-substitution pipe (`--early-input <(vault ...)`) is still accepted when it satisfies those checks. See the Daemon Mode notes below for the file formats and the plaintext/TLS caveat |
|
| `--password-file=FILE` | Read daemon password from file | ❌ Divergent | A7 daemon auth. Client: `--password-file` supplies `user:password` for a `host::module/path` destination (the username is taken from this file, so `user@host::module` stays rejected); the literal password is held client-side only for the SCRAM handshake and wiped at teardown. Server (`fastsync-server --daemon --password-file FILE`): the salted-PBKDF2 verifier store that modules with `auth users` are verified against. **Neither the password nor any replayable bearer value crosses the wire or is stored server-side** — the store holds a per-user salt plus derived keys, and the daemon proves the secret with a per-connection nonce challenge. The file must be private to its owner: both the client and server verify the exact inode they read (open-then-`fstat`, so the check cannot be raced) and refuse a `--password-file`/`--early-input` that is not owned by the current user or grants any group/other permission bit (mode 0600), mirroring the TLS private-key check. A process-substitution pipe (`--early-input <(vault ...)`) is still accepted when it satisfies those checks. **Hardening follow-up:** the file is opened with `O_NOFOLLOW`, so a symlinked credential path fails closed (`ELOOP`) instead of being followed before the owner/mode gate; literal fd-backed paths (`/dev/fd/<digits>`, `/proc/self/fd/<digits>`, which is what a bash process substitution passes) are exempt, so process substitution still works. A FIFO/process-substitution read now waits under a bounded ~3 s deadline for its writer, so a slow producer works while a connected-but-silent FIFO fails instead of hanging. See the Daemon Mode notes below for the file formats and the plaintext/TLS caveat |
|
||||||
| `--early-input=FILE` | Use FILE for daemon early exec | ❌ Divergent | Server-only (requires `--daemon`): a second credential-store file, same new-format grammar as `--password-file`, read before the listener accepts connections (a secrets-manager / process-substitution source). Its entries layer over `--password-file`: byte-identical verifiers dedupe, a conflicting verifier for the same user is a startup error. A daemon whose modules declare `auth users` must be given at least one of the two, or it refuses to start (fail closed) |
|
| `--early-input=FILE` | Use FILE for daemon early exec | ❌ Divergent | Server-only (requires `--daemon`): a second credential-store file, same new-format grammar as `--password-file`, read before the listener accepts connections (a secrets-manager / process-substitution source). Its entries layer over `--password-file`: byte-identical verifiers dedupe, a conflicting verifier for the same user is a startup error. Opened with the same `O_NOFOLLOW` hardening as `--password-file` (a symlinked path fails closed with `ELOOP`; fd-backed `/dev/fd/N`/`/proc/self/fd/N` process-substitution paths are exempt) and a FIFO read is bound-waited (~3 s) so a slow producer works while a writer-less FIFO cannot hang. A daemon whose modules declare `auth users` must be given at least one of the two, or it refuses to start (fail closed) |
|
||||||
| `--hash-credentials=FILE`, `--iterations N` | Hash a plaintext credential file | ❌ Divergent | Server-only offline tool (A7): reads the `user:password` lines of FILE (same owner-only 0600 check) and prints one new-format store line per entry to stdout, then exits. `--iterations` sets the PBKDF2 work factor (default 600000, range 100000–10000000). Dependency-free and does not run a listener. Use its output as `--password-file` for `--daemon`. There is no auto-upgrade: a legacy store line is hard-rejected by the loader and must be regenerated |
|
| `--hash-credentials=FILE`, `--iterations N` | Hash a plaintext credential file | ❌ Divergent | Server-only offline tool (A7): reads the `user:password` lines of FILE (same owner-only 0600 check) and prints one new-format store line per entry to stdout, then exits. `--iterations` sets the PBKDF2 work factor (default 600000, range 100000–10000000). Dependency-free and does not run a listener. Use its output as `--password-file` for `--daemon`. There is no auto-upgrade: a legacy store line is hard-rejected by the loader and must be regenerated |
|
||||||
|
|
||||||
**Daemon Mode notes (Wave A protocol 2.15.0; A7 auth protocol 2.19.0; MOTD no bump):** FastSync daemon mode is supported in FastSync's own protocol/config grammar, not rsync's SMB/daemon option encoding.
|
**Daemon Mode notes (Wave A protocol 2.15.0; A7 auth protocol 2.19.0; MOTD no bump):** FastSync daemon mode is supported in FastSync's own protocol/config grammar, not rsync's SMB/daemon option encoding.
|
||||||
@@ -733,7 +754,7 @@ targets verbatim, matching rsync.
|
|||||||
| Flag | Rsync Description | FastSync Status | Notes |
|
| Flag | Rsync Description | FastSync Status | Notes |
|
||||||
|------|-------------------|-----------------|-------|
|
|------|-------------------|-----------------|-------|
|
||||||
| Path escape detection | Ensure files stay within root | ✅ Parity | `has_path_traversal()` + realpath |
|
| Path escape detection | Ensure files stay within root | ✅ Parity | `has_path_traversal()` + realpath |
|
||||||
| Symlink-safe delete | Skip symlinks in delete walk | ✅ Parity | `delete_extras_walk()` |
|
| Symlink-safe delete | Skip symlinks in delete walk | ✅ Parity | `delete_extras_fd()` (`src/shared/utils.c`) and `manifest_delete_extras()` (`src/shared/file_receive.c`) |
|
||||||
| Protocol version check | Verify compatible versions | ✅ Parity | `config_receive()` |
|
| Protocol version check | Verify compatible versions | ✅ Parity | `config_receive()` |
|
||||||
| Max data/string/chunk sizes | Prevent OOM attacks | ✅ Parity | Per-message limits |
|
| Max data/string/chunk sizes | Prevent OOM attacks | ✅ Parity | Per-message limits |
|
||||||
| Per-connection memory limit | Cap memory per connection | ✅ Parity | `MAX_CONNECTION_MEMORY` is **256 MiB per connection** (256 * 1024 * 1024 bytes), charged across protocol reservations and decompression/chunk allocations. This is a FastSync-internal bound with no direct rsync analogue |
|
| Per-connection memory limit | Cap memory per connection | ✅ Parity | `MAX_CONNECTION_MEMORY` is **256 MiB per connection** (256 * 1024 * 1024 bytes), charged across protocol reservations and decompression/chunk allocations. This is a FastSync-internal bound with no direct rsync analogue |
|
||||||
@@ -930,10 +951,12 @@ These are the last compatibility items and the closing phase toward rsync flag p
|
|||||||
|
|
||||||
**Wire:** two trailing config-frame blocks after the `--iconv` spec, in fixed order — `send_privilege_options`/`receive_privilege_options` (one `super_mode` int, validated `0..2`), then `send_copy_as_options`/`receive_copy_as_options` (presence int + two int32 ids, validated `>= 0`, with `copy_as_set ⇒ use_metadata`). `PROTOCOL_VERSION` bumped **2.17.0 → 2.18.0**. **Divergences from rsync:** rsync's `--super` elevates the receiver and `--copy-as` actually switches its credentials; FastSync never elevates and only permits/forwards confined attempts, and `--copy-as` forces ownership rather than switching identity.
|
**Wire:** two trailing config-frame blocks after the `--iconv` spec, in fixed order — `send_privilege_options`/`receive_privilege_options` (one `super_mode` int, validated `0..2`), then `send_copy_as_options`/`receive_copy_as_options` (presence int + two int32 ids, validated `>= 0`, with `copy_as_set ⇒ use_metadata`). `PROTOCOL_VERSION` bumped **2.17.0 → 2.18.0**. **Divergences from rsync:** rsync's `--super` elevates the receiver and `--copy-as` actually switches its credentials; FastSync never elevates and only permits/forwards confined attempts, and `--copy-as` forces ownership rather than switching identity.
|
||||||
|
|
||||||
**Honest status after the parity 2.28.0 cycle (protocol 2.28.0), updated by the rsync-parity-stats, rsync-parity-options, rsync-parity-fs, parity-review, no-wire parity-track-1/2b and wire parity-track-4a/5a passes.** ✅ Parity 116 / ⚠️ Caveat 14 / ❌ Divergent 27 = 157 rows. Earlier revisions of this document reported "143 ✅ / 0 divergence / 0 partial"; that conflated "parsed and tested" with "rsync parity", because many rows carried documented behavioral differences and some short options were not parsed at all. This reclassification makes every difference explicit. The completion wave closed 23 previously-caveated rows (9 that triage showed were already parity, plus 14 genuine fixes) and turned the 17 inherently non-rsync rows — native daemon config/auth, the FastSync batch container, the safe-subset device/privilege flags, `-X`'s privileged namespaces, `--fake-super`'s native xattr format, and the `--old-args` no-op — into explicit ❌ divergences. The stats pass flipped `--delete-delay` to ✅ (actual-removal accounting), but the parity-review pass moved it back to ⚠️ because FastSync charged the `--max-delete` budget at plan/snapshot time and left a refilled snapshotted directory in place, whereas rsync charges on actual removals and recursively removes a queued directory (including content created after its plan). The no-wire parity-track-1 pass fixed both (actual-removal charging plus recursive deferred removal with an independent deferred-list cap), narrowing the caveat to the partial-delete ordering. The stats pass also reclassified `--out-format` to ❌ (protocol-specific `%b`/delta-`%c`), and sharpened the `--stats`/`--progress`/`--checksum-choice` residuals. The options pass flipped `--bwlimit` and `--ignore-errors` to ✅ (rsync-exact size parsing and ~100 ms leaky-bucket throttling, and rsync's skip-unreadable-subdir plus IO-error-suppressed deletion with exit 23) and emits rsync-format `--info=name/flist/del/remove/nonreg/progress` lines (real-run `deleting`/`*deleting` carried over a new trailing `report_deletes` wire bool, `PROTOCOL_VERSION` 2.26.0 → 2.27.0), while reclassifying `-M` over daemon/TCP
|
**Honest status after the parity 2.29 cycle (protocol 2.28.0, no wire change), updated by the parity cycle 2.29 pass and the audit-cycle follow-ups.** ✅ Parity 119 / ⚠️ Caveat 11 / ❌ Divergent 27 = 157 rows. The 2.29 cycle closed the scanner-order, delete-timing, relative-basis, and fuzzy-eligibility residuals (moving `-n`/`--delete`/`--del`/`--delete-delay` to ✅) and improved the `--info`/`--stats`/`--debug` partial rows; the remaining ⚠️ rows are `--info`, `--debug`, `--msgs2stderr`, `--stats`, `--progress`, `--delete-before`, `--filter`, the three basis-dir options, and `-y/--fuzzy`. Earlier: **Honest status after the parity 2.28.0 cycle (protocol 2.28.0), updated by the rsync-parity-stats, rsync-parity-options, rsync-parity-fs, parity-review, no-wire parity-track-1/2b and wire parity-track-4a/5a passes.** ✅ Parity 116 / ⚠️ Caveat 14 / ❌ Divergent 27 = 157 rows. Earlier revisions of this document reported "143 ✅ / 0 divergence / 0 partial"; that conflated "parsed and tested" with "rsync parity", because many rows carried documented behavioral differences and some short options were not parsed at all. This reclassification makes every difference explicit. The completion wave closed 23 previously-caveated rows (9 that triage showed were already parity, plus 14 genuine fixes) and turned the 17 inherently non-rsync rows — native daemon config/auth, the FastSync batch container, the safe-subset device/privilege flags, `-X`'s privileged namespaces, `--fake-super`'s native xattr format, and the `--old-args` no-op — into explicit ❌ divergences. The stats pass flipped `--delete-delay` to ✅ (actual-removal accounting), but the parity-review pass moved it back to ⚠️ because FastSync charged the `--max-delete` budget at plan/snapshot time and left a refilled snapshotted directory in place, whereas rsync charges on actual removals and recursively removes a queued directory (including content created after its plan). The no-wire parity-track-1 pass fixed both (actual-removal charging plus recursive deferred removal with an independent deferred-list cap), narrowing the caveat to the partial-delete ordering. The stats pass also reclassified `--out-format` to ❌ (protocol-specific `%b`/delta-`%c`), and sharpened the `--stats`/`--progress`/`--checksum-choice` residuals. The options pass flipped `--bwlimit` and `--ignore-errors` to ✅ (rsync-exact size parsing and ~100 ms leaky-bucket throttling, and rsync's skip-unreadable-subdir plus IO-error-suppressed deletion with exit 23) and emits rsync-format `--info=name/flist/del/remove/nonreg/progress` lines (real-run `deleting`/`*deleting` carried over a new trailing `report_deletes` wire bool, `PROTOCOL_VERSION` 2.26.0 → 2.27.0), while reclassifying `-M` over daemon/TCP
|
||||||
and receiver-side `protect`/`risk` re-derivation to ❌ (no argv channel /
|
and receiver-side `protect`/`risk` re-derivation to ❌ (no argv channel /
|
||||||
receiver filter engine); the wire parity-track-4a pass later added that
|
receiver filter engine); the wire parity-track-4a pass later added that
|
||||||
receiver filter engine, flipping `--filter=RULE` back to ✅ (see above). The fs pass flips `-d/--dirs` and `--iconv` to ✅ — recursive transfers now recreate empty source directories (and replace a blocking destination non-directory with an incoming directory); `-R --no-implied-dirs --files-from` places a listed file under a missing implied parent with default attributes instead of refusing; and `--iconv` now reproduces rsync's push direction (destination charset = the spec's REMOTE half) — and reclassified six rows to ❌ after reproducing their exact residual with differential tests: `--temp-dir` (the receiver confines the scratch dir to the receive root, so an absolute temp dir is deliberately rejected although standalone rsync follows it), the three basis-dir options (FastSync xxHash-verifies a basis hit while rsync's `--size-only` quick check installs the wrong basis content), `--delay-updates` (fixed staging name wipes an unrelated destination entry of that name), and `--dry-run` (would-delete report over-reports). `--fuzzy` was also reclassified to ❌ (deterministic heuristic with a 10× size window, not rsync's matcher), but its residual is the candidate-selection heuristic itself: the final tree is byte-exact by design, so no destination differential can expose it and the row is pinned by the `TestFuzzy` threshold suite rather than a byte-level rsync differential. (Track 5b later showed the name heuristic is in fact rsync's own and moved the row ❌ → ⚠️, leaving only the narrower delta size window as the residual; see the track 5b paragraph above.) The remaining ⚠️ rows are the ones with a documented residual (see the row notes and the **Parity Completion Wave (protocol 2.26.0)** section below).
|
receiver filter engine, flipping `--filter=RULE` back to ✅ (see above; the
|
||||||
|
audit-cycle follow-ups later moved it to ⚠️ for the accepted-but-ignored merge
|
||||||
|
modifiers, see the audit-cycle note). The fs pass flips `-d/--dirs` and `--iconv` to ✅ — recursive transfers now recreate empty source directories (and replace a blocking destination non-directory with an incoming directory); `-R --no-implied-dirs --files-from` places a listed file under a missing implied parent with default attributes instead of refusing; and `--iconv` now reproduces rsync's push direction (destination charset = the spec's REMOTE half) — and reclassified six rows to ❌ after reproducing their exact residual with differential tests: `--temp-dir` (the receiver confines the scratch dir to the receive root, so an absolute temp dir is deliberately rejected although standalone rsync follows it), the three basis-dir options (FastSync xxHash-verifies a basis hit while rsync's `--size-only` quick check installs the wrong basis content), `--delay-updates` (fixed staging name wipes an unrelated destination entry of that name), and `--dry-run` (would-delete report over-reports). `--fuzzy` was also reclassified to ❌ (deterministic heuristic with a 10× size window, not rsync's matcher), but its residual is the candidate-selection heuristic itself: the final tree is byte-exact by design, so no destination differential can expose it and the row is pinned by the `TestFuzzy` threshold suite rather than a byte-level rsync differential. (Track 5b later showed the name heuristic is in fact rsync's own and moved the row ❌ → ⚠️, leaving only the narrower delta size window as the residual; see the track 5b paragraph above.) The remaining ⚠️ rows are the ones with a documented residual (see the row notes and the **Parity Completion Wave (protocol 2.26.0)** section below).
|
||||||
|
|
||||||
**Preserve-attribute split (protocol 2.21.0 → 2.22.0) — ✅ implemented.** FastSync splits the former single metadata bundle into four independent, rsync-compatible per-attribute flags — `-p/--perms`, `-t/--times`, `-o/--owner`, `-g/--group` — each with a negation (`--no-perms`/`--no-times`/`--no-owner`/`--no-group`, short `--no-p`/`--no-t`/`--no-o`/`--no-g`), plus `--no-preserve` clearing all four. `-a/--archive` is now full rsync `-rlptgoD` (owner and group included, though their application stays privilege-gated), `-A/--acls` implies `-p`, `-X/--xattrs` does not, `-E/--executability` sets only executability, and `-U`/`-N` do not imply `-t`. `--incremental`/`--delta` still auto-preserve perms+times unless the user explicitly negated them. Wire: the binary config frame gains four appended booleans (`preserve_perms`/`preserve_times`/`preserve_owner`/`preserve_group`) after `omit_link_times`, so `PROTOCOL_VERSION` is bumped **2.21.0 → 2.22.0**; the fixed-width `FileMetadata` layout is unchanged and the receiver gates the metadata frame on a derived `use_metadata`. Receiver behavior: each attribute is applied independently, directory modes are applied under `-p` (at the end of the transfer, alongside dir times), symlink mode under `-p`, and `-O/--omit-dir-times` suppresses directory times only. Documented divergences as of 2.22.0, **all but (d)/(e) removed by the rsync-parity wave (protocol 2.23.0)**: (a) the mode-masking divergence is **gone** — under `-p` the source mode is now copied exactly, including `S_IWGRP`/`S_IWOTH` and setuid/setgid/sticky; (b) a brand-new file without `-p` still gets `source_mode & ~umask` when metadata is present (else the historical fixed `0644`), and a new *directory* without `-p` still uses FastSync's `0755` default; (c) the `--chmod`-implies-`-p` divergence is **gone** — `--chmod` no longer implies `-p` (rsync parity); (d) `-o`/`-g` map by name on the receiver with a raw-numeric fallback (only numeric ids cross the wire); (e) a daemon module without `client owner = yes` does not refuse a plain `-a`/`-o`/`-g` — it forces super off, applies no ownership, and logs a warning, while explicit `--chown`/`--usermap`/`--groupmap`/`--numeric-ids`/`--copy-as`/`--super` are still refused.
|
**Preserve-attribute split (protocol 2.21.0 → 2.22.0) — ✅ implemented.** FastSync splits the former single metadata bundle into four independent, rsync-compatible per-attribute flags — `-p/--perms`, `-t/--times`, `-o/--owner`, `-g/--group` — each with a negation (`--no-perms`/`--no-times`/`--no-owner`/`--no-group`, short `--no-p`/`--no-t`/`--no-o`/`--no-g`), plus `--no-preserve` clearing all four. `-a/--archive` is now full rsync `-rlptgoD` (owner and group included, though their application stays privilege-gated), `-A/--acls` implies `-p`, `-X/--xattrs` does not, `-E/--executability` sets only executability, and `-U`/`-N` do not imply `-t`. `--incremental`/`--delta` still auto-preserve perms+times unless the user explicitly negated them. Wire: the binary config frame gains four appended booleans (`preserve_perms`/`preserve_times`/`preserve_owner`/`preserve_group`) after `omit_link_times`, so `PROTOCOL_VERSION` is bumped **2.21.0 → 2.22.0**; the fixed-width `FileMetadata` layout is unchanged and the receiver gates the metadata frame on a derived `use_metadata`. Receiver behavior: each attribute is applied independently, directory modes are applied under `-p` (at the end of the transfer, alongside dir times), symlink mode under `-p`, and `-O/--omit-dir-times` suppresses directory times only. Documented divergences as of 2.22.0, **all but (d)/(e) removed by the rsync-parity wave (protocol 2.23.0)**: (a) the mode-masking divergence is **gone** — under `-p` the source mode is now copied exactly, including `S_IWGRP`/`S_IWOTH` and setuid/setgid/sticky; (b) a brand-new file without `-p` still gets `source_mode & ~umask` when metadata is present (else the historical fixed `0644`), and a new *directory* without `-p` still uses FastSync's `0755` default; (c) the `--chmod`-implies-`-p` divergence is **gone** — `--chmod` no longer implies `-p` (rsync parity); (d) `-o`/`-g` map by name on the receiver with a raw-numeric fallback (only numeric ids cross the wire); (e) a daemon module without `client owner = yes` does not refuse a plain `-a`/`-o`/`-g` — it forces super off, applies no ownership, and logs a warning, while explicit `--chown`/`--usermap`/`--groupmap`/`--numeric-ids`/`--copy-as`/`--super` are still refused.
|
||||||
|
|
||||||
@@ -1066,7 +1089,9 @@ These remain after the wave; they are the reasons a row above is ⚠️.
|
|||||||
count) are reported as 0; `--progress` is an aggregate line, not per-file.
|
count) are reported as 0; `--progress` is an aggregate line, not per-file.
|
||||||
- **`--password-file`/`--early-input`/`--hash-credentials`/`--iterations` are
|
- **`--password-file`/`--early-input`/`--hash-credentials`/`--iterations` are
|
||||||
FastSync-native** (SCRAM/PBKDF2), not rsync semantics; the batch format is not
|
FastSync-native** (SCRAM/PBKDF2), not rsync semantics; the batch format is not
|
||||||
rsync-interoperable.
|
rsync-interoperable. Credential files are opened with `O_NOFOLLOW` (a symlinked
|
||||||
|
path fails closed; fd-backed process-substitution paths are exempt) and a FIFO
|
||||||
|
read is bound-waited (~3 s).
|
||||||
- **xattr/ACL namespace policy** permits only `user.*` and
|
- **xattr/ACL namespace policy** permits only `user.*` and
|
||||||
`system.posix_acl_*` when `-A` is negotiated (stricter than rsync).
|
`system.posix_acl_*` when `-A` is negotiated (stricter than rsync).
|
||||||
- **`--stop-at` remains a FastSync-flexible parser** (client-only, not
|
- **`--stop-at` remains a FastSync-flexible parser** (client-only, not
|
||||||
@@ -1146,7 +1171,11 @@ wire protocol three times (full rationale in `src/shared/config.h`):
|
|||||||
- **Filter grammar:** `merge`/`.`, `dir-merge`/`:`, `hide`/`H`, `show`/`S`,
|
- **Filter grammar:** `merge`/`.`, `dir-merge`/`:`, `hide`/`H`, `show`/`S`,
|
||||||
`protect`/`P`, `risk`/`R`, `clear`/`!`, include/exclude and the `:`/`.`
|
`protect`/`P`, `risk`/`R`, `clear`/`!`, include/exclude and the `:`/`.`
|
||||||
modifiers; `-f` is bound to `--filter`; a single `-F` transfers
|
modifiers; `-f` is bound to `--filter`; a single `-F` transfers
|
||||||
`.rsync-filter` and `-FF` excludes it.
|
`.rsync-filter` and `-FF` excludes it. The xattr-name `x` modifier is **not
|
||||||
|
implemented** and is rejected with a clear error everywhere. The merge-only
|
||||||
|
`e`/`n`/`w` and `-` modifiers are accepted and consumed on `merge`/`dir-merge`
|
||||||
|
rules (rejected elsewhere, matching rsync), but their semantics are **not
|
||||||
|
implemented** (accepted-but-ignored).
|
||||||
- **Absolute basis directories** are used verbatim (rsync semantics) and
|
- **Absolute basis directories** are used verbatim (rsync semantics) and
|
||||||
**`--link-dest`** relinks an already up-to-date destination.
|
**`--link-dest`** relinks an already up-to-date destination.
|
||||||
|
|
||||||
|
|||||||
+63
-14
@@ -54,13 +54,26 @@ bool client_abort_pending(void) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#ifndef FASTSYNC_TEST_BUILD
|
#ifndef FASTSYNC_TEST_BUILD
|
||||||
|
/* SIG_DFL disposition used by the handler's "not armed" fallback. It is built
|
||||||
|
* once at load time so the handler can restore the default action with
|
||||||
|
* sigaction(2) -- which is async-signal-safe -- instead of signal(3), which is
|
||||||
|
* not. The zero-initialized sa_mask is the empty set. */
|
||||||
|
static const struct sigaction client_default_action = {
|
||||||
|
.sa_handler = SIG_DFL,
|
||||||
|
.sa_flags = 0,
|
||||||
|
};
|
||||||
|
|
||||||
/* Signal handler: perform NO work beyond storing the flag. Logging, protocol
|
/* Signal handler: perform NO work beyond storing the flag. Logging, protocol
|
||||||
* I/O and the STATUS_ABORT frame are all done later on the normal send path,
|
* I/O and the STATUS_ABORT frame are all done later on the normal send path,
|
||||||
* which is not async-signal-safe. When no transfer is armed, fall back to the
|
* which is not async-signal-safe. When no transfer is armed, restore the
|
||||||
* default action so local-only modes remain interruptible. */
|
* default disposition (async-signal-safe sigaction) and re-raise so local-only
|
||||||
|
* modes remain interruptible. The handler deliberately stays installed while a
|
||||||
|
* transfer is armed -- rather than using SA_RESETHAND -- so a second Ctrl-C
|
||||||
|
* during the graceful abort keeps setting the flag instead of hard-killing the
|
||||||
|
* process mid-cleanup. */
|
||||||
static void client_signal_handler(int signo) {
|
static void client_signal_handler(int signo) {
|
||||||
if (!client_abort_armed) {
|
if (!client_abort_armed) {
|
||||||
signal(signo, SIG_DFL);
|
sigaction(signo, &client_default_action, NULL);
|
||||||
raise(signo);
|
raise(signo);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -504,9 +517,8 @@ static bool split_flag_level(const char* token, char* name, size_t name_size, in
|
|||||||
* of rsync's `symsafe`, `hlink`, and `own`. */
|
* of rsync's `symsafe`, `hlink`, and `own`. */
|
||||||
static bool is_accepted_debug_category(const char* name) {
|
static bool is_accepted_debug_category(const char* name) {
|
||||||
static const char* const categories[] = {
|
static const char* const categories[] = {
|
||||||
"acl", "backup", "bind", "chdir", "cmd", "connect", "del", "deltasum",
|
"acl", "backup", "bind", "chdir", "cmd", "connect", "dup", "exit", "fuzzy",
|
||||||
"dup", "exit", "filter", "flist", "fuzzy", "genr", "hash", "hl",
|
"genr", "hl", "hlink", "iconv", "nstr", "own", "owner", "time",
|
||||||
"hlink", "iconv", "nstr", "own", "owner", "recv", "send", "time",
|
|
||||||
};
|
};
|
||||||
for (size_t i = 0; i < sizeof(categories) / sizeof(categories[0]); i++) {
|
for (size_t i = 0; i < sizeof(categories) / sizeof(categories[0]); i++) {
|
||||||
if (strcmp(name, categories[i]) == 0)
|
if (strcmp(name, categories[i]) == 0)
|
||||||
@@ -518,7 +530,6 @@ static bool is_accepted_debug_category(const char* name) {
|
|||||||
static bool is_accepted_info_category(const char* name) {
|
static bool is_accepted_info_category(const char* name) {
|
||||||
static const char* const categories[] = {
|
static const char* const categories[] = {
|
||||||
"backup",
|
"backup",
|
||||||
"mount",
|
|
||||||
"syms",
|
"syms",
|
||||||
"symsafe",
|
"symsafe",
|
||||||
};
|
};
|
||||||
@@ -571,6 +582,18 @@ static int parse_debug_flags(const char* value, Config* config) {
|
|||||||
flag = LOG_DEBUG_PACK;
|
flag = LOG_DEBUG_PACK;
|
||||||
} else if (strcmp(name, "util") == 0) {
|
} else if (strcmp(name, "util") == 0) {
|
||||||
flag = LOG_DEBUG_UTIL;
|
flag = LOG_DEBUG_UTIL;
|
||||||
|
} else if (strcmp(name, "flist") == 0) {
|
||||||
|
flag = LOG_DEBUG_FLIST;
|
||||||
|
} else if (strcmp(name, "del") == 0) {
|
||||||
|
flag = LOG_DEBUG_DEL;
|
||||||
|
} else if (strcmp(name, "hash") == 0 || strcmp(name, "deltasum") == 0) {
|
||||||
|
flag = LOG_DEBUG_HASH;
|
||||||
|
} else if (strcmp(name, "recv") == 0) {
|
||||||
|
flag = LOG_DEBUG_RECV;
|
||||||
|
} else if (strcmp(name, "filter") == 0) {
|
||||||
|
flag = LOG_DEBUG_FILTER;
|
||||||
|
} else if (strcmp(name, "send") == 0) {
|
||||||
|
flag = LOG_DEBUG_SEND;
|
||||||
} else if (is_accepted_debug_category(name)) {
|
} else if (is_accepted_debug_category(name)) {
|
||||||
continue;
|
continue;
|
||||||
} else {
|
} else {
|
||||||
@@ -645,9 +668,12 @@ static int parse_info_flags(const char* value, Config* config) {
|
|||||||
flag = LOG_INFO_MISC;
|
flag = LOG_INFO_MISC;
|
||||||
else if (strcmp(name, "skip") == 0)
|
else if (strcmp(name, "skip") == 0)
|
||||||
flag = LOG_INFO_SKIP;
|
flag = LOG_INFO_SKIP;
|
||||||
else if (strcmp(name, "stats") == 0)
|
else if (strcmp(name, "stats") == 0) {
|
||||||
flag = LOG_INFO_STATS;
|
flag = LOG_INFO_STATS;
|
||||||
else if (strcmp(name, "del") == 0)
|
/* `--info=stats` requests the same transfer-statistics block as
|
||||||
|
`--stats`; `--info=stats0` turns it back off. */
|
||||||
|
config->stats = level > 0;
|
||||||
|
} else if (strcmp(name, "del") == 0)
|
||||||
flag = LOG_INFO_DEL;
|
flag = LOG_INFO_DEL;
|
||||||
else if (strcmp(name, "remove") == 0)
|
else if (strcmp(name, "remove") == 0)
|
||||||
flag = LOG_INFO_REMOVE;
|
flag = LOG_INFO_REMOVE;
|
||||||
@@ -655,6 +681,8 @@ static int parse_info_flags(const char* value, Config* config) {
|
|||||||
flag = LOG_INFO_FLIST;
|
flag = LOG_INFO_FLIST;
|
||||||
else if (strcmp(name, "nonreg") == 0)
|
else if (strcmp(name, "nonreg") == 0)
|
||||||
flag = LOG_INFO_NONREG;
|
flag = LOG_INFO_NONREG;
|
||||||
|
else if (strcmp(name, "mount") == 0)
|
||||||
|
flag = LOG_INFO_MOUNT;
|
||||||
else if (strcmp(name, "progress") == 0)
|
else if (strcmp(name, "progress") == 0)
|
||||||
flag = LOG_INFO_PROGRESS;
|
flag = LOG_INFO_PROGRESS;
|
||||||
else if (is_accepted_info_category(name))
|
else if (is_accepted_info_category(name))
|
||||||
@@ -1213,6 +1241,12 @@ static int apply_table_option(Config* config, const OptionEntry* entry, const ch
|
|||||||
void* field = (char*)config + entry->offset;
|
void* field = (char*)config + entry->offset;
|
||||||
switch (entry->kind) {
|
switch (entry->kind) {
|
||||||
case OPT_FLAG:
|
case OPT_FLAG:
|
||||||
|
/* -x/--one-file-system is repeatable in rsync: `-xx` increments the level so
|
||||||
|
the scanner drops mount-point directories instead of recreating them
|
||||||
|
empty. Everything else is a plain boolean. */
|
||||||
|
if (entry->offset == offsetof(Config, one_file_system))
|
||||||
|
(*(int*)field)++;
|
||||||
|
else
|
||||||
*(bool*)field = true;
|
*(bool*)field = true;
|
||||||
return 0;
|
return 0;
|
||||||
case OPT_NOOP:
|
case OPT_NOOP:
|
||||||
@@ -2574,7 +2608,11 @@ static bool cli_handle_outbuf_option(CliParseCtx* ctx) {
|
|||||||
* load --files-from once every argument has been seen. Returns 0 on success,
|
* load --files-from once every argument has been seen. Returns 0 on success,
|
||||||
* -1 on error. */
|
* -1 on error. */
|
||||||
static int cli_finalize_config(Config* config, bool verbose, bool no_delta, bool no_incremental) {
|
static int cli_finalize_config(Config* config, bool verbose, bool no_delta, bool no_incremental) {
|
||||||
set_log_level(config->quiet ? LOG_LEVEL_ERROR : (verbose ? LOG_LEVEL_DEBUG : LOG_LEVEL_WARNING));
|
/* An explicit --debug=FLAGS enables the debug log level by itself (rsync
|
||||||
|
behaviour); -v enables every other INFO-level message. */
|
||||||
|
bool debug_enabled = verbose || config->debug_level != 0;
|
||||||
|
set_log_level(config->quiet ? LOG_LEVEL_ERROR
|
||||||
|
: (debug_enabled ? LOG_LEVEL_DEBUG : LOG_LEVEL_WARNING));
|
||||||
/* rsync's plain --delete defaults to delete-during (--del): each directory's
|
/* rsync's plain --delete defaults to delete-during (--del): each directory's
|
||||||
extras are removed as that directory is processed, so space is freed
|
extras are removed as that directory is processed, so space is freed
|
||||||
progressively and a tight destination never has to hold the whole old+new
|
progressively and a tight destination never has to hold the whole old+new
|
||||||
@@ -2587,6 +2625,15 @@ static int cli_finalize_config(Config* config, bool verbose, bool no_delta, bool
|
|||||||
if (config->use_delete && !config->delete_before && !config->delete_during &&
|
if (config->use_delete && !config->delete_before && !config->delete_during &&
|
||||||
!config->delete_delay && !config->delete_after)
|
!config->delete_delay && !config->delete_after)
|
||||||
config->delete_during = true;
|
config->delete_during = true;
|
||||||
|
/* rsync parity: --partial-dir=DIR chooses where an interrupted transfer's
|
||||||
|
partial file is kept, so it implies --partial. rsync applies the
|
||||||
|
implication after option parsing, so it wins over an explicit --no-partial
|
||||||
|
regardless of the order the two options appear in (verified on rsync
|
||||||
|
3.4.1). --inplace is the exception: the destination file is written in
|
||||||
|
place with no partial/temp staging, so the partial machinery is bypassed
|
||||||
|
and the implication is skipped to leave --inplace behavior untouched. */
|
||||||
|
if (config->partial_dir && !config->inplace)
|
||||||
|
config->partial = true;
|
||||||
if (config->compress_choice) {
|
if (config->compress_choice) {
|
||||||
int algo = compression_algo_from_name(config->compress_choice);
|
int algo = compression_algo_from_name(config->compress_choice);
|
||||||
if (algo >= 0) {
|
if (algo >= 0) {
|
||||||
@@ -2764,10 +2811,12 @@ static int cli_finalize_config(Config* config, bool verbose, bool no_delta, bool
|
|||||||
}
|
}
|
||||||
/* --info=del on a real --delete run asks the receiver to report the paths it
|
/* --info=del on a real --delete run asks the receiver to report the paths it
|
||||||
actually removed; the report rides the STATUS_STATS path list, so the wire
|
actually removed; the report rides the STATUS_STATS path list, so the wire
|
||||||
stats frame must be negotiated too. */
|
stats frame must be negotiated too. --debug=del needs the same paths, so
|
||||||
config->report_deletes = config->use_delete && !config->dry_run &&
|
it opts into the existing report (no new wire field). */
|
||||||
|
config->report_deletes =
|
||||||
|
config->use_delete && !config->dry_run &&
|
||||||
((config->info_level & LOG_INFO_DEL) != 0 || config->itemize_changes ||
|
((config->info_level & LOG_INFO_DEL) != 0 || config->itemize_changes ||
|
||||||
config->out_format != NULL);
|
config->out_format != NULL || (config->debug_level & LOG_DEBUG_DEL) != 0);
|
||||||
config->report_stats = config->stats || config->show_progress ||
|
config->report_stats = config->stats || config->show_progress ||
|
||||||
(config->info_level & LOG_INFO_PROGRESS) || format_needs_wire ||
|
(config->info_level & LOG_INFO_PROGRESS) || format_needs_wire ||
|
||||||
config->report_deletes || (config->dry_run && config->use_delete);
|
config->report_deletes || (config->dry_run && config->use_delete);
|
||||||
@@ -3247,7 +3296,7 @@ int main(int argc, char* argv[]) {
|
|||||||
exit_code = 1;
|
exit_code = 1;
|
||||||
}
|
}
|
||||||
} else if (config->use_multithreading) {
|
} else if (config->use_multithreading) {
|
||||||
exit_code = send_files_multithreaded(&config);
|
exit_code = send_files_multithreaded(config);
|
||||||
} else {
|
} else {
|
||||||
exit_code = send_files(config);
|
exit_code = send_files(config);
|
||||||
}
|
}
|
||||||
|
|||||||
+76
-76
@@ -37,8 +37,6 @@
|
|||||||
#include <sys/stat.h>
|
#include <sys/stat.h>
|
||||||
#include <unistd.h>
|
#include <unistd.h>
|
||||||
|
|
||||||
#define STREAM_THRESHOLD (64ULL * 1024 * 1024)
|
|
||||||
|
|
||||||
/* Aggregate loaded payload bytes the sender may buffer across the loader queue
|
/* Aggregate loaded payload bytes the sender may buffer across the loader queue
|
||||||
and the chunk in flight. Sending one chunk adds up to ~2 * MAX_CHUNK_SIZE of
|
and the chunk in flight. Sending one chunk adds up to ~2 * MAX_CHUNK_SIZE of
|
||||||
transient serialize/compress buffers on top of the queued payloads, so this
|
transient serialize/compress buffers on top of the queued payloads, so this
|
||||||
@@ -129,6 +127,21 @@ static void stats_type_breakdown(const TransferStats* stats, char* out, size_t o
|
|||||||
out_size);
|
out_size);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* rsync's `Number of files` counts every directory. A recursive scan that
|
||||||
|
preserves a directory attribute captures them in `dir_entries`; a `-r` scan
|
||||||
|
(no -t/-p) captures nothing, so fall back to the scanner's shared counter of
|
||||||
|
traversed directories that are not already represented by an inline
|
||||||
|
directory entry. The -d generator counts its explicit directory entries
|
||||||
|
inline and does not traverse, so it is excluded here. */
|
||||||
|
static unsigned long long dir_count_for_stats(const Config* config, const ArrayList* dir_entries,
|
||||||
|
atomic_ullong* counter) {
|
||||||
|
if (config == NULL || config->dirs || config->list_only)
|
||||||
|
return 0;
|
||||||
|
if (dir_metadata_should_capture(config))
|
||||||
|
return dir_entries != NULL ? (unsigned long long)dir_entries->size : 0;
|
||||||
|
return counter != NULL ? (unsigned long long)atomic_load(counter) : 0;
|
||||||
|
}
|
||||||
|
|
||||||
/* Print the rsync `--stats` block on stdout. The source-side flist and
|
/* Print the rsync `--stats` block on stdout. The source-side flist and
|
||||||
transferred counters come from `stats` (filled while scanning/sending), the
|
transferred counters come from `stats` (filled while scanning/sending), the
|
||||||
receiver-only counters from the STATUS_STATS frame, and the wire byte totals
|
receiver-only counters from the STATUS_STATS frame, and the wire byte totals
|
||||||
@@ -387,6 +400,15 @@ static bool info_flag_enabled(const Config* config, LogInfoFlag flag) {
|
|||||||
static void print_delete_reports(const Config* config, const ArrayList* paths) {
|
static void print_delete_reports(const Config* config, const ArrayList* paths) {
|
||||||
if (!config || !paths || config->quiet)
|
if (!config || !paths || config->quiet)
|
||||||
return;
|
return;
|
||||||
|
/* --debug=del is independent of the --info=del/itemize/out-format display:
|
||||||
|
emit the debug trace even when no deletion line would be printed. */
|
||||||
|
if (log_debug_enabled(LOG_DEBUG_DEL)) {
|
||||||
|
for (int i = 0; i < paths->size; i++) {
|
||||||
|
const char* raw = (const char*)paths->items[i];
|
||||||
|
const char* path = delete_display_path(config, raw);
|
||||||
|
log_debug_message(LOG_DEBUG_DEL, "del: %s", path ? path : raw);
|
||||||
|
}
|
||||||
|
}
|
||||||
if (!(config->itemize_changes || config->out_format != NULL ||
|
if (!(config->itemize_changes || config->out_format != NULL ||
|
||||||
info_flag_enabled(config, LOG_INFO_DEL)))
|
info_flag_enabled(config, LOG_INFO_DEL)))
|
||||||
return;
|
return;
|
||||||
@@ -665,6 +687,7 @@ static bool prepare_scanner(const Config* config, int num_threads, PreparedScann
|
|||||||
options->ignore_io_errors = config->ignore_errors;
|
options->ignore_io_errors = config->ignore_errors;
|
||||||
options->ignore_missing_args = config->ignore_missing_args || config->delete_missing_args;
|
options->ignore_missing_args = config->ignore_missing_args || config->delete_missing_args;
|
||||||
options->note_nonreg = (config->info_level & LOG_INFO_NONREG) != 0 && !config->quiet;
|
options->note_nonreg = (config->info_level & LOG_INFO_NONREG) != 0 && !config->quiet;
|
||||||
|
options->note_mount = (config->info_level & LOG_INFO_MOUNT) != 0 && !config->quiet;
|
||||||
options->send_directory = config->send_directory;
|
options->send_directory = config->send_directory;
|
||||||
options->eight_bit_output = config->eight_bit_output;
|
options->eight_bit_output = config->eight_bit_output;
|
||||||
options->excluded_paths = NULL;
|
options->excluded_paths = NULL;
|
||||||
@@ -672,6 +695,9 @@ static bool prepare_scanner(const Config* config, int num_threads, PreparedScann
|
|||||||
options->size_skipped_paths = NULL;
|
options->size_skipped_paths = NULL;
|
||||||
options->synced_dirs = NULL;
|
options->synced_dirs = NULL;
|
||||||
options->hardlinks = NULL;
|
options->hardlinks = NULL;
|
||||||
|
/* Set by the real send paths; NULL for the metadata-only scans (progress
|
||||||
|
pre-count, batch) that must not perturb the sender's --stats counter. */
|
||||||
|
options->dir_count = NULL;
|
||||||
/* P7 Wave D: capture source directory metadata when a directory attribute is
|
/* P7 Wave D: capture source directory metadata when a directory attribute is
|
||||||
requested (-p for modes, -t for times unless -O omits them). Whether they
|
requested (-p for modes, -t for times unless -O omits them). Whether they
|
||||||
are APPLIED is decided receiver-side. */
|
are APPLIED is decided receiver-side. */
|
||||||
@@ -730,6 +756,8 @@ static bool progress_precount_scan(const Config* config, ProgressPrecount* out)
|
|||||||
ScannerOptions local = prepared.options;
|
ScannerOptions local = prepared.options;
|
||||||
local.list_dirs = true;
|
local.list_dirs = true;
|
||||||
local.note_nonreg = false;
|
local.note_nonreg = false;
|
||||||
|
local.note_mount = false;
|
||||||
|
local.dir_count = NULL;
|
||||||
local.use_metadata = false;
|
local.use_metadata = false;
|
||||||
local.preserve_xattrs = false;
|
local.preserve_xattrs = false;
|
||||||
local.preserve_acls = false;
|
local.preserve_acls = false;
|
||||||
@@ -1085,7 +1113,7 @@ static Client* connect_transfer_client(const Config* config) {
|
|||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
return client_connect_ssh(config->ssh_destination, config->ssh_port,
|
return client_connect_ssh(config->ssh_destination, config->ssh_port,
|
||||||
config->fastsync_server_path, config->old_args, config->rsh_command,
|
config->fastsync_server_path, config->rsh_command,
|
||||||
config->blocking_io, config->remote_options,
|
config->blocking_io, config->remote_options,
|
||||||
config->remote_option_count);
|
config->remote_option_count);
|
||||||
}
|
}
|
||||||
@@ -1336,6 +1364,7 @@ static bool finalize_transfer(Client* client, const Config* config, ArrayList* r
|
|||||||
return false;
|
return false;
|
||||||
if (status == STATUS_STATS) {
|
if (status == STATUS_STATS) {
|
||||||
ReceiverStats scratch;
|
ReceiverStats scratch;
|
||||||
|
log_debug_message(LOG_DEBUG_RECV, "recv: receiver stats");
|
||||||
/* A real --info=del run carries the actually-removed paths in the stats
|
/* A real --info=del run carries the actually-removed paths in the stats
|
||||||
frame's path list; collect and print them in rsync's format. */
|
frame's path list; collect and print them in rsync's format. */
|
||||||
ArrayList* deleted = config->report_deletes ? array_list_create(free) : NULL;
|
ArrayList* deleted = config->report_deletes ? array_list_create(free) : NULL;
|
||||||
@@ -1855,25 +1884,6 @@ static bool scan_paths_only(const Config* config, const ScannerOptions* options,
|
|||||||
return ok;
|
return ok;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Transmit any not-yet-sent per-directory delete plan needed by the entries in
|
|
||||||
* `chunk` (ancestors root-first, then the entry's own directory for --dirs
|
|
||||||
* entries) before its data frames go out, so --delete-during/--delete-delay
|
|
||||||
* clear a directory's extras (and any type conflict) before the directory's
|
|
||||||
* first write. */
|
|
||||||
static int send_chunk_delete_plans(Client* client, DeletePlanSender* plans, const Chunk* chunk) {
|
|
||||||
if (!plans)
|
|
||||||
return 0;
|
|
||||||
for (int i = 0; i < chunk->element_count; i++) {
|
|
||||||
File* f = chunk->items[i];
|
|
||||||
if (!f)
|
|
||||||
continue;
|
|
||||||
if (delete_plan_send_for_path(client->file_descriptor, plans, file_wire_path(f), f->is_dir) !=
|
|
||||||
0)
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
static int incremental_check(Client* client, File* file, const Config* config,
|
static int incremental_check(Client* client, File* file, const Config* config,
|
||||||
DeltaSignature** out_sig, unsigned long long* resume_offset) {
|
DeltaSignature** out_sig, unsigned long long* resume_offset) {
|
||||||
*out_sig = NULL;
|
*out_sig = NULL;
|
||||||
@@ -1904,6 +1914,8 @@ static int incremental_check(Client* client, File* file, const Config* config,
|
|||||||
if (!file_checksum(file, (ChecksumAlgo)config->checksum_algo, config->checksum_seed, digest,
|
if (!file_checksum(file, (ChecksumAlgo)config->checksum_algo, config->checksum_seed, digest,
|
||||||
sizeof(digest), &digest_len))
|
sizeof(digest), &digest_len))
|
||||||
return -1;
|
return -1;
|
||||||
|
log_debug_message(LOG_DEBUG_HASH, "hash: %s (algo %d)", file_wire_path(file),
|
||||||
|
config->checksum_algo);
|
||||||
uint8_t wire_len = (uint8_t)digest_len;
|
uint8_t wire_len = (uint8_t)digest_len;
|
||||||
if (!send_n_data(client->file_descriptor, &wire_len, sizeof(wire_len)) ||
|
if (!send_n_data(client->file_descriptor, &wire_len, sizeof(wire_len)) ||
|
||||||
!send_n_data(client->file_descriptor, digest, wire_len))
|
!send_n_data(client->file_descriptor, digest, wire_len))
|
||||||
@@ -1938,6 +1950,7 @@ static int incremental_check(Client* client, File* file, const Config* config,
|
|||||||
log_server_rejection("Server reported error for file");
|
log_server_rejection("Server reported error for file");
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
log_debug_message(LOG_DEBUG_RECV, "recv: check reply for %s", file_wire_path(file));
|
||||||
if (s == STATUS_OK)
|
if (s == STATUS_OK)
|
||||||
return 1;
|
return 1;
|
||||||
if (s == STATUS_DELTA_SIGNATURE) {
|
if (s == STATUS_DELTA_SIGNATURE) {
|
||||||
@@ -1952,6 +1965,8 @@ static int incremental_check(Client* client, File* file, const Config* config,
|
|||||||
send_status(client->file_descriptor, STATUS_ERROR);
|
send_status(client->file_descriptor, STATUS_ERROR);
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
log_debug_message(LOG_DEBUG_RECV, "recv: delta signature for %s (%u blocks)",
|
||||||
|
file_wire_path(file), sig->block_count);
|
||||||
*out_sig = sig;
|
*out_sig = sig;
|
||||||
return 2;
|
return 2;
|
||||||
}
|
}
|
||||||
@@ -1992,6 +2007,7 @@ static int incremental_check(Client* client, File* file, const Config* config,
|
|||||||
static int send_delta(Client* client, File* file, DeltaSignature* sig, Config* config) {
|
static int send_delta(Client* client, File* file, DeltaSignature* sig, Config* config) {
|
||||||
Delta* delta = delta_compute_seeded(file->data->data, file->data->size, sig,
|
Delta* delta = delta_compute_seeded(file->data->data, file->data->size, sig,
|
||||||
config->delta_block_size, (uint32_t)config->checksum_seed);
|
config->delta_block_size, (uint32_t)config->checksum_seed);
|
||||||
|
log_debug_message(LOG_DEBUG_HASH, "deltasum: %s", file_wire_path(file));
|
||||||
/* The receiver is blocked after sending the signature. Every local
|
/* The receiver is blocked after sending the signature. Every local
|
||||||
fallback therefore needs the explicit NEXT response before full data. */
|
fallback therefore needs the explicit NEXT response before full data. */
|
||||||
if (!delta)
|
if (!delta)
|
||||||
@@ -2465,6 +2481,7 @@ static int send_single_file(Client* client, File* file, Config* config, bool use
|
|||||||
bool use_sendfile) {
|
bool use_sendfile) {
|
||||||
int compression_level = config->use_compression ? config->compression_level : 0;
|
int compression_level = config->use_compression ? config->compression_level : 0;
|
||||||
log_info_message(LOG_INFO_COPY, "Transferring %s", file->path);
|
log_info_message(LOG_INFO_COPY, "Transferring %s", file->path);
|
||||||
|
log_debug_message(LOG_DEBUG_SEND, "send: %s", file_wire_path(file));
|
||||||
|
|
||||||
if (!use_incremental) {
|
if (!use_incremental) {
|
||||||
if (use_sendfile) {
|
if (use_sendfile) {
|
||||||
@@ -2753,9 +2770,12 @@ static int send_chunks_multithreaded(void* pipeline_context) {
|
|||||||
return thrd_error;
|
return thrd_error;
|
||||||
}
|
}
|
||||||
} else if (context->delete_plans) {
|
} else if (context->delete_plans) {
|
||||||
/* --delete-during/--delete-delay: transmit the receive root's plan before
|
/* --delete-during/--delete-delay: transmit the COMPLETE per-directory plan
|
||||||
any data, exactly like rsync's first generator directory. */
|
set before any data, so a mid-transfer abort has already applied every
|
||||||
if (delete_plan_send_root(client->file_descriptor, context->delete_plans) != 0) {
|
planned removal exactly like rsync's generator (which runs ahead of its
|
||||||
|
throttled sender). A completed run is unaffected. */
|
||||||
|
if (delete_plan_send_all(client->file_descriptor, context->delete_plans, context->plan_dirs) !=
|
||||||
|
0) {
|
||||||
pipeline_cancel(context);
|
pipeline_cancel(context);
|
||||||
disconnect_transfer_client(client);
|
disconnect_transfer_client(client);
|
||||||
mark_sender_done(context);
|
mark_sender_done(context);
|
||||||
@@ -2802,15 +2822,6 @@ static int send_chunks_multithreaded(void* pipeline_context) {
|
|||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
if (send_chunk_delete_plans(client, context->delete_plans, current_chunk) != 0) {
|
|
||||||
log_message(LOG_LEVEL_ERROR, "unexpected error while sending delete plan");
|
|
||||||
chunk_destroy(current_chunk);
|
|
||||||
pipeline_cancel(context);
|
|
||||||
disconnect_transfer_client(client);
|
|
||||||
mark_sender_done(context);
|
|
||||||
protocol_session_unbind();
|
|
||||||
return thrd_error;
|
|
||||||
}
|
|
||||||
if (send_chunk_with_removal(client, current_chunk, context->config,
|
if (send_chunk_with_removal(client, current_chunk, context->config,
|
||||||
context->remove_source_files, &context->stats) != 0) {
|
context->remove_source_files, &context->stats) != 0) {
|
||||||
log_message(LOG_LEVEL_ERROR, "unexpected error while sending chunk");
|
log_message(LOG_LEVEL_ERROR, "unexpected error while sending chunk");
|
||||||
@@ -2893,13 +2904,6 @@ static int send_chunks_multithreaded(void* pipeline_context) {
|
|||||||
NULL) != 0)
|
NULL) != 0)
|
||||||
goto send_fail;
|
goto send_fail;
|
||||||
}
|
}
|
||||||
/* Emit the plans for source directories the data stream never triggered
|
|
||||||
(empty directories): their extras are still cleared while the directory
|
|
||||||
itself is kept. */
|
|
||||||
if (!context->scan_stopped_early && context->delete_plans && context->plan_dirs &&
|
|
||||||
delete_plan_send_remaining(client->file_descriptor, context->delete_plans,
|
|
||||||
context->plan_dirs) != 0)
|
|
||||||
goto send_fail;
|
|
||||||
/* P7 Wave D: transmit the captured directory times last. The scanner thread
|
/* P7 Wave D: transmit the captured directory times last. The scanner thread
|
||||||
(and all parallel workers) has been joined before scanner_done was set, so
|
(and all parallel workers) has been joined before scanner_done was set, so
|
||||||
the list is complete and race-free; on an early stop the list may be
|
the list is complete and race-free; on an early stop the list may be
|
||||||
@@ -2918,8 +2922,8 @@ static int send_chunks_multithreaded(void* pipeline_context) {
|
|||||||
"server reported a deletion failure (--delete); see the server log for the reason");
|
"server reported a deletion failure (--delete); see the server log for the reason");
|
||||||
if (ok)
|
if (ok)
|
||||||
remove_transferred_sources(context->config, context->remove_source_files);
|
remove_transferred_sources(context->config, context->remove_source_files);
|
||||||
if (context->dir_entries)
|
context->stats.flist_dir +=
|
||||||
context->stats.flist_dir += (unsigned long long)context->dir_entries->size;
|
dir_count_for_stats(context->config, context->dir_entries, &context->dir_count);
|
||||||
report_transfer_stats(context->config, &context->stats, start, &recv_stats);
|
report_transfer_stats(context->config, &context->stats, start, &recv_stats);
|
||||||
log_info_message(LOG_INFO_STATS, "Transfer summary: %llu files, %.1f MB",
|
log_info_message(LOG_INFO_STATS, "Transfer summary: %llu files, %.1f MB",
|
||||||
context->stats.transferred_regular,
|
context->stats.transferred_regular,
|
||||||
@@ -2956,6 +2960,7 @@ static int scan_directory_multithreaded(void* pipeline_context) {
|
|||||||
parallel workers append under the context's dedicated mutex. */
|
parallel workers append under the context's dedicated mutex. */
|
||||||
prepared.options.dir_entries = context->dir_entries;
|
prepared.options.dir_entries = context->dir_entries;
|
||||||
prepared.options.dir_entries_mutex = &context->dir_entries_mutex;
|
prepared.options.dir_entries_mutex = &context->dir_entries_mutex;
|
||||||
|
prepared.options.dir_count = context->config->stats ? &context->dir_count : NULL;
|
||||||
if (!append_implied_dir_times(context->config, context->dir_entries)) {
|
if (!append_implied_dir_times(context->config, context->dir_entries)) {
|
||||||
pipeline_cancel(context);
|
pipeline_cancel(context);
|
||||||
protocol_session_unbind();
|
protocol_session_unbind();
|
||||||
@@ -3224,6 +3229,9 @@ int send_files(Config* config) {
|
|||||||
/* P7 Wave D: captured source directory times, transmitted in trailing
|
/* P7 Wave D: captured source directory times, transmitted in trailing
|
||||||
STATUS_DIR_TIMES frame(s) (only when metadata rides the wire). */
|
STATUS_DIR_TIMES frame(s) (only when metadata rides the wire). */
|
||||||
ArrayList* dir_entries = NULL;
|
ArrayList* dir_entries = NULL;
|
||||||
|
/* --stats directory accounting for the no-metadata (-r) case. */
|
||||||
|
atomic_ullong dir_count;
|
||||||
|
atomic_init(&dir_count, 0);
|
||||||
/* Protected excluded prefixes (delete-excluded default protection). */
|
/* Protected excluded prefixes (delete-excluded default protection). */
|
||||||
ArrayList* excluded = NULL;
|
ArrayList* excluded = NULL;
|
||||||
/* Size-pruned prefixes (always protected) and synchronized directories. */
|
/* Size-pruned prefixes (always protected) and synchronized directories. */
|
||||||
@@ -3232,10 +3240,12 @@ int send_files(Config* config) {
|
|||||||
/* Traversed source directories for the per-directory delete keep set. */
|
/* Traversed source directories for the per-directory delete keep set. */
|
||||||
ArrayList* plan_dirs = NULL;
|
ArrayList* plan_dirs = NULL;
|
||||||
bool delete_early = config->use_delete && config_delete_timing_early(config);
|
bool delete_early = config->use_delete && config_delete_timing_early(config);
|
||||||
/* -d/--dirs does not recurse, so a per-directory plan would carry no child
|
/* --delete-during/--delete-delay use per-directory plans for every transfer
|
||||||
information and could delete the contents of an untraversed directory;
|
shape. For -d/--dirs the generator records only the directories whose
|
||||||
fall back to the whole-tree end-of-transfer commit for that mode. */
|
direct children it actually enumerated, so the plan removes extras directly
|
||||||
bool delete_per_dir = config->use_delete && config_delete_timing_per_dir(config) && !config->dirs;
|
inside a listed directory while an untraversed (kept) subdirectory is
|
||||||
|
shielded -- rsync's `-d DIR/ --delete`. */
|
||||||
|
bool delete_per_dir = config->use_delete && config_delete_timing_per_dir(config);
|
||||||
bool send_failed = false;
|
bool send_failed = false;
|
||||||
bool had_scan_io = false;
|
bool had_scan_io = false;
|
||||||
unsigned long long per_dir_non_dir_count = 0;
|
unsigned long long per_dir_non_dir_count = 0;
|
||||||
@@ -3287,12 +3297,12 @@ int send_files(Config* config) {
|
|||||||
prepared.options.synced_dirs = synced_dirs;
|
prepared.options.synced_dirs = synced_dirs;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
/* The late-timing modes (--delete-after/--delete-commit and a plain --delete
|
/* The late-timing modes (--delete-after/--delete-commit) build the manifest
|
||||||
that fell back from per-dir mode because of -d/--dirs) build the manifest
|
|
||||||
while streaming and send it after the last data frame. --delete-before
|
while streaming and send it after the last data frame. --delete-before
|
||||||
sends a whole-tree keep-set up front; --delete-during/--delete-delay build a
|
sends a whole-tree keep-set up front; --delete-during/--delete-delay build
|
||||||
per-directory plan set up front (paths only) and stream the plans alongside
|
the complete per-directory plan set up front (paths only) and transmit it
|
||||||
the data, so no manifest is kept during the data pass. */
|
all before the first data frame, so a mid-transfer abort has already
|
||||||
|
applied every planned removal. */
|
||||||
if (delete_early) {
|
if (delete_early) {
|
||||||
/* Pass 1: collect the complete keep-set (paths only, no data loaded) and
|
/* Pass 1: collect the complete keep-set (paths only, no data loaded) and
|
||||||
transmit it now, before any file data. The receiver removes extras and
|
transmit it now, before any file data. The receiver removes extras and
|
||||||
@@ -3335,9 +3345,10 @@ int send_files(Config* config) {
|
|||||||
goto send_fail;
|
goto send_fail;
|
||||||
} else if (delete_per_dir) {
|
} else if (delete_per_dir) {
|
||||||
/* --delete-during/--delete-delay: build one plan per source directory from a
|
/* --delete-during/--delete-delay: build one plan per source directory from a
|
||||||
path-only pre-scan and transmit the root plan now, before any data, so the
|
path-only pre-scan and transmit the COMPLETE plan set now, before any data,
|
||||||
receive root's extras are handled exactly like rsync's first generator
|
so every planned removal has already been applied when a later transfer
|
||||||
directory. The remaining plans are streamed with the data below. */
|
phase fails -- exactly like rsync's generator, whose deletion list runs
|
||||||
|
ahead of its throttled sender. A completed run is unaffected. */
|
||||||
plan_sender = delete_plan_sender_create();
|
plan_sender = delete_plan_sender_create();
|
||||||
plan_dirs = array_list_create(free);
|
plan_dirs = array_list_create(free);
|
||||||
if (!plan_sender || !plan_dirs)
|
if (!plan_sender || !plan_dirs)
|
||||||
@@ -3368,7 +3379,7 @@ int send_files(Config* config) {
|
|||||||
plan_dirs = NULL;
|
plan_dirs = NULL;
|
||||||
skip_delete = true;
|
skip_delete = true;
|
||||||
} else {
|
} else {
|
||||||
plans_ok = delete_plan_send_root(client->file_descriptor, plan_sender) == 0;
|
plans_ok = delete_plan_send_all(client->file_descriptor, plan_sender, plan_dirs) == 0;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
prepared.options.excluded_paths = NULL;
|
prepared.options.excluded_paths = NULL;
|
||||||
@@ -3402,6 +3413,7 @@ int send_files(Config* config) {
|
|||||||
the directory-time list (otherwise every directory would be captured
|
the directory-time list (otherwise every directory would be captured
|
||||||
twice). */
|
twice). */
|
||||||
prepared.options.dir_entries = dir_entries;
|
prepared.options.dir_entries = dir_entries;
|
||||||
|
prepared.options.dir_count = config->stats ? &dir_count : NULL;
|
||||||
scanner = directory_scanner_create_with_options(config->send_directory, &prepared.options);
|
scanner = directory_scanner_create_with_options(config->send_directory, &prepared.options);
|
||||||
if (!scanner)
|
if (!scanner)
|
||||||
goto send_fail;
|
goto send_fail;
|
||||||
@@ -3455,11 +3467,6 @@ int send_files(Config* config) {
|
|||||||
goto send_fail;
|
goto send_fail;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (send_chunk_delete_plans(client, plan_sender, current_chunk) != 0) {
|
|
||||||
chunk_destroy(current_chunk);
|
|
||||||
send_failed = true;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
if (send_chunk_with_removal(client, current_chunk, config, remove_sources, &transfer_stats) !=
|
if (send_chunk_with_removal(client, current_chunk, config, remove_sources, &transfer_stats) !=
|
||||||
0) {
|
0) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Failed to send chunk");
|
log_message(LOG_LEVEL_ERROR, "Failed to send chunk");
|
||||||
@@ -3542,12 +3549,6 @@ int send_files(Config* config) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
/* Emit the plans for any source directories the data stream never triggered
|
|
||||||
(an empty directory has no file frame). Sending them now still clears that
|
|
||||||
directory's destination extras while keeping the directory itself. */
|
|
||||||
if (!scan_stopped_early && plan_sender && plan_dirs &&
|
|
||||||
delete_plan_send_remaining(client->file_descriptor, plan_sender, plan_dirs) != 0)
|
|
||||||
goto send_fail;
|
|
||||||
/* P7 Wave D: every directory has now been traversed (or the scan stopped
|
/* P7 Wave D: every directory has now been traversed (or the scan stopped
|
||||||
early), so transmit the captured directory times last. The receiver defers
|
early), so transmit the captured directory times last. The receiver defers
|
||||||
applying them until after its own deletion/publication phase. */
|
applying them until after its own deletion/publication phase. */
|
||||||
@@ -3566,8 +3567,7 @@ int send_files(Config* config) {
|
|||||||
from the scanner's captured directory list (present whenever a directory
|
from the scanner's captured directory list (present whenever a directory
|
||||||
attribute is preserved, e.g. -a/-t/-p). The -d generator counts its
|
attribute is preserved, e.g. -a/-t/-p). The -d generator counts its
|
||||||
explicit directory entries inline instead. */
|
explicit directory entries inline instead. */
|
||||||
if (dir_entries)
|
transfer_stats.flist_dir += dir_count_for_stats(config, dir_entries, &dir_count);
|
||||||
transfer_stats.flist_dir += (unsigned long long)dir_entries->size;
|
|
||||||
report_transfer_stats(config, &transfer_stats, start, &recv_stats);
|
report_transfer_stats(config, &transfer_stats, start, &recv_stats);
|
||||||
log_info_message(LOG_INFO_STATS, "Transfer summary: %llu files, %.1f MB",
|
log_info_message(LOG_INFO_STATS, "Transfer summary: %llu files, %.1f MB",
|
||||||
transfer_stats.transferred_regular,
|
transfer_stats.transferred_regular,
|
||||||
@@ -3615,10 +3615,9 @@ send_fail:
|
|||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
|
|
||||||
int send_files_multithreaded(Config** config_ptr) {
|
int send_files_multithreaded(Config* config) {
|
||||||
if (!config_ptr || !*config_ptr)
|
if (!config)
|
||||||
return 1;
|
return 1;
|
||||||
Config* config = *config_ptr;
|
|
||||||
if (config->list_only)
|
if (config->list_only)
|
||||||
return send_list_only(config);
|
return send_list_only(config);
|
||||||
if (config->dry_run)
|
if (config->dry_run)
|
||||||
@@ -3714,10 +3713,11 @@ int send_files_multithreaded(Config** config_ptr) {
|
|||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
/* -d/--dirs does not recurse, so a per-directory plan would carry no child
|
/* --delete-during/--delete-delay use per-directory plans for every transfer
|
||||||
information and could delete the contents of an untraversed directory;
|
shape. The -d/--dirs generator records only the directories whose direct
|
||||||
fall back to the whole-tree end-of-transfer commit for that mode. */
|
children it enumerated, so extras directly inside a listed directory are
|
||||||
bool per_dir = config_delete_timing_per_dir(config) && !config->dirs;
|
removed while an untraversed (kept) subdirectory is shielded. */
|
||||||
|
bool per_dir = config_delete_timing_per_dir(config);
|
||||||
if (config_delete_timing_early(config) || per_dir) {
|
if (config_delete_timing_early(config) || per_dir) {
|
||||||
/* --delete-before / --delete-during / --delete-delay: build the keep-set
|
/* --delete-before / --delete-during / --delete-delay: build the keep-set
|
||||||
(paths only, nothing loaded or sent) up front so the sender thread can
|
(paths only, nothing loaded or sent) up front so the sender thread can
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ void client_set_abort_armed(bool armed);
|
|||||||
* never free it, and the caller retains ownership (freeing it with
|
* never free it, and the caller retains ownership (freeing it with
|
||||||
* config_delete() once the call returns). */
|
* config_delete() once the call returns). */
|
||||||
int send_files(Config* config);
|
int send_files(Config* config);
|
||||||
int send_files_multithreaded(Config** config);
|
int send_files_multithreaded(Config* config);
|
||||||
/* rsync's --ignore-errors deletion gate: with no I/O error during the scan the
|
/* rsync's --ignore-errors deletion gate: with no I/O error during the scan the
|
||||||
* deletion phase always proceeds; with one it is suppressed unless
|
* deletion phase always proceeds; with one it is suppressed unless
|
||||||
* `--ignore-errors` was given. Exposed so the decision can be unit-tested
|
* `--ignore-errors` was given. Exposed so the decision can be unit-tested
|
||||||
|
|||||||
@@ -53,7 +53,7 @@ bool validate_config(const Config* config) {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
if (config->compression_threads > 0 && !config->use_compression) {
|
if (config->compression_threads > 0 && !config->use_compression) {
|
||||||
log_message(LOG_LEVEL_ERROR, "--compress-threads requires compression (-c or -z)");
|
log_message(LOG_LEVEL_ERROR, "--compress-threads requires compression (-z/--compress)");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
if (config->transport == TRANSPORT_SSH && config->use_sendfile) {
|
if (config->transport == TRANSPORT_SSH && config->use_sendfile) {
|
||||||
@@ -75,6 +75,13 @@ bool validate_config(const Config* config) {
|
|||||||
log_message(LOG_LEVEL_ERROR, "-4/--ipv4 and -6/--ipv6 are mutually exclusive");
|
log_message(LOG_LEVEL_ERROR, "-4/--ipv4 and -6/--ipv6 are mutually exclusive");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
/* rsync 3.4.1 rejects --inplace together with --partial-dir (exit 1): the
|
||||||
|
inplace write path bypasses partial staging, so a partial-dir name would be
|
||||||
|
silently ignored. Match rsync's message and refuse before any I/O. */
|
||||||
|
if (config->inplace && config->partial_dir) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "--inplace cannot be used with --partial-dir");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
if (config->log_file_format && !config->log_file) {
|
if (config->log_file_format && !config->log_file) {
|
||||||
log_message(LOG_LEVEL_ERROR, "--log-file-format requires --log-file");
|
log_message(LOG_LEVEL_ERROR, "--log-file-format requires --log-file");
|
||||||
return false;
|
return false;
|
||||||
@@ -102,6 +109,16 @@ bool validate_config(const Config* config) {
|
|||||||
log_message(LOG_LEVEL_ERROR, "%s", invariants_error);
|
log_message(LOG_LEVEL_ERROR, "%s", invariants_error);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
/* The receiver rejects a protect-rule block with more than MAX_FILTER_RULES
|
||||||
|
entries as an opaque protocol error; reject an over-limit --filter set here,
|
||||||
|
before any network I/O, with an actionable message. send_protect_entries()
|
||||||
|
re-checks the final built count because cvs-exclude / merge rules can
|
||||||
|
expand it beyond config->filters->size. */
|
||||||
|
if (config->filters && config->filters->size > MAX_FILTER_RULES) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "too many filter rules: %d (maximum %d)", config->filters->size,
|
||||||
|
MAX_FILTER_RULES);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
/* --protocol: FastSync has exactly one wire format, so the forced version
|
/* --protocol: FastSync has exactly one wire format, so the forced version
|
||||||
must equal the current PROTOCOL_VERSION exactly. Rejected here, before any
|
must equal the current PROTOCOL_VERSION exactly. Rejected here, before any
|
||||||
network I/O, rather than letting the server hit its own mismatch check. */
|
network I/O, rather than letting the server hit its own mismatch check. */
|
||||||
|
|||||||
+248
-49
@@ -205,11 +205,22 @@ typedef struct {
|
|||||||
bool referent_error;
|
bool referent_error;
|
||||||
} ScannerEntry;
|
} ScannerEntry;
|
||||||
|
|
||||||
|
/* One inspected directory entry buffered so the sequential scanner can emit the
|
||||||
|
stream in rsync's flist order. `name` is the raw dirent name (owned here);
|
||||||
|
`entry` is the scanner_inspect_entry() result whose path/link_target are owned
|
||||||
|
when `inspection == 1`; `inspection` is that call's return code (1 keep,
|
||||||
|
0 skip, <0 fatal). */
|
||||||
|
typedef struct {
|
||||||
|
char* name;
|
||||||
|
ScannerEntry entry;
|
||||||
|
int inspection;
|
||||||
|
} SortedEntry;
|
||||||
|
|
||||||
/* --one-file-system (-x) decision. Only directories can carry a different
|
/* --one-file-system (-x) decision. Only directories can carry a different
|
||||||
* device than their parent (mount points), so this is checked when a child
|
* device than their parent (mount points), so this is checked when a child
|
||||||
* directory is about to be descended into. */
|
* directory is about to be descended into. */
|
||||||
bool scanner_same_filesystem(bool one_file_system, dev_t root_device, dev_t entry_device) {
|
bool scanner_same_filesystem(int one_file_system, dev_t root_device, dev_t entry_device) {
|
||||||
return !one_file_system || entry_device == root_device;
|
return one_file_system <= 0 || entry_device == root_device;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Build a payload-less directory File carrying the captured metadata (when
|
/* Build a payload-less directory File carrying the captured metadata (when
|
||||||
@@ -445,6 +456,40 @@ static void scanner_note_nonreg(const ScannerOptions* options, const char* fs_pa
|
|||||||
fflush(stdout);
|
fflush(stdout);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* rsync 3.4.1's `--info=mount` line, emitted when `-xx` drops a mount-point
|
||||||
|
* directory: `[sender] skipping mount-point dir NAME` (the client is the
|
||||||
|
* sender). Plain `-x` keeps the empty directory and prints nothing, matching
|
||||||
|
* rsync. */
|
||||||
|
static void scanner_note_mount(const ScannerOptions* options, const char* fs_path) {
|
||||||
|
if (!options || !options->note_mount || !fs_path)
|
||||||
|
return;
|
||||||
|
const char* rel = utils_strip_transfer_root(fs_path, options->send_directory);
|
||||||
|
char* escaped = output_escape(rel, options->eight_bit_output);
|
||||||
|
printf("[sender] skipping mount-point dir %s\n", escaped ? escaped : rel);
|
||||||
|
free(escaped);
|
||||||
|
fflush(stdout);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* --debug=filter: a selection/filter decision dropped an entry. */
|
||||||
|
static void scanner_note_filter(const ScannerOptions* options, const char* name) {
|
||||||
|
if (!options || !log_debug_enabled(LOG_DEBUG_FILTER) || !name)
|
||||||
|
return;
|
||||||
|
log_debug_message(LOG_DEBUG_FILTER, "filter: excluded %s", name);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Account for a directory that will not be represented by an inline directory
|
||||||
|
* entry. Paired with scanner_dir_count_uncount for empty directories that are
|
||||||
|
* emitted inline, so every traversed directory is counted exactly once. */
|
||||||
|
static void scanner_dir_count_count(const ScannerOptions* options) {
|
||||||
|
if (options && options->dir_count)
|
||||||
|
atomic_fetch_add(options->dir_count, 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void scanner_dir_count_uncount(const ScannerOptions* options) {
|
||||||
|
if (options && options->dir_count)
|
||||||
|
atomic_fetch_sub(options->dir_count, 1);
|
||||||
|
}
|
||||||
|
|
||||||
/* A user-selection exclusion (--filter/-C/per-dir or --exclude/--include). */
|
/* A user-selection exclusion (--filter/-C/per-dir or --exclude/--include). */
|
||||||
static void scanner_record_excluded(DirectoryScanner* scanner, const char* fs_path) {
|
static void scanner_record_excluded(DirectoryScanner* scanner, const char* fs_path) {
|
||||||
scanner_record_protected(scanner, fs_path, scanner->options.excluded_paths);
|
scanner_record_protected(scanner, fs_path, scanner->options.excluded_paths);
|
||||||
@@ -687,6 +732,114 @@ skip:
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void sorted_entry_destroy(void* item) {
|
||||||
|
SortedEntry* se = (SortedEntry*)item;
|
||||||
|
if (!se)
|
||||||
|
return;
|
||||||
|
free(se->name);
|
||||||
|
free(se->entry.path);
|
||||||
|
free(se->entry.link_target);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* rsync flist order within one directory: non-directories first, then
|
||||||
|
directories, each group by ascending name. strcmp() compares as unsigned
|
||||||
|
char, matching rsync's f_name_cmp(). */
|
||||||
|
static int sorted_entry_cmp(const void* a, const void* b) {
|
||||||
|
const SortedEntry* x = (const SortedEntry*)a;
|
||||||
|
const SortedEntry* y = (const SortedEntry*)b;
|
||||||
|
bool x_dir = x->inspection > 0 && x->entry.is_directory;
|
||||||
|
bool y_dir = y->inspection > 0 && y->entry.is_directory;
|
||||||
|
if (x_dir != y_dir)
|
||||||
|
return x_dir ? 1 : -1;
|
||||||
|
return strcmp(x->name, y->name);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void scanner_free_sorted(DirectoryScanner* scanner) {
|
||||||
|
SortedEntry* entries = (SortedEntry*)scanner->sorted_entries;
|
||||||
|
for (size_t i = 0; i < scanner->sorted_count; i++)
|
||||||
|
sorted_entry_destroy(&entries[i]);
|
||||||
|
free(entries);
|
||||||
|
scanner->sorted_entries = NULL;
|
||||||
|
scanner->sorted_count = 0;
|
||||||
|
scanner->sorted_index = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Read every entry of the open directory, inspect it once and store it sorted in
|
||||||
|
rsync's flist order. Returns 0 on success, -1 on a fatal error (the caller
|
||||||
|
aborts the scan). */
|
||||||
|
static int scanner_buffer_current_directory(DirectoryScanner* scanner) {
|
||||||
|
size_t capacity = 64;
|
||||||
|
size_t count = 0;
|
||||||
|
SortedEntry* entries = malloc(capacity * sizeof(*entries));
|
||||||
|
if (!entries) {
|
||||||
|
scanner->failed = true;
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
const struct dirent* dirent;
|
||||||
|
while ((dirent = readdir(scanner->current_dir)) != NULL) {
|
||||||
|
if (strcmp(dirent->d_name, ".") == 0 || strcmp(dirent->d_name, "..") == 0)
|
||||||
|
continue;
|
||||||
|
if (count == capacity) {
|
||||||
|
size_t next = capacity * 2;
|
||||||
|
SortedEntry* grown = realloc(entries, next * sizeof(*entries));
|
||||||
|
if (!grown) {
|
||||||
|
scanner->failed = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
entries = grown;
|
||||||
|
capacity = next;
|
||||||
|
}
|
||||||
|
char* name = str_dup(dirent->d_name);
|
||||||
|
if (!name) {
|
||||||
|
scanner->failed = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
char* link_rel = child_rel_path(scanner->current_rel, dirent->d_name);
|
||||||
|
if (!link_rel) {
|
||||||
|
free(name);
|
||||||
|
scanner->failed = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
int inspection = scanner_inspect_entry(&scanner->options, scanner->current_path, link_rel,
|
||||||
|
dirent->d_name, &entries[count].entry);
|
||||||
|
free(link_rel);
|
||||||
|
if (inspection < 0) {
|
||||||
|
free(name);
|
||||||
|
scanner->failed = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
entries[count].name = name;
|
||||||
|
entries[count].inspection = inspection;
|
||||||
|
count++;
|
||||||
|
}
|
||||||
|
if (scanner->failed) {
|
||||||
|
for (size_t i = 0; i < count; i++)
|
||||||
|
sorted_entry_destroy(&entries[i]);
|
||||||
|
free(entries);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
qsort(entries, count, sizeof(*entries), sorted_entry_cmp);
|
||||||
|
scanner->sorted_entries = entries;
|
||||||
|
scanner->sorted_count = count;
|
||||||
|
scanner->sorted_index = 0;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Push this directory's collected child directories onto the LIFO stack in
|
||||||
|
reverse so the first (ascending) child is popped first (depth-first). */
|
||||||
|
static void scanner_push_pending_dirs(DirectoryScanner* scanner) {
|
||||||
|
ArrayList* pending = (ArrayList*)scanner->pending_dirs;
|
||||||
|
if (!pending)
|
||||||
|
return;
|
||||||
|
for (int i = pending->size - 1; i >= 0; i--) {
|
||||||
|
if (!queue_push(scanner->directories, pending->items[i])) {
|
||||||
|
dir_entry_destroy(pending->items[i]);
|
||||||
|
scanner->failed = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pending->size = 0;
|
||||||
|
}
|
||||||
|
|
||||||
DirectoryScanner* directory_scanner_create_with_options(const char* root_directory,
|
DirectoryScanner* directory_scanner_create_with_options(const char* root_directory,
|
||||||
const ScannerOptions* options) {
|
const ScannerOptions* options) {
|
||||||
if (!root_directory || !options)
|
if (!root_directory || !options)
|
||||||
@@ -704,12 +857,22 @@ DirectoryScanner* directory_scanner_create_with_options(const char* root_directo
|
|||||||
free(scanner);
|
free(scanner);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
scanner->pending_dirs = array_list_create(NULL);
|
||||||
|
if (!scanner->pending_dirs) {
|
||||||
|
queue_destroy(scanner->directories);
|
||||||
|
free(scanner);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
scanner->current_dir = NULL;
|
scanner->current_dir = NULL;
|
||||||
scanner->current_path = NULL;
|
scanner->current_path = NULL;
|
||||||
scanner->current_depth = 0;
|
scanner->current_depth = 0;
|
||||||
scanner->failed = false;
|
scanner->failed = false;
|
||||||
|
scanner->sorted_entries = NULL;
|
||||||
|
scanner->sorted_count = 0;
|
||||||
|
scanner->sorted_index = 0;
|
||||||
scanner->root_path = str_dup(root_directory);
|
scanner->root_path = str_dup(root_directory);
|
||||||
if (!scanner->root_path) {
|
if (!scanner->root_path) {
|
||||||
|
array_list_delete(scanner->pending_dirs);
|
||||||
queue_destroy(scanner->directories);
|
queue_destroy(scanner->directories);
|
||||||
free(scanner);
|
free(scanner);
|
||||||
return NULL;
|
return NULL;
|
||||||
@@ -729,6 +892,7 @@ DirectoryScanner* directory_scanner_create_with_options(const char* root_directo
|
|||||||
scanner->filter_nodes = array_list_create(filter_node_destroy);
|
scanner->filter_nodes = array_list_create(filter_node_destroy);
|
||||||
if (!scanner->filter_nodes) {
|
if (!scanner->filter_nodes) {
|
||||||
free(scanner->root_path);
|
free(scanner->root_path);
|
||||||
|
array_list_delete(scanner->pending_dirs);
|
||||||
queue_destroy(scanner->directories);
|
queue_destroy(scanner->directories);
|
||||||
free(scanner);
|
free(scanner);
|
||||||
return NULL;
|
return NULL;
|
||||||
@@ -739,6 +903,7 @@ DirectoryScanner* directory_scanner_create_with_options(const char* root_directo
|
|||||||
if (stat(root_directory, &root_stats) != 0) {
|
if (stat(root_directory, &root_stats) != 0) {
|
||||||
log_perror("Could not stat source directory");
|
log_perror("Could not stat source directory");
|
||||||
free(scanner->root_path);
|
free(scanner->root_path);
|
||||||
|
array_list_delete(scanner->pending_dirs);
|
||||||
queue_destroy(scanner->directories);
|
queue_destroy(scanner->directories);
|
||||||
array_list_delete(scanner->filter_nodes);
|
array_list_delete(scanner->filter_nodes);
|
||||||
free(scanner);
|
free(scanner);
|
||||||
@@ -757,6 +922,7 @@ DirectoryScanner* directory_scanner_create_with_options(const char* root_directo
|
|||||||
if (!queue_enqueue(scanner->directories, root)) {
|
if (!queue_enqueue(scanner->directories, root)) {
|
||||||
dir_entry_destroy(root);
|
dir_entry_destroy(root);
|
||||||
free(scanner->root_path);
|
free(scanner->root_path);
|
||||||
|
array_list_delete(scanner->pending_dirs);
|
||||||
queue_destroy(scanner->directories);
|
queue_destroy(scanner->directories);
|
||||||
array_list_delete(scanner->filter_nodes);
|
array_list_delete(scanner->filter_nodes);
|
||||||
free(scanner);
|
free(scanner);
|
||||||
@@ -808,6 +974,13 @@ void directory_scanner_destroy(DirectoryScanner* scanner) {
|
|||||||
free(scanner->current_path);
|
free(scanner->current_path);
|
||||||
free(scanner->current_rel);
|
free(scanner->current_rel);
|
||||||
free(scanner->root_path);
|
free(scanner->root_path);
|
||||||
|
scanner_free_sorted(scanner);
|
||||||
|
ArrayList* pending = (ArrayList*)scanner->pending_dirs;
|
||||||
|
if (pending) {
|
||||||
|
for (int i = 0; i < pending->size; i++)
|
||||||
|
dir_entry_destroy(pending->items[i]);
|
||||||
|
array_list_delete(pending);
|
||||||
|
}
|
||||||
array_list_delete(scanner->filter_nodes);
|
array_list_delete(scanner->filter_nodes);
|
||||||
array_list_delete(scanner->dirs_batch);
|
array_list_delete(scanner->dirs_batch);
|
||||||
queue_destroy(scanner->directories);
|
queue_destroy(scanner->directories);
|
||||||
@@ -957,6 +1130,9 @@ static bool scanner_emit_empty_dir(DirectoryScanner* scanner, ArrayList* chunk_d
|
|||||||
file_destroy(dir);
|
file_destroy(dir);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
/* The directory was counted when it was opened; this inline entry represents
|
||||||
|
it, so drop the counter to avoid counting it twice in --stats. */
|
||||||
|
scanner_dir_count_uncount(&scanner->options);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -975,7 +1151,7 @@ static int open_next_directory(DirectoryScanner* scanner) {
|
|||||||
scanner->current_path = NULL;
|
scanner->current_path = NULL;
|
||||||
|
|
||||||
while (!queue_is_empty(scanner->directories)) {
|
while (!queue_is_empty(scanner->directories)) {
|
||||||
DirEntry* de = (DirEntry*)queue_dequeue(scanner->directories);
|
DirEntry* de = (DirEntry*)queue_pop(scanner->directories);
|
||||||
scanner->current_path = de->path;
|
scanner->current_path = de->path;
|
||||||
scanner->current_depth = de->depth;
|
scanner->current_depth = de->depth;
|
||||||
/* The seed directory inherits the scanner's configured context (the root
|
/* The seed directory inherits the scanner's configured context (the root
|
||||||
@@ -1046,6 +1222,8 @@ static int open_next_directory(DirectoryScanner* scanner) {
|
|||||||
scanner->failed = true;
|
scanner->failed = true;
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
scanner_dir_count_count(&scanner->options);
|
||||||
|
log_debug_message(LOG_DEBUG_FLIST, "flist: scanning %s", scanner->current_path);
|
||||||
if (scanner->options.capture_dir_times &&
|
if (scanner->options.capture_dir_times &&
|
||||||
!scanner_capture_dir_time(
|
!scanner_capture_dir_time(
|
||||||
scanner->options.dir_entries, scanner->options.dir_entries_mutex, scanner->root_path,
|
scanner->options.dir_entries, scanner->options.dir_entries_mutex, scanner->root_path,
|
||||||
@@ -1060,6 +1238,14 @@ static int open_next_directory(DirectoryScanner* scanner) {
|
|||||||
scanner->failed = true;
|
scanner->failed = true;
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
/* Buffer and sort this directory's entries in rsync's flist order. */
|
||||||
|
if (scanner_buffer_current_directory(scanner) != 0) {
|
||||||
|
closedir(scanner->current_dir);
|
||||||
|
scanner->current_dir = NULL;
|
||||||
|
free(scanner->current_path);
|
||||||
|
scanner->current_path = NULL;
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
return 0;
|
return 0;
|
||||||
@@ -1304,6 +1490,17 @@ static File* dirs_next_file(DirectoryScanner* scanner) {
|
|||||||
scanner->dirs_root_emitted = true;
|
scanner->dirs_root_emitted = true;
|
||||||
if (scanner->options.prune_empty_dirs && dirs_source_dir_is_empty(scanner->root_path))
|
if (scanner->options.prune_empty_dirs && dirs_source_dir_is_empty(scanner->root_path))
|
||||||
return NULL;
|
return NULL;
|
||||||
|
/* The listed directory's direct children are about to be enumerated, so
|
||||||
|
its destination mirror is a synchronized directory: record it for the
|
||||||
|
per-directory delete plan. The plan keeps the enumerated children and
|
||||||
|
shields untraversed subdirectories, so --delete-during removes extras
|
||||||
|
directly inside the listed directory without descending into a kept
|
||||||
|
(but untraversed) child -- exactly rsync's `-d DIR/ --delete`. */
|
||||||
|
if (!scanner_record_synced_dir(&scanner->options, scanner->root_path, "",
|
||||||
|
scanner->relative_mode)) {
|
||||||
|
scanner->failed = true;
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
scanner->current_dir = opendir(scanner->root_path);
|
scanner->current_dir = opendir(scanner->root_path);
|
||||||
if (!scanner->current_dir) {
|
if (!scanner->current_dir) {
|
||||||
scanner->io_error = true;
|
scanner->io_error = true;
|
||||||
@@ -1415,8 +1612,7 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
|
|||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
const struct dirent* entry = readdir(scanner->current_dir);
|
if (scanner->sorted_index >= scanner->sorted_count) {
|
||||||
if (entry == NULL) {
|
|
||||||
/* The directory is exhausted: if nothing was transferred or descended
|
/* The directory is exhausted: if nothing was transferred or descended
|
||||||
from it, recreate it at the destination as an explicit entry. */
|
from it, recreate it at the destination as an explicit entry. */
|
||||||
if (scanner->options.emit_empty_dirs && !scanner->current_dir_produced &&
|
if (scanner->options.emit_empty_dirs && !scanner->current_dir_produced &&
|
||||||
@@ -1425,10 +1621,12 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
|
|||||||
if (!scanner_emit_empty_dir(scanner, chunk_data))
|
if (!scanner_emit_empty_dir(scanner, chunk_data))
|
||||||
scanner->failed = true;
|
scanner->failed = true;
|
||||||
}
|
}
|
||||||
|
scanner_push_pending_dirs(scanner);
|
||||||
closedir(scanner->current_dir);
|
closedir(scanner->current_dir);
|
||||||
scanner->current_dir = NULL;
|
scanner->current_dir = NULL;
|
||||||
free(scanner->current_path);
|
free(scanner->current_path);
|
||||||
scanner->current_path = NULL;
|
scanner->current_path = NULL;
|
||||||
|
scanner_free_sorted(scanner);
|
||||||
if (scanner->failed) {
|
if (scanner->failed) {
|
||||||
array_list_delete(chunk_data);
|
array_list_delete(chunk_data);
|
||||||
return NULL;
|
return NULL;
|
||||||
@@ -1436,26 +1634,15 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
|
SortedEntry* sorted = &((SortedEntry*)scanner->sorted_entries)[scanner->sorted_index++];
|
||||||
continue;
|
const char* name = sorted->name;
|
||||||
|
ScannerEntry* inspected = &sorted->entry;
|
||||||
|
int inspection = sorted->inspection;
|
||||||
|
|
||||||
ScannerEntry inspected;
|
|
||||||
char* link_rel = child_rel_path(scanner->current_rel, entry->d_name);
|
|
||||||
if (!link_rel) {
|
|
||||||
scanner->failed = true;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
int inspection = scanner_inspect_entry(&scanner->options, scanner->current_path, link_rel,
|
|
||||||
entry->d_name, &inspected);
|
|
||||||
free(link_rel);
|
|
||||||
if (inspection < 0) {
|
|
||||||
scanner->failed = true;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
if (inspection == 0) {
|
if (inspection == 0) {
|
||||||
/* A dereferenced symlink with no referent is a partial-transfer error
|
/* A dereferenced symlink with no referent is a partial-transfer error
|
||||||
(rsync exit 23): record it as a non-fatal scan I/O error. */
|
(rsync exit 23): record it as a non-fatal scan I/O error. */
|
||||||
if (inspected.referent_error)
|
if (inspected->referent_error)
|
||||||
scanner->io_error = true;
|
scanner->io_error = true;
|
||||||
/* A user-selection exclude protects its destination mirror from --delete
|
/* A user-selection exclude protects its destination mirror from --delete
|
||||||
unless --delete-excluded; a size prune is always protected. Other
|
unless --delete-excluded; a size prune is always protected. Other
|
||||||
@@ -1463,23 +1650,23 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
|
|||||||
--files-from the protected prefix must be the entry's bare relative
|
--files-from the protected prefix must be the entry's bare relative
|
||||||
wire path, not its source path (which would not match the destination
|
wire path, not its source path (which would not match the destination
|
||||||
layout and would leave the mirror deletable). */
|
layout and would leave the mirror deletable). */
|
||||||
if (inspected.excluded) {
|
if (inspected->excluded) {
|
||||||
char* protected_path;
|
char* protected_path;
|
||||||
if (scanner->relative_mode) {
|
if (scanner->relative_mode) {
|
||||||
protected_path = child_rel_path(scanner->current_rel, entry->d_name);
|
protected_path = child_rel_path(scanner->current_rel, name);
|
||||||
} else if (scanner->options.relative_prefix) {
|
} else if (scanner->options.relative_prefix) {
|
||||||
char* relc = child_rel_path(scanner->current_rel, entry->d_name);
|
char* relc = child_rel_path(scanner->current_rel, name);
|
||||||
protected_path =
|
protected_path =
|
||||||
relc ? scanner_prefix_send_path(scanner->options.relative_prefix, relc) : NULL;
|
relc ? scanner_prefix_send_path(scanner->options.relative_prefix, relc) : NULL;
|
||||||
free(relc);
|
free(relc);
|
||||||
} else {
|
} else {
|
||||||
protected_path = path_cat(scanner->current_path, entry->d_name);
|
protected_path = path_cat(scanner->current_path, name);
|
||||||
}
|
}
|
||||||
if (!protected_path) {
|
if (!protected_path) {
|
||||||
scanner->failed = true;
|
scanner->failed = true;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
if (inspected.size_excluded)
|
if (inspected->size_excluded)
|
||||||
scanner_record_size_skipped(scanner, protected_path);
|
scanner_record_size_skipped(scanner, protected_path);
|
||||||
else
|
else
|
||||||
scanner_record_excluded(scanner, protected_path);
|
scanner_record_excluded(scanner, protected_path);
|
||||||
@@ -1487,23 +1674,22 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
|
|||||||
}
|
}
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
char* cur_path = inspected.path;
|
char* cur_path = inspected->path;
|
||||||
struct stat stats = inspected.stats;
|
struct stat stats = inspected->stats;
|
||||||
|
|
||||||
/* --files-from allow-set and the filter layer apply to files and to
|
/* --files-from allow-set and the filter layer apply to files and to
|
||||||
* directories (an excluded directory is not descended into). */
|
* directories (an excluded directory is not descended into). */
|
||||||
bool is_dir = inspected.is_directory;
|
bool is_dir = inspected->is_directory;
|
||||||
char* rel = child_rel_path(scanner->current_rel, entry->d_name);
|
char* rel = child_rel_path(scanner->current_rel, name);
|
||||||
if (!rel) {
|
if (!rel) {
|
||||||
free(cur_path);
|
|
||||||
scanner->failed = true;
|
scanner->failed = true;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
bool protect = false;
|
bool protect = false;
|
||||||
bool passes_selection = entry_passes_selection(
|
bool passes_selection = entry_passes_selection(
|
||||||
scanner->options.file_list, scanner->options.base_filters, scanner->current_node, rel,
|
scanner->options.file_list, scanner->options.base_filters, scanner->current_node, rel, name,
|
||||||
entry->d_name, is_dir, scanner->options.per_dir_filters,
|
is_dir, scanner->options.per_dir_filters, scanner->options.exclude_per_dir_filter_files,
|
||||||
scanner->options.exclude_per_dir_filter_files, &protect);
|
&protect);
|
||||||
/* A sender-side hide leaves the entry out of the transfer; an independent
|
/* A sender-side hide leaves the entry out of the transfer; an independent
|
||||||
receiver-side protect rule keeps a transferred entry's destination mirror
|
receiver-side protect rule keeps a transferred entry's destination mirror
|
||||||
from being deleted. Both are recorded in the same protection set. */
|
from being deleted. Both are recorded in the same protection set. */
|
||||||
@@ -1525,7 +1711,6 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
|
|||||||
char* wrel = scanner_prefix_send_path(scanner->options.relative_prefix, rel);
|
char* wrel = scanner_prefix_send_path(scanner->options.relative_prefix, rel);
|
||||||
if (!wrel) {
|
if (!wrel) {
|
||||||
free(rel);
|
free(rel);
|
||||||
free(cur_path);
|
|
||||||
scanner->failed = true;
|
scanner->failed = true;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
@@ -1542,13 +1727,12 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
|
|||||||
char* rel_copy = needs_rel ? str_dup(rel) : NULL;
|
char* rel_copy = needs_rel ? str_dup(rel) : NULL;
|
||||||
free(rel);
|
free(rel);
|
||||||
if (rel_copy == NULL && needs_rel) {
|
if (rel_copy == NULL && needs_rel) {
|
||||||
free(cur_path);
|
|
||||||
scanner->failed = true;
|
scanner->failed = true;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
if (!passes_selection) {
|
if (!passes_selection) {
|
||||||
|
scanner_note_filter(&scanner->options, name);
|
||||||
free(rel_copy);
|
free(rel_copy);
|
||||||
free(cur_path);
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1556,6 +1740,13 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
|
|||||||
free(rel_copy);
|
free(rel_copy);
|
||||||
if (!scanner_same_filesystem(scanner->options.one_file_system, scanner->root_dev,
|
if (!scanner_same_filesystem(scanner->options.one_file_system, scanner->root_dev,
|
||||||
stats.st_dev)) {
|
stats.st_dev)) {
|
||||||
|
if (scanner->options.one_file_system > 1) {
|
||||||
|
/* rsync's -xx drops the mount-point directory entirely (the plain -x
|
||||||
|
path below keeps it as an empty directory) and prints the
|
||||||
|
--info=mount line when that category is enabled. */
|
||||||
|
scanner_note_mount(&scanner->options, cur_path);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
/* rsync's -x/--one-file-system emits the mount-point directory entry
|
/* rsync's -x/--one-file-system emits the mount-point directory entry
|
||||||
itself (so the destination gets an empty directory) but does NOT
|
itself (so the destination gets an empty directory) but does NOT
|
||||||
descend into it. Build a payload-less directory File and hand it to
|
descend into it. Build a payload-less directory File and hand it to
|
||||||
@@ -1563,12 +1754,10 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
|
|||||||
File* mount = scanner_build_dir_file(cur_path, &stats, &scanner->options);
|
File* mount = scanner_build_dir_file(cur_path, &stats, &scanner->options);
|
||||||
if (mount == NULL || !array_list_add(chunk_data, mount)) {
|
if (mount == NULL || !array_list_add(chunk_data, mount)) {
|
||||||
file_destroy(mount);
|
file_destroy(mount);
|
||||||
free(cur_path);
|
|
||||||
scanner->failed = true;
|
scanner->failed = true;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
scanner->current_dir_produced = true;
|
scanner->current_dir_produced = true;
|
||||||
free(cur_path);
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
/* --list-only: list directory entries too (rsync prints them), even
|
/* --list-only: list directory entries too (rsync prints them), even
|
||||||
@@ -1577,7 +1766,6 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
|
|||||||
File* dir = scanner_build_dir_file(cur_path, &stats, &scanner->options);
|
File* dir = scanner_build_dir_file(cur_path, &stats, &scanner->options);
|
||||||
if (dir == NULL || !array_list_add(chunk_data, dir)) {
|
if (dir == NULL || !array_list_add(chunk_data, dir)) {
|
||||||
file_destroy(dir);
|
file_destroy(dir);
|
||||||
free(cur_path);
|
|
||||||
scanner->failed = true;
|
scanner->failed = true;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
@@ -1586,32 +1774,29 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
|
|||||||
int next_depth = scanner->current_depth + 1;
|
int next_depth = scanner->current_depth + 1;
|
||||||
if (scanner->options.max_depth <= 0 || next_depth < scanner->options.max_depth) {
|
if (scanner->options.max_depth <= 0 || next_depth < scanner->options.max_depth) {
|
||||||
DirEntry* de = dir_entry_create(cur_path, next_depth, scanner->current_node);
|
DirEntry* de = dir_entry_create(cur_path, next_depth, scanner->current_node);
|
||||||
if (!de || !queue_enqueue(scanner->directories, de)) {
|
if (!de || !array_list_add((ArrayList*)scanner->pending_dirs, de)) {
|
||||||
dir_entry_destroy(de);
|
dir_entry_destroy(de);
|
||||||
scanner->failed = true;
|
scanner->failed = true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
free(cur_path);
|
|
||||||
} else {
|
} else {
|
||||||
if (scanner->options.max_depth > 0 &&
|
if (scanner->options.max_depth > 0 &&
|
||||||
scanner->current_depth + 1 > scanner->options.max_depth) {
|
scanner->current_depth + 1 > scanner->options.max_depth) {
|
||||||
free(rel_copy);
|
free(rel_copy);
|
||||||
free(cur_path);
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
File* file = file_create(cur_path);
|
File* file = file_create(cur_path);
|
||||||
free(cur_path);
|
|
||||||
if (file == NULL) {
|
if (file == NULL) {
|
||||||
free(rel_copy);
|
free(rel_copy);
|
||||||
free(inspected.link_target);
|
free(inspected->link_target);
|
||||||
inspected.link_target = NULL;
|
inspected->link_target = NULL;
|
||||||
scanner->failed = true;
|
scanner->failed = true;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if (inspected.is_symlink) {
|
if (inspected->is_symlink) {
|
||||||
file->is_symlink = true;
|
file->is_symlink = true;
|
||||||
file->symlink_target = inspected.link_target;
|
file->symlink_target = inspected->link_target;
|
||||||
inspected.link_target = NULL;
|
inspected->link_target = NULL;
|
||||||
} else {
|
} else {
|
||||||
file->data->size = stats.st_size;
|
file->data->size = stats.st_size;
|
||||||
}
|
}
|
||||||
@@ -1975,6 +2160,7 @@ static void scan_root_entry(const ScannerOptions* options, const FilterNode* roo
|
|||||||
free(prefixed);
|
free(prefixed);
|
||||||
}
|
}
|
||||||
if (!passes) {
|
if (!passes) {
|
||||||
|
scanner_note_filter(options, entry->d_name);
|
||||||
free(rel);
|
free(rel);
|
||||||
free(cur_path);
|
free(cur_path);
|
||||||
return;
|
return;
|
||||||
@@ -1982,6 +2168,14 @@ static void scan_root_entry(const ScannerOptions* options, const FilterNode* roo
|
|||||||
}
|
}
|
||||||
if (is_dir) {
|
if (is_dir) {
|
||||||
if (!scanner_same_filesystem(options->one_file_system, root_dev, st.st_dev)) {
|
if (!scanner_same_filesystem(options->one_file_system, root_dev, st.st_dev)) {
|
||||||
|
if (options->one_file_system > 1) {
|
||||||
|
/* -xx: drop the mount-point directory entirely (rsync) and print the
|
||||||
|
--info=mount line when enabled. */
|
||||||
|
scanner_note_mount(options, cur_path);
|
||||||
|
free(rel);
|
||||||
|
free(cur_path);
|
||||||
|
return;
|
||||||
|
}
|
||||||
/* -x/--one-file-system: emit the mount-point directory entry (empty) but
|
/* -x/--one-file-system: emit the mount-point directory entry (empty) but
|
||||||
do not descend into it (see the sequential scanner for the same rule). */
|
do not descend into it (see the sequential scanner for the same rule). */
|
||||||
File* mount = file_create(cur_path);
|
File* mount = file_create(cur_path);
|
||||||
@@ -2119,6 +2313,7 @@ static bool scan_root_directory(ParallelScanner* ps, const char* root_directory,
|
|||||||
ps->failed = true;
|
ps->failed = true;
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
log_debug_message(LOG_DEBUG_FLIST, "flist: scanning %s", root_directory);
|
||||||
const struct dirent* entry;
|
const struct dirent* entry;
|
||||||
while ((entry = readdir(dir)) != NULL) {
|
while ((entry = readdir(dir)) != NULL) {
|
||||||
if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
|
if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
|
||||||
@@ -2283,6 +2478,10 @@ ParallelScanner* parallel_scanner_create_with_options(const char* root_directory
|
|||||||
parallel_scanner_destroy(ps);
|
parallel_scanner_destroy(ps);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
/* The root itself is a traversed directory (rsync counts it in
|
||||||
|
`Number of files`); the worker DirectoryScanners account for every
|
||||||
|
subdirectory below it. */
|
||||||
|
scanner_dir_count_count(options);
|
||||||
/* P7 Wave D: the parallel scanner never runs a DirectoryScanner over the
|
/* P7 Wave D: the parallel scanner never runs a DirectoryScanner over the
|
||||||
transfer root itself (it hands the root's immediate subdirectories to
|
transfer root itself (it hands the root's immediate subdirectories to
|
||||||
workers), so capture the root's directory time here. */
|
workers), so capture the root's directory time here. */
|
||||||
|
|||||||
+25
-2
@@ -10,6 +10,7 @@
|
|||||||
#include "stop_condition.h"
|
#include "stop_condition.h"
|
||||||
#include <dirent.h>
|
#include <dirent.h>
|
||||||
#include <stdbool.h>
|
#include <stdbool.h>
|
||||||
|
#include <stddef.h>
|
||||||
#include <stdatomic.h>
|
#include <stdatomic.h>
|
||||||
#include <sys/types.h>
|
#include <sys/types.h>
|
||||||
#include <threads.h>
|
#include <threads.h>
|
||||||
@@ -50,7 +51,7 @@ typedef struct {
|
|||||||
bool copy_dirlinks;
|
bool copy_dirlinks;
|
||||||
bool munge_links;
|
bool munge_links;
|
||||||
bool checksum;
|
bool checksum;
|
||||||
bool one_file_system;
|
int one_file_system;
|
||||||
/* Phase 4 special/devices: whether device nodes (--devices) and special files
|
/* Phase 4 special/devices: whether device nodes (--devices) and special files
|
||||||
* (--specials) are preserved via recreation, and whether --copy-devices
|
* (--specials) are preserved via recreation, and whether --copy-devices
|
||||||
* copies a device's content as an ordinary regular file. */
|
* copies a device's content as an ordinary regular file. */
|
||||||
@@ -129,6 +130,17 @@ typedef struct {
|
|||||||
/* --info=nonreg: print rsync's `skipping non-regular file "NAME"` line for a
|
/* --info=nonreg: print rsync's `skipping non-regular file "NAME"` line for a
|
||||||
* non-regular entry that is not being preserved. Client-only. */
|
* non-regular entry that is not being preserved. Client-only. */
|
||||||
bool note_nonreg;
|
bool note_nonreg;
|
||||||
|
/* --info=mount: print rsync's `[sender] skipping mount-point dir NAME` when
|
||||||
|
* -xx drops a mount-point directory. Client-only. */
|
||||||
|
bool note_mount;
|
||||||
|
/* --stats directory accounting for a `-r` run (no -t/-p): a shared counter of
|
||||||
|
* traversed directories that are NOT otherwise represented by an inline
|
||||||
|
* directory entry (rsync still counts every directory in `Number of files`).
|
||||||
|
* Incremented when a directory is opened and decremented when an empty
|
||||||
|
* directory is emitted inline (so it is counted exactly once). Atomic
|
||||||
|
* because the parallel scanner's workers share it; NULL disables the
|
||||||
|
* accounting. Client-only. */
|
||||||
|
atomic_ullong* dir_count;
|
||||||
/* Source root and 8-bit-output policy used to render a `--info=nonreg` name
|
/* Source root and 8-bit-output policy used to render a `--info=nonreg` name
|
||||||
* relative to the transfer root. Borrowed read-only. */
|
* relative to the transfer root. Borrowed read-only. */
|
||||||
const char* send_directory;
|
const char* send_directory;
|
||||||
@@ -188,6 +200,17 @@ typedef struct {
|
|||||||
int current_depth;
|
int current_depth;
|
||||||
dev_t root_dev;
|
dev_t root_dev;
|
||||||
bool failed;
|
bool failed;
|
||||||
|
/* rsync-order traversal: each opened directory's entries are inspected once
|
||||||
|
and buffered (an internal SortedEntry[] owned here) sorted as rsync's flist
|
||||||
|
orders them -- non-directories ascending, then directories ascending. The
|
||||||
|
entries are walked in order and child directories are collected in
|
||||||
|
`pending_dirs` (an ArrayList of DirEntry*, owned here) and pushed onto the
|
||||||
|
LIFO `directories` stack in reverse at directory exhaustion, so the emitted
|
||||||
|
stream is depth-first like rsync. `sorted_*` are reset per directory. */
|
||||||
|
void* sorted_entries;
|
||||||
|
size_t sorted_count;
|
||||||
|
size_t sorted_index;
|
||||||
|
void* pending_dirs;
|
||||||
/* Recursive scan: whether the open directory yielded any transferred or
|
/* Recursive scan: whether the open directory yielded any transferred or
|
||||||
descended entry. When it did not, closing it emits a directory entry so
|
descended entry. When it did not, closing it emits a directory entry so
|
||||||
the empty source directory is recreated at the destination (rsync
|
the empty source directory is recreated at the destination (rsync
|
||||||
@@ -254,7 +277,7 @@ void directory_scanner_destroy(DirectoryScanner* scanner);
|
|||||||
/* --one-file-system (-x) decision: a directory entry may be descended into
|
/* --one-file-system (-x) decision: a directory entry may be descended into
|
||||||
* only when the option is disabled or the entry lives on the same device as
|
* only when the option is disabled or the entry lives on the same device as
|
||||||
* the transfer root. Exposed so tests can exercise the rule directly. */
|
* the transfer root. Exposed so tests can exercise the rule directly. */
|
||||||
bool scanner_same_filesystem(bool one_file_system, dev_t root_device, dev_t entry_device);
|
bool scanner_same_filesystem(int one_file_system, dev_t root_device, dev_t entry_device);
|
||||||
|
|
||||||
/* Relative path of an on-disk path below `root` ("" == the root itself, NULL
|
/* Relative path of an on-disk path below `root` ("" == the root itself, NULL
|
||||||
* when `fs_path` is not under `root`). Handles trailing slashes and a root of
|
* when `fs_path` is not under `root`). Handles trailing slashes and a root of
|
||||||
|
|||||||
+24
-12
@@ -19,7 +19,9 @@ void print_usage(void) {
|
|||||||
printf("\n");
|
printf("\n");
|
||||||
printf("Options:\n");
|
printf("Options:\n");
|
||||||
printf(" -c, --checksum Verify content by checksum instead of size+mtime\n");
|
printf(" -c, --checksum Verify content by checksum instead of size+mtime\n");
|
||||||
printf(" -z, --compress [level] Enable compression (level 1-22, default 5)\n");
|
printf(" -z, --compress [level] Enable compression. The default level is\n");
|
||||||
|
printf(" per-codec: zstd 3 (range 1-22), zlib/zlibx 6, lz4\n");
|
||||||
|
printf(" ignores the level\n");
|
||||||
printf(" -a, --archive rsync archive mode (-rlptgoD): links, perms, times,\n");
|
printf(" -a, --archive rsync archive mode (-rlptgoD): links, perms, times,\n");
|
||||||
printf(" owner, group, devices and specials; not\n");
|
printf(" owner, group, devices and specials; not\n");
|
||||||
printf(" compression/multithreading\n");
|
printf(" compression/multithreading\n");
|
||||||
@@ -36,8 +38,8 @@ void print_usage(void) {
|
|||||||
printf(" arguments, e.g. -e \"ssh -p 2222\"\n");
|
printf(" arguments, e.g. -e \"ssh -p 2222\"\n");
|
||||||
printf(" --rsync-path <path> Alias for --fastsync-server-path (path to the\n");
|
printf(" --rsync-path <path> Alias for --fastsync-server-path (path to the\n");
|
||||||
printf(" fastsync server binary on the remote side)\n");
|
printf(" fastsync server binary on the remote side)\n");
|
||||||
printf(" --blocking-io Leave the SSH transport socket without read/write\n");
|
printf(" --blocking-io SSH transport only: leave the socket without read/write\n");
|
||||||
printf(" timeouts so it blocks naturally\n");
|
printf(" timeouts so it blocks naturally (no effect on TCP)\n");
|
||||||
printf(" --outbuf=MODE stdout/stderr buffering: N (none/unbuffered),\n");
|
printf(" --outbuf=MODE stdout/stderr buffering: N (none/unbuffered),\n");
|
||||||
printf(" L (line-buffered), or B (block-buffered, default)\n");
|
printf(" L (line-buffered), or B (block-buffered, default)\n");
|
||||||
printf(" --progress Show transfer progress\n");
|
printf(" --progress Show transfer progress\n");
|
||||||
@@ -49,6 +51,7 @@ void print_usage(void) {
|
|||||||
printf(" converted before transmission and back on receipt; a\n");
|
printf(" converted before transmission and back on receipt; a\n");
|
||||||
printf(" name that cannot be represented in the target charset\n");
|
printf(" name that cannot be represented in the target charset\n");
|
||||||
printf(" fails that transfer cleanly (rsync-compatible)\n");
|
printf(" fails that transfer cleanly (rsync-compatible)\n");
|
||||||
|
printf(" --no-iconv Disable --iconv charset conversion (same as --iconv=-)\n");
|
||||||
printf(" --protocol=NUM Force the wire protocol version (must equal the current\n");
|
printf(" --protocol=NUM Force the wire protocol version (must equal the current\n");
|
||||||
printf(" PROTOCOL_VERSION; FastSync cannot speak older/virtual\n");
|
printf(" PROTOCOL_VERSION; FastSync cannot speak older/virtual\n");
|
||||||
printf(" wire formats)\n");
|
printf(" wire formats)\n");
|
||||||
@@ -162,7 +165,7 @@ void print_usage(void) {
|
|||||||
printf(" --no-delta, or --no-incremental)\n");
|
printf(" --no-delta, or --no-incremental)\n");
|
||||||
printf(" --no-fuzzy Disable --fuzzy\n");
|
printf(" --no-fuzzy Disable --fuzzy\n");
|
||||||
printf(" -B <n>, --block-size <n>, --delta-block <n>\n");
|
printf(" -B <n>, --block-size <n>, --delta-block <n>\n");
|
||||||
printf(" Delta block size in bytes (default: %d)\n", DELTA_BLOCK_SIZE_DEFAULT);
|
printf(" Delta block size in bytes (default: %u)\n", DELTA_BLOCK_SIZE_DEFAULT);
|
||||||
printf(" --delta-max <n> Max file size for delta transfer (default: %llu)\n",
|
printf(" --delta-max <n> Max file size for delta transfer (default: %llu)\n",
|
||||||
DELTA_MAX_FILE_SIZE);
|
DELTA_MAX_FILE_SIZE);
|
||||||
printf(" -j, --threads[=N] Enable the multithreaded scanner/loader/sender\n");
|
printf(" -j, --threads[=N] Enable the multithreaded scanner/loader/sender\n");
|
||||||
@@ -192,6 +195,10 @@ void print_usage(void) {
|
|||||||
printf(" -U, --atimes Preserve access times\n");
|
printf(" -U, --atimes Preserve access times\n");
|
||||||
printf(" -N, --crtimes Capture birth time; cannot be applied (documented\n");
|
printf(" -N, --crtimes Capture birth time; cannot be applied (documented\n");
|
||||||
printf(" divergence)\n");
|
printf(" divergence)\n");
|
||||||
|
printf(" -O, --omit-dir-times Do not apply modification times to directories\n");
|
||||||
|
printf(" -J, --omit-link-times Do not apply times to symlinks\n");
|
||||||
|
printf(" --open-noatime Open source files with O_NOATIME so reading for a\n");
|
||||||
|
printf(" transfer does not update their access time\n");
|
||||||
printf(" -X, --xattrs Preserve user extended attributes (user.* only;\n");
|
printf(" -X, --xattrs Preserve user extended attributes (user.* only;\n");
|
||||||
printf(" privileged security.*/trusted.* namespaces are\n");
|
printf(" privileged security.*/trusted.* namespaces are\n");
|
||||||
printf(" never captured or applied)\n");
|
printf(" never captured or applied)\n");
|
||||||
@@ -287,8 +294,12 @@ void print_usage(void) {
|
|||||||
printf(" -x, --one-file-system Do not cross filesystem boundaries\n");
|
printf(" -x, --one-file-system Do not cross filesystem boundaries\n");
|
||||||
printf(" --log-file <path>, --log-file=<path> Write log messages to file\n");
|
printf(" --log-file <path>, --log-file=<path> Write log messages to file\n");
|
||||||
printf(" --stderr=MODE Route logging to stderr: errors or all\n");
|
printf(" --stderr=MODE Route logging to stderr: errors or all\n");
|
||||||
|
printf(" --msgs2stderr Route all messages to stderr (deprecated spelling of\n");
|
||||||
|
printf(" --stderr=all)\n");
|
||||||
|
printf(" --no-msgs2stderr Select errors-only stderr (deprecated spelling; the\n");
|
||||||
|
printf(" default)\n");
|
||||||
printf(" --partial Keep partial files on interrupted transfer\n");
|
printf(" --partial Keep partial files on interrupted transfer\n");
|
||||||
printf(" --partial-dir <dir> Directory for partial files\n");
|
printf(" --partial-dir <dir> Directory for partial files (implies --partial)\n");
|
||||||
printf(" -T, --temp-dir <dir> Scratch dir for temp files before atomic install.\n");
|
printf(" -T, --temp-dir <dir> Scratch dir for temp files before atomic install.\n");
|
||||||
printf(" Confined to the receive root: a relative dir resolves below\n");
|
printf(" Confined to the receive root: a relative dir resolves below\n");
|
||||||
printf(" it and an absolute/traversal dir is rejected. The dir must\n");
|
printf(" it and an absolute/traversal dir is rejected. The dir must\n");
|
||||||
@@ -337,7 +348,8 @@ void print_usage(void) {
|
|||||||
printf(" --append-verify Like --append, but verifies the retained prefix checksum\n");
|
printf(" --append-verify Like --append, but verifies the retained prefix checksum\n");
|
||||||
printf(" before appending (falls back to a full transfer on mismatch)\n");
|
printf(" before appending (falls back to a full transfer on mismatch)\n");
|
||||||
printf(" --fsync Fsync every written file before publication\n");
|
printf(" --fsync Fsync every written file before publication\n");
|
||||||
printf(" --compress-level <n> Compression level (default: 5)\n");
|
printf(" --compress-level <n> Compression level (per-codec default: zstd 3,\n");
|
||||||
|
printf(" zlib/zlibx 6, lz4 ignores it)\n");
|
||||||
printf(" --zl <n> Alias for --compress-level\n");
|
printf(" --zl <n> Alias for --compress-level\n");
|
||||||
printf(" --skip-compress=LIST Skip compression for suffixes in LIST (separated by\n");
|
printf(" --skip-compress=LIST Skip compression for suffixes in LIST (separated by\n");
|
||||||
printf(" '/' as in rsync, or ','); a leading dot is optional. The\n");
|
printf(" '/' as in rsync, or ','); a leading dot is optional. The\n");
|
||||||
@@ -349,19 +361,19 @@ void print_usage(void) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
void print_debug_usage(void) {
|
void print_debug_usage(void) {
|
||||||
printf("Emitting debug flags: IO,PROTO,PACK,UTIL,ALL,NONE\n");
|
printf("Emitting debug flags: IO,PROTO,PACK,UTIL,FLIST,DEL,HASH,DELTASUM,\n");
|
||||||
|
printf("RECV,FILTER,SEND,ALL,NONE\n");
|
||||||
printf("Also accepted for rsync CLI parity (silent): ACL,BACKUP,BIND,CHDIR,\n");
|
printf("Also accepted for rsync CLI parity (silent): ACL,BACKUP,BIND,CHDIR,\n");
|
||||||
printf("CONNECT,CMD,DEL,DELTASUM,DUP,EXIT,FILTER,FLIST,FUZZY,GENR,HASH,HLINK,\n");
|
printf("CONNECT,CMD,DUP,EXIT,FUZZY,GENR,HLINK,ICONV,NSTR,OWN,TIME.\n");
|
||||||
printf("ICONV,NSTR,OWN,RECV,SEND,TIME.\n");
|
|
||||||
printf("Flags may be comma-separated, for example: --debug=io,proto\n");
|
printf("Flags may be comma-separated, for example: --debug=io,proto\n");
|
||||||
printf("An optional level suffix is accepted (e.g. --debug=io2); level 0\n");
|
printf("An optional level suffix is accepted (e.g. --debug=io2); level 0\n");
|
||||||
printf("silences that item. Unknown names are rejected.\n");
|
printf("silences that item. Unknown names are rejected.\n");
|
||||||
}
|
}
|
||||||
|
|
||||||
void print_info_usage(void) {
|
void print_info_usage(void) {
|
||||||
printf("Emitting info flags: COPY,NAME,MISC,SKIP,STATS,ALL,NONE\n");
|
printf("Emitting info flags: COPY,MISC,SKIP,STATS,DEL,REMOVE,NAME,FLIST,\n");
|
||||||
printf("Also accepted for rsync CLI parity (silent): BACKUP,DEL,FLIST,MOUNT,\n");
|
printf("NONREG,PROGRESS,MOUNT,ALL,NONE\n");
|
||||||
printf("NONREG,PROGRESS,REMOVE,SYMSAFE.\n");
|
printf("Also accepted for rsync CLI parity (silent): BACKUP,SYMS,SYMSAFE.\n");
|
||||||
printf("Flags may be comma-separated, for example: --info=name,stats\n");
|
printf("Flags may be comma-separated, for example: --info=name,stats\n");
|
||||||
printf("An optional level suffix is accepted (e.g. --info=stats2); level 0\n");
|
printf("An optional level suffix is accepted (e.g. --info=stats2); level 0\n");
|
||||||
printf("silences that item. Unknown names are rejected.\n");
|
printf("silences that item. Unknown names are rejected.\n");
|
||||||
|
|||||||
+44
-18
@@ -226,11 +226,6 @@ static void release_authorization(void) {
|
|||||||
close(root_fd);
|
close(root_fd);
|
||||||
}
|
}
|
||||||
|
|
||||||
static bool path_is_within(const char* root, const char* path) {
|
|
||||||
size_t n = strlen(root);
|
|
||||||
return strncmp(root, path, n) == 0 && (path[n] == '\0' || path[n] == '/');
|
|
||||||
}
|
|
||||||
|
|
||||||
/* --mkpath contract: when the client's destination root directory does not
|
/* --mkpath contract: when the client's destination root directory does not
|
||||||
exist yet on the server side, --mkpath tells the server to create it (and
|
exist yet on the server side, --mkpath tells the server to create it (and
|
||||||
any missing leading components) below the authorized root at connection
|
any missing leading components) below the authorized root at connection
|
||||||
@@ -790,7 +785,7 @@ void handler(int file_descriptor) {
|
|||||||
if (joined_destination)
|
if (joined_destination)
|
||||||
destination = joined_destination;
|
destination = joined_destination;
|
||||||
if (!destination || has_path_traversal(destination) ||
|
if (!destination || has_path_traversal(destination) ||
|
||||||
!path_is_within(authorized_root, destination)) {
|
!path_is_within_root(authorized_root, destination)) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Rejected destination outside authorized root");
|
log_message(LOG_LEVEL_ERROR, "Rejected destination outside authorized root");
|
||||||
free(joined_destination);
|
free(joined_destination);
|
||||||
joined_destination = NULL;
|
joined_destination = NULL;
|
||||||
@@ -1055,17 +1050,42 @@ done:
|
|||||||
#ifndef FASTSYNC_SERVER_AS_LIB
|
#ifndef FASTSYNC_SERVER_AS_LIB
|
||||||
static Server* g_server = NULL;
|
static Server* g_server = NULL;
|
||||||
|
|
||||||
|
/* Signal handler for the foreground daemon/standalone listener.
|
||||||
|
*
|
||||||
|
* Async-signal-safety: _exit(2) is on the POSIX async-signal-safe list and is
|
||||||
|
* the ONLY thing done here. The previous body called server_delete()
|
||||||
|
* (close/free/SSL_CTX_free), daemon_conf_free() and credentials_free(); none of
|
||||||
|
* those (free/malloc, and much of OpenSSL teardown) are async-signal-safe, so a
|
||||||
|
* signal delivered while the main thread was inside malloc/free could deadlock
|
||||||
|
* or corrupt the heap.
|
||||||
|
*
|
||||||
|
* Residual (documented, not hidden): the in-memory teardown is skipped on the
|
||||||
|
* signal path. That is safe because the parent daemon owns no persistent
|
||||||
|
* resource that survives process exit -- the listening socket is closed by the
|
||||||
|
* kernel, the connection registry is an anonymous MAP_SHARED mapping with no
|
||||||
|
* named backing object, and the daemon config/credential stores are plain heap
|
||||||
|
* allocations. Connection children are separate processes and handle their own
|
||||||
|
* temp files/locks. The normal (non-signal) shutdown path in main() still runs
|
||||||
|
* the full teardown, so no cleanup is dropped on the common path. Wiring the
|
||||||
|
* accept loop (transport_tcp.c, outside this change's scope) to a flag-based
|
||||||
|
* self-pipe shutdown would let the frees run context-safely; it is deliberately
|
||||||
|
* deferred rather than risk restructuring the daemon loop. */
|
||||||
static void cleanup(int sig) {
|
static void cleanup(int sig) {
|
||||||
(void)sig;
|
(void)sig;
|
||||||
if (g_server)
|
|
||||||
server_delete(&g_server);
|
|
||||||
daemon_conf_free(g_daemon_conf);
|
|
||||||
g_daemon_conf = NULL;
|
|
||||||
credentials_free(g_credentials);
|
|
||||||
g_credentials = NULL;
|
|
||||||
_exit(0);
|
_exit(0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Install a signal handler with sigaction(2) (the required async-signal-safe
|
||||||
|
* install primitive; signal(3) is not specified to be async-signal-safe). */
|
||||||
|
static void install_cleanup_handler(int signo) {
|
||||||
|
struct sigaction action;
|
||||||
|
memset(&action, 0, sizeof(action));
|
||||||
|
action.sa_handler = cleanup;
|
||||||
|
sigemptyset(&action.sa_mask);
|
||||||
|
action.sa_flags = 0;
|
||||||
|
sigaction(signo, &action, NULL);
|
||||||
|
}
|
||||||
|
|
||||||
static void print_server_usage(void) {
|
static void print_server_usage(void) {
|
||||||
printf("FastSync Server\n");
|
printf("FastSync Server\n");
|
||||||
printf("Usage: fastsync-server [options]\n\n");
|
printf("Usage: fastsync-server [options]\n\n");
|
||||||
@@ -1178,13 +1198,19 @@ static bool daemonize(void) {
|
|||||||
close(devnull);
|
close(devnull);
|
||||||
}
|
}
|
||||||
/* Do not pin the launch CWD (module-relative 'path' entries would resolve
|
/* Do not pin the launch CWD (module-relative 'path' entries would resolve
|
||||||
* against an unstable working directory) and drop the restrictive host umask
|
* against an unstable working directory). Set a conservative daemon umask
|
||||||
* so modules can create files/dirs with the modes the config requests. */
|
* of 022 (the conventional service default): rsync never forces umask 0 --
|
||||||
|
* it reads and restores the inherited umask and creates new entries as
|
||||||
|
* 0777 & ~umask / source & ~umask without -p. Forcing 0 here made every
|
||||||
|
* implied parent directory world-writable (0777) whenever -p metadata was not
|
||||||
|
* applied. 022 gives 0755 directories and source&~022 files, matching rsync
|
||||||
|
* under a normal daemon umask; -p/-a still restore the exact source mode via
|
||||||
|
* fchmod, which is unaffected by the umask. */
|
||||||
if (chdir("/") != 0)
|
if (chdir("/") != 0)
|
||||||
log_message(LOG_LEVEL_WARNING, "daemon: chdir to / failed: %s", strerror(errno));
|
log_message(LOG_LEVEL_WARNING, "daemon: chdir to / failed: %s", strerror(errno));
|
||||||
umask(0);
|
umask(022);
|
||||||
/* Refresh the cached umask: main() captured the launch umask before this
|
/* Refresh the cached umask: main() captured the launch umask before this
|
||||||
* (single-threaded) umask(0), and file_mode_base() must see the daemon's
|
* (single-threaded) umask(022), and file_mode_base() must see the daemon's
|
||||||
* actual umask. */
|
* actual umask. */
|
||||||
file_umask_capture();
|
file_umask_capture();
|
||||||
return true;
|
return true;
|
||||||
@@ -1253,8 +1279,8 @@ int main(int argc, char* argv[]) {
|
|||||||
* this process-global policy cannot be re-enabled by a future caller. */
|
* this process-global policy cannot be re-enabled by a future caller. */
|
||||||
server_allow_super = opts.allow_super && !opts.stdio_mode;
|
server_allow_super = opts.allow_super && !opts.stdio_mode;
|
||||||
server_iconv_spec = opts.iconv_spec;
|
server_iconv_spec = opts.iconv_spec;
|
||||||
signal(SIGINT, cleanup);
|
install_cleanup_handler(SIGINT);
|
||||||
signal(SIGTERM, cleanup);
|
install_cleanup_handler(SIGTERM);
|
||||||
/* Server-owned socket deadline floor: the client default --timeout=0 would
|
/* Server-owned socket deadline floor: the client default --timeout=0 would
|
||||||
* otherwise leave accepted sockets without SO_RCVTIMEO/SO_SNDTIMEO and let a
|
* otherwise leave accepted sockets without SO_RCVTIMEO/SO_SNDTIMEO and let a
|
||||||
* silent peer hold a connection (and its process slot) forever. */
|
* silent peer hold a connection (and its process slot) forever. */
|
||||||
|
|||||||
@@ -3,20 +3,12 @@
|
|||||||
#include "credentials.h"
|
#include "credentials.h"
|
||||||
#include "utils.h"
|
#include "utils.h"
|
||||||
#include <limits.h>
|
#include <limits.h>
|
||||||
#include <stdarg.h>
|
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
#include <sys/socket.h>
|
#include <sys/socket.h>
|
||||||
|
|
||||||
static void set_error(char* err, size_t err_size, const char* fmt, ...) {
|
#define set_error utils_set_error
|
||||||
if (!err || err_size == 0)
|
|
||||||
return;
|
|
||||||
va_list args;
|
|
||||||
va_start(args, fmt);
|
|
||||||
vsnprintf(err, err_size, fmt, args);
|
|
||||||
va_end(args);
|
|
||||||
}
|
|
||||||
|
|
||||||
void server_cli_options_default(ServerCliOptions* opts) {
|
void server_cli_options_default(ServerCliOptions* opts) {
|
||||||
if (!opts)
|
if (!opts)
|
||||||
|
|||||||
@@ -9,7 +9,7 @@
|
|||||||
ArrayList* array_list_create(void (*item_destroyer)(void* item)) {
|
ArrayList* array_list_create(void (*item_destroyer)(void* item)) {
|
||||||
ArrayList* list = (ArrayList*)protocol_alloc(sizeof(ArrayList));
|
ArrayList* list = (ArrayList*)protocol_alloc(sizeof(ArrayList));
|
||||||
if (list == NULL) {
|
if (list == NULL) {
|
||||||
log_perror("ERROR: Could not allocate memory for array list struct");
|
log_message(LOG_LEVEL_ERROR, "%s", "ERROR: Could not allocate memory for array list struct");
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -47,7 +47,7 @@ static bool array_list_extend(ArrayList* array_list) {
|
|||||||
new_capacity = INITIAL_ARRAY_SIZE;
|
new_capacity = INITIAL_ARRAY_SIZE;
|
||||||
void* new_items = protocol_realloc(array_list->items, new_capacity * sizeof(void*));
|
void* new_items = protocol_realloc(array_list->items, new_capacity * sizeof(void*));
|
||||||
if (new_items == NULL) {
|
if (new_items == NULL) {
|
||||||
log_perror("ERROR: Could not reallocate memory for array list items");
|
log_message(LOG_LEVEL_ERROR, "%s", "ERROR: Could not reallocate memory for array list items");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
array_list->items = new_items;
|
array_list->items = new_items;
|
||||||
@@ -73,7 +73,7 @@ void** array_list_to_array(const ArrayList* array_list) {
|
|||||||
}
|
}
|
||||||
void** array = protocol_alloc(array_list->size * sizeof(void*));
|
void** array = protocol_alloc(array_list->size * sizeof(void*));
|
||||||
if (array == NULL) {
|
if (array == NULL) {
|
||||||
log_perror("Could not malloc space for array from array list!");
|
log_message(LOG_LEVEL_ERROR, "%s", "Could not malloc space for array from array list!");
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
memcpy(array, array_list->items, array_list->size * sizeof(void*));
|
memcpy(array, array_list->items, array_list->size * sizeof(void*));
|
||||||
|
|||||||
+5
-4
@@ -17,8 +17,9 @@
|
|||||||
#include "protocol.h"
|
#include "protocol.h"
|
||||||
#include "utils.h"
|
#include "utils.h"
|
||||||
|
|
||||||
/* Maximum individual file data size within a chunk (64 MB) */
|
/* Maximum individual file data size within a chunk (64 MB). Distinct from the
|
||||||
#define MAX_FILE_DATA_SIZE (64ULL * 1024 * 1024)
|
* receiver's whole-file MAX_FILE_DATA_SIZE (256 MB) in file_receive.c. */
|
||||||
|
#define MAX_CHUNK_FILE_DATA_SIZE (64ULL * 1024 * 1024)
|
||||||
#define MAX_FILES_PER_CHUNK 65536U
|
#define MAX_FILES_PER_CHUNK 65536U
|
||||||
|
|
||||||
/* Reserve `charge` against `session`'s connection budget. This mirrors the
|
/* Reserve `charge` against `session`'s connection budget. This mirrors the
|
||||||
@@ -382,9 +383,9 @@ Chunk* chunk_deserialize(Data* data, bool use_metadata) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Reject individual file data larger than the maximum allowed size.
|
// Reject individual file data larger than the maximum allowed size.
|
||||||
if (file_data_size > MAX_FILE_DATA_SIZE) {
|
if (file_data_size > MAX_CHUNK_FILE_DATA_SIZE) {
|
||||||
log_message(LOG_LEVEL_ERROR, "File data size %zu exceeds maximum %llu", file_data_size,
|
log_message(LOG_LEVEL_ERROR, "File data size %zu exceeds maximum %llu", file_data_size,
|
||||||
(unsigned long long)MAX_FILE_DATA_SIZE);
|
(unsigned long long)MAX_CHUNK_FILE_DATA_SIZE);
|
||||||
goto error;
|
goto error;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -16,7 +16,14 @@
|
|||||||
#include <zstd.h>
|
#include <zstd.h>
|
||||||
|
|
||||||
#define INITIAL_DECOMPRESS_BUF_SIZE (1024 * 1024)
|
#define INITIAL_DECOMPRESS_BUF_SIZE (1024 * 1024)
|
||||||
#define MAX_DECOMPRESSED_SIZE (100ULL * 1024 * 1024) /* 100 MB hard ceiling */
|
|
||||||
|
/* Hard ceiling for a single decompression. The sender compresses whole files
|
||||||
|
* up to the protocol's whole-file receive bound, so the decompressor must
|
||||||
|
* accept payloads that large; referencing the protocol constant keeps the two
|
||||||
|
* bounds from drifting apart (they previously did: a 100 MB ceiling rejected
|
||||||
|
* 100-256 MB files). This remains a real bomb guard -- every allocation in the
|
||||||
|
* paths below is clamped to it -- so it must not exceed the protocol bound. */
|
||||||
|
#define MAX_DECOMPRESSED_SIZE MAX_RECEIVE_WHOLE_FILE_SIZE
|
||||||
|
|
||||||
/* rsync 3.4.1's built-in skip-compress suffix list (the `--skip-compress`
|
/* rsync 3.4.1's built-in skip-compress suffix list (the `--skip-compress`
|
||||||
* defaults, in the man page's order). rsync stores it as space-separated
|
* defaults, in the man page's order). rsync stores it as space-separated
|
||||||
@@ -637,8 +644,7 @@ static Data* zstd_decompress(Data* compressed_data, size_t maximum_size) {
|
|||||||
}
|
}
|
||||||
if (ret > 0 && output.pos == output.size) {
|
if (ret > 0 && output.pos == output.size) {
|
||||||
if (buf_size >= hard_limit || buf_size > SIZE_MAX / 2) {
|
if (buf_size >= hard_limit || buf_size > SIZE_MAX / 2) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Decompressed data exceeds %llu bytes",
|
log_message(LOG_LEVEL_ERROR, "Decompressed data exceeds %llu bytes", hard_limit);
|
||||||
(unsigned long long)MAX_DECOMPRESSED_SIZE);
|
|
||||||
data_destroy(uncompressed_data);
|
data_destroy(uncompressed_data);
|
||||||
uncompressed_data = NULL;
|
uncompressed_data = NULL;
|
||||||
goto cleanup;
|
goto cleanup;
|
||||||
|
|||||||
+26
-3
@@ -79,7 +79,7 @@ static void config_set_defaults(Config* config) {
|
|||||||
config->cvs_exclude = false;
|
config->cvs_exclude = false;
|
||||||
config->per_dir_filter = false;
|
config->per_dir_filter = false;
|
||||||
config->per_dir_filter_count = 0;
|
config->per_dir_filter_count = 0;
|
||||||
config->one_file_system = false;
|
config->one_file_system = 0;
|
||||||
config->no_implied_dirs = false;
|
config->no_implied_dirs = false;
|
||||||
config->dirs = false;
|
config->dirs = false;
|
||||||
config->rsh_command = NULL;
|
config->rsh_command = NULL;
|
||||||
@@ -230,6 +230,13 @@ Config* config_create(void) {
|
|||||||
if (!config)
|
if (!config)
|
||||||
return NULL;
|
return NULL;
|
||||||
config_set_defaults(config);
|
config_set_defaults(config);
|
||||||
|
/* config_set_defaults() dups the default server host; a failure there leaves
|
||||||
|
* server_host NULL and would crash later consumers, so fail the whole create
|
||||||
|
* (every caller already handles a NULL return). */
|
||||||
|
if (!config->server_host) {
|
||||||
|
config_delete(config);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
return config;
|
return config;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -686,9 +693,15 @@ int config_parse_ssh_dest(Config* config) {
|
|||||||
return daemon_dest_parse_error("invalid remote destination user@host (must not be empty or "
|
return daemon_dest_parse_error("invalid remote destination user@host (must not be empty or "
|
||||||
"start with '-')",
|
"start with '-')",
|
||||||
dest);
|
dest);
|
||||||
config->transport = TRANSPORT_SSH;
|
char* ssh_destination = str_dup(dest);
|
||||||
config->ssh_destination = str_dup(dest);
|
|
||||||
char* path = str_dup(colon + 1);
|
char* path = str_dup(colon + 1);
|
||||||
|
if (!ssh_destination || !path) {
|
||||||
|
free(ssh_destination);
|
||||||
|
free(path);
|
||||||
|
return daemon_dest_parse_error("out of memory parsing remote destination", dest);
|
||||||
|
}
|
||||||
|
config->transport = TRANSPORT_SSH;
|
||||||
|
config->ssh_destination = ssh_destination;
|
||||||
free(config->receive_root_directory);
|
free(config->receive_root_directory);
|
||||||
config->receive_root_directory = path;
|
config->receive_root_directory = path;
|
||||||
return 0;
|
return 0;
|
||||||
@@ -1052,6 +1065,16 @@ static bool send_protect_entries(int fd, const Config* c) {
|
|||||||
log_message(LOG_LEVEL_ERROR, "invalid filter rule: %s", err);
|
log_message(LOG_LEVEL_ERROR, "invalid filter rule: %s", err);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
/* The receiver rejects any block with more than MAX_FILTER_RULES entries as a
|
||||||
|
* protocol error; refuse to emit such a frame at all. filter_base_build()
|
||||||
|
* can expand the client rule set (cvs-exclude, merge files), so this is the
|
||||||
|
* authoritative bound, not config->filters->size. */
|
||||||
|
if (rules->count < 0 || rules->count > MAX_FILTER_RULES) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "too many filter rules: %d (maximum %d)", rules->count,
|
||||||
|
MAX_FILTER_RULES);
|
||||||
|
filter_rule_list_free(rules);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
bool ok = send_int(fd, rules->count);
|
bool ok = send_int(fd, rules->count);
|
||||||
for (int i = 0; ok && i < rules->count; i++) {
|
for (int i = 0; ok && i < rules->count; i++) {
|
||||||
const FilterRule* r = rules->items[i];
|
const FilterRule* r = rules->items[i];
|
||||||
|
|||||||
+3
-1
@@ -463,7 +463,9 @@ typedef struct Config {
|
|||||||
* /.rsync-filter' (the .rsync-filter files themselves are transferred); a
|
* /.rsync-filter' (the .rsync-filter files themselves are transferred); a
|
||||||
* repeated -F adds --filter='- .rsync-filter' so they are excluded too. */
|
* repeated -F adds --filter='- .rsync-filter' so they are excluded too. */
|
||||||
int per_dir_filter_count;
|
int per_dir_filter_count;
|
||||||
bool one_file_system; /* -x/--one-file-system: do not cross filesystem boundaries */
|
int one_file_system; /* -x/--one-file-system: do not cross filesystem boundaries.
|
||||||
|
Repeated -x (rsync's -xx) drops the mount-point
|
||||||
|
directory entirely instead of recreating it empty. */
|
||||||
/* --no-implied-dirs: client-only. With -R, do not transfer the source
|
/* --no-implied-dirs: client-only. With -R, do not transfer the source
|
||||||
* metadata of the parent directories implied by a listed path; an unlisted
|
* metadata of the parent directories implied by a listed path; an unlisted
|
||||||
* implied parent is still created (with default attributes) so the listed
|
* implied parent is still created (with default attributes) so the listed
|
||||||
|
|||||||
+204
-18
@@ -8,12 +8,13 @@
|
|||||||
#include <openssl/evp.h>
|
#include <openssl/evp.h>
|
||||||
#include <openssl/params.h>
|
#include <openssl/params.h>
|
||||||
#include <openssl/rand.h>
|
#include <openssl/rand.h>
|
||||||
#include <stdarg.h>
|
#include <poll.h>
|
||||||
#include <stdint.h>
|
#include <stdint.h>
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
#include <sys/stat.h>
|
#include <sys/stat.h>
|
||||||
|
#include <time.h>
|
||||||
#include <unistd.h>
|
#include <unistd.h>
|
||||||
|
|
||||||
/* One store entry: a username and its salted PBKDF2 verifier. The plaintext
|
/* One store entry: a username and its salted PBKDF2 verifier. The plaintext
|
||||||
@@ -58,19 +59,37 @@ struct CredentialStore {
|
|||||||
static const uint8_t k_dummy_stored_key[CREDENTIAL_KEY_LEN] = {0};
|
static const uint8_t k_dummy_stored_key[CREDENTIAL_KEY_LEN] = {0};
|
||||||
static const uint8_t k_dummy_server_key[CREDENTIAL_KEY_LEN] = {0};
|
static const uint8_t k_dummy_server_key[CREDENTIAL_KEY_LEN] = {0};
|
||||||
|
|
||||||
static void set_error(char* err, size_t err_size, const char* fmt, ...) {
|
#define set_error utils_set_error
|
||||||
if (!err || err_size == 0)
|
|
||||||
return;
|
|
||||||
va_list args;
|
|
||||||
va_start(args, fmt);
|
|
||||||
vsnprintf(err, err_size, fmt, args);
|
|
||||||
va_end(args);
|
|
||||||
}
|
|
||||||
|
|
||||||
static bool is_comment_char(char c) {
|
static bool is_comment_char(char c) {
|
||||||
return c == '#' || c == ';';
|
return c == '#' || c == ';';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* True for a literal fd-backed store path: exactly "/dev/fd/<digits>" or
|
||||||
|
* "/proc/self/fd/<digits>", with no trailing component and no "..". These name
|
||||||
|
* the calling process's own open descriptors (e.g. a bash process substitution
|
||||||
|
* `<(...)`, which passes /dev/fd/N), and both prefixes are symlinks by
|
||||||
|
* construction. */
|
||||||
|
static bool is_fd_backed_path(const char* path) {
|
||||||
|
static const char* const prefixes[] = {"/dev/fd/", "/proc/self/fd/"};
|
||||||
|
if (!path)
|
||||||
|
return false;
|
||||||
|
for (size_t i = 0; i < sizeof(prefixes) / sizeof(prefixes[0]); i++) {
|
||||||
|
const char* prefix = prefixes[i];
|
||||||
|
size_t prefix_len = strlen(prefix);
|
||||||
|
if (strncmp(path, prefix, prefix_len) != 0)
|
||||||
|
continue;
|
||||||
|
const char* digits = path + prefix_len;
|
||||||
|
if (*digits < '0' || *digits > '9')
|
||||||
|
return false;
|
||||||
|
const char* p = digits;
|
||||||
|
while (*p >= '0' && *p <= '9')
|
||||||
|
p++;
|
||||||
|
return *p == '\0';
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
/* Open a --password-file / --early-input after verifying the EXACT inode we
|
/* Open a --password-file / --early-input after verifying the EXACT inode we
|
||||||
* will read: it must be owned by the effective user and grant no group/other
|
* will read: it must be owned by the effective user and grant no group/other
|
||||||
* permission bit (so 0600 and stricter modes such as 0400 are accepted),
|
* permission bit (so 0600 and stricter modes such as 0400 are accepted),
|
||||||
@@ -80,12 +99,31 @@ static bool is_comment_char(char c) {
|
|||||||
* path and then fstat the resulting fd (rather than stat()ing the path first
|
* path and then fstat the resulting fd (rather than stat()ing the path first
|
||||||
* and reopening it), so the permission decision is made on the same inode that
|
* and reopening it), so the permission decision is made on the same inode that
|
||||||
* is read and cannot be raced by swapping the path between check and open.
|
* is read and cannot be raced by swapping the path between check and open.
|
||||||
* The path may be a process-substitution pipe (`<(...)` -> /dev/fd/N), so
|
* O_NOFOLLOW refuses a symlinked path outright (ELOOP fails closed) instead of
|
||||||
* regular files and FIFOs are accepted when the ownership/mode checks pass.
|
* following it before the owner/mode gate can run. The one exception is a
|
||||||
|
* literal fd-backed path (/dev/fd/N or /proc/self/fd/N, see
|
||||||
|
* is_fd_backed_path): those entries are symlinks to the CALLING process's own
|
||||||
|
* descriptors, so following them is not the untrusted-symlink hazard
|
||||||
|
* O_NOFOLLOW guards against, and requiring O_NOFOLLOW would break the
|
||||||
|
* documented process-substitution/FIFO usage. For them only, O_NOFOLLOW is
|
||||||
|
* omitted; the same fstat owner/mode gate still applies to the resolved inode.
|
||||||
|
* O_NONBLOCK keeps the OPEN itself from
|
||||||
|
* blocking forever on a writer-less FIFO (a blocking O_RDONLY open would wait
|
||||||
|
* for a writer). The fd is left nonblocking for FIFOs so a read never blocks
|
||||||
|
* either; the read loop (secret_read_line) absorbs the resulting EAGAIN by
|
||||||
|
* waiting, under a bounded deadline, for the writer -- this is what makes a
|
||||||
|
* slow process substitution (`--password-file <(sleep 1; ...)`) work while a
|
||||||
|
* writer-less FIFO still fails after the deadline instead of hanging. Only
|
||||||
|
* regular files and FIFOs pass the ownership/mode checks; O_NONBLOCK is
|
||||||
|
* cleared for regular files, where it is a no-op anyway and no EAGAIN can
|
||||||
|
* occur, so their stdio read path is byte-for-byte unchanged.
|
||||||
*
|
*
|
||||||
* Returns a FILE* the caller must fclose, or NULL with `err` filled. */
|
* Returns a FILE* the caller must fclose, or NULL with `err` filled. */
|
||||||
static FILE* secret_file_open(const char* path, char* err, size_t err_size) {
|
static FILE* secret_file_open(const char* path, char* err, size_t err_size) {
|
||||||
int fd = open(path, O_RDONLY | O_CLOEXEC);
|
int flags = O_RDONLY | O_NONBLOCK | O_CLOEXEC;
|
||||||
|
if (!is_fd_backed_path(path))
|
||||||
|
flags |= O_NOFOLLOW;
|
||||||
|
int fd = open(path, flags);
|
||||||
if (fd < 0) {
|
if (fd < 0) {
|
||||||
set_error(err, err_size, "cannot open secret file '%s': %s", path, strerror(errno));
|
set_error(err, err_size, "cannot open secret file '%s': %s", path, strerror(errno));
|
||||||
return NULL;
|
return NULL;
|
||||||
@@ -105,6 +143,15 @@ static FILE* secret_file_open(const char* path, char* err, size_t err_size) {
|
|||||||
close(fd);
|
close(fd);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
/* O_NONBLOCK is only meaningful for the FIFO allowance. Restore blocking
|
||||||
|
* mode on a regular file so its read path is exactly as before; a no-op on
|
||||||
|
* most systems, but explicit. Failures here are ignored: O_NONBLOCK on a
|
||||||
|
* regular file does not affect reads either way. */
|
||||||
|
if (S_ISREG(st.st_mode)) {
|
||||||
|
int status_flags = fcntl(fd, F_GETFL);
|
||||||
|
if (status_flags >= 0)
|
||||||
|
(void)fcntl(fd, F_SETFL, status_flags & ~O_NONBLOCK);
|
||||||
|
}
|
||||||
FILE* fp = fdopen(fd, "r");
|
FILE* fp = fdopen(fd, "r");
|
||||||
if (!fp) {
|
if (!fp) {
|
||||||
set_error(err, err_size, "cannot read secret file '%s': %s", path, strerror(errno));
|
set_error(err, err_size, "cannot read secret file '%s': %s", path, strerror(errno));
|
||||||
@@ -114,6 +161,123 @@ static FILE* secret_file_open(const char* path, char* err, size_t err_size) {
|
|||||||
return fp;
|
return fp;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Overall bound on how long the reader waits for a process-substitution/FIFO
|
||||||
|
* writer to produce data before giving up. It must comfortably exceed a
|
||||||
|
* producer's startup delay (e.g. `--password-file <(sleep 1; ...)`) while still
|
||||||
|
* bounding a writer-less FIFO, so a stray or hostile FIFO cannot stall the
|
||||||
|
* daemon or client indefinitely. */
|
||||||
|
#define CREDENTIAL_FIFO_READ_TIMEOUT_MS 3000
|
||||||
|
|
||||||
|
/* Monotonic milliseconds, used only for the read deadline (wall-clock changes
|
||||||
|
* must not extend or shorten the wait). */
|
||||||
|
static int64_t credential_monotonic_ms(void) {
|
||||||
|
struct timespec ts;
|
||||||
|
if (clock_gettime(CLOCK_MONOTONIC, &ts) != 0)
|
||||||
|
return 0;
|
||||||
|
return (int64_t)ts.tv_sec * 1000 + (int64_t)(ts.tv_nsec / 1000000);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Wait until `fd` is readable or the deadline passes. Returns true when it is
|
||||||
|
* readable, false on timeout or a poll error (err filled). EINTR is retried
|
||||||
|
* against the same deadline, so signals cannot extend the wait. */
|
||||||
|
static bool credential_wait_readable(int fd, int64_t deadline, const char* label, const char* path,
|
||||||
|
char* err, size_t err_size) {
|
||||||
|
for (;;) {
|
||||||
|
int64_t remaining = deadline - credential_monotonic_ms();
|
||||||
|
if (remaining <= 0)
|
||||||
|
break;
|
||||||
|
if (remaining > INT_MAX)
|
||||||
|
remaining = INT_MAX;
|
||||||
|
struct pollfd pfd = {.fd = fd, .events = POLLIN, .revents = 0};
|
||||||
|
int rc = poll(&pfd, 1, (int)remaining);
|
||||||
|
if (rc > 0)
|
||||||
|
return true;
|
||||||
|
if (rc == 0)
|
||||||
|
break;
|
||||||
|
if (errno != EINTR) {
|
||||||
|
set_error(err, err_size, "error waiting for %s '%s': %s", label, path, strerror(errno));
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
set_error(err, err_size, "timed out after %d ms waiting for %s '%s'",
|
||||||
|
CREDENTIAL_FIFO_READ_TIMEOUT_MS, label, path);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
typedef enum {
|
||||||
|
SECRET_READ_LINE,
|
||||||
|
SECRET_READ_EOF,
|
||||||
|
SECRET_READ_ERROR,
|
||||||
|
} SecretReadResult;
|
||||||
|
|
||||||
|
/* Read one complete line from `fp` into `line` (capacity `cap`), including the
|
||||||
|
* trailing newline when present and always NUL-terminating. `*out_len`
|
||||||
|
* receives strlen(line).
|
||||||
|
*
|
||||||
|
* A regular file is read exactly as before: secret_file_open leaves it
|
||||||
|
* blocking, so fgets never sees EAGAIN. A FIFO stays nonblocking, so fgets
|
||||||
|
* returns NULL (or a partial line) with EAGAIN while the writer is still
|
||||||
|
* starting up; instead of treating that as a fatal error the loop clearerr()s
|
||||||
|
* and polls for readability against one overall deadline. The `used`
|
||||||
|
* accumulator reassembles a line that arrived in several write()s into a single
|
||||||
|
* line, so a split write is not misparsed as two entries.
|
||||||
|
*
|
||||||
|
* Returns SECRET_READ_LINE, SECRET_READ_EOF, or SECRET_READ_ERROR (err filled)
|
||||||
|
* on timeout or a genuine read error. */
|
||||||
|
static SecretReadResult secret_read_line(char* line, size_t cap, FILE* fp, const char* label,
|
||||||
|
const char* path, size_t* out_len, char* err,
|
||||||
|
size_t err_size) {
|
||||||
|
int fd = fileno(fp);
|
||||||
|
int64_t deadline = credential_monotonic_ms() + CREDENTIAL_FIFO_READ_TIMEOUT_MS;
|
||||||
|
size_t used = 0;
|
||||||
|
line[0] = '\0';
|
||||||
|
for (;;) {
|
||||||
|
errno = 0;
|
||||||
|
if (fgets(line + used, (int)(cap - used), fp)) {
|
||||||
|
used += strlen(line + used);
|
||||||
|
if (used > 0 && line[used - 1] == '\n') {
|
||||||
|
*out_len = used;
|
||||||
|
return SECRET_READ_LINE;
|
||||||
|
}
|
||||||
|
if (feof(fp)) {
|
||||||
|
*out_len = used; /* final unterminated line */
|
||||||
|
return SECRET_READ_LINE;
|
||||||
|
}
|
||||||
|
/* No newline and not EOF. A full buffer is the caller's over-long-line
|
||||||
|
* case; otherwise the line is only partially available (a nonblocking
|
||||||
|
* FIFO under a slow writer), so any genuine read error fails and anything
|
||||||
|
* else waits for the rest. */
|
||||||
|
if (used >= cap - 1) {
|
||||||
|
*out_len = used;
|
||||||
|
return SECRET_READ_LINE;
|
||||||
|
}
|
||||||
|
int e = ferror(fp) ? errno : 0;
|
||||||
|
if (e != 0 && e != EAGAIN && e != EWOULDBLOCK) {
|
||||||
|
set_error(err, err_size, "error reading %s '%s': %s", label, path, strerror(e));
|
||||||
|
return SECRET_READ_ERROR;
|
||||||
|
}
|
||||||
|
clearerr(fp);
|
||||||
|
if (!credential_wait_readable(fd, deadline, label, path, err, err_size))
|
||||||
|
return SECRET_READ_ERROR;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
/* fgets returned NULL: EOF, a not-yet-readable FIFO, or a real error. */
|
||||||
|
if (feof(fp)) {
|
||||||
|
*out_len = used;
|
||||||
|
return used > 0 ? SECRET_READ_LINE : SECRET_READ_EOF;
|
||||||
|
}
|
||||||
|
if (errno == EAGAIN || errno == EWOULDBLOCK) {
|
||||||
|
clearerr(fp);
|
||||||
|
if (!credential_wait_readable(fd, deadline, label, path, err, err_size))
|
||||||
|
return SECRET_READ_ERROR;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
set_error(err, err_size, "error reading %s '%s': %s", label, path,
|
||||||
|
errno != 0 ? strerror(errno) : "read failed");
|
||||||
|
return SECRET_READ_ERROR;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/* Trim leading/trailing ASCII space and tab in place; returns the new start. */
|
/* Trim leading/trailing ASCII space and tab in place; returns the new start. */
|
||||||
static char* trim_space(char* s) {
|
static char* trim_space(char* s) {
|
||||||
while (*s == ' ' || *s == '\t')
|
while (*s == ' ' || *s == '\t')
|
||||||
@@ -509,9 +673,17 @@ static CredentialStore* load_store_file(const char* path, char* err, size_t err_
|
|||||||
char line[CREDENTIAL_MAX_LINE + 2];
|
char line[CREDENTIAL_MAX_LINE + 2];
|
||||||
bool ok = true;
|
bool ok = true;
|
||||||
|
|
||||||
while (fgets(line, sizeof(line), fp)) {
|
for (;;) {
|
||||||
|
size_t len = 0;
|
||||||
|
SecretReadResult rr =
|
||||||
|
secret_read_line(line, sizeof(line), fp, "credential file", path, &len, err, err_size);
|
||||||
|
if (rr == SECRET_READ_EOF)
|
||||||
|
break;
|
||||||
|
if (rr == SECRET_READ_ERROR) {
|
||||||
|
ok = false;
|
||||||
|
break;
|
||||||
|
}
|
||||||
line_no++;
|
line_no++;
|
||||||
size_t len = strlen(line);
|
|
||||||
if (len == CREDENTIAL_MAX_LINE + 1 && line[len - 1] != '\n' && !feof(fp)) {
|
if (len == CREDENTIAL_MAX_LINE + 1 && line[len - 1] != '\n' && !feof(fp)) {
|
||||||
set_error(err, err_size, "credential file '%s' line %d exceeds the %d-byte limit", path,
|
set_error(err, err_size, "credential file '%s' line %d exceeds the %d-byte limit", path,
|
||||||
line_no, CREDENTIAL_MAX_LINE);
|
line_no, CREDENTIAL_MAX_LINE);
|
||||||
@@ -1148,9 +1320,17 @@ int credentials_hash_file(const char* path, uint32_t iters, FILE* out, char* err
|
|||||||
int line_no = 0;
|
int line_no = 0;
|
||||||
int result = 0;
|
int result = 0;
|
||||||
char line[CREDENTIAL_MAX_LINE + 2];
|
char line[CREDENTIAL_MAX_LINE + 2];
|
||||||
while (fgets(line, sizeof(line), fp)) {
|
for (;;) {
|
||||||
|
size_t len = 0;
|
||||||
|
SecretReadResult rr =
|
||||||
|
secret_read_line(line, sizeof(line), fp, "plaintext file", path, &len, err, err_size);
|
||||||
|
if (rr == SECRET_READ_EOF)
|
||||||
|
break;
|
||||||
|
if (rr == SECRET_READ_ERROR) {
|
||||||
|
result = -1;
|
||||||
|
break;
|
||||||
|
}
|
||||||
line_no++;
|
line_no++;
|
||||||
size_t len = strlen(line);
|
|
||||||
if (len == CREDENTIAL_MAX_LINE + 1 && line[len - 1] != '\n' && !feof(fp)) {
|
if (len == CREDENTIAL_MAX_LINE + 1 && line[len - 1] != '\n' && !feof(fp)) {
|
||||||
set_error(err, err_size, "plaintext file '%s' line %d exceeds the %d-byte limit", path,
|
set_error(err, err_size, "plaintext file '%s' line %d exceeds the %d-byte limit", path,
|
||||||
line_no, CREDENTIAL_MAX_LINE);
|
line_no, CREDENTIAL_MAX_LINE);
|
||||||
@@ -1228,9 +1408,15 @@ int credentials_read_secret_file(const char* path, char** user_out, char** passw
|
|||||||
char line[CREDENTIAL_MAX_LINE + 2];
|
char line[CREDENTIAL_MAX_LINE + 2];
|
||||||
int result = -1;
|
int result = -1;
|
||||||
|
|
||||||
while (fgets(line, sizeof(line), fp)) {
|
for (;;) {
|
||||||
|
size_t len = 0;
|
||||||
|
SecretReadResult rr =
|
||||||
|
secret_read_line(line, sizeof(line), fp, "password file", path, &len, err, err_size);
|
||||||
|
if (rr == SECRET_READ_EOF)
|
||||||
|
break;
|
||||||
|
if (rr == SECRET_READ_ERROR)
|
||||||
|
goto done;
|
||||||
line_no++;
|
line_no++;
|
||||||
size_t len = strlen(line);
|
|
||||||
if (len == CREDENTIAL_MAX_LINE + 1 && line[len - 1] != '\n' && !feof(fp)) {
|
if (len == CREDENTIAL_MAX_LINE + 1 && line[len - 1] != '\n' && !feof(fp)) {
|
||||||
set_error(err, err_size, "password file '%s' line %d exceeds the %d-byte limit", path,
|
set_error(err, err_size, "password file '%s' line %d exceeds the %d-byte limit", path,
|
||||||
line_no, CREDENTIAL_MAX_LINE);
|
line_no, CREDENTIAL_MAX_LINE);
|
||||||
|
|||||||
@@ -6,7 +6,6 @@
|
|||||||
#include <errno.h>
|
#include <errno.h>
|
||||||
#include <limits.h>
|
#include <limits.h>
|
||||||
#include <netinet/in.h>
|
#include <netinet/in.h>
|
||||||
#include <stdarg.h>
|
|
||||||
#include <stdint.h>
|
#include <stdint.h>
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
@@ -17,14 +16,7 @@
|
|||||||
/* helpers */
|
/* helpers */
|
||||||
/* ------------------------------------------------------------------ */
|
/* ------------------------------------------------------------------ */
|
||||||
|
|
||||||
static void set_error(char* err, size_t err_size, const char* fmt, ...) {
|
#define set_error utils_set_error
|
||||||
if (!err || err_size == 0)
|
|
||||||
return;
|
|
||||||
va_list args;
|
|
||||||
va_start(args, fmt);
|
|
||||||
vsnprintf(err, err_size, fmt, args);
|
|
||||||
va_end(args);
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Trim leading and trailing ASCII space/tab in place; returns the new start. */
|
/* Trim leading and trailing ASCII space/tab in place; returns the new start. */
|
||||||
static char* trim_ws(char* s) {
|
static char* trim_ws(char* s) {
|
||||||
|
|||||||
+109
-52
@@ -432,6 +432,17 @@ int delete_plan_send_remaining(int fd, DeletePlanSender* sender, const ArrayList
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
int delete_plan_send_all(int fd, DeletePlanSender* sender, const ArrayList* dirs) {
|
||||||
|
if (!sender)
|
||||||
|
return -1;
|
||||||
|
/* Root first: this also transmits the one-shot per-run config block on its
|
||||||
|
own carrier frame (see send_config_only), so it reaches the receiver even
|
||||||
|
when the scope permits no directory plan at all. */
|
||||||
|
if (delete_plan_send_root(fd, sender) != 0)
|
||||||
|
return -1;
|
||||||
|
return delete_plan_send_remaining(fd, sender, dirs);
|
||||||
|
}
|
||||||
|
|
||||||
/* ------------------------------------------------------------------ */
|
/* ------------------------------------------------------------------ */
|
||||||
/* Receiver: delete session */
|
/* Receiver: delete session */
|
||||||
/* ------------------------------------------------------------------ */
|
/* ------------------------------------------------------------------ */
|
||||||
@@ -471,6 +482,24 @@ static void notify_deleted(DeletePlanSession* session, const char* rel) {
|
|||||||
session->observer(session->observer_context, rel);
|
session->observer(session->observer_context, rel);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* A removed directory is reported with rsync's trailing slash (`deleting dir/`)
|
||||||
|
while files keep their bare path. */
|
||||||
|
static void notify_deleted_dir(DeletePlanSession* session, const char* rel) {
|
||||||
|
if (!session || !session->observer || !rel)
|
||||||
|
return;
|
||||||
|
size_t len = strlen(rel);
|
||||||
|
char* with_slash = malloc(len + 2);
|
||||||
|
if (!with_slash) {
|
||||||
|
session->observer(session->observer_context, rel);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
memcpy(with_slash, rel, len);
|
||||||
|
with_slash[len] = '/';
|
||||||
|
with_slash[len + 1] = '\0';
|
||||||
|
session->observer(session->observer_context, with_slash);
|
||||||
|
free(with_slash);
|
||||||
|
}
|
||||||
|
|
||||||
DeletePlanSession* delete_plan_session_create(const Config* config) {
|
DeletePlanSession* delete_plan_session_create(const Config* config) {
|
||||||
if (!config)
|
if (!config)
|
||||||
return NULL;
|
return NULL;
|
||||||
@@ -696,7 +725,7 @@ static bool process_extra_dir(int dirfd, const char* name, const char* child_rel
|
|||||||
session->deleted++;
|
session->deleted++;
|
||||||
session->planned++;
|
session->planned++;
|
||||||
log_deleted(child_rel);
|
log_deleted(child_rel);
|
||||||
notify_deleted(session, child_rel);
|
notify_deleted_dir(session, child_rel);
|
||||||
*removed = true;
|
*removed = true;
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
@@ -733,81 +762,108 @@ static bool process_children(int dirfd, const char* dir_rel, const ArrayList* ke
|
|||||||
const ArrayList* keep_files, bool at_root, bool force_now,
|
const ArrayList* keep_files, bool at_root, bool force_now,
|
||||||
const PlanSkips* skips, DeletePlanSession* session, bool* survives) {
|
const PlanSkips* skips, DeletePlanSession* session, bool* survives) {
|
||||||
*survives = false;
|
*survives = false;
|
||||||
int scanfd = openat(dirfd, ".", O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
DeleteDirEntry* entries = NULL;
|
||||||
if (scanfd < 0)
|
size_t count = 0;
|
||||||
|
bool collect_ok = true;
|
||||||
|
if (!delete_dir_entries_collect(dirfd, &entries, &count, &collect_ok))
|
||||||
return false;
|
return false;
|
||||||
DIR* dir = fdopendir(scanfd);
|
bool operation_ok = collect_ok;
|
||||||
if (!dir) {
|
bool local_survives = false;
|
||||||
close(scanfd);
|
bool* shielded = calloc(count ? count : 1, sizeof(bool));
|
||||||
|
bool* is_extra = calloc(count ? count : 1, sizeof(bool));
|
||||||
|
bool* force = calloc(count ? count : 1, sizeof(bool));
|
||||||
|
if (!shielded || !is_extra || !force) {
|
||||||
|
free(shielded);
|
||||||
|
free(is_extra);
|
||||||
|
free(force);
|
||||||
|
delete_dir_entries_free(entries, count);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
bool operation_ok = true;
|
|
||||||
bool local_survives = false;
|
/* rsync's order: extraneous subdirectories in descending name order, then
|
||||||
const struct dirent* entry;
|
extraneous files in descending name order (kept entries survive and are not
|
||||||
while ((entry = readdir(dir)) != NULL) {
|
touched here — a kept subdirectory gets its own per-directory plan). */
|
||||||
if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
|
if (count > 1)
|
||||||
continue;
|
qsort(entries, count, sizeof(*entries), delete_dir_entry_cmp_desc);
|
||||||
|
size_t dir_count = 0;
|
||||||
|
while (dir_count < count && entries[dir_count].is_dir)
|
||||||
|
dir_count++;
|
||||||
|
|
||||||
|
for (size_t i = 0; i < count; i++) {
|
||||||
char* child_rel =
|
char* child_rel =
|
||||||
(strcmp(dir_rel, ".") == 0) ? str_dup(entry->d_name) : path_cat(dir_rel, entry->d_name);
|
(strcmp(dir_rel, ".") == 0) ? str_dup(entries[i].name) : path_cat(dir_rel, entries[i].name);
|
||||||
if (!child_rel) {
|
if (!child_rel) {
|
||||||
operation_ok = false;
|
operation_ok = false;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if (path_under_skip_prefix(child_rel, at_root, skips->entries, skips->count)) {
|
if (path_under_skip_prefix(child_rel, at_root, skips->entries, skips->count)) {
|
||||||
|
shielded[i] = true;
|
||||||
local_survives = true;
|
local_survives = true;
|
||||||
free(child_rel);
|
free(child_rel);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
struct stat st;
|
bool is_dir = entries[i].is_dir;
|
||||||
if (fstatat(dirfd, entry->d_name, &st, AT_SYMLINK_NOFOLLOW) != 0) {
|
bool in_keep_dirs = is_dir && list_contains_str(keep_dirs, entries[i].name);
|
||||||
if (errno != ENOENT)
|
bool in_keep_files = !is_dir && list_contains_str(keep_files, entries[i].name);
|
||||||
operation_ok = false;
|
|
||||||
free(child_rel);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
bool is_dir = S_ISDIR(st.st_mode);
|
|
||||||
bool in_keep_dirs = is_dir && list_contains_str(keep_dirs, entry->d_name);
|
|
||||||
bool in_keep_files = !is_dir && list_contains_str(keep_files, entry->d_name);
|
|
||||||
bool rule_protected =
|
bool rule_protected =
|
||||||
skips->protect_rules &&
|
skips->protect_rules &&
|
||||||
filter_rules_apply_side(skips->protect_rules, child_rel, entry->d_name, is_dir,
|
filter_rules_apply_side(skips->protect_rules, child_rel, entries[i].name, is_dir,
|
||||||
FILTER_SIDE_RECEIVER) == FILTER_ACTION_PROTECT;
|
FILTER_SIDE_RECEIVER) == FILTER_ACTION_PROTECT;
|
||||||
if (in_keep_dirs) {
|
if (in_keep_dirs || in_keep_files || rule_protected) {
|
||||||
local_survives = true;
|
shielded[i] = true;
|
||||||
} else if (keep_dirs && !is_dir && list_contains_str(keep_dirs, entry->d_name)) {
|
|
||||||
/* Destination file blocks a source directory: clear it now, whatever the
|
|
||||||
delete timing, so the directory can be created. */
|
|
||||||
if (!process_extra_file(dirfd, entry->d_name, child_rel, true, session))
|
|
||||||
operation_ok = false;
|
|
||||||
} else if (in_keep_files) {
|
|
||||||
local_survives = true;
|
|
||||||
} else if (keep_files && is_dir && list_contains_str(keep_files, entry->d_name)) {
|
|
||||||
/* Destination directory blocks a source file: remove it now. */
|
|
||||||
bool removed = false;
|
|
||||||
if (!process_extra_dir(dirfd, entry->d_name, child_rel, true, skips, session, &removed))
|
|
||||||
operation_ok = false;
|
|
||||||
else if (!removed)
|
|
||||||
local_survives = true;
|
local_survives = true;
|
||||||
} else if (is_dir) {
|
} else if (is_dir) {
|
||||||
if (rule_protected) {
|
/* A destination directory blocks a source file of the same name: remove
|
||||||
local_survives = true;
|
it now, whatever the delete timing, so the file can be created. */
|
||||||
|
is_extra[i] = true;
|
||||||
|
force[i] = keep_files && list_contains_str(keep_files, entries[i].name);
|
||||||
} else {
|
} else {
|
||||||
|
/* A destination file blocks a source directory of the same name: clear it
|
||||||
|
now so the directory can be created. */
|
||||||
|
is_extra[i] = true;
|
||||||
|
force[i] = keep_dirs && list_contains_str(keep_dirs, entries[i].name);
|
||||||
|
}
|
||||||
|
free(child_rel);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Pass 1: extraneous subdirectories, descending. */
|
||||||
|
for (size_t i = 0; i < dir_count; i++) {
|
||||||
|
if (!is_extra[i])
|
||||||
|
continue;
|
||||||
|
char* child_rel =
|
||||||
|
(strcmp(dir_rel, ".") == 0) ? str_dup(entries[i].name) : path_cat(dir_rel, entries[i].name);
|
||||||
|
if (!child_rel) {
|
||||||
|
operation_ok = false;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
bool removed = false;
|
bool removed = false;
|
||||||
if (!process_extra_dir(dirfd, entry->d_name, child_rel, force_now, skips, session,
|
if (!process_extra_dir(dirfd, entries[i].name, child_rel, force[i] || force_now, skips, session,
|
||||||
&removed))
|
&removed))
|
||||||
operation_ok = false;
|
operation_ok = false;
|
||||||
else if (!removed)
|
else if (!removed)
|
||||||
local_survives = true;
|
local_survives = true;
|
||||||
}
|
|
||||||
} else if (rule_protected) {
|
|
||||||
local_survives = true;
|
|
||||||
} else {
|
|
||||||
if (!process_extra_file(dirfd, entry->d_name, child_rel, force_now, session))
|
|
||||||
operation_ok = false;
|
|
||||||
}
|
|
||||||
free(child_rel);
|
free(child_rel);
|
||||||
}
|
}
|
||||||
closedir(dir);
|
|
||||||
|
/* Pass 2: extraneous files, descending. */
|
||||||
|
for (size_t i = dir_count; i < count; i++) {
|
||||||
|
if (!is_extra[i])
|
||||||
|
continue;
|
||||||
|
char* child_rel =
|
||||||
|
(strcmp(dir_rel, ".") == 0) ? str_dup(entries[i].name) : path_cat(dir_rel, entries[i].name);
|
||||||
|
if (!child_rel) {
|
||||||
|
operation_ok = false;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (!process_extra_file(dirfd, entries[i].name, child_rel, force[i] || force_now, session))
|
||||||
|
operation_ok = false;
|
||||||
|
free(child_rel);
|
||||||
|
}
|
||||||
|
|
||||||
|
free(shielded);
|
||||||
|
free(is_extra);
|
||||||
|
free(force);
|
||||||
|
delete_dir_entries_free(entries, count);
|
||||||
*survives = local_survives;
|
*survives = local_survives;
|
||||||
return operation_ok;
|
return operation_ok;
|
||||||
}
|
}
|
||||||
@@ -932,10 +988,11 @@ static bool apply_deferred_path(DeletePlanSession* session, const Config* config
|
|||||||
return false;
|
return false;
|
||||||
char* leaf = NULL;
|
char* leaf = NULL;
|
||||||
int parent_fd = file_open_secure_parent(full, &leaf, false);
|
int parent_fd = file_open_secure_parent(full, &leaf, false);
|
||||||
|
int open_errno = errno;
|
||||||
free(full);
|
free(full);
|
||||||
if (parent_fd < 0) {
|
if (parent_fd < 0) {
|
||||||
free(leaf);
|
free(leaf);
|
||||||
return errno == ENOENT || errno == ENOTDIR;
|
return open_errno == ENOENT || open_errno == ENOTDIR;
|
||||||
}
|
}
|
||||||
struct stat st;
|
struct stat st;
|
||||||
if (fstatat(parent_fd, leaf, &st, AT_SYMLINK_NOFOLLOW) != 0) {
|
if (fstatat(parent_fd, leaf, &st, AT_SYMLINK_NOFOLLOW) != 0) {
|
||||||
@@ -981,7 +1038,7 @@ static bool apply_deferred_path(DeletePlanSession* session, const Config* config
|
|||||||
session->deleted++;
|
session->deleted++;
|
||||||
session->planned++;
|
session->planned++;
|
||||||
log_deleted(rel);
|
log_deleted(rel);
|
||||||
notify_deleted(session, rel);
|
notify_deleted_dir(session, rel);
|
||||||
} else if (errno != ENOENT && errno != ENOTEMPTY && errno != EEXIST) {
|
} else if (errno != ENOENT && errno != ENOTEMPTY && errno != EEXIST) {
|
||||||
close(parent_fd);
|
close(parent_fd);
|
||||||
free(leaf);
|
free(leaf);
|
||||||
|
|||||||
@@ -61,9 +61,19 @@ int delete_plan_send_root(int fd, DeletePlanSender* sender);
|
|||||||
* for `path` itself; already-sent plans are skipped. */
|
* for `path` itself; already-sent plans are skipped. */
|
||||||
int delete_plan_send_for_path(int fd, DeletePlanSender* sender, const char* path, bool is_dir);
|
int delete_plan_send_for_path(int fd, DeletePlanSender* sender, const char* path, bool is_dir);
|
||||||
/* Send the plan for every directory in `dirs` that has not been transmitted
|
/* Send the plan for every directory in `dirs` that has not been transmitted
|
||||||
* yet. Called after the data stream so an empty source directory's plan still
|
* yet. */
|
||||||
* clears its destination extras even though no file frame triggered it. */
|
|
||||||
int delete_plan_send_remaining(int fd, DeletePlanSender* sender, const ArrayList* dirs);
|
int delete_plan_send_remaining(int fd, DeletePlanSender* sender, const ArrayList* dirs);
|
||||||
|
/* Transmit the COMPLETE per-directory plan set in one pass, before any data
|
||||||
|
* frame: the root plan (with the one-shot per-run config block on its carrier
|
||||||
|
* frame) followed by every directory in `dirs`. Because the whole plan set is
|
||||||
|
* known from the path-only pre-scan, sending it all up front means a
|
||||||
|
* mid-transfer abort has already applied every planned removal, matching
|
||||||
|
* rsync's generator (which runs ahead of its throttled sender). A completed
|
||||||
|
* run is unaffected. `dirs` is the set of directories whose direct children
|
||||||
|
* were enumerated (the scanner's plan_dirs sink), so a merely listed but
|
||||||
|
* untraversed directory never gets a plan and its mirror is left intact.
|
||||||
|
* Returns -1 on I/O error. */
|
||||||
|
int delete_plan_send_all(int fd, DeletePlanSender* sender, const ArrayList* dirs);
|
||||||
|
|
||||||
/* ---- Receiver: delete session ---- */
|
/* ---- Receiver: delete session ---- */
|
||||||
|
|
||||||
|
|||||||
+43
-16
@@ -25,13 +25,6 @@
|
|||||||
#include "utils.h"
|
#include "utils.h"
|
||||||
#include "protocol.h"
|
#include "protocol.h"
|
||||||
#include "xattr.h"
|
#include "xattr.h"
|
||||||
#include <fcntl.h>
|
|
||||||
#include <unistd.h>
|
|
||||||
|
|
||||||
/* Files larger than this are not loaded whole for transfer (the sender streams
|
|
||||||
* them); a whole-file digest is computed from the path instead. Kept in sync
|
|
||||||
* with the sender's streaming threshold. */
|
|
||||||
#define STREAM_THRESHOLD (64ULL * 1024 * 1024)
|
|
||||||
|
|
||||||
static bool write_all(int fd, const void* data, unsigned long long size) {
|
static bool write_all(int fd, const void* data, unsigned long long size) {
|
||||||
const unsigned char* p = data;
|
const unsigned char* p = data;
|
||||||
@@ -1136,17 +1129,51 @@ int file_open_private_dir(const char* dir_path) {
|
|||||||
return fd;
|
return fd;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Open a --temp-dir scratch directory exactly as rsync does: the directory must
|
/* Open a --temp-dir scratch directory. The directory must already exist (rsync
|
||||||
* already exist and is used as given (an absolute path is used verbatim, a
|
* never creates it); a relative path was already resolved against the
|
||||||
* relative one was already resolved against the destination root by the
|
* destination root by the caller. Unlike file_open_private_dir this neither
|
||||||
* caller). Unlike file_open_private_dir this neither creates it nor confines
|
* creates it nor requires it to be a direct child of the receive root, because
|
||||||
* it below the receive root, because rsync accepts any temp dir -- including
|
* rsync permits a scratch dir that (via a symlink) lands on another filesystem
|
||||||
* one outside the destination tree or on another filesystem. Returns an
|
* -- but it MUST resolve inside the authorized receive root. The directory is
|
||||||
* O_DIRECTORY|O_CLOEXEC fd, or -1 on error. */
|
* opened following symlinks and then judged by the REAL path of the opened fd
|
||||||
|
* (through /proc/self/fd), so a client-planted symlink under the receive root
|
||||||
|
* can never redirect receiver scratch files outside the sandbox while an
|
||||||
|
* in-root link to another filesystem (the EXDEV fallback case) still works.
|
||||||
|
* Returns an O_DIRECTORY|O_CLOEXEC fd, or -1 on error (errno set; an escaping
|
||||||
|
* target is reported as EACCES with a logged reason). */
|
||||||
int file_open_temp_dir(const char* dir_path) {
|
int file_open_temp_dir(const char* dir_path) {
|
||||||
if (!dir_path)
|
if (!dir_path)
|
||||||
return -1;
|
return -1;
|
||||||
return open(dir_path, O_RDONLY | O_DIRECTORY | O_CLOEXEC);
|
int fd = open(dir_path, O_RDONLY | O_DIRECTORY | O_CLOEXEC);
|
||||||
|
if (fd < 0)
|
||||||
|
return -1;
|
||||||
|
const char* root = utils_get_authorized_root_path();
|
||||||
|
if (!root) {
|
||||||
|
/* No authorized root (e.g. a local batch apply): nothing to confine
|
||||||
|
against, so preserve the historical open-as-given behavior. */
|
||||||
|
return fd;
|
||||||
|
}
|
||||||
|
char fd_path[64];
|
||||||
|
int fd_path_length = snprintf(fd_path, sizeof(fd_path), "/proc/self/fd/%d", fd);
|
||||||
|
char resolved[PATH_MAX];
|
||||||
|
if (fd_path_length < 0 || (size_t)fd_path_length >= sizeof(fd_path) ||
|
||||||
|
!realpath(fd_path, resolved)) {
|
||||||
|
int saved_errno = errno;
|
||||||
|
close(fd);
|
||||||
|
errno = saved_errno;
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
if (!path_is_within_root(root, resolved)) {
|
||||||
|
char* escaped = output_escape(dir_path, log_get_8_bit_output());
|
||||||
|
log_message(LOG_LEVEL_ERROR,
|
||||||
|
"--temp-dir '%s' resolves outside the authorized receive root; refusing",
|
||||||
|
escaped ? escaped : "<allocation failed>");
|
||||||
|
free(escaped);
|
||||||
|
close(fd);
|
||||||
|
errno = EACCES;
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
return fd;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* After the content and mode/times are restored on the just-written file, apply
|
/* After the content and mode/times are restored on the just-written file, apply
|
||||||
@@ -1859,6 +1886,6 @@ bool file_write_to_disk(const char* path, const void* data, unsigned long long d
|
|||||||
bool inplace, bool sparse) {
|
bool inplace, bool sparse) {
|
||||||
if (!path || (!data && data_size != 0) || has_path_traversal(path))
|
if (!path || (!data && data_size != 0) || has_path_traversal(path))
|
||||||
return false;
|
return false;
|
||||||
FileAttrPolicy policy = {false, false, false, false};
|
FileAttrPolicy policy = {0};
|
||||||
return file_to_disk_secure(path, data, data_size, inplace, sparse, false, NULL, policy, NULL);
|
return file_to_disk_secure(path, data, data_size, inplace, sparse, false, NULL, policy, NULL);
|
||||||
}
|
}
|
||||||
+6
-2
@@ -103,8 +103,12 @@ bool file_remove_tree_secure(const char* path);
|
|||||||
the authorized root. Used for the --delay-updates staging directory. */
|
the authorized root. Used for the --delay-updates staging directory. */
|
||||||
int file_open_private_dir(const char* dir_path);
|
int file_open_private_dir(const char* dir_path);
|
||||||
|
|
||||||
/* Open an existing --temp-dir scratch directory as-is (absolute or relative;
|
/* Open an existing --temp-dir scratch directory (relative or absolute; no
|
||||||
no creation, no root confinement), matching rsync's --temp-dir handling. */
|
creation). When an authorized receive root is configured the directory's
|
||||||
|
REAL path (symlinks resolved) must lie within it, so a client-planted
|
||||||
|
symlink cannot redirect receiver scratch files outside the sandbox; an
|
||||||
|
in-root symlink to another filesystem is still allowed for rsync's EXDEV
|
||||||
|
fallback. */
|
||||||
int file_open_temp_dir(const char* dir_path);
|
int file_open_temp_dir(const char* dir_path);
|
||||||
|
|
||||||
/* The file_to_disk_secure* variants write a temporary copy in the destination
|
/* The file_to_disk_secure* variants write a temporary copy in the destination
|
||||||
|
|||||||
@@ -29,6 +29,12 @@ typedef struct FileAttrPolicy {
|
|||||||
bool times; /* config->preserve_times: apply the source mtime */
|
bool times; /* config->preserve_times: apply the source mtime */
|
||||||
bool atimes; /* config->preserve_atimes (-U): apply the source atime */
|
bool atimes; /* config->preserve_atimes (-U): apply the source atime */
|
||||||
bool executability; /* config->use_executability (-E): exec-bits-only mode */
|
bool executability; /* config->use_executability (-E): exec-bits-only mode */
|
||||||
|
/* privilege_super_mode_permitted(): when false (SUPER_MODE_OFF / --no-super,
|
||||||
|
or a daemon that did not grant `client owner = yes`), the setuid/setgid/
|
||||||
|
sticky bits are stripped from every applied mode (source mode and any
|
||||||
|
--chmod result) even under --perms. When true, rsync's exact semantics are
|
||||||
|
preserved: -p copies the special bits and the kernel decides. */
|
||||||
|
bool super_permitted;
|
||||||
} FileAttrPolicy;
|
} FileAttrPolicy;
|
||||||
|
|
||||||
/* Build the per-attribute policy from a connection's Config. A NULL config
|
/* Build the per-attribute policy from a connection's Config. A NULL config
|
||||||
|
|||||||
+94
-41
@@ -474,9 +474,13 @@ static FileSaveResult file_save_special_to_disk(const char* root_directory, cons
|
|||||||
/* Under -p/--perms rsync copies the source's permission and special bits; a
|
/* Under -p/--perms rsync copies the source's permission and special bits; a
|
||||||
* kernel that denies setuid/setgid/sticky reports the failure rather than
|
* kernel that denies setuid/setgid/sticky reports the failure rather than
|
||||||
* having them masked here. Without -p the node is created like any other new
|
* having them masked here. Without -p the node is created like any other new
|
||||||
* entry: source_mode & 0777 & ~umask. */
|
* entry: source_mode & 0777 & ~umask. When super-user activities are
|
||||||
|
* forbidden, the special bits are stripped even under -p (they are
|
||||||
|
* super-user activities just like device-node creation). */
|
||||||
mode_t perms = config->preserve_perms ? (mode & (mode_t)(S_ISUID | S_ISGID | S_ISVTX | 0777))
|
mode_t perms = config->preserve_perms ? (mode & (mode_t)(S_ISUID | S_ISGID | S_ISVTX | 0777))
|
||||||
: (mode & 0777 & ~(mode_t)file_process_umask());
|
: (mode & 0777 & ~(mode_t)file_process_umask());
|
||||||
|
if (!privilege_super_mode_permitted(config->super_mode))
|
||||||
|
perms &= ~(mode_t)(S_ISUID | S_ISGID | S_ISVTX);
|
||||||
|
|
||||||
int rc = is_fifo ? mkfifoat(parent_fd, leaf, perms)
|
int rc = is_fifo ? mkfifoat(parent_fd, leaf, perms)
|
||||||
: mknodat(parent_fd, leaf, create_mode | perms, rdev);
|
: mknodat(parent_fd, leaf, create_mode | perms, rdev);
|
||||||
@@ -1389,6 +1393,43 @@ bool file_basis_content_required(const Config* config) {
|
|||||||
return config != NULL && config->verify_basis;
|
return config != NULL && config->verify_basis;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Probe one candidate basis file: open it (confined, O_NOFOLLOW) and apply
|
||||||
|
rsync's metadata quick-check; under --verify-basis also hash its bytes and
|
||||||
|
require the sender's digest. On a hit record `candidate` in `out` and return
|
||||||
|
true. The caller retains ownership of `candidate`. */
|
||||||
|
static bool basis_match_probe(const Config* config, const char* candidate,
|
||||||
|
unsigned long long check_size, time_t check_mtime,
|
||||||
|
long check_mtime_nsec, const uint8_t* check_digest,
|
||||||
|
size_t check_digest_len, BasisDestType type, BasisMatch* out) {
|
||||||
|
int fd;
|
||||||
|
struct stat st;
|
||||||
|
if (!basis_open_regular(candidate, check_size, &fd, &st))
|
||||||
|
return false;
|
||||||
|
bool hit = false;
|
||||||
|
if (file_basis_quick_match(config, &st, check_mtime, check_mtime_nsec)) {
|
||||||
|
hit = true;
|
||||||
|
if (file_basis_content_required(config)) {
|
||||||
|
uint8_t basis_digest[CHECKSUM_MAX_DIGEST_LEN];
|
||||||
|
size_t basis_len = 0;
|
||||||
|
bool hashed = checksum_digest_fd((ChecksumAlgo)config->checksum_algo, config->checksum_seed,
|
||||||
|
fd, basis_digest, sizeof(basis_digest), &basis_len);
|
||||||
|
hit = hashed && basis_len == check_digest_len && check_digest_len > 0 &&
|
||||||
|
memcmp(basis_digest, check_digest, check_digest_len) == 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
close(fd);
|
||||||
|
if (!hit)
|
||||||
|
return false;
|
||||||
|
char* owned = str_dup(candidate);
|
||||||
|
if (!owned)
|
||||||
|
return false;
|
||||||
|
out->hit = true;
|
||||||
|
out->type = type;
|
||||||
|
out->basis_path = owned;
|
||||||
|
out->st = st;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
/* Search the basis-dir list in command-line order and return the first match.
|
/* Search the basis-dir list in command-line order and return the first match.
|
||||||
By default (no --verify-basis) rsync's metadata quick-check is sufficient:
|
By default (no --verify-basis) rsync's metadata quick-check is sufficient:
|
||||||
basis_open_regular has already required an equal size, and
|
basis_open_regular has already required an equal size, and
|
||||||
@@ -1403,7 +1444,22 @@ bool file_basis_content_required(const Config* config) {
|
|||||||
--dry-run passes false because hashing a basis against a client-supplied
|
--dry-run passes false because hashing a basis against a client-supplied
|
||||||
digest would be a 1-bit content oracle. Without --verify-basis a dry-run can
|
digest would be a 1-bit content oracle. Without --verify-basis a dry-run can
|
||||||
still confirm the metadata-only hit without reading any basis bytes, matching
|
still confirm the metadata-only hit without reading any basis bytes, matching
|
||||||
rsync's read-only quick-check. */
|
rsync's read-only quick-check.
|
||||||
|
|
||||||
|
Path resolution (rsync 3.4.1 parity): rsync resolves a relative
|
||||||
|
--compare-dest/--copy-dest/--link-dest DIR against the destination directory
|
||||||
|
(the receiver's cwd) and appends the file's TRANSFER-RELATIVE name, e.g.
|
||||||
|
`--compare-dest=basis` with `rsync src/ dst/` probes `dst/basis/<name-inside-src>`.
|
||||||
|
FastSync's receive root IS the destination directory, but its default transfer
|
||||||
|
mirrors the absolute source path below that root, so check_path carries the
|
||||||
|
source-root scaffolding rsync would not append. Recover rsync's spelling with
|
||||||
|
utils_strip_transfer_root for a relative DIR; under -R/--files-from the wire
|
||||||
|
path is already transfer-relative, so it is used as-is. A relative DIR also
|
||||||
|
probes the historical mirror-appended spelling as a fallback, so existing
|
||||||
|
FastSync-laid-out snapshot trees keep resolving. An absolute DIR is used
|
||||||
|
verbatim and keeps appending the destination-relative check_path (FastSync's
|
||||||
|
mirrored layout). Every candidate stays confined to the authorized root by
|
||||||
|
file_open_secure_parent. */
|
||||||
static bool basis_match_find(const Config* config, const char* check_path,
|
static bool basis_match_find(const Config* config, const char* check_path,
|
||||||
unsigned long long check_size, time_t check_mtime,
|
unsigned long long check_size, time_t check_mtime,
|
||||||
long check_mtime_nsec, const uint8_t* check_digest,
|
long check_mtime_nsec, const uint8_t* check_digest,
|
||||||
@@ -1416,48 +1472,40 @@ static bool basis_match_find(const Config* config, const char* check_path,
|
|||||||
the basis bytes. */
|
the basis bytes. */
|
||||||
if (file_basis_content_required(config) && !hash_content)
|
if (file_basis_content_required(config) && !hash_content)
|
||||||
return false;
|
return false;
|
||||||
|
const char* transfer_rel = check_path;
|
||||||
|
if (!config->relative && config->files_from_set == NULL)
|
||||||
|
transfer_rel = utils_strip_transfer_root(check_path, config->send_directory);
|
||||||
for (int i = 0; i < config->basis_count; i++) {
|
for (int i = 0; i < config->basis_count; i++) {
|
||||||
const BasisDest* entry = &config->basis_dirs[i];
|
const BasisDest* entry = &config->basis_dirs[i];
|
||||||
/* An absolute basis path is used verbatim (rsync semantics); a relative one
|
/* An absolute basis path is used verbatim (rsync semantics); a relative one
|
||||||
is resolved below the receive root. Both remain subject to the receiver's
|
is resolved below the receive root. Both remain subject to the receiver's
|
||||||
authorized-root confinement inside file_open_secure_parent. */
|
authorized-root confinement inside file_open_secure_parent. */
|
||||||
char* basis_dir = entry->path[0] == '/' ? str_dup(entry->path)
|
bool absolute = entry->path[0] == '/';
|
||||||
: path_cat(config->receive_root_directory, entry->path);
|
char* basis_dir =
|
||||||
|
absolute ? str_dup(entry->path) : path_cat(config->receive_root_directory, entry->path);
|
||||||
if (!basis_dir)
|
if (!basis_dir)
|
||||||
continue;
|
continue;
|
||||||
char* candidate = path_cat(basis_dir, check_path);
|
const char* names[2];
|
||||||
free(basis_dir);
|
int name_count = 0;
|
||||||
|
if (absolute)
|
||||||
|
names[name_count++] = check_path;
|
||||||
|
else
|
||||||
|
names[name_count++] = transfer_rel;
|
||||||
|
if (!absolute && strcmp(transfer_rel, check_path) != 0)
|
||||||
|
names[name_count++] = check_path; /* historical mirror-appended spelling */
|
||||||
|
bool found = false;
|
||||||
|
for (int n = 0; n < name_count && !found; n++) {
|
||||||
|
char* candidate = path_cat(basis_dir, names[n]);
|
||||||
if (!candidate)
|
if (!candidate)
|
||||||
continue;
|
continue;
|
||||||
|
found = basis_match_probe(config, candidate, check_size, check_mtime, check_mtime_nsec,
|
||||||
int fd;
|
check_digest, check_digest_len, entry->type, out);
|
||||||
struct stat st;
|
|
||||||
if (basis_open_regular(candidate, check_size, &fd, &st)) {
|
|
||||||
if (file_basis_quick_match(config, &st, check_mtime, check_mtime_nsec)) {
|
|
||||||
bool hit = true;
|
|
||||||
if (file_basis_content_required(config)) {
|
|
||||||
uint8_t basis_digest[CHECKSUM_MAX_DIGEST_LEN];
|
|
||||||
size_t basis_len = 0;
|
|
||||||
bool hashed =
|
|
||||||
checksum_digest_fd((ChecksumAlgo)config->checksum_algo, config->checksum_seed, fd,
|
|
||||||
basis_digest, sizeof(basis_digest), &basis_len);
|
|
||||||
hit = hashed && basis_len == check_digest_len && check_digest_len > 0 &&
|
|
||||||
memcmp(basis_digest, check_digest, check_digest_len) == 0;
|
|
||||||
}
|
|
||||||
if (hit) {
|
|
||||||
out->hit = true;
|
|
||||||
out->type = entry->type;
|
|
||||||
out->basis_path = candidate;
|
|
||||||
candidate = NULL; /* ownership transferred to out */
|
|
||||||
out->st = st;
|
|
||||||
close(fd);
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
close(fd);
|
|
||||||
}
|
|
||||||
free(candidate);
|
free(candidate);
|
||||||
}
|
}
|
||||||
|
free(basis_dir);
|
||||||
|
if (found)
|
||||||
|
return true;
|
||||||
|
}
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1489,9 +1537,12 @@ static bool basis_match_find(const Config* config, const char* check_path,
|
|||||||
* followed and nothing outside the destination root is ever read;
|
* followed and nothing outside the destination root is ever read;
|
||||||
* * dotfiles, directories, the target's own name, and the .fastsync-stage /
|
* * dotfiles, directories, the target's own name, and the .fastsync-stage /
|
||||||
* temp scratch names are never candidates;
|
* temp scratch names are never candidates;
|
||||||
* * size gate = the delta engine's own bounds (delta_should_attempt: both
|
* * size gate = rsync's, NOT the ordinary delta engine's bounds: any
|
||||||
* files >= DELTA_MIN_FILE_SIZE, <= delta_max_file_size, ratio <= 10x),
|
* non-empty regular sibling up to the receiver's whole-file buffer cap is
|
||||||
* because FastSync's delta engine cannot use a basis outside them;
|
* eligible, regardless of the 16 KiB delta minimum or the 10x delta size
|
||||||
|
* ratio (rsync's find_fuzzy has no delta-size gate at all). The delta
|
||||||
|
* engine consumes the fuzzy basis through the same signature handshake
|
||||||
|
* whether or not it is inside delta_should_attempt's window;
|
||||||
* * first pass = an exact size+mtime match wins regardless of name (rsync's
|
* * first pass = an exact size+mtime match wins regardless of name (rsync's
|
||||||
* "fuzzy size/modtime match");
|
* "fuzzy size/modtime match");
|
||||||
* * otherwise the winner minimizes rsync's weighted Levenshtein distance
|
* * otherwise the winner minimizes rsync's weighted Levenshtein distance
|
||||||
@@ -1639,8 +1690,7 @@ static void* fuzzy_basis_find_and_load(const Config* config, const char* check_p
|
|||||||
long check_mtime_nsec, unsigned long long* out_size) {
|
long check_mtime_nsec, unsigned long long* out_size) {
|
||||||
*out_size = 0;
|
*out_size = 0;
|
||||||
if (!config || !config->receive_root_directory || !config->fuzzy || !config->use_delta ||
|
if (!config || !config->receive_root_directory || !config->fuzzy || !config->use_delta ||
|
||||||
!check_path || check_size < DELTA_MIN_FILE_SIZE || check_size > config->delta_max_file_size ||
|
!check_path || check_size > MAX_RECEIVE_WHOLE_FILE_SIZE)
|
||||||
check_size > MAX_RECEIVE_WHOLE_FILE_SIZE)
|
|
||||||
return NULL;
|
return NULL;
|
||||||
|
|
||||||
char* full_path = path_cat(config->receive_root_directory, check_path);
|
char* full_path = path_cat(config->receive_root_directory, check_path);
|
||||||
@@ -1717,8 +1767,7 @@ static void* fuzzy_basis_find_and_load(const Config* config, const char* check_p
|
|||||||
if (fstatat(dir_fd, name, &st, AT_SYMLINK_NOFOLLOW) != 0 || !S_ISREG(st.st_mode))
|
if (fstatat(dir_fd, name, &st, AT_SYMLINK_NOFOLLOW) != 0 || !S_ISREG(st.st_mode))
|
||||||
continue;
|
continue;
|
||||||
unsigned long long cand_size = (unsigned long long)st.st_size;
|
unsigned long long cand_size = (unsigned long long)st.st_size;
|
||||||
if (cand_size == 0 || cand_size > MAX_RECEIVE_WHOLE_FILE_SIZE ||
|
if (cand_size == 0 || cand_size > MAX_RECEIVE_WHOLE_FILE_SIZE)
|
||||||
!delta_should_attempt(cand_size, check_size, config->delta_max_file_size))
|
|
||||||
continue;
|
continue;
|
||||||
long cand_nsec = 0;
|
long cand_nsec = 0;
|
||||||
#ifdef __linux__
|
#ifdef __linux__
|
||||||
@@ -2904,8 +2953,12 @@ void dir_metadata_list_apply(const DirTimeList* list, const char* root_directory
|
|||||||
}
|
}
|
||||||
if (mode_ready) {
|
if (mode_ready) {
|
||||||
/* rsync -p copies the source directory mode exactly, including
|
/* rsync -p copies the source directory mode exactly, including
|
||||||
* group/other write and the setgid/sticky bits. */
|
* group/other write and the setgid/sticky bits. Setuid/setgid/sticky
|
||||||
|
* are super-user activities: when the connection forbade them
|
||||||
|
* (SUPER_MODE_OFF / --no-super), strip them even under -p. */
|
||||||
mode_t safe_mode = dir_mode & (mode_t)(S_ISUID | S_ISGID | S_ISVTX | 0777);
|
mode_t safe_mode = dir_mode & (mode_t)(S_ISUID | S_ISGID | S_ISVTX | 0777);
|
||||||
|
if (!privilege_super_mode_permitted(config->super_mode))
|
||||||
|
safe_mode &= ~(mode_t)(S_ISUID | S_ISGID | S_ISVTX);
|
||||||
if (dir_fd < 0) {
|
if (dir_fd < 0) {
|
||||||
char* escaped_path = output_escape(dir_path, log_get_8_bit_output());
|
char* escaped_path = output_escape(dir_path, log_get_8_bit_output());
|
||||||
log_message(LOG_LEVEL_WARNING, "Failed to open directory %s to set its mode: %s",
|
log_message(LOG_LEVEL_WARNING, "Failed to open directory %s to set its mode: %s",
|
||||||
|
|||||||
@@ -166,6 +166,8 @@ bool file_send_sendfile_with_skip(File* file, int file_descriptor, bool use_meta
|
|||||||
}
|
}
|
||||||
struct pollfd pfd = {.fd = file_descriptor, .events = POLLOUT};
|
struct pollfd pfd = {.fd = file_descriptor, .events = POLLOUT};
|
||||||
int polled = poll(&pfd, 1, timeout);
|
int polled = poll(&pfd, 1, timeout);
|
||||||
|
if (polled < 0 && errno == EINTR)
|
||||||
|
continue;
|
||||||
if (polled <= 0 || (pfd.revents & (POLLERR | POLLHUP | POLLNVAL))) {
|
if (polled <= 0 || (pfd.revents & (POLLERR | POLLHUP | POLLNVAL))) {
|
||||||
close(fd);
|
close(fd);
|
||||||
return false;
|
return false;
|
||||||
@@ -183,6 +185,7 @@ bool file_send_sendfile_with_skip(File* file, int file_descriptor, bool use_meta
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
protocol_note_bytes_written((unsigned long long)sent);
|
protocol_note_bytes_written((unsigned long long)sent);
|
||||||
|
protocol_throttle_bytes((size_t)sent);
|
||||||
}
|
}
|
||||||
|
|
||||||
close(fd);
|
close(fd);
|
||||||
|
|||||||
+93
-25
@@ -4,22 +4,12 @@
|
|||||||
#include <ctype.h>
|
#include <ctype.h>
|
||||||
#include <errno.h>
|
#include <errno.h>
|
||||||
#include <limits.h>
|
#include <limits.h>
|
||||||
#include <stdarg.h>
|
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
|
|
||||||
/* Write a diagnostic message into the caller's optional buffer. A NULL `err`
|
/* Write a diagnostic message into the caller's optional buffer. */
|
||||||
* (or a zero size) is a no-op, so a caller that only needs the boolean status
|
#define filter_set_error utils_set_error
|
||||||
* may pass NULL without the snprintf-on-NULL undefined behaviour. */
|
|
||||||
static void filter_set_error(char* err, size_t err_size, const char* fmt, ...) {
|
|
||||||
if (!err || err_size == 0)
|
|
||||||
return;
|
|
||||||
va_list ap;
|
|
||||||
va_start(ap, fmt);
|
|
||||||
vsnprintf(err, err_size, fmt, ap);
|
|
||||||
va_end(ap);
|
|
||||||
}
|
|
||||||
|
|
||||||
/* ---- Ordered rule lists ---- */
|
/* ---- Ordered rule lists ---- */
|
||||||
|
|
||||||
@@ -172,14 +162,74 @@ static bool is_modifier_char(char c) {
|
|||||||
return c == 's' || c == 'r' || c == 'p' || c == 'x' || c == '/' || c == '!' || c == 'C';
|
return c == 's' || c == 'r' || c == 'p' || c == 'x' || c == '/' || c == '!' || c == 'C';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* merge/dir-merge rules are the only rules rsync accepts the merge-file
|
||||||
|
* modifiers on. */
|
||||||
|
static bool is_merge_rule(RuleKind kind) {
|
||||||
|
return kind == RULE_KIND_MERGE || kind == RULE_KIND_DIR_MERGE;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Merge-file modifiers rsync defines but FastSync does not implement:
|
||||||
|
* 'e' exclude the merge file itself, 'n' do not inherit the merge file, 'w'
|
||||||
|
* word-split the merge file. They are recognized as part of a modifier run on
|
||||||
|
* every rule (so a pure e/n/w token is rejected rather than folded into the
|
||||||
|
* pattern), but are accepted (and ignored) only on merge/dir-merge rules. */
|
||||||
|
static bool is_unsupported_modifier_char(char c) {
|
||||||
|
return c == 'e' || c == 'n' || c == 'w';
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Merge-file modifiers rsync accepts on merge/dir-merge rules: 'e', 'n', 'w'
|
||||||
|
* and '-' (do not transfer the merge file). */
|
||||||
|
static bool is_merge_modifier_char(char c) {
|
||||||
|
return c == 'e' || c == 'n' || c == 'w' || c == '-';
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Characters that count as part of a modifier run for `kind` when deciding
|
||||||
|
* whether a token is a pure modifier run. e/n/w count on every rule so that a
|
||||||
|
* pure e/n/w token is rejected on non-merge rules; '-' only on merge rules. */
|
||||||
|
static bool is_modifier_scan_char(char c, RuleKind kind) {
|
||||||
|
return is_modifier_char(c) || is_unsupported_modifier_char(c) ||
|
||||||
|
(is_merge_rule(kind) && is_merge_modifier_char(c));
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Characters actually consumed as modifiers for `kind`. The merge-file
|
||||||
|
* modifiers are consumed only on merge/dir-merge rules; elsewhere e/n/w fall
|
||||||
|
* through to the pattern (so mixed tokens such as "H,!secret" keep their
|
||||||
|
* historical "ecret" pattern). */
|
||||||
|
static bool is_consumed_modifier_char(char c, RuleKind kind) {
|
||||||
|
return is_modifier_char(c) || (is_merge_rule(kind) && is_merge_modifier_char(c));
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Inspect the token that follows a rule name (up to the first space/underscore
|
||||||
|
* or the end). If the token is composed *solely* of modifier characters and
|
||||||
|
* includes one that is invalid for `kind`, it is unambiguously a modifier run:
|
||||||
|
* return that character so the caller can reject it. A token that contains any
|
||||||
|
* non-modifier character is a pattern (e.g. "-newfile") and returns '\0', which
|
||||||
|
* keeps the historical parsing of mixed tokens such as "H,!secret" intact. */
|
||||||
|
static char unsupported_modifier_in_token(const char* tok, RuleKind kind) {
|
||||||
|
if (*tok == '\0' || *tok == ' ' || *tok == '_')
|
||||||
|
return '\0';
|
||||||
|
char bad = '\0';
|
||||||
|
for (const char* q = tok; *q != '\0' && *q != ' ' && *q != '_'; q++) {
|
||||||
|
if (!is_modifier_scan_char(*q, kind))
|
||||||
|
return '\0';
|
||||||
|
if (!is_merge_rule(kind) && is_unsupported_modifier_char(*q))
|
||||||
|
bad = *q;
|
||||||
|
}
|
||||||
|
return bad;
|
||||||
|
}
|
||||||
|
|
||||||
/* Parse "RULE[,MODIFIERS] [PATTERN]". On success `kind`, `sides`,
|
/* Parse "RULE[,MODIFIERS] [PATTERN]". On success `kind`, `sides`,
|
||||||
* `sides_explicit`, `negate`, `anchored_mod`, `perishable`, `xattr`,
|
* `sides_explicit`, `negate`, `anchored_mod`, `perishable`, `xattr`,
|
||||||
* `cvs_inject` and the pattern span (`pat_start`/`pat_len`, possibly 0 for
|
* `cvs_inject` and the pattern span (`pat_start`/`pat_len`, possibly 0 for
|
||||||
* merge/clear) are filled. Returns true on success. */
|
* merge/clear) are filled. Returns true on success.
|
||||||
|
*
|
||||||
|
* On failure `*bad_mod` is set to the offending modifier character when the
|
||||||
|
* rule carried a modifier FastSync does not implement, and left '\0' for a
|
||||||
|
* generic syntax error so callers can emit a precise diagnostic. */
|
||||||
static bool parse_rule_syntax(const char* text, RuleKind* kind, unsigned* sides,
|
static bool parse_rule_syntax(const char* text, RuleKind* kind, unsigned* sides,
|
||||||
bool* sides_explicit, bool* negate, bool* anchored_mod,
|
bool* sides_explicit, bool* negate, bool* anchored_mod,
|
||||||
bool* perishable, bool* xattr, bool* cvs_inject,
|
bool* perishable, bool* xattr, bool* cvs_inject,
|
||||||
const char** pat_start, size_t* pat_len) {
|
const char** pat_start, size_t* pat_len, char* bad_mod) {
|
||||||
const char* p = text;
|
const char* p = text;
|
||||||
*sides = FILTER_SIDE_SENDER | FILTER_SIDE_RECEIVER;
|
*sides = FILTER_SIDE_SENDER | FILTER_SIDE_RECEIVER;
|
||||||
*sides_explicit = false;
|
*sides_explicit = false;
|
||||||
@@ -190,6 +240,7 @@ static bool parse_rule_syntax(const char* text, RuleKind* kind, unsigned* sides,
|
|||||||
*cvs_inject = false;
|
*cvs_inject = false;
|
||||||
*pat_start = NULL;
|
*pat_start = NULL;
|
||||||
*pat_len = 0;
|
*pat_len = 0;
|
||||||
|
*bad_mod = '\0';
|
||||||
|
|
||||||
bool is_short = false;
|
bool is_short = false;
|
||||||
if (short_rule_char(*p, kind)) {
|
if (short_rule_char(*p, kind)) {
|
||||||
@@ -210,17 +261,25 @@ static bool parse_rule_syntax(const char* text, RuleKind* kind, unsigned* sides,
|
|||||||
/* Modifiers: long names require a comma; short names may attach directly.
|
/* Modifiers: long names require a comma; short names may attach directly.
|
||||||
Only commit a modifier run that terminates at a separator or the end, so a
|
Only commit a modifier run that terminates at a separator or the end, so a
|
||||||
pattern such as "*.tmp" written as "-*.tmp" is not mistaken for modifiers. */
|
pattern such as "*.tmp" written as "-*.tmp" is not mistaken for modifiers. */
|
||||||
|
if (*p == ',') {
|
||||||
|
*bad_mod = unsupported_modifier_in_token(p + 1, *kind);
|
||||||
|
} else if (is_short) {
|
||||||
|
*bad_mod = unsupported_modifier_in_token(p, *kind);
|
||||||
|
}
|
||||||
|
if (*bad_mod != '\0')
|
||||||
|
return false;
|
||||||
|
|
||||||
const char* mod_start = p;
|
const char* mod_start = p;
|
||||||
const char* mod_end = p;
|
const char* mod_end = p;
|
||||||
if (*p == ',') {
|
if (*p == ',') {
|
||||||
p++;
|
p++;
|
||||||
mod_start = p;
|
mod_start = p;
|
||||||
while (is_modifier_char(*p))
|
while (is_consumed_modifier_char(*p, *kind))
|
||||||
p++;
|
p++;
|
||||||
mod_end = p;
|
mod_end = p;
|
||||||
} else if (is_short) {
|
} else if (is_short) {
|
||||||
const char* scan = p;
|
const char* scan = p;
|
||||||
while (is_modifier_char(*scan))
|
while (is_consumed_modifier_char(*scan, *kind))
|
||||||
scan++;
|
scan++;
|
||||||
if (*scan == '\0' || *scan == ' ' || *scan == '_') {
|
if (*scan == '\0' || *scan == ' ' || *scan == '_') {
|
||||||
mod_start = p;
|
mod_start = p;
|
||||||
@@ -290,8 +349,12 @@ FilterRule* filter_rule_parse(const char* line, const FilterParseOptions* opts,
|
|||||||
bool sides_explicit, negate, anchored_mod, perishable, xattr, cvs_inject;
|
bool sides_explicit, negate, anchored_mod, perishable, xattr, cvs_inject;
|
||||||
const char* pat;
|
const char* pat;
|
||||||
size_t pat_len;
|
size_t pat_len;
|
||||||
|
char bad_mod;
|
||||||
if (!parse_rule_syntax(p, &kind, &sides, &sides_explicit, &negate, &anchored_mod, &perishable,
|
if (!parse_rule_syntax(p, &kind, &sides, &sides_explicit, &negate, &anchored_mod, &perishable,
|
||||||
&xattr, &cvs_inject, &pat, &pat_len)) {
|
&xattr, &cvs_inject, &pat, &pat_len, &bad_mod)) {
|
||||||
|
if (bad_mod != '\0')
|
||||||
|
filter_set_error(err, err_size, "unsupported filter modifier '%c'", bad_mod);
|
||||||
|
else
|
||||||
filter_set_error(err, err_size, "unrecognized filter rule syntax");
|
filter_set_error(err, err_size, "unrecognized filter rule syntax");
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
@@ -401,7 +464,6 @@ FilterRule* filter_rule_parse(const char* line, const FilterParseOptions* opts,
|
|||||||
rule->dir_only = dir_only;
|
rule->dir_only = dir_only;
|
||||||
rule->negate = negate;
|
rule->negate = negate;
|
||||||
rule->perishable = perishable;
|
rule->perishable = perishable;
|
||||||
(void)xattr; /* xattr-name rules never match file/dir names; accepted/ignored */
|
|
||||||
return rule;
|
return rule;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -530,8 +592,12 @@ static bool filter_list_parse_append_depth(FilterRuleList* list, const char* lin
|
|||||||
bool sides_explicit, negate, anchored_mod, perishable, xattr, cvs_inject;
|
bool sides_explicit, negate, anchored_mod, perishable, xattr, cvs_inject;
|
||||||
const char* pat;
|
const char* pat;
|
||||||
size_t pat_len;
|
size_t pat_len;
|
||||||
|
char bad_mod;
|
||||||
if (!parse_rule_syntax(p, &kind, &sides, &sides_explicit, &negate, &anchored_mod, &perishable,
|
if (!parse_rule_syntax(p, &kind, &sides, &sides_explicit, &negate, &anchored_mod, &perishable,
|
||||||
&xattr, &cvs_inject, &pat, &pat_len)) {
|
&xattr, &cvs_inject, &pat, &pat_len, &bad_mod)) {
|
||||||
|
if (bad_mod != '\0')
|
||||||
|
filter_set_error(err, err_size, "unsupported filter modifier '%c': %s", bad_mod, p);
|
||||||
|
else
|
||||||
filter_set_error(err, err_size, "unrecognized filter rule syntax: %s", p);
|
filter_set_error(err, err_size, "unrecognized filter rule syntax: %s", p);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
@@ -539,7 +605,14 @@ static bool filter_list_parse_append_depth(FilterRuleList* list, const char* lin
|
|||||||
(void)negate;
|
(void)negate;
|
||||||
(void)anchored_mod;
|
(void)anchored_mod;
|
||||||
(void)perishable;
|
(void)perishable;
|
||||||
(void)xattr;
|
|
||||||
|
/* xattr-name rules are not implemented; reject them everywhere (including on
|
||||||
|
* merge/dir-merge, where the flag would otherwise be silently dropped) with
|
||||||
|
* the same diagnostic the standalone parser gives. */
|
||||||
|
if (xattr) {
|
||||||
|
filter_set_error(err, err_size, "xattr-name filter rules (the x modifier) are not supported");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
if (cvs_inject) {
|
if (cvs_inject) {
|
||||||
/* "C" injects the CVS defaults in place; no pattern is expected. */
|
/* "C" injects the CVS defaults in place; no pattern is expected. */
|
||||||
@@ -811,8 +884,3 @@ FilterAction filter_rules_apply_side(const FilterRuleList* list, const char* rel
|
|||||||
}
|
}
|
||||||
return FILTER_ACTION_NONE;
|
return FILTER_ACTION_NONE;
|
||||||
}
|
}
|
||||||
|
|
||||||
FilterAction filter_rules_apply(const FilterRuleList* list, const char* rel_path, const char* leaf,
|
|
||||||
bool is_dir) {
|
|
||||||
return filter_rules_apply_side(list, rel_path, leaf, is_dir, FILTER_SIDE_SENDER);
|
|
||||||
}
|
|
||||||
+7
-6
@@ -23,7 +23,13 @@
|
|||||||
* dir-merge/: per-directory merge file (registered for the scanner)
|
* dir-merge/: per-directory merge file (registered for the scanner)
|
||||||
* clear/! clear the current rule list (takes no argument)
|
* clear/! clear the current rule list (takes no argument)
|
||||||
* Modifiers: '/' absolute anchor, '!' negate match, 'C' inject CVS defaults,
|
* Modifiers: '/' absolute anchor, '!' negate match, 'C' inject CVS defaults,
|
||||||
* 's' sender side, 'r' receiver side, 'p' perishable, 'x' xattr name rule.
|
* 's' sender side, 'r' receiver side, 'p' perishable. The rsync 'x'
|
||||||
|
* (xattr-name) modifier is not implemented and is rejected explicitly
|
||||||
|
* everywhere. The merge-file modifiers 'e' (exclude the merge file itself),
|
||||||
|
* 'n' (do not inherit the merge file), 'w' (word-split the merge file) and '-'
|
||||||
|
* (do not transfer the merge file) are accepted and consumed only on merge/
|
||||||
|
* dir-merge rules (rejected on every other rule, matching rsync); their
|
||||||
|
* semantics are not implemented and they are otherwise ignored.
|
||||||
* A trailing '/' makes a pattern match directories only. A leading '/' anchors
|
* A trailing '/' makes a pattern match directories only. A leading '/' anchors
|
||||||
* the pattern to its owner directory.
|
* the pattern to its owner directory.
|
||||||
*/
|
*/
|
||||||
@@ -129,9 +135,4 @@ FilterRuleList* filter_file_read(const char* dir_path, const char* owner_rel, bo
|
|||||||
FilterAction filter_rules_apply_side(const FilterRuleList* list, const char* rel_path,
|
FilterAction filter_rules_apply_side(const FilterRuleList* list, const char* rel_path,
|
||||||
const char* leaf, bool is_dir, unsigned side);
|
const char* leaf, bool is_dir, unsigned side);
|
||||||
|
|
||||||
/* Sender-side convenience wrapper (kept for callers/tests that only need the
|
|
||||||
* transfer decision). */
|
|
||||||
FilterAction filter_rules_apply(const FilterRuleList* list, const char* rel_path, const char* leaf,
|
|
||||||
bool is_dir);
|
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
+30
-5
@@ -5,6 +5,15 @@
|
|||||||
#include <stdbool.h>
|
#include <stdbool.h>
|
||||||
#include <stdint.h>
|
#include <stdint.h>
|
||||||
|
|
||||||
|
/* Ask the compiler to type-check the printf-style arguments of the variadic
|
||||||
|
* logging helpers. Only enabled for GNU-compatible compilers (gcc/clang). */
|
||||||
|
#if defined(__GNUC__)
|
||||||
|
#define LOG_PRINTF_ATTR(fmt_idx, first_vararg_idx) \
|
||||||
|
__attribute__((format(printf, fmt_idx, first_vararg_idx)))
|
||||||
|
#else
|
||||||
|
#define LOG_PRINTF_ATTR(fmt_idx, first_vararg_idx)
|
||||||
|
#endif
|
||||||
|
|
||||||
typedef enum { LOG_LEVEL_DEBUG, LOG_LEVEL_INFO, LOG_LEVEL_WARNING, LOG_LEVEL_ERROR } LogLevel;
|
typedef enum { LOG_LEVEL_DEBUG, LOG_LEVEL_INFO, LOG_LEVEL_WARNING, LOG_LEVEL_ERROR } LogLevel;
|
||||||
typedef enum { LOG_STDERR_ERRORS, LOG_STDERR_ALL } LogStderrMode;
|
typedef enum { LOG_STDERR_ERRORS, LOG_STDERR_ALL } LogStderrMode;
|
||||||
|
|
||||||
@@ -13,7 +22,19 @@ typedef enum {
|
|||||||
LOG_DEBUG_PROTO = 1u << 1,
|
LOG_DEBUG_PROTO = 1u << 1,
|
||||||
LOG_DEBUG_PACK = 1u << 2,
|
LOG_DEBUG_PACK = 1u << 2,
|
||||||
LOG_DEBUG_UTIL = 1u << 3,
|
LOG_DEBUG_UTIL = 1u << 3,
|
||||||
LOG_DEBUG_ALL = (1u << 4) - 1,
|
/* rsync --debug categories that now map to a natural FastSync event:
|
||||||
|
* flist (file-list scan progress), del (deletions), hash/deltasum
|
||||||
|
* (whole-file hashing and delta-sum generation), recv (receiver
|
||||||
|
* responses/signatures), filter (selection/exclusion decisions) and send
|
||||||
|
* (files handed to the sender). Only emitted when the category is
|
||||||
|
* explicitly enabled; a normal run stays silent. */
|
||||||
|
LOG_DEBUG_FLIST = 1u << 4,
|
||||||
|
LOG_DEBUG_DEL = 1u << 5,
|
||||||
|
LOG_DEBUG_HASH = 1u << 6,
|
||||||
|
LOG_DEBUG_RECV = 1u << 7,
|
||||||
|
LOG_DEBUG_FILTER = 1u << 8,
|
||||||
|
LOG_DEBUG_SEND = 1u << 9,
|
||||||
|
LOG_DEBUG_ALL = (1u << 10) - 1,
|
||||||
} LogDebugFlag;
|
} LogDebugFlag;
|
||||||
|
|
||||||
typedef enum {
|
typedef enum {
|
||||||
@@ -38,12 +59,16 @@ typedef enum {
|
|||||||
the info_level bitset (there is no separate Config field) and is never set
|
the info_level bitset (there is no separate Config field) and is never set
|
||||||
by --info=all (which selects level 1). */
|
by --info=all (which selects level 1). */
|
||||||
LOG_INFO_NAME_UPTODATE = 1u << 10,
|
LOG_INFO_NAME_UPTODATE = 1u << 10,
|
||||||
|
/* --info=mount: print rsync's `[sender] skipping mount-point dir NAME` when
|
||||||
|
* -xx/--one-file-system drops a mount-point directory (FastSync's client is
|
||||||
|
* the sender). */
|
||||||
|
LOG_INFO_MOUNT = 1u << 11,
|
||||||
LOG_INFO_ALL = LOG_INFO_COPY | LOG_INFO_MISC | LOG_INFO_SKIP | LOG_INFO_STATS | LOG_INFO_DEL |
|
LOG_INFO_ALL = LOG_INFO_COPY | LOG_INFO_MISC | LOG_INFO_SKIP | LOG_INFO_STATS | LOG_INFO_DEL |
|
||||||
LOG_INFO_REMOVE | LOG_INFO_NAME | LOG_INFO_FLIST | LOG_INFO_NONREG |
|
LOG_INFO_REMOVE | LOG_INFO_NAME | LOG_INFO_FLIST | LOG_INFO_NONREG |
|
||||||
LOG_INFO_PROGRESS,
|
LOG_INFO_PROGRESS | LOG_INFO_MOUNT,
|
||||||
} LogInfoFlag;
|
} LogInfoFlag;
|
||||||
|
|
||||||
void log_message(LogLevel log_level, const char* message, ...);
|
void log_message(LogLevel log_level, const char* message, ...) LOG_PRINTF_ATTR(2, 3);
|
||||||
void log_perror(const char* context);
|
void log_perror(const char* context);
|
||||||
void set_log_level(LogLevel level);
|
void set_log_level(LogLevel level);
|
||||||
void set_log_debug_flags(uint32_t flags);
|
void set_log_debug_flags(uint32_t flags);
|
||||||
@@ -52,10 +77,10 @@ uint32_t get_log_debug_flags(void);
|
|||||||
* the debug log level is enabled AND the flag is selected. Hot paths use this
|
* the debug log level is enabled AND the flag is selected. Hot paths use this
|
||||||
* to skip expensive message formatting/escaping when the line is filtered. */
|
* to skip expensive message formatting/escaping when the line is filtered. */
|
||||||
bool log_debug_enabled(LogDebugFlag flag);
|
bool log_debug_enabled(LogDebugFlag flag);
|
||||||
void log_debug_message(LogDebugFlag flag, const char* message, ...);
|
void log_debug_message(LogDebugFlag flag, const char* message, ...) LOG_PRINTF_ATTR(2, 3);
|
||||||
void set_log_info_flags(uint32_t flags);
|
void set_log_info_flags(uint32_t flags);
|
||||||
uint32_t get_log_info_flags(void);
|
uint32_t get_log_info_flags(void);
|
||||||
void log_info_message(LogInfoFlag flag, const char* message, ...);
|
void log_info_message(LogInfoFlag flag, const char* message, ...) LOG_PRINTF_ATTR(2, 3);
|
||||||
void log_set_file(FILE* fp);
|
void log_set_file(FILE* fp);
|
||||||
void log_set_8_bit_output(bool enabled);
|
void log_set_8_bit_output(bool enabled);
|
||||||
bool log_get_8_bit_output(void);
|
bool log_get_8_bit_output(void);
|
||||||
|
|||||||
+20
-5
@@ -211,13 +211,22 @@ FileMetadata* metadata_receive(int file_descriptor, int* ok) {
|
|||||||
|
|
||||||
bool metadata_mode_for_policy(mode_t source_mode, mode_t current_mode, FileAttrPolicy policy,
|
bool metadata_mode_for_policy(mode_t source_mode, mode_t current_mode, FileAttrPolicy policy,
|
||||||
mode_t* out_mode) {
|
mode_t* out_mode) {
|
||||||
|
const mode_t special_bits = (mode_t)(S_ISUID | S_ISGID | S_ISVTX);
|
||||||
const mode_t execute_bits = S_IXUSR | S_IXGRP | S_IXOTH;
|
const mode_t execute_bits = S_IXUSR | S_IXGRP | S_IXOTH;
|
||||||
if (policy.perms) {
|
if (policy.perms) {
|
||||||
/* rsync --perms copies the source's permission and special bits exactly,
|
/* rsync --perms copies the source's permission and special bits exactly,
|
||||||
* including group/other write and setuid/setgid/sticky. The kernel may
|
* including group/other write and setuid/setgid/sticky. The kernel may
|
||||||
* still clear setgid when the receiver is not in the file's group; the
|
* still clear setgid when the receiver is not in the file's group; the
|
||||||
* caller logs a failed chmod rather than silently masking the bits here. */
|
* caller logs a failed chmod rather than silently masking the bits here.
|
||||||
*out_mode = source_mode & (mode_t)(S_ISUID | S_ISGID | S_ISVTX | 0777);
|
* Setuid/setgid/sticky are super-user activities: when the connection did
|
||||||
|
* not permit them (SUPER_MODE_OFF / --no-super) they are stripped, so a
|
||||||
|
* client can never install a privileged bit on a receiver that forbade
|
||||||
|
* super-user activities. This also covers bits introduced by --chmod,
|
||||||
|
* whose result is fed in as source_mode. */
|
||||||
|
mode_t bits = source_mode & (mode_t)(special_bits | 0777);
|
||||||
|
if (!policy.super_permitted)
|
||||||
|
bits &= ~special_bits;
|
||||||
|
*out_mode = bits;
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
if (policy.executability) {
|
if (policy.executability) {
|
||||||
@@ -227,8 +236,11 @@ bool metadata_mode_for_policy(mode_t source_mode, mode_t current_mode, FileAttrP
|
|||||||
* execute); otherwise clear every execute bit. This runs on the
|
* execute); otherwise clear every execute bit. This runs on the
|
||||||
* destination-derived base (pre-existing dest mode, or source&~umask for a
|
* destination-derived base (pre-existing dest mode, or source&~umask for a
|
||||||
* new file), and leaves the special bits untouched. --perms wins when both
|
* new file), and leaves the special bits untouched. --perms wins when both
|
||||||
* are set (handled above). */
|
* are set (handled above). The destination's own special bits survive
|
||||||
mode_t base = current_mode & (mode_t)(S_ISUID | S_ISGID | S_ISVTX | 0777);
|
* unless super-user activities are forbidden. */
|
||||||
|
mode_t base = current_mode & (mode_t)(special_bits | 0777);
|
||||||
|
if (!policy.super_permitted)
|
||||||
|
base &= ~special_bits;
|
||||||
if (source_mode & 0111)
|
if (source_mode & 0111)
|
||||||
*out_mode = base | ((base & 0444) >> 2);
|
*out_mode = base | ((base & 0444) >> 2);
|
||||||
else
|
else
|
||||||
@@ -240,12 +252,13 @@ bool metadata_mode_for_policy(mode_t source_mode, mode_t current_mode, FileAttrP
|
|||||||
}
|
}
|
||||||
|
|
||||||
FileAttrPolicy file_attr_policy_from_config(const Config* config) {
|
FileAttrPolicy file_attr_policy_from_config(const Config* config) {
|
||||||
FileAttrPolicy policy = {false, false, false, false};
|
FileAttrPolicy policy = {0};
|
||||||
if (config) {
|
if (config) {
|
||||||
policy.perms = config->preserve_perms;
|
policy.perms = config->preserve_perms;
|
||||||
policy.times = config->preserve_times;
|
policy.times = config->preserve_times;
|
||||||
policy.atimes = config->preserve_atimes;
|
policy.atimes = config->preserve_atimes;
|
||||||
policy.executability = config->use_executability;
|
policy.executability = config->use_executability;
|
||||||
|
policy.super_permitted = privilege_super_mode_permitted(config->super_mode);
|
||||||
}
|
}
|
||||||
return policy;
|
return policy;
|
||||||
}
|
}
|
||||||
@@ -314,6 +327,8 @@ bool file_restore_symlink_metadata(const char* path, const FileMetadata* metadat
|
|||||||
transfer never fails over it. */
|
transfer never fails over it. */
|
||||||
if (policy.perms) {
|
if (policy.perms) {
|
||||||
mode_t link_mode = metadata->mode & (mode_t)(S_ISUID | S_ISGID | S_ISVTX | 0777);
|
mode_t link_mode = metadata->mode & (mode_t)(S_ISUID | S_ISGID | S_ISVTX | 0777);
|
||||||
|
if (!policy.super_permitted)
|
||||||
|
link_mode &= ~(mode_t)(S_ISUID | S_ISGID | S_ISVTX);
|
||||||
if (fchmodat(parent_fd, leaf, link_mode, AT_SYMLINK_NOFOLLOW) != 0 && errno != EOPNOTSUPP &&
|
if (fchmodat(parent_fd, leaf, link_mode, AT_SYMLINK_NOFOLLOW) != 0 && errno != EOPNOTSUPP &&
|
||||||
errno != ENOTSUP && errno != ENOSYS) {
|
errno != ENOTSUP && errno != ENOSYS) {
|
||||||
log_message(LOG_LEVEL_DEBUG, "Could not set symlink mode on %s: %s", path, strerror(errno));
|
log_message(LOG_LEVEL_DEBUG, "Could not set symlink mode on %s: %s", path, strerror(errno));
|
||||||
|
|||||||
@@ -50,6 +50,7 @@ PipelineContextSender* pipeline_context_sender_create(Config* config, Queue* que
|
|||||||
protocol_session_set_max_alloc(&context->allocation_session, config->max_alloc);
|
protocol_session_set_max_alloc(&context->allocation_session, config->max_alloc);
|
||||||
context->dir_entries = NULL;
|
context->dir_entries = NULL;
|
||||||
context->dir_entries_mutex_init = false;
|
context->dir_entries_mutex_init = false;
|
||||||
|
atomic_init(&context->dir_count, 0);
|
||||||
context->delete_limit = false;
|
context->delete_limit = false;
|
||||||
int init = 0;
|
int init = 0;
|
||||||
if (config->use_metadata) {
|
if (config->use_metadata) {
|
||||||
@@ -85,11 +86,13 @@ PipelineContextSender* pipeline_context_sender_create(Config* config, Queue* que
|
|||||||
return context;
|
return context;
|
||||||
|
|
||||||
fail:
|
fail:
|
||||||
log_perror("Error initializing synchronization objects");
|
log_message(LOG_LEVEL_ERROR, "%s", "Error initializing synchronization objects");
|
||||||
if (context->dir_entries_mutex_init)
|
if (context->dir_entries_mutex_init)
|
||||||
mtx_destroy(&context->dir_entries_mutex);
|
mtx_destroy(&context->dir_entries_mutex);
|
||||||
if (context->dir_entries)
|
if (context->dir_entries)
|
||||||
array_list_delete(context->dir_entries);
|
array_list_delete(context->dir_entries);
|
||||||
|
if (init >= 7)
|
||||||
|
mtx_destroy(&context->mutex_progress);
|
||||||
if (init >= 6)
|
if (init >= 6)
|
||||||
cnd_destroy(&context->condition_not_empty_loader);
|
cnd_destroy(&context->condition_not_empty_loader);
|
||||||
if (init >= 5)
|
if (init >= 5)
|
||||||
|
|||||||
@@ -119,6 +119,10 @@ typedef struct {
|
|||||||
ArrayList* dir_entries;
|
ArrayList* dir_entries;
|
||||||
mtx_t dir_entries_mutex;
|
mtx_t dir_entries_mutex;
|
||||||
bool dir_entries_mutex_init;
|
bool dir_entries_mutex_init;
|
||||||
|
/* --stats directory accounting for a `-r` scan (no directory metadata):
|
||||||
|
shared by the parallel scanner workers, read by the sender thread once the
|
||||||
|
scanner is done. See ScannerOptions.dir_count. */
|
||||||
|
atomic_ullong dir_count;
|
||||||
/* Set by the sender thread when the receiver reported a --max-delete-capped
|
/* Set by the sender thread when the receiver reported a --max-delete-capped
|
||||||
deletion (STATUS_DELETE_LIMIT): the transfer succeeded and the process must
|
deletion (STATUS_DELETE_LIMIT): the transfer succeeded and the process must
|
||||||
exit 25 like rsync. Read by the caller after the sender thread is joined. */
|
exit 25 like rsync. Read by the caller after the sender thread is joined. */
|
||||||
|
|||||||
+42
-7
@@ -301,6 +301,14 @@ static ProtocolSession* legacy_session(int read_fd, int write_fd) {
|
|||||||
return &legacy_io_session;
|
return &legacy_io_session;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Pace an out-of-band write that bypassed protocol_send_n_data (the plaintext
|
||||||
|
* sendfile fast path). The bound/legacy session is resolved exactly as
|
||||||
|
* send_n_data resolves it, so the same token-bucket state is throttled and the
|
||||||
|
* TLS and plaintext transports share identical --bwlimit semantics. */
|
||||||
|
void protocol_throttle_bytes(size_t bytes) {
|
||||||
|
bw_throttle_session(legacy_session(-1, -1), bytes);
|
||||||
|
}
|
||||||
|
|
||||||
bool send_n_data(int file_descriptor, const void* data, size_t data_size) {
|
bool send_n_data(int file_descriptor, const void* data, size_t data_size) {
|
||||||
return protocol_send_n_data(legacy_session(-1, file_descriptor), data, data_size);
|
return protocol_send_n_data(legacy_session(-1, file_descriptor), data, data_size);
|
||||||
}
|
}
|
||||||
@@ -382,7 +390,7 @@ bool protocol_send_n_data(ProtocolSession* session, const void* data, size_t dat
|
|||||||
if (session->ssl)
|
if (session->ssl)
|
||||||
wait_events = POLLOUT;
|
wait_events = POLLOUT;
|
||||||
}
|
}
|
||||||
log_debug_message(LOG_DEBUG_IO, " Send n Data: %zu", total_bytes_send);
|
log_debug_message(LOG_DEBUG_IO, " Send n Data: %zd", total_bytes_send);
|
||||||
atomic_fetch_add(&io_bytes_written, (unsigned long long)total_bytes_send);
|
atomic_fetch_add(&io_bytes_written, (unsigned long long)total_bytes_send);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
@@ -439,12 +447,18 @@ static bool protocol_receive_n_data_until(ProtocolSession* session, void* data,
|
|||||||
}
|
}
|
||||||
|
|
||||||
ssize_t bytes_received;
|
ssize_t bytes_received;
|
||||||
if (session->ssl)
|
if (session->ssl) {
|
||||||
bytes_received = SSL_read(session->ssl, (char*)data + total_bytes_received,
|
/* SSL_read takes an int length; clamp a >INT_MAX request into chunks
|
||||||
data_size - total_bytes_received);
|
* (mirrors the send path) so the size_t downcast can never truncate into
|
||||||
else
|
* a negative/partial read. */
|
||||||
|
size_t ssl_chunk = data_size - total_bytes_received > (size_t)INT_MAX
|
||||||
|
? (size_t)INT_MAX
|
||||||
|
: data_size - total_bytes_received;
|
||||||
|
bytes_received = SSL_read(session->ssl, (char*)data + total_bytes_received, (int)ssl_chunk);
|
||||||
|
} else {
|
||||||
bytes_received =
|
bytes_received =
|
||||||
read(fd, (char*)data + total_bytes_received, data_size - total_bytes_received);
|
read(fd, (char*)data + total_bytes_received, data_size - total_bytes_received);
|
||||||
|
}
|
||||||
if (bytes_received <= 0) {
|
if (bytes_received <= 0) {
|
||||||
if (session->ssl) {
|
if (session->ssl) {
|
||||||
int ssl_err = SSL_get_error(session->ssl, (int)bytes_received);
|
int ssl_err = SSL_get_error(session->ssl, (int)bytes_received);
|
||||||
@@ -550,6 +564,15 @@ static const char* status_to_string(Status status) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Reject a raw wire status outside the known enum range before it is handed to
|
||||||
|
* callers, so an unknown/corrupt frame fails as a protocol error instead of
|
||||||
|
* being silently interpreted as an unexpected-but-valid verdict. STATUS_OK is
|
||||||
|
* the first enumerator and STATUS_STATS the last, so the range check accepts
|
||||||
|
* every status the protocol defines. */
|
||||||
|
static bool status_is_valid(Status status) {
|
||||||
|
return status >= STATUS_OK && status <= STATUS_STATS;
|
||||||
|
}
|
||||||
|
|
||||||
/* Shared string send/receive implementation. `redact` selects whether the
|
/* Shared string send/receive implementation. `redact` selects whether the
|
||||||
* payload body is written to the LOG_DEBUG_PROTO debug log: daemon auth material
|
* payload body is written to the LOG_DEBUG_PROTO debug log: daemon auth material
|
||||||
* (the username and the proof/signature fields) sets it so a --verbose log never
|
* (the username and the proof/signature fields) sets it so a --verbose log never
|
||||||
@@ -633,7 +656,7 @@ bool protocol_send_data(ProtocolSession* session, const Data* data) {
|
|||||||
return false;
|
return false;
|
||||||
if (!protocol_send_n_data(session, data->data, data_size))
|
if (!protocol_send_n_data(session, data->data, data_size))
|
||||||
return false;
|
return false;
|
||||||
log_debug_message(LOG_DEBUG_PROTO, "Send %lld data", data_size);
|
log_debug_message(LOG_DEBUG_PROTO, "Send %llu data", data_size);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -667,7 +690,7 @@ Data* protocol_receive_data_limited(ProtocolSession* session, unsigned long long
|
|||||||
protocol_release_memory_for_session(session, allocation_size);
|
protocol_release_memory_for_session(session, allocation_size);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
log_debug_message(LOG_DEBUG_PROTO, "Received %lld data", size);
|
log_debug_message(LOG_DEBUG_PROTO, "Received %llu data", size);
|
||||||
Data* result = data_create(data, (size_t)size);
|
Data* result = data_create(data, (size_t)size);
|
||||||
if (!result) {
|
if (!result) {
|
||||||
protocol_release_memory_for_session(session, allocation_size);
|
protocol_release_memory_for_session(session, allocation_size);
|
||||||
@@ -777,6 +800,10 @@ bool protocol_receive_status(ProtocolSession* session, Status* status) {
|
|||||||
}
|
}
|
||||||
if (!protocol_receive_n_data_until(session, status, sizeof(Status), deadline_ptr))
|
if (!protocol_receive_n_data_until(session, status, sizeof(Status), deadline_ptr))
|
||||||
return false;
|
return false;
|
||||||
|
if (!status_is_valid(*status)) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Received unknown protocol status %d", *status);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
if (!protocol_capture_error_detail(session, status, deadline_ptr, NULL))
|
if (!protocol_capture_error_detail(session, status, deadline_ptr, NULL))
|
||||||
return false;
|
return false;
|
||||||
log_debug_message(LOG_DEBUG_PROTO, "Received Status: %s", status_to_string(*status));
|
log_debug_message(LOG_DEBUG_PROTO, "Received Status: %s", status_to_string(*status));
|
||||||
@@ -798,6 +825,10 @@ bool protocol_receive_status_timed(ProtocolSession* session, Status* status, int
|
|||||||
deadline.tv_sec += timeout_sec;
|
deadline.tv_sec += timeout_sec;
|
||||||
if (!protocol_receive_n_data_until(session, status, sizeof(Status), &deadline))
|
if (!protocol_receive_n_data_until(session, status, sizeof(Status), &deadline))
|
||||||
return false;
|
return false;
|
||||||
|
if (!status_is_valid(*status)) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Received unknown protocol status %d", *status);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
if (!protocol_capture_error_detail(session, status, &deadline, NULL))
|
if (!protocol_capture_error_detail(session, status, &deadline, NULL))
|
||||||
return false;
|
return false;
|
||||||
log_debug_message(LOG_DEBUG_PROTO, "Received Status: %s", status_to_string(*status));
|
log_debug_message(LOG_DEBUG_PROTO, "Received Status: %s", status_to_string(*status));
|
||||||
@@ -911,6 +942,10 @@ bool protocol_receive_status_keepalive(ProtocolSession* session, Status* status,
|
|||||||
Status received;
|
Status received;
|
||||||
if (!protocol_read_status_until(session, &received, &deadline))
|
if (!protocol_read_status_until(session, &received, &deadline))
|
||||||
return false;
|
return false;
|
||||||
|
if (!status_is_valid(received)) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Received unknown protocol status %d", received);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
if (!protocol_capture_error_detail(session, &received, &deadline, abort_check))
|
if (!protocol_capture_error_detail(session, &received, &deadline, abort_check))
|
||||||
return false;
|
return false;
|
||||||
if (received == STATUS_KEEPALIVE) {
|
if (received == STATUS_KEEPALIVE) {
|
||||||
|
|||||||
@@ -28,6 +28,10 @@
|
|||||||
|
|
||||||
/* Maximum chunk size (64 MB) — prevents unbounded allocation from the wire */
|
/* Maximum chunk size (64 MB) — prevents unbounded allocation from the wire */
|
||||||
#define MAX_CHUNK_SIZE (64ULL * 1024 * 1024)
|
#define MAX_CHUNK_SIZE (64ULL * 1024 * 1024)
|
||||||
|
/* Files larger than this are not kept fully in memory while loading: the
|
||||||
|
* loader skips them so the sender streams from the path, and file_checksum
|
||||||
|
* hashes them from disk in bounded buffers instead of forcing a full load. */
|
||||||
|
#define STREAM_THRESHOLD (64ULL * 1024 * 1024)
|
||||||
#define MAX_MANIFEST_ENTRIES (1024 * 1024)
|
#define MAX_MANIFEST_ENTRIES (1024 * 1024)
|
||||||
/* Aggregate bytes retained by one received deletion manifest. */
|
/* Aggregate bytes retained by one received deletion manifest. */
|
||||||
#define MAX_MANIFEST_BYTES (16ULL * 1024 * 1024)
|
#define MAX_MANIFEST_BYTES (16ULL * 1024 * 1024)
|
||||||
@@ -220,6 +224,12 @@ SSL* io_get_ssl(void);
|
|||||||
unsigned long long protocol_bytes_written(void);
|
unsigned long long protocol_bytes_written(void);
|
||||||
unsigned long long protocol_bytes_read(void);
|
unsigned long long protocol_bytes_read(void);
|
||||||
void protocol_note_bytes_written(unsigned long long bytes);
|
void protocol_note_bytes_written(unsigned long long bytes);
|
||||||
|
/* Apply --bwlimit pacing to bytes written outside protocol_send_n_data (the
|
||||||
|
* plaintext zero-copy sendfile fast path). Resolves the bound/legacy session
|
||||||
|
* exactly as send_n_data does and runs the same token-bucket throttle, so the
|
||||||
|
* sendfile transport is paced identically to the buffered/TLS paths. A no-op
|
||||||
|
* when the effective session has no bandwidth limit. */
|
||||||
|
void protocol_throttle_bytes(size_t bytes);
|
||||||
|
|
||||||
void protocol_session_init(ProtocolSession* session, int read_fd, int write_fd);
|
void protocol_session_init(ProtocolSession* session, int read_fd, int write_fd);
|
||||||
/* Transitional bridge for helpers whose signatures still carry only an fd. */
|
/* Transitional bridge for helpers whose signatures still carry only an fd. */
|
||||||
@@ -263,6 +273,9 @@ bool protocol_send_int(ProtocolSession* session, int data);
|
|||||||
bool protocol_receive_int(ProtocolSession* session, int* data);
|
bool protocol_receive_int(ProtocolSession* session, int* data);
|
||||||
bool protocol_send_status(ProtocolSession* session, Status status);
|
bool protocol_send_status(ProtocolSession* session, Status status);
|
||||||
bool protocol_receive_status(ProtocolSession* session, Status* status);
|
bool protocol_receive_status(ProtocolSession* session, Status* status);
|
||||||
|
/* As protocol_receive_status, but with an explicit per-message deadline
|
||||||
|
* (seconds) instead of the session's configured io_timeout_sec. */
|
||||||
|
bool protocol_receive_status_timed(ProtocolSession* session, Status* status, int timeout_sec);
|
||||||
bool send_n_data(int file_descriptor, const void* data, size_t data_size);
|
bool send_n_data(int file_descriptor, const void* data, size_t data_size);
|
||||||
bool receive_n_data(int file_descriptor, void* data, size_t data_size);
|
bool receive_n_data(int file_descriptor, void* data, size_t data_size);
|
||||||
|
|
||||||
|
|||||||
+18
-1
@@ -128,7 +128,7 @@ bool queue_enqueue_multithreaded_cancel(Queue* queue, void* item, mtx_t* mutex,
|
|||||||
|
|
||||||
void* queue_dequeue(Queue* queue) {
|
void* queue_dequeue(Queue* queue) {
|
||||||
if (queue == NULL || queue_is_empty(queue)) {
|
if (queue == NULL || queue_is_empty(queue)) {
|
||||||
log_perror("ERROR: Could not dequeue from null or empty queue.");
|
log_message(LOG_LEVEL_ERROR, "%s", "ERROR: Could not dequeue from null or empty queue.");
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -139,6 +139,23 @@ void* queue_dequeue(Queue* queue) {
|
|||||||
return item;
|
return item;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool queue_push(Queue* queue, void* item) {
|
||||||
|
return queue_enqueue(queue, item);
|
||||||
|
}
|
||||||
|
|
||||||
|
void* queue_pop(Queue* queue) {
|
||||||
|
if (queue == NULL || queue_is_empty(queue)) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "%s", "ERROR: Could not pop from null or empty queue.");
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
queue->rear = (queue->rear - 1 + queue->capacity) % queue->capacity;
|
||||||
|
void* item = queue->items[queue->rear];
|
||||||
|
queue->items[queue->rear] = NULL;
|
||||||
|
queue->size--;
|
||||||
|
return item;
|
||||||
|
}
|
||||||
|
|
||||||
void* queue_dequeue_multithreaded(Queue* queue, mtx_t* mutex, cnd_t* condition_not_empty,
|
void* queue_dequeue_multithreaded(Queue* queue, mtx_t* mutex, cnd_t* condition_not_empty,
|
||||||
cnd_t* condition_not_full, const bool* other_thread_done) {
|
cnd_t* condition_not_full, const bool* other_thread_done) {
|
||||||
mtx_lock(mutex);
|
mtx_lock(mutex);
|
||||||
|
|||||||
@@ -28,4 +28,11 @@ void* queue_dequeue(Queue* queue);
|
|||||||
void* queue_dequeue_multithreaded(Queue* queue, mtx_t* mutex, cnd_t* condition_not_empty,
|
void* queue_dequeue_multithreaded(Queue* queue, mtx_t* mutex, cnd_t* condition_not_empty,
|
||||||
cnd_t* condition_not_full, const bool* other_thread_done);
|
cnd_t* condition_not_full, const bool* other_thread_done);
|
||||||
|
|
||||||
|
/* LIFO stack operations over the same ring buffer. queue_push() is the enqueue
|
||||||
|
primitive; queue_pop() removes from the rear, so a sequence of pushes is
|
||||||
|
returned in reverse order. Used by the sequential scanner's depth-first
|
||||||
|
traversal. */
|
||||||
|
bool queue_push(Queue* queue, void* item);
|
||||||
|
void* queue_pop(Queue* queue);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
@@ -87,7 +87,7 @@ static int parse_remote_dest(const char* dest, RemoteDest* r) {
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
char* ssh_build_remote_command(const char* server_path, bool old_args, char* const* remote_options,
|
char* ssh_build_remote_command(const char* server_path, char* const* remote_options,
|
||||||
int remote_option_count) {
|
int remote_option_count) {
|
||||||
const char* path = server_path ? server_path : "fastsync-server";
|
const char* path = server_path ? server_path : "fastsync-server";
|
||||||
const char* suffix = " --stdio";
|
const char* suffix = " --stdio";
|
||||||
@@ -105,9 +105,7 @@ char* ssh_build_remote_command(const char* server_path, bool old_args, char* con
|
|||||||
shell word (remote options below reuse the same escaping), then
|
shell word (remote options below reuse the same escaping), then
|
||||||
" --stdio". Quoting the path is the only injection-safe construction: an
|
" --stdio". Quoting the path is the only injection-safe construction: an
|
||||||
unquoted path would carry shell metacharacters straight into the remote
|
unquoted path would carry shell metacharacters straight into the remote
|
||||||
shell command. --old-args is kept for CLI/ABI compatibility but no longer
|
shell command. (rsync's --old-args no longer disables that protection.) */
|
||||||
disables that protection. */
|
|
||||||
(void)old_args;
|
|
||||||
size_t quote_count = 0;
|
size_t quote_count = 0;
|
||||||
for (const char* p = path; *p; p++)
|
for (const char* p = path; *p; p++)
|
||||||
if (*p == '\'')
|
if (*p == '\'')
|
||||||
@@ -296,8 +294,8 @@ void ssh_free_client_argv(char** argv) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
Client* client_connect_ssh(const char* destination, int port, const char* server_path,
|
Client* client_connect_ssh(const char* destination, int port, const char* server_path,
|
||||||
bool old_args, const char* rsh_command, bool blocking_io,
|
const char* rsh_command, bool blocking_io, char* const* remote_options,
|
||||||
char* const* remote_options, int remote_option_count) {
|
int remote_option_count) {
|
||||||
RemoteDest r;
|
RemoteDest r;
|
||||||
if (parse_remote_dest(destination, &r) != 0) {
|
if (parse_remote_dest(destination, &r) != 0) {
|
||||||
char* escaped = output_escape(destination, false);
|
char* escaped = output_escape(destination, false);
|
||||||
@@ -376,7 +374,7 @@ Client* client_connect_ssh(const char* destination, int port, const char* server
|
|||||||
snprintf(ssh_user, ssh_user_len, "%s", r.host);
|
snprintf(ssh_user, ssh_user_len, "%s", r.host);
|
||||||
|
|
||||||
char* remote_command =
|
char* remote_command =
|
||||||
ssh_build_remote_command(server_path, old_args, remote_options, remote_option_count);
|
ssh_build_remote_command(server_path, remote_options, remote_option_count);
|
||||||
if (!remote_command)
|
if (!remote_command)
|
||||||
ssh_child_setup_failed(exec_pipe[1]);
|
ssh_child_setup_failed(exec_pipe[1]);
|
||||||
char** ssh_argv = ssh_build_client_argv(rsh_command, port, ssh_user, remote_command);
|
char** ssh_argv = ssh_build_client_argv(rsh_command, port, ssh_user, remote_command);
|
||||||
|
|||||||
@@ -4,17 +4,17 @@
|
|||||||
#include "transport_tcp.h"
|
#include "transport_tcp.h"
|
||||||
|
|
||||||
Client* client_connect_ssh(const char* destination, int port, const char* server_path,
|
Client* client_connect_ssh(const char* destination, int port, const char* server_path,
|
||||||
bool old_args, const char* rsh_command, bool blocking_io,
|
const char* rsh_command, bool blocking_io, char* const* remote_options,
|
||||||
char* const* remote_options, int remote_option_count);
|
int remote_option_count);
|
||||||
/* Build the escaped remote-shell command string (the server program path always
|
/* Build the escaped remote-shell command string (the server program path always
|
||||||
* quoted as one remote-shell word, followed by ` --stdio` and each
|
* quoted as one remote-shell word, followed by ` --stdio` and each
|
||||||
* --remote-option value appended as an individually single-quoted shell word).
|
* --remote-option value appended as an individually single-quoted shell word).
|
||||||
* `old_args` is accepted for CLI/ABI compatibility but no longer disables
|
* The path is always escaped so a metacharacter-bearing --rsync-path can never
|
||||||
* quoting: the path is always escaped so a metacharacter-bearing
|
* be interpreted by the remote shell (the --old-args no-op does not disable
|
||||||
* --rsync-path can never be interpreted by the remote shell. Every
|
* quoting). Every --remote-option value is individually escaped with the '\''
|
||||||
* --remote-option value is individually escaped with the '\'' sequence and
|
* sequence and values with empty/control characters are rejected at the CLI
|
||||||
* values with empty/control characters are rejected at the CLI parse layer. */
|
* parse layer. */
|
||||||
char* ssh_build_remote_command(const char* server_path, bool old_args, char* const* remote_options,
|
char* ssh_build_remote_command(const char* server_path, char* const* remote_options,
|
||||||
int remote_option_count);
|
int remote_option_count);
|
||||||
/* Build the NULL-terminated child argv for the remote-shell client (argv[0] is
|
/* Build the NULL-terminated child argv for the remote-shell client (argv[0] is
|
||||||
* the exec/execvp program). rsh_command is whitespace-split into leading argv
|
* the exec/execvp program). rsh_command is whitespace-split into leading argv
|
||||||
|
|||||||
+321
-100
@@ -7,6 +7,7 @@
|
|||||||
#include <errno.h>
|
#include <errno.h>
|
||||||
#include <fcntl.h>
|
#include <fcntl.h>
|
||||||
#include <netinet/in.h>
|
#include <netinet/in.h>
|
||||||
|
#include <stdarg.h>
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
@@ -48,6 +49,15 @@ const char* utils_get_authorized_root_path(void) {
|
|||||||
return authorized_root_path;
|
return authorized_root_path;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void utils_set_error(char* err, size_t err_size, const char* fmt, ...) {
|
||||||
|
if (!err || err_size == 0)
|
||||||
|
return;
|
||||||
|
va_list args;
|
||||||
|
va_start(args, fmt);
|
||||||
|
vsnprintf(err, err_size, fmt, args);
|
||||||
|
va_end(args);
|
||||||
|
}
|
||||||
|
|
||||||
bool path_is_within_root(const char* root, const char* path) {
|
bool path_is_within_root(const char* root, const char* path) {
|
||||||
size_t root_len = strlen(root);
|
size_t root_len = strlen(root);
|
||||||
return strncmp(root, path, root_len) == 0 && (path[root_len] == '\0' || path[root_len] == '/');
|
return strncmp(root, path, root_len) == 0 && (path[root_len] == '\0' || path[root_len] == '/');
|
||||||
@@ -672,22 +682,24 @@ static bool is_synced_dir(const PathIndex* dirs, const char* rel) {
|
|||||||
return path_index_contains(dirs, rel[0] == '\0' ? "." : rel);
|
return path_index_contains(dirs, rel[0] == '\0' ? "." : rel);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Remove the extras directly inside the directory open on `dirfd`, recursing
|
/* Unsigned byte-wise string compare, matching rsync's u_strcmp (a signed
|
||||||
into every child directory so kept content below a synchronized prefix is
|
strcmp would order bytes >= 0x80 differently). */
|
||||||
reached. `all_removed` reports whether every child entry was removed (so the
|
static int delete_name_cmp(const char* a, const char* b) {
|
||||||
caller may rmdir this directory). A child directory is never removed when it
|
const unsigned char* pa = (const unsigned char*)a;
|
||||||
is itself a synchronized directory or holds kept content; with a dirs index
|
const unsigned char* pb = (const unsigned char*)b;
|
||||||
supplied, direct children of a non-synchronized directory are never extras at
|
while (*pa != '\0' && *pa == *pb) {
|
||||||
all (they are left in place but still descended into). Symlinks are unlinked
|
pa++;
|
||||||
like any other non-directory extra (never followed). */
|
pb++;
|
||||||
static bool delete_extras_fd(int dirfd, const char* rel_path, const PathIndex* keep,
|
}
|
||||||
const PathIndex* dirs, DeleteBudget* budget,
|
return (int)*pa - (int)*pb;
|
||||||
const DeleteSkipEntry* skips, int skip_count,
|
}
|
||||||
const FilterRuleList* protect_rules, bool parent_deletable,
|
|
||||||
bool* all_removed, DeletePathObserver observer,
|
bool delete_dir_entries_collect(int dirfd, DeleteDirEntry** out, size_t* count,
|
||||||
void* observer_context) {
|
bool* operation_ok) {
|
||||||
/* openat(dirfd, ".") opens an independent file description: a dup() would
|
*out = NULL;
|
||||||
share dirfd's file offset and a prior pass could leave the stream drained. */
|
*count = 0;
|
||||||
|
if (operation_ok)
|
||||||
|
*operation_ok = true;
|
||||||
int scanfd = openat(dirfd, ".", O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
int scanfd = openat(dirfd, ".", O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||||
if (scanfd < 0)
|
if (scanfd < 0)
|
||||||
return false;
|
return false;
|
||||||
@@ -696,17 +708,127 @@ static bool delete_extras_fd(int dirfd, const char* rel_path, const PathIndex* k
|
|||||||
close(scanfd);
|
close(scanfd);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
bool operation_ok = true;
|
DeleteDirEntry* entries = NULL;
|
||||||
bool local_survives = false;
|
size_t used = 0;
|
||||||
/* A directory is deletable when it or ANY ancestor is synchronized; the
|
size_t capacity = 0;
|
||||||
`parent_deletable` flag carries that down the recursion so dest-only
|
bool ok = true;
|
||||||
directories below a synchronized root are removed wholesale. */
|
|
||||||
bool deletable = parent_deletable || is_synced_dir(dirs, rel_path);
|
|
||||||
const struct dirent* entry;
|
const struct dirent* entry;
|
||||||
while ((entry = readdir(dir)) != NULL) {
|
while ((entry = readdir(dir)) != NULL) {
|
||||||
if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
|
if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
|
||||||
continue;
|
continue;
|
||||||
char* child_rel = path_cat((char*)rel_path, entry->d_name);
|
struct stat st;
|
||||||
|
if (fstatat(dirfd, entry->d_name, &st, AT_SYMLINK_NOFOLLOW) != 0) {
|
||||||
|
if (errno != ENOENT && operation_ok)
|
||||||
|
*operation_ok = false;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (used == capacity) {
|
||||||
|
size_t next = capacity == 0 ? 16 : capacity * 2;
|
||||||
|
DeleteDirEntry* grown = realloc(entries, next * sizeof(*grown));
|
||||||
|
if (!grown) {
|
||||||
|
ok = false;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
entries = grown;
|
||||||
|
capacity = next;
|
||||||
|
}
|
||||||
|
entries[used].name = str_dup(entry->d_name);
|
||||||
|
if (!entries[used].name) {
|
||||||
|
ok = false;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
entries[used].is_dir = S_ISDIR(st.st_mode);
|
||||||
|
used++;
|
||||||
|
}
|
||||||
|
closedir(dir);
|
||||||
|
if (!ok) {
|
||||||
|
delete_dir_entries_free(entries, used);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
*out = entries;
|
||||||
|
*count = used;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
void delete_dir_entries_free(DeleteDirEntry* entries, size_t count) {
|
||||||
|
if (!entries)
|
||||||
|
return;
|
||||||
|
for (size_t i = 0; i < count; i++)
|
||||||
|
free(entries[i].name);
|
||||||
|
free(entries);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* rsync's extraneous-entry order: subdirectories before files, each group in
|
||||||
|
descending name order. */
|
||||||
|
int delete_dir_entry_cmp_desc(const void* a, const void* b) {
|
||||||
|
const DeleteDirEntry* ea = a;
|
||||||
|
const DeleteDirEntry* eb = b;
|
||||||
|
if (ea->is_dir != eb->is_dir)
|
||||||
|
return ea->is_dir ? -1 : 1;
|
||||||
|
return -delete_name_cmp(ea->name, eb->name);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* rsync's kept-subdirectory order: plain ascending name. */
|
||||||
|
int delete_dir_entry_cmp_asc(const void* a, const void* b) {
|
||||||
|
const DeleteDirEntry* ea = a;
|
||||||
|
const DeleteDirEntry* eb = b;
|
||||||
|
return delete_name_cmp(ea->name, eb->name);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Remove the extras directly inside the directory open on `dirfd`, recursing
|
||||||
|
into every child directory so kept content below a synchronized prefix is
|
||||||
|
reached. `all_removed` reports whether every child entry was removed (so the
|
||||||
|
caller may rmdir this directory). A child directory is never removed when it
|
||||||
|
is itself a synchronized directory or holds kept content; with a dirs index
|
||||||
|
supplied, direct children of a non-synchronized directory are never extras at
|
||||||
|
all (they are left in place but still descended into). Symlinks are unlinked
|
||||||
|
like any other non-directory extra (never followed).
|
||||||
|
|
||||||
|
Entries are processed in rsync's order (extraneous subdirectories in
|
||||||
|
descending name order, then extraneous files, then kept subdirectories in
|
||||||
|
ascending order) rather than readdir() order, so `--max-delete` leaves the
|
||||||
|
same survivors and the `--info=del`/dry-run line order matches rsync. */
|
||||||
|
static bool delete_extras_fd(int dirfd, const char* rel_path, const PathIndex* keep,
|
||||||
|
const PathIndex* dirs, DeleteBudget* budget,
|
||||||
|
const DeleteSkipEntry* skips, int skip_count,
|
||||||
|
const FilterRuleList* protect_rules, bool parent_deletable,
|
||||||
|
bool* all_removed, DeletePathObserver observer,
|
||||||
|
void* observer_context) {
|
||||||
|
DeleteDirEntry* entries = NULL;
|
||||||
|
size_t count = 0;
|
||||||
|
bool collect_ok = true;
|
||||||
|
if (!delete_dir_entries_collect(dirfd, &entries, &count, &collect_ok))
|
||||||
|
return false;
|
||||||
|
bool operation_ok = collect_ok;
|
||||||
|
bool local_survives = false;
|
||||||
|
bool* shielded = calloc(count ? count : 1, sizeof(bool));
|
||||||
|
bool* is_extra = calloc(count ? count : 1, sizeof(bool));
|
||||||
|
if (!shielded || !is_extra) {
|
||||||
|
free(shielded);
|
||||||
|
free(is_extra);
|
||||||
|
delete_dir_entries_free(entries, count);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
/* A directory is deletable when it or ANY ancestor is synchronized; the
|
||||||
|
`parent_deletable` flag carries that down the recursion so dest-only
|
||||||
|
directories below a synchronized root are removed wholesale. */
|
||||||
|
bool deletable = parent_deletable || is_synced_dir(dirs, rel_path);
|
||||||
|
bool at_root = rel_path[0] == '\0';
|
||||||
|
|
||||||
|
/* Reproduce rsync's traversal order: extraneous subdirectories in descending
|
||||||
|
name order, then extraneous files in descending name order, and kept
|
||||||
|
subdirectories only afterwards (ascending). Sorting up front also fixes the
|
||||||
|
identity of the survivors under a partial --max-delete. */
|
||||||
|
if (count > 1)
|
||||||
|
qsort(entries, count, sizeof(*entries), delete_dir_entry_cmp_desc);
|
||||||
|
size_t dir_count = 0;
|
||||||
|
while (dir_count < count && entries[dir_count].is_dir)
|
||||||
|
dir_count++;
|
||||||
|
|
||||||
|
/* Classify every entry up front (the verdict does not depend on processing
|
||||||
|
order) so the ordered passes below can act on it. */
|
||||||
|
for (size_t i = 0; i < count; i++) {
|
||||||
|
char* child_rel = path_cat((char*)rel_path, entries[i].name);
|
||||||
if (!child_rel) {
|
if (!child_rel) {
|
||||||
operation_ok = false;
|
operation_ok = false;
|
||||||
continue;
|
continue;
|
||||||
@@ -718,30 +840,40 @@ static bool delete_extras_fd(int dirfd, const char* rel_path, const PathIndex* k
|
|||||||
top-level-only prefix) and the basis prefixes are protected: a nested
|
top-level-only prefix) and the basis prefixes are protected: a nested
|
||||||
destination directory that happens to be called .fastsync-stage is
|
destination directory that happens to be called .fastsync-stage is
|
||||||
ordinary content. */
|
ordinary content. */
|
||||||
if (path_under_skip_prefix(child_rel, rel_path[0] == '\0', skips, skip_count)) {
|
if (path_under_skip_prefix(child_rel, at_root, skips, skip_count)) {
|
||||||
|
shielded[i] = true;
|
||||||
local_survives = true;
|
local_survives = true;
|
||||||
free(child_rel);
|
} else if (protect_rules &&
|
||||||
continue;
|
filter_rules_apply_side(protect_rules, child_rel, entries[i].name, entries[i].is_dir,
|
||||||
}
|
|
||||||
struct stat st;
|
|
||||||
if (fstatat(dirfd, entry->d_name, &st, AT_SYMLINK_NOFOLLOW) != 0) {
|
|
||||||
if (errno != ENOENT)
|
|
||||||
operation_ok = false;
|
|
||||||
free(child_rel);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
bool is_dir = S_ISDIR(st.st_mode);
|
|
||||||
if (protect_rules && filter_rules_apply_side(protect_rules, child_rel, entry->d_name, is_dir,
|
|
||||||
FILTER_SIDE_RECEIVER) == FILTER_ACTION_PROTECT) {
|
FILTER_SIDE_RECEIVER) == FILTER_ACTION_PROTECT) {
|
||||||
/* A first-match protect rule shields the extra; for a directory the whole
|
/* A first-match protect rule shields the extra; for a directory the whole
|
||||||
subtree is shielded (rsync prunes an excluded directory), so do not
|
subtree is shielded (rsync prunes an excluded directory), so do not
|
||||||
descend. */
|
descend. */
|
||||||
|
shielded[i] = true;
|
||||||
local_survives = true;
|
local_survives = true;
|
||||||
|
} else if (entries[i].is_dir) {
|
||||||
|
bool child_synced = dirs && path_index_contains(dirs, child_rel);
|
||||||
|
is_extra[i] = deletable && !child_synced && !keep_is_dir(keep, child_rel);
|
||||||
|
if (!is_extra[i])
|
||||||
|
local_survives = true;
|
||||||
|
} else {
|
||||||
|
is_extra[i] = deletable && !keep_is_file(keep, child_rel);
|
||||||
|
if (!is_extra[i])
|
||||||
|
local_survives = true;
|
||||||
|
}
|
||||||
free(child_rel);
|
free(child_rel);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Pass 1: extraneous subdirectories, descending. */
|
||||||
|
for (size_t i = 0; i < dir_count; i++) {
|
||||||
|
if (!is_extra[i])
|
||||||
|
continue;
|
||||||
|
char* child_rel = path_cat((char*)rel_path, entries[i].name);
|
||||||
|
if (!child_rel) {
|
||||||
|
operation_ok = false;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if (is_dir) {
|
int childfd = openat(dirfd, entries[i].name, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||||
int childfd = openat(dirfd, entry->d_name, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
|
||||||
bool child_all_removed = false;
|
bool child_all_removed = false;
|
||||||
if (childfd >= 0) {
|
if (childfd >= 0) {
|
||||||
if (!delete_extras_fd(childfd, child_rel, keep, dirs, budget, skips, skip_count,
|
if (!delete_extras_fd(childfd, child_rel, keep, dirs, budget, skips, skip_count,
|
||||||
@@ -752,59 +884,98 @@ static bool delete_extras_fd(int dirfd, const char* rel_path, const PathIndex* k
|
|||||||
} else if (errno != ENOENT) {
|
} else if (errno != ENOENT) {
|
||||||
operation_ok = false;
|
operation_ok = false;
|
||||||
}
|
}
|
||||||
bool child_synced = dirs && path_index_contains(dirs, child_rel);
|
if (child_all_removed && deletable) {
|
||||||
if (child_synced || keep_is_dir(keep, child_rel)) {
|
|
||||||
/* A synchronized directory and a directory holding kept content are
|
|
||||||
never removed. */
|
|
||||||
local_survives = true;
|
|
||||||
} else if (child_all_removed && deletable) {
|
|
||||||
if (budget->deleted >= budget->max_delete) {
|
if (budget->deleted >= budget->max_delete) {
|
||||||
budget->limit_hit = true;
|
budget->limit_hit = true;
|
||||||
budget->skipped++;
|
budget->skipped++;
|
||||||
local_survives = true;
|
local_survives = true;
|
||||||
} else if (unlinkat(dirfd, entry->d_name, AT_REMOVEDIR) != 0) {
|
} else if (unlinkat(dirfd, entries[i].name, AT_REMOVEDIR) != 0) {
|
||||||
/* ENOENT: already gone (fine). ENOTEMPTY/EEXIST: the directory
|
/* ENOENT: already gone (fine). ENOTEMPTY/EEXIST: the directory still
|
||||||
still holds entries the walker leaves in place (a protected
|
holds entries the walker leaves in place (a protected excluded
|
||||||
excluded prefix, a kept file the manifest protects, a symlink);
|
prefix, a kept file the manifest protects, a symlink); rsync leaves
|
||||||
rsync leaves such a directory behind, so this is not an error.
|
such a directory behind, so this is not an error. Only genuine I/O
|
||||||
Only genuine I/O failures abort the deletion. */
|
failures abort the deletion. */
|
||||||
if (errno != ENOENT && errno != ENOTEMPTY && errno != EEXIST)
|
if (errno != ENOENT && errno != ENOTEMPTY && errno != EEXIST)
|
||||||
operation_ok = false;
|
operation_ok = false;
|
||||||
local_survives = true;
|
local_survives = true;
|
||||||
} else {
|
} else {
|
||||||
budget->deleted++;
|
budget->deleted++;
|
||||||
if (observer)
|
/* rsync reports a removed directory with a trailing slash. */
|
||||||
|
if (observer) {
|
||||||
|
size_t len = strlen(child_rel);
|
||||||
|
char* with_slash = malloc(len + 2);
|
||||||
|
if (with_slash) {
|
||||||
|
memcpy(with_slash, child_rel, len);
|
||||||
|
with_slash[len] = '/';
|
||||||
|
with_slash[len + 1] = '\0';
|
||||||
|
observer(observer_context, with_slash);
|
||||||
|
free(with_slash);
|
||||||
|
} else {
|
||||||
observer(observer_context, child_rel);
|
observer(observer_context, child_rel);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
local_survives = true;
|
local_survives = true;
|
||||||
}
|
}
|
||||||
} else {
|
free(child_rel);
|
||||||
bool found = keep_is_file(keep, child_rel);
|
}
|
||||||
if (found || !deletable) {
|
|
||||||
/* Kept file, or a child of a directory that is not synchronized: never
|
/* Pass 2: extraneous files, descending. */
|
||||||
an extra for this run. */
|
for (size_t i = dir_count; i < count; i++) {
|
||||||
local_survives = true;
|
if (!is_extra[i])
|
||||||
} else if (budget->deleted >= budget->max_delete) {
|
continue;
|
||||||
|
if (budget->deleted >= budget->max_delete) {
|
||||||
budget->limit_hit = true;
|
budget->limit_hit = true;
|
||||||
budget->skipped++;
|
budget->skipped++;
|
||||||
local_survives = true;
|
local_survives = true;
|
||||||
} else if (unlinkat(dirfd, entry->d_name, 0) != 0) {
|
} else if (unlinkat(dirfd, entries[i].name, 0) != 0) {
|
||||||
if (errno != ENOENT)
|
if (errno != ENOENT)
|
||||||
operation_ok = false;
|
operation_ok = false;
|
||||||
local_survives = true;
|
local_survives = true;
|
||||||
} else {
|
} else {
|
||||||
budget->deleted++;
|
budget->deleted++;
|
||||||
|
char* child_rel = path_cat((char*)rel_path, entries[i].name);
|
||||||
|
if (child_rel) {
|
||||||
if (observer)
|
if (observer)
|
||||||
observer(observer_context, child_rel);
|
observer(observer_context, child_rel);
|
||||||
char* escaped_path = output_escape(child_rel, log_get_8_bit_output());
|
char* escaped_path = output_escape(child_rel, log_get_8_bit_output());
|
||||||
fprintf(stderr, " Deleted: %s\n", escaped_path ? escaped_path : "<allocation failed>");
|
fprintf(stderr, " Deleted: %s\n", escaped_path ? escaped_path : "<allocation failed>");
|
||||||
free(escaped_path);
|
free(escaped_path);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
free(child_rel);
|
free(child_rel);
|
||||||
}
|
}
|
||||||
closedir(dir);
|
}
|
||||||
|
|
||||||
|
/* Pass 3: kept subdirectories, ascending (rsync descends into these only
|
||||||
|
after the parent's own extras have been handled). */
|
||||||
|
for (size_t i = dir_count; i-- > 0;) {
|
||||||
|
if (is_extra[i] || shielded[i])
|
||||||
|
continue;
|
||||||
|
char* child_rel = path_cat((char*)rel_path, entries[i].name);
|
||||||
|
if (!child_rel) {
|
||||||
|
operation_ok = false;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
int childfd = openat(dirfd, entries[i].name, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||||
|
bool child_all_removed = false;
|
||||||
|
if (childfd >= 0) {
|
||||||
|
if (!delete_extras_fd(childfd, child_rel, keep, dirs, budget, skips, skip_count,
|
||||||
|
protect_rules, deletable, &child_all_removed, observer,
|
||||||
|
observer_context))
|
||||||
|
operation_ok = false;
|
||||||
|
close(childfd);
|
||||||
|
} else if (errno != ENOENT) {
|
||||||
|
operation_ok = false;
|
||||||
|
}
|
||||||
|
/* A kept/synchronized directory is never removed. */
|
||||||
|
local_survives = true;
|
||||||
|
free(child_rel);
|
||||||
|
}
|
||||||
|
|
||||||
|
free(shielded);
|
||||||
|
free(is_extra);
|
||||||
|
delete_dir_entries_free(entries, count);
|
||||||
*all_removed = !local_survives;
|
*all_removed = !local_survives;
|
||||||
return operation_ok;
|
return operation_ok;
|
||||||
}
|
}
|
||||||
@@ -817,49 +988,71 @@ static bool list_extras_fd(int dirfd, const char* rel_path, const PathIndex* kee
|
|||||||
const DeleteSkipEntry* skips, int skip_count,
|
const DeleteSkipEntry* skips, int skip_count,
|
||||||
const FilterRuleList* protect_rules, bool parent_deletable,
|
const FilterRuleList* protect_rules, bool parent_deletable,
|
||||||
bool* all_removed) {
|
bool* all_removed) {
|
||||||
int scanfd = openat(dirfd, ".", O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
DeleteDirEntry* entries = NULL;
|
||||||
if (scanfd < 0)
|
size_t count = 0;
|
||||||
|
bool collect_ok = true;
|
||||||
|
if (!delete_dir_entries_collect(dirfd, &entries, &count, &collect_ok))
|
||||||
return false;
|
return false;
|
||||||
DIR* dir = fdopendir(scanfd);
|
bool operation_ok = collect_ok;
|
||||||
if (!dir) {
|
bool local_survives = false;
|
||||||
close(scanfd);
|
bool* shielded = calloc(count ? count : 1, sizeof(bool));
|
||||||
|
bool* is_extra = calloc(count ? count : 1, sizeof(bool));
|
||||||
|
if (!shielded || !is_extra) {
|
||||||
|
free(shielded);
|
||||||
|
free(is_extra);
|
||||||
|
delete_dir_entries_free(entries, count);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
bool operation_ok = true;
|
|
||||||
bool local_survives = false;
|
|
||||||
bool deletable = parent_deletable || is_synced_dir(dirs, rel_path);
|
bool deletable = parent_deletable || is_synced_dir(dirs, rel_path);
|
||||||
const struct dirent* entry;
|
bool at_root = rel_path[0] == '\0';
|
||||||
while ((entry = readdir(dir)) != NULL) {
|
|
||||||
if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
|
/* Mirror the delete walk's rsync order (extraneous subdirectories descending,
|
||||||
continue;
|
then extraneous files descending, then kept subdirectories ascending). */
|
||||||
char* child_rel = path_cat((char*)rel_path, entry->d_name);
|
if (count > 1)
|
||||||
|
qsort(entries, count, sizeof(*entries), delete_dir_entry_cmp_desc);
|
||||||
|
size_t dir_count = 0;
|
||||||
|
while (dir_count < count && entries[dir_count].is_dir)
|
||||||
|
dir_count++;
|
||||||
|
|
||||||
|
for (size_t i = 0; i < count; i++) {
|
||||||
|
char* child_rel = path_cat((char*)rel_path, entries[i].name);
|
||||||
if (!child_rel) {
|
if (!child_rel) {
|
||||||
operation_ok = false;
|
operation_ok = false;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if (path_under_skip_prefix(child_rel, rel_path[0] == '\0', skips, skip_count)) {
|
if (path_under_skip_prefix(child_rel, at_root, skips, skip_count)) {
|
||||||
|
shielded[i] = true;
|
||||||
local_survives = true;
|
local_survives = true;
|
||||||
free(child_rel);
|
} else if (protect_rules &&
|
||||||
continue;
|
filter_rules_apply_side(protect_rules, child_rel, entries[i].name, entries[i].is_dir,
|
||||||
}
|
|
||||||
struct stat st;
|
|
||||||
if (fstatat(dirfd, entry->d_name, &st, AT_SYMLINK_NOFOLLOW) != 0) {
|
|
||||||
if (errno != ENOENT)
|
|
||||||
operation_ok = false;
|
|
||||||
free(child_rel);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
bool is_dir = S_ISDIR(st.st_mode);
|
|
||||||
if (protect_rules && filter_rules_apply_side(protect_rules, child_rel, entry->d_name, is_dir,
|
|
||||||
FILTER_SIDE_RECEIVER) == FILTER_ACTION_PROTECT) {
|
FILTER_SIDE_RECEIVER) == FILTER_ACTION_PROTECT) {
|
||||||
/* Mirror the delete walk: a protected entry is never reported as a
|
/* Mirror the delete walk: a protected entry is never reported as a
|
||||||
would-delete and a protected directory's subtree is not enumerated. */
|
would-delete and a protected directory's subtree is not enumerated. */
|
||||||
|
shielded[i] = true;
|
||||||
local_survives = true;
|
local_survives = true;
|
||||||
|
} else if (entries[i].is_dir) {
|
||||||
|
bool child_synced = dirs && path_index_contains(dirs, child_rel);
|
||||||
|
is_extra[i] = deletable && !child_synced && !keep_is_dir(keep, child_rel);
|
||||||
|
if (!is_extra[i])
|
||||||
|
local_survives = true;
|
||||||
|
} else {
|
||||||
|
is_extra[i] = deletable && !keep_is_file(keep, child_rel);
|
||||||
|
if (!is_extra[i])
|
||||||
|
local_survives = true;
|
||||||
|
}
|
||||||
free(child_rel);
|
free(child_rel);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Pass 1: extraneous subdirectories, descending (recorded after contents). */
|
||||||
|
for (size_t i = 0; i < dir_count; i++) {
|
||||||
|
if (!is_extra[i])
|
||||||
|
continue;
|
||||||
|
char* child_rel = path_cat((char*)rel_path, entries[i].name);
|
||||||
|
if (!child_rel) {
|
||||||
|
operation_ok = false;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if (is_dir) {
|
int childfd = openat(dirfd, entries[i].name, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||||
int childfd = openat(dirfd, entry->d_name, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
|
||||||
bool child_all_removed = false;
|
bool child_all_removed = false;
|
||||||
if (childfd >= 0) {
|
if (childfd >= 0) {
|
||||||
if (!list_extras_fd(childfd, child_rel, keep, dirs, out, recorded, skips, skip_count,
|
if (!list_extras_fd(childfd, child_rel, keep, dirs, out, recorded, skips, skip_count,
|
||||||
@@ -869,10 +1062,7 @@ static bool list_extras_fd(int dirfd, const char* rel_path, const PathIndex* kee
|
|||||||
} else if (errno != ENOENT) {
|
} else if (errno != ENOENT) {
|
||||||
operation_ok = false;
|
operation_ok = false;
|
||||||
}
|
}
|
||||||
bool child_synced = dirs && path_index_contains(dirs, child_rel);
|
if (child_all_removed && deletable) {
|
||||||
if (child_synced || keep_is_dir(keep, child_rel)) {
|
|
||||||
local_survives = true;
|
|
||||||
} else if (child_all_removed && deletable) {
|
|
||||||
size_t len = strlen(child_rel);
|
size_t len = strlen(child_rel);
|
||||||
char* copy = malloc(len + 2);
|
char* copy = malloc(len + 2);
|
||||||
if (!copy) {
|
if (!copy) {
|
||||||
@@ -891,11 +1081,18 @@ static bool list_extras_fd(int dirfd, const char* rel_path, const PathIndex* kee
|
|||||||
} else {
|
} else {
|
||||||
local_survives = true;
|
local_survives = true;
|
||||||
}
|
}
|
||||||
} else {
|
free(child_rel);
|
||||||
bool found = keep_is_file(keep, child_rel);
|
}
|
||||||
if (found || !deletable) {
|
|
||||||
local_survives = true;
|
/* Pass 2: extraneous files, descending. */
|
||||||
} else {
|
for (size_t i = dir_count; i < count; i++) {
|
||||||
|
if (!is_extra[i])
|
||||||
|
continue;
|
||||||
|
char* child_rel = path_cat((char*)rel_path, entries[i].name);
|
||||||
|
if (!child_rel) {
|
||||||
|
operation_ok = false;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
char* copy = str_dup(child_rel);
|
char* copy = str_dup(child_rel);
|
||||||
if (!copy || !array_list_add(out, copy)) {
|
if (!copy || !array_list_add(out, copy)) {
|
||||||
free(copy);
|
free(copy);
|
||||||
@@ -903,11 +1100,35 @@ static bool list_extras_fd(int dirfd, const char* rel_path, const PathIndex* kee
|
|||||||
} else {
|
} else {
|
||||||
(*recorded)++;
|
(*recorded)++;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
free(child_rel);
|
free(child_rel);
|
||||||
}
|
}
|
||||||
closedir(dir);
|
|
||||||
|
/* Pass 3: kept subdirectories, ascending. */
|
||||||
|
for (size_t i = dir_count; i-- > 0;) {
|
||||||
|
if (is_extra[i] || shielded[i])
|
||||||
|
continue;
|
||||||
|
char* child_rel = path_cat((char*)rel_path, entries[i].name);
|
||||||
|
if (!child_rel) {
|
||||||
|
operation_ok = false;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
int childfd = openat(dirfd, entries[i].name, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||||
|
bool child_all_removed = false;
|
||||||
|
if (childfd >= 0) {
|
||||||
|
if (!list_extras_fd(childfd, child_rel, keep, dirs, out, recorded, skips, skip_count,
|
||||||
|
protect_rules, deletable, &child_all_removed))
|
||||||
|
operation_ok = false;
|
||||||
|
close(childfd);
|
||||||
|
} else if (errno != ENOENT) {
|
||||||
|
operation_ok = false;
|
||||||
|
}
|
||||||
|
local_survives = true;
|
||||||
|
free(child_rel);
|
||||||
|
}
|
||||||
|
|
||||||
|
free(shielded);
|
||||||
|
free(is_extra);
|
||||||
|
delete_dir_entries_free(entries, count);
|
||||||
*all_removed = !local_survives;
|
*all_removed = !local_survives;
|
||||||
return operation_ok;
|
return operation_ok;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -134,6 +134,28 @@ typedef struct {
|
|||||||
only DIRECT children of the destination root, i.e. child_rel has no '/'). */
|
only DIRECT children of the destination root, i.e. child_rel has no '/'). */
|
||||||
bool path_under_skip_prefix(const char* child_rel, bool at_root, const DeleteSkipEntry* skips,
|
bool path_under_skip_prefix(const char* child_rel, bool at_root, const DeleteSkipEntry* skips,
|
||||||
int skip_count);
|
int skip_count);
|
||||||
|
/* One destination-directory entry collected up front so the delete walkers can
|
||||||
|
reproduce rsync's traversal order instead of readdir() order. rsync processes
|
||||||
|
a directory's extraneous subdirectories first (descending name, depth-first),
|
||||||
|
then its extraneous files (descending name), and only afterwards descends into
|
||||||
|
its kept subdirectories (ascending name). */
|
||||||
|
typedef struct {
|
||||||
|
char* name;
|
||||||
|
bool is_dir;
|
||||||
|
} DeleteDirEntry;
|
||||||
|
/* Collect the entries of the directory open on `dirfd` (excluding "." and ".."),
|
||||||
|
stat'ing each with AT_SYMLINK_NOFOLLOW. On success *out is a malloc'd array of
|
||||||
|
*count entries whose names the caller frees with delete_dir_entries_free().
|
||||||
|
Returns false on an allocation/readdir failure; a vanished entry (ENOENT) is
|
||||||
|
skipped, any other stat failure is reported through *operation_ok while the
|
||||||
|
walk continues. */
|
||||||
|
bool delete_dir_entries_collect(int dirfd, DeleteDirEntry** out, size_t* count, bool* operation_ok);
|
||||||
|
void delete_dir_entries_free(DeleteDirEntry* entries, size_t count);
|
||||||
|
/* Sort comparators: `_desc` orders subdirectories before files and each group by
|
||||||
|
descending name (rsync's extraneous-entry order); `_asc` orders plain ascending
|
||||||
|
name (rsync's kept-subdirectory order). */
|
||||||
|
int delete_dir_entry_cmp_desc(const void* a, const void* b);
|
||||||
|
int delete_dir_entry_cmp_asc(const void* a, const void* b);
|
||||||
/* Remove files/dirs/symlinks under dest_root that are not listed in manifest
|
/* Remove files/dirs/symlinks under dest_root that are not listed in manifest
|
||||||
without ever descending into a protected prefix (see DeleteSkipEntry). When
|
without ever descending into a protected prefix (see DeleteSkipEntry). When
|
||||||
`synced_dirs` is non-NULL, extras are only removed directly inside a directory
|
`synced_dirs` is non-NULL, extras are only removed directly inside a directory
|
||||||
@@ -203,6 +225,11 @@ void utils_set_authorized_root_fd(int fd);
|
|||||||
* threads spawn; see utils.c). */
|
* threads spawn; see utils.c). */
|
||||||
int utils_get_authorized_root_fd(void);
|
int utils_get_authorized_root_fd(void);
|
||||||
const char* utils_get_authorized_root_path(void);
|
const char* utils_get_authorized_root_path(void);
|
||||||
|
/* Write a diagnostic message into a caller-supplied buffer, mirroring
|
||||||
|
* vsnprintf. A NULL `err` or a zero `err_size` is a no-op, so a caller that
|
||||||
|
* only needs the boolean status may safely pass NULL. Returns nothing; the
|
||||||
|
* buffer is always NUL-terminated by vsnprintf when err_size > 0. */
|
||||||
|
void utils_set_error(char* err, size_t err_size, const char* fmt, ...);
|
||||||
/* True when `path` is `root` itself or lies directly beneath it: a lexical
|
/* True when `path` is `root` itself or lies directly beneath it: a lexical
|
||||||
* prefix test requiring the byte after `root` to be '\0' or '/'. Both `root`
|
* prefix test requiring the byte after `root` to be '\0' or '/'. Both `root`
|
||||||
* and `path` must be absolute canonical paths free of "."/".." components (the
|
* and `path` must be absolute canonical paths free of "."/".." components (the
|
||||||
|
|||||||
@@ -1 +0,0 @@
|
|||||||
OLDDEST
|
|
||||||
@@ -1 +0,0 @@
|
|||||||
NEWCONTENT
|
|
||||||
@@ -1 +0,0 @@
|
|||||||
NEWCONTENT
|
|
||||||
+55
-12
@@ -20,6 +20,12 @@ CLIENT_CMD = [os.path.join(BUILD_DIR, "client")]
|
|||||||
_WORKER = os.environ.get("PYTEST_XDIST_WORKER")
|
_WORKER = os.environ.get("PYTEST_XDIST_WORKER")
|
||||||
TEST_DATA_DIR = os.path.join(PROJECT_ROOT, f"test_data-{_WORKER}" if _WORKER else "test_data")
|
TEST_DATA_DIR = os.path.join(PROJECT_ROOT, f"test_data-{_WORKER}" if _WORKER else "test_data")
|
||||||
|
|
||||||
|
# Default wall-clock budget for a short-lived client invocation. Every client
|
||||||
|
# is expected to finish well within this; the bound exists so a hung client
|
||||||
|
# fails the test instead of stalling the whole CI run indefinitely. Callers
|
||||||
|
# that legitimately need longer can pass an explicit ``timeout``.
|
||||||
|
CLIENT_TIMEOUT = 180
|
||||||
|
|
||||||
|
|
||||||
class ServerManager:
|
class ServerManager:
|
||||||
"""Manages a long-lived server process. Reuses across test cases."""
|
"""Manages a long-lived server process. Reuses across test cases."""
|
||||||
@@ -137,7 +143,40 @@ class CountingProxy:
|
|||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
def run_client(source_dir, dest_dir, flags=None, port=None, extra_args=None):
|
def _run_client_cmd(cmd, timeout):
|
||||||
|
"""Run one client command, returning ``(result, duration)``.
|
||||||
|
|
||||||
|
On timeout the client is killed and a result-like ``CompletedProcess`` with
|
||||||
|
a non-zero returncode is returned instead of raising, so callers keep the
|
||||||
|
established ``(result, duration)`` contract and the failure carries the
|
||||||
|
command plus whatever output was captured for diagnosis.
|
||||||
|
"""
|
||||||
|
start = time.monotonic()
|
||||||
|
try:
|
||||||
|
result = subprocess.run(cmd, text=True, capture_output=True, timeout=timeout)
|
||||||
|
except subprocess.TimeoutExpired as exc:
|
||||||
|
duration = time.monotonic() - start
|
||||||
|
stdout = exc.stdout or ""
|
||||||
|
stderr = exc.stderr or ""
|
||||||
|
if isinstance(stdout, bytes):
|
||||||
|
stdout = stdout.decode(errors="replace")
|
||||||
|
if isinstance(stderr, bytes):
|
||||||
|
stderr = stderr.decode(errors="replace")
|
||||||
|
diagnostic = (
|
||||||
|
f"client timed out after {timeout}s\n"
|
||||||
|
f"command: {cmd!r}\n"
|
||||||
|
f"--- captured stdout ---\n{stdout}\n"
|
||||||
|
f"--- captured stderr ---\n{stderr}"
|
||||||
|
)
|
||||||
|
result = subprocess.CompletedProcess(cmd, returncode=-1,
|
||||||
|
stdout=stdout, stderr=diagnostic)
|
||||||
|
return result, duration
|
||||||
|
duration = time.monotonic() - start
|
||||||
|
return result, duration
|
||||||
|
|
||||||
|
|
||||||
|
def run_client(source_dir, dest_dir, flags=None, port=None, extra_args=None,
|
||||||
|
timeout=CLIENT_TIMEOUT):
|
||||||
"""Run the client and return (result, duration)."""
|
"""Run the client and return (result, duration)."""
|
||||||
cmd = CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir, "--save-to-disk"]
|
cmd = CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir, "--save-to-disk"]
|
||||||
if port:
|
if port:
|
||||||
@@ -146,23 +185,18 @@ def run_client(source_dir, dest_dir, flags=None, port=None, extra_args=None):
|
|||||||
cmd += flags
|
cmd += flags
|
||||||
if extra_args:
|
if extra_args:
|
||||||
cmd += extra_args
|
cmd += extra_args
|
||||||
start = time.monotonic()
|
return _run_client_cmd(cmd, timeout)
|
||||||
result = subprocess.run(cmd, text=True, capture_output=True)
|
|
||||||
duration = time.monotonic() - start
|
|
||||||
return result, duration
|
|
||||||
|
|
||||||
|
|
||||||
def run_client_posix(source_dir, dest_dir, flags=None, port=None):
|
def run_client_posix(source_dir, dest_dir, flags=None, port=None,
|
||||||
|
timeout=CLIENT_TIMEOUT):
|
||||||
"""Run the client with positional args (rsync-style)."""
|
"""Run the client with positional args (rsync-style)."""
|
||||||
cmd = CLIENT_CMD + [source_dir, dest_dir, "--save-to-disk"]
|
cmd = CLIENT_CMD + [source_dir, dest_dir, "--save-to-disk"]
|
||||||
if port:
|
if port:
|
||||||
cmd += ["--server-port", str(port)]
|
cmd += ["--server-port", str(port)]
|
||||||
if flags:
|
if flags:
|
||||||
cmd += flags
|
cmd += flags
|
||||||
start = time.monotonic()
|
return _run_client_cmd(cmd, timeout)
|
||||||
result = subprocess.run(cmd, text=True, capture_output=True)
|
|
||||||
duration = time.monotonic() - start
|
|
||||||
return result, duration
|
|
||||||
|
|
||||||
|
|
||||||
def generate_test_files(source_dir, full=False):
|
def generate_test_files(source_dir, full=False):
|
||||||
@@ -238,8 +272,17 @@ def make_result(name, success, duration=None, error=""):
|
|||||||
|
|
||||||
|
|
||||||
def get_dest_received_dir(dest_dir, source_dir):
|
def get_dest_received_dir(dest_dir, source_dir):
|
||||||
"""Get the path where received files land inside dest_dir."""
|
"""Get the path where received files land inside dest_dir.
|
||||||
return os.path.join(dest_dir, os.path.abspath(source_dir).lstrip(os.sep))
|
|
||||||
|
FastSync mirrors the absolute source path below the receive root with the
|
||||||
|
leading root separator removed. Strip that separator explicitly rather
|
||||||
|
than with ``str.lstrip(os.sep)``: ``lstrip`` removes a *set* of characters
|
||||||
|
rather than a path prefix, which is not the same operation.
|
||||||
|
"""
|
||||||
|
abs_source = os.path.abspath(source_dir)
|
||||||
|
if abs_source.startswith(os.sep):
|
||||||
|
abs_source = abs_source[len(os.sep):]
|
||||||
|
return os.path.join(dest_dir, abs_source)
|
||||||
|
|
||||||
|
|
||||||
def _find_free_port():
|
def _find_free_port():
|
||||||
|
|||||||
@@ -63,6 +63,10 @@ DETACH_MODULE = os.path.join(MODULE_ROOT, "detach")
|
|||||||
DETACH_CONF = os.path.join(TEST_DATA_DIR, "fastsyncd_detach.conf")
|
DETACH_CONF = os.path.join(TEST_DATA_DIR, "fastsyncd_detach.conf")
|
||||||
DETACH_PORT = None
|
DETACH_PORT = None
|
||||||
|
|
||||||
|
# A dedicated config for the umask test: the daemon must be launched in the real
|
||||||
|
# (double-fork) detach path, whose daemonize() applies umask(022).
|
||||||
|
UMASK_CONF = os.path.join(TEST_DATA_DIR, "fastsyncd_umask.conf")
|
||||||
|
|
||||||
# Passwords are never sent as plaintext and never logged; these literals are
|
# Passwords are never sent as plaintext and never logged; these literals are
|
||||||
# only hashed into the server credential file / client password file.
|
# only hashed into the server credential file / client password file.
|
||||||
ALICE_PASS = "alice-s3cret"
|
ALICE_PASS = "alice-s3cret"
|
||||||
@@ -332,6 +336,44 @@ class TestDaemonModuleSelection:
|
|||||||
assert not missing, f"missing: {missing[:5]}"
|
assert not missing, f"missing: {missing[:5]}"
|
||||||
assert not mismatches, f"mismatch: {mismatches[:5]}"
|
assert not mismatches, f"mismatch: {mismatches[:5]}"
|
||||||
|
|
||||||
|
def test_daemon_new_dirs_not_world_writable(self):
|
||||||
|
"""The daemon must not force umask 0: implied parent directories created
|
||||||
|
without -p are the source default (0755 under the daemon's 022 umask),
|
||||||
|
never world-writable 0777.
|
||||||
|
|
||||||
|
This drives the real double-fork detach path, where the umask(022) fix
|
||||||
|
lives (daemonize()); the --no-detach path never calls it. The launcher
|
||||||
|
is run with umask 0, so without the fix the daemon would inherit 0 and
|
||||||
|
create a 0777 directory; with the fix the assertion below fails only if
|
||||||
|
the fix regresses."""
|
||||||
|
port = _find_free_port()
|
||||||
|
with open(UMASK_CONF, "w") as f:
|
||||||
|
f.write("port = %d\n\n[files]\npath = %s\n" % (port, FILES_MODULE))
|
||||||
|
sub = os.path.join(FILES_MODULE, "umask_check")
|
||||||
|
shutil.rmtree(sub, ignore_errors=True)
|
||||||
|
os.makedirs(sub, exist_ok=True)
|
||||||
|
log_path = os.path.join(TEST_DATA_DIR, "fastsyncd_umask.log")
|
||||||
|
log = open(log_path, "w")
|
||||||
|
cmd = SERVER_CMD + ["--daemon", "--config", UMASK_CONF, "--allow-unauthenticated"]
|
||||||
|
proc = subprocess.Popen(cmd, stdout=log, stderr=log, stdin=subprocess.DEVNULL,
|
||||||
|
preexec_fn=lambda: os.umask(0))
|
||||||
|
try:
|
||||||
|
_wait_for_port(port, timeout=15)
|
||||||
|
result = _push("127.0.0.1::files/umask_check", port)
|
||||||
|
assert result.returncode == 0, result.stderr or result.stdout
|
||||||
|
received = get_dest_received_dir(sub, SOURCE_DIR)
|
||||||
|
nested = os.path.join(received, "nested")
|
||||||
|
assert os.path.isdir(nested), f"nested dir missing under {received}"
|
||||||
|
mode = stat.S_IMODE(os.stat(nested).st_mode)
|
||||||
|
assert (mode & 0o022) == 0, f"implied directory is group/other writable: {oct(mode)}"
|
||||||
|
finally:
|
||||||
|
_kill_by_cmdline_marker(UMASK_CONF)
|
||||||
|
log.close()
|
||||||
|
try:
|
||||||
|
proc.wait(timeout=5)
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
proc.kill()
|
||||||
|
|
||||||
|
|
||||||
class TestDaemonRejection:
|
class TestDaemonRejection:
|
||||||
def _tree_files(self):
|
def _tree_files(self):
|
||||||
|
|||||||
@@ -0,0 +1,291 @@
|
|||||||
|
"""Differential coverage for the delete-timing ABORT BOUNDARY (A9/A10).
|
||||||
|
|
||||||
|
rsync's generator runs ahead of its throttled sender, so on a mid-transfer abort
|
||||||
|
it has already removed every extra it planned. FastSync now transmits the
|
||||||
|
COMPLETE per-directory plan set before the first data frame, so an abort has the
|
||||||
|
same effect. Before that change FastSync only removed the extras of the
|
||||||
|
directories its (slower) data stream had reached, and ``-d/--dirs`` used an
|
||||||
|
end-of-transfer commit that removed nothing on abort.
|
||||||
|
|
||||||
|
These tests abort both tools mid-transfer and assert the destination extras
|
||||||
|
removed match real ``rsync 3.4.1``. The rsync side is driven locally with
|
||||||
|
``--bwlimit`` and a small timing window (its generator's delete list is computed
|
||||||
|
long before the throttled payload finishes); the FastSync side uses the
|
||||||
|
byte-deterministic slicing proxy from ``test_delete_timing_parity``.
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
sys.path.insert(0, os.path.dirname(__file__))
|
||||||
|
from common import ( # noqa: E402
|
||||||
|
TEST_DATA_DIR,
|
||||||
|
ServerManager,
|
||||||
|
clean_dir,
|
||||||
|
get_dest_received_dir,
|
||||||
|
run_client,
|
||||||
|
)
|
||||||
|
from test_delete_timing_parity import _SlicingProxy # noqa: E402
|
||||||
|
|
||||||
|
RSYNC = shutil.which("rsync")
|
||||||
|
requires_rsync = pytest.mark.skipif(RSYNC is None, reason="rsync 3.4.1 not installed")
|
||||||
|
|
||||||
|
# Exceeds the 10 MiB scanner chunk, so the next directory lands in a later chunk
|
||||||
|
# (still unreached when the proxy cuts the stream).
|
||||||
|
BIG_BYTES = 16 * 1024 * 1024
|
||||||
|
# Cut well past the (small) config + delete-plan frames and into the big payload,
|
||||||
|
# so the receiver has provably processed every plan before the abort.
|
||||||
|
MID_TRANSFER_BYTES = 256 * 1024
|
||||||
|
PROXY_THROTTLE = 0.001
|
||||||
|
# Throttle rsync's sender so the generator has deleted long before the payload
|
||||||
|
# finishes, then interrupt it mid-transfer.
|
||||||
|
RSYNC_BWLIMIT = 512 # KiB/s -> ~32 s for 16 MiB
|
||||||
|
RSYNC_ABORT_DELAY = 1.5
|
||||||
|
|
||||||
|
|
||||||
|
def _write(path, content):
|
||||||
|
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||||
|
with open(path, "wb") as fh:
|
||||||
|
fh.write(content)
|
||||||
|
|
||||||
|
|
||||||
|
def _rsync_aborted(args, delay=RSYNC_ABORT_DELAY):
|
||||||
|
"""Start rsync, let its generator run, then interrupt it mid-transfer."""
|
||||||
|
env = dict(os.environ, LC_ALL="C")
|
||||||
|
proc = subprocess.Popen([RSYNC] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
||||||
|
text=True, env=env)
|
||||||
|
time.sleep(delay)
|
||||||
|
proc.terminate()
|
||||||
|
try:
|
||||||
|
proc.wait(timeout=10)
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
proc.kill()
|
||||||
|
proc.wait(timeout=5)
|
||||||
|
return proc
|
||||||
|
|
||||||
|
|
||||||
|
class TestDeleteDuringAbortBoundary:
|
||||||
|
"""A9: on an abort, every planned removal has already been applied."""
|
||||||
|
|
||||||
|
def _seed_recursive(self, tag):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, f"dab_{tag}_src")
|
||||||
|
clean_dir(source)
|
||||||
|
# ``a/keep.bin`` sorts first, so the client streams it (and the proxy
|
||||||
|
# cuts) before the data pass ever reaches ``z/deep``.
|
||||||
|
_write(os.path.join(source, "a", "keep.bin"), b"B" * BIG_BYTES)
|
||||||
|
_write(os.path.join(source, "z", "deep", "keep.txt"), b"keep\n")
|
||||||
|
return source
|
||||||
|
|
||||||
|
@requires_rsync
|
||||||
|
def test_recursive_abort_removes_all_planned_extras(self):
|
||||||
|
# ---- FastSync: abort mid ``a/keep.bin``; ``z/deep`` is never reached.
|
||||||
|
source = self._seed_recursive("rec_fs")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "dab_rec_fs_dst")
|
||||||
|
clean_dir(dest)
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
os.makedirs(os.path.join(received, "a"), exist_ok=True)
|
||||||
|
_write(os.path.join(received, "a", "a_extra"), b"stale\n")
|
||||||
|
os.makedirs(os.path.join(received, "z", "deep"), exist_ok=True)
|
||||||
|
_write(os.path.join(received, "z", "deep", "old_extra"), b"stale\n")
|
||||||
|
|
||||||
|
with ServerManager() as server:
|
||||||
|
server.start(extra_args=["--allow-delete"])
|
||||||
|
proxy = _SlicingProxy(server.port, forward_limit=MID_TRANSFER_BYTES,
|
||||||
|
throttle=PROXY_THROTTLE)
|
||||||
|
result, _ = run_client(source, dest, flags=["--delete-during"], port=proxy.port)
|
||||||
|
proxy.finish()
|
||||||
|
assert result.returncode != 0, "truncated transfer reported success"
|
||||||
|
assert not os.path.exists(os.path.join(received, "a", "a_extra"))
|
||||||
|
assert not os.path.exists(os.path.join(received, "z", "deep", "old_extra")), (
|
||||||
|
"FastSync left an extra in a directory it never reached before the abort"
|
||||||
|
)
|
||||||
|
|
||||||
|
# ---- rsync 3.4.1: same tree, same abort, same delete outcome.
|
||||||
|
source = self._seed_recursive("rec_rs")
|
||||||
|
rsync_dst = os.path.join(TEST_DATA_DIR, "dab_rec_rs_dst")
|
||||||
|
clean_dir(rsync_dst)
|
||||||
|
os.makedirs(os.path.join(rsync_dst, "a"), exist_ok=True)
|
||||||
|
_write(os.path.join(rsync_dst, "a", "a_extra"), b"stale\n")
|
||||||
|
os.makedirs(os.path.join(rsync_dst, "z", "deep"), exist_ok=True)
|
||||||
|
_write(os.path.join(rsync_dst, "z", "deep", "old_extra"), b"stale\n")
|
||||||
|
|
||||||
|
proc = _rsync_aborted(["-a", "--delete-during", f"--bwlimit={RSYNC_BWLIMIT}",
|
||||||
|
source + "/", rsync_dst + "/"])
|
||||||
|
assert proc.returncode != 0, "rsync was not actually interrupted"
|
||||||
|
assert not os.path.exists(os.path.join(rsync_dst, "a", "a_extra"))
|
||||||
|
assert not os.path.exists(os.path.join(rsync_dst, "z", "deep", "old_extra")), (
|
||||||
|
"rsync's generator did not delete ahead of its sender"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestDirsDeleteAbortBoundary:
|
||||||
|
"""A10: ``-d/--dirs`` uses per-directory plans like rsync.
|
||||||
|
|
||||||
|
The listed directory's direct extras are removed by the up-front plan while
|
||||||
|
a kept but untraversed subdirectory (and its destination content) is
|
||||||
|
shielded.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def _seed_dirs(self, tag):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, f"ddb_{tag}_src")
|
||||||
|
clean_dir(source)
|
||||||
|
_write(os.path.join(source, "big.bin"), b"B" * BIG_BYTES)
|
||||||
|
_write(os.path.join(source, "subdir", "keep.txt"), b"inner\n")
|
||||||
|
return source
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("fs_flag,rs_flag", [("--delete-during", "--delete-during"),
|
||||||
|
("--delete", "--delete")])
|
||||||
|
@requires_rsync
|
||||||
|
def test_dirs_abort_removes_direct_extras_only(self, fs_flag, rs_flag):
|
||||||
|
label = f"{fs_flag.lstrip('-')}_{rs_flag.lstrip('-')}"
|
||||||
|
# ---- FastSync: ``-d`` lists the immediate children; big.bin streams and
|
||||||
|
# the abort lands mid-payload.
|
||||||
|
source = self._seed_dirs(f"dirs_{label}_fs")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, f"ddb_{label}_fs_dst")
|
||||||
|
clean_dir(dest)
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
_write(os.path.join(received, "old_extra"), b"stale\n")
|
||||||
|
os.makedirs(os.path.join(received, "subdir"), exist_ok=True)
|
||||||
|
_write(os.path.join(received, "subdir", "stale.txt"), b"stale inner\n")
|
||||||
|
|
||||||
|
with ServerManager() as server:
|
||||||
|
server.start(extra_args=["--allow-delete"])
|
||||||
|
proxy = _SlicingProxy(server.port, forward_limit=MID_TRANSFER_BYTES,
|
||||||
|
throttle=PROXY_THROTTLE)
|
||||||
|
result, _ = run_client(source + "/", dest, flags=["-d", fs_flag], port=proxy.port)
|
||||||
|
proxy.finish()
|
||||||
|
assert result.returncode != 0, f"{fs_flag}: truncated transfer reported success"
|
||||||
|
assert not os.path.exists(os.path.join(received, "old_extra")), (
|
||||||
|
f"{fs_flag}: the listed directory's direct extra survived the abort"
|
||||||
|
)
|
||||||
|
assert os.path.exists(os.path.join(received, "subdir", "stale.txt")), (
|
||||||
|
f"{fs_flag}: descended into a kept, untraversed subdirectory"
|
||||||
|
)
|
||||||
|
|
||||||
|
# ---- rsync 3.4.1: same shape and same abort.
|
||||||
|
source = self._seed_dirs(f"dirs_{label}_rs")
|
||||||
|
rsync_dst = os.path.join(TEST_DATA_DIR, f"ddb_{label}_rs_dst")
|
||||||
|
clean_dir(rsync_dst)
|
||||||
|
_write(os.path.join(rsync_dst, "old_extra"), b"stale\n")
|
||||||
|
os.makedirs(os.path.join(rsync_dst, "subdir"), exist_ok=True)
|
||||||
|
_write(os.path.join(rsync_dst, "subdir", "stale.txt"), b"stale inner\n")
|
||||||
|
|
||||||
|
proc = _rsync_aborted(["-d", rs_flag, f"--bwlimit={RSYNC_BWLIMIT}",
|
||||||
|
source + "/", rsync_dst + "/"])
|
||||||
|
assert proc.returncode != 0, "rsync was not actually interrupted"
|
||||||
|
assert not os.path.exists(os.path.join(rsync_dst, "old_extra")), (
|
||||||
|
f"rsync {rs_flag}: the listed directory's direct extra survived the abort"
|
||||||
|
)
|
||||||
|
assert os.path.exists(os.path.join(rsync_dst, "subdir", "stale.txt")), (
|
||||||
|
f"rsync {rs_flag}: descended into a kept, untraversed subdirectory"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _tree(root):
|
||||||
|
out = []
|
||||||
|
for dirpath, dirs, files in os.walk(root):
|
||||||
|
for name in dirs:
|
||||||
|
out.append(os.path.relpath(os.path.join(dirpath, name), root))
|
||||||
|
for name in files:
|
||||||
|
out.append(os.path.relpath(os.path.join(dirpath, name), root))
|
||||||
|
return sorted(out)
|
||||||
|
|
||||||
|
|
||||||
|
class TestDirsDeleteFinalStateParity:
|
||||||
|
"""A10 completed run: ``-d DIR/ --delete`` (during default) and
|
||||||
|
``--delete-during`` match rsync's final tree, including a kept but
|
||||||
|
untraversed subdirectory whose destination content survives."""
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("flag", ["--delete", "--delete-during"])
|
||||||
|
@requires_rsync
|
||||||
|
def test_dirs_final_state_matches_rsync(self, flag):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, f"ddf_{flag.lstrip('-')}_src")
|
||||||
|
clean_dir(source)
|
||||||
|
_write(os.path.join(source, "keep.txt"), b"new keep\n")
|
||||||
|
_write(os.path.join(source, "subdir", "inner.txt"), b"inner\n")
|
||||||
|
|
||||||
|
def seed_dest(root):
|
||||||
|
clean_dir(root)
|
||||||
|
_write(os.path.join(root, "keep.txt"), b"old keep\n")
|
||||||
|
_write(os.path.join(root, "extra.txt"), b"extra\n")
|
||||||
|
_write(os.path.join(root, "extrasub", "ex.txt"), b"extra sub\n")
|
||||||
|
_write(os.path.join(root, "subdir", "stale.txt"), b"stale inner\n")
|
||||||
|
|
||||||
|
rsync_dst = os.path.join(TEST_DATA_DIR, f"ddf_{flag.lstrip('-')}_rs_dst")
|
||||||
|
seed_dest(rsync_dst)
|
||||||
|
env = dict(os.environ, LC_ALL="C")
|
||||||
|
rsync_result = subprocess.run(
|
||||||
|
[RSYNC, "-d", flag, source + "/", rsync_dst + "/"],
|
||||||
|
capture_output=True, text=True, env=env, timeout=120)
|
||||||
|
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||||
|
rsync_tree = _tree(rsync_dst)
|
||||||
|
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, f"ddf_{flag.lstrip('-')}_fs_dst")
|
||||||
|
clean_dir(dest)
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
seed_dest(received)
|
||||||
|
with ServerManager() as server:
|
||||||
|
server.start(extra_args=["--allow-delete"])
|
||||||
|
result, _ = run_client(source + "/", dest, flags=["-d", flag], port=server.port)
|
||||||
|
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||||
|
fastsync_tree = _tree(received)
|
||||||
|
assert fastsync_tree == rsync_tree, (
|
||||||
|
f"-d {flag}: fastsync tree {fastsync_tree} != rsync tree {rsync_tree}")
|
||||||
|
|
||||||
|
|
||||||
|
class TestOneFileSystemDeleteParity:
|
||||||
|
"""A9 side effect: the per-directory plan is now emitted only for directories
|
||||||
|
whose children were enumerated, so a ``-x`` mount-point directory that is
|
||||||
|
emitted but never traversed is shielded -- its destination content survives,
|
||||||
|
exactly as rsync keeps a non-descended mount point under ``--delete``."""
|
||||||
|
|
||||||
|
@requires_rsync
|
||||||
|
def test_mountpoint_content_survives_delete(self):
|
||||||
|
local = os.stat(".")
|
||||||
|
shm = "/dev/shm"
|
||||||
|
if not os.path.isdir(shm) or os.stat(shm).st_dev == local.st_dev:
|
||||||
|
pytest.skip("no cross-device filesystem available")
|
||||||
|
probe = os.path.join(shm, f"fastsync_dofs_{os.getpid()}")
|
||||||
|
clean_dir(probe)
|
||||||
|
_write(os.path.join(probe, "inside.txt"), b"cross\n")
|
||||||
|
try:
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "dofs_src")
|
||||||
|
clean_dir(source)
|
||||||
|
_write(os.path.join(source, "keep.txt"), b"keep\n")
|
||||||
|
os.symlink(probe, os.path.join(source, "nested_link"))
|
||||||
|
|
||||||
|
def seed_dest(root):
|
||||||
|
clean_dir(root)
|
||||||
|
_write(os.path.join(root, "keep.txt"), b"old\n")
|
||||||
|
_write(os.path.join(root, "nested_link", "stale.txt"), b"stale\n")
|
||||||
|
|
||||||
|
rsync_dst = os.path.join(TEST_DATA_DIR, "dofs_rs_dst")
|
||||||
|
seed_dest(rsync_dst)
|
||||||
|
env = dict(os.environ, LC_ALL="C")
|
||||||
|
rsync_result = subprocess.run(
|
||||||
|
[RSYNC, "-a", "--copy-links", "-x", "--delete-during",
|
||||||
|
source + "/", rsync_dst + "/"],
|
||||||
|
capture_output=True, text=True, env=env, timeout=120)
|
||||||
|
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||||
|
assert os.path.exists(os.path.join(rsync_dst, "nested_link", "stale.txt")), (
|
||||||
|
"rsync unexpectedly descended into the mount point")
|
||||||
|
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "dofs_fs_dst")
|
||||||
|
clean_dir(dest)
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
seed_dest(received)
|
||||||
|
with ServerManager() as server:
|
||||||
|
server.start(extra_args=["--allow-delete"])
|
||||||
|
result, _ = run_client(source, dest,
|
||||||
|
flags=["-a", "--copy-links", "-x", "--delete-during"],
|
||||||
|
port=server.port)
|
||||||
|
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||||
|
assert os.path.exists(os.path.join(received, "nested_link", "stale.txt")), (
|
||||||
|
"FastSync descended into a non-traversed mount point under --delete")
|
||||||
|
finally:
|
||||||
|
clean_dir(probe)
|
||||||
|
|
||||||
@@ -2275,6 +2275,36 @@ class TestPartialDir:
|
|||||||
partial = os.path.join(dest, ".partial", os.path.relpath(source_file, os.path.sep))
|
partial = os.path.join(dest, ".partial", os.path.relpath(source_file, os.path.sep))
|
||||||
assert not os.path.exists(partial)
|
assert not os.path.exists(partial)
|
||||||
|
|
||||||
|
def test_partial_dir_alone_implies_partial(self, shared_server):
|
||||||
|
"""--partial-dir=DIR with no --partial implies --partial, like rsync.
|
||||||
|
|
||||||
|
rsync 3.4.1 retains the staged partial when --partial-dir is given by
|
||||||
|
itself; before the implication was added FastSync discarded it. The
|
||||||
|
transfer is made to fail deterministically by placing a non-empty
|
||||||
|
directory at the destination path, so the final partial-dir ->
|
||||||
|
destination rename fails and whatever was staged under the partial dir
|
||||||
|
stays on disk."""
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "partial_dir_implied_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "partial_dir_implied_dst")
|
||||||
|
clean_dir(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
source_file = os.path.join(source, "f.bin")
|
||||||
|
with open(source_file, "wb") as f:
|
||||||
|
f.write(b"partial payload")
|
||||||
|
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
os.makedirs(os.path.join(received, "f.bin"))
|
||||||
|
with open(os.path.join(received, "f.bin", "keep"), "wb") as f:
|
||||||
|
f.write(b"keep")
|
||||||
|
|
||||||
|
result, _ = run_client(source, dest, flags=["--partial-dir=.partial"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode != 0, "expected the blocked install to fail"
|
||||||
|
|
||||||
|
partial = os.path.join(dest, ".partial", os.path.relpath(source_file, os.path.sep))
|
||||||
|
assert os.path.exists(partial), \
|
||||||
|
"--partial-dir alone must imply --partial and retain the partial file"
|
||||||
|
|
||||||
|
|
||||||
class TestLargeFile:
|
class TestLargeFile:
|
||||||
def test_transfer_100mb_file(self, shared_server):
|
def test_transfer_100mb_file(self, shared_server):
|
||||||
@@ -2606,35 +2636,30 @@ class TestTimeoutAndAllocLimits:
|
|||||||
mismatches, missing = verify_transfer(source, received)
|
mismatches, missing = verify_transfer(source, received)
|
||||||
assert not missing and not mismatches
|
assert not missing and not mismatches
|
||||||
|
|
||||||
def test_temp_dir_cross_filesystem_fallback(self, shared_server):
|
def test_temp_dir_symlink_escape_rejected(self, shared_server):
|
||||||
"""A confined relative --temp-dir that resolves (via a symlink under the
|
"""A symlink planted inside the destination root pointing outside it
|
||||||
destination root) to another filesystem must fall back to a non-atomic
|
must not redirect receiver scratch files: --temp-dir=<that link> is
|
||||||
copy instead of aborting (rsync parity). Skipped when no second
|
refused and nothing is written at the link target. An in-root symlink
|
||||||
filesystem is available."""
|
(e.g. to a mount point that stays inside the authorized root) is still
|
||||||
shm = "/dev/shm"
|
accepted, preserving the engine's EXDEV cross-filesystem fallback."""
|
||||||
if not os.path.isdir(shm):
|
source, dest = self._seed("tempdir_escape_src")
|
||||||
pytest.skip("/dev/shm not available")
|
outside = "/tmp/fastsync_tempdir_escape_%d" % os.getpid()
|
||||||
if os.stat(shm).st_dev == os.stat(TEST_DATA_DIR).st_dev:
|
shutil.rmtree(outside, ignore_errors=True)
|
||||||
pytest.skip("/dev/shm is on the same filesystem as the test data")
|
os.makedirs(outside)
|
||||||
scratch = os.path.join(shm, f"fastsync_tmp_{os.getpid()}")
|
link = os.path.join(dest, "escape_scratch")
|
||||||
shutil.rmtree(scratch, ignore_errors=True)
|
if os.path.lexists(link):
|
||||||
os.makedirs(scratch)
|
os.unlink(link)
|
||||||
|
os.symlink(outside, link)
|
||||||
try:
|
try:
|
||||||
source, dest = self._seed("tempdir_xdev_src")
|
result, _ = run_client(source, dest, flags=["--temp-dir", "escape_scratch"],
|
||||||
# The receiver resolves a relative temp dir under the destination
|
|
||||||
# root; a symlink there points the scratch at the second filesystem.
|
|
||||||
link = os.path.join(dest, "xdev_scratch")
|
|
||||||
os.symlink(scratch, link)
|
|
||||||
result, _ = run_client(source, dest, flags=["--temp-dir", "xdev_scratch"],
|
|
||||||
port=shared_server.port)
|
port=shared_server.port)
|
||||||
assert result.returncode == 0, f"cross-fs temp-dir failed: {result.stderr[:300]}"
|
assert result.returncode != 0, "an escaping --temp-dir symlink must be refused"
|
||||||
received = get_dest_received_dir(dest, source)
|
received = get_dest_received_dir(dest, source)
|
||||||
mismatches, missing = verify_transfer(source, received)
|
assert not os.path.exists(os.path.join(received, "f.txt")), \
|
||||||
assert not missing, f"Missing: {missing}"
|
"the receiver must not fall back to writing the file"
|
||||||
assert not mismatches, f"Mismatch: {mismatches}"
|
assert os.listdir(outside) == [], "receiver wrote outside the authorized root"
|
||||||
assert os.listdir(scratch) == [], "temp files left behind in the cross-fs scratch"
|
|
||||||
finally:
|
finally:
|
||||||
shutil.rmtree(scratch, ignore_errors=True)
|
shutil.rmtree(outside, ignore_errors=True)
|
||||||
|
|
||||||
|
|
||||||
class TestRemoteOptionTransport:
|
class TestRemoteOptionTransport:
|
||||||
@@ -5782,6 +5807,35 @@ class TestStandaloneSuperDefault:
|
|||||||
"standalone server accepted --copy-as without --allow-super"
|
"standalone server accepted --copy-as without --allow-super"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@pytest.mark.skipif(
|
||||||
|
os.geteuid() != 0,
|
||||||
|
reason="root triggers the SUPER_MODE_OFF default and can create setuid sources",
|
||||||
|
)
|
||||||
|
def test_special_bits_masked_without_allow_super(self):
|
||||||
|
"""A root standalone server without --allow-super forces SUPER_MODE_OFF,
|
||||||
|
so client-supplied setuid/setgid/sticky bits must be stripped even under
|
||||||
|
-p (they are super-user activities just like device-node creation)."""
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "super_default_mode_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "super_default_mode_dst")
|
||||||
|
clean_dir(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
src_file = os.path.join(source, "priv.sh")
|
||||||
|
with open(src_file, "wb") as f:
|
||||||
|
f.write(b"#!/bin/sh\necho hi\n")
|
||||||
|
os.chmod(src_file, 0o4755)
|
||||||
|
server = ServerManager()
|
||||||
|
server.start() # deliberately no --allow-super -> SUPER_MODE_OFF as root
|
||||||
|
try:
|
||||||
|
result, _ = run_client(source, dest, flags=["-p"], port=server.port)
|
||||||
|
finally:
|
||||||
|
server.stop()
|
||||||
|
assert result.returncode == 0, f"exit {result.returncode}: {(result.stderr or '')[:200]}"
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
mode = stat.S_IMODE(os.stat(os.path.join(received, "priv.sh")).st_mode)
|
||||||
|
assert (mode & (stat.S_ISUID | stat.S_ISGID | stat.S_ISVTX)) == 0, \
|
||||||
|
f"--no-super receiver kept a privileged bit: {oct(mode)}"
|
||||||
|
assert (mode & 0o777) == 0o755, f"ordinary permission bits lost: {oct(mode)}"
|
||||||
|
|
||||||
@pytest.mark.skipif(os.geteuid() != 0, reason="root can create the source device node")
|
@pytest.mark.skipif(os.geteuid() != 0, reason="root can create the source device node")
|
||||||
def test_devices_skipped_without_allow_super(self):
|
def test_devices_skipped_without_allow_super(self):
|
||||||
"""Root standalone server without --allow-super must skip device-node
|
"""Root standalone server without --allow-super must skip device-node
|
||||||
|
|||||||
@@ -662,6 +662,46 @@ class TestWireStatsParity:
|
|||||||
assert re.match(r"Number of created files: 1 \(reg: 1\)$", r_created), r_created
|
assert re.match(r"Number of created files: 1 \(reg: 1\)$", r_created), r_created
|
||||||
assert f_created == r_created, (r_created, f_created)
|
assert f_created == r_created, (r_created, f_created)
|
||||||
|
|
||||||
|
@requires_rsync
|
||||||
|
@pytest.mark.ci
|
||||||
|
@pytest.mark.parametrize("mt", [False, True])
|
||||||
|
def test_stats_r_directory_breakdown_matches_rsync(self, shared_server, mt):
|
||||||
|
"""A recursive `-r` scan (no -t/-p) exposes no directory metadata, but
|
||||||
|
rsync still counts every directory in `Number of files`; the sender's
|
||||||
|
lightweight directory counter must reproduce the `dir: N` category."""
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "wire_stdir_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "wire_stdir_dst")
|
||||||
|
rdst = os.path.join(TEST_DATA_DIR, "wire_stdir_rdst")
|
||||||
|
clean_dir(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
clean_dir(rdst)
|
||||||
|
os.makedirs(os.path.join(source, "sub", "deep"))
|
||||||
|
os.makedirs(os.path.join(source, "empty"))
|
||||||
|
for rel in ("a.txt", os.path.join("sub", "b.txt"), os.path.join("sub", "deep", "c.txt")):
|
||||||
|
with open(os.path.join(source, rel), "wb") as fh:
|
||||||
|
fh.write(b"x\n")
|
||||||
|
os.makedirs(get_dest_received_dir(dest, source), exist_ok=True)
|
||||||
|
|
||||||
|
rsync_result = _rsync(["-r", "--stats", source + "/", rdst + "/"])
|
||||||
|
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||||
|
flags = ["-r", "--stats"] + (["--threads"] if mt else [])
|
||||||
|
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
|
||||||
|
assert result.returncode == 0, result.stderr[:300]
|
||||||
|
|
||||||
|
def stats_line(text, key):
|
||||||
|
for line in text.splitlines():
|
||||||
|
if line.startswith(key + ":"):
|
||||||
|
return line
|
||||||
|
return None
|
||||||
|
|
||||||
|
r_files = stats_line(rsync_result.stdout, "Number of files")
|
||||||
|
f_files = stats_line(result.stdout, "Number of files")
|
||||||
|
# 3 regular files, 4 directories (root, sub, sub/deep, empty).
|
||||||
|
assert re.match(r"Number of files: 7 \(reg: 3, dir: 4\)$", r_files), r_files
|
||||||
|
assert f_files == r_files, (r_files, f_files)
|
||||||
|
assert (stats_line(result.stdout, "Number of regular files transferred") ==
|
||||||
|
stats_line(rsync_result.stdout, "Number of regular files transferred"))
|
||||||
|
|
||||||
@requires_rsync
|
@requires_rsync
|
||||||
@pytest.mark.ci
|
@pytest.mark.ci
|
||||||
@pytest.mark.parametrize("mt", [False, True])
|
@pytest.mark.parametrize("mt", [False, True])
|
||||||
|
|||||||
@@ -0,0 +1,198 @@
|
|||||||
|
"""Differential rsync-parity coverage for two residuals closed on this branch.
|
||||||
|
|
||||||
|
* A4 -- ``--compare-dest``/``--copy-dest``/``--link-dest`` relative-DIR
|
||||||
|
resolution: rsync resolves a relative DIR against the destination directory
|
||||||
|
and appends the file's TRANSFER-RELATIVE name. FastSync's default transfer
|
||||||
|
mirrors the absolute source path below its receive root, so a naive relative
|
||||||
|
DIR used to probe a different tree. These tests seed the basis at rsync's
|
||||||
|
spelling and assert FastSync finds it (byte-exact / hard-linked / sparse),
|
||||||
|
matching real rsync 3.4.1.
|
||||||
|
|
||||||
|
* A5 -- ``-y``/``--fuzzy`` candidate eligibility: rsync's ``find_fuzzy`` has no
|
||||||
|
delta-size gate, so it reuses an oversized (>10x) or sub-16-KiB sibling;
|
||||||
|
FastSync used to decline both. These tests assert FastSync now uses the same
|
||||||
|
sibling as rsync (observable as ``Matched data``) with a byte-exact result.
|
||||||
|
|
||||||
|
Every test skips cleanly when rsync is absent.
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
sys.path.insert(0, os.path.dirname(__file__))
|
||||||
|
from common import ( # noqa: E402
|
||||||
|
TEST_DATA_DIR,
|
||||||
|
clean_dir,
|
||||||
|
get_dest_received_dir,
|
||||||
|
run_client,
|
||||||
|
)
|
||||||
|
|
||||||
|
RSYNC = shutil.which("rsync")
|
||||||
|
requires_rsync = pytest.mark.skipif(RSYNC is None, reason="rsync 3.4.1 not installed")
|
||||||
|
|
||||||
|
OLD_MTIME = 1_500_000_000
|
||||||
|
|
||||||
|
|
||||||
|
def _write(path, content, mtime=None):
|
||||||
|
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||||
|
with open(path, "wb") as fh:
|
||||||
|
fh.write(content)
|
||||||
|
if mtime is not None:
|
||||||
|
os.utime(path, (mtime, mtime))
|
||||||
|
|
||||||
|
|
||||||
|
def _read(path):
|
||||||
|
with open(path, "rb") as fh:
|
||||||
|
return fh.read()
|
||||||
|
|
||||||
|
|
||||||
|
def _rsync(args):
|
||||||
|
env = dict(os.environ, LC_ALL="C")
|
||||||
|
return subprocess.run([RSYNC] + args, capture_output=True, text=True, env=env, timeout=120)
|
||||||
|
|
||||||
|
|
||||||
|
def _stat_bytes(text, label):
|
||||||
|
for line in text.splitlines():
|
||||||
|
if line.startswith(label + ":"):
|
||||||
|
return int(line.split(":", 1)[1].strip().split()[0].replace(",", ""))
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
class TestRelativeBasisDirResolution:
|
||||||
|
"""A4: a relative basis DIR must resolve to the same tree as rsync's."""
|
||||||
|
|
||||||
|
_FILES = {
|
||||||
|
"root.txt": b"root-basis-content\n",
|
||||||
|
"sub/nested.txt": b"nested-basis-content\n",
|
||||||
|
}
|
||||||
|
|
||||||
|
def _seed_source(self, source):
|
||||||
|
clean_dir(source)
|
||||||
|
for rel, data in self._FILES.items():
|
||||||
|
_write(os.path.join(source, rel), data, OLD_MTIME)
|
||||||
|
return self._FILES
|
||||||
|
|
||||||
|
@requires_rsync
|
||||||
|
@pytest.mark.parametrize("flag", ["--compare-dest", "--link-dest"])
|
||||||
|
def test_relative_dir_resolves_like_rsync(self, shared_server, flag):
|
||||||
|
tag = flag.lstrip("-")
|
||||||
|
source = os.path.join(TEST_DATA_DIR, f"relbasis_{tag}_src")
|
||||||
|
rdst = os.path.join(TEST_DATA_DIR, f"relbasis_{tag}_rdst")
|
||||||
|
fdst = os.path.join(TEST_DATA_DIR, f"relbasis_{tag}_fdst")
|
||||||
|
self._seed_source(source)
|
||||||
|
|
||||||
|
# rsync: relative DIR -> dest/basis/<transfer-relative name>.
|
||||||
|
clean_dir(rdst)
|
||||||
|
for rel, data in self._FILES.items():
|
||||||
|
_write(os.path.join(rdst, "basis", rel), data, OLD_MTIME)
|
||||||
|
rs = _rsync(["-a", f"{flag}=basis", source + "/", rdst + "/"])
|
||||||
|
assert rs.returncode == 0, rs.stderr
|
||||||
|
|
||||||
|
# FastSync: the SAME relative spelling seeded at the SAME
|
||||||
|
# transfer-relative location under its destination root.
|
||||||
|
clean_dir(fdst)
|
||||||
|
for rel, data in self._FILES.items():
|
||||||
|
_write(os.path.join(fdst, "basis", rel), data, OLD_MTIME)
|
||||||
|
result, _ = run_client(source, fdst,
|
||||||
|
flags=["-a", f"{flag}=basis", "--incremental"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||||
|
received = get_dest_received_dir(fdst, source)
|
||||||
|
|
||||||
|
for rel, data in self._FILES.items():
|
||||||
|
rfile = os.path.join(rdst, rel)
|
||||||
|
ffile = os.path.join(received, rel)
|
||||||
|
basis = os.path.join(fdst, "basis", rel)
|
||||||
|
if flag == "--compare-dest":
|
||||||
|
# compare-dest never copies: both destinations stay sparse.
|
||||||
|
assert not os.path.exists(rfile), f"rsync copied {rel}"
|
||||||
|
assert not os.path.exists(ffile), (
|
||||||
|
f"FastSync did not resolve the relative basis DIR at {basis!r} "
|
||||||
|
f"(expected {rel!r} to stay sparse like rsync)")
|
||||||
|
else:
|
||||||
|
# link-dest hard-links; a basis miss would transfer a new file.
|
||||||
|
assert os.path.exists(ffile), f"FastSync lost {rel}"
|
||||||
|
assert _read(ffile) == data
|
||||||
|
assert os.stat(ffile).st_ino == os.stat(basis).st_ino, (
|
||||||
|
f"FastSync did not hard-link {rel!r} to the relative basis at "
|
||||||
|
f"{basis!r} (basis not resolved like rsync)")
|
||||||
|
|
||||||
|
@requires_rsync
|
||||||
|
def test_relative_dir_copy_dest_content(self, shared_server):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "relbasis_copy_src")
|
||||||
|
fdst = os.path.join(TEST_DATA_DIR, "relbasis_copy_fdst")
|
||||||
|
self._seed_source(source)
|
||||||
|
clean_dir(fdst)
|
||||||
|
for rel, data in self._FILES.items():
|
||||||
|
_write(os.path.join(fdst, "basis", rel), data, OLD_MTIME)
|
||||||
|
result, _ = run_client(source, fdst,
|
||||||
|
flags=["-a", "--copy-dest=basis", "--incremental"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||||
|
received = get_dest_received_dir(fdst, source)
|
||||||
|
for rel, data in self._FILES.items():
|
||||||
|
ffile = os.path.join(received, rel)
|
||||||
|
assert os.path.exists(ffile), f"copy-dest did not materialize {rel}"
|
||||||
|
assert _read(ffile) == data
|
||||||
|
assert os.stat(ffile).st_ino != os.stat(os.path.join(fdst, "basis", rel)).st_ino
|
||||||
|
|
||||||
|
|
||||||
|
class TestFuzzyEligibilityWindow:
|
||||||
|
"""A5: --fuzzy candidate eligibility must match rsync's uncapped window."""
|
||||||
|
|
||||||
|
BASE = b"the quick brown fox jumps over the lazy dog\n" * 4000
|
||||||
|
|
||||||
|
def _run_pair(self, shared_server, tag, payload, sibling):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, f"fzw_{tag}_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, f"fzw_{tag}_dst")
|
||||||
|
rdst = os.path.join(TEST_DATA_DIR, f"fzw_{tag}_rdst")
|
||||||
|
clean_dir(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
clean_dir(rdst)
|
||||||
|
_write(os.path.join(source, "report_v2.txt"), payload)
|
||||||
|
for root in (rdst, get_dest_received_dir(dest, source)):
|
||||||
|
_write(os.path.join(root, "report_v1.txt"), sibling)
|
||||||
|
|
||||||
|
rs = _rsync(["-a", "--no-whole-file", "--fuzzy", "--stats",
|
||||||
|
source + "/", rdst + "/"])
|
||||||
|
assert rs.returncode == 0, rs.stderr
|
||||||
|
result, _ = run_client(
|
||||||
|
source, dest,
|
||||||
|
flags=["-a", "--incremental", "--delta", "--fuzzy", "--stats"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||||
|
|
||||||
|
# The reconstructed file is byte-exact in every case.
|
||||||
|
assert _read(os.path.join(get_dest_received_dir(dest, source),
|
||||||
|
"report_v2.txt")) == payload
|
||||||
|
return rs, result
|
||||||
|
|
||||||
|
@requires_rsync
|
||||||
|
def test_oversized_sibling_eligible_like_rsync(self, shared_server):
|
||||||
|
"""A sibling 20x the source is used by rsync; FastSync must too (its old
|
||||||
|
10x delta-size gate declined it)."""
|
||||||
|
n = 65536
|
||||||
|
payload = (self.BASE * ((n // len(self.BASE)) + 1))[:n]
|
||||||
|
sibling = (self.BASE * 200)[: n * 20]
|
||||||
|
rs, result = self._run_pair(shared_server, "big", payload, sibling)
|
||||||
|
assert _stat_bytes(rs.stdout, "Matched data") > 0, \
|
||||||
|
"rsync should use a >10x fuzzy basis"
|
||||||
|
assert _stat_bytes(result.stdout, "Matched data") > 0, (
|
||||||
|
"FastSync's fuzzy eligibility must accept a >10x sibling like rsync "
|
||||||
|
f"(Matched data={_stat_bytes(result.stdout, 'Matched data')})")
|
||||||
|
|
||||||
|
@requires_rsync
|
||||||
|
def test_small_source_sibling_eligible_like_rsync(self, shared_server):
|
||||||
|
"""A sub-16-KiB source with an identical sibling is used by rsync;
|
||||||
|
FastSync's old 16 KiB delta minimum declined it."""
|
||||||
|
n = 8192
|
||||||
|
payload = (self.BASE * ((n // len(self.BASE)) + 1))[:n]
|
||||||
|
rs, result = self._run_pair(shared_server, "small", payload, payload)
|
||||||
|
assert _stat_bytes(rs.stdout, "Matched data") > 0, \
|
||||||
|
"rsync applies --fuzzy below 16 KiB"
|
||||||
|
assert _stat_bytes(result.stdout, "Matched data") > 0, (
|
||||||
|
"FastSync's fuzzy eligibility must accept a sub-16-KiB source like "
|
||||||
|
f"rsync (Matched data={_stat_bytes(result.stdout, 'Matched data')})")
|
||||||
@@ -0,0 +1,105 @@
|
|||||||
|
"""`--debug=FLAGS` natural-event categories (no-wire).
|
||||||
|
|
||||||
|
FastSync maps the rsync `--debug` categories that correspond to a real event it
|
||||||
|
already performs (``flist``, ``del``, ``hash``/``deltasum``, ``recv``,
|
||||||
|
``filter`` and ``send``) onto debug output. A normal run prints none of it.
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
sys.path.insert(0, os.path.dirname(__file__))
|
||||||
|
from common import TEST_DATA_DIR, run_client, clean_dir, get_dest_received_dir, ServerManager
|
||||||
|
|
||||||
|
|
||||||
|
def _make_tree(root):
|
||||||
|
clean_dir(root)
|
||||||
|
os.makedirs(os.path.join(root, "sub"))
|
||||||
|
with open(os.path.join(root, "a.txt"), "wb") as fh:
|
||||||
|
fh.write(b"alpha\n")
|
||||||
|
with open(os.path.join(root, "keep.log"), "wb") as fh:
|
||||||
|
fh.write(b"log\n")
|
||||||
|
with open(os.path.join(root, "sub", "b.txt"), "wb") as fh:
|
||||||
|
fh.write(b"beta\n")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_debug_flist_and_send_emit_output(shared_server):
|
||||||
|
"""`--debug=flist,send` produces category-tagged debug output."""
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "dbg_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "dbg_dst")
|
||||||
|
_make_tree(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
result, _ = run_client(source, dest, flags=["-a", "--debug=flist,send"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||||
|
assert "flist: scanning" in result.stdout, result.stdout
|
||||||
|
assert "send: " in result.stdout, result.stdout
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_debug_filter_emits_excluded_entry(shared_server):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "dbg_filter_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "dbg_filter_dst")
|
||||||
|
_make_tree(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
result, _ = run_client(source, dest,
|
||||||
|
flags=["-a", "--debug=filter", "--exclude=*.log"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||||
|
assert "filter: excluded keep.log" in result.stdout, result.stdout
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_debug_hash_and_recv_emit_on_incremental(shared_server):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "dbg_hash_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "dbg_hash_dst")
|
||||||
|
_make_tree(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
result, _ = run_client(source, dest,
|
||||||
|
flags=["-a", "--incremental", "--checksum",
|
||||||
|
"--debug=hash,recv"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||||
|
assert "hash: " in result.stdout, result.stdout
|
||||||
|
assert "recv: " in result.stdout, result.stdout
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_debug_del_emits_deleted_path():
|
||||||
|
"""`--debug=del` reports the paths the receiver actually removed.
|
||||||
|
|
||||||
|
A deletion-capable server is required (the shared fixture refuses
|
||||||
|
client-requested deletion)."""
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "dbg_del_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "dbg_del_dst")
|
||||||
|
_make_tree(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
seeded = get_dest_received_dir(dest, source)
|
||||||
|
os.makedirs(seeded)
|
||||||
|
with open(os.path.join(seeded, "extra.tmp"), "wb") as fh:
|
||||||
|
fh.write(b"stale\n")
|
||||||
|
server = ServerManager()
|
||||||
|
server.start(extra_args=["--allow-super", "--allow-delete"])
|
||||||
|
try:
|
||||||
|
result, _ = run_client(source, dest, flags=["-a", "--delete", "--debug=del"],
|
||||||
|
port=server.port)
|
||||||
|
finally:
|
||||||
|
server.stop()
|
||||||
|
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||||
|
assert "del: " in result.stdout and "extra.tmp" in result.stdout, result.stdout
|
||||||
|
assert not os.path.exists(os.path.join(seeded, "extra.tmp"))
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_normal_run_has_no_debug_output(shared_server):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "dbg_quiet_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "dbg_quiet_dst")
|
||||||
|
_make_tree(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
result, _ = run_client(source, dest, flags=["-a"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||||
|
assert "[DEBUG]" not in result.stdout
|
||||||
|
assert "flist: scanning" not in result.stdout
|
||||||
|
assert "send: " not in result.stdout
|
||||||
@@ -0,0 +1,174 @@
|
|||||||
|
"""Differential parity for `--info=mount` and `--info=stats` (no-wire).
|
||||||
|
|
||||||
|
Both behaviours are compared against real rsync 3.4.1:
|
||||||
|
|
||||||
|
* `--info=mount` prints rsync's ``[sender] skipping mount-point dir NAME`` line
|
||||||
|
when ``-xx`` drops a mount-point directory. Plain ``-x`` keeps the empty
|
||||||
|
directory and stays silent, exactly like rsync.
|
||||||
|
* `--info=stats` requests the same transfer-statistics block as `--stats`
|
||||||
|
(rsync spells the full block ``--info=stats2``/``--stats``).
|
||||||
|
|
||||||
|
The tests are skipped when rsync is unavailable.
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
sys.path.insert(0, os.path.dirname(__file__))
|
||||||
|
from common import TEST_DATA_DIR, run_client, clean_dir, get_dest_received_dir
|
||||||
|
|
||||||
|
RSYNC = shutil.which("rsync")
|
||||||
|
requires_rsync = pytest.mark.skipif(RSYNC is None, reason="rsync 3.4.1 not installed")
|
||||||
|
|
||||||
|
|
||||||
|
def _rsync(args):
|
||||||
|
env = dict(os.environ, LC_ALL="C")
|
||||||
|
return subprocess.run([RSYNC] + args, capture_output=True, text=True, env=env, timeout=120)
|
||||||
|
|
||||||
|
|
||||||
|
def _cross_device_mount_tree(source):
|
||||||
|
"""Build a source whose ``nested_link`` is a symlink onto a tmpfs directory.
|
||||||
|
|
||||||
|
``--copy-links`` dereferences it so ``-x`` sees a mount-point directory on a
|
||||||
|
different device. Returns the probe path to remove, or skips the test when
|
||||||
|
no cross-device filesystem is available.
|
||||||
|
"""
|
||||||
|
local = os.stat(".")
|
||||||
|
shm = "/dev/shm"
|
||||||
|
try:
|
||||||
|
shm_stat = os.stat(shm)
|
||||||
|
except OSError:
|
||||||
|
pytest.skip("/dev/shm not available")
|
||||||
|
if shm_stat.st_dev == local.st_dev:
|
||||||
|
pytest.skip("no cross-device filesystem available")
|
||||||
|
|
||||||
|
clean_dir(source)
|
||||||
|
with open(os.path.join(source, "keep.txt"), "wb") as fh:
|
||||||
|
fh.write(b"keep\n")
|
||||||
|
probe = os.path.join(shm, f"fastsync_info_mount_{os.getpid()}")
|
||||||
|
shutil.rmtree(probe, ignore_errors=True)
|
||||||
|
os.makedirs(probe)
|
||||||
|
with open(os.path.join(probe, "inside.txt"), "wb") as fh:
|
||||||
|
fh.write(b"cross\n")
|
||||||
|
try:
|
||||||
|
os.symlink(probe, os.path.join(source, "nested_link"))
|
||||||
|
except OSError:
|
||||||
|
shutil.rmtree(probe, ignore_errors=True)
|
||||||
|
pytest.skip("cannot create symlink")
|
||||||
|
return probe
|
||||||
|
|
||||||
|
|
||||||
|
@requires_rsync
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_info_mount_xx_matches_rsync(shared_server):
|
||||||
|
"""`-xx --info=mount` drops the mount-point dir and prints rsync's line."""
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "info_mount_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "info_mount_dst")
|
||||||
|
rdst = os.path.join(TEST_DATA_DIR, "info_mount_rdst")
|
||||||
|
probe = _cross_device_mount_tree(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
clean_dir(rdst)
|
||||||
|
flags = ["-a", "--copy-links", "-xx", "--info=mount"]
|
||||||
|
try:
|
||||||
|
rsync_result = _rsync(flags + [source + "/", rdst + "/"])
|
||||||
|
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||||
|
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
|
||||||
|
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||||
|
|
||||||
|
expected = "[sender] skipping mount-point dir nested_link"
|
||||||
|
assert expected in rsync_result.stdout, rsync_result.stdout
|
||||||
|
assert expected in result.stdout, (result.stdout, result.stderr)
|
||||||
|
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
assert os.path.exists(os.path.join(received, "keep.txt"))
|
||||||
|
# -xx omits the mount-point directory entirely.
|
||||||
|
assert not os.path.exists(os.path.join(received, "nested_link"))
|
||||||
|
assert not os.path.exists(os.path.join(rdst, "nested_link"))
|
||||||
|
finally:
|
||||||
|
shutil.rmtree(probe, ignore_errors=True)
|
||||||
|
|
||||||
|
|
||||||
|
@requires_rsync
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_info_mount_single_x_is_silent(shared_server):
|
||||||
|
"""Plain `-x` keeps the empty mount-point directory and prints no line."""
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "info_mount1_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "info_mount1_dst")
|
||||||
|
rdst = os.path.join(TEST_DATA_DIR, "info_mount1_rdst")
|
||||||
|
probe = _cross_device_mount_tree(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
clean_dir(rdst)
|
||||||
|
flags = ["-a", "--copy-links", "-x", "--info=mount"]
|
||||||
|
try:
|
||||||
|
rsync_result = _rsync(flags + [source + "/", rdst + "/"])
|
||||||
|
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||||
|
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
|
||||||
|
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||||
|
|
||||||
|
assert "skipping mount-point dir" not in rsync_result.stdout
|
||||||
|
assert "skipping mount-point dir" not in result.stdout
|
||||||
|
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
assert os.path.isdir(os.path.join(received, "nested_link"))
|
||||||
|
assert not os.path.exists(os.path.join(received, "nested_link", "inside.txt"))
|
||||||
|
assert os.path.isdir(os.path.join(rdst, "nested_link"))
|
||||||
|
assert not os.path.exists(os.path.join(rdst, "nested_link", "inside.txt"))
|
||||||
|
finally:
|
||||||
|
shutil.rmtree(probe, ignore_errors=True)
|
||||||
|
|
||||||
|
|
||||||
|
def _make_stats_tree(root):
|
||||||
|
clean_dir(root)
|
||||||
|
os.makedirs(os.path.join(root, "sub"))
|
||||||
|
with open(os.path.join(root, "a.txt"), "wb") as fh:
|
||||||
|
fh.write(b"alpha\n")
|
||||||
|
with open(os.path.join(root, "sub", "b.txt"), "wb") as fh:
|
||||||
|
fh.write(b"beta\n")
|
||||||
|
|
||||||
|
|
||||||
|
def _pick_stats(text):
|
||||||
|
keys = ("Number of files", "Number of regular files transferred", "Total file size",
|
||||||
|
"Total transferred file size", "Literal data", "Matched data")
|
||||||
|
out = {}
|
||||||
|
for line in text.splitlines():
|
||||||
|
for key in keys:
|
||||||
|
if line.startswith(key + ":"):
|
||||||
|
out[key] = line
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
@requires_rsync
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_info_stats_emits_full_stats_block(shared_server):
|
||||||
|
"""`--info=stats` is the same full block as `--stats` and matches rsync."""
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "info_stats_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "info_stats_dst")
|
||||||
|
rdst = os.path.join(TEST_DATA_DIR, "info_stats_rdst")
|
||||||
|
dest2 = os.path.join(TEST_DATA_DIR, "info_stats_dst2")
|
||||||
|
_make_stats_tree(source)
|
||||||
|
for path in (dest, rdst, dest2):
|
||||||
|
clean_dir(path)
|
||||||
|
os.makedirs(get_dest_received_dir(path, source), exist_ok=True)
|
||||||
|
|
||||||
|
rsync_result = _rsync(["-a", "--stats", source + "/", rdst + "/"])
|
||||||
|
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||||
|
|
||||||
|
info_result, _ = run_client(source, dest, flags=["-a", "--info=stats"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert info_result.returncode == 0, (info_result.stderr or info_result.stdout)[:300]
|
||||||
|
stats_result, _ = run_client(source, dest2, flags=["-a", "--stats"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert stats_result.returncode == 0, (stats_result.stderr or stats_result.stdout)[:300]
|
||||||
|
|
||||||
|
# --info=stats must print the same block as --stats...
|
||||||
|
assert _pick_stats(info_result.stdout) == _pick_stats(stats_result.stdout), (
|
||||||
|
f"info={info_result.stdout} stats={stats_result.stdout}")
|
||||||
|
# ...and the protocol-independent counters must match real rsync.
|
||||||
|
assert _pick_stats(info_result.stdout) == _pick_stats(rsync_result.stdout), (
|
||||||
|
f"rsync={_pick_stats(rsync_result.stdout)} fastsync={_pick_stats(info_result.stdout)}")
|
||||||
|
assert re.search(r"^Number of files: \d+ \(reg: 2, dir: 2\)$", info_result.stdout,
|
||||||
|
re.MULTILINE), info_result.stdout
|
||||||
@@ -0,0 +1,185 @@
|
|||||||
|
"""Differential rsync-parity coverage for FastSync's transfer/delete ORDER.
|
||||||
|
|
||||||
|
rsync walks a source tree in its sorted flist order: within each directory the
|
||||||
|
non-directories come first (ascending name), then the subdirectories (ascending
|
||||||
|
name), each subdirectory immediately followed by its own subtree (depth-first).
|
||||||
|
The sequential scanner now reproduces that order, which makes both the
|
||||||
|
``--info=name`` stream and the ``--delete-during`` deletion sequence match real
|
||||||
|
``rsync 3.4.1`` exactly. ``--threads`` has no rsync analogue and is unordered.
|
||||||
|
|
||||||
|
Every test skips cleanly when rsync is absent.
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
sys.path.insert(0, os.path.dirname(__file__))
|
||||||
|
from common import ( # noqa: E402
|
||||||
|
TEST_DATA_DIR,
|
||||||
|
ServerManager,
|
||||||
|
clean_dir,
|
||||||
|
get_dest_received_dir,
|
||||||
|
run_client,
|
||||||
|
)
|
||||||
|
|
||||||
|
RSYNC = shutil.which("rsync")
|
||||||
|
requires_rsync = pytest.mark.skipif(RSYNC is None, reason="rsync 3.4.1 not installed")
|
||||||
|
|
||||||
|
MTIME = 1_500_000_000
|
||||||
|
|
||||||
|
_TREE = {
|
||||||
|
"a.txt": b"a\n",
|
||||||
|
"b.txt": b"b\n",
|
||||||
|
"z.txt": b"z\n",
|
||||||
|
"a_dir/f.txt": b"f\n",
|
||||||
|
"a_dir/deep/g.txt": b"g\n",
|
||||||
|
"m_dir/h.txt": b"h\n",
|
||||||
|
"Z_dir/i.txt": b"i\n",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _write(path, data):
|
||||||
|
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||||
|
with open(path, "wb") as fh:
|
||||||
|
fh.write(data)
|
||||||
|
os.utime(path, (MTIME, MTIME))
|
||||||
|
|
||||||
|
|
||||||
|
def _rsync(args):
|
||||||
|
env = dict(os.environ, LC_ALL="C")
|
||||||
|
return subprocess.run([RSYNC] + args, capture_output=True, text=True, env=env, timeout=120)
|
||||||
|
|
||||||
|
|
||||||
|
def _deleting(text):
|
||||||
|
out = []
|
||||||
|
for line in text.splitlines():
|
||||||
|
stripped = line.strip()
|
||||||
|
if stripped.startswith("*deleting") or stripped.startswith("deleting"):
|
||||||
|
out.append(stripped.split()[-1])
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
class TestTransferOrderParity:
|
||||||
|
@requires_rsync
|
||||||
|
def test_info_name_file_order_matches_rsync(self, shared_server):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "order_name_src")
|
||||||
|
clean_dir(source)
|
||||||
|
for rel, data in _TREE.items():
|
||||||
|
_write(os.path.join(source, rel), data)
|
||||||
|
|
||||||
|
rdst = os.path.join(TEST_DATA_DIR, "order_name_rdst")
|
||||||
|
clean_dir(rdst)
|
||||||
|
rs = _rsync(["-a", "--info=name", source + "/", rdst + "/"])
|
||||||
|
assert rs.returncode == 0, rs.stderr
|
||||||
|
# rsync also names the directories (trailing '/'); FastSync names the
|
||||||
|
# transferred entries. Compare the file/symlink sequence, which is what
|
||||||
|
# the traversal order determines.
|
||||||
|
rsync_files = [l for l in rs.stdout.splitlines() if l.strip() and not l.endswith("/")]
|
||||||
|
|
||||||
|
fdst = os.path.join(TEST_DATA_DIR, "order_name_fdst")
|
||||||
|
clean_dir(fdst)
|
||||||
|
result, _ = run_client(source, fdst, flags=["-a", "--info=name"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||||
|
fsync_files = [
|
||||||
|
l for l in result.stdout.splitlines()
|
||||||
|
if l.strip() and l.strip() != "./" and not l.startswith("sending")
|
||||||
|
]
|
||||||
|
assert fsync_files == rsync_files, (
|
||||||
|
f"transfer order differs\nrsync={rsync_files}\nfastsync={fsync_files}")
|
||||||
|
|
||||||
|
|
||||||
|
class TestDeleteOrderParity:
|
||||||
|
_EXTRA = {
|
||||||
|
"a_extra.txt": b"a\n",
|
||||||
|
"z_extra.txt": b"z\n",
|
||||||
|
"a_extra_dir/f": b"f\n",
|
||||||
|
"z_extra_dir/f": b"f\n",
|
||||||
|
"a_extra_dir/sub/g": b"g\n",
|
||||||
|
}
|
||||||
|
|
||||||
|
def _seed_source(self):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "order_del_src")
|
||||||
|
clean_dir(source)
|
||||||
|
_write(os.path.join(source, "keep.txt"), b"k\n")
|
||||||
|
_write(os.path.join(source, "keepdir", "x.txt"), b"x\n")
|
||||||
|
_write(os.path.join(source, "keep2", "y.txt"), b"y\n")
|
||||||
|
return source
|
||||||
|
|
||||||
|
def _assert_order(self, timing, dry_run=False):
|
||||||
|
source = self._seed_source()
|
||||||
|
rdst = os.path.join(TEST_DATA_DIR, f"order_{timing}_rdst")
|
||||||
|
clean_dir(rdst)
|
||||||
|
for rel, data in self._EXTRA.items():
|
||||||
|
_write(os.path.join(rdst, rel), data)
|
||||||
|
rs_flags = ["-a", "-n"] if dry_run else ["-a"]
|
||||||
|
rs = _rsync(rs_flags + [timing, "--info=del", source + "/", rdst + "/"])
|
||||||
|
assert rs.returncode == 0, rs.stderr
|
||||||
|
|
||||||
|
fdst = os.path.join(TEST_DATA_DIR, f"order_{timing}_fdst")
|
||||||
|
clean_dir(fdst)
|
||||||
|
received = get_dest_received_dir(fdst, source)
|
||||||
|
for rel, data in self._EXTRA.items():
|
||||||
|
_write(os.path.join(received, rel), data)
|
||||||
|
fs_flags = ["-a", "-n"] if dry_run else ["-a"]
|
||||||
|
with ServerManager() as server:
|
||||||
|
server.start(extra_args=["--allow-delete"])
|
||||||
|
result, _ = run_client(source, fdst, flags=fs_flags + [timing, "--info=del"],
|
||||||
|
port=server.port)
|
||||||
|
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||||
|
|
||||||
|
rsync_order = _deleting(rs.stdout)
|
||||||
|
fsync_order = _deleting(result.stdout)
|
||||||
|
assert sorted(fsync_order) == sorted(rsync_order), (
|
||||||
|
f"{timing} deleted set differs\nrsync={rsync_order}\nfastsync={fsync_order}")
|
||||||
|
assert fsync_order == rsync_order, (
|
||||||
|
f"{timing} deletion order differs\nrsync={rsync_order}\nfastsync={fsync_order}")
|
||||||
|
|
||||||
|
@requires_rsync
|
||||||
|
def test_delete_during_deletion_order_matches_rsync(self):
|
||||||
|
self._assert_order("--delete-during")
|
||||||
|
|
||||||
|
@requires_rsync
|
||||||
|
def test_delete_delay_deletion_order_matches_rsync(self):
|
||||||
|
self._assert_order("--delete-delay")
|
||||||
|
|
||||||
|
@requires_rsync
|
||||||
|
def test_dry_run_delete_order_matches_rsync(self):
|
||||||
|
self._assert_order("--delete", dry_run=True)
|
||||||
|
|
||||||
|
@requires_rsync
|
||||||
|
def test_partial_max_delete_survivor_order_matches_rsync(self):
|
||||||
|
"""With the exact removal order matching rsync, a --max-delete cap stops
|
||||||
|
after the same entries, so the survivor set is identical too."""
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "order_maxdel_src")
|
||||||
|
clean_dir(source)
|
||||||
|
_write(os.path.join(source, "keep.txt"), b"k\n")
|
||||||
|
extra = {f"e{i}.txt": b"x\n" for i in range(6)}
|
||||||
|
extra["ed/f"] = b"f\n"
|
||||||
|
extra["ed/g"] = b"g\n"
|
||||||
|
|
||||||
|
rdst = os.path.join(TEST_DATA_DIR, "order_maxdel_rdst")
|
||||||
|
clean_dir(rdst)
|
||||||
|
for rel, data in extra.items():
|
||||||
|
_write(os.path.join(rdst, rel), data)
|
||||||
|
rs = _rsync(["-a", "--delete-during", "--max-delete=3", "--info=del",
|
||||||
|
source + "/", rdst + "/"])
|
||||||
|
assert rs.returncode in (0, 25), (rs.returncode, rs.stderr)
|
||||||
|
|
||||||
|
fdst = os.path.join(TEST_DATA_DIR, "order_maxdel_fdst")
|
||||||
|
clean_dir(fdst)
|
||||||
|
received = get_dest_received_dir(fdst, source)
|
||||||
|
for rel, data in extra.items():
|
||||||
|
_write(os.path.join(received, rel), data)
|
||||||
|
with ServerManager() as server:
|
||||||
|
server.start(extra_args=["--allow-delete"])
|
||||||
|
result, _ = run_client(source, fdst,
|
||||||
|
flags=["-a", "--delete-during", "--max-delete=3", "--info=del"],
|
||||||
|
port=server.port)
|
||||||
|
assert result.returncode in (0, 25), (result.returncode, result.stderr[:300])
|
||||||
|
assert _deleting(result.stdout) == _deleting(rs.stdout), (
|
||||||
|
f"partial --max-delete survivor order differs\n"
|
||||||
|
f"rsync={_deleting(rs.stdout)}\nfastsync={_deleting(result.stdout)}")
|
||||||
@@ -861,11 +861,11 @@ class TestFuzzy:
|
|||||||
byte-exact result. FastSync ports rsync 3.4.1's weighted-Levenshtein name
|
byte-exact result. FastSync ports rsync 3.4.1's weighted-Levenshtein name
|
||||||
heuristic, so where both delta engines admit the candidate the tools pick
|
heuristic, so where both delta engines admit the candidate the tools pick
|
||||||
the same basis (the ``fuzzy_basis`` differential asserts the tree and the
|
the same basis (the ``fuzzy_basis`` differential asserts the tree and the
|
||||||
Matched/Literal counters match with the block size pinned). The residual is
|
Matched/Literal counters match with the block size pinned). Candidate
|
||||||
candidate ELIGIBILITY: FastSync's delta size gate (both files >= 16 KiB and
|
ELIGIBILITY is now rsync's too: the fuzzy search no longer inherits the
|
||||||
a <= 10x size ratio) is narrower than rsync's, which empirically uses a
|
ordinary delta engine's 16 KiB minimum or 10x size-ratio bound, so an
|
||||||
fuzzy basis well beyond 10x and below 16 KiB. These tests pin the window
|
oversized or sub-16-KiB sibling is reused exactly as rsync reuses it.
|
||||||
boundary and prove the byte-exact fallback on both sides of it."""
|
These tests pin that window on both sides."""
|
||||||
|
|
||||||
_BASE = b"the quick brown fox jumps over the lazy dog\n" * 4000
|
_BASE = b"the quick brown fox jumps over the lazy dog\n" * 4000
|
||||||
|
|
||||||
@@ -900,9 +900,10 @@ class TestFuzzy:
|
|||||||
return rs, result
|
return rs, result
|
||||||
|
|
||||||
@requires_rsync
|
@requires_rsync
|
||||||
def test_fuzzy_above_size_window_declines_but_tree_exact(self, shared_server):
|
def test_fuzzy_above_size_window_matches_rsync(self, shared_server):
|
||||||
"""A sibling >10x the source is used by rsync but declined by FastSync's
|
"""A sibling >10x the source is used by rsync and by FastSync: fuzzy
|
||||||
delta size-ratio gate; both destinations stay byte-identical."""
|
eligibility is rsync's, not the ordinary delta size-ratio gate; both
|
||||||
|
destinations stay byte-identical and both reuse the basis."""
|
||||||
n = 65536
|
n = 65536
|
||||||
payload = (self._BASE * ((n // len(self._BASE)) + 1))[:n]
|
payload = (self._BASE * ((n // len(self._BASE)) + 1))[:n]
|
||||||
sibling = (self._BASE * 200)[: n * 20]
|
sibling = (self._BASE * 200)[: n * 20]
|
||||||
@@ -911,15 +912,16 @@ class TestFuzzy:
|
|||||||
rs, result = self._run_both(shared_server, source, dest, rdst,
|
rs, result = self._run_both(shared_server, source, dest, rdst,
|
||||||
payload, sibling)
|
payload, sibling)
|
||||||
assert _stat_bytes(rs.stdout, "Matched data") > 0, \
|
assert _stat_bytes(rs.stdout, "Matched data") > 0, \
|
||||||
"rsync should still use a >10x fuzzy basis"
|
"rsync should use a >10x fuzzy basis"
|
||||||
assert _stat_bytes(result.stdout, "Matched data") == 0, \
|
assert _stat_bytes(result.stdout, "Matched data") > 0, \
|
||||||
"FastSync's 10x delta size-ratio gate must decline the oversized basis"
|
"FastSync must accept a >10x fuzzy basis like rsync"
|
||||||
assert _stat_bytes(result.stdout, "Literal data") == n
|
assert _stat_bytes(result.stdout, "Literal data") < n
|
||||||
|
|
||||||
@requires_rsync
|
@requires_rsync
|
||||||
def test_fuzzy_below_delta_minimum_declines_but_tree_exact(self, shared_server):
|
def test_fuzzy_below_delta_minimum_matches_rsync(self, shared_server):
|
||||||
"""A sibling below the 16 KiB delta minimum is used by rsync but never
|
"""A sibling below the 16 KiB delta minimum is used by rsync and by
|
||||||
enters FastSync's delta/fuzzy path; both trees stay byte-identical."""
|
FastSync: fuzzy eligibility no longer inherits the delta engine's
|
||||||
|
minimum; both trees stay byte-identical and both reuse the basis."""
|
||||||
n = 8192
|
n = 8192
|
||||||
payload = (self._BASE * ((n // len(self._BASE)) + 1))[:n]
|
payload = (self._BASE * ((n // len(self._BASE)) + 1))[:n]
|
||||||
source, dest, rdst = (self._src("small"), self._dst("small"),
|
source, dest, rdst = (self._src("small"), self._dst("small"),
|
||||||
@@ -928,9 +930,9 @@ class TestFuzzy:
|
|||||||
payload, payload)
|
payload, payload)
|
||||||
assert _stat_bytes(rs.stdout, "Matched data") > 0, \
|
assert _stat_bytes(rs.stdout, "Matched data") > 0, \
|
||||||
"rsync applies --fuzzy below 16 KiB"
|
"rsync applies --fuzzy below 16 KiB"
|
||||||
assert _stat_bytes(result.stdout, "Matched data") == 0, \
|
assert _stat_bytes(result.stdout, "Matched data") > 0, \
|
||||||
"FastSync's 16 KiB delta minimum must bypass the fuzzy basis"
|
"FastSync must apply --fuzzy below 16 KiB like rsync"
|
||||||
assert _stat_bytes(result.stdout, "Literal data") == n
|
assert _stat_bytes(result.stdout, "Literal data") < n
|
||||||
|
|
||||||
|
|
||||||
class TestIgnoreExistingShortCircuit:
|
class TestIgnoreExistingShortCircuit:
|
||||||
|
|||||||
@@ -1,23 +1,57 @@
|
|||||||
"""CLI validation and preflight checks."""
|
"""CLI validation and preflight checks."""
|
||||||
|
import socket
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
import os
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
|
import time
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
sys.path.insert(0, os.path.dirname(__file__))
|
sys.path.insert(0, os.path.dirname(__file__))
|
||||||
from common import BUILD_DIR, CLIENT_CMD, SERVER_CMD, TEST_DATA_DIR, run_client, verify_transfer
|
from common import (
|
||||||
|
BUILD_DIR,
|
||||||
|
CLIENT_CMD,
|
||||||
|
CLIENT_TIMEOUT,
|
||||||
|
SERVER_CMD,
|
||||||
|
TEST_DATA_DIR,
|
||||||
|
get_dest_received_dir,
|
||||||
|
run_client,
|
||||||
|
verify_transfer,
|
||||||
|
)
|
||||||
|
|
||||||
|
DEFAULT_PORT = 8080
|
||||||
|
|
||||||
|
|
||||||
|
def _port_is_listening(host, port, timeout=0.3):
|
||||||
|
"""True if something accepts a TCP connection on host:port right now."""
|
||||||
|
try:
|
||||||
|
with socket.create_connection((host, port), timeout=timeout):
|
||||||
|
return True
|
||||||
|
except OSError:
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _wait_for_listener(host, port, timeout=5.0):
|
||||||
|
"""Poll host:port until a listener accepts, or the deadline passes."""
|
||||||
|
deadline = time.monotonic() + timeout
|
||||||
|
while time.monotonic() < deadline:
|
||||||
|
if _port_is_listening(host, port):
|
||||||
|
return True
|
||||||
|
time.sleep(0.05)
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
class TestHelp:
|
class TestHelp:
|
||||||
def test_client_help(self):
|
def test_client_help(self):
|
||||||
r = subprocess.run(CLIENT_CMD + ["--help"], capture_output=True, text=True)
|
r = subprocess.run(CLIENT_CMD + ["--help"], capture_output=True,
|
||||||
|
text=True, timeout=CLIENT_TIMEOUT)
|
||||||
assert r.returncode == 0
|
assert r.returncode == 0
|
||||||
assert "Usage:" in r.stdout
|
assert "Usage:" in r.stdout
|
||||||
assert "SSH transport" in r.stdout
|
assert "SSH transport" in r.stdout
|
||||||
|
|
||||||
def test_server_help(self):
|
def test_server_help(self):
|
||||||
r = subprocess.run(SERVER_CMD + ["--help"], capture_output=True, text=True)
|
r = subprocess.run(SERVER_CMD + ["--help"], capture_output=True,
|
||||||
|
text=True, timeout=CLIENT_TIMEOUT)
|
||||||
assert r.returncode == 0
|
assert r.returncode == 0
|
||||||
assert "Usage:" in r.stdout
|
assert "Usage:" in r.stdout
|
||||||
|
|
||||||
@@ -67,19 +101,24 @@ class TestServerPort:
|
|||||||
|
|
||||||
def test_default_port(self):
|
def test_default_port(self):
|
||||||
"""Server should start on default port 8080."""
|
"""Server should start on default port 8080."""
|
||||||
|
if _port_is_listening("127.0.0.1", DEFAULT_PORT):
|
||||||
|
pytest.skip(f"port {DEFAULT_PORT} already in use by another process")
|
||||||
|
|
||||||
proc = subprocess.Popen(
|
proc = subprocess.Popen(
|
||||||
SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None,
|
SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None,
|
||||||
)
|
)
|
||||||
try:
|
try:
|
||||||
import socket, time
|
if not _wait_for_listener("127.0.0.1", DEFAULT_PORT, timeout=5.0):
|
||||||
time.sleep(0.5)
|
if proc.poll() is not None and _port_is_listening("127.0.0.1", DEFAULT_PORT):
|
||||||
with socket.create_connection(("127.0.0.1", 8080), timeout=2):
|
pytest.skip(f"port {DEFAULT_PORT} was taken by another process")
|
||||||
pass # Port is listening
|
pytest.fail(f"Server not listening on default port {DEFAULT_PORT}")
|
||||||
except (ConnectionRefusedError, OSError):
|
|
||||||
pytest.fail("Server not listening on default port 8080")
|
|
||||||
finally:
|
finally:
|
||||||
proc.terminate()
|
proc.terminate()
|
||||||
|
try:
|
||||||
proc.wait(timeout=5)
|
proc.wait(timeout=5)
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
proc.kill()
|
||||||
|
proc.wait()
|
||||||
|
|
||||||
|
|
||||||
def _seed_protocol_source(source):
|
def _seed_protocol_source(source):
|
||||||
@@ -105,7 +144,7 @@ class TestProtocol:
|
|||||||
port=shared_server.port)
|
port=shared_server.port)
|
||||||
assert result.returncode == 0, \
|
assert result.returncode == 0, \
|
||||||
f"--protocol current run failed: {(result.stderr or result.stdout)[:400]}"
|
f"--protocol current run failed: {(result.stderr or result.stdout)[:400]}"
|
||||||
received = os.path.join(dest, os.path.abspath(source).lstrip(os.sep))
|
received = get_dest_received_dir(dest, source)
|
||||||
mismatches, missing = verify_transfer(source, received)
|
mismatches, missing = verify_transfer(source, received)
|
||||||
assert not mismatches and not missing, \
|
assert not mismatches and not missing, \
|
||||||
f"transfer mismatch: missing={missing} mismatches={mismatches}"
|
f"transfer mismatch: missing={missing} mismatches={mismatches}"
|
||||||
|
|||||||
@@ -16,6 +16,7 @@
|
|||||||
#include "test_file.h"
|
#include "test_file.h"
|
||||||
#include "test_file_list.h"
|
#include "test_file_list.h"
|
||||||
#include "test_file_sendfile.h"
|
#include "test_file_sendfile.h"
|
||||||
|
#include "test_filter.h"
|
||||||
#include "test_format.h"
|
#include "test_format.h"
|
||||||
#include "test_fuzz_smoke.h"
|
#include "test_fuzz_smoke.h"
|
||||||
#include "test_glob.h"
|
#include "test_glob.h"
|
||||||
@@ -80,6 +81,7 @@ int main() {
|
|||||||
RUN_TEST(test_receiver_timeout);
|
RUN_TEST(test_receiver_timeout);
|
||||||
RUN_TEST(test_metadata);
|
RUN_TEST(test_metadata);
|
||||||
RUN_TEST(test_glob);
|
RUN_TEST(test_glob);
|
||||||
|
RUN_TEST(test_filter);
|
||||||
RUN_TEST(test_iconv);
|
RUN_TEST(test_iconv);
|
||||||
RUN_TEST(test_file);
|
RUN_TEST(test_file);
|
||||||
RUN_TEST(test_file_list);
|
RUN_TEST(test_file_list);
|
||||||
|
|||||||
+167
-28
@@ -171,6 +171,38 @@ static void test_validate_config_unified_invariants() {
|
|||||||
config_delete(cfg);
|
config_delete(cfg);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* The receiver enforces MAX_FILTER_RULES on the protect-rule block and would
|
||||||
|
otherwise fail the session with an opaque protocol error. The client must
|
||||||
|
accept exactly the limit and reject one more up front, before any network
|
||||||
|
I/O, with an actionable message. */
|
||||||
|
static void test_validate_config_filter_rule_limit() {
|
||||||
|
Config* cfg = valid_client_config();
|
||||||
|
cfg->filters = array_list_create(free);
|
||||||
|
EXPECT_NOT_NULL(cfg->filters);
|
||||||
|
for (int i = 0; i < MAX_FILTER_RULES; i++)
|
||||||
|
EXPECT_TRUE(array_list_add(cfg->filters, str_dup("- *.tmp")));
|
||||||
|
EXPECT_TRUE(validate_config(cfg)); /* exactly the limit is accepted */
|
||||||
|
|
||||||
|
FILE* log_capture = tmpfile();
|
||||||
|
EXPECT_NOT_NULL(log_capture);
|
||||||
|
log_set_file(log_capture);
|
||||||
|
EXPECT_TRUE(array_list_add(cfg->filters, str_dup("- *.bak")));
|
||||||
|
EXPECT_FALSE(validate_config(cfg)); /* one over the limit is rejected */
|
||||||
|
fflush(log_capture);
|
||||||
|
rewind(log_capture);
|
||||||
|
char line[512];
|
||||||
|
bool saw_message = false;
|
||||||
|
while (fgets(line, sizeof(line), log_capture) != NULL) {
|
||||||
|
if (strstr(line, "too many filter rules") != NULL && strstr(line, "(maximum 1024)") != NULL)
|
||||||
|
saw_message = true;
|
||||||
|
}
|
||||||
|
log_set_file(NULL);
|
||||||
|
fclose(log_capture);
|
||||||
|
EXPECT_TRUE(saw_message);
|
||||||
|
|
||||||
|
config_delete(cfg);
|
||||||
|
}
|
||||||
|
|
||||||
/* Test main() with --help flag (early return path, no server connection needed) */
|
/* Test main() with --help flag (early return path, no server connection needed) */
|
||||||
static void test_cli_help() {
|
static void test_cli_help() {
|
||||||
/* We can't easily call main() because it calls send_files which needs a server.
|
/* We can't easily call main() because it calls send_files which needs a server.
|
||||||
@@ -511,6 +543,66 @@ static void test_parse_args_ignore_existing() {
|
|||||||
config_delete(cfg);
|
config_delete(cfg);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* --partial-dir=DIR implies --partial, matching rsync 3.4.1. rsync resolves
|
||||||
|
* this after option parsing, so the implication wins over an explicit
|
||||||
|
* --no-partial in either order. It is skipped under --inplace, where partial
|
||||||
|
* staging is bypassed and the destination is written in place. */
|
||||||
|
static void test_parse_args_partial_dir_implies_partial() {
|
||||||
|
{
|
||||||
|
Config* cfg = config_create();
|
||||||
|
char* argv[] = {"fastsync", "--partial-dir=.partial", "/src", "/dst"};
|
||||||
|
int positional_args[2];
|
||||||
|
int positional_count = 0;
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
|
||||||
|
EXPECT_TRUE(cfg->partial);
|
||||||
|
config_delete(cfg);
|
||||||
|
}
|
||||||
|
{
|
||||||
|
/* Explicit --no-partial before --partial-dir: --partial-dir still wins. */
|
||||||
|
Config* cfg = config_create();
|
||||||
|
char* argv[] = {"fastsync", "--no-partial", "--partial-dir=.partial", "/src", "/dst"};
|
||||||
|
int positional_args[2];
|
||||||
|
int positional_count = 0;
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0);
|
||||||
|
EXPECT_TRUE(cfg->partial);
|
||||||
|
config_delete(cfg);
|
||||||
|
}
|
||||||
|
{
|
||||||
|
/* Reversed order must not change the precedence. */
|
||||||
|
Config* cfg = config_create();
|
||||||
|
char* argv[] = {"fastsync", "--partial-dir=.partial", "--no-partial", "/src", "/dst"};
|
||||||
|
int positional_args[2];
|
||||||
|
int positional_count = 0;
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0);
|
||||||
|
EXPECT_TRUE(cfg->partial);
|
||||||
|
config_delete(cfg);
|
||||||
|
}
|
||||||
|
{
|
||||||
|
/* --inplace bypasses partial staging, so parse_args must not set the
|
||||||
|
implied --partial; the combination itself is invalid (rsync parity:
|
||||||
|
"--inplace cannot be used with --partial-dir"), so validation rejects. */
|
||||||
|
Config* cfg = config_create();
|
||||||
|
char* argv[] = {"fastsync", "--inplace", "--partial-dir=.partial", "/src", "/dst"};
|
||||||
|
int positional_args[2];
|
||||||
|
int positional_count = 0;
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0);
|
||||||
|
EXPECT_FALSE(cfg->partial);
|
||||||
|
cfg->send_directory = str_dup("/src");
|
||||||
|
cfg->receive_root_directory = str_dup("/dst");
|
||||||
|
EXPECT_FALSE(validate_config(cfg));
|
||||||
|
config_delete(cfg);
|
||||||
|
}
|
||||||
|
{
|
||||||
|
Config* cfg = config_create();
|
||||||
|
char* argv[] = {"fastsync", "--no-partial", "/src", "/dst"};
|
||||||
|
int positional_args[2];
|
||||||
|
int positional_count = 0;
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
|
||||||
|
EXPECT_FALSE(cfg->partial);
|
||||||
|
config_delete(cfg);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
static void test_parse_args_executability() {
|
static void test_parse_args_executability() {
|
||||||
Config* cfg = config_create();
|
Config* cfg = config_create();
|
||||||
char* argv[] = {"fastsync", "-E", "/src", "/dst"};
|
char* argv[] = {"fastsync", "-E", "/src", "/dst"};
|
||||||
@@ -762,8 +854,9 @@ static void test_parse_args_debug_flags() {
|
|||||||
int positional_count = 0;
|
int positional_count = 0;
|
||||||
|
|
||||||
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
|
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
|
||||||
EXPECT_EQ_INT(cfg->debug_level, LOG_DEBUG_ALL);
|
EXPECT_EQ_INT(cfg->debug_level, LOG_DEBUG_IO | LOG_DEBUG_PROTO | LOG_DEBUG_PACK | LOG_DEBUG_UTIL);
|
||||||
EXPECT_EQ_INT(get_log_debug_flags(), LOG_DEBUG_ALL);
|
EXPECT_EQ_INT(get_log_debug_flags(),
|
||||||
|
LOG_DEBUG_IO | LOG_DEBUG_PROTO | LOG_DEBUG_PACK | LOG_DEBUG_UTIL);
|
||||||
config_delete(cfg);
|
config_delete(cfg);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1241,35 +1334,78 @@ static void test_parse_args_delete_timing_without_delete_rejected() {
|
|||||||
config_delete(cfg);
|
config_delete(cfg);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Parsed-but-unimplemented options must fail instead of being silently accepted. */
|
/* Truly-unknown options (including server-only spellings) must be rejected
|
||||||
static void test_parse_args_rejects_unimplemented_options() {
|
* through the unknown-option path instead of being silently accepted. */
|
||||||
static const char* const options[] = {"--silent",
|
static void test_parse_args_rejects_unknown_options() {
|
||||||
"--queue-size",
|
static const char* const options[] = {"--silent", "--queue-size", "--bind-address",
|
||||||
"-A",
|
"--daemon", "--config", "--server"};
|
||||||
"--acls",
|
|
||||||
"-X",
|
|
||||||
"--xattrs",
|
|
||||||
"-D",
|
|
||||||
"--devices",
|
|
||||||
"--delete-excluded",
|
|
||||||
"--max-delete",
|
|
||||||
"--prune-empty-dirs",
|
|
||||||
"--bind-address",
|
|
||||||
"--daemon",
|
|
||||||
"--config",
|
|
||||||
"--server"};
|
|
||||||
|
|
||||||
for (size_t i = 0; i < sizeof(options) / sizeof(options[0]); i++) {
|
for (size_t i = 0; i < sizeof(options) / sizeof(options[0]); i++) {
|
||||||
Config* cfg = config_create();
|
Config* cfg = config_create();
|
||||||
char* argv[] = {"fastsync", (char*)options[i], "dummy", "/src", "/dst"};
|
char* argv[] = {"fastsync", (char*)options[i], "/src", "/dst"};
|
||||||
int positional_args[2];
|
int positional_args[2];
|
||||||
int positional_count = 0;
|
int positional_count = 0;
|
||||||
|
|
||||||
EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), -1);
|
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), -1);
|
||||||
config_delete(cfg);
|
config_delete(cfg);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Options that are genuinely implemented must parse successfully and record
|
||||||
|
* their effect, rather than being lumped in with the unknown-option set. */
|
||||||
|
static void test_parse_args_accepts_implemented_metadata_options() {
|
||||||
|
Config* cfg = config_create();
|
||||||
|
char* argv_x[] = {"fastsync", "-X", "/src", "/dst"};
|
||||||
|
int positional_args[2];
|
||||||
|
int positional_count = 0;
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 4, argv_x, positional_args, &positional_count), 0);
|
||||||
|
EXPECT_TRUE(cfg->preserve_xattrs);
|
||||||
|
config_delete(cfg);
|
||||||
|
|
||||||
|
cfg = config_create();
|
||||||
|
positional_count = 0;
|
||||||
|
char* argv_acls[] = {"fastsync", "--acls", "/src", "/dst"};
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 4, argv_acls, positional_args, &positional_count), 0);
|
||||||
|
EXPECT_TRUE(cfg->preserve_acls);
|
||||||
|
config_delete(cfg);
|
||||||
|
|
||||||
|
cfg = config_create();
|
||||||
|
positional_count = 0;
|
||||||
|
char* argv_d[] = {"fastsync", "-D", "/src", "/dst"};
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 4, argv_d, positional_args, &positional_count), 0);
|
||||||
|
EXPECT_TRUE(cfg->preserve_devices);
|
||||||
|
EXPECT_TRUE(cfg->preserve_specials);
|
||||||
|
config_delete(cfg);
|
||||||
|
|
||||||
|
cfg = config_create();
|
||||||
|
positional_count = 0;
|
||||||
|
char* argv_devices[] = {"fastsync", "--devices", "/src", "/dst"};
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 4, argv_devices, positional_args, &positional_count), 0);
|
||||||
|
EXPECT_TRUE(cfg->preserve_devices);
|
||||||
|
config_delete(cfg);
|
||||||
|
|
||||||
|
cfg = config_create();
|
||||||
|
positional_count = 0;
|
||||||
|
char* argv_delete_excluded[] = {"fastsync", "--delete-excluded", "/src", "/dst"};
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 4, argv_delete_excluded, positional_args, &positional_count), 0);
|
||||||
|
EXPECT_TRUE(cfg->delete_excluded);
|
||||||
|
config_delete(cfg);
|
||||||
|
|
||||||
|
cfg = config_create();
|
||||||
|
positional_count = 0;
|
||||||
|
char* argv_max_delete[] = {"fastsync", "--max-delete=5", "/src", "/dst"};
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 4, argv_max_delete, positional_args, &positional_count), 0);
|
||||||
|
EXPECT_EQ_INT(cfg->max_delete, 5);
|
||||||
|
config_delete(cfg);
|
||||||
|
|
||||||
|
cfg = config_create();
|
||||||
|
positional_count = 0;
|
||||||
|
char* argv_prune[] = {"fastsync", "--prune-empty-dirs", "/src", "/dst"};
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 4, argv_prune, positional_args, &positional_count), 0);
|
||||||
|
EXPECT_TRUE(cfg->prune_empty_dirs);
|
||||||
|
config_delete(cfg);
|
||||||
|
}
|
||||||
|
|
||||||
/* Test both rsync-compatible quiet spellings and option ordering. */
|
/* Test both rsync-compatible quiet spellings and option ordering. */
|
||||||
static void test_parse_args_quiet() {
|
static void test_parse_args_quiet() {
|
||||||
static const char* const options[][2] = {
|
static const char* const options[][2] = {
|
||||||
@@ -1421,10 +1557,9 @@ static void test_parse_args_info_name_and_help() {
|
|||||||
config_delete(cfg);
|
config_delete(cfg);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* rsync 3.4.1's full --info/--debug vocabulary parses. The info categories
|
/* rsync 3.4.1's full --info/--debug vocabulary parses. The categories with a
|
||||||
* with a FastSync event set their flag; the remaining rsync-only categories
|
* FastSync event set their flag; the remaining rsync-only categories
|
||||||
* (backup/mount/symsafe/syms) parse but stay silent. Every --debug category
|
* (backup/symsafe/syms, acl/bind/chdir/...) parse but stay silent. */
|
||||||
* listed here is FastSync-silent, so debug_level stays 0. */
|
|
||||||
static void test_parse_args_rsync_flag_vocabulary_accepted() {
|
static void test_parse_args_rsync_flag_vocabulary_accepted() {
|
||||||
Config* cfg = config_create();
|
Config* cfg = config_create();
|
||||||
char* argv[] = {"fastsync", "--info=backup,del,flist,mount,nonreg,progress,remove,symsafe,syms",
|
char* argv[] = {"fastsync", "--info=backup,del,flist,mount,nonreg,progress,remove,symsafe,syms",
|
||||||
@@ -1436,9 +1571,10 @@ static void test_parse_args_rsync_flag_vocabulary_accepted() {
|
|||||||
int positional_count = 0;
|
int positional_count = 0;
|
||||||
|
|
||||||
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
|
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
|
||||||
EXPECT_EQ_INT(cfg->info_level, LOG_INFO_DEL | LOG_INFO_FLIST | LOG_INFO_NONREG |
|
EXPECT_EQ_INT(cfg->info_level, LOG_INFO_DEL | LOG_INFO_FLIST | LOG_INFO_MOUNT | LOG_INFO_NONREG |
|
||||||
LOG_INFO_PROGRESS | LOG_INFO_REMOVE);
|
LOG_INFO_PROGRESS | LOG_INFO_REMOVE);
|
||||||
EXPECT_EQ_INT(cfg->debug_level, 0);
|
EXPECT_EQ_INT(cfg->debug_level, LOG_DEBUG_DEL | LOG_DEBUG_FLIST | LOG_DEBUG_HASH |
|
||||||
|
LOG_DEBUG_RECV | LOG_DEBUG_FILTER | LOG_DEBUG_SEND);
|
||||||
config_delete(cfg);
|
config_delete(cfg);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -4768,6 +4904,7 @@ void test_client_cli() {
|
|||||||
test_validate_config_credentials_require_tls_or_loopback();
|
test_validate_config_credentials_require_tls_or_loopback();
|
||||||
test_validate_config_delta_sendfile_constraints();
|
test_validate_config_delta_sendfile_constraints();
|
||||||
test_validate_config_unified_invariants();
|
test_validate_config_unified_invariants();
|
||||||
|
test_validate_config_filter_rule_limit();
|
||||||
test_cli_help();
|
test_cli_help();
|
||||||
test_cli_archive_flags();
|
test_cli_archive_flags();
|
||||||
test_cli_dry_run();
|
test_cli_dry_run();
|
||||||
@@ -4783,6 +4920,7 @@ void test_client_cli() {
|
|||||||
test_parse_args_valid_port();
|
test_parse_args_valid_port();
|
||||||
test_parse_args_size_only();
|
test_parse_args_size_only();
|
||||||
test_parse_args_ignore_existing();
|
test_parse_args_ignore_existing();
|
||||||
|
test_parse_args_partial_dir_implies_partial();
|
||||||
test_parse_args_executability();
|
test_parse_args_executability();
|
||||||
test_parse_args_chmod();
|
test_parse_args_chmod();
|
||||||
test_parse_args_numeric_chmod();
|
test_parse_args_numeric_chmod();
|
||||||
@@ -4818,7 +4956,8 @@ void test_client_cli() {
|
|||||||
test_parse_args_delete_default_timing_and_commit();
|
test_parse_args_delete_default_timing_and_commit();
|
||||||
test_parse_args_delete_timing_conflict_rejected();
|
test_parse_args_delete_timing_conflict_rejected();
|
||||||
test_parse_args_delete_timing_without_delete_rejected();
|
test_parse_args_delete_timing_without_delete_rejected();
|
||||||
test_parse_args_rejects_unimplemented_options();
|
test_parse_args_rejects_unknown_options();
|
||||||
|
test_parse_args_accepts_implemented_metadata_options();
|
||||||
test_parse_args_quiet();
|
test_parse_args_quiet();
|
||||||
test_parse_args_human_readable();
|
test_parse_args_human_readable();
|
||||||
test_parse_args_hard_links();
|
test_parse_args_hard_links();
|
||||||
|
|||||||
@@ -3,7 +3,9 @@
|
|||||||
#include "compression.h"
|
#include "compression.h"
|
||||||
#include "data.h"
|
#include "data.h"
|
||||||
#include "file.h"
|
#include "file.h"
|
||||||
|
#include "protocol.h"
|
||||||
#include "utils.h"
|
#include "utils.h"
|
||||||
|
#include <stdint.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
#include <sys/stat.h>
|
#include <sys/stat.h>
|
||||||
@@ -216,6 +218,67 @@ static void test_data_decompress_unknown_size_frame() {
|
|||||||
data_destroy(frame);
|
data_destroy(frame);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* The receiver advertises MAX_RECEIVE_WHOLE_FILE_SIZE (256 MiB) and the sender
|
||||||
|
* compresses whole files, so the decompressor's internal ceiling must match that
|
||||||
|
* protocol bound. A 130 MiB payload -- above the old 100 MiB ceiling but below
|
||||||
|
* the protocol bound -- must round-trip through
|
||||||
|
* data_decompress_limited(..., MAX_RECEIVE_WHOLE_FILE_SIZE). The payload is all
|
||||||
|
* zeros so it compresses to a tiny frame while still declaring its full size. */
|
||||||
|
static void test_data_decompress_limited_whole_file_ceiling() {
|
||||||
|
const size_t size = 130ULL * 1024 * 1024;
|
||||||
|
Data* input = data_create_empty(size);
|
||||||
|
EXPECT_NOT_NULL(input);
|
||||||
|
memset(input->data, 0, size);
|
||||||
|
input->size = size;
|
||||||
|
|
||||||
|
/* Threaded zstd stores the content size in the frame header, as the sender
|
||||||
|
* does for whole files, so the decompressor sees the exact declared size. */
|
||||||
|
Data* compressed = data_compress_codec(input, COMPRESSION_ALGO_ZSTD, 1, 2);
|
||||||
|
EXPECT_NOT_NULL(compressed);
|
||||||
|
/* Premise: the declared size sits between the old 100 MiB cap and the
|
||||||
|
* protocol whole-file bound -- exactly the range that used to be rejected. */
|
||||||
|
unsigned long long declared =
|
||||||
|
ZSTD_getFrameContentSize((uint8_t*)compressed->data + 1, compressed->size - 1);
|
||||||
|
EXPECT_TRUE(declared > (100ULL * 1024 * 1024));
|
||||||
|
EXPECT_TRUE(declared <= MAX_RECEIVE_WHOLE_FILE_SIZE);
|
||||||
|
|
||||||
|
Data* out = data_decompress_limited(compressed, MAX_RECEIVE_WHOLE_FILE_SIZE);
|
||||||
|
EXPECT_NOT_NULL(out);
|
||||||
|
EXPECT_EQ_INT((int)out->size, (int)size);
|
||||||
|
EXPECT_EQ_INT(memcmp(out->data, input->data, size), 0);
|
||||||
|
|
||||||
|
data_destroy(out);
|
||||||
|
data_destroy(compressed);
|
||||||
|
data_destroy(input);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* A frame declaring an uncompressed size above the hard ceiling is still
|
||||||
|
* rejected before any allocation, even when the caller passes a limit higher
|
||||||
|
* than the protocol bound. The 13-byte header is a valid zstd frame header with
|
||||||
|
* an 8-byte content size and no blocks; rejection happens at the size check. */
|
||||||
|
static void test_data_decompress_limited_rejects_over_ceiling() {
|
||||||
|
const uint64_t declared = MAX_RECEIVE_WHOLE_FILE_SIZE + 1;
|
||||||
|
Data* frame = data_create_empty(1 + 13);
|
||||||
|
EXPECT_NOT_NULL(frame);
|
||||||
|
uint8_t* p = (uint8_t*)frame->data;
|
||||||
|
p[0] = (uint8_t)COMPRESSION_ALGO_ZSTD;
|
||||||
|
p[1] = 0x28; /* zstd magic number, little-endian */
|
||||||
|
p[2] = 0xB5;
|
||||||
|
p[3] = 0x2F;
|
||||||
|
p[4] = 0xFD;
|
||||||
|
p[5] = 0xE0; /* Frame_Header_Descriptor: 8-byte content size, single segment */
|
||||||
|
for (int i = 0; i < 8; i++)
|
||||||
|
p[6 + i] = (uint8_t)((declared >> (8 * i)) & 0xff);
|
||||||
|
frame->size = 1 + 13;
|
||||||
|
/* Guard the premise: zstd reads back exactly the declared over-ceiling size. */
|
||||||
|
EXPECT_EQ_INT((int)ZSTD_getFrameContentSize(p + 1, frame->size - 1), (int)declared);
|
||||||
|
|
||||||
|
EXPECT_NULL(data_decompress_limited(frame, MAX_RECEIVE_WHOLE_FILE_SIZE * 2));
|
||||||
|
EXPECT_NULL(data_decompress_limited(frame, MAX_RECEIVE_WHOLE_FILE_SIZE));
|
||||||
|
|
||||||
|
data_destroy(frame);
|
||||||
|
}
|
||||||
|
|
||||||
typedef struct {
|
typedef struct {
|
||||||
int id;
|
int id;
|
||||||
int iterations;
|
int iterations;
|
||||||
@@ -467,6 +530,8 @@ void test_compression() {
|
|||||||
test_data_compress_decompress_roundtrip();
|
test_data_compress_decompress_roundtrip();
|
||||||
test_data_compress_decompress_large();
|
test_data_compress_decompress_large();
|
||||||
test_data_decompress_unknown_size_frame();
|
test_data_decompress_unknown_size_frame();
|
||||||
|
test_data_decompress_limited_whole_file_ceiling();
|
||||||
|
test_data_decompress_limited_rejects_over_ceiling();
|
||||||
test_data_decompress_truncated_frame_fails();
|
test_data_decompress_truncated_frame_fails();
|
||||||
test_skip_compress_suffix_matching();
|
test_skip_compress_suffix_matching();
|
||||||
test_data_compress_with_threads_roundtrip();
|
test_data_compress_with_threads_roundtrip();
|
||||||
|
|||||||
@@ -3,12 +3,14 @@
|
|||||||
#include "test_utils.h"
|
#include "test_utils.h"
|
||||||
#include "utils.h"
|
#include "utils.h"
|
||||||
#include <errno.h>
|
#include <errno.h>
|
||||||
|
#include <fcntl.h>
|
||||||
#include <glob.h>
|
#include <glob.h>
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
#include <sys/stat.h>
|
#include <sys/stat.h>
|
||||||
#include <sys/types.h>
|
#include <sys/types.h>
|
||||||
|
#include <sys/wait.h>
|
||||||
#include <unistd.h>
|
#include <unistd.h>
|
||||||
|
|
||||||
/* Known-answer vector, independently recomputed with Python
|
/* Known-answer vector, independently recomputed with Python
|
||||||
@@ -719,6 +721,238 @@ static void test_credentials_read_secret_file_bad() {
|
|||||||
EXPECT_EQ_INT(credentials_read_secret_file(missing, NULL, NULL, err, sizeof(err)), -1);
|
EXPECT_EQ_INT(credentials_read_secret_file(missing, NULL, NULL, err, sizeof(err)), -1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* A symlink planted at a password-file path is refused (O_NOFOLLOW) instead of
|
||||||
|
* being followed before the owner/mode gate, even when it resolves to a valid
|
||||||
|
* owner-only regular file. */
|
||||||
|
static void test_credentials_read_secret_file_symlink_rejected() {
|
||||||
|
char err[512];
|
||||||
|
char* target = make_tmp_file("alice:correct horse battery staple\n");
|
||||||
|
EXPECT_NOT_NULL(target);
|
||||||
|
|
||||||
|
char link[256];
|
||||||
|
snprintf(link, sizeof(link), "/tmp/fs_cred_pwlink_%d_%d", (int)getpid(), g_file_counter++);
|
||||||
|
unlink(link);
|
||||||
|
EXPECT_EQ_INT(symlink(target, link), 0);
|
||||||
|
|
||||||
|
char* user = (char*)1;
|
||||||
|
char* password = (char*)1;
|
||||||
|
EXPECT_EQ_INT(credentials_read_secret_file(link, &user, &password, err, sizeof(err)), -1);
|
||||||
|
EXPECT_NULL(user);
|
||||||
|
EXPECT_NULL(password);
|
||||||
|
EXPECT_TRUE(err[0] != '\0');
|
||||||
|
|
||||||
|
unlink(link); /* remove the symlink itself, not its target */
|
||||||
|
rm_temp(target);
|
||||||
|
free(target);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* A named FIFO with no writer must not hang in fgets (O_NONBLOCK): the read
|
||||||
|
* fails cleanly with "no user:password line" instead of blocking forever. */
|
||||||
|
static void test_credentials_read_secret_file_fifo_no_hang() {
|
||||||
|
char err[512];
|
||||||
|
char fifo[256];
|
||||||
|
snprintf(fifo, sizeof(fifo), "/tmp/fs_cred_pwfifo_%d_%d", (int)getpid(), g_file_counter++);
|
||||||
|
unlink(fifo);
|
||||||
|
EXPECT_EQ_INT(mkfifo(fifo, 0600), 0);
|
||||||
|
|
||||||
|
char* user = (char*)1;
|
||||||
|
char* password = (char*)1;
|
||||||
|
EXPECT_EQ_INT(credentials_read_secret_file(fifo, &user, &password, err, sizeof(err)), -1);
|
||||||
|
EXPECT_NULL(user);
|
||||||
|
EXPECT_NULL(password);
|
||||||
|
EXPECT_TRUE(strstr(err, "no 'user:password'") != NULL || err[0] != '\0');
|
||||||
|
|
||||||
|
unlink(fifo);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Write `s` fully to `fd`, retrying EINTR. */
|
||||||
|
static void write_all_fd(int fd, const char* s) {
|
||||||
|
size_t total = strlen(s);
|
||||||
|
size_t off = 0;
|
||||||
|
while (off < total) {
|
||||||
|
ssize_t w = write(fd, s + off, total - off);
|
||||||
|
if (w < 0) {
|
||||||
|
if (errno == EINTR)
|
||||||
|
continue;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
off += (size_t)w;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Deterministically model a slow process substitution (`--password-file
|
||||||
|
* <(sleep N; ...)`): attach a writer to the FIFO (so the reader sees EAGAIN --
|
||||||
|
* the empty/no-writer FIFO instead yields an immediate EOF), have it sleep
|
||||||
|
* `delay_ms`, then write `first` and, after another `delay_ms`, `second` (NULL
|
||||||
|
* for a single write). Splitting across the delay exercises reassembly of a
|
||||||
|
* line delivered by several write()s.
|
||||||
|
*
|
||||||
|
* The parent keeps a spare read end open for the lifetime of the test so the
|
||||||
|
* writer always has a reader; the caller must close(*hold_out), waitpid() the
|
||||||
|
* returned pid and unlink the FIFO. Returns the child pid, or -1 on setup
|
||||||
|
* failure. */
|
||||||
|
static pid_t fifo_writer_sleep_then_write(const char* fifo, const char* first, unsigned delay_ms,
|
||||||
|
const char* second, int* hold_out) {
|
||||||
|
int sync[2];
|
||||||
|
if (pipe(sync) != 0)
|
||||||
|
return -1;
|
||||||
|
pid_t pid = fork();
|
||||||
|
if (pid < 0) {
|
||||||
|
close(sync[0]);
|
||||||
|
close(sync[1]);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
if (pid == 0) {
|
||||||
|
close(sync[0]);
|
||||||
|
int wfd = open(fifo, O_WRONLY | O_CLOEXEC);
|
||||||
|
char ready = wfd >= 0 ? 1 : 0;
|
||||||
|
if (write(sync[1], &ready, 1) != 1)
|
||||||
|
_exit(1);
|
||||||
|
close(sync[1]);
|
||||||
|
if (wfd >= 0) {
|
||||||
|
usleep(delay_ms * 1000);
|
||||||
|
write_all_fd(wfd, first);
|
||||||
|
if (second) {
|
||||||
|
usleep(delay_ms * 1000);
|
||||||
|
write_all_fd(wfd, second);
|
||||||
|
}
|
||||||
|
close(wfd);
|
||||||
|
}
|
||||||
|
_exit(0);
|
||||||
|
}
|
||||||
|
close(sync[1]);
|
||||||
|
int hold = open(fifo, O_RDONLY | O_NONBLOCK | O_CLOEXEC);
|
||||||
|
char ready = 0;
|
||||||
|
ssize_t got = read(sync[0], &ready, 1);
|
||||||
|
close(sync[0]);
|
||||||
|
if (got != 1 || ready != 1) {
|
||||||
|
if (hold >= 0)
|
||||||
|
close(hold);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
*hold_out = hold;
|
||||||
|
return pid;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* A FIFO writer that produces its data after a short delay must be read
|
||||||
|
* successfully (the regression: O_NONBLOCK made fgets fail with EAGAIN before
|
||||||
|
* the writer ran). */
|
||||||
|
static void test_credentials_read_secret_file_fifo_delayed_writer() {
|
||||||
|
char err[512];
|
||||||
|
char fifo[256];
|
||||||
|
snprintf(fifo, sizeof(fifo), "/tmp/fs_cred_pwfifo_slow_%d_%d", (int)getpid(), g_file_counter++);
|
||||||
|
unlink(fifo);
|
||||||
|
EXPECT_EQ_INT(mkfifo(fifo, 0600), 0);
|
||||||
|
|
||||||
|
int hold = -1;
|
||||||
|
pid_t writer =
|
||||||
|
fifo_writer_sleep_then_write(fifo, "alice:correct horse battery staple\n", 250, NULL, &hold);
|
||||||
|
EXPECT_TRUE(writer > 0);
|
||||||
|
|
||||||
|
char* user = NULL;
|
||||||
|
char* password = NULL;
|
||||||
|
EXPECT_EQ_INT(credentials_read_secret_file(fifo, &user, &password, err, sizeof(err)), 0);
|
||||||
|
EXPECT_EQ_STR(user, "alice");
|
||||||
|
EXPECT_EQ_STR(password, "correct horse battery staple");
|
||||||
|
free(user);
|
||||||
|
free(password);
|
||||||
|
|
||||||
|
int status = 0;
|
||||||
|
waitpid(writer, &status, 0);
|
||||||
|
if (hold >= 0)
|
||||||
|
close(hold);
|
||||||
|
unlink(fifo);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* The same, but the line is written in two chunks separated by the delay: the
|
||||||
|
* reader must reassemble one line rather than parse the first chunk as an
|
||||||
|
* empty-password entry. */
|
||||||
|
static void test_credentials_read_secret_file_fifo_split_write() {
|
||||||
|
char err[512];
|
||||||
|
char fifo[256];
|
||||||
|
snprintf(fifo, sizeof(fifo), "/tmp/fs_cred_pwfifo_split_%d_%d", (int)getpid(), g_file_counter++);
|
||||||
|
unlink(fifo);
|
||||||
|
EXPECT_EQ_INT(mkfifo(fifo, 0600), 0);
|
||||||
|
|
||||||
|
int hold = -1;
|
||||||
|
pid_t writer =
|
||||||
|
fifo_writer_sleep_then_write(fifo, "alice:correct horse", 200, " battery staple\n", &hold);
|
||||||
|
EXPECT_TRUE(writer > 0);
|
||||||
|
|
||||||
|
char* user = NULL;
|
||||||
|
char* password = NULL;
|
||||||
|
EXPECT_EQ_INT(credentials_read_secret_file(fifo, &user, &password, err, sizeof(err)), 0);
|
||||||
|
EXPECT_EQ_STR(user, "alice");
|
||||||
|
EXPECT_EQ_STR(password, "correct horse battery staple");
|
||||||
|
free(user);
|
||||||
|
free(password);
|
||||||
|
|
||||||
|
int status = 0;
|
||||||
|
waitpid(writer, &status, 0);
|
||||||
|
if (hold >= 0)
|
||||||
|
close(hold);
|
||||||
|
unlink(fifo);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* The server-side store loader (--password-file / --early-input) must also
|
||||||
|
* accept a FIFO whose writer appears after a delay. */
|
||||||
|
static void test_credentials_store_fifo_delayed_writer() {
|
||||||
|
char err[512];
|
||||||
|
char fifo[256];
|
||||||
|
snprintf(fifo, sizeof(fifo), "/tmp/fs_cred_storefifo_%d_%d", (int)getpid(), g_file_counter++);
|
||||||
|
unlink(fifo);
|
||||||
|
EXPECT_EQ_INT(mkfifo(fifo, 0600), 0);
|
||||||
|
|
||||||
|
int hold = -1;
|
||||||
|
pid_t writer = fifo_writer_sleep_then_write(fifo, KAT_STORE_LINE "\n", 250, NULL, &hold);
|
||||||
|
EXPECT_TRUE(writer > 0);
|
||||||
|
|
||||||
|
CredentialStore* store = credentials_load(fifo, NULL, err, sizeof(err));
|
||||||
|
EXPECT_NOT_NULL(store);
|
||||||
|
EXPECT_EQ_INT(credentials_store_size(store), 1);
|
||||||
|
credentials_free(store);
|
||||||
|
|
||||||
|
int status = 0;
|
||||||
|
waitpid(writer, &status, 0);
|
||||||
|
if (hold >= 0)
|
||||||
|
close(hold);
|
||||||
|
rm_temp(fifo);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* fd-backed store paths (bash process substitution `<(...)`, i.e. /dev/fd/N and
|
||||||
|
* /proc/self/fd/N) are symlinks, so the ordinary O_NOFOLLOW rule would reject
|
||||||
|
* them with ELOOP. They name the calling process's own descriptors, so they
|
||||||
|
* are exempt: opening one that points at an owner-only regular file is
|
||||||
|
* accepted, while a symlink at a NORMAL path is still rejected
|
||||||
|
* (test_credentials_read_secret_file_symlink_rejected). */
|
||||||
|
static void test_credentials_read_secret_file_fd_backed_accepted() {
|
||||||
|
char err[512];
|
||||||
|
char* path = make_tmp_file("alice:correct horse battery staple\n");
|
||||||
|
EXPECT_NOT_NULL(path);
|
||||||
|
|
||||||
|
int fd = open(path, O_RDONLY | O_CLOEXEC);
|
||||||
|
EXPECT_TRUE(fd >= 0);
|
||||||
|
|
||||||
|
const char* prefixes[] = {"/proc/self/fd/", "/dev/fd/"};
|
||||||
|
for (size_t i = 0; i < sizeof(prefixes) / sizeof(prefixes[0]); i++) {
|
||||||
|
if (i == 1 && access("/dev/fd", F_OK) != 0)
|
||||||
|
continue; /* /dev/fd is not present on every system */
|
||||||
|
char fd_path[64];
|
||||||
|
snprintf(fd_path, sizeof(fd_path), "%s%d", prefixes[i], fd);
|
||||||
|
char* user = (char*)1;
|
||||||
|
char* password = (char*)1;
|
||||||
|
EXPECT_EQ_INT(credentials_read_secret_file(fd_path, &user, &password, err, sizeof(err)), 0);
|
||||||
|
EXPECT_EQ_STR(user, "alice");
|
||||||
|
EXPECT_EQ_STR(password, "correct horse battery staple");
|
||||||
|
free(user);
|
||||||
|
free(password);
|
||||||
|
}
|
||||||
|
|
||||||
|
close(fd);
|
||||||
|
rm_temp(path);
|
||||||
|
free(path);
|
||||||
|
}
|
||||||
|
|
||||||
static void test_credentials_hash_file() {
|
static void test_credentials_hash_file() {
|
||||||
char* plaintext = make_tmp_file("# comment\n\n alice :" KAT_PASSWORD "\nbob:bob-s3cret\n");
|
char* plaintext = make_tmp_file("# comment\n\n alice :" KAT_PASSWORD "\nbob:bob-s3cret\n");
|
||||||
EXPECT_NOT_NULL(plaintext);
|
EXPECT_NOT_NULL(plaintext);
|
||||||
@@ -1103,6 +1337,12 @@ void test_credentials(void) {
|
|||||||
test_credentials_early_input_merge();
|
test_credentials_early_input_merge();
|
||||||
test_credentials_read_secret_file();
|
test_credentials_read_secret_file();
|
||||||
test_credentials_read_secret_file_bad();
|
test_credentials_read_secret_file_bad();
|
||||||
|
test_credentials_read_secret_file_symlink_rejected();
|
||||||
|
test_credentials_read_secret_file_fifo_no_hang();
|
||||||
|
test_credentials_read_secret_file_fifo_delayed_writer();
|
||||||
|
test_credentials_read_secret_file_fifo_split_write();
|
||||||
|
test_credentials_store_fifo_delayed_writer();
|
||||||
|
test_credentials_read_secret_file_fd_backed_accepted();
|
||||||
test_credentials_hash_file();
|
test_credentials_hash_file();
|
||||||
test_credentials_rejects_group_or_other_accessible();
|
test_credentials_rejects_group_or_other_accessible();
|
||||||
test_credentials_dummy_key_persisted();
|
test_credentials_dummy_key_persisted();
|
||||||
|
|||||||
+109
-16
@@ -377,6 +377,98 @@ static void test_file_save_to_disk_temp_dir_confined() {
|
|||||||
rmdir(outside);
|
rmdir(outside);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* A client-planted symlink under the receive root must never redirect the
|
||||||
|
* --temp-dir scratch directory outside the authorized root: the REAL path of
|
||||||
|
* the opened dir is checked. An in-root symlink (the EXDEV cross-filesystem
|
||||||
|
* case) must still be accepted. */
|
||||||
|
static void test_file_open_temp_dir_symlink_confinement() {
|
||||||
|
const char* root = "test_tempdir_link_root";
|
||||||
|
const char* outside = "test_tempdir_link_outside";
|
||||||
|
char root_abs[PATH_MAX];
|
||||||
|
char outside_abs[PATH_MAX];
|
||||||
|
unlink("test_tempdir_link_root/escape");
|
||||||
|
unlink("test_tempdir_link_root/inside_link");
|
||||||
|
rmdir("test_tempdir_link_root/scratch");
|
||||||
|
rmdir(root);
|
||||||
|
rmdir(outside);
|
||||||
|
|
||||||
|
int mkdir_root_ret = mkdir(root, 0755);
|
||||||
|
int mkdir_outside_ret = mkdir(outside, 0755);
|
||||||
|
bool root_resolved = realpath(root, root_abs) != NULL;
|
||||||
|
bool outside_resolved = realpath(outside, outside_abs) != NULL;
|
||||||
|
int root_fd = root_resolved ? open(root_abs, O_RDONLY | O_DIRECTORY | O_CLOEXEC) : -1;
|
||||||
|
|
||||||
|
bool root_set = false;
|
||||||
|
bool scratch_ok = false;
|
||||||
|
int scratch_fd = -1;
|
||||||
|
bool escape_staged = false;
|
||||||
|
int escape_fd = 0;
|
||||||
|
bool inside_staged = false;
|
||||||
|
int inside_fd = -1;
|
||||||
|
char* scratch = NULL;
|
||||||
|
char* escape = NULL;
|
||||||
|
char* inside_link = NULL;
|
||||||
|
|
||||||
|
/* Only touch the global authorized root and the scratch fixtures once the
|
||||||
|
setup succeeded; the teardown below always runs regardless. */
|
||||||
|
if (root_fd >= 0 && outside_resolved) {
|
||||||
|
root_set = utils_set_authorized_root(root_fd, root_abs);
|
||||||
|
|
||||||
|
/* An existing in-root scratch dir opens normally. */
|
||||||
|
scratch = path_cat(root_abs, "scratch");
|
||||||
|
if (scratch && mkdir(scratch, 0755) == 0) {
|
||||||
|
scratch_ok = true;
|
||||||
|
scratch_fd = file_open_temp_dir(scratch);
|
||||||
|
if (scratch_fd >= 0)
|
||||||
|
close(scratch_fd);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* A symlink whose target is outside the root is refused. */
|
||||||
|
escape = path_cat(root_abs, "escape");
|
||||||
|
if (escape && symlink(outside_abs, escape) == 0) {
|
||||||
|
escape_staged = true;
|
||||||
|
escape_fd = file_open_temp_dir(escape);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* A symlink that stays inside the root is accepted (EXDEV fallback). */
|
||||||
|
inside_link = path_cat(root_abs, "inside_link");
|
||||||
|
if (inside_link && scratch && symlink(scratch, inside_link) == 0) {
|
||||||
|
inside_staged = true;
|
||||||
|
inside_fd = file_open_temp_dir(inside_link);
|
||||||
|
if (inside_fd >= 0)
|
||||||
|
close(inside_fd);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Release the global authorized root and all fixtures BEFORE asserting:
|
||||||
|
EXPECT_* returns early on failure, so a failed assertion must not be able
|
||||||
|
to leave the process state poisoned or leak root_fd. */
|
||||||
|
utils_set_authorized_root(-1, NULL);
|
||||||
|
if (root_fd >= 0)
|
||||||
|
close(root_fd);
|
||||||
|
free(inside_link);
|
||||||
|
free(escape);
|
||||||
|
free(scratch);
|
||||||
|
unlink("test_tempdir_link_root/escape");
|
||||||
|
unlink("test_tempdir_link_root/inside_link");
|
||||||
|
rmdir("test_tempdir_link_root/scratch");
|
||||||
|
rmdir(root);
|
||||||
|
rmdir(outside);
|
||||||
|
|
||||||
|
EXPECT_EQ_INT(mkdir_root_ret, 0);
|
||||||
|
EXPECT_EQ_INT(mkdir_outside_ret, 0);
|
||||||
|
EXPECT_TRUE(root_resolved);
|
||||||
|
EXPECT_TRUE(outside_resolved);
|
||||||
|
EXPECT_TRUE(root_fd >= 0);
|
||||||
|
EXPECT_TRUE(root_set);
|
||||||
|
EXPECT_TRUE(scratch_ok);
|
||||||
|
EXPECT_TRUE(scratch_fd >= 0);
|
||||||
|
EXPECT_TRUE(escape_staged);
|
||||||
|
EXPECT_EQ_INT(escape_fd, -1);
|
||||||
|
EXPECT_TRUE(inside_staged);
|
||||||
|
EXPECT_TRUE(inside_fd >= 0);
|
||||||
|
}
|
||||||
|
|
||||||
/* Issue #251: file_save_to_disk_full must distinguish receiver-side skips
|
/* Issue #251: file_save_to_disk_full must distinguish receiver-side skips
|
||||||
(--existing/--ignore-existing/--update) from real writes so the sender can
|
(--existing/--ignore-existing/--update) from real writes so the sender can
|
||||||
decide whether --remove-source-files may unlink its source. */
|
decide whether --remove-source-files may unlink its source. */
|
||||||
@@ -1040,8 +1132,8 @@ static void test_atomic_no_perms_preserves_destination_mode() {
|
|||||||
|
|
||||||
/* No -p/-E: the pre-existing 0640 survives the atomic overwrite. */
|
/* No -p/-E: the pre-existing 0640 survives the atomic overwrite. */
|
||||||
bool ok = file_to_disk_secure_attrs(path, "data", 4, false, false, false, &m,
|
bool ok = file_to_disk_secure_attrs(path, "data", 4, false, false, false, &m,
|
||||||
(FileAttrPolicy){false, false, false, false}, false, false,
|
(FileAttrPolicy){false, false, false, false, true}, false,
|
||||||
false, NULL, false, false, NULL);
|
false, false, NULL, false, false, NULL);
|
||||||
EXPECT_TRUE(ok);
|
EXPECT_TRUE(ok);
|
||||||
struct stat st;
|
struct stat st;
|
||||||
EXPECT_EQ_INT(stat(path, &st), 0);
|
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||||
@@ -1049,8 +1141,8 @@ static void test_atomic_no_perms_preserves_destination_mode() {
|
|||||||
|
|
||||||
/* -p: the source mode wins. */
|
/* -p: the source mode wins. */
|
||||||
ok = file_to_disk_secure_attrs(path, "data2", 5, false, false, false, &m,
|
ok = file_to_disk_secure_attrs(path, "data2", 5, false, false, false, &m,
|
||||||
(FileAttrPolicy){true, true, false, false}, false, false, false,
|
(FileAttrPolicy){true, true, false, false, true}, false, false,
|
||||||
NULL, false, false, NULL);
|
false, NULL, false, false, NULL);
|
||||||
EXPECT_TRUE(ok);
|
EXPECT_TRUE(ok);
|
||||||
EXPECT_EQ_INT(stat(path, &st), 0);
|
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||||
EXPECT_EQ_INT((int)(st.st_mode & 0777), 0755);
|
EXPECT_EQ_INT((int)(st.st_mode & 0777), 0755);
|
||||||
@@ -1060,8 +1152,8 @@ static void test_atomic_no_perms_preserves_destination_mode() {
|
|||||||
source 0755 gives 0750, not 0751 and not the scratch 0711. */
|
source 0755 gives 0750, not 0751 and not the scratch 0711. */
|
||||||
EXPECT_EQ_INT(chmod(path, 0640), 0);
|
EXPECT_EQ_INT(chmod(path, 0640), 0);
|
||||||
ok = file_to_disk_secure_attrs(path, "data3", 6, false, false, false, &m,
|
ok = file_to_disk_secure_attrs(path, "data3", 6, false, false, false, &m,
|
||||||
(FileAttrPolicy){false, false, false, true}, false, false, false,
|
(FileAttrPolicy){false, false, false, true, true}, false, false,
|
||||||
NULL, false, false, NULL);
|
false, NULL, false, false, NULL);
|
||||||
EXPECT_TRUE(ok);
|
EXPECT_TRUE(ok);
|
||||||
EXPECT_EQ_INT(stat(path, &st), 0);
|
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||||
EXPECT_EQ_INT((int)(st.st_mode & 0777), 0750);
|
EXPECT_EQ_INT((int)(st.st_mode & 0777), 0750);
|
||||||
@@ -1073,8 +1165,8 @@ static void test_atomic_no_perms_preserves_destination_mode() {
|
|||||||
const char* fresh = "test_attr_split_fresh.txt";
|
const char* fresh = "test_attr_split_fresh.txt";
|
||||||
unlink(fresh);
|
unlink(fresh);
|
||||||
ok = file_to_disk_secure_attrs(fresh, "data", 4, false, false, false, &m,
|
ok = file_to_disk_secure_attrs(fresh, "data", 4, false, false, false, &m,
|
||||||
(FileAttrPolicy){false, false, false, false}, false, false, false,
|
(FileAttrPolicy){false, false, false, false, true}, false, false,
|
||||||
NULL, false, false, NULL);
|
false, NULL, false, false, NULL);
|
||||||
EXPECT_TRUE(ok);
|
EXPECT_TRUE(ok);
|
||||||
EXPECT_EQ_INT(stat(fresh, &st), 0);
|
EXPECT_EQ_INT(stat(fresh, &st), 0);
|
||||||
EXPECT_EQ_INT((int)(st.st_mode & 0777), (int)(m.mode & 0777 & ~(mode_t)file_process_umask()));
|
EXPECT_EQ_INT((int)(st.st_mode & 0777), (int)(m.mode & 0777 & ~(mode_t)file_process_umask()));
|
||||||
@@ -1083,8 +1175,8 @@ static void test_atomic_no_perms_preserves_destination_mode() {
|
|||||||
/* Without any metadata the historical fixed 0644 default still applies. */
|
/* Without any metadata the historical fixed 0644 default still applies. */
|
||||||
unlink(fresh);
|
unlink(fresh);
|
||||||
ok = file_to_disk_secure_attrs(fresh, "data", 4, false, false, false, NULL,
|
ok = file_to_disk_secure_attrs(fresh, "data", 4, false, false, false, NULL,
|
||||||
(FileAttrPolicy){false, false, false, false}, false, false, false,
|
(FileAttrPolicy){false, false, false, false, true}, false, false,
|
||||||
NULL, false, false, NULL);
|
false, NULL, false, false, NULL);
|
||||||
EXPECT_TRUE(ok);
|
EXPECT_TRUE(ok);
|
||||||
EXPECT_EQ_INT(stat(fresh, &st), 0);
|
EXPECT_EQ_INT(stat(fresh, &st), 0);
|
||||||
EXPECT_EQ_INT((int)(st.st_mode & 0777), 0644);
|
EXPECT_EQ_INT((int)(st.st_mode & 0777), 0644);
|
||||||
@@ -1104,8 +1196,8 @@ static void test_new_file_mode_honors_source_and_umask() {
|
|||||||
m.gid = getegid();
|
m.gid = getegid();
|
||||||
|
|
||||||
bool ok = file_to_disk_secure_attrs(path, "x", 1, false, false, false, &m,
|
bool ok = file_to_disk_secure_attrs(path, "x", 1, false, false, false, &m,
|
||||||
(FileAttrPolicy){false, false, false, false}, false, false,
|
(FileAttrPolicy){false, false, false, false, true}, false,
|
||||||
false, NULL, false, false, NULL);
|
false, false, NULL, false, false, NULL);
|
||||||
EXPECT_TRUE(ok);
|
EXPECT_TRUE(ok);
|
||||||
struct stat st;
|
struct stat st;
|
||||||
EXPECT_EQ_INT(stat(path, &st), 0);
|
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||||
@@ -1663,8 +1755,8 @@ static void test_file_write_to_disk_partial_retention() {
|
|||||||
m.atime_valid = false;
|
m.atime_valid = false;
|
||||||
m.crtime_valid = false;
|
m.crtime_valid = false;
|
||||||
bool ok = file_to_disk_secure_attrs(path, content, strlen(content), false, false, true, &m,
|
bool ok = file_to_disk_secure_attrs(path, content, strlen(content), false, false, true, &m,
|
||||||
(FileAttrPolicy){true, true, false, false}, false, false,
|
(FileAttrPolicy){true, true, false, false, true}, false,
|
||||||
false, NULL, false, true, NULL);
|
false, false, NULL, false, true, NULL);
|
||||||
EXPECT_FALSE(ok); /* the write itself succeeded, but metadata restore failed */
|
EXPECT_FALSE(ok); /* the write itself succeeded, but metadata restore failed */
|
||||||
/* Retained: the already-written temp now sits at the destination path. */
|
/* Retained: the already-written temp now sits at the destination path. */
|
||||||
int fd = open(path, O_RDONLY);
|
int fd = open(path, O_RDONLY);
|
||||||
@@ -1683,8 +1775,8 @@ static void test_file_write_to_disk_partial_retention() {
|
|||||||
|
|
||||||
/* Same failure with keep_partial=false: temp is unlinked, nothing retained. */
|
/* Same failure with keep_partial=false: temp is unlinked, nothing retained. */
|
||||||
ok = file_to_disk_secure_attrs(path, content, strlen(content), false, false, true, &m,
|
ok = file_to_disk_secure_attrs(path, content, strlen(content), false, false, true, &m,
|
||||||
(FileAttrPolicy){true, true, false, false}, false, false, false,
|
(FileAttrPolicy){true, true, false, false, true}, false, false,
|
||||||
NULL, false, false, NULL);
|
false, NULL, false, false, NULL);
|
||||||
EXPECT_FALSE(ok);
|
EXPECT_FALSE(ok);
|
||||||
EXPECT_TRUE(access(path, F_OK) == -1);
|
EXPECT_TRUE(access(path, F_OK) == -1);
|
||||||
}
|
}
|
||||||
@@ -2264,6 +2356,7 @@ void test_file() {
|
|||||||
test_file_save_to_disk_ignore_existing_entry_types();
|
test_file_save_to_disk_ignore_existing_entry_types();
|
||||||
test_file_save_to_disk_partial_install();
|
test_file_save_to_disk_partial_install();
|
||||||
test_file_save_to_disk_temp_dir_confined();
|
test_file_save_to_disk_temp_dir_confined();
|
||||||
|
test_file_open_temp_dir_symlink_confinement();
|
||||||
test_file_save_to_disk_reports_skips();
|
test_file_save_to_disk_reports_skips();
|
||||||
test_file_write_to_disk_sparse_preserves_holes();
|
test_file_write_to_disk_sparse_preserves_holes();
|
||||||
test_file_write_to_disk_partial_retention();
|
test_file_write_to_disk_partial_retention();
|
||||||
|
|||||||
@@ -0,0 +1,294 @@
|
|||||||
|
#include "test_filter.h"
|
||||||
|
#include "filter.h"
|
||||||
|
#include "test_utils.h"
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <string.h>
|
||||||
|
#include <sys/stat.h>
|
||||||
|
#include <unistd.h>
|
||||||
|
|
||||||
|
/* Every `-f`/`--filter` rule string is validated through the list parser, so
|
||||||
|
* the list parser must reject the modifiers the standalone parser rejects
|
||||||
|
* rather than silently folding them into a pattern. */
|
||||||
|
|
||||||
|
static void test_filter_list_rejects_xattr_modifier() {
|
||||||
|
static const char* const rules[] = {
|
||||||
|
"-x user.foo", /* short exclude + x */
|
||||||
|
"exclude,x user.foo", /* long exclude + x */
|
||||||
|
"+x user.foo", /* include + x */
|
||||||
|
"hide,x *.tmp", /* hide + x */
|
||||||
|
"dir-merge,x .rules", /* x must not be dropped on dir-merge */
|
||||||
|
"merge,x /tmp/nonexistent" /* x must not be dropped on merge */
|
||||||
|
};
|
||||||
|
for (size_t i = 0; i < sizeof(rules) / sizeof(rules[0]); i++) {
|
||||||
|
FilterRuleList* list = filter_rule_list_create();
|
||||||
|
EXPECT_NOT_NULL(list);
|
||||||
|
char err[256] = "";
|
||||||
|
bool ok = filter_rule_list_parse_append(list, rules[i], NULL, NULL, err, sizeof(err));
|
||||||
|
EXPECT_FALSE(ok);
|
||||||
|
EXPECT_TRUE(strstr(err, "xattr") != NULL);
|
||||||
|
filter_rule_list_free(list);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* A bare "x" is not a rule at all: rejected as generic bad syntax. */
|
||||||
|
FilterRuleList* list = filter_rule_list_create();
|
||||||
|
EXPECT_NOT_NULL(list);
|
||||||
|
char err[256] = "";
|
||||||
|
EXPECT_FALSE(filter_rule_list_parse_append(list, "x user.foo", NULL, NULL, err, sizeof(err)));
|
||||||
|
EXPECT_TRUE(err[0] != '\0');
|
||||||
|
filter_rule_list_free(list);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_filter_list_rejects_unsupported_modifiers() {
|
||||||
|
/* The merge-file modifiers e/n/w/- are invalid on every non-merge rule; a
|
||||||
|
token made up solely of modifier characters is a modifier run, so it must
|
||||||
|
be rejected rather than folded into the pattern. */
|
||||||
|
static const char* const rules[] = {
|
||||||
|
"-e foo", /* e: merge-only in rsync */
|
||||||
|
"-n foo", /* n: merge-only in rsync */
|
||||||
|
"-w foo", /* w: merge-only in rsync */
|
||||||
|
"-new", /* pure modifier letters (n/e/w) */
|
||||||
|
"-press", /* pure modifier letters (p/r/e/s) */
|
||||||
|
"exclude,w foo", "exclude,e foo", "exclude,n foo",
|
||||||
|
"hide,w foo", "protect,n foo", "risk,e foo",
|
||||||
|
};
|
||||||
|
for (size_t i = 0; i < sizeof(rules) / sizeof(rules[0]); i++) {
|
||||||
|
FilterRuleList* list = filter_rule_list_create();
|
||||||
|
EXPECT_NOT_NULL(list);
|
||||||
|
char err[256] = "";
|
||||||
|
bool ok = filter_rule_list_parse_append(list, rules[i], NULL, NULL, err, sizeof(err));
|
||||||
|
EXPECT_FALSE(ok);
|
||||||
|
EXPECT_TRUE(strstr(err, "unsupported filter modifier") != NULL);
|
||||||
|
filter_rule_list_free(list);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* rsync accepts the merge-file modifiers e/n/w/- on merge and dir-merge rules.
|
||||||
|
* They must be consumed so they never leak into the merge filename. */
|
||||||
|
static void test_filter_list_accepts_merge_modifiers() {
|
||||||
|
char tmpl[] = "/tmp/fastsync_filter_mmod_XXXXXX";
|
||||||
|
EXPECT_TRUE(mkdtemp(tmpl) != NULL);
|
||||||
|
char path[512];
|
||||||
|
snprintf(path, sizeof(path), "%s/rules", tmpl);
|
||||||
|
FILE* fp = fopen(path, "w");
|
||||||
|
EXPECT_NOT_NULL(fp);
|
||||||
|
fputs("- *.tmp\n", fp);
|
||||||
|
fclose(fp);
|
||||||
|
|
||||||
|
/* merge with e/n/w/- consumes the modifiers and reads the right file. */
|
||||||
|
static const char* const fmts[] = {
|
||||||
|
"merge,e %s", "merge,n %s", "merge,w %s", "merge,- %s", ".e %s", ".- %s",
|
||||||
|
};
|
||||||
|
for (size_t i = 0; i < sizeof(fmts) / sizeof(fmts[0]); i++) {
|
||||||
|
FilterRuleList* list = filter_rule_list_create();
|
||||||
|
EXPECT_NOT_NULL(list);
|
||||||
|
char rule[600];
|
||||||
|
char err[256] = "";
|
||||||
|
snprintf(rule, sizeof(rule), fmts[i], path);
|
||||||
|
bool ok = filter_rule_list_parse_append(list, rule, NULL, NULL, err, sizeof(err));
|
||||||
|
if (!ok)
|
||||||
|
printf(" merge rule '%s' errored: %s\n", rule, err);
|
||||||
|
EXPECT_TRUE(ok);
|
||||||
|
EXPECT_EQ_INT(list->count, 1);
|
||||||
|
EXPECT_EQ_STR(list->items[0]->pattern, "*.tmp");
|
||||||
|
filter_rule_list_free(list);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* dir-merge with e/n/w/- registers the basename without the modifiers. */
|
||||||
|
static const struct {
|
||||||
|
const char* rule;
|
||||||
|
const char* want;
|
||||||
|
} drules[] = {
|
||||||
|
{"dir-merge,e .rules", ".rules"}, {"dir-merge,n .rules", ".rules"},
|
||||||
|
{"dir-merge,w .rules", ".rules"}, {"dir-merge,- .rules", ".rules"},
|
||||||
|
{":e .rules", ".rules"}, {":- .rules", ".rules"},
|
||||||
|
};
|
||||||
|
for (size_t i = 0; i < sizeof(drules) / sizeof(drules[0]); i++) {
|
||||||
|
FilterRuleList* list = filter_rule_list_create();
|
||||||
|
EXPECT_NOT_NULL(list);
|
||||||
|
char err[256] = "";
|
||||||
|
bool ok = filter_rule_list_parse_append(list, drules[i].rule, NULL, NULL, err, sizeof(err));
|
||||||
|
if (!ok)
|
||||||
|
printf(" dir-merge rule '%s' errored: %s\n", drules[i].rule, err);
|
||||||
|
EXPECT_TRUE(ok);
|
||||||
|
EXPECT_EQ_INT(list->dir_merge_count, 1);
|
||||||
|
EXPECT_EQ_STR(list->dir_merge_names[0], drules[i].want);
|
||||||
|
filter_rule_list_free(list);
|
||||||
|
}
|
||||||
|
|
||||||
|
unlink(path);
|
||||||
|
rmdir(tmpl);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_filter_list_accepts_supported_rules_and_modifiers() {
|
||||||
|
static const char* const rules[] = {
|
||||||
|
"- *.tmp", "+ /a.txt", "-s foo", "-r foo", "-p foo",
|
||||||
|
"-! *.o", "-/ foo", "hide *.tmp", "show *.txt", "protect *.bak",
|
||||||
|
"risk *.o", "dir-merge .rules", "-C",
|
||||||
|
};
|
||||||
|
for (size_t i = 0; i < sizeof(rules) / sizeof(rules[0]); i++) {
|
||||||
|
FilterRuleList* list = filter_rule_list_create();
|
||||||
|
EXPECT_NOT_NULL(list);
|
||||||
|
char err[256] = "";
|
||||||
|
bool ok = filter_rule_list_parse_append(list, rules[i], NULL, NULL, err, sizeof(err));
|
||||||
|
if (!ok)
|
||||||
|
printf(" rule '%s' errored: %s\n", rules[i], err);
|
||||||
|
EXPECT_TRUE(ok);
|
||||||
|
filter_rule_list_free(list);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* A glued word is a pattern, not a modifier run (no separator). */
|
||||||
|
{
|
||||||
|
FilterRuleList* list = filter_rule_list_create();
|
||||||
|
char err[128] = "";
|
||||||
|
EXPECT_TRUE(filter_rule_list_parse_append(list, "-newfile", NULL, NULL, err, sizeof(err)));
|
||||||
|
EXPECT_EQ_INT(list->count, 1);
|
||||||
|
EXPECT_EQ_STR(list->items[0]->pattern, "newfile");
|
||||||
|
filter_rule_list_free(list);
|
||||||
|
|
||||||
|
list = filter_rule_list_create();
|
||||||
|
EXPECT_TRUE(filter_rule_list_parse_append(list, "-e2e", NULL, NULL, err, sizeof(err)));
|
||||||
|
EXPECT_EQ_INT(list->count, 1);
|
||||||
|
EXPECT_EQ_STR(list->items[0]->pattern, "e2e");
|
||||||
|
filter_rule_list_free(list);
|
||||||
|
|
||||||
|
list = filter_rule_list_create();
|
||||||
|
EXPECT_TRUE(filter_rule_list_parse_append(list, "-*.o", NULL, NULL, err, sizeof(err)));
|
||||||
|
EXPECT_EQ_INT(list->count, 1);
|
||||||
|
EXPECT_EQ_STR(list->items[0]->pattern, "*.o");
|
||||||
|
filter_rule_list_free(list);
|
||||||
|
|
||||||
|
/* A comma with no modifier still treats the rest as the pattern. */
|
||||||
|
list = filter_rule_list_create();
|
||||||
|
EXPECT_TRUE(filter_rule_list_parse_append(list, "exclude,foo", NULL, NULL, err, sizeof(err)));
|
||||||
|
EXPECT_EQ_INT(list->count, 1);
|
||||||
|
EXPECT_EQ_STR(list->items[0]->pattern, "foo");
|
||||||
|
filter_rule_list_free(list);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* `!` clears the list. */
|
||||||
|
{
|
||||||
|
FilterRuleList* list = filter_rule_list_create();
|
||||||
|
char err[128] = "";
|
||||||
|
EXPECT_TRUE(filter_rule_list_parse_append(list, "- *.tmp", NULL, NULL, err, sizeof(err)));
|
||||||
|
EXPECT_EQ_INT(list->count, 1);
|
||||||
|
EXPECT_TRUE(filter_rule_list_parse_append(list, "!", NULL, NULL, err, sizeof(err)));
|
||||||
|
EXPECT_EQ_INT(list->count, 0);
|
||||||
|
filter_rule_list_free(list);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* -C injects the CVS defaults. */
|
||||||
|
{
|
||||||
|
FilterRuleList* list = filter_rule_list_create();
|
||||||
|
char err[128] = "";
|
||||||
|
EXPECT_TRUE(filter_rule_list_parse_append(list, "-C", NULL, NULL, err, sizeof(err)));
|
||||||
|
EXPECT_TRUE(list->count > 0);
|
||||||
|
filter_rule_list_free(list);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_filter_list_merge_file_still_supported() {
|
||||||
|
char tmpl[] = "/tmp/fastsync_filter_XXXXXX";
|
||||||
|
EXPECT_TRUE(mkdtemp(tmpl) != NULL);
|
||||||
|
char path[512];
|
||||||
|
snprintf(path, sizeof(path), "%s/rules", tmpl);
|
||||||
|
FILE* fp = fopen(path, "w");
|
||||||
|
EXPECT_NOT_NULL(fp);
|
||||||
|
fputs("- *.tmp\n", fp);
|
||||||
|
fclose(fp);
|
||||||
|
|
||||||
|
FilterRuleList* list = filter_rule_list_create();
|
||||||
|
char err[256] = "";
|
||||||
|
char rule[600];
|
||||||
|
snprintf(rule, sizeof(rule), "merge %s", path);
|
||||||
|
EXPECT_TRUE(filter_rule_list_parse_append(list, rule, NULL, NULL, err, sizeof(err)));
|
||||||
|
EXPECT_EQ_INT(list->count, 1);
|
||||||
|
filter_rule_list_free(list);
|
||||||
|
|
||||||
|
/* The same merge with the x modifier is rejected, not silently read. */
|
||||||
|
list = filter_rule_list_create();
|
||||||
|
snprintf(rule, sizeof(rule), "merge,x %s", path);
|
||||||
|
EXPECT_FALSE(filter_rule_list_parse_append(list, rule, NULL, NULL, err, sizeof(err)));
|
||||||
|
EXPECT_TRUE(strstr(err, "xattr") != NULL);
|
||||||
|
filter_rule_list_free(list);
|
||||||
|
|
||||||
|
unlink(path);
|
||||||
|
rmdir(tmpl);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_filter_rule_parse_rejects_unsupported_and_keeps_supported() {
|
||||||
|
char err[256] = "";
|
||||||
|
|
||||||
|
EXPECT_NULL(filter_rule_parse("-x user.foo", NULL, err, sizeof(err)));
|
||||||
|
EXPECT_TRUE(strstr(err, "xattr") != NULL);
|
||||||
|
|
||||||
|
EXPECT_NULL(filter_rule_parse("-e foo", NULL, err, sizeof(err)));
|
||||||
|
EXPECT_TRUE(strstr(err, "unsupported filter modifier") != NULL);
|
||||||
|
|
||||||
|
FilterRule* rule = filter_rule_parse("- *.tmp", NULL, err, sizeof(err));
|
||||||
|
EXPECT_NOT_NULL(rule);
|
||||||
|
EXPECT_EQ_STR(rule->pattern, "*.tmp");
|
||||||
|
filter_rule_free(rule);
|
||||||
|
|
||||||
|
rule = filter_rule_parse("-newfile", NULL, err, sizeof(err));
|
||||||
|
EXPECT_NOT_NULL(rule);
|
||||||
|
EXPECT_EQ_STR(rule->pattern, "newfile");
|
||||||
|
filter_rule_free(rule);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_filter_rules_apply_supported_modifiers() {
|
||||||
|
/* exclude */
|
||||||
|
{
|
||||||
|
const char* texts[] = {"- *.tmp"};
|
||||||
|
FilterRuleList* list = filter_base_build(texts, 1, false, false, NULL, 0);
|
||||||
|
EXPECT_NOT_NULL(list);
|
||||||
|
EXPECT_EQ_INT(filter_rules_apply_side(list, "b.tmp", "b.tmp", false, FILTER_SIDE_SENDER),
|
||||||
|
FILTER_ACTION_EXCLUDE);
|
||||||
|
EXPECT_EQ_INT(filter_rules_apply_side(list, "a.txt", "a.txt", false, FILTER_SIDE_SENDER),
|
||||||
|
FILTER_ACTION_NONE);
|
||||||
|
filter_rule_list_free(list);
|
||||||
|
}
|
||||||
|
/* anchored include then exclude-all */
|
||||||
|
{
|
||||||
|
const char* texts[] = {"+ /a.txt", "- *"};
|
||||||
|
FilterRuleList* list = filter_base_build(texts, 2, false, false, NULL, 0);
|
||||||
|
EXPECT_NOT_NULL(list);
|
||||||
|
EXPECT_EQ_INT(filter_rules_apply_side(list, "a.txt", "a.txt", false, FILTER_SIDE_SENDER),
|
||||||
|
FILTER_ACTION_INCLUDE);
|
||||||
|
EXPECT_EQ_INT(filter_rules_apply_side(list, "b.txt", "b.txt", false, FILTER_SIDE_SENDER),
|
||||||
|
FILTER_ACTION_EXCLUDE);
|
||||||
|
filter_rule_list_free(list);
|
||||||
|
}
|
||||||
|
/* negate */
|
||||||
|
{
|
||||||
|
const char* texts[] = {"-! *.o"};
|
||||||
|
FilterRuleList* list = filter_base_build(texts, 1, false, false, NULL, 0);
|
||||||
|
EXPECT_NOT_NULL(list);
|
||||||
|
EXPECT_EQ_INT(filter_rules_apply_side(list, "foo.c", "foo.c", false, FILTER_SIDE_SENDER),
|
||||||
|
FILTER_ACTION_EXCLUDE);
|
||||||
|
EXPECT_EQ_INT(filter_rules_apply_side(list, "foo.o", "foo.o", false, FILTER_SIDE_SENDER),
|
||||||
|
FILTER_ACTION_NONE);
|
||||||
|
filter_rule_list_free(list);
|
||||||
|
}
|
||||||
|
/* dir-only trailing slash */
|
||||||
|
{
|
||||||
|
const char* texts[] = {"+ dir/", "- *"};
|
||||||
|
FilterRuleList* list = filter_base_build(texts, 2, false, false, NULL, 0);
|
||||||
|
EXPECT_NOT_NULL(list);
|
||||||
|
EXPECT_EQ_INT(filter_rules_apply_side(list, "dir", "dir", true, FILTER_SIDE_SENDER),
|
||||||
|
FILTER_ACTION_INCLUDE);
|
||||||
|
EXPECT_EQ_INT(filter_rules_apply_side(list, "dir", "dir", false, FILTER_SIDE_SENDER),
|
||||||
|
FILTER_ACTION_EXCLUDE);
|
||||||
|
filter_rule_list_free(list);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void test_filter() {
|
||||||
|
test_filter_list_rejects_xattr_modifier();
|
||||||
|
test_filter_list_rejects_unsupported_modifiers();
|
||||||
|
test_filter_list_accepts_merge_modifiers();
|
||||||
|
test_filter_list_accepts_supported_rules_and_modifiers();
|
||||||
|
test_filter_list_merge_file_still_supported();
|
||||||
|
test_filter_rule_parse_rejects_unsupported_and_keeps_supported();
|
||||||
|
test_filter_rules_apply_supported_modifiers();
|
||||||
|
}
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
#ifndef TEST_FILTER_H
|
||||||
|
#define TEST_FILTER_H
|
||||||
|
|
||||||
|
void test_filter(void);
|
||||||
|
|
||||||
|
#endif
|
||||||
+66
-32
@@ -339,7 +339,7 @@ static void test_file_restore_metadata_applies_atime() {
|
|||||||
m.crtime_sec = 0;
|
m.crtime_sec = 0;
|
||||||
m.crtime_nsec = 0;
|
m.crtime_nsec = 0;
|
||||||
|
|
||||||
file_restore_metadata(path, &m, (FileAttrPolicy){true, true, true, false});
|
file_restore_metadata(path, &m, (FileAttrPolicy){true, true, true, false, true});
|
||||||
|
|
||||||
struct stat st;
|
struct stat st;
|
||||||
EXPECT_EQ_INT(stat(path, &st), 0);
|
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||||
@@ -378,7 +378,7 @@ static void test_file_restore_metadata() {
|
|||||||
.atime_valid = false,
|
.atime_valid = false,
|
||||||
.crtime_valid = false};
|
.crtime_valid = false};
|
||||||
|
|
||||||
file_restore_metadata(path, &m, (FileAttrPolicy){true, true, false, false});
|
file_restore_metadata(path, &m, (FileAttrPolicy){true, true, false, false, true});
|
||||||
|
|
||||||
struct stat st;
|
struct stat st;
|
||||||
EXPECT_EQ_INT(stat(path, &st), 0);
|
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||||
@@ -395,7 +395,7 @@ static void test_file_restore_executability_only() {
|
|||||||
|
|
||||||
FileMetadata m = {
|
FileMetadata m = {
|
||||||
.mode = 0751, .uid = getuid(), .gid = getgid(), .mtime_sec = 0, .mtime_nsec = 0};
|
.mode = 0751, .uid = getuid(), .gid = getgid(), .mtime_sec = 0, .mtime_nsec = 0};
|
||||||
file_restore_metadata(path, &m, (FileAttrPolicy){false, false, false, true});
|
file_restore_metadata(path, &m, (FileAttrPolicy){false, false, false, true, true});
|
||||||
|
|
||||||
struct stat st;
|
struct stat st;
|
||||||
EXPECT_EQ_INT(stat(path, &st), 0);
|
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||||
@@ -409,7 +409,7 @@ static void test_directory_restore_executability_only() {
|
|||||||
|
|
||||||
FileMetadata m = {
|
FileMetadata m = {
|
||||||
.mode = 0755, .uid = getuid(), .gid = getgid(), .mtime_sec = 0, .mtime_nsec = 0};
|
.mode = 0755, .uid = getuid(), .gid = getgid(), .mtime_sec = 0, .mtime_nsec = 0};
|
||||||
file_restore_metadata(path, &m, (FileAttrPolicy){false, false, false, true});
|
file_restore_metadata(path, &m, (FileAttrPolicy){false, false, false, true, true});
|
||||||
|
|
||||||
struct stat st;
|
struct stat st;
|
||||||
EXPECT_EQ_INT(stat(path, &st), 0);
|
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||||
@@ -437,7 +437,7 @@ static void test_file_restore_executability_rsync_rule() {
|
|||||||
EXPECT_TRUE(file_write_to_disk(path, "x", 1, false, false));
|
EXPECT_TRUE(file_write_to_disk(path, "x", 1, false, false));
|
||||||
EXPECT_EQ_INT(chmod(path, cases[i].dest), 0);
|
EXPECT_EQ_INT(chmod(path, cases[i].dest), 0);
|
||||||
FileMetadata m = {.mode = cases[i].src, .uid = getuid(), .gid = getgid()};
|
FileMetadata m = {.mode = cases[i].src, .uid = getuid(), .gid = getgid()};
|
||||||
file_restore_metadata(path, &m, (FileAttrPolicy){false, false, false, true});
|
file_restore_metadata(path, &m, (FileAttrPolicy){false, false, false, true, true});
|
||||||
struct stat st;
|
struct stat st;
|
||||||
EXPECT_EQ_INT(stat(path, &st), 0);
|
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||||
EXPECT_EQ_INT(st.st_mode & 0777, cases[i].want);
|
EXPECT_EQ_INT(st.st_mode & 0777, cases[i].want);
|
||||||
@@ -453,34 +453,60 @@ static void test_file_restore_executability_rsync_rule() {
|
|||||||
* bits from the destination and --perms wins when both are set. */
|
* bits from the destination and --perms wins when both are set. */
|
||||||
static void test_metadata_mode_for_policy() {
|
static void test_metadata_mode_for_policy() {
|
||||||
mode_t out = 0xdead;
|
mode_t out = 0xdead;
|
||||||
EXPECT_FALSE(
|
EXPECT_FALSE(metadata_mode_for_policy(0777, 0644,
|
||||||
metadata_mode_for_policy(0777, 0644, (FileAttrPolicy){false, false, false, false}, &out));
|
(FileAttrPolicy){false, false, false, false, true}, &out));
|
||||||
EXPECT_EQ_INT((int)out, 0xdead); /* untouched when no change is requested */
|
EXPECT_EQ_INT((int)out, 0xdead); /* untouched when no change is requested */
|
||||||
|
|
||||||
EXPECT_TRUE(
|
EXPECT_TRUE(metadata_mode_for_policy(0777, 0644,
|
||||||
metadata_mode_for_policy(0777, 0644, (FileAttrPolicy){true, false, false, false}, &out));
|
(FileAttrPolicy){true, false, false, false, true}, &out));
|
||||||
EXPECT_EQ_INT((int)(out & 0777), 0777); /* group/other write is preserved */
|
EXPECT_EQ_INT((int)(out & 0777), 0777); /* group/other write is preserved */
|
||||||
|
|
||||||
mode_t specials = (mode_t)(S_ISUID | S_ISGID | S_ISVTX | 0672);
|
mode_t specials = (mode_t)(S_ISUID | S_ISGID | S_ISVTX | 0672);
|
||||||
EXPECT_TRUE(
|
EXPECT_TRUE(metadata_mode_for_policy(specials, 0644,
|
||||||
metadata_mode_for_policy(specials, 0644, (FileAttrPolicy){true, false, false, false}, &out));
|
(FileAttrPolicy){true, false, false, false, true}, &out));
|
||||||
EXPECT_EQ_INT((int)(out & (S_ISUID | S_ISGID | S_ISVTX | 0777)),
|
EXPECT_EQ_INT((int)(out & (S_ISUID | S_ISGID | S_ISVTX | 0777)),
|
||||||
(int)(S_ISUID | S_ISGID | S_ISVTX | 0672));
|
(int)(S_ISUID | S_ISGID | S_ISVTX | 0672));
|
||||||
|
|
||||||
|
/* SUPER_MODE_OFF: the special bits are stripped even under -p (this also
|
||||||
|
* covers bits introduced by --chmod, whose result is fed in as source_mode),
|
||||||
|
* while the ordinary permission bits are still copied. */
|
||||||
|
EXPECT_TRUE(metadata_mode_for_policy(specials, 0644,
|
||||||
|
(FileAttrPolicy){true, false, false, false, false}, &out));
|
||||||
|
EXPECT_EQ_INT((int)(out & (S_ISUID | S_ISGID | S_ISVTX)), 0);
|
||||||
|
EXPECT_EQ_INT((int)(out & 0777), 0672);
|
||||||
|
EXPECT_TRUE(metadata_mode_for_policy(04755, 0644,
|
||||||
|
(FileAttrPolicy){true, false, false, false, false}, &out));
|
||||||
|
EXPECT_EQ_INT((int)(out & 07777), 0755);
|
||||||
|
/* The same holds for a special bit introduced by --chmod=+s. */
|
||||||
|
mode_t chmodded = 0;
|
||||||
|
EXPECT_TRUE(chmod_apply(0755, "u+s", &chmodded));
|
||||||
|
EXPECT_TRUE(metadata_mode_for_policy(chmodded, 0644,
|
||||||
|
(FileAttrPolicy){true, false, false, false, false}, &out));
|
||||||
|
EXPECT_EQ_INT((int)(out & 07777), 0755);
|
||||||
|
|
||||||
|
/* -E: a destination's own special bits survive unless super-user activities
|
||||||
|
* are forbidden, in which case they are stripped from the derived base. */
|
||||||
|
EXPECT_TRUE(metadata_mode_for_policy(0755, (mode_t)(S_ISUID | 0750),
|
||||||
|
(FileAttrPolicy){false, false, false, true, true}, &out));
|
||||||
|
EXPECT_EQ_INT((int)(out & (S_ISUID | 0777)), (int)(S_ISUID | 0750));
|
||||||
|
EXPECT_TRUE(metadata_mode_for_policy(0755, (mode_t)(S_ISUID | 0750),
|
||||||
|
(FileAttrPolicy){false, false, false, true, false}, &out));
|
||||||
|
EXPECT_EQ_INT((int)(out & (S_ISUID | 0777)), 0750);
|
||||||
|
|
||||||
/* -E: exec bits derive from the DESTINATION's read bits. */
|
/* -E: exec bits derive from the DESTINATION's read bits. */
|
||||||
EXPECT_TRUE(
|
EXPECT_TRUE(metadata_mode_for_policy(0755, 0644,
|
||||||
metadata_mode_for_policy(0755, 0644, (FileAttrPolicy){false, false, false, true}, &out));
|
(FileAttrPolicy){false, false, false, true, true}, &out));
|
||||||
EXPECT_EQ_INT((int)(out & 0777), 0755);
|
EXPECT_EQ_INT((int)(out & 0777), 0755);
|
||||||
EXPECT_TRUE(
|
EXPECT_TRUE(metadata_mode_for_policy(0644, 0755,
|
||||||
metadata_mode_for_policy(0644, 0755, (FileAttrPolicy){false, false, false, true}, &out));
|
(FileAttrPolicy){false, false, false, true, true}, &out));
|
||||||
EXPECT_EQ_INT((int)(out & 0777), 0644);
|
EXPECT_EQ_INT((int)(out & 0777), 0644);
|
||||||
EXPECT_TRUE(
|
EXPECT_TRUE(metadata_mode_for_policy(0755, 0600,
|
||||||
metadata_mode_for_policy(0755, 0600, (FileAttrPolicy){false, false, false, true}, &out));
|
(FileAttrPolicy){false, false, false, true, true}, &out));
|
||||||
EXPECT_EQ_INT((int)(out & 0777), 0700);
|
EXPECT_EQ_INT((int)(out & 0777), 0700);
|
||||||
|
|
||||||
/* --perms wins over -E when both are set. */
|
/* --perms wins over -E when both are set. */
|
||||||
EXPECT_TRUE(
|
EXPECT_TRUE(
|
||||||
metadata_mode_for_policy(0700, 0644, (FileAttrPolicy){true, false, false, true}, &out));
|
metadata_mode_for_policy(0700, 0644, (FileAttrPolicy){true, false, false, true, true}, &out));
|
||||||
EXPECT_EQ_INT((int)(out & 0777), 0700);
|
EXPECT_EQ_INT((int)(out & 0777), 0700);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -493,8 +519,8 @@ static void test_new_file_mode_from_source_and_umask() {
|
|||||||
mode_t want = (mode_t)(0751 & 0777 & ~(mode_t)file_process_umask());
|
mode_t want = (mode_t)(0751 & 0777 & ~(mode_t)file_process_umask());
|
||||||
|
|
||||||
bool ok = file_to_disk_secure_attrs(path, "x", 1, false, false, false, &m,
|
bool ok = file_to_disk_secure_attrs(path, "x", 1, false, false, false, &m,
|
||||||
(FileAttrPolicy){false, false, false, false}, false, false,
|
(FileAttrPolicy){false, false, false, false, true}, false,
|
||||||
false, NULL, false, false, NULL);
|
false, false, NULL, false, false, NULL);
|
||||||
EXPECT_TRUE(ok);
|
EXPECT_TRUE(ok);
|
||||||
struct stat st;
|
struct stat st;
|
||||||
EXPECT_EQ_INT(stat(path, &st), 0);
|
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||||
@@ -503,8 +529,8 @@ static void test_new_file_mode_from_source_and_umask() {
|
|||||||
|
|
||||||
/* -E on top of the source&~umask base (src 0751, umask 022 -> 0751). */
|
/* -E on top of the source&~umask base (src 0751, umask 022 -> 0751). */
|
||||||
ok = file_to_disk_secure_attrs(path, "x", 1, false, false, false, &m,
|
ok = file_to_disk_secure_attrs(path, "x", 1, false, false, false, &m,
|
||||||
(FileAttrPolicy){false, false, false, true}, false, false, false,
|
(FileAttrPolicy){false, false, false, true, true}, false, false,
|
||||||
NULL, false, false, NULL);
|
false, NULL, false, false, NULL);
|
||||||
EXPECT_TRUE(ok);
|
EXPECT_TRUE(ok);
|
||||||
EXPECT_EQ_INT(stat(path, &st), 0);
|
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||||
mode_t want_e =
|
mode_t want_e =
|
||||||
@@ -529,7 +555,7 @@ static void test_file_restore_attribute_split() {
|
|||||||
.crtime_valid = false};
|
.crtime_valid = false};
|
||||||
|
|
||||||
/* times only: mtime changes, mode stays 0640. */
|
/* times only: mtime changes, mode stays 0640. */
|
||||||
file_restore_metadata(path, &m, (FileAttrPolicy){false, true, false, false});
|
file_restore_metadata(path, &m, (FileAttrPolicy){false, true, false, false, true});
|
||||||
struct stat st;
|
struct stat st;
|
||||||
EXPECT_EQ_INT(stat(path, &st), 0);
|
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||||
EXPECT_EQ_INT(st.st_mode & 0777, 0640);
|
EXPECT_EQ_INT(st.st_mode & 0777, 0640);
|
||||||
@@ -543,7 +569,7 @@ static void test_file_restore_attribute_split() {
|
|||||||
FileMetadata m2 = m;
|
FileMetadata m2 = m;
|
||||||
m2.mode = 0700;
|
m2.mode = 0700;
|
||||||
m2.mtime_sec = 1600000000;
|
m2.mtime_sec = 1600000000;
|
||||||
file_restore_metadata(path, &m2, (FileAttrPolicy){false, false, false, false});
|
file_restore_metadata(path, &m2, (FileAttrPolicy){false, false, false, false, true});
|
||||||
EXPECT_EQ_INT(stat(path, &st), 0);
|
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||||
EXPECT_EQ_INT(st.st_mode & 0777, 0640);
|
EXPECT_EQ_INT(st.st_mode & 0777, 0640);
|
||||||
EXPECT_EQ_INT((int)st.st_mtime, 1000000000);
|
EXPECT_EQ_INT((int)st.st_mtime, 1000000000);
|
||||||
@@ -569,6 +595,11 @@ static void test_file_attr_policy_from_config() {
|
|||||||
EXPECT_TRUE(p.times);
|
EXPECT_TRUE(p.times);
|
||||||
EXPECT_TRUE(p.atimes);
|
EXPECT_TRUE(p.atimes);
|
||||||
EXPECT_TRUE(p.executability);
|
EXPECT_TRUE(p.executability);
|
||||||
|
/* Default super mode (AUTO) permits special bits. */
|
||||||
|
EXPECT_TRUE(p.super_permitted);
|
||||||
|
c->super_mode = SUPER_MODE_OFF;
|
||||||
|
p = file_attr_policy_from_config(c);
|
||||||
|
EXPECT_FALSE(p.super_permitted);
|
||||||
config_delete(c);
|
config_delete(c);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -582,8 +613,8 @@ static void test_perms_preserves_special_bits() {
|
|||||||
.mode = (mode_t)(S_ISUID | S_ISGID | S_ISVTX | 0755), .uid = getuid(), .gid = getgid()};
|
.mode = (mode_t)(S_ISUID | S_ISGID | S_ISVTX | 0755), .uid = getuid(), .gid = getgid()};
|
||||||
|
|
||||||
bool ok = file_to_disk_secure_attrs(path, "x", 1, false, false, false, &m,
|
bool ok = file_to_disk_secure_attrs(path, "x", 1, false, false, false, &m,
|
||||||
(FileAttrPolicy){true, false, false, false}, false, false,
|
(FileAttrPolicy){true, false, false, false, true}, false,
|
||||||
false, NULL, false, false, NULL);
|
false, false, NULL, false, false, NULL);
|
||||||
EXPECT_TRUE(ok);
|
EXPECT_TRUE(ok);
|
||||||
struct stat st;
|
struct stat st;
|
||||||
EXPECT_EQ_INT(stat(path, &st), 0);
|
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||||
@@ -670,7 +701,8 @@ static void test_file_restore_symlink_metadata() {
|
|||||||
|
|
||||||
/* Positive path: a non-omitted apply stamps the link's own mtime. */
|
/* Positive path: a non-omitted apply stamps the link's own mtime. */
|
||||||
FileMetadata applied = {.mtime_sec = 1000000000, .mtime_nsec = 0};
|
FileMetadata applied = {.mtime_sec = 1000000000, .mtime_nsec = 0};
|
||||||
file_restore_symlink_metadata(link, &applied, (FileAttrPolicy){false, true, false, false}, false);
|
file_restore_symlink_metadata(link, &applied, (FileAttrPolicy){false, true, false, false, true},
|
||||||
|
false);
|
||||||
struct stat st;
|
struct stat st;
|
||||||
EXPECT_EQ_INT(lstat(link, &st), 0);
|
EXPECT_EQ_INT(lstat(link, &st), 0);
|
||||||
EXPECT_TRUE(S_ISLNK(st.st_mode));
|
EXPECT_TRUE(S_ISLNK(st.st_mode));
|
||||||
@@ -679,7 +711,8 @@ static void test_file_restore_symlink_metadata() {
|
|||||||
|
|
||||||
/* -J: a different time must be left untouched. */
|
/* -J: a different time must be left untouched. */
|
||||||
FileMetadata newer = {.mtime_sec = 1234567890, .mtime_nsec = 0};
|
FileMetadata newer = {.mtime_sec = 1234567890, .mtime_nsec = 0};
|
||||||
file_restore_symlink_metadata(link, &newer, (FileAttrPolicy){false, true, false, false}, true);
|
file_restore_symlink_metadata(link, &newer, (FileAttrPolicy){false, true, false, false, true},
|
||||||
|
true);
|
||||||
EXPECT_EQ_INT(lstat(link, &st), 0);
|
EXPECT_EQ_INT(lstat(link, &st), 0);
|
||||||
EXPECT_EQ_INT((int)st.st_mtime, (int)t1);
|
EXPECT_EQ_INT((int)st.st_mtime, (int)t1);
|
||||||
if (symlink_times_supported)
|
if (symlink_times_supported)
|
||||||
@@ -723,14 +756,14 @@ static void test_file_restore_metadata_fd_attribute_split() {
|
|||||||
|
|
||||||
/* perms-only: mode applied, mtime untouched. */
|
/* perms-only: mode applied, mtime untouched. */
|
||||||
EXPECT_EQ_INT(fstat(fd, &before), 0);
|
EXPECT_EQ_INT(fstat(fd, &before), 0);
|
||||||
EXPECT_TRUE(file_restore_metadata_fd(fd, &m, (FileAttrPolicy){true, false, false, false}));
|
EXPECT_TRUE(file_restore_metadata_fd(fd, &m, (FileAttrPolicy){true, false, false, false, true}));
|
||||||
EXPECT_EQ_INT(fstat(fd, &st), 0);
|
EXPECT_EQ_INT(fstat(fd, &st), 0);
|
||||||
EXPECT_EQ_INT(st.st_mode & 0777, 0755);
|
EXPECT_EQ_INT(st.st_mode & 0777, 0755);
|
||||||
EXPECT_EQ_INT((int)st.st_mtime, (int)before.st_mtime);
|
EXPECT_EQ_INT((int)st.st_mtime, (int)before.st_mtime);
|
||||||
|
|
||||||
/* times-only: mtime applied, mode untouched. */
|
/* times-only: mtime applied, mode untouched. */
|
||||||
EXPECT_EQ_INT(chmod(path, 0600), 0);
|
EXPECT_EQ_INT(chmod(path, 0600), 0);
|
||||||
EXPECT_TRUE(file_restore_metadata_fd(fd, &m, (FileAttrPolicy){false, true, false, false}));
|
EXPECT_TRUE(file_restore_metadata_fd(fd, &m, (FileAttrPolicy){false, true, false, false, true}));
|
||||||
EXPECT_EQ_INT(fstat(fd, &st), 0);
|
EXPECT_EQ_INT(fstat(fd, &st), 0);
|
||||||
EXPECT_EQ_INT(st.st_mode & 0777, 0600);
|
EXPECT_EQ_INT(st.st_mode & 0777, 0600);
|
||||||
EXPECT_EQ_INT((int)st.st_mtime, 1234567890);
|
EXPECT_EQ_INT((int)st.st_mtime, 1234567890);
|
||||||
@@ -740,7 +773,7 @@ static void test_file_restore_metadata_fd_attribute_split() {
|
|||||||
{.tv_sec = 1000000000, .tv_nsec = 0}};
|
{.tv_sec = 1000000000, .tv_nsec = 0}};
|
||||||
EXPECT_EQ_INT(futimens(fd, reset), 0);
|
EXPECT_EQ_INT(futimens(fd, reset), 0);
|
||||||
EXPECT_EQ_INT(fstat(fd, &before), 0);
|
EXPECT_EQ_INT(fstat(fd, &before), 0);
|
||||||
EXPECT_TRUE(file_restore_metadata_fd(fd, &m, (FileAttrPolicy){false, false, true, false}));
|
EXPECT_TRUE(file_restore_metadata_fd(fd, &m, (FileAttrPolicy){false, false, true, false, true}));
|
||||||
EXPECT_EQ_INT(fstat(fd, &st), 0);
|
EXPECT_EQ_INT(fstat(fd, &st), 0);
|
||||||
EXPECT_EQ_INT((int)st.st_atime, 999999999);
|
EXPECT_EQ_INT((int)st.st_atime, 999999999);
|
||||||
EXPECT_EQ_INT((int)st.st_mtime, (int)before.st_mtime);
|
EXPECT_EQ_INT((int)st.st_mtime, (int)before.st_mtime);
|
||||||
@@ -753,7 +786,8 @@ static void test_file_restore_metadata_fd_attribute_split() {
|
|||||||
m2.mode = 0700;
|
m2.mode = 0700;
|
||||||
m2.mtime_sec = 1600000000;
|
m2.mtime_sec = 1600000000;
|
||||||
m2.atime_sec = 1700000000;
|
m2.atime_sec = 1700000000;
|
||||||
EXPECT_TRUE(file_restore_metadata_fd(fd, &m2, (FileAttrPolicy){false, false, false, false}));
|
EXPECT_TRUE(
|
||||||
|
file_restore_metadata_fd(fd, &m2, (FileAttrPolicy){false, false, false, false, true}));
|
||||||
EXPECT_EQ_INT(fstat(fd, &st), 0);
|
EXPECT_EQ_INT(fstat(fd, &st), 0);
|
||||||
EXPECT_EQ_INT(st.st_mode & 0777, 0640);
|
EXPECT_EQ_INT(st.st_mode & 0777, 0640);
|
||||||
EXPECT_EQ_INT((int)st.st_mtime, 1000000000);
|
EXPECT_EQ_INT((int)st.st_mtime, 1000000000);
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
#include "test_utils.h"
|
#include "test_utils.h"
|
||||||
#include <limits.h>
|
#include <limits.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
|
#include <time.h>
|
||||||
#include <unistd.h>
|
#include <unistd.h>
|
||||||
#include <threads.h>
|
#include <threads.h>
|
||||||
|
|
||||||
@@ -215,6 +216,38 @@ static void test_send_receive_status() {
|
|||||||
close(p[1]);
|
close(p[1]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* An unknown wire status outside the enum range must be rejected as a protocol
|
||||||
|
* error instead of being handed to the caller as an unexpected verdict. The
|
||||||
|
* last known enumerator (STATUS_STATS) must still be accepted, proving the
|
||||||
|
* validation does not reject legitimate statuses. */
|
||||||
|
static void test_receive_status_rejects_unknown() {
|
||||||
|
int p[2];
|
||||||
|
EXPECT_EQ_INT(pipe(p), 0);
|
||||||
|
ProtocolSession session;
|
||||||
|
protocol_session_init(&session, p[0], p[1]);
|
||||||
|
|
||||||
|
Status bogus = (Status)(STATUS_STATS + 1);
|
||||||
|
EXPECT_EQ_INT((int)write(p[1], &bogus, sizeof(bogus)), (int)sizeof(bogus));
|
||||||
|
Status received = STATUS_OK;
|
||||||
|
EXPECT_FALSE(protocol_receive_status(&session, &received));
|
||||||
|
|
||||||
|
Status negative = (Status)-1;
|
||||||
|
EXPECT_EQ_INT((int)write(p[1], &negative, sizeof(negative)), (int)sizeof(negative));
|
||||||
|
EXPECT_FALSE(protocol_receive_status(&session, &received));
|
||||||
|
|
||||||
|
Status top = STATUS_STATS;
|
||||||
|
EXPECT_EQ_INT((int)write(p[1], &top, sizeof(top)), (int)sizeof(top));
|
||||||
|
EXPECT_TRUE(protocol_receive_status(&session, &received));
|
||||||
|
EXPECT_EQ_INT((int)received, (int)STATUS_STATS);
|
||||||
|
|
||||||
|
Status timed_bogus = (Status)(STATUS_STATS + 7);
|
||||||
|
EXPECT_EQ_INT((int)write(p[1], &timed_bogus, sizeof(timed_bogus)), (int)sizeof(timed_bogus));
|
||||||
|
EXPECT_FALSE(protocol_receive_status_timed(&session, &received, 5));
|
||||||
|
|
||||||
|
close(p[0]);
|
||||||
|
close(p[1]);
|
||||||
|
}
|
||||||
|
|
||||||
static void test_receive_n_data_truncated() {
|
static void test_receive_n_data_truncated() {
|
||||||
int p[2];
|
int p[2];
|
||||||
EXPECT_EQ_INT(pipe(p), 0);
|
EXPECT_EQ_INT(pipe(p), 0);
|
||||||
@@ -669,6 +702,50 @@ static void test_receive_status_keepalive_emits() {
|
|||||||
close(to_peer[1]);
|
close(to_peer[1]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* protocol_throttle_bytes() must apply the same token-bucket pacing as the
|
||||||
|
* buffered protocol send path, so the plaintext sendfile fast path honors
|
||||||
|
* --bwlimit exactly like the TLS path. With bwlimit=1 MB/s the initial burst
|
||||||
|
* is 100 KB (bwlimit/10); pacing 150 KB therefore owes ~50 KB of debt, i.e. a
|
||||||
|
* ~50 ms sleep. */
|
||||||
|
static void test_protocol_throttle_bytes_paces() {
|
||||||
|
ProtocolSession session;
|
||||||
|
protocol_session_init(&session, -1, -1);
|
||||||
|
protocol_session_bind(&session);
|
||||||
|
protocol_session_set_bwlimit(&session, 1000000ULL);
|
||||||
|
|
||||||
|
struct timespec start;
|
||||||
|
clock_gettime(CLOCK_MONOTONIC, &start);
|
||||||
|
protocol_throttle_bytes(150000);
|
||||||
|
struct timespec now;
|
||||||
|
clock_gettime(CLOCK_MONOTONIC, &now);
|
||||||
|
long long elapsed_ms =
|
||||||
|
(now.tv_sec - start.tv_sec) * 1000LL + (now.tv_nsec - start.tv_nsec) / 1000000LL;
|
||||||
|
/* Allow for scheduler slack but require the bulk of the expected 50 ms. */
|
||||||
|
EXPECT_TRUE(elapsed_ms >= 40);
|
||||||
|
|
||||||
|
protocol_session_unbind();
|
||||||
|
}
|
||||||
|
|
||||||
|
/* With no bandwidth limit the primitive must not sleep, however many bytes it
|
||||||
|
* is handed. */
|
||||||
|
static void test_protocol_throttle_bytes_unlimited() {
|
||||||
|
ProtocolSession session;
|
||||||
|
protocol_session_init(&session, -1, -1);
|
||||||
|
protocol_session_bind(&session);
|
||||||
|
protocol_session_set_bwlimit(&session, 0);
|
||||||
|
|
||||||
|
struct timespec start;
|
||||||
|
clock_gettime(CLOCK_MONOTONIC, &start);
|
||||||
|
protocol_throttle_bytes(100000000ULL);
|
||||||
|
struct timespec now;
|
||||||
|
clock_gettime(CLOCK_MONOTONIC, &now);
|
||||||
|
long long elapsed_ms =
|
||||||
|
(now.tv_sec - start.tv_sec) * 1000LL + (now.tv_nsec - start.tv_nsec) / 1000000LL;
|
||||||
|
EXPECT_TRUE(elapsed_ms < 2000);
|
||||||
|
|
||||||
|
protocol_session_unbind();
|
||||||
|
}
|
||||||
|
|
||||||
void test_protocol() {
|
void test_protocol() {
|
||||||
test_send_receive_n_data();
|
test_send_receive_n_data();
|
||||||
test_send_receive_n_data_zero();
|
test_send_receive_n_data_zero();
|
||||||
@@ -678,6 +755,7 @@ void test_protocol() {
|
|||||||
test_send_receive_data();
|
test_send_receive_data();
|
||||||
test_send_receive_int();
|
test_send_receive_int();
|
||||||
test_send_receive_status();
|
test_send_receive_status();
|
||||||
|
test_receive_status_rejects_unknown();
|
||||||
test_protocol_session_io_timeout();
|
test_protocol_session_io_timeout();
|
||||||
test_protocol_server_io_timeout_floor();
|
test_protocol_server_io_timeout_floor();
|
||||||
test_send_receive_status_timed();
|
test_send_receive_status_timed();
|
||||||
@@ -698,4 +776,6 @@ void test_protocol() {
|
|||||||
test_protocol_accounting_release_does_not_underflow();
|
test_protocol_accounting_release_does_not_underflow();
|
||||||
test_receive_data_charge_follows_owning_session();
|
test_receive_data_charge_follows_owning_session();
|
||||||
test_data_create_starts_uncharged_and_unowned();
|
test_data_create_starts_uncharged_and_unowned();
|
||||||
|
test_protocol_throttle_bytes_paces();
|
||||||
|
test_protocol_throttle_bytes_unlimited();
|
||||||
}
|
}
|
||||||
+21
-3
@@ -458,6 +458,11 @@ static void test_incremental_check_size_mismatch_full_transfer() {
|
|||||||
File* file = receive_incremental_check(p[0], cfg, &skipped);
|
File* file = receive_incremental_check(p[0], cfg, &skipped);
|
||||||
bool ok = file != NULL && !skipped && file->path != NULL && strcmp(file->path, "file.txt") == 0;
|
bool ok = file != NULL && !skipped && file->path != NULL && strcmp(file->path, "file.txt") == 0;
|
||||||
file_destroy(file);
|
file_destroy(file);
|
||||||
|
/* Stay alive until the parent closes its write end so its send_data can
|
||||||
|
never race this exit into a spurious EPIPE. */
|
||||||
|
char drain;
|
||||||
|
while (read(p[0], &drain, 1) > 0) {
|
||||||
|
}
|
||||||
config_delete(cfg);
|
config_delete(cfg);
|
||||||
close(p[0]);
|
close(p[0]);
|
||||||
_exit(ok ? 0 : 1);
|
_exit(ok ? 0 : 1);
|
||||||
@@ -487,9 +492,9 @@ static void test_incremental_check_size_mismatch_full_transfer() {
|
|||||||
EXPECT_TRUE(send_data(p[1], body));
|
EXPECT_TRUE(send_data(p[1], body));
|
||||||
data_destroy(body);
|
data_destroy(body);
|
||||||
|
|
||||||
|
close(p[1]);
|
||||||
int status;
|
int status;
|
||||||
waitpid(pid, &status, 0);
|
waitpid(pid, &status, 0);
|
||||||
close(p[1]);
|
|
||||||
config_delete(cfg);
|
config_delete(cfg);
|
||||||
unlink(path);
|
unlink(path);
|
||||||
rmdir(root);
|
rmdir(root);
|
||||||
@@ -1025,6 +1030,11 @@ static void test_incremental_check_fifo_destination_does_not_hang() {
|
|||||||
File* file = receive_incremental_check(p[0], cfg, &skipped);
|
File* file = receive_incremental_check(p[0], cfg, &skipped);
|
||||||
bool ok = file != NULL && !skipped;
|
bool ok = file != NULL && !skipped;
|
||||||
file_destroy(file);
|
file_destroy(file);
|
||||||
|
/* Stay alive until the parent closes its write end so its send_data can
|
||||||
|
never race this exit into a spurious EPIPE. */
|
||||||
|
char drain;
|
||||||
|
while (read(p[0], &drain, 1) > 0) {
|
||||||
|
}
|
||||||
config_delete(cfg);
|
config_delete(cfg);
|
||||||
close(p[0]);
|
close(p[0]);
|
||||||
_exit(ok ? 0 : 1);
|
_exit(ok ? 0 : 1);
|
||||||
@@ -1051,9 +1061,9 @@ static void test_incremental_check_fifo_destination_does_not_hang() {
|
|||||||
EXPECT_TRUE(send_data(p[1], body));
|
EXPECT_TRUE(send_data(p[1], body));
|
||||||
data_destroy(body);
|
data_destroy(body);
|
||||||
|
|
||||||
|
close(p[1]);
|
||||||
int status;
|
int status;
|
||||||
waitpid(pid, &status, 0);
|
waitpid(pid, &status, 0);
|
||||||
close(p[1]);
|
|
||||||
config_delete(cfg);
|
config_delete(cfg);
|
||||||
unlink(path);
|
unlink(path);
|
||||||
rmdir(root);
|
rmdir(root);
|
||||||
@@ -1191,6 +1201,12 @@ static void test_incremental_check_basis_fifo_does_not_hang() {
|
|||||||
File* file = receive_incremental_check(p[0], cfg, &skipped);
|
File* file = receive_incremental_check(p[0], cfg, &skipped);
|
||||||
bool ok = file != NULL && !skipped;
|
bool ok = file != NULL && !skipped;
|
||||||
file_destroy(file);
|
file_destroy(file);
|
||||||
|
/* The parent sends the data body after the check reply and only then closes
|
||||||
|
its write end. Stay alive until that EOF so the parent's send_data can
|
||||||
|
never race this exit into a spurious EPIPE. */
|
||||||
|
char drain;
|
||||||
|
while (read(p[0], &drain, 1) > 0) {
|
||||||
|
}
|
||||||
config_delete(cfg);
|
config_delete(cfg);
|
||||||
close(p[0]);
|
close(p[0]);
|
||||||
_exit(ok ? 0 : 1);
|
_exit(ok ? 0 : 1);
|
||||||
@@ -1222,9 +1238,11 @@ static void test_incremental_check_basis_fifo_does_not_hang() {
|
|||||||
EXPECT_TRUE(send_data(p[1], body));
|
EXPECT_TRUE(send_data(p[1], body));
|
||||||
data_destroy(body);
|
data_destroy(body);
|
||||||
|
|
||||||
|
/* Close the write end before waiting: this hands the child EOF so it can
|
||||||
|
exit, and guarantees the child was still alive for the data write. */
|
||||||
|
close(p[1]);
|
||||||
int status;
|
int status;
|
||||||
waitpid(pid, &status, 0);
|
waitpid(pid, &status, 0);
|
||||||
close(p[1]);
|
|
||||||
config_delete(cfg);
|
config_delete(cfg);
|
||||||
unlink(basis_path);
|
unlink(basis_path);
|
||||||
rmdir(basis_dir);
|
rmdir(basis_dir);
|
||||||
|
|||||||
+16
-31
@@ -5,13 +5,13 @@
|
|||||||
static void test_ssh_connect_invalid_dest_no_colon() {
|
static void test_ssh_connect_invalid_dest_no_colon() {
|
||||||
/* cppcheck-suppress constVariablePointer */
|
/* cppcheck-suppress constVariablePointer */
|
||||||
Client* client =
|
Client* client =
|
||||||
client_connect_ssh("invalid-destination-no-colon", 22, NULL, false, NULL, false, NULL, 0);
|
client_connect_ssh("invalid-destination-no-colon", 22, NULL, NULL, false, NULL, 0);
|
||||||
EXPECT_NULL(client);
|
EXPECT_NULL(client);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void test_ssh_connect_invalid_dest_empty() {
|
static void test_ssh_connect_invalid_dest_empty() {
|
||||||
/* cppcheck-suppress constVariablePointer */
|
/* cppcheck-suppress constVariablePointer */
|
||||||
Client* client = client_connect_ssh("", 22, NULL, false, NULL, false, NULL, 0);
|
Client* client = client_connect_ssh("", 22, NULL, NULL, false, NULL, 0);
|
||||||
EXPECT_NULL(client);
|
EXPECT_NULL(client);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -22,7 +22,7 @@ static void test_ssh_connect_malformed() {
|
|||||||
setenv("PATH", "", 1);
|
setenv("PATH", "", 1);
|
||||||
|
|
||||||
/* cppcheck-suppress constVariablePointer */
|
/* cppcheck-suppress constVariablePointer */
|
||||||
Client* client = client_connect_ssh(":", 22, NULL, false, NULL, false, NULL, 0);
|
Client* client = client_connect_ssh(":", 22, NULL, NULL, false, NULL, 0);
|
||||||
|
|
||||||
if (saved_path) {
|
if (saved_path) {
|
||||||
setenv("PATH", saved_path, 1);
|
setenv("PATH", saved_path, 1);
|
||||||
@@ -38,7 +38,7 @@ static void test_ssh_connect_malformed() {
|
|||||||
* The function launches ssh which will fail to connect, returns a Client. */
|
* The function launches ssh which will fail to connect, returns a Client. */
|
||||||
static void test_ssh_connect_unreachable() {
|
static void test_ssh_connect_unreachable() {
|
||||||
Client* client =
|
Client* client =
|
||||||
client_connect_ssh("nonexistent.invalid:/remote/path", 22, NULL, false, NULL, false, NULL, 0);
|
client_connect_ssh("nonexistent.invalid:/remote/path", 22, NULL, NULL, false, NULL, 0);
|
||||||
if (client != NULL) {
|
if (client != NULL) {
|
||||||
client_disconnect(client);
|
client_disconnect(client);
|
||||||
client_delete(client);
|
client_delete(client);
|
||||||
@@ -47,23 +47,13 @@ static void test_ssh_connect_unreachable() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
static void test_ssh_remote_command_argument_modes() {
|
static void test_ssh_remote_command_argument_modes() {
|
||||||
char* command = ssh_build_remote_command("fast sync; touch /tmp/pwned", false, NULL, 0);
|
char* command = ssh_build_remote_command("fast sync; touch /tmp/pwned", NULL, 0);
|
||||||
EXPECT_EQ_STR(command, "'fast sync; touch /tmp/pwned' --stdio");
|
EXPECT_EQ_STR(command, "'fast sync; touch /tmp/pwned' --stdio");
|
||||||
free(command);
|
free(command);
|
||||||
|
|
||||||
command = ssh_build_remote_command("fast'sync", false, NULL, 0);
|
command = ssh_build_remote_command("fast'sync", NULL, 0);
|
||||||
EXPECT_EQ_STR(command, "'fast'\\''sync' --stdio");
|
EXPECT_EQ_STR(command, "'fast'\\''sync' --stdio");
|
||||||
free(command);
|
free(command);
|
||||||
|
|
||||||
/* --old-args no longer disables injection-safe quoting: the path is still one
|
|
||||||
single-quoted word, even when it carries shell metacharacters. */
|
|
||||||
command = ssh_build_remote_command("fast sync; touch /tmp/pwned", true, NULL, 0);
|
|
||||||
EXPECT_EQ_STR(command, "'fast sync; touch /tmp/pwned' --stdio");
|
|
||||||
free(command);
|
|
||||||
|
|
||||||
command = ssh_build_remote_command("fast'sync; rm -rf /", true, NULL, 0);
|
|
||||||
EXPECT_EQ_STR(command, "'fast'\\''sync; rm -rf /' --stdio");
|
|
||||||
free(command);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/* The build for a single-word argv is [prog, six -o args, "--", user, command]. */
|
/* The build for a single-word argv is [prog, six -o args, "--", user, command]. */
|
||||||
@@ -120,12 +110,12 @@ static void test_ssh_build_client_argv_whitespace_command_and_port() {
|
|||||||
* returned and no command can run. */
|
* returned and no command can run. */
|
||||||
static void test_ssh_connect_rejects_option_host() {
|
static void test_ssh_connect_rejects_option_host() {
|
||||||
/* cppcheck-suppress constVariablePointer */
|
/* cppcheck-suppress constVariablePointer */
|
||||||
Client* client = client_connect_ssh("-oProxyCommand=touch /tmp/pwned:/remote", 22, NULL, false,
|
Client* client =
|
||||||
NULL, false, NULL, 0);
|
client_connect_ssh("-oProxyCommand=touch /tmp/pwned:/remote", 22, NULL, NULL, false, NULL, 0);
|
||||||
EXPECT_NULL(client);
|
EXPECT_NULL(client);
|
||||||
client = client_connect_ssh("-evil:/remote", 22, NULL, false, NULL, false, NULL, 0);
|
client = client_connect_ssh("-evil:/remote", 22, NULL, NULL, false, NULL, 0);
|
||||||
EXPECT_NULL(client);
|
EXPECT_NULL(client);
|
||||||
client = client_connect_ssh("user@:/remote", 22, NULL, false, NULL, false, NULL, 0);
|
client = client_connect_ssh("user@:/remote", 22, NULL, NULL, false, NULL, 0);
|
||||||
EXPECT_NULL(client);
|
EXPECT_NULL(client);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -135,13 +125,13 @@ static void test_ssh_connect_rejects_option_host() {
|
|||||||
* ssh_build_remote_command safety boundary for the server path. */
|
* ssh_build_remote_command safety boundary for the server path. */
|
||||||
static void test_ssh_remote_command_with_remote_options() {
|
static void test_ssh_remote_command_with_remote_options() {
|
||||||
char* noop[] = {"--allow-delete"};
|
char* noop[] = {"--allow-delete"};
|
||||||
char* command = ssh_build_remote_command("fastsync-server", false, noop, 1);
|
char* command = ssh_build_remote_command("fastsync-server", noop, 1);
|
||||||
EXPECT_EQ_STR(command, "'fastsync-server' --stdio '--allow-delete'");
|
EXPECT_EQ_STR(command, "'fastsync-server' --stdio '--allow-delete'");
|
||||||
free(command);
|
free(command);
|
||||||
|
|
||||||
/* Multiple options append in order, each as its own quoted word. */
|
/* Multiple options append in order, each as its own quoted word. */
|
||||||
char* multi[] = {"-v", "--allow-delete"};
|
char* multi[] = {"-v", "--allow-delete"};
|
||||||
command = ssh_build_remote_command("srv", false, multi, 2);
|
command = ssh_build_remote_command("srv", multi, 2);
|
||||||
EXPECT_EQ_STR(command, "'srv' --stdio '-v' '--allow-delete'");
|
EXPECT_EQ_STR(command, "'srv' --stdio '-v' '--allow-delete'");
|
||||||
free(command);
|
free(command);
|
||||||
|
|
||||||
@@ -150,29 +140,24 @@ static void test_ssh_remote_command_with_remote_options() {
|
|||||||
break out into an arbitrary remote command. */
|
break out into an arbitrary remote command. */
|
||||||
char* val = strdup("--x=un'der; touch /tmp/pwned");
|
char* val = strdup("--x=un'der; touch /tmp/pwned");
|
||||||
char* dangerous[1] = {val};
|
char* dangerous[1] = {val};
|
||||||
command = ssh_build_remote_command("srv", false, dangerous, 1);
|
command = ssh_build_remote_command("srv", dangerous, 1);
|
||||||
EXPECT_EQ_STR(command, "'srv' --stdio '--x=un'\\''der; touch /tmp/pwned'");
|
EXPECT_EQ_STR(command, "'srv' --stdio '--x=un'\\''der; touch /tmp/pwned'");
|
||||||
free(command);
|
free(command);
|
||||||
free(val);
|
free(val);
|
||||||
|
|
||||||
/* --old-args still quotes both the server path and the remote options. */
|
|
||||||
command = ssh_build_remote_command("srv", true, multi, 2);
|
|
||||||
EXPECT_EQ_STR(command, "'srv' --stdio '-v' '--allow-delete'");
|
|
||||||
free(command);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/* The remote command builder refuses to forward an empty or control-character
|
/* The remote command builder refuses to forward an empty or control-character
|
||||||
* remote option (defense-in-depth independent of the CLI validation). */
|
* remote option (defense-in-depth independent of the CLI validation). */
|
||||||
static void test_ssh_remote_command_rejects_bad_options() {
|
static void test_ssh_remote_command_rejects_bad_options() {
|
||||||
char* empty[] = {""};
|
char* empty[] = {""};
|
||||||
EXPECT_NULL(ssh_build_remote_command("srv", false, empty, 1));
|
EXPECT_NULL(ssh_build_remote_command("srv", empty, 1));
|
||||||
|
|
||||||
char nl = '\n';
|
char nl = '\n';
|
||||||
char* newline[] = {&nl};
|
char* newline[] = {&nl};
|
||||||
EXPECT_NULL(ssh_build_remote_command("srv", false, newline, 1));
|
EXPECT_NULL(ssh_build_remote_command("srv", newline, 1));
|
||||||
|
|
||||||
char* with_null[] = {NULL};
|
char* with_null[] = {NULL};
|
||||||
EXPECT_NULL(ssh_build_remote_command("srv", false, with_null, 1));
|
EXPECT_NULL(ssh_build_remote_command("srv", with_null, 1));
|
||||||
}
|
}
|
||||||
|
|
||||||
void test_transport_ssh() {
|
void test_transport_ssh() {
|
||||||
|
|||||||
+3
-3
@@ -248,7 +248,7 @@ static void test_link_copy_fallback_preserves_xattrs() {
|
|||||||
m.crtime_valid = false;
|
m.crtime_valid = false;
|
||||||
|
|
||||||
bool ok = file_to_disk_secure_link_attrs(dest, basis_dir, "payload", 7, false, &m,
|
bool ok = file_to_disk_secure_link_attrs(dest, basis_dir, "payload", 7, false, &m,
|
||||||
(FileAttrPolicy){true, true, false, false}, false,
|
(FileAttrPolicy){true, true, false, false, true}, false,
|
||||||
xattrs, true, NULL);
|
xattrs, true, NULL);
|
||||||
xattr_list_free(xattrs);
|
xattr_list_free(xattrs);
|
||||||
EXPECT_TRUE(ok);
|
EXPECT_TRUE(ok);
|
||||||
@@ -369,7 +369,7 @@ static void test_fake_super_restore() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/* No xattr present yet: restore is a silent no-op (returns false, no crash). */
|
/* No xattr present yet: restore is a silent no-op (returns false, no crash). */
|
||||||
FileAttrPolicy policy = {true, true, false, false};
|
FileAttrPolicy policy = {true, true, false, false, true};
|
||||||
EXPECT_FALSE(fake_super_restore_fd(fd, policy));
|
EXPECT_FALSE(fake_super_restore_fd(fd, policy));
|
||||||
|
|
||||||
fake_super_store_fd(fd, 1001, 1002, 0751, 1700000000, 123456789);
|
fake_super_store_fd(fd, 1001, 1002, 0751, 1700000000, 123456789);
|
||||||
@@ -423,7 +423,7 @@ static void test_fake_super_no_real_chown() {
|
|||||||
fake_super_store_fd(fd, 12345, 12346, 0755, 1700000000, 0);
|
fake_super_store_fd(fd, 12345, 12346, 0755, 1700000000, 0);
|
||||||
|
|
||||||
Config* c = config_create();
|
Config* c = config_create();
|
||||||
FileAttrPolicy policy = {true, true, false, false};
|
FileAttrPolicy policy = {true, true, false, false, true};
|
||||||
EXPECT_NOT_NULL(c);
|
EXPECT_NOT_NULL(c);
|
||||||
/* The strongest ownership request available plus permitted super mode. */
|
/* The strongest ownership request available plus permitted super mode. */
|
||||||
c->preserve_owner = true;
|
c->preserve_owner = true;
|
||||||
|
|||||||
Reference in new issue
Block a user