Compare commits
55
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c062a0762e | ||
|
|
283f9f0823 | ||
|
|
5754b9a952 | ||
|
|
b8a0efef7b | ||
|
|
2e77c09447 | ||
|
|
06c4026b74 | ||
|
|
9f47b13712 | ||
|
|
b1eddf0133 | ||
|
|
338c27db73 | ||
|
|
8d46a26c04 | ||
|
|
707ba272df | ||
|
|
bc71a3c0a5 | ||
|
|
cee9b7647c | ||
|
|
3adb6dddb5 | ||
|
|
d77849774e | ||
|
|
b5c6f8b60f | ||
|
|
134dcd74cc | ||
|
|
42f2f845ac | ||
|
|
0669335ca5 | ||
|
|
391f76cd56 | ||
|
|
2021afe613 | ||
|
|
ba914e8ab3 | ||
|
|
df8fe1ae4e | ||
|
|
2c490d58b7 | ||
|
|
d55dabff2e | ||
|
|
b478a59a81 | ||
|
|
865941f850 | ||
|
|
221cefa7cc | ||
|
|
bb9b59024a | ||
|
|
5baf120243 | ||
|
|
84b7e1fd2f | ||
|
|
800a978e15 | ||
|
|
0f40e747f0 | ||
|
|
b07306d5bc | ||
|
|
f2c89b6e7c | ||
|
|
379f127859 | ||
|
|
3799200f71 | ||
|
|
91c4a967e6 | ||
|
|
88c8968b4c | ||
|
|
082b31886a | ||
|
|
423a62e691 | ||
|
|
bc18ae205b | ||
|
|
10a61c6101 | ||
|
|
bc1e1191af | ||
|
|
0fbb9de915 | ||
|
|
9b05972375 | ||
|
|
d119f35066 | ||
|
|
00829fd265 | ||
|
|
ff261bc38a | ||
|
|
b235721f8b | ||
|
|
79a28cdb96 | ||
|
|
402cae80ad | ||
|
|
9691dba6f0 | ||
|
|
558782d339 | ||
|
|
5597e74f6a |
@@ -12,3 +12,12 @@ build_docker2/
|
||||
# Test/run artifacts
|
||||
root/
|
||||
test_partial_install_tmp/
|
||||
|
||||
# Editor/tooling + test caches/artifacts
|
||||
.pytest_cache/
|
||||
*.gcda
|
||||
*.gcno
|
||||
*.gcov
|
||||
di/
|
||||
test_data-manual/
|
||||
*.log
|
||||
|
||||
@@ -4,9 +4,9 @@ FastSync is a high-performance file synchronization system written in C11. It su
|
||||
|
||||
## Dependency installation
|
||||
|
||||
**CI rule:** never add `apt-get install` / `pip install` steps to CI workflows — use the custom Docker image instead. The image is built from the repo-root `Dockerfile` and is the same image CI uses: `gitea.tap-tap.win/taptap/fastsync-ci:v11`. It contains the full toolchain: gcc/g++, CMake, libzstd-dev, libssl-dev, make, git, cppcheck, clang-format, python3 + pytest + pytest-xdist, openssh-client, Node.js, plus `rsync` 3.4.1 (with zstd/xxhash/lz4), `acl` and `attr` (setfacl/getfacl, setfattr/getfattr) for drop-in parity tests.
|
||||
**CI rule:** never add `apt-get install` / `pip install` steps to CI workflows — use the custom Docker image instead. The image is built from the repo-root `Dockerfile` and is the same image CI uses: `gitea.tap-tap.win/taptap/fastsync-ci:v11`. It contains the full toolchain: gcc/g++, CMake, libzstd-dev, zlib1g-dev, liblz4-dev, libxxhash-dev, libssl-dev, make, git, cppcheck, clang-format, python3 + pytest + pytest-xdist, openssh-client, Node.js, plus `rsync` 3.4.1 (with zstd/xxhash/lz4), `acl` and `attr` (setfacl/getfacl, setfattr/getfattr) for drop-in parity tests. (CMake hard-requires zstd, zlib, and lz4; xxHash is fetched via `FetchContent`.)
|
||||
|
||||
**Host rule:** for local development, use `nix-shell` (see `README.md`) which provides zstd, OpenSSL, CMake, and gcc. The Docker image can also be used locally for CI parity.
|
||||
**Host rule:** for local development, use `nix-shell` (see `README.md`) which provides zstd, zlib, lz4, OpenSSL, CMake, and gcc. The Docker image can also be used locally for CI parity.
|
||||
|
||||
```bash
|
||||
# Use the prebuilt CI image directly (faster, guaranteed CI parity)
|
||||
@@ -38,11 +38,12 @@ If a dependency is missing from the CI image, add it to the `Dockerfile` (and re
|
||||
When configuring for CI parity, use:
|
||||
```bash
|
||||
cmake -B build -S . -DSTRICT_WARNINGS=ON # -Wextra -Wpedantic -Werror
|
||||
cmake -B build -S . -DSANITIZER=address # AddressSanitizer (ASan)
|
||||
cmake -B build -S . -DSANITIZER=thread # ThreadSanitizer (TSan)
|
||||
cmake -B build -S . -DSANITIZER=address # AddressSanitizer (ASan); in the CI matrix
|
||||
cmake -B build -S . -DSANITIZER=undefined # UndefinedBehaviorSanitizer (UBSan); in the CI matrix
|
||||
cmake -B build -S . -DSANITIZER=thread # ThreadSanitizer (TSan); local-only, NOT in CI
|
||||
```
|
||||
|
||||
The CI workflow (`.gitea/workflows/ci.yaml`) runs lint (clang-format, cppcheck), then a **fast PR gate** — build + unit + a representative subset of integration tests marked `@pytest.mark.ci`, parallelized with pytest-xdist (`-n 4 --dist=load`). The full coverage jobs (full integration suite as `-m "not setpriv"`, sanitizer, fuzz, coverage, valgrind) run **only on push to `dev`/`main`**; pull requests skip them to keep PR CI under ~3 minutes. The two `setpriv` privilege tests are excluded from CI via a marker because their result depends on the runner/container uid and host mount permissions.
|
||||
The CI workflow (`.gitea/workflows/ci.yaml`) runs lint (clang-format, cppcheck), then a **fast PR gate** — build + unit + a representative subset of integration tests marked `@pytest.mark.ci`, parallelized with pytest-xdist (`-n 4 --dist=load`). The full coverage jobs (full integration suite as `-m "not setpriv"`, the `address`+`undefined` sanitizer matrix, fuzz, coverage, valgrind) run **only on push to `dev`/`main`**; pull requests skip them to keep PR CI under ~3 minutes. TSan is not part of the CI matrix and is a local-only configuration. The `setpriv`-marked privilege tests (four decorated functions, collecting to eight instances because two are parametrized) are excluded from CI via a marker because their result depends on the runner/container uid and host mount permissions.
|
||||
|
||||
## Build
|
||||
|
||||
@@ -105,7 +106,7 @@ Two main branches: `dev` (integration) and `main` (stable releases).
|
||||
|
||||
### Rules
|
||||
- **All PRs target `dev`** — never target `main` directly
|
||||
- **`dev` is the default branch** in Gitea repo settings
|
||||
- **`dev` is intended to be the default branch** in Gitea repo settings — verify in the repo settings, since this clone's `origin/HEAD` still points at `main`
|
||||
- **`main` is protected** — only merged from `dev` via PR with 2 approvals + full CI pass
|
||||
- **Feature/bug branches** branch from `dev`, PR back to `dev`
|
||||
- **`dev` → `main` merges** happen on-demand or weekly, requiring full CI + review
|
||||
|
||||
+131
@@ -4,6 +4,137 @@ All notable changes to FastSync are documented here. Versions match
|
||||
`PROTOCOL_VERSION` (printed by `fastsync --version`); the client and server must
|
||||
run the same version because the handshake is strict.
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
The rsync-parity cycle 2.29 (no wire change; `PROTOCOL_VERSION` stays 2.28.0).
|
||||
`RSYNC_COMPAT.md` moves from **116 ✅ / 14 ⚠️ / 27 ❌** to
|
||||
**120 ✅ / 10 ⚠️ / 27 ❌** of 157 rows.
|
||||
|
||||
An audit cycle follows on the same wire version (`PROTOCOL_VERSION` stays
|
||||
2.28.0): a security-and-correctness pass over the parity-2.29 baseline, plus a
|
||||
set of audit follow-ups (filter merge modifiers, the `--inplace`/`--partial-dir`
|
||||
conflict, credential-file hardening, and small leak/log/test fixes). It fixes
|
||||
a `--temp-dir` symlink escape, gates client-controlled special permission bits,
|
||||
corrects `--partial-dir`/`--bwlimit`/`-z` behavior, handles unsupported filter
|
||||
modifiers, and tightens client and wire validation. The only parity
|
||||
reclassification is `--filter=RULE` moving ✅ → ⚠️, because its merge-only
|
||||
`e`/`n`/`w`/`-` modifiers are now accepted and consumed but their semantics
|
||||
remain unimplemented (accepted-but-ignored); the matrix is therefore **119 ✅ /
|
||||
11 ⚠️ / 27 ❌** of 157 rows. The affected rows' notes and the summary tally in
|
||||
`RSYNC_COMPAT.md` were updated.
|
||||
|
||||
### Changed
|
||||
|
||||
- **rsync-exact traversal order.** The sequential scanner now walks each
|
||||
directory's entries in rsync 3.4.1's flist order (non-directories ascending,
|
||||
then directories ascending, depth-first), so `--info=name`, the
|
||||
`--delete-during`/`--delete-delay`/`-n` would-delete order and the partial
|
||||
`--max-delete` survivor set match rsync byte-for-byte. `--threads` has no
|
||||
rsync analogue and stays unordered.
|
||||
- **Delete timing.** The complete `--delete-during`/`--delete-delay`
|
||||
per-directory plan set is transmitted before the first data frame, so a
|
||||
mid-transfer abort has already removed every planned extra like rsync's
|
||||
generator; `-d/--dirs` uses per-directory plans (shielded untraversed
|
||||
subdirectories) instead of the end-of-transfer commit. `-n`, `--delete`,
|
||||
`--del`/`--delete-during` and `--delete-delay` are now ✅ Parity.
|
||||
- **Basis directories.** A relative `--compare-dest`/`--copy-dest`/`--link-dest`
|
||||
DIR resolves against the destination directory with the transfer-relative
|
||||
name appended, exactly like rsync 3.4.1.
|
||||
- **`-y`/`--fuzzy`.** The candidate search no longer inherits the ordinary delta
|
||||
engine's 16 KiB minimum or 10× size-ratio bound, so an oversized or
|
||||
sub-16-KiB sibling is reused exactly as rsync reuses it.
|
||||
- `--info=mount` prints rsync's mount-point skip line (repeated `-xx` drops the
|
||||
mount-point directory); `--info=stats` enables the `--stats` block; `-x` is
|
||||
repeatable. `--stats` counts traversed directories for the `Number of files`
|
||||
breakdown under a plain `-r` scan. `--debug` emits real output for
|
||||
`flist`/`del`/`hash`/`deltasum`/`recv`/`filter`/`send`.
|
||||
|
||||
### Known residuals
|
||||
|
||||
- `--progress` and `--info` still need a receiver→sender event channel for the
|
||||
root `./` line, ancestor-directory suppression, receiver-side `skip`/`backup`
|
||||
wording, and symlink/empty-directory quick-checks.
|
||||
- `--delete-before`'s phase-0 late-file divergence remains (rsync's pre-scan
|
||||
fixes the file list before the data pass).
|
||||
- A single file larger than 256 MiB cannot be streamed in the default path
|
||||
(a general whole-file limit, not basis-specific).
|
||||
- `--stats` byte totals and `--msgs2stderr` stay documented divergences.
|
||||
|
||||
### Security
|
||||
|
||||
- **`--temp-dir` symlink escape fixed.** The receiver's scratch directory was
|
||||
opened with a bare `open()`, so a symlink planted under the receive root could
|
||||
redirect receiver scratch files outside the authorized root. The opened
|
||||
directory is now judged by the real path of its fd (`/proc/self/fd` via
|
||||
`realpath`) and an escaping target is refused (`EACCES`, logged); an in-root
|
||||
link to another filesystem (the `EXDEV` fallback case) still works.
|
||||
- **Client-controlled special bits masked when super-user activities are not
|
||||
permitted.** Setuid/setgid/sticky bits (`--perms`, `--chmod`, the symlink and
|
||||
special-node paths, and deferred directory modes) are now stripped when the
|
||||
connection forbids super activities (`--no-super`, a non-opted daemon module,
|
||||
a privileged listener without `--allow-super`); exact rsync semantics are
|
||||
preserved wherever super activities are permitted.
|
||||
- **Daemon umask no longer forced to `0`.** `daemonize()` now sets the
|
||||
conventional `022`, so implied parent directories created without `-p` are no
|
||||
longer world-writable `0777`.
|
||||
- **Credentials and signal handling hardened.** Secret files are opened with
|
||||
`O_NOFOLLOW|O_NONBLOCK` (while allowing fd-backed store paths and bound-waiting
|
||||
a FIFO read for ~3 s so a slow process substitution works but a connected-but-
|
||||
silent FIFO cannot hang), and signal handlers use `sigaction` with
|
||||
async-signal-safe bodies.
|
||||
|
||||
### Fixed
|
||||
|
||||
- **`-z` on 100–256 MiB files.** The decompressor's internal ceiling was 100 MiB
|
||||
while the receiver advertises and the sender compresses whole files up to
|
||||
`MAX_RECEIVE_WHOLE_FILE_SIZE` (256 MiB), so `-z` on a 100–256 MiB regular file
|
||||
failed with `Declared decompressed size exceeds 104857600 bytes`. The ceiling
|
||||
is now defined in terms of the protocol whole-file bound (still an
|
||||
allocation-clamped bomb guard).
|
||||
- **`--bwlimit` now paces `--sendfile`.** The plaintext-TCP `--sendfile` fast
|
||||
path bypassed the protocol's token bucket, so the limit was ignored there. It
|
||||
now throttles through the same per-session leaky bucket as the TLS path.
|
||||
- **`--partial-dir` implies `--partial`.** Matching rsync 3.4.1 (which sets
|
||||
`keep_partial` after option parsing), `--partial-dir=DIR` alone retains an
|
||||
interrupted transfer's partial and wins over an explicit `--no-partial`;
|
||||
`--inplace` still bypasses the partial machinery, and combining `--inplace`
|
||||
with `--partial-dir` is now rejected up front with rsync's message
|
||||
(`--inplace cannot be used with --partial-dir`).
|
||||
- **Filter modifiers handled.** The `x` xattr-name modifier is rejected with a
|
||||
clear error everywhere. The merge-only `e`/`n`/`w` and `-` modifiers are now
|
||||
accepted and consumed on `merge`/`dir-merge` rules (so they no longer leak
|
||||
into the merge filename) while still being rejected on non-merge rules,
|
||||
matching rsync; their semantics remain unimplemented (accepted-but-ignored).
|
||||
Glued patterns (`-newfile`, `-e2e`) and mixed tokens (`H,!secret`) keep their
|
||||
historical parsing.
|
||||
- **Credential-file reads hardened.** Secret files (`--password-file`/
|
||||
`--early-input`/`--hash-credentials` input) are opened with `O_NOFOLLOW`, so a
|
||||
symlinked credential path now fails closed (`ELOOP`) instead of being followed
|
||||
before the owner/mode gate; literal fd-backed paths (`/dev/fd/<digits>`,
|
||||
`/proc/self/fd/<digits>`) are exempt so process substitution still works. A
|
||||
FIFO/process-substitution read now waits under a bounded ~3 s deadline for its
|
||||
writer, so a slow producer works while a connected-but-silent FIFO fails
|
||||
instead of hanging.
|
||||
- **Miscellaneous correctness fixes:** `--filter` rule count is checked
|
||||
client-side against `MAX_FILTER_RULES` before any network I/O (the receiver
|
||||
still re-checks the expanded count); unknown wire `Status` values are rejected
|
||||
as protocol errors; a mutex leak on an init-failure path, an `errno` read
|
||||
after `free()` in deferred delete application, `log_perror` misuse for
|
||||
non-`errno` conditions, and a `NULL` `server_host`/`ssh_destination`
|
||||
allocation path were fixed (the `config_create` failure now releases through
|
||||
`config_delete`); the decompression-limit log now prints the effective bound
|
||||
rather than the compile-time ceiling; the daemon umask and root test fixtures
|
||||
were hardened; `SSL_read` length is clamped and `sendfile` `poll()` retries on
|
||||
`EINTR`.
|
||||
|
||||
### Refactored / Docs
|
||||
|
||||
- Dropped dead `filter_rules_apply` and dead `--old-args` plumbing, unified
|
||||
`set_error`, deduplicated `path_is_within` and shared constants, and added
|
||||
printf format attributes (fixing format mismatches). `RSYNC_COMPAT.md`,
|
||||
`CHANGELOG.md` and `HANDOFF.md` were updated for the audit cycle; the
|
||||
`RSYNC_COMPAT.md` summary tally was corrected to match the rows.
|
||||
|
||||
## [2.28.0] - 2026-09-20
|
||||
|
||||
The rsync-parity cycle. `PROTOCOL_VERSION` moves `2.26.0 → 2.27.0 → 2.28.0`;
|
||||
|
||||
+3
-2
@@ -26,9 +26,9 @@ elseif(NOT SANITIZER STREQUAL "none")
|
||||
endif()
|
||||
|
||||
# --- Strict warnings option ---
|
||||
option(STRICT_WARNINGS "Enable strict warnings (Wextra, Wpedantic, Werror)" OFF)
|
||||
option(STRICT_WARNINGS "Enable strict warnings (Wextra, Wpedantic, Wformat-signedness, Werror)" OFF)
|
||||
if(STRICT_WARNINGS)
|
||||
add_compile_options(-Wextra -Wpedantic -Werror)
|
||||
add_compile_options(-Wextra -Wpedantic -Wformat-signedness -Werror)
|
||||
endif()
|
||||
|
||||
# --- Coverage option ---
|
||||
@@ -226,6 +226,7 @@ set(TEST_SRCS
|
||||
tests/test_file.c
|
||||
tests/test_file_list.c
|
||||
tests/test_file_sendfile.c
|
||||
tests/test_filter.c
|
||||
tests/test_format.c
|
||||
tests/test_fuzz_smoke.c
|
||||
tests/test_glob.c
|
||||
|
||||
+62
-23
@@ -1,18 +1,30 @@
|
||||
# FastSync — Session Handoff (2026-09-19)
|
||||
# FastSync — Session Handoff (2026-09-21)
|
||||
|
||||
## Current status
|
||||
- **rsync-parity tracks 1-6 landed on `dev`** via **PR #303** (`10159dc`,
|
||||
"feat(parity): rsync parity tracks 1-6 (protocol 2.28.0)"). Dev push CI run
|
||||
**581** fully green: lint, build-and-test, parity-full, ASan, UBSan,
|
||||
fuzz-build, coverage, valgrind.
|
||||
- **Release `v2.28.0`** is tagged and merged to `main`: tag `v2.28.0` points at
|
||||
`ee6523a`, and the PR #304 merge commit `b4d54504` is on `main`.
|
||||
- **`dev` is at `0fbb9de`** — the merge of parity cycle 2.29 (PR #305). The old
|
||||
`558782d` (incremental-check flake fix) is an ancestor.
|
||||
- **`PROTOCOL_VERSION` = `"2.28.0"`** (`src/shared/config.h`); CMake
|
||||
`project(FastFileTransfer VERSION 2.28.0)`. The cycle batched all wire
|
||||
changes (stats counters, filter-rule block, `--verify-basis`) under the one
|
||||
bump.
|
||||
- **`main` = `ef76c90`** (tag `v2.26.0`); the 2.27.0/2.28.0 work is on `dev`
|
||||
and not yet released. A `dev -> main` v2.28.0 release PR is the next step.
|
||||
- Parity matrix: **116 ✅ / 14 ⚠️ / 27 ❌ = 157** (was 111/13/33 at cycle start).
|
||||
- Working tree clean; feature branch deleted; no scratch trees or worktrees.
|
||||
`project(FastFileTransfer VERSION 2.28.0)`.
|
||||
- **Parity cycle 2.29 is merged to `dev`** (PR #305), no wire change. It closed
|
||||
the scanner-order, delete-timing, relative-basis and fuzzy-eligibility
|
||||
residuals and improved the `--info`/`--stats`/`--debug` partials. Parity
|
||||
matrix: **120 ✅ / 10 ⚠️ / 27 ❌ = 157**. Remaining ⚠️ rows: `--info`,
|
||||
`--debug`, `--msgs2stderr`, `--stats`, `--progress`, `--delete-before`, the
|
||||
three basis-dir options, and `-y`/`--fuzzy`.
|
||||
- **Audit cycle complete on branch `fix/audit-cycle`** (branched from `dev` @
|
||||
`0fbb9de`), integration PR to `dev` pending. No wire change
|
||||
(`PROTOCOL_VERSION` stays 2.28.0). It lands the receiver/client security and
|
||||
correctness fixes — `--temp-dir` symlink-escape confinement, special-bit
|
||||
masking under a super-off policy, daemon `umask(022)`, the `-z` decompression
|
||||
ceiling raised to the 256 MiB whole-file bound, `--bwlimit` pacing the
|
||||
plaintext `--sendfile` path, `--partial-dir` implying `--partial`, rejection
|
||||
of unsupported filter modifiers (`x`/`e`/`n`/`w`), client-side
|
||||
`MAX_FILTER_RULES` enforcement, unknown wire `Status` rejection, and the
|
||||
accompanying refactors/docs. The parity matrix is unchanged at
|
||||
**120 ✅ / 10 ⚠️ / 27 ❌ = 157**; this docs pass (worktree `fix/audit-docs2`)
|
||||
corrects the `RSYNC_COMPAT.md` summary tally to match the rows.
|
||||
|
||||
|
||||
## What landed this session
|
||||
@@ -98,7 +110,8 @@
|
||||
uptodate` plus the leading `./` root name line for `--info=name` (only the
|
||||
root-line trigger condition and receiver-side `skip` wording remain). Matrix
|
||||
now **111 ✅ / 14 ⚠️ / 32 ❌ = 157**; differential + unit tests added in
|
||||
`test_features.py`, `test_option_parity.py`, `test_delete_plan.c`,
|
||||
`test_features.py`, `test_option_parity.py`, the unit test
|
||||
`tests/test_delete_plan.c`,
|
||||
`test_delete_delay_budget_parity.py`, `test_delete_timing_parity.py`.
|
||||
12. **No-wire parity track 2b** on `feat/parity-2.28` (no protocol change):
|
||||
`--progress`/`-P`/`--info=progress` (when not `--quiet`) now run an opt-in
|
||||
@@ -195,17 +208,43 @@
|
||||
**116 ✅ / 14 ⚠️ / 27 ❌ = 157** (the `--delete`/`--delete-during` rows stay
|
||||
⚠️ for the abort boundary; `--delete-after` stays ✅).
|
||||
|
||||
17. **Audit cycle** on `fix/audit-cycle` (from `dev` @ `0fbb9de`;
|
||||
`PROTOCOL_VERSION` stays `2.28.0`): a security/correctness pass over the
|
||||
parity-2.29 baseline. It raises the decompression ceiling to the 256 MiB
|
||||
protocol whole-file bound (`-z` on 100–256 MiB files now works), paces the
|
||||
plaintext-TCP `--sendfile` path with `--bwlimit`, confines the `--temp-dir`
|
||||
scratch dir by the fd's real path (symlink escape refused), masks
|
||||
client-controlled setuid/setgid/sticky bits when super activities are not
|
||||
permitted, sets the daemon umask to `022`, makes `--partial-dir` imply
|
||||
`--partial`, rejects the unsupported filter modifiers (`x`/`e`/`n`/`w`),
|
||||
enforces `MAX_FILTER_RULES` client-side, rejects unknown wire `Status`
|
||||
values, and hardens credentials/signal handling (with the accompanying
|
||||
refactors and docs). No row changes classification, so the matrix stays
|
||||
**120 ✅ / 10 ⚠️ / 27 ❌ = 157**. This docs pass is on `fix/audit-docs2`.
|
||||
|
||||
## Next steps
|
||||
1. **Merge PR #284** (`dev` -> `main`) once reviewed (protected branch).
|
||||
2. **Deferred security items** (documented, not implemented):
|
||||
- Pre-auth config/daemon-auth handshake has no aggregate wall-clock deadline
|
||||
(per-message timeout only) — slowloris holds connection slots.
|
||||
- Per-source registry fails open when the shared table is full (per-module/global
|
||||
caps and host ACLs still apply); consider fail-closed or larger/evicting table.
|
||||
- SCRAM-like daemon auth has no TLS channel binding (and is not RFC 5802).
|
||||
- `cleanup()` signal handler calls non-async-signal-safe teardown; daemon `umask(0)`.
|
||||
- Wire protocol assumes homogeneous word size/endianness (lengths are native
|
||||
`size_t`) — document or move to fixed-width framing.
|
||||
1. **Open and merge the audit-cycle PR** (`fix/audit-cycle`, including this
|
||||
`fix/audit-docs2` docs pass) into `dev` once reviewed. `dev` is the default
|
||||
branch; all PRs target `dev`, never `main` directly.
|
||||
2. **Remaining deferred items:**
|
||||
- **Large structural refactors:** delete-engine consolidation
|
||||
(`delete_extras_fd`/`manifest_delete_extras`/the delete-plan path),
|
||||
god-function splits, and translation-unit splits.
|
||||
- **`--progress`/`--info` receiver→sender event channel:** the root `./`
|
||||
line, ancestor-directory suppression, receiver-side `skip`/`backup` echo,
|
||||
and symlink/empty-dir quick-check feedback.
|
||||
- **`--delete-before` phase-0 keep-set** (rsync fixes the file list before
|
||||
the data pass; FastSync keeps its pre-scan snapshot race).
|
||||
- **>256 MiB single-file streaming** (B4, the general whole-file limit).
|
||||
- **Wire native-size framing:** lengths are native `size_t` and the protocol
|
||||
assumes homogeneous word size/endianness — document or move to fixed-width
|
||||
framing.
|
||||
- **SCRAM-like daemon auth channel binding:** no TLS channel binding today
|
||||
(and it is not RFC 5802).
|
||||
- Still-open security nits: the pre-auth config/daemon-auth handshake has no
|
||||
aggregate wall-clock deadline (per-message timeout only — slowloris holds
|
||||
connection slots); the per-source registry fails open when the shared table
|
||||
is full (per-module/global caps and host ACLs still apply).
|
||||
3. **Out of scope / intentional:** pull (remote source) mode is **not** planned —
|
||||
FastSync is push-only; see `RSYNC_COMPAT.md#direction`.
|
||||
|
||||
|
||||
@@ -75,8 +75,9 @@ matrix is classified as parity, caveat, or divergent in
|
||||
owner, group, devices, and special files — and does not imply compression or
|
||||
multithreading (see [Client](#client)). Ownership application is still
|
||||
privilege-gated: a receiver that cannot `chown` logs a warning and skips it.
|
||||
Under `-p` the source mode is copied exactly, including setuid/setgid/sticky
|
||||
and group/other-write bits (strict rsync parity; see
|
||||
Under `-p` the source mode is copied exactly, including group/other-write
|
||||
bits; setuid/setgid/sticky bits are copied only when super-user activities are
|
||||
permitted, and are masked under `SUPER_MODE_OFF`/`--no-super` (see
|
||||
[`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)).
|
||||
- Symlink transfer stores targets **verbatim** (`-l`/`--links`), including
|
||||
absolute and `..`-bearing targets, matching rsync. The receiver does not
|
||||
@@ -172,7 +173,7 @@ This produces `./build/client` and `./build/server`. `compile_commands.json` is
|
||||
| `--preserve` | Preserve mode and mtime (`-p` + `-t`; add `-o`/`-g` for owner/group or `-U`/`--atimes` for atime; `-N`/`--crtimes` captures birth time but cannot apply it) |
|
||||
| `-U, --atimes` | Preserve access times. Captured with the metadata payload; does not enable ownership. |
|
||||
| `-N, --crtimes` | Capture birth time; cannot be applied (documented divergence) |
|
||||
| `-p, --perms` | Preserve permission bits. Strict rsync parity: the source mode is copied exactly, including setuid/setgid/sticky and group/other-write bits |
|
||||
| `-p, --perms` | Preserve permission bits. The source mode is copied exactly, including group/other-write bits; setuid/setgid/sticky are copied only when super-user activities are permitted (`SUPER_MODE_OFF`/`--no-super` masks them) |
|
||||
| `-t, --times` | Preserve modification times |
|
||||
| `-o, --owner` | Preserve the source owner (privilege-gated; mapped by name on the receiver with a numeric fallback) |
|
||||
| `-g, --group` | Preserve the source group (privilege-gated; mapped by name on the receiver with a numeric fallback) |
|
||||
@@ -204,9 +205,10 @@ This produces `./build/client` and `./build/server`. `compile_commands.json` is
|
||||
| `-u, --update` | Skip files newer than the source on the receiver |
|
||||
| `--incremental` | Skip files unchanged since last transfer (size + mtime). Auto-enables `--preserve`. Incompatible with `--chunk-serialization`. |
|
||||
| `--existing` | Skip files not already present at the destination; update existing files normally. |
|
||||
| `--compare-dest <dir>` | Extra comparison basis: unchanged files are not transferred (requires/implies `--incremental`) |
|
||||
| `--copy-dest <dir>` | Like `--compare-dest`, but copies the unchanged file from DIR into the destination |
|
||||
| `--link-dest <dir>` | Like `--copy-dest`, but hard-links the unchanged file from DIR (repeatable; earlier DIRs win) |
|
||||
| `--ignore-existing` | Skip files that already exist on the receiver; like rsync it does not apply to directories or symlinks. |
|
||||
| `--compare-dest <dir>` | Extra comparison basis: unchanged files are not transferred (requires/implies `--incremental`; a basis MISS above the 256 MiB whole-file payload bound is refused — see [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)) |
|
||||
| `--copy-dest <dir>` | Like `--compare-dest`, but copies the unchanged file from DIR into the destination (same basis-size caveat; see [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)) |
|
||||
| `--link-dest <dir>` | Like `--copy-dest`, but hard-links the unchanged file from DIR (repeatable; earlier DIRs win; same basis-size caveat; see [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)) |
|
||||
| `--verify-basis` | FastSync-only: require a basis hit (`--compare-dest`/`--copy-dest`/`--link-dest`) to match the source by whole-file digest instead of trusting the size+mtime quick-check (default matches rsync) |
|
||||
| `--delete` | Delete files on receiver not present in source (default timing: delete-during, matching rsync, so destination space is freed progressively). Scoped to the synchronized directories, so `--files-from` subsets are safe |
|
||||
| `--delete-before` | Delete extras before the transfer starts (implies `--delete`) |
|
||||
@@ -216,16 +218,16 @@ This produces `./build/client` and `./build/server`. `compile_commands.json` is
|
||||
| `--delete-commit` | FastSync-only: keep the pre-2.28 atomic timing — delete only after the whole transfer succeeded (identical timing to `--delete-after`) |
|
||||
| `--delete-excluded` | Also delete filter-excluded destination mirrors (size-pruned mirrors stay protected) |
|
||||
| `--max-delete <n>` | Delete at most n destination entries; the rest are skipped and the run exits 25 (partial), matching rsync |
|
||||
| `--delay-updates` | Put updated files into place only at the end of the transfer (`--force` is honored at publication) |
|
||||
| `--delay-updates` | Put updated files into place only at the end of the transfer (`--force` is honored at publication; the fixed `.fastsync-stage` staging name diverges from rsync — see [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)) |
|
||||
| `-T, --temp-dir <dir>` | Scratch directory for temp files before the atomic install; confined to the receive root (relative only), with an `EXDEV` non-atomic copy fallback |
|
||||
| `-n, --dry-run` | Report what would be transferred without mutating the destination. Since protocol 2.21.0 a server-routed target contacts the receiver and reports would-transfer based on receiver state; a plain local destination keeps the client-side scan. Never mutates or deletes. |
|
||||
| `-v, --verbose` | Enable debug logging |
|
||||
| `-q, --quiet` | Suppress non-error output |
|
||||
| `--progress` | Show rsync-style per-file progress blocks from the receiver's wire counters (FastSync does not print rsync's leading `./` line) |
|
||||
| `--progress` | Show rsync-style per-file progress blocks from the receiver's wire counters; the root `./` line is printed whenever progress is active (rsync prints it only when the transfer root is created) |
|
||||
| `-P` | Enables partial-transfer mode + progress output; interrupted writes retain the already-written temp for resumption |
|
||||
| `--stats` | Print transfer statistics at end (bytes, files, timing), including the receiver-only counters reported over the wire; rsync's per-type `Number of files` breakdown is not reproduced |
|
||||
| `--stats` | Print transfer statistics at end (bytes, files, timing), including the receiver-only counters reported over the wire; `Number of files` and `Number of created files` carry rsync's per-type breakdown (deleted files are reported as a single total) |
|
||||
| `-i, --itemize-changes` | Print an rsync-style per-file change line |
|
||||
| `--out-format=FORMAT` | Output format for changed files (`%f %n %l %b %M %%`) |
|
||||
| `--out-format=FORMAT` | Output format for changed files (`%f %n %l %b %c %C %i %M %%`) |
|
||||
| `--list-only` | List source files instead of transferring |
|
||||
| `--fsync` | Fsync every written file before publication |
|
||||
| `-h, --human-readable` | Format transfer byte/rate counts with rsync's decimal (base-1000) units |
|
||||
@@ -246,7 +248,7 @@ This produces `./build/client` and `./build/server`. `compile_commands.json` is
|
||||
| `-4, --ipv4` | Force IPv4 for destination resolution |
|
||||
| `-6, --ipv6` | Force IPv6 for destination resolution |
|
||||
| `--sockopts=OPTS` | Comma-separated OPT=VAL socket options applied before connect (`TCP_NODELAY`, `SO_KEEPALIVE`, `SO_RCVBUF`, `SO_SNDBUF`, `SO_REUSEADDR`) |
|
||||
| `--bwlimit <KB/s>` | Bandwidth limit in kilobytes per second |
|
||||
| `--bwlimit <KB/s>` | Bandwidth limit in kilobytes per second; also paces `--sendfile` transfers |
|
||||
| `--chunk-size <n>` | Chunk size in bytes (default: 10485760) |
|
||||
| `--timeout <sec>` | I/O timeout in seconds, applied to both the socket (`SO_RCVTIMEO`/`SO_SNDTIMEO`) and the per-message protocol poll deadline. Default `0` = disabled (matching rsync); `0` disables it. `--no-timeout` is the negation. The value is not sent on the wire; the server side keeps its own safe floor. |
|
||||
| `--contimeout <sec>` | Connection timeout in seconds (default: 60, matching rsync); `0` disables it (`--no-contimeout` is the negation) |
|
||||
@@ -314,17 +316,22 @@ transfer is never aborted.
|
||||
`timeout`, `contimeout`, `quiet`, `stats`, `max_depth`, and `log_file` are
|
||||
client-only.
|
||||
5. **Queue** — thread-safe bounded queue with condition variables.
|
||||
6. **DirectoryScanner** — recursive BFS traversal with exclude and include
|
||||
pattern support, max-depth enforcement.
|
||||
6. **DirectoryScanner** — recursive traversal that buffers and sorts each
|
||||
directory (non-directories ascending, then directories ascending) and walks
|
||||
depth-first in rsync flist order, with exclude and include pattern support and
|
||||
max-depth enforcement.
|
||||
|
||||
### Key Algorithms
|
||||
|
||||
1. **File scanning** — BFS directory traversal; entries matched against exclude
|
||||
and include patterns, with max-depth enforced.
|
||||
1. **File scanning** — sorted depth-first traversal in rsync flist order (each
|
||||
directory's non-directories ascending, then its directories ascending);
|
||||
entries matched against exclude and include patterns, with max-depth
|
||||
enforced. The `--threads` parallel scanner remains unordered.
|
||||
2. **Chunking** — files accumulated until the `chunk_size` threshold (default
|
||||
10 MiB) is reached, then flushed.
|
||||
3. **Compression** — streaming zstd via `ZSTD_compressStream2()` /
|
||||
`ZSTD_decompressStream()`.
|
||||
`ZSTD_decompressStream()`, with lz4 and zlib/zlibx codecs also supported
|
||||
(selectable with `--compress-choice`).
|
||||
4. **Network protocol** — status-code-driven exchange with metadata packing,
|
||||
keep-alive, and abort support.
|
||||
5. **Incremental check** — the client sends `STATUS_CHECK` + path + size +
|
||||
@@ -379,6 +386,8 @@ Received files are written to a temporary path (suffixed with `.tmp`) and then a
|
||||
- C11 compiler
|
||||
- CMake >= 3.22
|
||||
- zstd library
|
||||
- zlib library
|
||||
- lz4 library
|
||||
- OpenSSL (development headers and libraries)
|
||||
- pthreads
|
||||
- SSH client (for SSH transport mode only)
|
||||
@@ -387,12 +396,12 @@ Received files are written to a temporary path (suffixed with `.tmp`) and then a
|
||||
|
||||
**Ubuntu/Debian:**
|
||||
```bash
|
||||
sudo apt install cmake build-essential libzstd-dev libssl-dev openssh-client
|
||||
sudo apt install cmake build-essential libzstd-dev zlib1g-dev liblz4-dev libssl-dev openssh-client
|
||||
```
|
||||
|
||||
**Nix:**
|
||||
```bash
|
||||
nix-shell # provides zstd, openssl, cmake, gcc
|
||||
nix-shell # provides zstd, zlib, lz4, openssl, cmake, gcc
|
||||
```
|
||||
|
||||
## Building
|
||||
@@ -526,9 +535,9 @@ features without changing the meaning of ordinary compatibility options.
|
||||
| `--server-host <host>` | Select the TCP server host. |
|
||||
| `--server-port <port>` | Select the TCP server port (`--port <port>` and `--port=<port>` are rsync-friendly aliases). |
|
||||
| `--tls` | Enable TLS for TCP transport. |
|
||||
| `--bwlimit <KB/s>` | Apply token-bucket bandwidth limiting. |
|
||||
| `--progress` | Show rsync-style per-file progress blocks from the receiver's wire counters (FastSync omits rsync's leading `./` line). |
|
||||
| `--stats` | Print transfer statistics, including the receiver-only counters reported over the wire; rsync's per-type `Number of files` breakdown is not reproduced. |
|
||||
| `--bwlimit <KB/s>` | Apply token-bucket bandwidth limiting (also paces `--sendfile` transfers). |
|
||||
| `--progress` | Show rsync-style per-file progress blocks from the receiver's wire counters; the root `./` line is printed whenever progress is active (rsync prints it only when the transfer root is created). |
|
||||
| `--stats` | Print transfer statistics, including the receiver-only counters reported over the wire; `Number of files`/`Number of created files` carry rsync's per-type breakdown (deleted files are a single total). |
|
||||
| `--timeout <seconds>` | Set the socket **and** per-message protocol I/O timeout. Default `0` = disabled (matching rsync); `0` disables it. |
|
||||
| `--contimeout <seconds>` | Connection timeout (default 60, matching rsync); `0` disables it. |
|
||||
|
||||
@@ -562,23 +571,26 @@ remote SSH argv is already built injection-safe.
|
||||
| `--size-only` | Skip incremental files matching in size, ignoring mtime. |
|
||||
| `-I, --ignore-times` | Transfer files even when size and mtime match. |
|
||||
| `-u, --update` | Skip files newer than the source on the receiver. |
|
||||
| `--ignore-existing` | Skip files that already exist on the receiver; like rsync it does not apply to directories or symlinks. |
|
||||
| `-@, --modify-window <sec>` | Modification-time tolerance (seconds) for the incremental/basis quick-check; `0` requires an exact mtime match. |
|
||||
| `-W, --whole-file` | Transfer changed files without delta processing (`--no-whole-file` clears it). |
|
||||
| `-B <n>, --block-size <n>` | Delta block size in bytes (alias `--delta-block`). |
|
||||
| `-d, --dirs` | Transfer the named directory entries without recursing into their contents (aliases `--old-dirs`/`--old-d`). |
|
||||
| `-R, --relative` | Use rsync's relative path semantics (including the `/./` cut); with `--files-from`, preserve each listed entry's relative path below the destination root. |
|
||||
| `--files-from <file>` | Read the source file list from FILE (paths relative to the source root). |
|
||||
| `--delay-updates` | Put updated files into place only at the end of the transfer. |
|
||||
| `--compare-dest <dir>` | Extra comparison basis: unchanged files are not transferred (requires/implies `--incremental`). |
|
||||
| `--copy-dest <dir>` | Like `--compare-dest`, but copies the unchanged file from DIR into the destination. |
|
||||
| `--link-dest <dir>` | Like `--copy-dest`, but hard-links the unchanged file from DIR (repeatable; earlier DIRs win). |
|
||||
| `-0, --from0` | Treat entries in `--files-from` files as NUL-delimited instead of newline-delimited. |
|
||||
| `--delay-updates` | Put updated files into place only at the end of the transfer (the fixed `.fastsync-stage` staging name diverges from rsync; see [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)). |
|
||||
| `--compare-dest <dir>` | Extra comparison basis: unchanged files are not transferred (requires/implies `--incremental`; a basis MISS above the 256 MiB whole-file payload bound is refused — see [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)). |
|
||||
| `--copy-dest <dir>` | Like `--compare-dest`, but copies the unchanged file from DIR into the destination (same basis-size caveat; see [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)). |
|
||||
| `--link-dest <dir>` | Like `--copy-dest`, but hard-links the unchanged file from DIR (repeatable; earlier DIRs win; same basis-size caveat; see [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)). |
|
||||
| `--verify-basis` | FastSync-only: require a basis hit to match the source by whole-file digest instead of trusting the size+mtime quick-check (default matches rsync). |
|
||||
| `--preallocate` | Allocate destination file space up front (fail-fast on a full disk). |
|
||||
| `--append` | Resume a shorter destination by appending only its tail (prefix not verified; requires `--incremental`). |
|
||||
| `--append-verify` | Like `--append`, but verifies the retained prefix checksum first (falls back to a full transfer on mismatch). |
|
||||
| `--delete` | Request removal of destination entries absent from the source. The server must allow deletion. Default timing is delete-after: extras are removed only after the whole transfer succeeded. Scoped to the synchronized directories, so `--files-from` subsets are safe. |
|
||||
| `--delete` | Request removal of destination entries absent from the source. The server must allow deletion. Default timing is delete-during (matching rsync's `--del`): extras are removed per directory as the transfer proceeds, so destination space is freed progressively. Scoped to the synchronized directories, so `--files-from` subsets are safe. |
|
||||
| `--delete-before` | Delete extras before the transfer starts (implies `--delete`). |
|
||||
| `--delete-during`, `--del` | Delete extras once the keep-set manifest is known, before data is applied (implies `--delete`; early mode, same engine behaviour as `--delete-before`). |
|
||||
| `--delete-delay` | Delete extras only after a successful transfer (implies `--delete`; commit mode, same behaviour as `--delete-after`). |
|
||||
| `--delete-during`, `--del` | Delete each directory's extras as that directory is processed (implies `--delete`). Since protocol 2.24.0 the sender streams a per-directory `STATUS_DELETE_PLAN` frame as it reaches each source directory; this is also the default timing of a plain `--delete`. |
|
||||
| `--delete-delay` | Record extras per directory during the scan but remove them only after a successful transfer (implies `--delete`). Uses the same per-directory `STATUS_DELETE_PLAN` frames as `--delete-during`, applied late. |
|
||||
| `--delete-commit` | FastSync-only: atomic delete-after timing (only after the whole transfer succeeded). |
|
||||
| `--delete-after` | Explicit delete-after timing: delete only after the transfer succeeded (implies `--delete`). |
|
||||
| `--delete-excluded` | Also delete filter-excluded destination mirrors (size-pruned mirrors stay protected). |
|
||||
@@ -598,8 +610,8 @@ remote SSH argv is already built injection-safe.
|
||||
| `--backup-dir <dir>` | Store backups under a separate directory (requires `--backup`). |
|
||||
| `--suffix <suffix>` | Set the backup filename suffix (default: `~`). |
|
||||
| `--partial` | Select partial-transfer handling. On failed/interrupted writes the already-written temp file is retained (best-effort) for resumption. With `--partial --partial-dir <dir>`, completed files are written under the partial directory and installed atomically. |
|
||||
| `--partial-dir <dir>` | Set a relative partial-transfer directory below the server destination root. Use with `--partial`. |
|
||||
| `--inplace` | Write directly to the destination instead of using a temporary file. |
|
||||
| `--partial-dir <dir>` | Set a relative partial-transfer directory below the server destination root. Implies `--partial`. Rejected together with `--inplace` (`--inplace cannot be used with --partial-dir`, matching rsync), because the inplace path bypasses partial/temp staging. |
|
||||
| `--inplace` | Write directly to the destination instead of using a temporary file. Cannot be combined with `--partial-dir`. |
|
||||
| `--fsync` | Fsync every written file before publication. |
|
||||
| `--write-batch=FILE` | Run the normal live transfer and also emit a self-contained batch file of the source tree. |
|
||||
| `--only-write-batch=FILE` | Emit the batch file only (no destination, no server). |
|
||||
@@ -614,8 +626,11 @@ remote SSH argv is already built injection-safe.
|
||||
| `--preserve` | Preserve mode and mtime (long form only; equivalent to `-p` + `-t`). Add `-o`/`-g` for owner/group, `-U`/`--atimes` for atime, or an identity flag (`--chown`/`--usermap`/`--groupmap`/`--numeric-ids`/`--copy-as`) for mapped ownership. |
|
||||
| `-U`, `--atimes` | Preserve access times. Captured with the metadata payload; does not enable ownership. |
|
||||
| `-N`, `--crtimes` | Capture birth time and transmit it; it cannot be applied because no portable filesystem call can set a birth time (documented divergence). |
|
||||
| `-p`, `--perms` | Preserve permission bits. One of the four per-attribute preserve flags (with `-t`/`-o`/`-g`); under `-p` the source mode is copied exactly (setuid/setgid/sticky and group/other-write included), matching rsync. |
|
||||
| `-p`, `--perms` | Preserve permission bits. One of the four per-attribute preserve flags (with `-t`/`-o`/`-g`); under `-p` the source mode is copied exactly (group/other-write included; setuid/setgid/sticky included only when super-user activities are permitted, masked under `SUPER_MODE_OFF`/`--no-super`), matching rsync otherwise. |
|
||||
| `-t`, `--times` | Preserve modification times. Independent of the other attributes; `-O`/`--omit-dir-times` suppresses directories only. |
|
||||
| `-O`, `--omit-dir-times` | Do not apply modification times to directories. |
|
||||
| `-J`, `--omit-link-times` | Do not apply times to symlinks. |
|
||||
| `--open-noatime` | Open source files with `O_NOATIME` so reading for a transfer does not update their access time (client-only). |
|
||||
| `-o`, `--owner` | Preserve the source owner (uid). Mapped by name on the receiver with a raw-numeric fallback (only numeric ids cross the wire); application is privilege-gated. |
|
||||
| `-g`, `--group` | Preserve the source group (gid). Same name-mapping/numeric-fallback and privilege gating as `-o`. |
|
||||
| `--no-perms`, `--no-times`, `--no-owner`, `--no-group` | Negate each per-attribute flag (also `--no-p`/`--no-t`/`--no-o`/`--no-g`); `--no-preserve` clears all four. |
|
||||
@@ -642,6 +657,7 @@ remote SSH argv is already built injection-safe.
|
||||
| `-D` | Preserve device and special files (implies `--devices --specials`). |
|
||||
| `--devices` | Recreate device nodes on the destination (privileged; skipped without `CAP_MKNOD`). |
|
||||
| `--specials` | Recreate special files: FIFOs and unix sockets. |
|
||||
| `--copy-devices` | Copy a source device's content as an ordinary regular file on the destination (rsync's non-privileged safe mode) instead of recreating the device node. |
|
||||
| `-S`, `--sparse` | Sparse-file handling: receiver preserves holes (zero runs are written as holes; no wire change). |
|
||||
|
||||
### Output and logging
|
||||
@@ -650,12 +666,17 @@ remote SSH argv is already built injection-safe.
|
||||
|---|---|
|
||||
| `-v`, `--verbose` | Enable debug logging. |
|
||||
| `-q`, `--quiet` | Suppress non-error output. |
|
||||
| `--progress` | Show rsync-style per-file progress blocks (not rsync's leading `./` line). |
|
||||
| `--stats` | Print transfer statistics, including the receiver-only counters reported over the wire. |
|
||||
| `-i`, `--itemize-changes` | Print an rsync-style per-file change line. |
|
||||
| `--out-format=FORMAT` | Output format for changed files (`%f %n %l %b %M %%`). |
|
||||
| `--progress` | Show rsync-style per-file progress blocks; the root `./` line is printed whenever progress is active (rsync prints it only when the transfer root is created). |
|
||||
| `--stats` | Print transfer statistics, including the receiver-only counters reported over the wire; `Number of files`/`Number of created files` carry rsync's per-type breakdown (deleted files are a single total). |
|
||||
| `-i, --itemize-changes` | Print an rsync-style per-file change line. |
|
||||
| `--out-format=FORMAT` | Output format for changed files (`%f %n %l %b %c %C %i %M %%`). |
|
||||
| `--list-only` | List source files instead of transferring. |
|
||||
| `--outbuf=MODE` | stdout/stderr buffering: `N` (none/unbuffered), `L` (line-buffered), or `B` (block-buffered, default). |
|
||||
| `--log-file <path>` | Write log output to a file. |
|
||||
| `--log-file-format=FORMAT` | Per-file log-line format (requires `--log-file`). |
|
||||
| `--stderr=MODE` | Route logging to stderr: `errors` or `all`. |
|
||||
| `--msgs2stderr` | Route all messages to stderr (deprecated spelling of `--stderr=all`). |
|
||||
| `--no-msgs2stderr` | Select errors-only stderr (deprecated spelling; the default). |
|
||||
| `-V`, `--version` | Print the FastSync protocol version. |
|
||||
| `--help` | Print command usage. |
|
||||
|
||||
@@ -680,6 +701,11 @@ remote SSH argv is already built injection-safe.
|
||||
| `-4`, `--ipv4` | Force IPv4 for destination resolution. |
|
||||
| `-6`, `--ipv6` | Force IPv6 for destination resolution. |
|
||||
| `--sockopts=OPTS` | Comma-separated OPT=VAL socket options applied before connect. |
|
||||
| `--blocking-io` | SSH transport only: leave the socket without read/write timeouts so it blocks naturally (no effect on TCP). |
|
||||
| `--protocol=NUM` | Force the wire protocol version; must equal the current `PROTOCOL_VERSION` (FastSync cannot speak older/virtual wire formats). |
|
||||
| `--old-args` | Accepted for rsync CLI compatibility; no effect (the remote server path is always safely quoted). |
|
||||
| `--iconv=LOCAL[,REMOTE]` | Convert file-name charsets at the wire boundary (`LOCAL` is our names' charset, `REMOTE` the peer's, defaulting to `LOCAL`). |
|
||||
| `--no-iconv` | Disable `--iconv` charset conversion (same as `--iconv=-`). |
|
||||
| `--tls` | Enable TLS. Requires `--cert`, `--key`, and `--ca`. |
|
||||
| `--cert <path>` | TLS certificate file. |
|
||||
| `--key <path>` | TLS private key file. |
|
||||
|
||||
+57
-28
File diff suppressed because one or more lines are too long
+65
-16
@@ -54,13 +54,26 @@ bool client_abort_pending(void) {
|
||||
}
|
||||
|
||||
#ifndef FASTSYNC_TEST_BUILD
|
||||
/* SIG_DFL disposition used by the handler's "not armed" fallback. It is built
|
||||
* once at load time so the handler can restore the default action with
|
||||
* sigaction(2) -- which is async-signal-safe -- instead of signal(3), which is
|
||||
* not. The zero-initialized sa_mask is the empty set. */
|
||||
static const struct sigaction client_default_action = {
|
||||
.sa_handler = SIG_DFL,
|
||||
.sa_flags = 0,
|
||||
};
|
||||
|
||||
/* Signal handler: perform NO work beyond storing the flag. Logging, protocol
|
||||
* I/O and the STATUS_ABORT frame are all done later on the normal send path,
|
||||
* which is not async-signal-safe. When no transfer is armed, fall back to the
|
||||
* default action so local-only modes remain interruptible. */
|
||||
* which is not async-signal-safe. When no transfer is armed, restore the
|
||||
* default disposition (async-signal-safe sigaction) and re-raise so local-only
|
||||
* modes remain interruptible. The handler deliberately stays installed while a
|
||||
* transfer is armed -- rather than using SA_RESETHAND -- so a second Ctrl-C
|
||||
* during the graceful abort keeps setting the flag instead of hard-killing the
|
||||
* process mid-cleanup. */
|
||||
static void client_signal_handler(int signo) {
|
||||
if (!client_abort_armed) {
|
||||
signal(signo, SIG_DFL);
|
||||
sigaction(signo, &client_default_action, NULL);
|
||||
raise(signo);
|
||||
return;
|
||||
}
|
||||
@@ -504,9 +517,8 @@ static bool split_flag_level(const char* token, char* name, size_t name_size, in
|
||||
* of rsync's `symsafe`, `hlink`, and `own`. */
|
||||
static bool is_accepted_debug_category(const char* name) {
|
||||
static const char* const categories[] = {
|
||||
"acl", "backup", "bind", "chdir", "cmd", "connect", "del", "deltasum",
|
||||
"dup", "exit", "filter", "flist", "fuzzy", "genr", "hash", "hl",
|
||||
"hlink", "iconv", "nstr", "own", "owner", "recv", "send", "time",
|
||||
"acl", "backup", "bind", "chdir", "cmd", "connect", "dup", "exit", "fuzzy",
|
||||
"genr", "hl", "hlink", "iconv", "nstr", "own", "owner", "time",
|
||||
};
|
||||
for (size_t i = 0; i < sizeof(categories) / sizeof(categories[0]); i++) {
|
||||
if (strcmp(name, categories[i]) == 0)
|
||||
@@ -518,7 +530,6 @@ static bool is_accepted_debug_category(const char* name) {
|
||||
static bool is_accepted_info_category(const char* name) {
|
||||
static const char* const categories[] = {
|
||||
"backup",
|
||||
"mount",
|
||||
"syms",
|
||||
"symsafe",
|
||||
};
|
||||
@@ -571,6 +582,18 @@ static int parse_debug_flags(const char* value, Config* config) {
|
||||
flag = LOG_DEBUG_PACK;
|
||||
} else if (strcmp(name, "util") == 0) {
|
||||
flag = LOG_DEBUG_UTIL;
|
||||
} else if (strcmp(name, "flist") == 0) {
|
||||
flag = LOG_DEBUG_FLIST;
|
||||
} else if (strcmp(name, "del") == 0) {
|
||||
flag = LOG_DEBUG_DEL;
|
||||
} else if (strcmp(name, "hash") == 0 || strcmp(name, "deltasum") == 0) {
|
||||
flag = LOG_DEBUG_HASH;
|
||||
} else if (strcmp(name, "recv") == 0) {
|
||||
flag = LOG_DEBUG_RECV;
|
||||
} else if (strcmp(name, "filter") == 0) {
|
||||
flag = LOG_DEBUG_FILTER;
|
||||
} else if (strcmp(name, "send") == 0) {
|
||||
flag = LOG_DEBUG_SEND;
|
||||
} else if (is_accepted_debug_category(name)) {
|
||||
continue;
|
||||
} else {
|
||||
@@ -645,9 +668,12 @@ static int parse_info_flags(const char* value, Config* config) {
|
||||
flag = LOG_INFO_MISC;
|
||||
else if (strcmp(name, "skip") == 0)
|
||||
flag = LOG_INFO_SKIP;
|
||||
else if (strcmp(name, "stats") == 0)
|
||||
else if (strcmp(name, "stats") == 0) {
|
||||
flag = LOG_INFO_STATS;
|
||||
else if (strcmp(name, "del") == 0)
|
||||
/* `--info=stats` requests the same transfer-statistics block as
|
||||
`--stats`; `--info=stats0` turns it back off. */
|
||||
config->stats = level > 0;
|
||||
} else if (strcmp(name, "del") == 0)
|
||||
flag = LOG_INFO_DEL;
|
||||
else if (strcmp(name, "remove") == 0)
|
||||
flag = LOG_INFO_REMOVE;
|
||||
@@ -655,6 +681,8 @@ static int parse_info_flags(const char* value, Config* config) {
|
||||
flag = LOG_INFO_FLIST;
|
||||
else if (strcmp(name, "nonreg") == 0)
|
||||
flag = LOG_INFO_NONREG;
|
||||
else if (strcmp(name, "mount") == 0)
|
||||
flag = LOG_INFO_MOUNT;
|
||||
else if (strcmp(name, "progress") == 0)
|
||||
flag = LOG_INFO_PROGRESS;
|
||||
else if (is_accepted_info_category(name))
|
||||
@@ -1213,7 +1241,13 @@ static int apply_table_option(Config* config, const OptionEntry* entry, const ch
|
||||
void* field = (char*)config + entry->offset;
|
||||
switch (entry->kind) {
|
||||
case OPT_FLAG:
|
||||
*(bool*)field = true;
|
||||
/* -x/--one-file-system is repeatable in rsync: `-xx` increments the level so
|
||||
the scanner drops mount-point directories instead of recreating them
|
||||
empty. Everything else is a plain boolean. */
|
||||
if (entry->offset == offsetof(Config, one_file_system))
|
||||
(*(int*)field)++;
|
||||
else
|
||||
*(bool*)field = true;
|
||||
return 0;
|
||||
case OPT_NOOP:
|
||||
return 0;
|
||||
@@ -2574,7 +2608,11 @@ static bool cli_handle_outbuf_option(CliParseCtx* ctx) {
|
||||
* load --files-from once every argument has been seen. Returns 0 on success,
|
||||
* -1 on error. */
|
||||
static int cli_finalize_config(Config* config, bool verbose, bool no_delta, bool no_incremental) {
|
||||
set_log_level(config->quiet ? LOG_LEVEL_ERROR : (verbose ? LOG_LEVEL_DEBUG : LOG_LEVEL_WARNING));
|
||||
/* An explicit --debug=FLAGS enables the debug log level by itself (rsync
|
||||
behaviour); -v enables every other INFO-level message. */
|
||||
bool debug_enabled = verbose || config->debug_level != 0;
|
||||
set_log_level(config->quiet ? LOG_LEVEL_ERROR
|
||||
: (debug_enabled ? LOG_LEVEL_DEBUG : LOG_LEVEL_WARNING));
|
||||
/* rsync's plain --delete defaults to delete-during (--del): each directory's
|
||||
extras are removed as that directory is processed, so space is freed
|
||||
progressively and a tight destination never has to hold the whole old+new
|
||||
@@ -2587,6 +2625,15 @@ static int cli_finalize_config(Config* config, bool verbose, bool no_delta, bool
|
||||
if (config->use_delete && !config->delete_before && !config->delete_during &&
|
||||
!config->delete_delay && !config->delete_after)
|
||||
config->delete_during = true;
|
||||
/* rsync parity: --partial-dir=DIR chooses where an interrupted transfer's
|
||||
partial file is kept, so it implies --partial. rsync applies the
|
||||
implication after option parsing, so it wins over an explicit --no-partial
|
||||
regardless of the order the two options appear in (verified on rsync
|
||||
3.4.1). --inplace is the exception: the destination file is written in
|
||||
place with no partial/temp staging, so the partial machinery is bypassed
|
||||
and the implication is skipped to leave --inplace behavior untouched. */
|
||||
if (config->partial_dir && !config->inplace)
|
||||
config->partial = true;
|
||||
if (config->compress_choice) {
|
||||
int algo = compression_algo_from_name(config->compress_choice);
|
||||
if (algo >= 0) {
|
||||
@@ -2764,10 +2811,12 @@ static int cli_finalize_config(Config* config, bool verbose, bool no_delta, bool
|
||||
}
|
||||
/* --info=del on a real --delete run asks the receiver to report the paths it
|
||||
actually removed; the report rides the STATUS_STATS path list, so the wire
|
||||
stats frame must be negotiated too. */
|
||||
config->report_deletes = config->use_delete && !config->dry_run &&
|
||||
((config->info_level & LOG_INFO_DEL) != 0 || config->itemize_changes ||
|
||||
config->out_format != NULL);
|
||||
stats frame must be negotiated too. --debug=del needs the same paths, so
|
||||
it opts into the existing report (no new wire field). */
|
||||
config->report_deletes =
|
||||
config->use_delete && !config->dry_run &&
|
||||
((config->info_level & LOG_INFO_DEL) != 0 || config->itemize_changes ||
|
||||
config->out_format != NULL || (config->debug_level & LOG_DEBUG_DEL) != 0);
|
||||
config->report_stats = config->stats || config->show_progress ||
|
||||
(config->info_level & LOG_INFO_PROGRESS) || format_needs_wire ||
|
||||
config->report_deletes || (config->dry_run && config->use_delete);
|
||||
@@ -3247,7 +3296,7 @@ int main(int argc, char* argv[]) {
|
||||
exit_code = 1;
|
||||
}
|
||||
} else if (config->use_multithreading) {
|
||||
exit_code = send_files_multithreaded(&config);
|
||||
exit_code = send_files_multithreaded(config);
|
||||
} else {
|
||||
exit_code = send_files(config);
|
||||
}
|
||||
|
||||
+76
-76
@@ -37,8 +37,6 @@
|
||||
#include <sys/stat.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#define STREAM_THRESHOLD (64ULL * 1024 * 1024)
|
||||
|
||||
/* Aggregate loaded payload bytes the sender may buffer across the loader queue
|
||||
and the chunk in flight. Sending one chunk adds up to ~2 * MAX_CHUNK_SIZE of
|
||||
transient serialize/compress buffers on top of the queued payloads, so this
|
||||
@@ -129,6 +127,21 @@ static void stats_type_breakdown(const TransferStats* stats, char* out, size_t o
|
||||
out_size);
|
||||
}
|
||||
|
||||
/* rsync's `Number of files` counts every directory. A recursive scan that
|
||||
preserves a directory attribute captures them in `dir_entries`; a `-r` scan
|
||||
(no -t/-p) captures nothing, so fall back to the scanner's shared counter of
|
||||
traversed directories that are not already represented by an inline
|
||||
directory entry. The -d generator counts its explicit directory entries
|
||||
inline and does not traverse, so it is excluded here. */
|
||||
static unsigned long long dir_count_for_stats(const Config* config, const ArrayList* dir_entries,
|
||||
atomic_ullong* counter) {
|
||||
if (config == NULL || config->dirs || config->list_only)
|
||||
return 0;
|
||||
if (dir_metadata_should_capture(config))
|
||||
return dir_entries != NULL ? (unsigned long long)dir_entries->size : 0;
|
||||
return counter != NULL ? (unsigned long long)atomic_load(counter) : 0;
|
||||
}
|
||||
|
||||
/* Print the rsync `--stats` block on stdout. The source-side flist and
|
||||
transferred counters come from `stats` (filled while scanning/sending), the
|
||||
receiver-only counters from the STATUS_STATS frame, and the wire byte totals
|
||||
@@ -387,6 +400,15 @@ static bool info_flag_enabled(const Config* config, LogInfoFlag flag) {
|
||||
static void print_delete_reports(const Config* config, const ArrayList* paths) {
|
||||
if (!config || !paths || config->quiet)
|
||||
return;
|
||||
/* --debug=del is independent of the --info=del/itemize/out-format display:
|
||||
emit the debug trace even when no deletion line would be printed. */
|
||||
if (log_debug_enabled(LOG_DEBUG_DEL)) {
|
||||
for (int i = 0; i < paths->size; i++) {
|
||||
const char* raw = (const char*)paths->items[i];
|
||||
const char* path = delete_display_path(config, raw);
|
||||
log_debug_message(LOG_DEBUG_DEL, "del: %s", path ? path : raw);
|
||||
}
|
||||
}
|
||||
if (!(config->itemize_changes || config->out_format != NULL ||
|
||||
info_flag_enabled(config, LOG_INFO_DEL)))
|
||||
return;
|
||||
@@ -665,6 +687,7 @@ static bool prepare_scanner(const Config* config, int num_threads, PreparedScann
|
||||
options->ignore_io_errors = config->ignore_errors;
|
||||
options->ignore_missing_args = config->ignore_missing_args || config->delete_missing_args;
|
||||
options->note_nonreg = (config->info_level & LOG_INFO_NONREG) != 0 && !config->quiet;
|
||||
options->note_mount = (config->info_level & LOG_INFO_MOUNT) != 0 && !config->quiet;
|
||||
options->send_directory = config->send_directory;
|
||||
options->eight_bit_output = config->eight_bit_output;
|
||||
options->excluded_paths = NULL;
|
||||
@@ -672,6 +695,9 @@ static bool prepare_scanner(const Config* config, int num_threads, PreparedScann
|
||||
options->size_skipped_paths = NULL;
|
||||
options->synced_dirs = NULL;
|
||||
options->hardlinks = NULL;
|
||||
/* Set by the real send paths; NULL for the metadata-only scans (progress
|
||||
pre-count, batch) that must not perturb the sender's --stats counter. */
|
||||
options->dir_count = NULL;
|
||||
/* P7 Wave D: capture source directory metadata when a directory attribute is
|
||||
requested (-p for modes, -t for times unless -O omits them). Whether they
|
||||
are APPLIED is decided receiver-side. */
|
||||
@@ -730,6 +756,8 @@ static bool progress_precount_scan(const Config* config, ProgressPrecount* out)
|
||||
ScannerOptions local = prepared.options;
|
||||
local.list_dirs = true;
|
||||
local.note_nonreg = false;
|
||||
local.note_mount = false;
|
||||
local.dir_count = NULL;
|
||||
local.use_metadata = false;
|
||||
local.preserve_xattrs = false;
|
||||
local.preserve_acls = false;
|
||||
@@ -1085,7 +1113,7 @@ static Client* connect_transfer_client(const Config* config) {
|
||||
return NULL;
|
||||
}
|
||||
return client_connect_ssh(config->ssh_destination, config->ssh_port,
|
||||
config->fastsync_server_path, config->old_args, config->rsh_command,
|
||||
config->fastsync_server_path, config->rsh_command,
|
||||
config->blocking_io, config->remote_options,
|
||||
config->remote_option_count);
|
||||
}
|
||||
@@ -1336,6 +1364,7 @@ static bool finalize_transfer(Client* client, const Config* config, ArrayList* r
|
||||
return false;
|
||||
if (status == STATUS_STATS) {
|
||||
ReceiverStats scratch;
|
||||
log_debug_message(LOG_DEBUG_RECV, "recv: receiver stats");
|
||||
/* A real --info=del run carries the actually-removed paths in the stats
|
||||
frame's path list; collect and print them in rsync's format. */
|
||||
ArrayList* deleted = config->report_deletes ? array_list_create(free) : NULL;
|
||||
@@ -1855,25 +1884,6 @@ static bool scan_paths_only(const Config* config, const ScannerOptions* options,
|
||||
return ok;
|
||||
}
|
||||
|
||||
/* Transmit any not-yet-sent per-directory delete plan needed by the entries in
|
||||
* `chunk` (ancestors root-first, then the entry's own directory for --dirs
|
||||
* entries) before its data frames go out, so --delete-during/--delete-delay
|
||||
* clear a directory's extras (and any type conflict) before the directory's
|
||||
* first write. */
|
||||
static int send_chunk_delete_plans(Client* client, DeletePlanSender* plans, const Chunk* chunk) {
|
||||
if (!plans)
|
||||
return 0;
|
||||
for (int i = 0; i < chunk->element_count; i++) {
|
||||
File* f = chunk->items[i];
|
||||
if (!f)
|
||||
continue;
|
||||
if (delete_plan_send_for_path(client->file_descriptor, plans, file_wire_path(f), f->is_dir) !=
|
||||
0)
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int incremental_check(Client* client, File* file, const Config* config,
|
||||
DeltaSignature** out_sig, unsigned long long* resume_offset) {
|
||||
*out_sig = NULL;
|
||||
@@ -1904,6 +1914,8 @@ static int incremental_check(Client* client, File* file, const Config* config,
|
||||
if (!file_checksum(file, (ChecksumAlgo)config->checksum_algo, config->checksum_seed, digest,
|
||||
sizeof(digest), &digest_len))
|
||||
return -1;
|
||||
log_debug_message(LOG_DEBUG_HASH, "hash: %s (algo %d)", file_wire_path(file),
|
||||
config->checksum_algo);
|
||||
uint8_t wire_len = (uint8_t)digest_len;
|
||||
if (!send_n_data(client->file_descriptor, &wire_len, sizeof(wire_len)) ||
|
||||
!send_n_data(client->file_descriptor, digest, wire_len))
|
||||
@@ -1938,6 +1950,7 @@ static int incremental_check(Client* client, File* file, const Config* config,
|
||||
log_server_rejection("Server reported error for file");
|
||||
return -1;
|
||||
}
|
||||
log_debug_message(LOG_DEBUG_RECV, "recv: check reply for %s", file_wire_path(file));
|
||||
if (s == STATUS_OK)
|
||||
return 1;
|
||||
if (s == STATUS_DELTA_SIGNATURE) {
|
||||
@@ -1952,6 +1965,8 @@ static int incremental_check(Client* client, File* file, const Config* config,
|
||||
send_status(client->file_descriptor, STATUS_ERROR);
|
||||
return -1;
|
||||
}
|
||||
log_debug_message(LOG_DEBUG_RECV, "recv: delta signature for %s (%u blocks)",
|
||||
file_wire_path(file), sig->block_count);
|
||||
*out_sig = sig;
|
||||
return 2;
|
||||
}
|
||||
@@ -1992,6 +2007,7 @@ static int incremental_check(Client* client, File* file, const Config* config,
|
||||
static int send_delta(Client* client, File* file, DeltaSignature* sig, Config* config) {
|
||||
Delta* delta = delta_compute_seeded(file->data->data, file->data->size, sig,
|
||||
config->delta_block_size, (uint32_t)config->checksum_seed);
|
||||
log_debug_message(LOG_DEBUG_HASH, "deltasum: %s", file_wire_path(file));
|
||||
/* The receiver is blocked after sending the signature. Every local
|
||||
fallback therefore needs the explicit NEXT response before full data. */
|
||||
if (!delta)
|
||||
@@ -2465,6 +2481,7 @@ static int send_single_file(Client* client, File* file, Config* config, bool use
|
||||
bool use_sendfile) {
|
||||
int compression_level = config->use_compression ? config->compression_level : 0;
|
||||
log_info_message(LOG_INFO_COPY, "Transferring %s", file->path);
|
||||
log_debug_message(LOG_DEBUG_SEND, "send: %s", file_wire_path(file));
|
||||
|
||||
if (!use_incremental) {
|
||||
if (use_sendfile) {
|
||||
@@ -2753,9 +2770,12 @@ static int send_chunks_multithreaded(void* pipeline_context) {
|
||||
return thrd_error;
|
||||
}
|
||||
} else if (context->delete_plans) {
|
||||
/* --delete-during/--delete-delay: transmit the receive root's plan before
|
||||
any data, exactly like rsync's first generator directory. */
|
||||
if (delete_plan_send_root(client->file_descriptor, context->delete_plans) != 0) {
|
||||
/* --delete-during/--delete-delay: transmit the COMPLETE per-directory plan
|
||||
set before any data, so a mid-transfer abort has already applied every
|
||||
planned removal exactly like rsync's generator (which runs ahead of its
|
||||
throttled sender). A completed run is unaffected. */
|
||||
if (delete_plan_send_all(client->file_descriptor, context->delete_plans, context->plan_dirs) !=
|
||||
0) {
|
||||
pipeline_cancel(context);
|
||||
disconnect_transfer_client(client);
|
||||
mark_sender_done(context);
|
||||
@@ -2802,15 +2822,6 @@ static int send_chunks_multithreaded(void* pipeline_context) {
|
||||
}
|
||||
break;
|
||||
}
|
||||
if (send_chunk_delete_plans(client, context->delete_plans, current_chunk) != 0) {
|
||||
log_message(LOG_LEVEL_ERROR, "unexpected error while sending delete plan");
|
||||
chunk_destroy(current_chunk);
|
||||
pipeline_cancel(context);
|
||||
disconnect_transfer_client(client);
|
||||
mark_sender_done(context);
|
||||
protocol_session_unbind();
|
||||
return thrd_error;
|
||||
}
|
||||
if (send_chunk_with_removal(client, current_chunk, context->config,
|
||||
context->remove_source_files, &context->stats) != 0) {
|
||||
log_message(LOG_LEVEL_ERROR, "unexpected error while sending chunk");
|
||||
@@ -2893,13 +2904,6 @@ static int send_chunks_multithreaded(void* pipeline_context) {
|
||||
NULL) != 0)
|
||||
goto send_fail;
|
||||
}
|
||||
/* Emit the plans for source directories the data stream never triggered
|
||||
(empty directories): their extras are still cleared while the directory
|
||||
itself is kept. */
|
||||
if (!context->scan_stopped_early && context->delete_plans && context->plan_dirs &&
|
||||
delete_plan_send_remaining(client->file_descriptor, context->delete_plans,
|
||||
context->plan_dirs) != 0)
|
||||
goto send_fail;
|
||||
/* P7 Wave D: transmit the captured directory times last. The scanner thread
|
||||
(and all parallel workers) has been joined before scanner_done was set, so
|
||||
the list is complete and race-free; on an early stop the list may be
|
||||
@@ -2918,8 +2922,8 @@ static int send_chunks_multithreaded(void* pipeline_context) {
|
||||
"server reported a deletion failure (--delete); see the server log for the reason");
|
||||
if (ok)
|
||||
remove_transferred_sources(context->config, context->remove_source_files);
|
||||
if (context->dir_entries)
|
||||
context->stats.flist_dir += (unsigned long long)context->dir_entries->size;
|
||||
context->stats.flist_dir +=
|
||||
dir_count_for_stats(context->config, context->dir_entries, &context->dir_count);
|
||||
report_transfer_stats(context->config, &context->stats, start, &recv_stats);
|
||||
log_info_message(LOG_INFO_STATS, "Transfer summary: %llu files, %.1f MB",
|
||||
context->stats.transferred_regular,
|
||||
@@ -2956,6 +2960,7 @@ static int scan_directory_multithreaded(void* pipeline_context) {
|
||||
parallel workers append under the context's dedicated mutex. */
|
||||
prepared.options.dir_entries = context->dir_entries;
|
||||
prepared.options.dir_entries_mutex = &context->dir_entries_mutex;
|
||||
prepared.options.dir_count = context->config->stats ? &context->dir_count : NULL;
|
||||
if (!append_implied_dir_times(context->config, context->dir_entries)) {
|
||||
pipeline_cancel(context);
|
||||
protocol_session_unbind();
|
||||
@@ -3224,6 +3229,9 @@ int send_files(Config* config) {
|
||||
/* P7 Wave D: captured source directory times, transmitted in trailing
|
||||
STATUS_DIR_TIMES frame(s) (only when metadata rides the wire). */
|
||||
ArrayList* dir_entries = NULL;
|
||||
/* --stats directory accounting for the no-metadata (-r) case. */
|
||||
atomic_ullong dir_count;
|
||||
atomic_init(&dir_count, 0);
|
||||
/* Protected excluded prefixes (delete-excluded default protection). */
|
||||
ArrayList* excluded = NULL;
|
||||
/* Size-pruned prefixes (always protected) and synchronized directories. */
|
||||
@@ -3232,10 +3240,12 @@ int send_files(Config* config) {
|
||||
/* Traversed source directories for the per-directory delete keep set. */
|
||||
ArrayList* plan_dirs = NULL;
|
||||
bool delete_early = config->use_delete && config_delete_timing_early(config);
|
||||
/* -d/--dirs does not recurse, so a per-directory plan would carry no child
|
||||
information and could delete the contents of an untraversed directory;
|
||||
fall back to the whole-tree end-of-transfer commit for that mode. */
|
||||
bool delete_per_dir = config->use_delete && config_delete_timing_per_dir(config) && !config->dirs;
|
||||
/* --delete-during/--delete-delay use per-directory plans for every transfer
|
||||
shape. For -d/--dirs the generator records only the directories whose
|
||||
direct children it actually enumerated, so the plan removes extras directly
|
||||
inside a listed directory while an untraversed (kept) subdirectory is
|
||||
shielded -- rsync's `-d DIR/ --delete`. */
|
||||
bool delete_per_dir = config->use_delete && config_delete_timing_per_dir(config);
|
||||
bool send_failed = false;
|
||||
bool had_scan_io = false;
|
||||
unsigned long long per_dir_non_dir_count = 0;
|
||||
@@ -3287,12 +3297,12 @@ int send_files(Config* config) {
|
||||
prepared.options.synced_dirs = synced_dirs;
|
||||
}
|
||||
}
|
||||
/* The late-timing modes (--delete-after/--delete-commit and a plain --delete
|
||||
that fell back from per-dir mode because of -d/--dirs) build the manifest
|
||||
/* The late-timing modes (--delete-after/--delete-commit) build the manifest
|
||||
while streaming and send it after the last data frame. --delete-before
|
||||
sends a whole-tree keep-set up front; --delete-during/--delete-delay build a
|
||||
per-directory plan set up front (paths only) and stream the plans alongside
|
||||
the data, so no manifest is kept during the data pass. */
|
||||
sends a whole-tree keep-set up front; --delete-during/--delete-delay build
|
||||
the complete per-directory plan set up front (paths only) and transmit it
|
||||
all before the first data frame, so a mid-transfer abort has already
|
||||
applied every planned removal. */
|
||||
if (delete_early) {
|
||||
/* Pass 1: collect the complete keep-set (paths only, no data loaded) and
|
||||
transmit it now, before any file data. The receiver removes extras and
|
||||
@@ -3335,9 +3345,10 @@ int send_files(Config* config) {
|
||||
goto send_fail;
|
||||
} else if (delete_per_dir) {
|
||||
/* --delete-during/--delete-delay: build one plan per source directory from a
|
||||
path-only pre-scan and transmit the root plan now, before any data, so the
|
||||
receive root's extras are handled exactly like rsync's first generator
|
||||
directory. The remaining plans are streamed with the data below. */
|
||||
path-only pre-scan and transmit the COMPLETE plan set now, before any data,
|
||||
so every planned removal has already been applied when a later transfer
|
||||
phase fails -- exactly like rsync's generator, whose deletion list runs
|
||||
ahead of its throttled sender. A completed run is unaffected. */
|
||||
plan_sender = delete_plan_sender_create();
|
||||
plan_dirs = array_list_create(free);
|
||||
if (!plan_sender || !plan_dirs)
|
||||
@@ -3368,7 +3379,7 @@ int send_files(Config* config) {
|
||||
plan_dirs = NULL;
|
||||
skip_delete = true;
|
||||
} else {
|
||||
plans_ok = delete_plan_send_root(client->file_descriptor, plan_sender) == 0;
|
||||
plans_ok = delete_plan_send_all(client->file_descriptor, plan_sender, plan_dirs) == 0;
|
||||
}
|
||||
}
|
||||
prepared.options.excluded_paths = NULL;
|
||||
@@ -3402,6 +3413,7 @@ int send_files(Config* config) {
|
||||
the directory-time list (otherwise every directory would be captured
|
||||
twice). */
|
||||
prepared.options.dir_entries = dir_entries;
|
||||
prepared.options.dir_count = config->stats ? &dir_count : NULL;
|
||||
scanner = directory_scanner_create_with_options(config->send_directory, &prepared.options);
|
||||
if (!scanner)
|
||||
goto send_fail;
|
||||
@@ -3455,11 +3467,6 @@ int send_files(Config* config) {
|
||||
goto send_fail;
|
||||
}
|
||||
}
|
||||
if (send_chunk_delete_plans(client, plan_sender, current_chunk) != 0) {
|
||||
chunk_destroy(current_chunk);
|
||||
send_failed = true;
|
||||
break;
|
||||
}
|
||||
if (send_chunk_with_removal(client, current_chunk, config, remove_sources, &transfer_stats) !=
|
||||
0) {
|
||||
log_message(LOG_LEVEL_ERROR, "Failed to send chunk");
|
||||
@@ -3542,12 +3549,6 @@ int send_files(Config* config) {
|
||||
}
|
||||
}
|
||||
}
|
||||
/* Emit the plans for any source directories the data stream never triggered
|
||||
(an empty directory has no file frame). Sending them now still clears that
|
||||
directory's destination extras while keeping the directory itself. */
|
||||
if (!scan_stopped_early && plan_sender && plan_dirs &&
|
||||
delete_plan_send_remaining(client->file_descriptor, plan_sender, plan_dirs) != 0)
|
||||
goto send_fail;
|
||||
/* P7 Wave D: every directory has now been traversed (or the scan stopped
|
||||
early), so transmit the captured directory times last. The receiver defers
|
||||
applying them until after its own deletion/publication phase. */
|
||||
@@ -3566,8 +3567,7 @@ int send_files(Config* config) {
|
||||
from the scanner's captured directory list (present whenever a directory
|
||||
attribute is preserved, e.g. -a/-t/-p). The -d generator counts its
|
||||
explicit directory entries inline instead. */
|
||||
if (dir_entries)
|
||||
transfer_stats.flist_dir += (unsigned long long)dir_entries->size;
|
||||
transfer_stats.flist_dir += dir_count_for_stats(config, dir_entries, &dir_count);
|
||||
report_transfer_stats(config, &transfer_stats, start, &recv_stats);
|
||||
log_info_message(LOG_INFO_STATS, "Transfer summary: %llu files, %.1f MB",
|
||||
transfer_stats.transferred_regular,
|
||||
@@ -3615,10 +3615,9 @@ send_fail:
|
||||
return ret;
|
||||
}
|
||||
|
||||
int send_files_multithreaded(Config** config_ptr) {
|
||||
if (!config_ptr || !*config_ptr)
|
||||
int send_files_multithreaded(Config* config) {
|
||||
if (!config)
|
||||
return 1;
|
||||
Config* config = *config_ptr;
|
||||
if (config->list_only)
|
||||
return send_list_only(config);
|
||||
if (config->dry_run)
|
||||
@@ -3714,10 +3713,11 @@ int send_files_multithreaded(Config** config_ptr) {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
/* -d/--dirs does not recurse, so a per-directory plan would carry no child
|
||||
information and could delete the contents of an untraversed directory;
|
||||
fall back to the whole-tree end-of-transfer commit for that mode. */
|
||||
bool per_dir = config_delete_timing_per_dir(config) && !config->dirs;
|
||||
/* --delete-during/--delete-delay use per-directory plans for every transfer
|
||||
shape. The -d/--dirs generator records only the directories whose direct
|
||||
children it enumerated, so extras directly inside a listed directory are
|
||||
removed while an untraversed (kept) subdirectory is shielded. */
|
||||
bool per_dir = config_delete_timing_per_dir(config);
|
||||
if (config_delete_timing_early(config) || per_dir) {
|
||||
/* --delete-before / --delete-during / --delete-delay: build the keep-set
|
||||
(paths only, nothing loaded or sent) up front so the sender thread can
|
||||
|
||||
@@ -22,7 +22,7 @@ void client_set_abort_armed(bool armed);
|
||||
* never free it, and the caller retains ownership (freeing it with
|
||||
* config_delete() once the call returns). */
|
||||
int send_files(Config* config);
|
||||
int send_files_multithreaded(Config** config);
|
||||
int send_files_multithreaded(Config* config);
|
||||
/* rsync's --ignore-errors deletion gate: with no I/O error during the scan the
|
||||
* deletion phase always proceeds; with one it is suppressed unless
|
||||
* `--ignore-errors` was given. Exposed so the decision can be unit-tested
|
||||
|
||||
@@ -53,7 +53,7 @@ bool validate_config(const Config* config) {
|
||||
return false;
|
||||
}
|
||||
if (config->compression_threads > 0 && !config->use_compression) {
|
||||
log_message(LOG_LEVEL_ERROR, "--compress-threads requires compression (-c or -z)");
|
||||
log_message(LOG_LEVEL_ERROR, "--compress-threads requires compression (-z/--compress)");
|
||||
return false;
|
||||
}
|
||||
if (config->transport == TRANSPORT_SSH && config->use_sendfile) {
|
||||
@@ -75,6 +75,13 @@ bool validate_config(const Config* config) {
|
||||
log_message(LOG_LEVEL_ERROR, "-4/--ipv4 and -6/--ipv6 are mutually exclusive");
|
||||
return false;
|
||||
}
|
||||
/* rsync 3.4.1 rejects --inplace together with --partial-dir (exit 1): the
|
||||
inplace write path bypasses partial staging, so a partial-dir name would be
|
||||
silently ignored. Match rsync's message and refuse before any I/O. */
|
||||
if (config->inplace && config->partial_dir) {
|
||||
log_message(LOG_LEVEL_ERROR, "--inplace cannot be used with --partial-dir");
|
||||
return false;
|
||||
}
|
||||
if (config->log_file_format && !config->log_file) {
|
||||
log_message(LOG_LEVEL_ERROR, "--log-file-format requires --log-file");
|
||||
return false;
|
||||
@@ -102,6 +109,16 @@ bool validate_config(const Config* config) {
|
||||
log_message(LOG_LEVEL_ERROR, "%s", invariants_error);
|
||||
return false;
|
||||
}
|
||||
/* The receiver rejects a protect-rule block with more than MAX_FILTER_RULES
|
||||
entries as an opaque protocol error; reject an over-limit --filter set here,
|
||||
before any network I/O, with an actionable message. send_protect_entries()
|
||||
re-checks the final built count because cvs-exclude / merge rules can
|
||||
expand it beyond config->filters->size. */
|
||||
if (config->filters && config->filters->size > MAX_FILTER_RULES) {
|
||||
log_message(LOG_LEVEL_ERROR, "too many filter rules: %d (maximum %d)", config->filters->size,
|
||||
MAX_FILTER_RULES);
|
||||
return false;
|
||||
}
|
||||
/* --protocol: FastSync has exactly one wire format, so the forced version
|
||||
must equal the current PROTOCOL_VERSION exactly. Rejected here, before any
|
||||
network I/O, rather than letting the server hit its own mismatch check. */
|
||||
|
||||
+248
-49
@@ -205,11 +205,22 @@ typedef struct {
|
||||
bool referent_error;
|
||||
} ScannerEntry;
|
||||
|
||||
/* One inspected directory entry buffered so the sequential scanner can emit the
|
||||
stream in rsync's flist order. `name` is the raw dirent name (owned here);
|
||||
`entry` is the scanner_inspect_entry() result whose path/link_target are owned
|
||||
when `inspection == 1`; `inspection` is that call's return code (1 keep,
|
||||
0 skip, <0 fatal). */
|
||||
typedef struct {
|
||||
char* name;
|
||||
ScannerEntry entry;
|
||||
int inspection;
|
||||
} SortedEntry;
|
||||
|
||||
/* --one-file-system (-x) decision. Only directories can carry a different
|
||||
* device than their parent (mount points), so this is checked when a child
|
||||
* directory is about to be descended into. */
|
||||
bool scanner_same_filesystem(bool one_file_system, dev_t root_device, dev_t entry_device) {
|
||||
return !one_file_system || entry_device == root_device;
|
||||
bool scanner_same_filesystem(int one_file_system, dev_t root_device, dev_t entry_device) {
|
||||
return one_file_system <= 0 || entry_device == root_device;
|
||||
}
|
||||
|
||||
/* Build a payload-less directory File carrying the captured metadata (when
|
||||
@@ -445,6 +456,40 @@ static void scanner_note_nonreg(const ScannerOptions* options, const char* fs_pa
|
||||
fflush(stdout);
|
||||
}
|
||||
|
||||
/* rsync 3.4.1's `--info=mount` line, emitted when `-xx` drops a mount-point
|
||||
* directory: `[sender] skipping mount-point dir NAME` (the client is the
|
||||
* sender). Plain `-x` keeps the empty directory and prints nothing, matching
|
||||
* rsync. */
|
||||
static void scanner_note_mount(const ScannerOptions* options, const char* fs_path) {
|
||||
if (!options || !options->note_mount || !fs_path)
|
||||
return;
|
||||
const char* rel = utils_strip_transfer_root(fs_path, options->send_directory);
|
||||
char* escaped = output_escape(rel, options->eight_bit_output);
|
||||
printf("[sender] skipping mount-point dir %s\n", escaped ? escaped : rel);
|
||||
free(escaped);
|
||||
fflush(stdout);
|
||||
}
|
||||
|
||||
/* --debug=filter: a selection/filter decision dropped an entry. */
|
||||
static void scanner_note_filter(const ScannerOptions* options, const char* name) {
|
||||
if (!options || !log_debug_enabled(LOG_DEBUG_FILTER) || !name)
|
||||
return;
|
||||
log_debug_message(LOG_DEBUG_FILTER, "filter: excluded %s", name);
|
||||
}
|
||||
|
||||
/* Account for a directory that will not be represented by an inline directory
|
||||
* entry. Paired with scanner_dir_count_uncount for empty directories that are
|
||||
* emitted inline, so every traversed directory is counted exactly once. */
|
||||
static void scanner_dir_count_count(const ScannerOptions* options) {
|
||||
if (options && options->dir_count)
|
||||
atomic_fetch_add(options->dir_count, 1);
|
||||
}
|
||||
|
||||
static void scanner_dir_count_uncount(const ScannerOptions* options) {
|
||||
if (options && options->dir_count)
|
||||
atomic_fetch_sub(options->dir_count, 1);
|
||||
}
|
||||
|
||||
/* A user-selection exclusion (--filter/-C/per-dir or --exclude/--include). */
|
||||
static void scanner_record_excluded(DirectoryScanner* scanner, const char* fs_path) {
|
||||
scanner_record_protected(scanner, fs_path, scanner->options.excluded_paths);
|
||||
@@ -687,6 +732,114 @@ skip:
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void sorted_entry_destroy(void* item) {
|
||||
SortedEntry* se = (SortedEntry*)item;
|
||||
if (!se)
|
||||
return;
|
||||
free(se->name);
|
||||
free(se->entry.path);
|
||||
free(se->entry.link_target);
|
||||
}
|
||||
|
||||
/* rsync flist order within one directory: non-directories first, then
|
||||
directories, each group by ascending name. strcmp() compares as unsigned
|
||||
char, matching rsync's f_name_cmp(). */
|
||||
static int sorted_entry_cmp(const void* a, const void* b) {
|
||||
const SortedEntry* x = (const SortedEntry*)a;
|
||||
const SortedEntry* y = (const SortedEntry*)b;
|
||||
bool x_dir = x->inspection > 0 && x->entry.is_directory;
|
||||
bool y_dir = y->inspection > 0 && y->entry.is_directory;
|
||||
if (x_dir != y_dir)
|
||||
return x_dir ? 1 : -1;
|
||||
return strcmp(x->name, y->name);
|
||||
}
|
||||
|
||||
static void scanner_free_sorted(DirectoryScanner* scanner) {
|
||||
SortedEntry* entries = (SortedEntry*)scanner->sorted_entries;
|
||||
for (size_t i = 0; i < scanner->sorted_count; i++)
|
||||
sorted_entry_destroy(&entries[i]);
|
||||
free(entries);
|
||||
scanner->sorted_entries = NULL;
|
||||
scanner->sorted_count = 0;
|
||||
scanner->sorted_index = 0;
|
||||
}
|
||||
|
||||
/* Read every entry of the open directory, inspect it once and store it sorted in
|
||||
rsync's flist order. Returns 0 on success, -1 on a fatal error (the caller
|
||||
aborts the scan). */
|
||||
static int scanner_buffer_current_directory(DirectoryScanner* scanner) {
|
||||
size_t capacity = 64;
|
||||
size_t count = 0;
|
||||
SortedEntry* entries = malloc(capacity * sizeof(*entries));
|
||||
if (!entries) {
|
||||
scanner->failed = true;
|
||||
return -1;
|
||||
}
|
||||
const struct dirent* dirent;
|
||||
while ((dirent = readdir(scanner->current_dir)) != NULL) {
|
||||
if (strcmp(dirent->d_name, ".") == 0 || strcmp(dirent->d_name, "..") == 0)
|
||||
continue;
|
||||
if (count == capacity) {
|
||||
size_t next = capacity * 2;
|
||||
SortedEntry* grown = realloc(entries, next * sizeof(*entries));
|
||||
if (!grown) {
|
||||
scanner->failed = true;
|
||||
break;
|
||||
}
|
||||
entries = grown;
|
||||
capacity = next;
|
||||
}
|
||||
char* name = str_dup(dirent->d_name);
|
||||
if (!name) {
|
||||
scanner->failed = true;
|
||||
break;
|
||||
}
|
||||
char* link_rel = child_rel_path(scanner->current_rel, dirent->d_name);
|
||||
if (!link_rel) {
|
||||
free(name);
|
||||
scanner->failed = true;
|
||||
break;
|
||||
}
|
||||
int inspection = scanner_inspect_entry(&scanner->options, scanner->current_path, link_rel,
|
||||
dirent->d_name, &entries[count].entry);
|
||||
free(link_rel);
|
||||
if (inspection < 0) {
|
||||
free(name);
|
||||
scanner->failed = true;
|
||||
break;
|
||||
}
|
||||
entries[count].name = name;
|
||||
entries[count].inspection = inspection;
|
||||
count++;
|
||||
}
|
||||
if (scanner->failed) {
|
||||
for (size_t i = 0; i < count; i++)
|
||||
sorted_entry_destroy(&entries[i]);
|
||||
free(entries);
|
||||
return -1;
|
||||
}
|
||||
qsort(entries, count, sizeof(*entries), sorted_entry_cmp);
|
||||
scanner->sorted_entries = entries;
|
||||
scanner->sorted_count = count;
|
||||
scanner->sorted_index = 0;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Push this directory's collected child directories onto the LIFO stack in
|
||||
reverse so the first (ascending) child is popped first (depth-first). */
|
||||
static void scanner_push_pending_dirs(DirectoryScanner* scanner) {
|
||||
ArrayList* pending = (ArrayList*)scanner->pending_dirs;
|
||||
if (!pending)
|
||||
return;
|
||||
for (int i = pending->size - 1; i >= 0; i--) {
|
||||
if (!queue_push(scanner->directories, pending->items[i])) {
|
||||
dir_entry_destroy(pending->items[i]);
|
||||
scanner->failed = true;
|
||||
}
|
||||
}
|
||||
pending->size = 0;
|
||||
}
|
||||
|
||||
DirectoryScanner* directory_scanner_create_with_options(const char* root_directory,
|
||||
const ScannerOptions* options) {
|
||||
if (!root_directory || !options)
|
||||
@@ -704,12 +857,22 @@ DirectoryScanner* directory_scanner_create_with_options(const char* root_directo
|
||||
free(scanner);
|
||||
return NULL;
|
||||
}
|
||||
scanner->pending_dirs = array_list_create(NULL);
|
||||
if (!scanner->pending_dirs) {
|
||||
queue_destroy(scanner->directories);
|
||||
free(scanner);
|
||||
return NULL;
|
||||
}
|
||||
scanner->current_dir = NULL;
|
||||
scanner->current_path = NULL;
|
||||
scanner->current_depth = 0;
|
||||
scanner->failed = false;
|
||||
scanner->sorted_entries = NULL;
|
||||
scanner->sorted_count = 0;
|
||||
scanner->sorted_index = 0;
|
||||
scanner->root_path = str_dup(root_directory);
|
||||
if (!scanner->root_path) {
|
||||
array_list_delete(scanner->pending_dirs);
|
||||
queue_destroy(scanner->directories);
|
||||
free(scanner);
|
||||
return NULL;
|
||||
@@ -729,6 +892,7 @@ DirectoryScanner* directory_scanner_create_with_options(const char* root_directo
|
||||
scanner->filter_nodes = array_list_create(filter_node_destroy);
|
||||
if (!scanner->filter_nodes) {
|
||||
free(scanner->root_path);
|
||||
array_list_delete(scanner->pending_dirs);
|
||||
queue_destroy(scanner->directories);
|
||||
free(scanner);
|
||||
return NULL;
|
||||
@@ -739,6 +903,7 @@ DirectoryScanner* directory_scanner_create_with_options(const char* root_directo
|
||||
if (stat(root_directory, &root_stats) != 0) {
|
||||
log_perror("Could not stat source directory");
|
||||
free(scanner->root_path);
|
||||
array_list_delete(scanner->pending_dirs);
|
||||
queue_destroy(scanner->directories);
|
||||
array_list_delete(scanner->filter_nodes);
|
||||
free(scanner);
|
||||
@@ -757,6 +922,7 @@ DirectoryScanner* directory_scanner_create_with_options(const char* root_directo
|
||||
if (!queue_enqueue(scanner->directories, root)) {
|
||||
dir_entry_destroy(root);
|
||||
free(scanner->root_path);
|
||||
array_list_delete(scanner->pending_dirs);
|
||||
queue_destroy(scanner->directories);
|
||||
array_list_delete(scanner->filter_nodes);
|
||||
free(scanner);
|
||||
@@ -808,6 +974,13 @@ void directory_scanner_destroy(DirectoryScanner* scanner) {
|
||||
free(scanner->current_path);
|
||||
free(scanner->current_rel);
|
||||
free(scanner->root_path);
|
||||
scanner_free_sorted(scanner);
|
||||
ArrayList* pending = (ArrayList*)scanner->pending_dirs;
|
||||
if (pending) {
|
||||
for (int i = 0; i < pending->size; i++)
|
||||
dir_entry_destroy(pending->items[i]);
|
||||
array_list_delete(pending);
|
||||
}
|
||||
array_list_delete(scanner->filter_nodes);
|
||||
array_list_delete(scanner->dirs_batch);
|
||||
queue_destroy(scanner->directories);
|
||||
@@ -957,6 +1130,9 @@ static bool scanner_emit_empty_dir(DirectoryScanner* scanner, ArrayList* chunk_d
|
||||
file_destroy(dir);
|
||||
return false;
|
||||
}
|
||||
/* The directory was counted when it was opened; this inline entry represents
|
||||
it, so drop the counter to avoid counting it twice in --stats. */
|
||||
scanner_dir_count_uncount(&scanner->options);
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -975,7 +1151,7 @@ static int open_next_directory(DirectoryScanner* scanner) {
|
||||
scanner->current_path = NULL;
|
||||
|
||||
while (!queue_is_empty(scanner->directories)) {
|
||||
DirEntry* de = (DirEntry*)queue_dequeue(scanner->directories);
|
||||
DirEntry* de = (DirEntry*)queue_pop(scanner->directories);
|
||||
scanner->current_path = de->path;
|
||||
scanner->current_depth = de->depth;
|
||||
/* The seed directory inherits the scanner's configured context (the root
|
||||
@@ -1046,6 +1222,8 @@ static int open_next_directory(DirectoryScanner* scanner) {
|
||||
scanner->failed = true;
|
||||
return -1;
|
||||
}
|
||||
scanner_dir_count_count(&scanner->options);
|
||||
log_debug_message(LOG_DEBUG_FLIST, "flist: scanning %s", scanner->current_path);
|
||||
if (scanner->options.capture_dir_times &&
|
||||
!scanner_capture_dir_time(
|
||||
scanner->options.dir_entries, scanner->options.dir_entries_mutex, scanner->root_path,
|
||||
@@ -1060,6 +1238,14 @@ static int open_next_directory(DirectoryScanner* scanner) {
|
||||
scanner->failed = true;
|
||||
return -1;
|
||||
}
|
||||
/* Buffer and sort this directory's entries in rsync's flist order. */
|
||||
if (scanner_buffer_current_directory(scanner) != 0) {
|
||||
closedir(scanner->current_dir);
|
||||
scanner->current_dir = NULL;
|
||||
free(scanner->current_path);
|
||||
scanner->current_path = NULL;
|
||||
return -1;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
@@ -1304,6 +1490,17 @@ static File* dirs_next_file(DirectoryScanner* scanner) {
|
||||
scanner->dirs_root_emitted = true;
|
||||
if (scanner->options.prune_empty_dirs && dirs_source_dir_is_empty(scanner->root_path))
|
||||
return NULL;
|
||||
/* The listed directory's direct children are about to be enumerated, so
|
||||
its destination mirror is a synchronized directory: record it for the
|
||||
per-directory delete plan. The plan keeps the enumerated children and
|
||||
shields untraversed subdirectories, so --delete-during removes extras
|
||||
directly inside the listed directory without descending into a kept
|
||||
(but untraversed) child -- exactly rsync's `-d DIR/ --delete`. */
|
||||
if (!scanner_record_synced_dir(&scanner->options, scanner->root_path, "",
|
||||
scanner->relative_mode)) {
|
||||
scanner->failed = true;
|
||||
return NULL;
|
||||
}
|
||||
scanner->current_dir = opendir(scanner->root_path);
|
||||
if (!scanner->current_dir) {
|
||||
scanner->io_error = true;
|
||||
@@ -1415,8 +1612,7 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
|
||||
break;
|
||||
}
|
||||
|
||||
const struct dirent* entry = readdir(scanner->current_dir);
|
||||
if (entry == NULL) {
|
||||
if (scanner->sorted_index >= scanner->sorted_count) {
|
||||
/* The directory is exhausted: if nothing was transferred or descended
|
||||
from it, recreate it at the destination as an explicit entry. */
|
||||
if (scanner->options.emit_empty_dirs && !scanner->current_dir_produced &&
|
||||
@@ -1425,10 +1621,12 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
|
||||
if (!scanner_emit_empty_dir(scanner, chunk_data))
|
||||
scanner->failed = true;
|
||||
}
|
||||
scanner_push_pending_dirs(scanner);
|
||||
closedir(scanner->current_dir);
|
||||
scanner->current_dir = NULL;
|
||||
free(scanner->current_path);
|
||||
scanner->current_path = NULL;
|
||||
scanner_free_sorted(scanner);
|
||||
if (scanner->failed) {
|
||||
array_list_delete(chunk_data);
|
||||
return NULL;
|
||||
@@ -1436,26 +1634,15 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
|
||||
continue;
|
||||
SortedEntry* sorted = &((SortedEntry*)scanner->sorted_entries)[scanner->sorted_index++];
|
||||
const char* name = sorted->name;
|
||||
ScannerEntry* inspected = &sorted->entry;
|
||||
int inspection = sorted->inspection;
|
||||
|
||||
ScannerEntry inspected;
|
||||
char* link_rel = child_rel_path(scanner->current_rel, entry->d_name);
|
||||
if (!link_rel) {
|
||||
scanner->failed = true;
|
||||
break;
|
||||
}
|
||||
int inspection = scanner_inspect_entry(&scanner->options, scanner->current_path, link_rel,
|
||||
entry->d_name, &inspected);
|
||||
free(link_rel);
|
||||
if (inspection < 0) {
|
||||
scanner->failed = true;
|
||||
break;
|
||||
}
|
||||
if (inspection == 0) {
|
||||
/* A dereferenced symlink with no referent is a partial-transfer error
|
||||
(rsync exit 23): record it as a non-fatal scan I/O error. */
|
||||
if (inspected.referent_error)
|
||||
if (inspected->referent_error)
|
||||
scanner->io_error = true;
|
||||
/* A user-selection exclude protects its destination mirror from --delete
|
||||
unless --delete-excluded; a size prune is always protected. Other
|
||||
@@ -1463,23 +1650,23 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
|
||||
--files-from the protected prefix must be the entry's bare relative
|
||||
wire path, not its source path (which would not match the destination
|
||||
layout and would leave the mirror deletable). */
|
||||
if (inspected.excluded) {
|
||||
if (inspected->excluded) {
|
||||
char* protected_path;
|
||||
if (scanner->relative_mode) {
|
||||
protected_path = child_rel_path(scanner->current_rel, entry->d_name);
|
||||
protected_path = child_rel_path(scanner->current_rel, name);
|
||||
} else if (scanner->options.relative_prefix) {
|
||||
char* relc = child_rel_path(scanner->current_rel, entry->d_name);
|
||||
char* relc = child_rel_path(scanner->current_rel, name);
|
||||
protected_path =
|
||||
relc ? scanner_prefix_send_path(scanner->options.relative_prefix, relc) : NULL;
|
||||
free(relc);
|
||||
} else {
|
||||
protected_path = path_cat(scanner->current_path, entry->d_name);
|
||||
protected_path = path_cat(scanner->current_path, name);
|
||||
}
|
||||
if (!protected_path) {
|
||||
scanner->failed = true;
|
||||
break;
|
||||
}
|
||||
if (inspected.size_excluded)
|
||||
if (inspected->size_excluded)
|
||||
scanner_record_size_skipped(scanner, protected_path);
|
||||
else
|
||||
scanner_record_excluded(scanner, protected_path);
|
||||
@@ -1487,23 +1674,22 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
|
||||
}
|
||||
continue;
|
||||
}
|
||||
char* cur_path = inspected.path;
|
||||
struct stat stats = inspected.stats;
|
||||
char* cur_path = inspected->path;
|
||||
struct stat stats = inspected->stats;
|
||||
|
||||
/* --files-from allow-set and the filter layer apply to files and to
|
||||
* directories (an excluded directory is not descended into). */
|
||||
bool is_dir = inspected.is_directory;
|
||||
char* rel = child_rel_path(scanner->current_rel, entry->d_name);
|
||||
bool is_dir = inspected->is_directory;
|
||||
char* rel = child_rel_path(scanner->current_rel, name);
|
||||
if (!rel) {
|
||||
free(cur_path);
|
||||
scanner->failed = true;
|
||||
break;
|
||||
}
|
||||
bool protect = false;
|
||||
bool passes_selection = entry_passes_selection(
|
||||
scanner->options.file_list, scanner->options.base_filters, scanner->current_node, rel,
|
||||
entry->d_name, is_dir, scanner->options.per_dir_filters,
|
||||
scanner->options.exclude_per_dir_filter_files, &protect);
|
||||
scanner->options.file_list, scanner->options.base_filters, scanner->current_node, rel, name,
|
||||
is_dir, scanner->options.per_dir_filters, scanner->options.exclude_per_dir_filter_files,
|
||||
&protect);
|
||||
/* A sender-side hide leaves the entry out of the transfer; an independent
|
||||
receiver-side protect rule keeps a transferred entry's destination mirror
|
||||
from being deleted. Both are recorded in the same protection set. */
|
||||
@@ -1525,7 +1711,6 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
|
||||
char* wrel = scanner_prefix_send_path(scanner->options.relative_prefix, rel);
|
||||
if (!wrel) {
|
||||
free(rel);
|
||||
free(cur_path);
|
||||
scanner->failed = true;
|
||||
break;
|
||||
}
|
||||
@@ -1542,13 +1727,12 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
|
||||
char* rel_copy = needs_rel ? str_dup(rel) : NULL;
|
||||
free(rel);
|
||||
if (rel_copy == NULL && needs_rel) {
|
||||
free(cur_path);
|
||||
scanner->failed = true;
|
||||
break;
|
||||
}
|
||||
if (!passes_selection) {
|
||||
scanner_note_filter(&scanner->options, name);
|
||||
free(rel_copy);
|
||||
free(cur_path);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -1556,6 +1740,13 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
|
||||
free(rel_copy);
|
||||
if (!scanner_same_filesystem(scanner->options.one_file_system, scanner->root_dev,
|
||||
stats.st_dev)) {
|
||||
if (scanner->options.one_file_system > 1) {
|
||||
/* rsync's -xx drops the mount-point directory entirely (the plain -x
|
||||
path below keeps it as an empty directory) and prints the
|
||||
--info=mount line when that category is enabled. */
|
||||
scanner_note_mount(&scanner->options, cur_path);
|
||||
continue;
|
||||
}
|
||||
/* rsync's -x/--one-file-system emits the mount-point directory entry
|
||||
itself (so the destination gets an empty directory) but does NOT
|
||||
descend into it. Build a payload-less directory File and hand it to
|
||||
@@ -1563,12 +1754,10 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
|
||||
File* mount = scanner_build_dir_file(cur_path, &stats, &scanner->options);
|
||||
if (mount == NULL || !array_list_add(chunk_data, mount)) {
|
||||
file_destroy(mount);
|
||||
free(cur_path);
|
||||
scanner->failed = true;
|
||||
break;
|
||||
}
|
||||
scanner->current_dir_produced = true;
|
||||
free(cur_path);
|
||||
continue;
|
||||
}
|
||||
/* --list-only: list directory entries too (rsync prints them), even
|
||||
@@ -1577,7 +1766,6 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
|
||||
File* dir = scanner_build_dir_file(cur_path, &stats, &scanner->options);
|
||||
if (dir == NULL || !array_list_add(chunk_data, dir)) {
|
||||
file_destroy(dir);
|
||||
free(cur_path);
|
||||
scanner->failed = true;
|
||||
break;
|
||||
}
|
||||
@@ -1586,32 +1774,29 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
|
||||
int next_depth = scanner->current_depth + 1;
|
||||
if (scanner->options.max_depth <= 0 || next_depth < scanner->options.max_depth) {
|
||||
DirEntry* de = dir_entry_create(cur_path, next_depth, scanner->current_node);
|
||||
if (!de || !queue_enqueue(scanner->directories, de)) {
|
||||
if (!de || !array_list_add((ArrayList*)scanner->pending_dirs, de)) {
|
||||
dir_entry_destroy(de);
|
||||
scanner->failed = true;
|
||||
}
|
||||
}
|
||||
free(cur_path);
|
||||
} else {
|
||||
if (scanner->options.max_depth > 0 &&
|
||||
scanner->current_depth + 1 > scanner->options.max_depth) {
|
||||
free(rel_copy);
|
||||
free(cur_path);
|
||||
continue;
|
||||
}
|
||||
File* file = file_create(cur_path);
|
||||
free(cur_path);
|
||||
if (file == NULL) {
|
||||
free(rel_copy);
|
||||
free(inspected.link_target);
|
||||
inspected.link_target = NULL;
|
||||
free(inspected->link_target);
|
||||
inspected->link_target = NULL;
|
||||
scanner->failed = true;
|
||||
continue;
|
||||
}
|
||||
if (inspected.is_symlink) {
|
||||
if (inspected->is_symlink) {
|
||||
file->is_symlink = true;
|
||||
file->symlink_target = inspected.link_target;
|
||||
inspected.link_target = NULL;
|
||||
file->symlink_target = inspected->link_target;
|
||||
inspected->link_target = NULL;
|
||||
} else {
|
||||
file->data->size = stats.st_size;
|
||||
}
|
||||
@@ -1975,6 +2160,7 @@ static void scan_root_entry(const ScannerOptions* options, const FilterNode* roo
|
||||
free(prefixed);
|
||||
}
|
||||
if (!passes) {
|
||||
scanner_note_filter(options, entry->d_name);
|
||||
free(rel);
|
||||
free(cur_path);
|
||||
return;
|
||||
@@ -1982,6 +2168,14 @@ static void scan_root_entry(const ScannerOptions* options, const FilterNode* roo
|
||||
}
|
||||
if (is_dir) {
|
||||
if (!scanner_same_filesystem(options->one_file_system, root_dev, st.st_dev)) {
|
||||
if (options->one_file_system > 1) {
|
||||
/* -xx: drop the mount-point directory entirely (rsync) and print the
|
||||
--info=mount line when enabled. */
|
||||
scanner_note_mount(options, cur_path);
|
||||
free(rel);
|
||||
free(cur_path);
|
||||
return;
|
||||
}
|
||||
/* -x/--one-file-system: emit the mount-point directory entry (empty) but
|
||||
do not descend into it (see the sequential scanner for the same rule). */
|
||||
File* mount = file_create(cur_path);
|
||||
@@ -2119,6 +2313,7 @@ static bool scan_root_directory(ParallelScanner* ps, const char* root_directory,
|
||||
ps->failed = true;
|
||||
return false;
|
||||
}
|
||||
log_debug_message(LOG_DEBUG_FLIST, "flist: scanning %s", root_directory);
|
||||
const struct dirent* entry;
|
||||
while ((entry = readdir(dir)) != NULL) {
|
||||
if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
|
||||
@@ -2283,6 +2478,10 @@ ParallelScanner* parallel_scanner_create_with_options(const char* root_directory
|
||||
parallel_scanner_destroy(ps);
|
||||
return NULL;
|
||||
}
|
||||
/* The root itself is a traversed directory (rsync counts it in
|
||||
`Number of files`); the worker DirectoryScanners account for every
|
||||
subdirectory below it. */
|
||||
scanner_dir_count_count(options);
|
||||
/* P7 Wave D: the parallel scanner never runs a DirectoryScanner over the
|
||||
transfer root itself (it hands the root's immediate subdirectories to
|
||||
workers), so capture the root's directory time here. */
|
||||
|
||||
+25
-2
@@ -10,6 +10,7 @@
|
||||
#include "stop_condition.h"
|
||||
#include <dirent.h>
|
||||
#include <stdbool.h>
|
||||
#include <stddef.h>
|
||||
#include <stdatomic.h>
|
||||
#include <sys/types.h>
|
||||
#include <threads.h>
|
||||
@@ -50,7 +51,7 @@ typedef struct {
|
||||
bool copy_dirlinks;
|
||||
bool munge_links;
|
||||
bool checksum;
|
||||
bool one_file_system;
|
||||
int one_file_system;
|
||||
/* Phase 4 special/devices: whether device nodes (--devices) and special files
|
||||
* (--specials) are preserved via recreation, and whether --copy-devices
|
||||
* copies a device's content as an ordinary regular file. */
|
||||
@@ -129,6 +130,17 @@ typedef struct {
|
||||
/* --info=nonreg: print rsync's `skipping non-regular file "NAME"` line for a
|
||||
* non-regular entry that is not being preserved. Client-only. */
|
||||
bool note_nonreg;
|
||||
/* --info=mount: print rsync's `[sender] skipping mount-point dir NAME` when
|
||||
* -xx drops a mount-point directory. Client-only. */
|
||||
bool note_mount;
|
||||
/* --stats directory accounting for a `-r` run (no -t/-p): a shared counter of
|
||||
* traversed directories that are NOT otherwise represented by an inline
|
||||
* directory entry (rsync still counts every directory in `Number of files`).
|
||||
* Incremented when a directory is opened and decremented when an empty
|
||||
* directory is emitted inline (so it is counted exactly once). Atomic
|
||||
* because the parallel scanner's workers share it; NULL disables the
|
||||
* accounting. Client-only. */
|
||||
atomic_ullong* dir_count;
|
||||
/* Source root and 8-bit-output policy used to render a `--info=nonreg` name
|
||||
* relative to the transfer root. Borrowed read-only. */
|
||||
const char* send_directory;
|
||||
@@ -188,6 +200,17 @@ typedef struct {
|
||||
int current_depth;
|
||||
dev_t root_dev;
|
||||
bool failed;
|
||||
/* rsync-order traversal: each opened directory's entries are inspected once
|
||||
and buffered (an internal SortedEntry[] owned here) sorted as rsync's flist
|
||||
orders them -- non-directories ascending, then directories ascending. The
|
||||
entries are walked in order and child directories are collected in
|
||||
`pending_dirs` (an ArrayList of DirEntry*, owned here) and pushed onto the
|
||||
LIFO `directories` stack in reverse at directory exhaustion, so the emitted
|
||||
stream is depth-first like rsync. `sorted_*` are reset per directory. */
|
||||
void* sorted_entries;
|
||||
size_t sorted_count;
|
||||
size_t sorted_index;
|
||||
void* pending_dirs;
|
||||
/* Recursive scan: whether the open directory yielded any transferred or
|
||||
descended entry. When it did not, closing it emits a directory entry so
|
||||
the empty source directory is recreated at the destination (rsync
|
||||
@@ -254,7 +277,7 @@ void directory_scanner_destroy(DirectoryScanner* scanner);
|
||||
/* --one-file-system (-x) decision: a directory entry may be descended into
|
||||
* only when the option is disabled or the entry lives on the same device as
|
||||
* the transfer root. Exposed so tests can exercise the rule directly. */
|
||||
bool scanner_same_filesystem(bool one_file_system, dev_t root_device, dev_t entry_device);
|
||||
bool scanner_same_filesystem(int one_file_system, dev_t root_device, dev_t entry_device);
|
||||
|
||||
/* Relative path of an on-disk path below `root` ("" == the root itself, NULL
|
||||
* when `fs_path` is not under `root`). Handles trailing slashes and a root of
|
||||
|
||||
+24
-12
@@ -19,7 +19,9 @@ void print_usage(void) {
|
||||
printf("\n");
|
||||
printf("Options:\n");
|
||||
printf(" -c, --checksum Verify content by checksum instead of size+mtime\n");
|
||||
printf(" -z, --compress [level] Enable compression (level 1-22, default 5)\n");
|
||||
printf(" -z, --compress [level] Enable compression. The default level is\n");
|
||||
printf(" per-codec: zstd 3 (range 1-22), zlib/zlibx 6, lz4\n");
|
||||
printf(" ignores the level\n");
|
||||
printf(" -a, --archive rsync archive mode (-rlptgoD): links, perms, times,\n");
|
||||
printf(" owner, group, devices and specials; not\n");
|
||||
printf(" compression/multithreading\n");
|
||||
@@ -36,8 +38,8 @@ void print_usage(void) {
|
||||
printf(" arguments, e.g. -e \"ssh -p 2222\"\n");
|
||||
printf(" --rsync-path <path> Alias for --fastsync-server-path (path to the\n");
|
||||
printf(" fastsync server binary on the remote side)\n");
|
||||
printf(" --blocking-io Leave the SSH transport socket without read/write\n");
|
||||
printf(" timeouts so it blocks naturally\n");
|
||||
printf(" --blocking-io SSH transport only: leave the socket without read/write\n");
|
||||
printf(" timeouts so it blocks naturally (no effect on TCP)\n");
|
||||
printf(" --outbuf=MODE stdout/stderr buffering: N (none/unbuffered),\n");
|
||||
printf(" L (line-buffered), or B (block-buffered, default)\n");
|
||||
printf(" --progress Show transfer progress\n");
|
||||
@@ -49,6 +51,7 @@ void print_usage(void) {
|
||||
printf(" converted before transmission and back on receipt; a\n");
|
||||
printf(" name that cannot be represented in the target charset\n");
|
||||
printf(" fails that transfer cleanly (rsync-compatible)\n");
|
||||
printf(" --no-iconv Disable --iconv charset conversion (same as --iconv=-)\n");
|
||||
printf(" --protocol=NUM Force the wire protocol version (must equal the current\n");
|
||||
printf(" PROTOCOL_VERSION; FastSync cannot speak older/virtual\n");
|
||||
printf(" wire formats)\n");
|
||||
@@ -162,7 +165,7 @@ void print_usage(void) {
|
||||
printf(" --no-delta, or --no-incremental)\n");
|
||||
printf(" --no-fuzzy Disable --fuzzy\n");
|
||||
printf(" -B <n>, --block-size <n>, --delta-block <n>\n");
|
||||
printf(" Delta block size in bytes (default: %d)\n", DELTA_BLOCK_SIZE_DEFAULT);
|
||||
printf(" Delta block size in bytes (default: %u)\n", DELTA_BLOCK_SIZE_DEFAULT);
|
||||
printf(" --delta-max <n> Max file size for delta transfer (default: %llu)\n",
|
||||
DELTA_MAX_FILE_SIZE);
|
||||
printf(" -j, --threads[=N] Enable the multithreaded scanner/loader/sender\n");
|
||||
@@ -192,6 +195,10 @@ void print_usage(void) {
|
||||
printf(" -U, --atimes Preserve access times\n");
|
||||
printf(" -N, --crtimes Capture birth time; cannot be applied (documented\n");
|
||||
printf(" divergence)\n");
|
||||
printf(" -O, --omit-dir-times Do not apply modification times to directories\n");
|
||||
printf(" -J, --omit-link-times Do not apply times to symlinks\n");
|
||||
printf(" --open-noatime Open source files with O_NOATIME so reading for a\n");
|
||||
printf(" transfer does not update their access time\n");
|
||||
printf(" -X, --xattrs Preserve user extended attributes (user.* only;\n");
|
||||
printf(" privileged security.*/trusted.* namespaces are\n");
|
||||
printf(" never captured or applied)\n");
|
||||
@@ -287,8 +294,12 @@ void print_usage(void) {
|
||||
printf(" -x, --one-file-system Do not cross filesystem boundaries\n");
|
||||
printf(" --log-file <path>, --log-file=<path> Write log messages to file\n");
|
||||
printf(" --stderr=MODE Route logging to stderr: errors or all\n");
|
||||
printf(" --msgs2stderr Route all messages to stderr (deprecated spelling of\n");
|
||||
printf(" --stderr=all)\n");
|
||||
printf(" --no-msgs2stderr Select errors-only stderr (deprecated spelling; the\n");
|
||||
printf(" default)\n");
|
||||
printf(" --partial Keep partial files on interrupted transfer\n");
|
||||
printf(" --partial-dir <dir> Directory for partial files\n");
|
||||
printf(" --partial-dir <dir> Directory for partial files (implies --partial)\n");
|
||||
printf(" -T, --temp-dir <dir> Scratch dir for temp files before atomic install.\n");
|
||||
printf(" Confined to the receive root: a relative dir resolves below\n");
|
||||
printf(" it and an absolute/traversal dir is rejected. The dir must\n");
|
||||
@@ -337,7 +348,8 @@ void print_usage(void) {
|
||||
printf(" --append-verify Like --append, but verifies the retained prefix checksum\n");
|
||||
printf(" before appending (falls back to a full transfer on mismatch)\n");
|
||||
printf(" --fsync Fsync every written file before publication\n");
|
||||
printf(" --compress-level <n> Compression level (default: 5)\n");
|
||||
printf(" --compress-level <n> Compression level (per-codec default: zstd 3,\n");
|
||||
printf(" zlib/zlibx 6, lz4 ignores it)\n");
|
||||
printf(" --zl <n> Alias for --compress-level\n");
|
||||
printf(" --skip-compress=LIST Skip compression for suffixes in LIST (separated by\n");
|
||||
printf(" '/' as in rsync, or ','); a leading dot is optional. The\n");
|
||||
@@ -349,19 +361,19 @@ void print_usage(void) {
|
||||
}
|
||||
|
||||
void print_debug_usage(void) {
|
||||
printf("Emitting debug flags: IO,PROTO,PACK,UTIL,ALL,NONE\n");
|
||||
printf("Emitting debug flags: IO,PROTO,PACK,UTIL,FLIST,DEL,HASH,DELTASUM,\n");
|
||||
printf("RECV,FILTER,SEND,ALL,NONE\n");
|
||||
printf("Also accepted for rsync CLI parity (silent): ACL,BACKUP,BIND,CHDIR,\n");
|
||||
printf("CONNECT,CMD,DEL,DELTASUM,DUP,EXIT,FILTER,FLIST,FUZZY,GENR,HASH,HLINK,\n");
|
||||
printf("ICONV,NSTR,OWN,RECV,SEND,TIME.\n");
|
||||
printf("CONNECT,CMD,DUP,EXIT,FUZZY,GENR,HLINK,ICONV,NSTR,OWN,TIME.\n");
|
||||
printf("Flags may be comma-separated, for example: --debug=io,proto\n");
|
||||
printf("An optional level suffix is accepted (e.g. --debug=io2); level 0\n");
|
||||
printf("silences that item. Unknown names are rejected.\n");
|
||||
}
|
||||
|
||||
void print_info_usage(void) {
|
||||
printf("Emitting info flags: COPY,NAME,MISC,SKIP,STATS,ALL,NONE\n");
|
||||
printf("Also accepted for rsync CLI parity (silent): BACKUP,DEL,FLIST,MOUNT,\n");
|
||||
printf("NONREG,PROGRESS,REMOVE,SYMSAFE.\n");
|
||||
printf("Emitting info flags: COPY,MISC,SKIP,STATS,DEL,REMOVE,NAME,FLIST,\n");
|
||||
printf("NONREG,PROGRESS,MOUNT,ALL,NONE\n");
|
||||
printf("Also accepted for rsync CLI parity (silent): BACKUP,SYMS,SYMSAFE.\n");
|
||||
printf("Flags may be comma-separated, for example: --info=name,stats\n");
|
||||
printf("An optional level suffix is accepted (e.g. --info=stats2); level 0\n");
|
||||
printf("silences that item. Unknown names are rejected.\n");
|
||||
|
||||
+44
-18
@@ -226,11 +226,6 @@ static void release_authorization(void) {
|
||||
close(root_fd);
|
||||
}
|
||||
|
||||
static bool path_is_within(const char* root, const char* path) {
|
||||
size_t n = strlen(root);
|
||||
return strncmp(root, path, n) == 0 && (path[n] == '\0' || path[n] == '/');
|
||||
}
|
||||
|
||||
/* --mkpath contract: when the client's destination root directory does not
|
||||
exist yet on the server side, --mkpath tells the server to create it (and
|
||||
any missing leading components) below the authorized root at connection
|
||||
@@ -790,7 +785,7 @@ void handler(int file_descriptor) {
|
||||
if (joined_destination)
|
||||
destination = joined_destination;
|
||||
if (!destination || has_path_traversal(destination) ||
|
||||
!path_is_within(authorized_root, destination)) {
|
||||
!path_is_within_root(authorized_root, destination)) {
|
||||
log_message(LOG_LEVEL_ERROR, "Rejected destination outside authorized root");
|
||||
free(joined_destination);
|
||||
joined_destination = NULL;
|
||||
@@ -1055,17 +1050,42 @@ done:
|
||||
#ifndef FASTSYNC_SERVER_AS_LIB
|
||||
static Server* g_server = NULL;
|
||||
|
||||
/* Signal handler for the foreground daemon/standalone listener.
|
||||
*
|
||||
* Async-signal-safety: _exit(2) is on the POSIX async-signal-safe list and is
|
||||
* the ONLY thing done here. The previous body called server_delete()
|
||||
* (close/free/SSL_CTX_free), daemon_conf_free() and credentials_free(); none of
|
||||
* those (free/malloc, and much of OpenSSL teardown) are async-signal-safe, so a
|
||||
* signal delivered while the main thread was inside malloc/free could deadlock
|
||||
* or corrupt the heap.
|
||||
*
|
||||
* Residual (documented, not hidden): the in-memory teardown is skipped on the
|
||||
* signal path. That is safe because the parent daemon owns no persistent
|
||||
* resource that survives process exit -- the listening socket is closed by the
|
||||
* kernel, the connection registry is an anonymous MAP_SHARED mapping with no
|
||||
* named backing object, and the daemon config/credential stores are plain heap
|
||||
* allocations. Connection children are separate processes and handle their own
|
||||
* temp files/locks. The normal (non-signal) shutdown path in main() still runs
|
||||
* the full teardown, so no cleanup is dropped on the common path. Wiring the
|
||||
* accept loop (transport_tcp.c, outside this change's scope) to a flag-based
|
||||
* self-pipe shutdown would let the frees run context-safely; it is deliberately
|
||||
* deferred rather than risk restructuring the daemon loop. */
|
||||
static void cleanup(int sig) {
|
||||
(void)sig;
|
||||
if (g_server)
|
||||
server_delete(&g_server);
|
||||
daemon_conf_free(g_daemon_conf);
|
||||
g_daemon_conf = NULL;
|
||||
credentials_free(g_credentials);
|
||||
g_credentials = NULL;
|
||||
_exit(0);
|
||||
}
|
||||
|
||||
/* Install a signal handler with sigaction(2) (the required async-signal-safe
|
||||
* install primitive; signal(3) is not specified to be async-signal-safe). */
|
||||
static void install_cleanup_handler(int signo) {
|
||||
struct sigaction action;
|
||||
memset(&action, 0, sizeof(action));
|
||||
action.sa_handler = cleanup;
|
||||
sigemptyset(&action.sa_mask);
|
||||
action.sa_flags = 0;
|
||||
sigaction(signo, &action, NULL);
|
||||
}
|
||||
|
||||
static void print_server_usage(void) {
|
||||
printf("FastSync Server\n");
|
||||
printf("Usage: fastsync-server [options]\n\n");
|
||||
@@ -1178,13 +1198,19 @@ static bool daemonize(void) {
|
||||
close(devnull);
|
||||
}
|
||||
/* Do not pin the launch CWD (module-relative 'path' entries would resolve
|
||||
* against an unstable working directory) and drop the restrictive host umask
|
||||
* so modules can create files/dirs with the modes the config requests. */
|
||||
* against an unstable working directory). Set a conservative daemon umask
|
||||
* of 022 (the conventional service default): rsync never forces umask 0 --
|
||||
* it reads and restores the inherited umask and creates new entries as
|
||||
* 0777 & ~umask / source & ~umask without -p. Forcing 0 here made every
|
||||
* implied parent directory world-writable (0777) whenever -p metadata was not
|
||||
* applied. 022 gives 0755 directories and source&~022 files, matching rsync
|
||||
* under a normal daemon umask; -p/-a still restore the exact source mode via
|
||||
* fchmod, which is unaffected by the umask. */
|
||||
if (chdir("/") != 0)
|
||||
log_message(LOG_LEVEL_WARNING, "daemon: chdir to / failed: %s", strerror(errno));
|
||||
umask(0);
|
||||
umask(022);
|
||||
/* Refresh the cached umask: main() captured the launch umask before this
|
||||
* (single-threaded) umask(0), and file_mode_base() must see the daemon's
|
||||
* (single-threaded) umask(022), and file_mode_base() must see the daemon's
|
||||
* actual umask. */
|
||||
file_umask_capture();
|
||||
return true;
|
||||
@@ -1253,8 +1279,8 @@ int main(int argc, char* argv[]) {
|
||||
* this process-global policy cannot be re-enabled by a future caller. */
|
||||
server_allow_super = opts.allow_super && !opts.stdio_mode;
|
||||
server_iconv_spec = opts.iconv_spec;
|
||||
signal(SIGINT, cleanup);
|
||||
signal(SIGTERM, cleanup);
|
||||
install_cleanup_handler(SIGINT);
|
||||
install_cleanup_handler(SIGTERM);
|
||||
/* Server-owned socket deadline floor: the client default --timeout=0 would
|
||||
* otherwise leave accepted sockets without SO_RCVTIMEO/SO_SNDTIMEO and let a
|
||||
* silent peer hold a connection (and its process slot) forever. */
|
||||
|
||||
@@ -3,20 +3,12 @@
|
||||
#include "credentials.h"
|
||||
#include "utils.h"
|
||||
#include <limits.h>
|
||||
#include <stdarg.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/socket.h>
|
||||
|
||||
static void set_error(char* err, size_t err_size, const char* fmt, ...) {
|
||||
if (!err || err_size == 0)
|
||||
return;
|
||||
va_list args;
|
||||
va_start(args, fmt);
|
||||
vsnprintf(err, err_size, fmt, args);
|
||||
va_end(args);
|
||||
}
|
||||
#define set_error utils_set_error
|
||||
|
||||
void server_cli_options_default(ServerCliOptions* opts) {
|
||||
if (!opts)
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
ArrayList* array_list_create(void (*item_destroyer)(void* item)) {
|
||||
ArrayList* list = (ArrayList*)protocol_alloc(sizeof(ArrayList));
|
||||
if (list == NULL) {
|
||||
log_perror("ERROR: Could not allocate memory for array list struct");
|
||||
log_message(LOG_LEVEL_ERROR, "%s", "ERROR: Could not allocate memory for array list struct");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
@@ -47,7 +47,7 @@ static bool array_list_extend(ArrayList* array_list) {
|
||||
new_capacity = INITIAL_ARRAY_SIZE;
|
||||
void* new_items = protocol_realloc(array_list->items, new_capacity * sizeof(void*));
|
||||
if (new_items == NULL) {
|
||||
log_perror("ERROR: Could not reallocate memory for array list items");
|
||||
log_message(LOG_LEVEL_ERROR, "%s", "ERROR: Could not reallocate memory for array list items");
|
||||
return false;
|
||||
}
|
||||
array_list->items = new_items;
|
||||
@@ -73,7 +73,7 @@ void** array_list_to_array(const ArrayList* array_list) {
|
||||
}
|
||||
void** array = protocol_alloc(array_list->size * sizeof(void*));
|
||||
if (array == NULL) {
|
||||
log_perror("Could not malloc space for array from array list!");
|
||||
log_message(LOG_LEVEL_ERROR, "%s", "Could not malloc space for array from array list!");
|
||||
return NULL;
|
||||
}
|
||||
memcpy(array, array_list->items, array_list->size * sizeof(void*));
|
||||
|
||||
+5
-4
@@ -17,8 +17,9 @@
|
||||
#include "protocol.h"
|
||||
#include "utils.h"
|
||||
|
||||
/* Maximum individual file data size within a chunk (64 MB) */
|
||||
#define MAX_FILE_DATA_SIZE (64ULL * 1024 * 1024)
|
||||
/* Maximum individual file data size within a chunk (64 MB). Distinct from the
|
||||
* receiver's whole-file MAX_FILE_DATA_SIZE (256 MB) in file_receive.c. */
|
||||
#define MAX_CHUNK_FILE_DATA_SIZE (64ULL * 1024 * 1024)
|
||||
#define MAX_FILES_PER_CHUNK 65536U
|
||||
|
||||
/* Reserve `charge` against `session`'s connection budget. This mirrors the
|
||||
@@ -382,9 +383,9 @@ Chunk* chunk_deserialize(Data* data, bool use_metadata) {
|
||||
}
|
||||
|
||||
// Reject individual file data larger than the maximum allowed size.
|
||||
if (file_data_size > MAX_FILE_DATA_SIZE) {
|
||||
if (file_data_size > MAX_CHUNK_FILE_DATA_SIZE) {
|
||||
log_message(LOG_LEVEL_ERROR, "File data size %zu exceeds maximum %llu", file_data_size,
|
||||
(unsigned long long)MAX_FILE_DATA_SIZE);
|
||||
(unsigned long long)MAX_CHUNK_FILE_DATA_SIZE);
|
||||
goto error;
|
||||
}
|
||||
|
||||
|
||||
@@ -16,7 +16,14 @@
|
||||
#include <zstd.h>
|
||||
|
||||
#define INITIAL_DECOMPRESS_BUF_SIZE (1024 * 1024)
|
||||
#define MAX_DECOMPRESSED_SIZE (100ULL * 1024 * 1024) /* 100 MB hard ceiling */
|
||||
|
||||
/* Hard ceiling for a single decompression. The sender compresses whole files
|
||||
* up to the protocol's whole-file receive bound, so the decompressor must
|
||||
* accept payloads that large; referencing the protocol constant keeps the two
|
||||
* bounds from drifting apart (they previously did: a 100 MB ceiling rejected
|
||||
* 100-256 MB files). This remains a real bomb guard -- every allocation in the
|
||||
* paths below is clamped to it -- so it must not exceed the protocol bound. */
|
||||
#define MAX_DECOMPRESSED_SIZE MAX_RECEIVE_WHOLE_FILE_SIZE
|
||||
|
||||
/* rsync 3.4.1's built-in skip-compress suffix list (the `--skip-compress`
|
||||
* defaults, in the man page's order). rsync stores it as space-separated
|
||||
@@ -637,8 +644,7 @@ static Data* zstd_decompress(Data* compressed_data, size_t maximum_size) {
|
||||
}
|
||||
if (ret > 0 && output.pos == output.size) {
|
||||
if (buf_size >= hard_limit || buf_size > SIZE_MAX / 2) {
|
||||
log_message(LOG_LEVEL_ERROR, "Decompressed data exceeds %llu bytes",
|
||||
(unsigned long long)MAX_DECOMPRESSED_SIZE);
|
||||
log_message(LOG_LEVEL_ERROR, "Decompressed data exceeds %llu bytes", hard_limit);
|
||||
data_destroy(uncompressed_data);
|
||||
uncompressed_data = NULL;
|
||||
goto cleanup;
|
||||
|
||||
+26
-3
@@ -79,7 +79,7 @@ static void config_set_defaults(Config* config) {
|
||||
config->cvs_exclude = false;
|
||||
config->per_dir_filter = false;
|
||||
config->per_dir_filter_count = 0;
|
||||
config->one_file_system = false;
|
||||
config->one_file_system = 0;
|
||||
config->no_implied_dirs = false;
|
||||
config->dirs = false;
|
||||
config->rsh_command = NULL;
|
||||
@@ -230,6 +230,13 @@ Config* config_create(void) {
|
||||
if (!config)
|
||||
return NULL;
|
||||
config_set_defaults(config);
|
||||
/* config_set_defaults() dups the default server host; a failure there leaves
|
||||
* server_host NULL and would crash later consumers, so fail the whole create
|
||||
* (every caller already handles a NULL return). */
|
||||
if (!config->server_host) {
|
||||
config_delete(config);
|
||||
return NULL;
|
||||
}
|
||||
return config;
|
||||
}
|
||||
|
||||
@@ -686,9 +693,15 @@ int config_parse_ssh_dest(Config* config) {
|
||||
return daemon_dest_parse_error("invalid remote destination user@host (must not be empty or "
|
||||
"start with '-')",
|
||||
dest);
|
||||
config->transport = TRANSPORT_SSH;
|
||||
config->ssh_destination = str_dup(dest);
|
||||
char* ssh_destination = str_dup(dest);
|
||||
char* path = str_dup(colon + 1);
|
||||
if (!ssh_destination || !path) {
|
||||
free(ssh_destination);
|
||||
free(path);
|
||||
return daemon_dest_parse_error("out of memory parsing remote destination", dest);
|
||||
}
|
||||
config->transport = TRANSPORT_SSH;
|
||||
config->ssh_destination = ssh_destination;
|
||||
free(config->receive_root_directory);
|
||||
config->receive_root_directory = path;
|
||||
return 0;
|
||||
@@ -1052,6 +1065,16 @@ static bool send_protect_entries(int fd, const Config* c) {
|
||||
log_message(LOG_LEVEL_ERROR, "invalid filter rule: %s", err);
|
||||
return false;
|
||||
}
|
||||
/* The receiver rejects any block with more than MAX_FILTER_RULES entries as a
|
||||
* protocol error; refuse to emit such a frame at all. filter_base_build()
|
||||
* can expand the client rule set (cvs-exclude, merge files), so this is the
|
||||
* authoritative bound, not config->filters->size. */
|
||||
if (rules->count < 0 || rules->count > MAX_FILTER_RULES) {
|
||||
log_message(LOG_LEVEL_ERROR, "too many filter rules: %d (maximum %d)", rules->count,
|
||||
MAX_FILTER_RULES);
|
||||
filter_rule_list_free(rules);
|
||||
return false;
|
||||
}
|
||||
bool ok = send_int(fd, rules->count);
|
||||
for (int i = 0; ok && i < rules->count; i++) {
|
||||
const FilterRule* r = rules->items[i];
|
||||
|
||||
+3
-1
@@ -463,7 +463,9 @@ typedef struct Config {
|
||||
* /.rsync-filter' (the .rsync-filter files themselves are transferred); a
|
||||
* repeated -F adds --filter='- .rsync-filter' so they are excluded too. */
|
||||
int per_dir_filter_count;
|
||||
bool one_file_system; /* -x/--one-file-system: do not cross filesystem boundaries */
|
||||
int one_file_system; /* -x/--one-file-system: do not cross filesystem boundaries.
|
||||
Repeated -x (rsync's -xx) drops the mount-point
|
||||
directory entirely instead of recreating it empty. */
|
||||
/* --no-implied-dirs: client-only. With -R, do not transfer the source
|
||||
* metadata of the parent directories implied by a listed path; an unlisted
|
||||
* implied parent is still created (with default attributes) so the listed
|
||||
|
||||
+204
-18
@@ -8,12 +8,13 @@
|
||||
#include <openssl/evp.h>
|
||||
#include <openssl/params.h>
|
||||
#include <openssl/rand.h>
|
||||
#include <stdarg.h>
|
||||
#include <poll.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/stat.h>
|
||||
#include <time.h>
|
||||
#include <unistd.h>
|
||||
|
||||
/* One store entry: a username and its salted PBKDF2 verifier. The plaintext
|
||||
@@ -58,19 +59,37 @@ struct CredentialStore {
|
||||
static const uint8_t k_dummy_stored_key[CREDENTIAL_KEY_LEN] = {0};
|
||||
static const uint8_t k_dummy_server_key[CREDENTIAL_KEY_LEN] = {0};
|
||||
|
||||
static void set_error(char* err, size_t err_size, const char* fmt, ...) {
|
||||
if (!err || err_size == 0)
|
||||
return;
|
||||
va_list args;
|
||||
va_start(args, fmt);
|
||||
vsnprintf(err, err_size, fmt, args);
|
||||
va_end(args);
|
||||
}
|
||||
#define set_error utils_set_error
|
||||
|
||||
static bool is_comment_char(char c) {
|
||||
return c == '#' || c == ';';
|
||||
}
|
||||
|
||||
/* True for a literal fd-backed store path: exactly "/dev/fd/<digits>" or
|
||||
* "/proc/self/fd/<digits>", with no trailing component and no "..". These name
|
||||
* the calling process's own open descriptors (e.g. a bash process substitution
|
||||
* `<(...)`, which passes /dev/fd/N), and both prefixes are symlinks by
|
||||
* construction. */
|
||||
static bool is_fd_backed_path(const char* path) {
|
||||
static const char* const prefixes[] = {"/dev/fd/", "/proc/self/fd/"};
|
||||
if (!path)
|
||||
return false;
|
||||
for (size_t i = 0; i < sizeof(prefixes) / sizeof(prefixes[0]); i++) {
|
||||
const char* prefix = prefixes[i];
|
||||
size_t prefix_len = strlen(prefix);
|
||||
if (strncmp(path, prefix, prefix_len) != 0)
|
||||
continue;
|
||||
const char* digits = path + prefix_len;
|
||||
if (*digits < '0' || *digits > '9')
|
||||
return false;
|
||||
const char* p = digits;
|
||||
while (*p >= '0' && *p <= '9')
|
||||
p++;
|
||||
return *p == '\0';
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/* Open a --password-file / --early-input after verifying the EXACT inode we
|
||||
* will read: it must be owned by the effective user and grant no group/other
|
||||
* permission bit (so 0600 and stricter modes such as 0400 are accepted),
|
||||
@@ -80,12 +99,31 @@ static bool is_comment_char(char c) {
|
||||
* path and then fstat the resulting fd (rather than stat()ing the path first
|
||||
* and reopening it), so the permission decision is made on the same inode that
|
||||
* is read and cannot be raced by swapping the path between check and open.
|
||||
* The path may be a process-substitution pipe (`<(...)` -> /dev/fd/N), so
|
||||
* regular files and FIFOs are accepted when the ownership/mode checks pass.
|
||||
* O_NOFOLLOW refuses a symlinked path outright (ELOOP fails closed) instead of
|
||||
* following it before the owner/mode gate can run. The one exception is a
|
||||
* literal fd-backed path (/dev/fd/N or /proc/self/fd/N, see
|
||||
* is_fd_backed_path): those entries are symlinks to the CALLING process's own
|
||||
* descriptors, so following them is not the untrusted-symlink hazard
|
||||
* O_NOFOLLOW guards against, and requiring O_NOFOLLOW would break the
|
||||
* documented process-substitution/FIFO usage. For them only, O_NOFOLLOW is
|
||||
* omitted; the same fstat owner/mode gate still applies to the resolved inode.
|
||||
* O_NONBLOCK keeps the OPEN itself from
|
||||
* blocking forever on a writer-less FIFO (a blocking O_RDONLY open would wait
|
||||
* for a writer). The fd is left nonblocking for FIFOs so a read never blocks
|
||||
* either; the read loop (secret_read_line) absorbs the resulting EAGAIN by
|
||||
* waiting, under a bounded deadline, for the writer -- this is what makes a
|
||||
* slow process substitution (`--password-file <(sleep 1; ...)`) work while a
|
||||
* writer-less FIFO still fails after the deadline instead of hanging. Only
|
||||
* regular files and FIFOs pass the ownership/mode checks; O_NONBLOCK is
|
||||
* cleared for regular files, where it is a no-op anyway and no EAGAIN can
|
||||
* occur, so their stdio read path is byte-for-byte unchanged.
|
||||
*
|
||||
* Returns a FILE* the caller must fclose, or NULL with `err` filled. */
|
||||
static FILE* secret_file_open(const char* path, char* err, size_t err_size) {
|
||||
int fd = open(path, O_RDONLY | O_CLOEXEC);
|
||||
int flags = O_RDONLY | O_NONBLOCK | O_CLOEXEC;
|
||||
if (!is_fd_backed_path(path))
|
||||
flags |= O_NOFOLLOW;
|
||||
int fd = open(path, flags);
|
||||
if (fd < 0) {
|
||||
set_error(err, err_size, "cannot open secret file '%s': %s", path, strerror(errno));
|
||||
return NULL;
|
||||
@@ -105,6 +143,15 @@ static FILE* secret_file_open(const char* path, char* err, size_t err_size) {
|
||||
close(fd);
|
||||
return NULL;
|
||||
}
|
||||
/* O_NONBLOCK is only meaningful for the FIFO allowance. Restore blocking
|
||||
* mode on a regular file so its read path is exactly as before; a no-op on
|
||||
* most systems, but explicit. Failures here are ignored: O_NONBLOCK on a
|
||||
* regular file does not affect reads either way. */
|
||||
if (S_ISREG(st.st_mode)) {
|
||||
int status_flags = fcntl(fd, F_GETFL);
|
||||
if (status_flags >= 0)
|
||||
(void)fcntl(fd, F_SETFL, status_flags & ~O_NONBLOCK);
|
||||
}
|
||||
FILE* fp = fdopen(fd, "r");
|
||||
if (!fp) {
|
||||
set_error(err, err_size, "cannot read secret file '%s': %s", path, strerror(errno));
|
||||
@@ -114,6 +161,123 @@ static FILE* secret_file_open(const char* path, char* err, size_t err_size) {
|
||||
return fp;
|
||||
}
|
||||
|
||||
/* Overall bound on how long the reader waits for a process-substitution/FIFO
|
||||
* writer to produce data before giving up. It must comfortably exceed a
|
||||
* producer's startup delay (e.g. `--password-file <(sleep 1; ...)`) while still
|
||||
* bounding a writer-less FIFO, so a stray or hostile FIFO cannot stall the
|
||||
* daemon or client indefinitely. */
|
||||
#define CREDENTIAL_FIFO_READ_TIMEOUT_MS 3000
|
||||
|
||||
/* Monotonic milliseconds, used only for the read deadline (wall-clock changes
|
||||
* must not extend or shorten the wait). */
|
||||
static int64_t credential_monotonic_ms(void) {
|
||||
struct timespec ts;
|
||||
if (clock_gettime(CLOCK_MONOTONIC, &ts) != 0)
|
||||
return 0;
|
||||
return (int64_t)ts.tv_sec * 1000 + (int64_t)(ts.tv_nsec / 1000000);
|
||||
}
|
||||
|
||||
/* Wait until `fd` is readable or the deadline passes. Returns true when it is
|
||||
* readable, false on timeout or a poll error (err filled). EINTR is retried
|
||||
* against the same deadline, so signals cannot extend the wait. */
|
||||
static bool credential_wait_readable(int fd, int64_t deadline, const char* label, const char* path,
|
||||
char* err, size_t err_size) {
|
||||
for (;;) {
|
||||
int64_t remaining = deadline - credential_monotonic_ms();
|
||||
if (remaining <= 0)
|
||||
break;
|
||||
if (remaining > INT_MAX)
|
||||
remaining = INT_MAX;
|
||||
struct pollfd pfd = {.fd = fd, .events = POLLIN, .revents = 0};
|
||||
int rc = poll(&pfd, 1, (int)remaining);
|
||||
if (rc > 0)
|
||||
return true;
|
||||
if (rc == 0)
|
||||
break;
|
||||
if (errno != EINTR) {
|
||||
set_error(err, err_size, "error waiting for %s '%s': %s", label, path, strerror(errno));
|
||||
return false;
|
||||
}
|
||||
}
|
||||
set_error(err, err_size, "timed out after %d ms waiting for %s '%s'",
|
||||
CREDENTIAL_FIFO_READ_TIMEOUT_MS, label, path);
|
||||
return false;
|
||||
}
|
||||
|
||||
typedef enum {
|
||||
SECRET_READ_LINE,
|
||||
SECRET_READ_EOF,
|
||||
SECRET_READ_ERROR,
|
||||
} SecretReadResult;
|
||||
|
||||
/* Read one complete line from `fp` into `line` (capacity `cap`), including the
|
||||
* trailing newline when present and always NUL-terminating. `*out_len`
|
||||
* receives strlen(line).
|
||||
*
|
||||
* A regular file is read exactly as before: secret_file_open leaves it
|
||||
* blocking, so fgets never sees EAGAIN. A FIFO stays nonblocking, so fgets
|
||||
* returns NULL (or a partial line) with EAGAIN while the writer is still
|
||||
* starting up; instead of treating that as a fatal error the loop clearerr()s
|
||||
* and polls for readability against one overall deadline. The `used`
|
||||
* accumulator reassembles a line that arrived in several write()s into a single
|
||||
* line, so a split write is not misparsed as two entries.
|
||||
*
|
||||
* Returns SECRET_READ_LINE, SECRET_READ_EOF, or SECRET_READ_ERROR (err filled)
|
||||
* on timeout or a genuine read error. */
|
||||
static SecretReadResult secret_read_line(char* line, size_t cap, FILE* fp, const char* label,
|
||||
const char* path, size_t* out_len, char* err,
|
||||
size_t err_size) {
|
||||
int fd = fileno(fp);
|
||||
int64_t deadline = credential_monotonic_ms() + CREDENTIAL_FIFO_READ_TIMEOUT_MS;
|
||||
size_t used = 0;
|
||||
line[0] = '\0';
|
||||
for (;;) {
|
||||
errno = 0;
|
||||
if (fgets(line + used, (int)(cap - used), fp)) {
|
||||
used += strlen(line + used);
|
||||
if (used > 0 && line[used - 1] == '\n') {
|
||||
*out_len = used;
|
||||
return SECRET_READ_LINE;
|
||||
}
|
||||
if (feof(fp)) {
|
||||
*out_len = used; /* final unterminated line */
|
||||
return SECRET_READ_LINE;
|
||||
}
|
||||
/* No newline and not EOF. A full buffer is the caller's over-long-line
|
||||
* case; otherwise the line is only partially available (a nonblocking
|
||||
* FIFO under a slow writer), so any genuine read error fails and anything
|
||||
* else waits for the rest. */
|
||||
if (used >= cap - 1) {
|
||||
*out_len = used;
|
||||
return SECRET_READ_LINE;
|
||||
}
|
||||
int e = ferror(fp) ? errno : 0;
|
||||
if (e != 0 && e != EAGAIN && e != EWOULDBLOCK) {
|
||||
set_error(err, err_size, "error reading %s '%s': %s", label, path, strerror(e));
|
||||
return SECRET_READ_ERROR;
|
||||
}
|
||||
clearerr(fp);
|
||||
if (!credential_wait_readable(fd, deadline, label, path, err, err_size))
|
||||
return SECRET_READ_ERROR;
|
||||
continue;
|
||||
}
|
||||
/* fgets returned NULL: EOF, a not-yet-readable FIFO, or a real error. */
|
||||
if (feof(fp)) {
|
||||
*out_len = used;
|
||||
return used > 0 ? SECRET_READ_LINE : SECRET_READ_EOF;
|
||||
}
|
||||
if (errno == EAGAIN || errno == EWOULDBLOCK) {
|
||||
clearerr(fp);
|
||||
if (!credential_wait_readable(fd, deadline, label, path, err, err_size))
|
||||
return SECRET_READ_ERROR;
|
||||
continue;
|
||||
}
|
||||
set_error(err, err_size, "error reading %s '%s': %s", label, path,
|
||||
errno != 0 ? strerror(errno) : "read failed");
|
||||
return SECRET_READ_ERROR;
|
||||
}
|
||||
}
|
||||
|
||||
/* Trim leading/trailing ASCII space and tab in place; returns the new start. */
|
||||
static char* trim_space(char* s) {
|
||||
while (*s == ' ' || *s == '\t')
|
||||
@@ -509,9 +673,17 @@ static CredentialStore* load_store_file(const char* path, char* err, size_t err_
|
||||
char line[CREDENTIAL_MAX_LINE + 2];
|
||||
bool ok = true;
|
||||
|
||||
while (fgets(line, sizeof(line), fp)) {
|
||||
for (;;) {
|
||||
size_t len = 0;
|
||||
SecretReadResult rr =
|
||||
secret_read_line(line, sizeof(line), fp, "credential file", path, &len, err, err_size);
|
||||
if (rr == SECRET_READ_EOF)
|
||||
break;
|
||||
if (rr == SECRET_READ_ERROR) {
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
line_no++;
|
||||
size_t len = strlen(line);
|
||||
if (len == CREDENTIAL_MAX_LINE + 1 && line[len - 1] != '\n' && !feof(fp)) {
|
||||
set_error(err, err_size, "credential file '%s' line %d exceeds the %d-byte limit", path,
|
||||
line_no, CREDENTIAL_MAX_LINE);
|
||||
@@ -1148,9 +1320,17 @@ int credentials_hash_file(const char* path, uint32_t iters, FILE* out, char* err
|
||||
int line_no = 0;
|
||||
int result = 0;
|
||||
char line[CREDENTIAL_MAX_LINE + 2];
|
||||
while (fgets(line, sizeof(line), fp)) {
|
||||
for (;;) {
|
||||
size_t len = 0;
|
||||
SecretReadResult rr =
|
||||
secret_read_line(line, sizeof(line), fp, "plaintext file", path, &len, err, err_size);
|
||||
if (rr == SECRET_READ_EOF)
|
||||
break;
|
||||
if (rr == SECRET_READ_ERROR) {
|
||||
result = -1;
|
||||
break;
|
||||
}
|
||||
line_no++;
|
||||
size_t len = strlen(line);
|
||||
if (len == CREDENTIAL_MAX_LINE + 1 && line[len - 1] != '\n' && !feof(fp)) {
|
||||
set_error(err, err_size, "plaintext file '%s' line %d exceeds the %d-byte limit", path,
|
||||
line_no, CREDENTIAL_MAX_LINE);
|
||||
@@ -1228,9 +1408,15 @@ int credentials_read_secret_file(const char* path, char** user_out, char** passw
|
||||
char line[CREDENTIAL_MAX_LINE + 2];
|
||||
int result = -1;
|
||||
|
||||
while (fgets(line, sizeof(line), fp)) {
|
||||
for (;;) {
|
||||
size_t len = 0;
|
||||
SecretReadResult rr =
|
||||
secret_read_line(line, sizeof(line), fp, "password file", path, &len, err, err_size);
|
||||
if (rr == SECRET_READ_EOF)
|
||||
break;
|
||||
if (rr == SECRET_READ_ERROR)
|
||||
goto done;
|
||||
line_no++;
|
||||
size_t len = strlen(line);
|
||||
if (len == CREDENTIAL_MAX_LINE + 1 && line[len - 1] != '\n' && !feof(fp)) {
|
||||
set_error(err, err_size, "password file '%s' line %d exceeds the %d-byte limit", path,
|
||||
line_no, CREDENTIAL_MAX_LINE);
|
||||
|
||||
@@ -6,7 +6,6 @@
|
||||
#include <errno.h>
|
||||
#include <limits.h>
|
||||
#include <netinet/in.h>
|
||||
#include <stdarg.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
@@ -17,14 +16,7 @@
|
||||
/* helpers */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
static void set_error(char* err, size_t err_size, const char* fmt, ...) {
|
||||
if (!err || err_size == 0)
|
||||
return;
|
||||
va_list args;
|
||||
va_start(args, fmt);
|
||||
vsnprintf(err, err_size, fmt, args);
|
||||
va_end(args);
|
||||
}
|
||||
#define set_error utils_set_error
|
||||
|
||||
/* Trim leading and trailing ASCII space/tab in place; returns the new start. */
|
||||
static char* trim_ws(char* s) {
|
||||
|
||||
+113
-56
@@ -432,6 +432,17 @@ int delete_plan_send_remaining(int fd, DeletePlanSender* sender, const ArrayList
|
||||
return 0;
|
||||
}
|
||||
|
||||
int delete_plan_send_all(int fd, DeletePlanSender* sender, const ArrayList* dirs) {
|
||||
if (!sender)
|
||||
return -1;
|
||||
/* Root first: this also transmits the one-shot per-run config block on its
|
||||
own carrier frame (see send_config_only), so it reaches the receiver even
|
||||
when the scope permits no directory plan at all. */
|
||||
if (delete_plan_send_root(fd, sender) != 0)
|
||||
return -1;
|
||||
return delete_plan_send_remaining(fd, sender, dirs);
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Receiver: delete session */
|
||||
/* ------------------------------------------------------------------ */
|
||||
@@ -471,6 +482,24 @@ static void notify_deleted(DeletePlanSession* session, const char* rel) {
|
||||
session->observer(session->observer_context, rel);
|
||||
}
|
||||
|
||||
/* A removed directory is reported with rsync's trailing slash (`deleting dir/`)
|
||||
while files keep their bare path. */
|
||||
static void notify_deleted_dir(DeletePlanSession* session, const char* rel) {
|
||||
if (!session || !session->observer || !rel)
|
||||
return;
|
||||
size_t len = strlen(rel);
|
||||
char* with_slash = malloc(len + 2);
|
||||
if (!with_slash) {
|
||||
session->observer(session->observer_context, rel);
|
||||
return;
|
||||
}
|
||||
memcpy(with_slash, rel, len);
|
||||
with_slash[len] = '/';
|
||||
with_slash[len + 1] = '\0';
|
||||
session->observer(session->observer_context, with_slash);
|
||||
free(with_slash);
|
||||
}
|
||||
|
||||
DeletePlanSession* delete_plan_session_create(const Config* config) {
|
||||
if (!config)
|
||||
return NULL;
|
||||
@@ -696,7 +725,7 @@ static bool process_extra_dir(int dirfd, const char* name, const char* child_rel
|
||||
session->deleted++;
|
||||
session->planned++;
|
||||
log_deleted(child_rel);
|
||||
notify_deleted(session, child_rel);
|
||||
notify_deleted_dir(session, child_rel);
|
||||
*removed = true;
|
||||
return true;
|
||||
}
|
||||
@@ -733,81 +762,108 @@ static bool process_children(int dirfd, const char* dir_rel, const ArrayList* ke
|
||||
const ArrayList* keep_files, bool at_root, bool force_now,
|
||||
const PlanSkips* skips, DeletePlanSession* session, bool* survives) {
|
||||
*survives = false;
|
||||
int scanfd = openat(dirfd, ".", O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||
if (scanfd < 0)
|
||||
DeleteDirEntry* entries = NULL;
|
||||
size_t count = 0;
|
||||
bool collect_ok = true;
|
||||
if (!delete_dir_entries_collect(dirfd, &entries, &count, &collect_ok))
|
||||
return false;
|
||||
DIR* dir = fdopendir(scanfd);
|
||||
if (!dir) {
|
||||
close(scanfd);
|
||||
bool operation_ok = collect_ok;
|
||||
bool local_survives = false;
|
||||
bool* shielded = calloc(count ? count : 1, sizeof(bool));
|
||||
bool* is_extra = calloc(count ? count : 1, sizeof(bool));
|
||||
bool* force = calloc(count ? count : 1, sizeof(bool));
|
||||
if (!shielded || !is_extra || !force) {
|
||||
free(shielded);
|
||||
free(is_extra);
|
||||
free(force);
|
||||
delete_dir_entries_free(entries, count);
|
||||
return false;
|
||||
}
|
||||
bool operation_ok = true;
|
||||
bool local_survives = false;
|
||||
const struct dirent* entry;
|
||||
while ((entry = readdir(dir)) != NULL) {
|
||||
if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
|
||||
continue;
|
||||
|
||||
/* rsync's order: extraneous subdirectories in descending name order, then
|
||||
extraneous files in descending name order (kept entries survive and are not
|
||||
touched here — a kept subdirectory gets its own per-directory plan). */
|
||||
if (count > 1)
|
||||
qsort(entries, count, sizeof(*entries), delete_dir_entry_cmp_desc);
|
||||
size_t dir_count = 0;
|
||||
while (dir_count < count && entries[dir_count].is_dir)
|
||||
dir_count++;
|
||||
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
char* child_rel =
|
||||
(strcmp(dir_rel, ".") == 0) ? str_dup(entry->d_name) : path_cat(dir_rel, entry->d_name);
|
||||
(strcmp(dir_rel, ".") == 0) ? str_dup(entries[i].name) : path_cat(dir_rel, entries[i].name);
|
||||
if (!child_rel) {
|
||||
operation_ok = false;
|
||||
continue;
|
||||
}
|
||||
if (path_under_skip_prefix(child_rel, at_root, skips->entries, skips->count)) {
|
||||
shielded[i] = true;
|
||||
local_survives = true;
|
||||
free(child_rel);
|
||||
continue;
|
||||
}
|
||||
struct stat st;
|
||||
if (fstatat(dirfd, entry->d_name, &st, AT_SYMLINK_NOFOLLOW) != 0) {
|
||||
if (errno != ENOENT)
|
||||
operation_ok = false;
|
||||
free(child_rel);
|
||||
continue;
|
||||
}
|
||||
bool is_dir = S_ISDIR(st.st_mode);
|
||||
bool in_keep_dirs = is_dir && list_contains_str(keep_dirs, entry->d_name);
|
||||
bool in_keep_files = !is_dir && list_contains_str(keep_files, entry->d_name);
|
||||
bool is_dir = entries[i].is_dir;
|
||||
bool in_keep_dirs = is_dir && list_contains_str(keep_dirs, entries[i].name);
|
||||
bool in_keep_files = !is_dir && list_contains_str(keep_files, entries[i].name);
|
||||
bool rule_protected =
|
||||
skips->protect_rules &&
|
||||
filter_rules_apply_side(skips->protect_rules, child_rel, entry->d_name, is_dir,
|
||||
filter_rules_apply_side(skips->protect_rules, child_rel, entries[i].name, is_dir,
|
||||
FILTER_SIDE_RECEIVER) == FILTER_ACTION_PROTECT;
|
||||
if (in_keep_dirs) {
|
||||
if (in_keep_dirs || in_keep_files || rule_protected) {
|
||||
shielded[i] = true;
|
||||
local_survives = true;
|
||||
} else if (keep_dirs && !is_dir && list_contains_str(keep_dirs, entry->d_name)) {
|
||||
/* Destination file blocks a source directory: clear it now, whatever the
|
||||
delete timing, so the directory can be created. */
|
||||
if (!process_extra_file(dirfd, entry->d_name, child_rel, true, session))
|
||||
operation_ok = false;
|
||||
} else if (in_keep_files) {
|
||||
local_survives = true;
|
||||
} else if (keep_files && is_dir && list_contains_str(keep_files, entry->d_name)) {
|
||||
/* Destination directory blocks a source file: remove it now. */
|
||||
bool removed = false;
|
||||
if (!process_extra_dir(dirfd, entry->d_name, child_rel, true, skips, session, &removed))
|
||||
operation_ok = false;
|
||||
else if (!removed)
|
||||
local_survives = true;
|
||||
} else if (is_dir) {
|
||||
if (rule_protected) {
|
||||
local_survives = true;
|
||||
} else {
|
||||
bool removed = false;
|
||||
if (!process_extra_dir(dirfd, entry->d_name, child_rel, force_now, skips, session,
|
||||
&removed))
|
||||
operation_ok = false;
|
||||
else if (!removed)
|
||||
local_survives = true;
|
||||
}
|
||||
} else if (rule_protected) {
|
||||
local_survives = true;
|
||||
/* A destination directory blocks a source file of the same name: remove
|
||||
it now, whatever the delete timing, so the file can be created. */
|
||||
is_extra[i] = true;
|
||||
force[i] = keep_files && list_contains_str(keep_files, entries[i].name);
|
||||
} else {
|
||||
if (!process_extra_file(dirfd, entry->d_name, child_rel, force_now, session))
|
||||
operation_ok = false;
|
||||
/* A destination file blocks a source directory of the same name: clear it
|
||||
now so the directory can be created. */
|
||||
is_extra[i] = true;
|
||||
force[i] = keep_dirs && list_contains_str(keep_dirs, entries[i].name);
|
||||
}
|
||||
free(child_rel);
|
||||
}
|
||||
closedir(dir);
|
||||
|
||||
/* Pass 1: extraneous subdirectories, descending. */
|
||||
for (size_t i = 0; i < dir_count; i++) {
|
||||
if (!is_extra[i])
|
||||
continue;
|
||||
char* child_rel =
|
||||
(strcmp(dir_rel, ".") == 0) ? str_dup(entries[i].name) : path_cat(dir_rel, entries[i].name);
|
||||
if (!child_rel) {
|
||||
operation_ok = false;
|
||||
continue;
|
||||
}
|
||||
bool removed = false;
|
||||
if (!process_extra_dir(dirfd, entries[i].name, child_rel, force[i] || force_now, skips, session,
|
||||
&removed))
|
||||
operation_ok = false;
|
||||
else if (!removed)
|
||||
local_survives = true;
|
||||
free(child_rel);
|
||||
}
|
||||
|
||||
/* Pass 2: extraneous files, descending. */
|
||||
for (size_t i = dir_count; i < count; i++) {
|
||||
if (!is_extra[i])
|
||||
continue;
|
||||
char* child_rel =
|
||||
(strcmp(dir_rel, ".") == 0) ? str_dup(entries[i].name) : path_cat(dir_rel, entries[i].name);
|
||||
if (!child_rel) {
|
||||
operation_ok = false;
|
||||
continue;
|
||||
}
|
||||
if (!process_extra_file(dirfd, entries[i].name, child_rel, force[i] || force_now, session))
|
||||
operation_ok = false;
|
||||
free(child_rel);
|
||||
}
|
||||
|
||||
free(shielded);
|
||||
free(is_extra);
|
||||
free(force);
|
||||
delete_dir_entries_free(entries, count);
|
||||
*survives = local_survives;
|
||||
return operation_ok;
|
||||
}
|
||||
@@ -932,10 +988,11 @@ static bool apply_deferred_path(DeletePlanSession* session, const Config* config
|
||||
return false;
|
||||
char* leaf = NULL;
|
||||
int parent_fd = file_open_secure_parent(full, &leaf, false);
|
||||
int open_errno = errno;
|
||||
free(full);
|
||||
if (parent_fd < 0) {
|
||||
free(leaf);
|
||||
return errno == ENOENT || errno == ENOTDIR;
|
||||
return open_errno == ENOENT || open_errno == ENOTDIR;
|
||||
}
|
||||
struct stat st;
|
||||
if (fstatat(parent_fd, leaf, &st, AT_SYMLINK_NOFOLLOW) != 0) {
|
||||
@@ -981,7 +1038,7 @@ static bool apply_deferred_path(DeletePlanSession* session, const Config* config
|
||||
session->deleted++;
|
||||
session->planned++;
|
||||
log_deleted(rel);
|
||||
notify_deleted(session, rel);
|
||||
notify_deleted_dir(session, rel);
|
||||
} else if (errno != ENOENT && errno != ENOTEMPTY && errno != EEXIST) {
|
||||
close(parent_fd);
|
||||
free(leaf);
|
||||
|
||||
@@ -61,9 +61,19 @@ int delete_plan_send_root(int fd, DeletePlanSender* sender);
|
||||
* for `path` itself; already-sent plans are skipped. */
|
||||
int delete_plan_send_for_path(int fd, DeletePlanSender* sender, const char* path, bool is_dir);
|
||||
/* Send the plan for every directory in `dirs` that has not been transmitted
|
||||
* yet. Called after the data stream so an empty source directory's plan still
|
||||
* clears its destination extras even though no file frame triggered it. */
|
||||
* yet. */
|
||||
int delete_plan_send_remaining(int fd, DeletePlanSender* sender, const ArrayList* dirs);
|
||||
/* Transmit the COMPLETE per-directory plan set in one pass, before any data
|
||||
* frame: the root plan (with the one-shot per-run config block on its carrier
|
||||
* frame) followed by every directory in `dirs`. Because the whole plan set is
|
||||
* known from the path-only pre-scan, sending it all up front means a
|
||||
* mid-transfer abort has already applied every planned removal, matching
|
||||
* rsync's generator (which runs ahead of its throttled sender). A completed
|
||||
* run is unaffected. `dirs` is the set of directories whose direct children
|
||||
* were enumerated (the scanner's plan_dirs sink), so a merely listed but
|
||||
* untraversed directory never gets a plan and its mirror is left intact.
|
||||
* Returns -1 on I/O error. */
|
||||
int delete_plan_send_all(int fd, DeletePlanSender* sender, const ArrayList* dirs);
|
||||
|
||||
/* ---- Receiver: delete session ---- */
|
||||
|
||||
|
||||
+43
-16
@@ -25,13 +25,6 @@
|
||||
#include "utils.h"
|
||||
#include "protocol.h"
|
||||
#include "xattr.h"
|
||||
#include <fcntl.h>
|
||||
#include <unistd.h>
|
||||
|
||||
/* Files larger than this are not loaded whole for transfer (the sender streams
|
||||
* them); a whole-file digest is computed from the path instead. Kept in sync
|
||||
* with the sender's streaming threshold. */
|
||||
#define STREAM_THRESHOLD (64ULL * 1024 * 1024)
|
||||
|
||||
static bool write_all(int fd, const void* data, unsigned long long size) {
|
||||
const unsigned char* p = data;
|
||||
@@ -1136,17 +1129,51 @@ int file_open_private_dir(const char* dir_path) {
|
||||
return fd;
|
||||
}
|
||||
|
||||
/* Open a --temp-dir scratch directory exactly as rsync does: the directory must
|
||||
* already exist and is used as given (an absolute path is used verbatim, a
|
||||
* relative one was already resolved against the destination root by the
|
||||
* caller). Unlike file_open_private_dir this neither creates it nor confines
|
||||
* it below the receive root, because rsync accepts any temp dir -- including
|
||||
* one outside the destination tree or on another filesystem. Returns an
|
||||
* O_DIRECTORY|O_CLOEXEC fd, or -1 on error. */
|
||||
/* Open a --temp-dir scratch directory. The directory must already exist (rsync
|
||||
* never creates it); a relative path was already resolved against the
|
||||
* destination root by the caller. Unlike file_open_private_dir this neither
|
||||
* creates it nor requires it to be a direct child of the receive root, because
|
||||
* rsync permits a scratch dir that (via a symlink) lands on another filesystem
|
||||
* -- but it MUST resolve inside the authorized receive root. The directory is
|
||||
* opened following symlinks and then judged by the REAL path of the opened fd
|
||||
* (through /proc/self/fd), so a client-planted symlink under the receive root
|
||||
* can never redirect receiver scratch files outside the sandbox while an
|
||||
* in-root link to another filesystem (the EXDEV fallback case) still works.
|
||||
* Returns an O_DIRECTORY|O_CLOEXEC fd, or -1 on error (errno set; an escaping
|
||||
* target is reported as EACCES with a logged reason). */
|
||||
int file_open_temp_dir(const char* dir_path) {
|
||||
if (!dir_path)
|
||||
return -1;
|
||||
return open(dir_path, O_RDONLY | O_DIRECTORY | O_CLOEXEC);
|
||||
int fd = open(dir_path, O_RDONLY | O_DIRECTORY | O_CLOEXEC);
|
||||
if (fd < 0)
|
||||
return -1;
|
||||
const char* root = utils_get_authorized_root_path();
|
||||
if (!root) {
|
||||
/* No authorized root (e.g. a local batch apply): nothing to confine
|
||||
against, so preserve the historical open-as-given behavior. */
|
||||
return fd;
|
||||
}
|
||||
char fd_path[64];
|
||||
int fd_path_length = snprintf(fd_path, sizeof(fd_path), "/proc/self/fd/%d", fd);
|
||||
char resolved[PATH_MAX];
|
||||
if (fd_path_length < 0 || (size_t)fd_path_length >= sizeof(fd_path) ||
|
||||
!realpath(fd_path, resolved)) {
|
||||
int saved_errno = errno;
|
||||
close(fd);
|
||||
errno = saved_errno;
|
||||
return -1;
|
||||
}
|
||||
if (!path_is_within_root(root, resolved)) {
|
||||
char* escaped = output_escape(dir_path, log_get_8_bit_output());
|
||||
log_message(LOG_LEVEL_ERROR,
|
||||
"--temp-dir '%s' resolves outside the authorized receive root; refusing",
|
||||
escaped ? escaped : "<allocation failed>");
|
||||
free(escaped);
|
||||
close(fd);
|
||||
errno = EACCES;
|
||||
return -1;
|
||||
}
|
||||
return fd;
|
||||
}
|
||||
|
||||
/* After the content and mode/times are restored on the just-written file, apply
|
||||
@@ -1859,6 +1886,6 @@ bool file_write_to_disk(const char* path, const void* data, unsigned long long d
|
||||
bool inplace, bool sparse) {
|
||||
if (!path || (!data && data_size != 0) || has_path_traversal(path))
|
||||
return false;
|
||||
FileAttrPolicy policy = {false, false, false, false};
|
||||
FileAttrPolicy policy = {0};
|
||||
return file_to_disk_secure(path, data, data_size, inplace, sparse, false, NULL, policy, NULL);
|
||||
}
|
||||
|
||||
+6
-2
@@ -103,8 +103,12 @@ bool file_remove_tree_secure(const char* path);
|
||||
the authorized root. Used for the --delay-updates staging directory. */
|
||||
int file_open_private_dir(const char* dir_path);
|
||||
|
||||
/* Open an existing --temp-dir scratch directory as-is (absolute or relative;
|
||||
no creation, no root confinement), matching rsync's --temp-dir handling. */
|
||||
/* Open an existing --temp-dir scratch directory (relative or absolute; no
|
||||
creation). When an authorized receive root is configured the directory's
|
||||
REAL path (symlinks resolved) must lie within it, so a client-planted
|
||||
symlink cannot redirect receiver scratch files outside the sandbox; an
|
||||
in-root symlink to another filesystem is still allowed for rsync's EXDEV
|
||||
fallback. */
|
||||
int file_open_temp_dir(const char* dir_path);
|
||||
|
||||
/* The file_to_disk_secure* variants write a temporary copy in the destination
|
||||
|
||||
@@ -29,6 +29,12 @@ typedef struct FileAttrPolicy {
|
||||
bool times; /* config->preserve_times: apply the source mtime */
|
||||
bool atimes; /* config->preserve_atimes (-U): apply the source atime */
|
||||
bool executability; /* config->use_executability (-E): exec-bits-only mode */
|
||||
/* privilege_super_mode_permitted(): when false (SUPER_MODE_OFF / --no-super,
|
||||
or a daemon that did not grant `client owner = yes`), the setuid/setgid/
|
||||
sticky bits are stripped from every applied mode (source mode and any
|
||||
--chmod result) even under --perms. When true, rsync's exact semantics are
|
||||
preserved: -p copies the special bits and the kernel decides. */
|
||||
bool super_permitted;
|
||||
} FileAttrPolicy;
|
||||
|
||||
/* Build the per-attribute policy from a connection's Config. A NULL config
|
||||
|
||||
+96
-43
@@ -474,9 +474,13 @@ static FileSaveResult file_save_special_to_disk(const char* root_directory, cons
|
||||
/* Under -p/--perms rsync copies the source's permission and special bits; a
|
||||
* kernel that denies setuid/setgid/sticky reports the failure rather than
|
||||
* having them masked here. Without -p the node is created like any other new
|
||||
* entry: source_mode & 0777 & ~umask. */
|
||||
* entry: source_mode & 0777 & ~umask. When super-user activities are
|
||||
* forbidden, the special bits are stripped even under -p (they are
|
||||
* super-user activities just like device-node creation). */
|
||||
mode_t perms = config->preserve_perms ? (mode & (mode_t)(S_ISUID | S_ISGID | S_ISVTX | 0777))
|
||||
: (mode & 0777 & ~(mode_t)file_process_umask());
|
||||
if (!privilege_super_mode_permitted(config->super_mode))
|
||||
perms &= ~(mode_t)(S_ISUID | S_ISGID | S_ISVTX);
|
||||
|
||||
int rc = is_fifo ? mkfifoat(parent_fd, leaf, perms)
|
||||
: mknodat(parent_fd, leaf, create_mode | perms, rdev);
|
||||
@@ -1389,6 +1393,43 @@ bool file_basis_content_required(const Config* config) {
|
||||
return config != NULL && config->verify_basis;
|
||||
}
|
||||
|
||||
/* Probe one candidate basis file: open it (confined, O_NOFOLLOW) and apply
|
||||
rsync's metadata quick-check; under --verify-basis also hash its bytes and
|
||||
require the sender's digest. On a hit record `candidate` in `out` and return
|
||||
true. The caller retains ownership of `candidate`. */
|
||||
static bool basis_match_probe(const Config* config, const char* candidate,
|
||||
unsigned long long check_size, time_t check_mtime,
|
||||
long check_mtime_nsec, const uint8_t* check_digest,
|
||||
size_t check_digest_len, BasisDestType type, BasisMatch* out) {
|
||||
int fd;
|
||||
struct stat st;
|
||||
if (!basis_open_regular(candidate, check_size, &fd, &st))
|
||||
return false;
|
||||
bool hit = false;
|
||||
if (file_basis_quick_match(config, &st, check_mtime, check_mtime_nsec)) {
|
||||
hit = true;
|
||||
if (file_basis_content_required(config)) {
|
||||
uint8_t basis_digest[CHECKSUM_MAX_DIGEST_LEN];
|
||||
size_t basis_len = 0;
|
||||
bool hashed = checksum_digest_fd((ChecksumAlgo)config->checksum_algo, config->checksum_seed,
|
||||
fd, basis_digest, sizeof(basis_digest), &basis_len);
|
||||
hit = hashed && basis_len == check_digest_len && check_digest_len > 0 &&
|
||||
memcmp(basis_digest, check_digest, check_digest_len) == 0;
|
||||
}
|
||||
}
|
||||
close(fd);
|
||||
if (!hit)
|
||||
return false;
|
||||
char* owned = str_dup(candidate);
|
||||
if (!owned)
|
||||
return false;
|
||||
out->hit = true;
|
||||
out->type = type;
|
||||
out->basis_path = owned;
|
||||
out->st = st;
|
||||
return true;
|
||||
}
|
||||
|
||||
/* Search the basis-dir list in command-line order and return the first match.
|
||||
By default (no --verify-basis) rsync's metadata quick-check is sufficient:
|
||||
basis_open_regular has already required an equal size, and
|
||||
@@ -1403,7 +1444,22 @@ bool file_basis_content_required(const Config* config) {
|
||||
--dry-run passes false because hashing a basis against a client-supplied
|
||||
digest would be a 1-bit content oracle. Without --verify-basis a dry-run can
|
||||
still confirm the metadata-only hit without reading any basis bytes, matching
|
||||
rsync's read-only quick-check. */
|
||||
rsync's read-only quick-check.
|
||||
|
||||
Path resolution (rsync 3.4.1 parity): rsync resolves a relative
|
||||
--compare-dest/--copy-dest/--link-dest DIR against the destination directory
|
||||
(the receiver's cwd) and appends the file's TRANSFER-RELATIVE name, e.g.
|
||||
`--compare-dest=basis` with `rsync src/ dst/` probes `dst/basis/<name-inside-src>`.
|
||||
FastSync's receive root IS the destination directory, but its default transfer
|
||||
mirrors the absolute source path below that root, so check_path carries the
|
||||
source-root scaffolding rsync would not append. Recover rsync's spelling with
|
||||
utils_strip_transfer_root for a relative DIR; under -R/--files-from the wire
|
||||
path is already transfer-relative, so it is used as-is. A relative DIR also
|
||||
probes the historical mirror-appended spelling as a fallback, so existing
|
||||
FastSync-laid-out snapshot trees keep resolving. An absolute DIR is used
|
||||
verbatim and keeps appending the destination-relative check_path (FastSync's
|
||||
mirrored layout). Every candidate stays confined to the authorized root by
|
||||
file_open_secure_parent. */
|
||||
static bool basis_match_find(const Config* config, const char* check_path,
|
||||
unsigned long long check_size, time_t check_mtime,
|
||||
long check_mtime_nsec, const uint8_t* check_digest,
|
||||
@@ -1416,47 +1472,39 @@ static bool basis_match_find(const Config* config, const char* check_path,
|
||||
the basis bytes. */
|
||||
if (file_basis_content_required(config) && !hash_content)
|
||||
return false;
|
||||
const char* transfer_rel = check_path;
|
||||
if (!config->relative && config->files_from_set == NULL)
|
||||
transfer_rel = utils_strip_transfer_root(check_path, config->send_directory);
|
||||
for (int i = 0; i < config->basis_count; i++) {
|
||||
const BasisDest* entry = &config->basis_dirs[i];
|
||||
/* An absolute basis path is used verbatim (rsync semantics); a relative one
|
||||
is resolved below the receive root. Both remain subject to the receiver's
|
||||
authorized-root confinement inside file_open_secure_parent. */
|
||||
char* basis_dir = entry->path[0] == '/' ? str_dup(entry->path)
|
||||
: path_cat(config->receive_root_directory, entry->path);
|
||||
bool absolute = entry->path[0] == '/';
|
||||
char* basis_dir =
|
||||
absolute ? str_dup(entry->path) : path_cat(config->receive_root_directory, entry->path);
|
||||
if (!basis_dir)
|
||||
continue;
|
||||
char* candidate = path_cat(basis_dir, check_path);
|
||||
free(basis_dir);
|
||||
if (!candidate)
|
||||
continue;
|
||||
|
||||
int fd;
|
||||
struct stat st;
|
||||
if (basis_open_regular(candidate, check_size, &fd, &st)) {
|
||||
if (file_basis_quick_match(config, &st, check_mtime, check_mtime_nsec)) {
|
||||
bool hit = true;
|
||||
if (file_basis_content_required(config)) {
|
||||
uint8_t basis_digest[CHECKSUM_MAX_DIGEST_LEN];
|
||||
size_t basis_len = 0;
|
||||
bool hashed =
|
||||
checksum_digest_fd((ChecksumAlgo)config->checksum_algo, config->checksum_seed, fd,
|
||||
basis_digest, sizeof(basis_digest), &basis_len);
|
||||
hit = hashed && basis_len == check_digest_len && check_digest_len > 0 &&
|
||||
memcmp(basis_digest, check_digest, check_digest_len) == 0;
|
||||
}
|
||||
if (hit) {
|
||||
out->hit = true;
|
||||
out->type = entry->type;
|
||||
out->basis_path = candidate;
|
||||
candidate = NULL; /* ownership transferred to out */
|
||||
out->st = st;
|
||||
close(fd);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
close(fd);
|
||||
const char* names[2];
|
||||
int name_count = 0;
|
||||
if (absolute)
|
||||
names[name_count++] = check_path;
|
||||
else
|
||||
names[name_count++] = transfer_rel;
|
||||
if (!absolute && strcmp(transfer_rel, check_path) != 0)
|
||||
names[name_count++] = check_path; /* historical mirror-appended spelling */
|
||||
bool found = false;
|
||||
for (int n = 0; n < name_count && !found; n++) {
|
||||
char* candidate = path_cat(basis_dir, names[n]);
|
||||
if (!candidate)
|
||||
continue;
|
||||
found = basis_match_probe(config, candidate, check_size, check_mtime, check_mtime_nsec,
|
||||
check_digest, check_digest_len, entry->type, out);
|
||||
free(candidate);
|
||||
}
|
||||
free(candidate);
|
||||
free(basis_dir);
|
||||
if (found)
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
@@ -1489,9 +1537,12 @@ static bool basis_match_find(const Config* config, const char* check_path,
|
||||
* followed and nothing outside the destination root is ever read;
|
||||
* * dotfiles, directories, the target's own name, and the .fastsync-stage /
|
||||
* temp scratch names are never candidates;
|
||||
* * size gate = the delta engine's own bounds (delta_should_attempt: both
|
||||
* files >= DELTA_MIN_FILE_SIZE, <= delta_max_file_size, ratio <= 10x),
|
||||
* because FastSync's delta engine cannot use a basis outside them;
|
||||
* * size gate = rsync's, NOT the ordinary delta engine's bounds: any
|
||||
* non-empty regular sibling up to the receiver's whole-file buffer cap is
|
||||
* eligible, regardless of the 16 KiB delta minimum or the 10x delta size
|
||||
* ratio (rsync's find_fuzzy has no delta-size gate at all). The delta
|
||||
* engine consumes the fuzzy basis through the same signature handshake
|
||||
* whether or not it is inside delta_should_attempt's window;
|
||||
* * first pass = an exact size+mtime match wins regardless of name (rsync's
|
||||
* "fuzzy size/modtime match");
|
||||
* * otherwise the winner minimizes rsync's weighted Levenshtein distance
|
||||
@@ -1639,8 +1690,7 @@ static void* fuzzy_basis_find_and_load(const Config* config, const char* check_p
|
||||
long check_mtime_nsec, unsigned long long* out_size) {
|
||||
*out_size = 0;
|
||||
if (!config || !config->receive_root_directory || !config->fuzzy || !config->use_delta ||
|
||||
!check_path || check_size < DELTA_MIN_FILE_SIZE || check_size > config->delta_max_file_size ||
|
||||
check_size > MAX_RECEIVE_WHOLE_FILE_SIZE)
|
||||
!check_path || check_size > MAX_RECEIVE_WHOLE_FILE_SIZE)
|
||||
return NULL;
|
||||
|
||||
char* full_path = path_cat(config->receive_root_directory, check_path);
|
||||
@@ -1717,8 +1767,7 @@ static void* fuzzy_basis_find_and_load(const Config* config, const char* check_p
|
||||
if (fstatat(dir_fd, name, &st, AT_SYMLINK_NOFOLLOW) != 0 || !S_ISREG(st.st_mode))
|
||||
continue;
|
||||
unsigned long long cand_size = (unsigned long long)st.st_size;
|
||||
if (cand_size == 0 || cand_size > MAX_RECEIVE_WHOLE_FILE_SIZE ||
|
||||
!delta_should_attempt(cand_size, check_size, config->delta_max_file_size))
|
||||
if (cand_size == 0 || cand_size > MAX_RECEIVE_WHOLE_FILE_SIZE)
|
||||
continue;
|
||||
long cand_nsec = 0;
|
||||
#ifdef __linux__
|
||||
@@ -2904,8 +2953,12 @@ void dir_metadata_list_apply(const DirTimeList* list, const char* root_directory
|
||||
}
|
||||
if (mode_ready) {
|
||||
/* rsync -p copies the source directory mode exactly, including
|
||||
* group/other write and the setgid/sticky bits. */
|
||||
* group/other write and the setgid/sticky bits. Setuid/setgid/sticky
|
||||
* are super-user activities: when the connection forbade them
|
||||
* (SUPER_MODE_OFF / --no-super), strip them even under -p. */
|
||||
mode_t safe_mode = dir_mode & (mode_t)(S_ISUID | S_ISGID | S_ISVTX | 0777);
|
||||
if (!privilege_super_mode_permitted(config->super_mode))
|
||||
safe_mode &= ~(mode_t)(S_ISUID | S_ISGID | S_ISVTX);
|
||||
if (dir_fd < 0) {
|
||||
char* escaped_path = output_escape(dir_path, log_get_8_bit_output());
|
||||
log_message(LOG_LEVEL_WARNING, "Failed to open directory %s to set its mode: %s",
|
||||
|
||||
@@ -166,6 +166,8 @@ bool file_send_sendfile_with_skip(File* file, int file_descriptor, bool use_meta
|
||||
}
|
||||
struct pollfd pfd = {.fd = file_descriptor, .events = POLLOUT};
|
||||
int polled = poll(&pfd, 1, timeout);
|
||||
if (polled < 0 && errno == EINTR)
|
||||
continue;
|
||||
if (polled <= 0 || (pfd.revents & (POLLERR | POLLHUP | POLLNVAL))) {
|
||||
close(fd);
|
||||
return false;
|
||||
@@ -183,6 +185,7 @@ bool file_send_sendfile_with_skip(File* file, int file_descriptor, bool use_meta
|
||||
return false;
|
||||
}
|
||||
protocol_note_bytes_written((unsigned long long)sent);
|
||||
protocol_throttle_bytes((size_t)sent);
|
||||
}
|
||||
|
||||
close(fd);
|
||||
|
||||
+95
-27
@@ -4,22 +4,12 @@
|
||||
#include <ctype.h>
|
||||
#include <errno.h>
|
||||
#include <limits.h>
|
||||
#include <stdarg.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
/* Write a diagnostic message into the caller's optional buffer. A NULL `err`
|
||||
* (or a zero size) is a no-op, so a caller that only needs the boolean status
|
||||
* may pass NULL without the snprintf-on-NULL undefined behaviour. */
|
||||
static void filter_set_error(char* err, size_t err_size, const char* fmt, ...) {
|
||||
if (!err || err_size == 0)
|
||||
return;
|
||||
va_list ap;
|
||||
va_start(ap, fmt);
|
||||
vsnprintf(err, err_size, fmt, ap);
|
||||
va_end(ap);
|
||||
}
|
||||
/* Write a diagnostic message into the caller's optional buffer. */
|
||||
#define filter_set_error utils_set_error
|
||||
|
||||
/* ---- Ordered rule lists ---- */
|
||||
|
||||
@@ -172,14 +162,74 @@ static bool is_modifier_char(char c) {
|
||||
return c == 's' || c == 'r' || c == 'p' || c == 'x' || c == '/' || c == '!' || c == 'C';
|
||||
}
|
||||
|
||||
/* merge/dir-merge rules are the only rules rsync accepts the merge-file
|
||||
* modifiers on. */
|
||||
static bool is_merge_rule(RuleKind kind) {
|
||||
return kind == RULE_KIND_MERGE || kind == RULE_KIND_DIR_MERGE;
|
||||
}
|
||||
|
||||
/* Merge-file modifiers rsync defines but FastSync does not implement:
|
||||
* 'e' exclude the merge file itself, 'n' do not inherit the merge file, 'w'
|
||||
* word-split the merge file. They are recognized as part of a modifier run on
|
||||
* every rule (so a pure e/n/w token is rejected rather than folded into the
|
||||
* pattern), but are accepted (and ignored) only on merge/dir-merge rules. */
|
||||
static bool is_unsupported_modifier_char(char c) {
|
||||
return c == 'e' || c == 'n' || c == 'w';
|
||||
}
|
||||
|
||||
/* Merge-file modifiers rsync accepts on merge/dir-merge rules: 'e', 'n', 'w'
|
||||
* and '-' (do not transfer the merge file). */
|
||||
static bool is_merge_modifier_char(char c) {
|
||||
return c == 'e' || c == 'n' || c == 'w' || c == '-';
|
||||
}
|
||||
|
||||
/* Characters that count as part of a modifier run for `kind` when deciding
|
||||
* whether a token is a pure modifier run. e/n/w count on every rule so that a
|
||||
* pure e/n/w token is rejected on non-merge rules; '-' only on merge rules. */
|
||||
static bool is_modifier_scan_char(char c, RuleKind kind) {
|
||||
return is_modifier_char(c) || is_unsupported_modifier_char(c) ||
|
||||
(is_merge_rule(kind) && is_merge_modifier_char(c));
|
||||
}
|
||||
|
||||
/* Characters actually consumed as modifiers for `kind`. The merge-file
|
||||
* modifiers are consumed only on merge/dir-merge rules; elsewhere e/n/w fall
|
||||
* through to the pattern (so mixed tokens such as "H,!secret" keep their
|
||||
* historical "ecret" pattern). */
|
||||
static bool is_consumed_modifier_char(char c, RuleKind kind) {
|
||||
return is_modifier_char(c) || (is_merge_rule(kind) && is_merge_modifier_char(c));
|
||||
}
|
||||
|
||||
/* Inspect the token that follows a rule name (up to the first space/underscore
|
||||
* or the end). If the token is composed *solely* of modifier characters and
|
||||
* includes one that is invalid for `kind`, it is unambiguously a modifier run:
|
||||
* return that character so the caller can reject it. A token that contains any
|
||||
* non-modifier character is a pattern (e.g. "-newfile") and returns '\0', which
|
||||
* keeps the historical parsing of mixed tokens such as "H,!secret" intact. */
|
||||
static char unsupported_modifier_in_token(const char* tok, RuleKind kind) {
|
||||
if (*tok == '\0' || *tok == ' ' || *tok == '_')
|
||||
return '\0';
|
||||
char bad = '\0';
|
||||
for (const char* q = tok; *q != '\0' && *q != ' ' && *q != '_'; q++) {
|
||||
if (!is_modifier_scan_char(*q, kind))
|
||||
return '\0';
|
||||
if (!is_merge_rule(kind) && is_unsupported_modifier_char(*q))
|
||||
bad = *q;
|
||||
}
|
||||
return bad;
|
||||
}
|
||||
|
||||
/* Parse "RULE[,MODIFIERS] [PATTERN]". On success `kind`, `sides`,
|
||||
* `sides_explicit`, `negate`, `anchored_mod`, `perishable`, `xattr`,
|
||||
* `cvs_inject` and the pattern span (`pat_start`/`pat_len`, possibly 0 for
|
||||
* merge/clear) are filled. Returns true on success. */
|
||||
* merge/clear) are filled. Returns true on success.
|
||||
*
|
||||
* On failure `*bad_mod` is set to the offending modifier character when the
|
||||
* rule carried a modifier FastSync does not implement, and left '\0' for a
|
||||
* generic syntax error so callers can emit a precise diagnostic. */
|
||||
static bool parse_rule_syntax(const char* text, RuleKind* kind, unsigned* sides,
|
||||
bool* sides_explicit, bool* negate, bool* anchored_mod,
|
||||
bool* perishable, bool* xattr, bool* cvs_inject,
|
||||
const char** pat_start, size_t* pat_len) {
|
||||
const char** pat_start, size_t* pat_len, char* bad_mod) {
|
||||
const char* p = text;
|
||||
*sides = FILTER_SIDE_SENDER | FILTER_SIDE_RECEIVER;
|
||||
*sides_explicit = false;
|
||||
@@ -190,6 +240,7 @@ static bool parse_rule_syntax(const char* text, RuleKind* kind, unsigned* sides,
|
||||
*cvs_inject = false;
|
||||
*pat_start = NULL;
|
||||
*pat_len = 0;
|
||||
*bad_mod = '\0';
|
||||
|
||||
bool is_short = false;
|
||||
if (short_rule_char(*p, kind)) {
|
||||
@@ -210,17 +261,25 @@ static bool parse_rule_syntax(const char* text, RuleKind* kind, unsigned* sides,
|
||||
/* Modifiers: long names require a comma; short names may attach directly.
|
||||
Only commit a modifier run that terminates at a separator or the end, so a
|
||||
pattern such as "*.tmp" written as "-*.tmp" is not mistaken for modifiers. */
|
||||
if (*p == ',') {
|
||||
*bad_mod = unsupported_modifier_in_token(p + 1, *kind);
|
||||
} else if (is_short) {
|
||||
*bad_mod = unsupported_modifier_in_token(p, *kind);
|
||||
}
|
||||
if (*bad_mod != '\0')
|
||||
return false;
|
||||
|
||||
const char* mod_start = p;
|
||||
const char* mod_end = p;
|
||||
if (*p == ',') {
|
||||
p++;
|
||||
mod_start = p;
|
||||
while (is_modifier_char(*p))
|
||||
while (is_consumed_modifier_char(*p, *kind))
|
||||
p++;
|
||||
mod_end = p;
|
||||
} else if (is_short) {
|
||||
const char* scan = p;
|
||||
while (is_modifier_char(*scan))
|
||||
while (is_consumed_modifier_char(*scan, *kind))
|
||||
scan++;
|
||||
if (*scan == '\0' || *scan == ' ' || *scan == '_') {
|
||||
mod_start = p;
|
||||
@@ -290,9 +349,13 @@ FilterRule* filter_rule_parse(const char* line, const FilterParseOptions* opts,
|
||||
bool sides_explicit, negate, anchored_mod, perishable, xattr, cvs_inject;
|
||||
const char* pat;
|
||||
size_t pat_len;
|
||||
char bad_mod;
|
||||
if (!parse_rule_syntax(p, &kind, &sides, &sides_explicit, &negate, &anchored_mod, &perishable,
|
||||
&xattr, &cvs_inject, &pat, &pat_len)) {
|
||||
filter_set_error(err, err_size, "unrecognized filter rule syntax");
|
||||
&xattr, &cvs_inject, &pat, &pat_len, &bad_mod)) {
|
||||
if (bad_mod != '\0')
|
||||
filter_set_error(err, err_size, "unsupported filter modifier '%c'", bad_mod);
|
||||
else
|
||||
filter_set_error(err, err_size, "unrecognized filter rule syntax");
|
||||
return NULL;
|
||||
}
|
||||
if (cvs_inject) {
|
||||
@@ -401,7 +464,6 @@ FilterRule* filter_rule_parse(const char* line, const FilterParseOptions* opts,
|
||||
rule->dir_only = dir_only;
|
||||
rule->negate = negate;
|
||||
rule->perishable = perishable;
|
||||
(void)xattr; /* xattr-name rules never match file/dir names; accepted/ignored */
|
||||
return rule;
|
||||
}
|
||||
|
||||
@@ -530,16 +592,27 @@ static bool filter_list_parse_append_depth(FilterRuleList* list, const char* lin
|
||||
bool sides_explicit, negate, anchored_mod, perishable, xattr, cvs_inject;
|
||||
const char* pat;
|
||||
size_t pat_len;
|
||||
char bad_mod;
|
||||
if (!parse_rule_syntax(p, &kind, &sides, &sides_explicit, &negate, &anchored_mod, &perishable,
|
||||
&xattr, &cvs_inject, &pat, &pat_len)) {
|
||||
filter_set_error(err, err_size, "unrecognized filter rule syntax: %s", p);
|
||||
&xattr, &cvs_inject, &pat, &pat_len, &bad_mod)) {
|
||||
if (bad_mod != '\0')
|
||||
filter_set_error(err, err_size, "unsupported filter modifier '%c': %s", bad_mod, p);
|
||||
else
|
||||
filter_set_error(err, err_size, "unrecognized filter rule syntax: %s", p);
|
||||
return false;
|
||||
}
|
||||
(void)sides_explicit;
|
||||
(void)negate;
|
||||
(void)anchored_mod;
|
||||
(void)perishable;
|
||||
(void)xattr;
|
||||
|
||||
/* xattr-name rules are not implemented; reject them everywhere (including on
|
||||
* merge/dir-merge, where the flag would otherwise be silently dropped) with
|
||||
* the same diagnostic the standalone parser gives. */
|
||||
if (xattr) {
|
||||
filter_set_error(err, err_size, "xattr-name filter rules (the x modifier) are not supported");
|
||||
return false;
|
||||
}
|
||||
|
||||
if (cvs_inject) {
|
||||
/* "C" injects the CVS defaults in place; no pattern is expected. */
|
||||
@@ -811,8 +884,3 @@ FilterAction filter_rules_apply_side(const FilterRuleList* list, const char* rel
|
||||
}
|
||||
return FILTER_ACTION_NONE;
|
||||
}
|
||||
|
||||
FilterAction filter_rules_apply(const FilterRuleList* list, const char* rel_path, const char* leaf,
|
||||
bool is_dir) {
|
||||
return filter_rules_apply_side(list, rel_path, leaf, is_dir, FILTER_SIDE_SENDER);
|
||||
}
|
||||
|
||||
+7
-6
@@ -23,7 +23,13 @@
|
||||
* dir-merge/: per-directory merge file (registered for the scanner)
|
||||
* clear/! clear the current rule list (takes no argument)
|
||||
* Modifiers: '/' absolute anchor, '!' negate match, 'C' inject CVS defaults,
|
||||
* 's' sender side, 'r' receiver side, 'p' perishable, 'x' xattr name rule.
|
||||
* 's' sender side, 'r' receiver side, 'p' perishable. The rsync 'x'
|
||||
* (xattr-name) modifier is not implemented and is rejected explicitly
|
||||
* everywhere. The merge-file modifiers 'e' (exclude the merge file itself),
|
||||
* 'n' (do not inherit the merge file), 'w' (word-split the merge file) and '-'
|
||||
* (do not transfer the merge file) are accepted and consumed only on merge/
|
||||
* dir-merge rules (rejected on every other rule, matching rsync); their
|
||||
* semantics are not implemented and they are otherwise ignored.
|
||||
* A trailing '/' makes a pattern match directories only. A leading '/' anchors
|
||||
* the pattern to its owner directory.
|
||||
*/
|
||||
@@ -129,9 +135,4 @@ FilterRuleList* filter_file_read(const char* dir_path, const char* owner_rel, bo
|
||||
FilterAction filter_rules_apply_side(const FilterRuleList* list, const char* rel_path,
|
||||
const char* leaf, bool is_dir, unsigned side);
|
||||
|
||||
/* Sender-side convenience wrapper (kept for callers/tests that only need the
|
||||
* transfer decision). */
|
||||
FilterAction filter_rules_apply(const FilterRuleList* list, const char* rel_path, const char* leaf,
|
||||
bool is_dir);
|
||||
|
||||
#endif
|
||||
|
||||
+30
-5
@@ -5,6 +5,15 @@
|
||||
#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
|
||||
/* Ask the compiler to type-check the printf-style arguments of the variadic
|
||||
* logging helpers. Only enabled for GNU-compatible compilers (gcc/clang). */
|
||||
#if defined(__GNUC__)
|
||||
#define LOG_PRINTF_ATTR(fmt_idx, first_vararg_idx) \
|
||||
__attribute__((format(printf, fmt_idx, first_vararg_idx)))
|
||||
#else
|
||||
#define LOG_PRINTF_ATTR(fmt_idx, first_vararg_idx)
|
||||
#endif
|
||||
|
||||
typedef enum { LOG_LEVEL_DEBUG, LOG_LEVEL_INFO, LOG_LEVEL_WARNING, LOG_LEVEL_ERROR } LogLevel;
|
||||
typedef enum { LOG_STDERR_ERRORS, LOG_STDERR_ALL } LogStderrMode;
|
||||
|
||||
@@ -13,7 +22,19 @@ typedef enum {
|
||||
LOG_DEBUG_PROTO = 1u << 1,
|
||||
LOG_DEBUG_PACK = 1u << 2,
|
||||
LOG_DEBUG_UTIL = 1u << 3,
|
||||
LOG_DEBUG_ALL = (1u << 4) - 1,
|
||||
/* rsync --debug categories that now map to a natural FastSync event:
|
||||
* flist (file-list scan progress), del (deletions), hash/deltasum
|
||||
* (whole-file hashing and delta-sum generation), recv (receiver
|
||||
* responses/signatures), filter (selection/exclusion decisions) and send
|
||||
* (files handed to the sender). Only emitted when the category is
|
||||
* explicitly enabled; a normal run stays silent. */
|
||||
LOG_DEBUG_FLIST = 1u << 4,
|
||||
LOG_DEBUG_DEL = 1u << 5,
|
||||
LOG_DEBUG_HASH = 1u << 6,
|
||||
LOG_DEBUG_RECV = 1u << 7,
|
||||
LOG_DEBUG_FILTER = 1u << 8,
|
||||
LOG_DEBUG_SEND = 1u << 9,
|
||||
LOG_DEBUG_ALL = (1u << 10) - 1,
|
||||
} LogDebugFlag;
|
||||
|
||||
typedef enum {
|
||||
@@ -38,12 +59,16 @@ typedef enum {
|
||||
the info_level bitset (there is no separate Config field) and is never set
|
||||
by --info=all (which selects level 1). */
|
||||
LOG_INFO_NAME_UPTODATE = 1u << 10,
|
||||
/* --info=mount: print rsync's `[sender] skipping mount-point dir NAME` when
|
||||
* -xx/--one-file-system drops a mount-point directory (FastSync's client is
|
||||
* the sender). */
|
||||
LOG_INFO_MOUNT = 1u << 11,
|
||||
LOG_INFO_ALL = LOG_INFO_COPY | LOG_INFO_MISC | LOG_INFO_SKIP | LOG_INFO_STATS | LOG_INFO_DEL |
|
||||
LOG_INFO_REMOVE | LOG_INFO_NAME | LOG_INFO_FLIST | LOG_INFO_NONREG |
|
||||
LOG_INFO_PROGRESS,
|
||||
LOG_INFO_PROGRESS | LOG_INFO_MOUNT,
|
||||
} LogInfoFlag;
|
||||
|
||||
void log_message(LogLevel log_level, const char* message, ...);
|
||||
void log_message(LogLevel log_level, const char* message, ...) LOG_PRINTF_ATTR(2, 3);
|
||||
void log_perror(const char* context);
|
||||
void set_log_level(LogLevel level);
|
||||
void set_log_debug_flags(uint32_t flags);
|
||||
@@ -52,10 +77,10 @@ uint32_t get_log_debug_flags(void);
|
||||
* the debug log level is enabled AND the flag is selected. Hot paths use this
|
||||
* to skip expensive message formatting/escaping when the line is filtered. */
|
||||
bool log_debug_enabled(LogDebugFlag flag);
|
||||
void log_debug_message(LogDebugFlag flag, const char* message, ...);
|
||||
void log_debug_message(LogDebugFlag flag, const char* message, ...) LOG_PRINTF_ATTR(2, 3);
|
||||
void set_log_info_flags(uint32_t flags);
|
||||
uint32_t get_log_info_flags(void);
|
||||
void log_info_message(LogInfoFlag flag, const char* message, ...);
|
||||
void log_info_message(LogInfoFlag flag, const char* message, ...) LOG_PRINTF_ATTR(2, 3);
|
||||
void log_set_file(FILE* fp);
|
||||
void log_set_8_bit_output(bool enabled);
|
||||
bool log_get_8_bit_output(void);
|
||||
|
||||
+20
-5
@@ -211,13 +211,22 @@ FileMetadata* metadata_receive(int file_descriptor, int* ok) {
|
||||
|
||||
bool metadata_mode_for_policy(mode_t source_mode, mode_t current_mode, FileAttrPolicy policy,
|
||||
mode_t* out_mode) {
|
||||
const mode_t special_bits = (mode_t)(S_ISUID | S_ISGID | S_ISVTX);
|
||||
const mode_t execute_bits = S_IXUSR | S_IXGRP | S_IXOTH;
|
||||
if (policy.perms) {
|
||||
/* rsync --perms copies the source's permission and special bits exactly,
|
||||
* including group/other write and setuid/setgid/sticky. The kernel may
|
||||
* still clear setgid when the receiver is not in the file's group; the
|
||||
* caller logs a failed chmod rather than silently masking the bits here. */
|
||||
*out_mode = source_mode & (mode_t)(S_ISUID | S_ISGID | S_ISVTX | 0777);
|
||||
* caller logs a failed chmod rather than silently masking the bits here.
|
||||
* Setuid/setgid/sticky are super-user activities: when the connection did
|
||||
* not permit them (SUPER_MODE_OFF / --no-super) they are stripped, so a
|
||||
* client can never install a privileged bit on a receiver that forbade
|
||||
* super-user activities. This also covers bits introduced by --chmod,
|
||||
* whose result is fed in as source_mode. */
|
||||
mode_t bits = source_mode & (mode_t)(special_bits | 0777);
|
||||
if (!policy.super_permitted)
|
||||
bits &= ~special_bits;
|
||||
*out_mode = bits;
|
||||
return true;
|
||||
}
|
||||
if (policy.executability) {
|
||||
@@ -227,8 +236,11 @@ bool metadata_mode_for_policy(mode_t source_mode, mode_t current_mode, FileAttrP
|
||||
* execute); otherwise clear every execute bit. This runs on the
|
||||
* destination-derived base (pre-existing dest mode, or source&~umask for a
|
||||
* new file), and leaves the special bits untouched. --perms wins when both
|
||||
* are set (handled above). */
|
||||
mode_t base = current_mode & (mode_t)(S_ISUID | S_ISGID | S_ISVTX | 0777);
|
||||
* are set (handled above). The destination's own special bits survive
|
||||
* unless super-user activities are forbidden. */
|
||||
mode_t base = current_mode & (mode_t)(special_bits | 0777);
|
||||
if (!policy.super_permitted)
|
||||
base &= ~special_bits;
|
||||
if (source_mode & 0111)
|
||||
*out_mode = base | ((base & 0444) >> 2);
|
||||
else
|
||||
@@ -240,12 +252,13 @@ bool metadata_mode_for_policy(mode_t source_mode, mode_t current_mode, FileAttrP
|
||||
}
|
||||
|
||||
FileAttrPolicy file_attr_policy_from_config(const Config* config) {
|
||||
FileAttrPolicy policy = {false, false, false, false};
|
||||
FileAttrPolicy policy = {0};
|
||||
if (config) {
|
||||
policy.perms = config->preserve_perms;
|
||||
policy.times = config->preserve_times;
|
||||
policy.atimes = config->preserve_atimes;
|
||||
policy.executability = config->use_executability;
|
||||
policy.super_permitted = privilege_super_mode_permitted(config->super_mode);
|
||||
}
|
||||
return policy;
|
||||
}
|
||||
@@ -314,6 +327,8 @@ bool file_restore_symlink_metadata(const char* path, const FileMetadata* metadat
|
||||
transfer never fails over it. */
|
||||
if (policy.perms) {
|
||||
mode_t link_mode = metadata->mode & (mode_t)(S_ISUID | S_ISGID | S_ISVTX | 0777);
|
||||
if (!policy.super_permitted)
|
||||
link_mode &= ~(mode_t)(S_ISUID | S_ISGID | S_ISVTX);
|
||||
if (fchmodat(parent_fd, leaf, link_mode, AT_SYMLINK_NOFOLLOW) != 0 && errno != EOPNOTSUPP &&
|
||||
errno != ENOTSUP && errno != ENOSYS) {
|
||||
log_message(LOG_LEVEL_DEBUG, "Could not set symlink mode on %s: %s", path, strerror(errno));
|
||||
|
||||
@@ -50,6 +50,7 @@ PipelineContextSender* pipeline_context_sender_create(Config* config, Queue* que
|
||||
protocol_session_set_max_alloc(&context->allocation_session, config->max_alloc);
|
||||
context->dir_entries = NULL;
|
||||
context->dir_entries_mutex_init = false;
|
||||
atomic_init(&context->dir_count, 0);
|
||||
context->delete_limit = false;
|
||||
int init = 0;
|
||||
if (config->use_metadata) {
|
||||
@@ -85,11 +86,13 @@ PipelineContextSender* pipeline_context_sender_create(Config* config, Queue* que
|
||||
return context;
|
||||
|
||||
fail:
|
||||
log_perror("Error initializing synchronization objects");
|
||||
log_message(LOG_LEVEL_ERROR, "%s", "Error initializing synchronization objects");
|
||||
if (context->dir_entries_mutex_init)
|
||||
mtx_destroy(&context->dir_entries_mutex);
|
||||
if (context->dir_entries)
|
||||
array_list_delete(context->dir_entries);
|
||||
if (init >= 7)
|
||||
mtx_destroy(&context->mutex_progress);
|
||||
if (init >= 6)
|
||||
cnd_destroy(&context->condition_not_empty_loader);
|
||||
if (init >= 5)
|
||||
|
||||
@@ -119,6 +119,10 @@ typedef struct {
|
||||
ArrayList* dir_entries;
|
||||
mtx_t dir_entries_mutex;
|
||||
bool dir_entries_mutex_init;
|
||||
/* --stats directory accounting for a `-r` scan (no directory metadata):
|
||||
shared by the parallel scanner workers, read by the sender thread once the
|
||||
scanner is done. See ScannerOptions.dir_count. */
|
||||
atomic_ullong dir_count;
|
||||
/* Set by the sender thread when the receiver reported a --max-delete-capped
|
||||
deletion (STATUS_DELETE_LIMIT): the transfer succeeded and the process must
|
||||
exit 25 like rsync. Read by the caller after the sender thread is joined. */
|
||||
|
||||
+42
-7
@@ -301,6 +301,14 @@ static ProtocolSession* legacy_session(int read_fd, int write_fd) {
|
||||
return &legacy_io_session;
|
||||
}
|
||||
|
||||
/* Pace an out-of-band write that bypassed protocol_send_n_data (the plaintext
|
||||
* sendfile fast path). The bound/legacy session is resolved exactly as
|
||||
* send_n_data resolves it, so the same token-bucket state is throttled and the
|
||||
* TLS and plaintext transports share identical --bwlimit semantics. */
|
||||
void protocol_throttle_bytes(size_t bytes) {
|
||||
bw_throttle_session(legacy_session(-1, -1), bytes);
|
||||
}
|
||||
|
||||
bool send_n_data(int file_descriptor, const void* data, size_t data_size) {
|
||||
return protocol_send_n_data(legacy_session(-1, file_descriptor), data, data_size);
|
||||
}
|
||||
@@ -382,7 +390,7 @@ bool protocol_send_n_data(ProtocolSession* session, const void* data, size_t dat
|
||||
if (session->ssl)
|
||||
wait_events = POLLOUT;
|
||||
}
|
||||
log_debug_message(LOG_DEBUG_IO, " Send n Data: %zu", total_bytes_send);
|
||||
log_debug_message(LOG_DEBUG_IO, " Send n Data: %zd", total_bytes_send);
|
||||
atomic_fetch_add(&io_bytes_written, (unsigned long long)total_bytes_send);
|
||||
return true;
|
||||
}
|
||||
@@ -439,12 +447,18 @@ static bool protocol_receive_n_data_until(ProtocolSession* session, void* data,
|
||||
}
|
||||
|
||||
ssize_t bytes_received;
|
||||
if (session->ssl)
|
||||
bytes_received = SSL_read(session->ssl, (char*)data + total_bytes_received,
|
||||
data_size - total_bytes_received);
|
||||
else
|
||||
if (session->ssl) {
|
||||
/* SSL_read takes an int length; clamp a >INT_MAX request into chunks
|
||||
* (mirrors the send path) so the size_t downcast can never truncate into
|
||||
* a negative/partial read. */
|
||||
size_t ssl_chunk = data_size - total_bytes_received > (size_t)INT_MAX
|
||||
? (size_t)INT_MAX
|
||||
: data_size - total_bytes_received;
|
||||
bytes_received = SSL_read(session->ssl, (char*)data + total_bytes_received, (int)ssl_chunk);
|
||||
} else {
|
||||
bytes_received =
|
||||
read(fd, (char*)data + total_bytes_received, data_size - total_bytes_received);
|
||||
}
|
||||
if (bytes_received <= 0) {
|
||||
if (session->ssl) {
|
||||
int ssl_err = SSL_get_error(session->ssl, (int)bytes_received);
|
||||
@@ -550,6 +564,15 @@ static const char* status_to_string(Status status) {
|
||||
}
|
||||
}
|
||||
|
||||
/* Reject a raw wire status outside the known enum range before it is handed to
|
||||
* callers, so an unknown/corrupt frame fails as a protocol error instead of
|
||||
* being silently interpreted as an unexpected-but-valid verdict. STATUS_OK is
|
||||
* the first enumerator and STATUS_STATS the last, so the range check accepts
|
||||
* every status the protocol defines. */
|
||||
static bool status_is_valid(Status status) {
|
||||
return status >= STATUS_OK && status <= STATUS_STATS;
|
||||
}
|
||||
|
||||
/* Shared string send/receive implementation. `redact` selects whether the
|
||||
* payload body is written to the LOG_DEBUG_PROTO debug log: daemon auth material
|
||||
* (the username and the proof/signature fields) sets it so a --verbose log never
|
||||
@@ -633,7 +656,7 @@ bool protocol_send_data(ProtocolSession* session, const Data* data) {
|
||||
return false;
|
||||
if (!protocol_send_n_data(session, data->data, data_size))
|
||||
return false;
|
||||
log_debug_message(LOG_DEBUG_PROTO, "Send %lld data", data_size);
|
||||
log_debug_message(LOG_DEBUG_PROTO, "Send %llu data", data_size);
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -667,7 +690,7 @@ Data* protocol_receive_data_limited(ProtocolSession* session, unsigned long long
|
||||
protocol_release_memory_for_session(session, allocation_size);
|
||||
return NULL;
|
||||
}
|
||||
log_debug_message(LOG_DEBUG_PROTO, "Received %lld data", size);
|
||||
log_debug_message(LOG_DEBUG_PROTO, "Received %llu data", size);
|
||||
Data* result = data_create(data, (size_t)size);
|
||||
if (!result) {
|
||||
protocol_release_memory_for_session(session, allocation_size);
|
||||
@@ -777,6 +800,10 @@ bool protocol_receive_status(ProtocolSession* session, Status* status) {
|
||||
}
|
||||
if (!protocol_receive_n_data_until(session, status, sizeof(Status), deadline_ptr))
|
||||
return false;
|
||||
if (!status_is_valid(*status)) {
|
||||
log_message(LOG_LEVEL_ERROR, "Received unknown protocol status %d", *status);
|
||||
return false;
|
||||
}
|
||||
if (!protocol_capture_error_detail(session, status, deadline_ptr, NULL))
|
||||
return false;
|
||||
log_debug_message(LOG_DEBUG_PROTO, "Received Status: %s", status_to_string(*status));
|
||||
@@ -798,6 +825,10 @@ bool protocol_receive_status_timed(ProtocolSession* session, Status* status, int
|
||||
deadline.tv_sec += timeout_sec;
|
||||
if (!protocol_receive_n_data_until(session, status, sizeof(Status), &deadline))
|
||||
return false;
|
||||
if (!status_is_valid(*status)) {
|
||||
log_message(LOG_LEVEL_ERROR, "Received unknown protocol status %d", *status);
|
||||
return false;
|
||||
}
|
||||
if (!protocol_capture_error_detail(session, status, &deadline, NULL))
|
||||
return false;
|
||||
log_debug_message(LOG_DEBUG_PROTO, "Received Status: %s", status_to_string(*status));
|
||||
@@ -911,6 +942,10 @@ bool protocol_receive_status_keepalive(ProtocolSession* session, Status* status,
|
||||
Status received;
|
||||
if (!protocol_read_status_until(session, &received, &deadline))
|
||||
return false;
|
||||
if (!status_is_valid(received)) {
|
||||
log_message(LOG_LEVEL_ERROR, "Received unknown protocol status %d", received);
|
||||
return false;
|
||||
}
|
||||
if (!protocol_capture_error_detail(session, &received, &deadline, abort_check))
|
||||
return false;
|
||||
if (received == STATUS_KEEPALIVE) {
|
||||
|
||||
@@ -28,6 +28,10 @@
|
||||
|
||||
/* Maximum chunk size (64 MB) — prevents unbounded allocation from the wire */
|
||||
#define MAX_CHUNK_SIZE (64ULL * 1024 * 1024)
|
||||
/* Files larger than this are not kept fully in memory while loading: the
|
||||
* loader skips them so the sender streams from the path, and file_checksum
|
||||
* hashes them from disk in bounded buffers instead of forcing a full load. */
|
||||
#define STREAM_THRESHOLD (64ULL * 1024 * 1024)
|
||||
#define MAX_MANIFEST_ENTRIES (1024 * 1024)
|
||||
/* Aggregate bytes retained by one received deletion manifest. */
|
||||
#define MAX_MANIFEST_BYTES (16ULL * 1024 * 1024)
|
||||
@@ -220,6 +224,12 @@ SSL* io_get_ssl(void);
|
||||
unsigned long long protocol_bytes_written(void);
|
||||
unsigned long long protocol_bytes_read(void);
|
||||
void protocol_note_bytes_written(unsigned long long bytes);
|
||||
/* Apply --bwlimit pacing to bytes written outside protocol_send_n_data (the
|
||||
* plaintext zero-copy sendfile fast path). Resolves the bound/legacy session
|
||||
* exactly as send_n_data does and runs the same token-bucket throttle, so the
|
||||
* sendfile transport is paced identically to the buffered/TLS paths. A no-op
|
||||
* when the effective session has no bandwidth limit. */
|
||||
void protocol_throttle_bytes(size_t bytes);
|
||||
|
||||
void protocol_session_init(ProtocolSession* session, int read_fd, int write_fd);
|
||||
/* Transitional bridge for helpers whose signatures still carry only an fd. */
|
||||
@@ -263,6 +273,9 @@ bool protocol_send_int(ProtocolSession* session, int data);
|
||||
bool protocol_receive_int(ProtocolSession* session, int* data);
|
||||
bool protocol_send_status(ProtocolSession* session, Status status);
|
||||
bool protocol_receive_status(ProtocolSession* session, Status* status);
|
||||
/* As protocol_receive_status, but with an explicit per-message deadline
|
||||
* (seconds) instead of the session's configured io_timeout_sec. */
|
||||
bool protocol_receive_status_timed(ProtocolSession* session, Status* status, int timeout_sec);
|
||||
bool send_n_data(int file_descriptor, const void* data, size_t data_size);
|
||||
bool receive_n_data(int file_descriptor, void* data, size_t data_size);
|
||||
|
||||
|
||||
+18
-1
@@ -128,7 +128,7 @@ bool queue_enqueue_multithreaded_cancel(Queue* queue, void* item, mtx_t* mutex,
|
||||
|
||||
void* queue_dequeue(Queue* queue) {
|
||||
if (queue == NULL || queue_is_empty(queue)) {
|
||||
log_perror("ERROR: Could not dequeue from null or empty queue.");
|
||||
log_message(LOG_LEVEL_ERROR, "%s", "ERROR: Could not dequeue from null or empty queue.");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
@@ -139,6 +139,23 @@ void* queue_dequeue(Queue* queue) {
|
||||
return item;
|
||||
}
|
||||
|
||||
bool queue_push(Queue* queue, void* item) {
|
||||
return queue_enqueue(queue, item);
|
||||
}
|
||||
|
||||
void* queue_pop(Queue* queue) {
|
||||
if (queue == NULL || queue_is_empty(queue)) {
|
||||
log_message(LOG_LEVEL_ERROR, "%s", "ERROR: Could not pop from null or empty queue.");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
queue->rear = (queue->rear - 1 + queue->capacity) % queue->capacity;
|
||||
void* item = queue->items[queue->rear];
|
||||
queue->items[queue->rear] = NULL;
|
||||
queue->size--;
|
||||
return item;
|
||||
}
|
||||
|
||||
void* queue_dequeue_multithreaded(Queue* queue, mtx_t* mutex, cnd_t* condition_not_empty,
|
||||
cnd_t* condition_not_full, const bool* other_thread_done) {
|
||||
mtx_lock(mutex);
|
||||
|
||||
@@ -28,4 +28,11 @@ void* queue_dequeue(Queue* queue);
|
||||
void* queue_dequeue_multithreaded(Queue* queue, mtx_t* mutex, cnd_t* condition_not_empty,
|
||||
cnd_t* condition_not_full, const bool* other_thread_done);
|
||||
|
||||
/* LIFO stack operations over the same ring buffer. queue_push() is the enqueue
|
||||
primitive; queue_pop() removes from the rear, so a sequence of pushes is
|
||||
returned in reverse order. Used by the sequential scanner's depth-first
|
||||
traversal. */
|
||||
bool queue_push(Queue* queue, void* item);
|
||||
void* queue_pop(Queue* queue);
|
||||
|
||||
#endif
|
||||
|
||||
@@ -87,7 +87,7 @@ static int parse_remote_dest(const char* dest, RemoteDest* r) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
char* ssh_build_remote_command(const char* server_path, bool old_args, char* const* remote_options,
|
||||
char* ssh_build_remote_command(const char* server_path, char* const* remote_options,
|
||||
int remote_option_count) {
|
||||
const char* path = server_path ? server_path : "fastsync-server";
|
||||
const char* suffix = " --stdio";
|
||||
@@ -105,9 +105,7 @@ char* ssh_build_remote_command(const char* server_path, bool old_args, char* con
|
||||
shell word (remote options below reuse the same escaping), then
|
||||
" --stdio". Quoting the path is the only injection-safe construction: an
|
||||
unquoted path would carry shell metacharacters straight into the remote
|
||||
shell command. --old-args is kept for CLI/ABI compatibility but no longer
|
||||
disables that protection. */
|
||||
(void)old_args;
|
||||
shell command. (rsync's --old-args no longer disables that protection.) */
|
||||
size_t quote_count = 0;
|
||||
for (const char* p = path; *p; p++)
|
||||
if (*p == '\'')
|
||||
@@ -296,8 +294,8 @@ void ssh_free_client_argv(char** argv) {
|
||||
}
|
||||
|
||||
Client* client_connect_ssh(const char* destination, int port, const char* server_path,
|
||||
bool old_args, const char* rsh_command, bool blocking_io,
|
||||
char* const* remote_options, int remote_option_count) {
|
||||
const char* rsh_command, bool blocking_io, char* const* remote_options,
|
||||
int remote_option_count) {
|
||||
RemoteDest r;
|
||||
if (parse_remote_dest(destination, &r) != 0) {
|
||||
char* escaped = output_escape(destination, false);
|
||||
@@ -376,7 +374,7 @@ Client* client_connect_ssh(const char* destination, int port, const char* server
|
||||
snprintf(ssh_user, ssh_user_len, "%s", r.host);
|
||||
|
||||
char* remote_command =
|
||||
ssh_build_remote_command(server_path, old_args, remote_options, remote_option_count);
|
||||
ssh_build_remote_command(server_path, remote_options, remote_option_count);
|
||||
if (!remote_command)
|
||||
ssh_child_setup_failed(exec_pipe[1]);
|
||||
char** ssh_argv = ssh_build_client_argv(rsh_command, port, ssh_user, remote_command);
|
||||
|
||||
@@ -4,17 +4,17 @@
|
||||
#include "transport_tcp.h"
|
||||
|
||||
Client* client_connect_ssh(const char* destination, int port, const char* server_path,
|
||||
bool old_args, const char* rsh_command, bool blocking_io,
|
||||
char* const* remote_options, int remote_option_count);
|
||||
const char* rsh_command, bool blocking_io, char* const* remote_options,
|
||||
int remote_option_count);
|
||||
/* Build the escaped remote-shell command string (the server program path always
|
||||
* quoted as one remote-shell word, followed by ` --stdio` and each
|
||||
* --remote-option value appended as an individually single-quoted shell word).
|
||||
* `old_args` is accepted for CLI/ABI compatibility but no longer disables
|
||||
* quoting: the path is always escaped so a metacharacter-bearing
|
||||
* --rsync-path can never be interpreted by the remote shell. Every
|
||||
* --remote-option value is individually escaped with the '\'' sequence and
|
||||
* values with empty/control characters are rejected at the CLI parse layer. */
|
||||
char* ssh_build_remote_command(const char* server_path, bool old_args, char* const* remote_options,
|
||||
* The path is always escaped so a metacharacter-bearing --rsync-path can never
|
||||
* be interpreted by the remote shell (the --old-args no-op does not disable
|
||||
* quoting). Every --remote-option value is individually escaped with the '\''
|
||||
* sequence and values with empty/control characters are rejected at the CLI
|
||||
* parse layer. */
|
||||
char* ssh_build_remote_command(const char* server_path, char* const* remote_options,
|
||||
int remote_option_count);
|
||||
/* Build the NULL-terminated child argv for the remote-shell client (argv[0] is
|
||||
* the exec/execvp program). rsh_command is whitespace-split into leading argv
|
||||
|
||||
+373
-152
@@ -7,6 +7,7 @@
|
||||
#include <errno.h>
|
||||
#include <fcntl.h>
|
||||
#include <netinet/in.h>
|
||||
#include <stdarg.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
@@ -48,6 +49,15 @@ const char* utils_get_authorized_root_path(void) {
|
||||
return authorized_root_path;
|
||||
}
|
||||
|
||||
void utils_set_error(char* err, size_t err_size, const char* fmt, ...) {
|
||||
if (!err || err_size == 0)
|
||||
return;
|
||||
va_list args;
|
||||
va_start(args, fmt);
|
||||
vsnprintf(err, err_size, fmt, args);
|
||||
va_end(args);
|
||||
}
|
||||
|
||||
bool path_is_within_root(const char* root, const char* path) {
|
||||
size_t root_len = strlen(root);
|
||||
return strncmp(root, path, root_len) == 0 && (path[root_len] == '\0' || path[root_len] == '/');
|
||||
@@ -672,22 +682,24 @@ static bool is_synced_dir(const PathIndex* dirs, const char* rel) {
|
||||
return path_index_contains(dirs, rel[0] == '\0' ? "." : rel);
|
||||
}
|
||||
|
||||
/* Remove the extras directly inside the directory open on `dirfd`, recursing
|
||||
into every child directory so kept content below a synchronized prefix is
|
||||
reached. `all_removed` reports whether every child entry was removed (so the
|
||||
caller may rmdir this directory). A child directory is never removed when it
|
||||
is itself a synchronized directory or holds kept content; with a dirs index
|
||||
supplied, direct children of a non-synchronized directory are never extras at
|
||||
all (they are left in place but still descended into). Symlinks are unlinked
|
||||
like any other non-directory extra (never followed). */
|
||||
static bool delete_extras_fd(int dirfd, const char* rel_path, const PathIndex* keep,
|
||||
const PathIndex* dirs, DeleteBudget* budget,
|
||||
const DeleteSkipEntry* skips, int skip_count,
|
||||
const FilterRuleList* protect_rules, bool parent_deletable,
|
||||
bool* all_removed, DeletePathObserver observer,
|
||||
void* observer_context) {
|
||||
/* openat(dirfd, ".") opens an independent file description: a dup() would
|
||||
share dirfd's file offset and a prior pass could leave the stream drained. */
|
||||
/* Unsigned byte-wise string compare, matching rsync's u_strcmp (a signed
|
||||
strcmp would order bytes >= 0x80 differently). */
|
||||
static int delete_name_cmp(const char* a, const char* b) {
|
||||
const unsigned char* pa = (const unsigned char*)a;
|
||||
const unsigned char* pb = (const unsigned char*)b;
|
||||
while (*pa != '\0' && *pa == *pb) {
|
||||
pa++;
|
||||
pb++;
|
||||
}
|
||||
return (int)*pa - (int)*pb;
|
||||
}
|
||||
|
||||
bool delete_dir_entries_collect(int dirfd, DeleteDirEntry** out, size_t* count,
|
||||
bool* operation_ok) {
|
||||
*out = NULL;
|
||||
*count = 0;
|
||||
if (operation_ok)
|
||||
*operation_ok = true;
|
||||
int scanfd = openat(dirfd, ".", O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||
if (scanfd < 0)
|
||||
return false;
|
||||
@@ -696,17 +708,127 @@ static bool delete_extras_fd(int dirfd, const char* rel_path, const PathIndex* k
|
||||
close(scanfd);
|
||||
return false;
|
||||
}
|
||||
bool operation_ok = true;
|
||||
bool local_survives = false;
|
||||
/* A directory is deletable when it or ANY ancestor is synchronized; the
|
||||
`parent_deletable` flag carries that down the recursion so dest-only
|
||||
directories below a synchronized root are removed wholesale. */
|
||||
bool deletable = parent_deletable || is_synced_dir(dirs, rel_path);
|
||||
DeleteDirEntry* entries = NULL;
|
||||
size_t used = 0;
|
||||
size_t capacity = 0;
|
||||
bool ok = true;
|
||||
const struct dirent* entry;
|
||||
while ((entry = readdir(dir)) != NULL) {
|
||||
if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
|
||||
continue;
|
||||
char* child_rel = path_cat((char*)rel_path, entry->d_name);
|
||||
struct stat st;
|
||||
if (fstatat(dirfd, entry->d_name, &st, AT_SYMLINK_NOFOLLOW) != 0) {
|
||||
if (errno != ENOENT && operation_ok)
|
||||
*operation_ok = false;
|
||||
continue;
|
||||
}
|
||||
if (used == capacity) {
|
||||
size_t next = capacity == 0 ? 16 : capacity * 2;
|
||||
DeleteDirEntry* grown = realloc(entries, next * sizeof(*grown));
|
||||
if (!grown) {
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
entries = grown;
|
||||
capacity = next;
|
||||
}
|
||||
entries[used].name = str_dup(entry->d_name);
|
||||
if (!entries[used].name) {
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
entries[used].is_dir = S_ISDIR(st.st_mode);
|
||||
used++;
|
||||
}
|
||||
closedir(dir);
|
||||
if (!ok) {
|
||||
delete_dir_entries_free(entries, used);
|
||||
return false;
|
||||
}
|
||||
*out = entries;
|
||||
*count = used;
|
||||
return true;
|
||||
}
|
||||
|
||||
void delete_dir_entries_free(DeleteDirEntry* entries, size_t count) {
|
||||
if (!entries)
|
||||
return;
|
||||
for (size_t i = 0; i < count; i++)
|
||||
free(entries[i].name);
|
||||
free(entries);
|
||||
}
|
||||
|
||||
/* rsync's extraneous-entry order: subdirectories before files, each group in
|
||||
descending name order. */
|
||||
int delete_dir_entry_cmp_desc(const void* a, const void* b) {
|
||||
const DeleteDirEntry* ea = a;
|
||||
const DeleteDirEntry* eb = b;
|
||||
if (ea->is_dir != eb->is_dir)
|
||||
return ea->is_dir ? -1 : 1;
|
||||
return -delete_name_cmp(ea->name, eb->name);
|
||||
}
|
||||
|
||||
/* rsync's kept-subdirectory order: plain ascending name. */
|
||||
int delete_dir_entry_cmp_asc(const void* a, const void* b) {
|
||||
const DeleteDirEntry* ea = a;
|
||||
const DeleteDirEntry* eb = b;
|
||||
return delete_name_cmp(ea->name, eb->name);
|
||||
}
|
||||
|
||||
/* Remove the extras directly inside the directory open on `dirfd`, recursing
|
||||
into every child directory so kept content below a synchronized prefix is
|
||||
reached. `all_removed` reports whether every child entry was removed (so the
|
||||
caller may rmdir this directory). A child directory is never removed when it
|
||||
is itself a synchronized directory or holds kept content; with a dirs index
|
||||
supplied, direct children of a non-synchronized directory are never extras at
|
||||
all (they are left in place but still descended into). Symlinks are unlinked
|
||||
like any other non-directory extra (never followed).
|
||||
|
||||
Entries are processed in rsync's order (extraneous subdirectories in
|
||||
descending name order, then extraneous files, then kept subdirectories in
|
||||
ascending order) rather than readdir() order, so `--max-delete` leaves the
|
||||
same survivors and the `--info=del`/dry-run line order matches rsync. */
|
||||
static bool delete_extras_fd(int dirfd, const char* rel_path, const PathIndex* keep,
|
||||
const PathIndex* dirs, DeleteBudget* budget,
|
||||
const DeleteSkipEntry* skips, int skip_count,
|
||||
const FilterRuleList* protect_rules, bool parent_deletable,
|
||||
bool* all_removed, DeletePathObserver observer,
|
||||
void* observer_context) {
|
||||
DeleteDirEntry* entries = NULL;
|
||||
size_t count = 0;
|
||||
bool collect_ok = true;
|
||||
if (!delete_dir_entries_collect(dirfd, &entries, &count, &collect_ok))
|
||||
return false;
|
||||
bool operation_ok = collect_ok;
|
||||
bool local_survives = false;
|
||||
bool* shielded = calloc(count ? count : 1, sizeof(bool));
|
||||
bool* is_extra = calloc(count ? count : 1, sizeof(bool));
|
||||
if (!shielded || !is_extra) {
|
||||
free(shielded);
|
||||
free(is_extra);
|
||||
delete_dir_entries_free(entries, count);
|
||||
return false;
|
||||
}
|
||||
/* A directory is deletable when it or ANY ancestor is synchronized; the
|
||||
`parent_deletable` flag carries that down the recursion so dest-only
|
||||
directories below a synchronized root are removed wholesale. */
|
||||
bool deletable = parent_deletable || is_synced_dir(dirs, rel_path);
|
||||
bool at_root = rel_path[0] == '\0';
|
||||
|
||||
/* Reproduce rsync's traversal order: extraneous subdirectories in descending
|
||||
name order, then extraneous files in descending name order, and kept
|
||||
subdirectories only afterwards (ascending). Sorting up front also fixes the
|
||||
identity of the survivors under a partial --max-delete. */
|
||||
if (count > 1)
|
||||
qsort(entries, count, sizeof(*entries), delete_dir_entry_cmp_desc);
|
||||
size_t dir_count = 0;
|
||||
while (dir_count < count && entries[dir_count].is_dir)
|
||||
dir_count++;
|
||||
|
||||
/* Classify every entry up front (the verdict does not depend on processing
|
||||
order) so the ordered passes below can act on it. */
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
char* child_rel = path_cat((char*)rel_path, entries[i].name);
|
||||
if (!child_rel) {
|
||||
operation_ok = false;
|
||||
continue;
|
||||
@@ -718,93 +840,142 @@ static bool delete_extras_fd(int dirfd, const char* rel_path, const PathIndex* k
|
||||
top-level-only prefix) and the basis prefixes are protected: a nested
|
||||
destination directory that happens to be called .fastsync-stage is
|
||||
ordinary content. */
|
||||
if (path_under_skip_prefix(child_rel, rel_path[0] == '\0', skips, skip_count)) {
|
||||
if (path_under_skip_prefix(child_rel, at_root, skips, skip_count)) {
|
||||
shielded[i] = true;
|
||||
local_survives = true;
|
||||
free(child_rel);
|
||||
continue;
|
||||
}
|
||||
struct stat st;
|
||||
if (fstatat(dirfd, entry->d_name, &st, AT_SYMLINK_NOFOLLOW) != 0) {
|
||||
if (errno != ENOENT)
|
||||
operation_ok = false;
|
||||
free(child_rel);
|
||||
continue;
|
||||
}
|
||||
bool is_dir = S_ISDIR(st.st_mode);
|
||||
if (protect_rules && filter_rules_apply_side(protect_rules, child_rel, entry->d_name, is_dir,
|
||||
FILTER_SIDE_RECEIVER) == FILTER_ACTION_PROTECT) {
|
||||
} else if (protect_rules &&
|
||||
filter_rules_apply_side(protect_rules, child_rel, entries[i].name, entries[i].is_dir,
|
||||
FILTER_SIDE_RECEIVER) == FILTER_ACTION_PROTECT) {
|
||||
/* A first-match protect rule shields the extra; for a directory the whole
|
||||
subtree is shielded (rsync prunes an excluded directory), so do not
|
||||
descend. */
|
||||
shielded[i] = true;
|
||||
local_survives = true;
|
||||
free(child_rel);
|
||||
} else if (entries[i].is_dir) {
|
||||
bool child_synced = dirs && path_index_contains(dirs, child_rel);
|
||||
is_extra[i] = deletable && !child_synced && !keep_is_dir(keep, child_rel);
|
||||
if (!is_extra[i])
|
||||
local_survives = true;
|
||||
} else {
|
||||
is_extra[i] = deletable && !keep_is_file(keep, child_rel);
|
||||
if (!is_extra[i])
|
||||
local_survives = true;
|
||||
}
|
||||
free(child_rel);
|
||||
}
|
||||
|
||||
/* Pass 1: extraneous subdirectories, descending. */
|
||||
for (size_t i = 0; i < dir_count; i++) {
|
||||
if (!is_extra[i])
|
||||
continue;
|
||||
char* child_rel = path_cat((char*)rel_path, entries[i].name);
|
||||
if (!child_rel) {
|
||||
operation_ok = false;
|
||||
continue;
|
||||
}
|
||||
if (is_dir) {
|
||||
int childfd = openat(dirfd, entry->d_name, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||
bool child_all_removed = false;
|
||||
if (childfd >= 0) {
|
||||
if (!delete_extras_fd(childfd, child_rel, keep, dirs, budget, skips, skip_count,
|
||||
protect_rules, deletable, &child_all_removed, observer,
|
||||
observer_context))
|
||||
operation_ok = false;
|
||||
close(childfd);
|
||||
} else if (errno != ENOENT) {
|
||||
int childfd = openat(dirfd, entries[i].name, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||
bool child_all_removed = false;
|
||||
if (childfd >= 0) {
|
||||
if (!delete_extras_fd(childfd, child_rel, keep, dirs, budget, skips, skip_count,
|
||||
protect_rules, deletable, &child_all_removed, observer,
|
||||
observer_context))
|
||||
operation_ok = false;
|
||||
}
|
||||
bool child_synced = dirs && path_index_contains(dirs, child_rel);
|
||||
if (child_synced || keep_is_dir(keep, child_rel)) {
|
||||
/* A synchronized directory and a directory holding kept content are
|
||||
never removed. */
|
||||
local_survives = true;
|
||||
} else if (child_all_removed && deletable) {
|
||||
if (budget->deleted >= budget->max_delete) {
|
||||
budget->limit_hit = true;
|
||||
budget->skipped++;
|
||||
local_survives = true;
|
||||
} else if (unlinkat(dirfd, entry->d_name, AT_REMOVEDIR) != 0) {
|
||||
/* ENOENT: already gone (fine). ENOTEMPTY/EEXIST: the directory
|
||||
still holds entries the walker leaves in place (a protected
|
||||
excluded prefix, a kept file the manifest protects, a symlink);
|
||||
rsync leaves such a directory behind, so this is not an error.
|
||||
Only genuine I/O failures abort the deletion. */
|
||||
if (errno != ENOENT && errno != ENOTEMPTY && errno != EEXIST)
|
||||
operation_ok = false;
|
||||
local_survives = true;
|
||||
} else {
|
||||
budget->deleted++;
|
||||
if (observer)
|
||||
observer(observer_context, child_rel);
|
||||
}
|
||||
} else {
|
||||
local_survives = true;
|
||||
}
|
||||
} else {
|
||||
bool found = keep_is_file(keep, child_rel);
|
||||
if (found || !deletable) {
|
||||
/* Kept file, or a child of a directory that is not synchronized: never
|
||||
an extra for this run. */
|
||||
local_survives = true;
|
||||
} else if (budget->deleted >= budget->max_delete) {
|
||||
close(childfd);
|
||||
} else if (errno != ENOENT) {
|
||||
operation_ok = false;
|
||||
}
|
||||
if (child_all_removed && deletable) {
|
||||
if (budget->deleted >= budget->max_delete) {
|
||||
budget->limit_hit = true;
|
||||
budget->skipped++;
|
||||
local_survives = true;
|
||||
} else if (unlinkat(dirfd, entry->d_name, 0) != 0) {
|
||||
if (errno != ENOENT)
|
||||
} else if (unlinkat(dirfd, entries[i].name, AT_REMOVEDIR) != 0) {
|
||||
/* ENOENT: already gone (fine). ENOTEMPTY/EEXIST: the directory still
|
||||
holds entries the walker leaves in place (a protected excluded
|
||||
prefix, a kept file the manifest protects, a symlink); rsync leaves
|
||||
such a directory behind, so this is not an error. Only genuine I/O
|
||||
failures abort the deletion. */
|
||||
if (errno != ENOENT && errno != ENOTEMPTY && errno != EEXIST)
|
||||
operation_ok = false;
|
||||
local_survives = true;
|
||||
} else {
|
||||
budget->deleted++;
|
||||
/* rsync reports a removed directory with a trailing slash. */
|
||||
if (observer) {
|
||||
size_t len = strlen(child_rel);
|
||||
char* with_slash = malloc(len + 2);
|
||||
if (with_slash) {
|
||||
memcpy(with_slash, child_rel, len);
|
||||
with_slash[len] = '/';
|
||||
with_slash[len + 1] = '\0';
|
||||
observer(observer_context, with_slash);
|
||||
free(with_slash);
|
||||
} else {
|
||||
observer(observer_context, child_rel);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
local_survives = true;
|
||||
}
|
||||
free(child_rel);
|
||||
}
|
||||
|
||||
/* Pass 2: extraneous files, descending. */
|
||||
for (size_t i = dir_count; i < count; i++) {
|
||||
if (!is_extra[i])
|
||||
continue;
|
||||
if (budget->deleted >= budget->max_delete) {
|
||||
budget->limit_hit = true;
|
||||
budget->skipped++;
|
||||
local_survives = true;
|
||||
} else if (unlinkat(dirfd, entries[i].name, 0) != 0) {
|
||||
if (errno != ENOENT)
|
||||
operation_ok = false;
|
||||
local_survives = true;
|
||||
} else {
|
||||
budget->deleted++;
|
||||
char* child_rel = path_cat((char*)rel_path, entries[i].name);
|
||||
if (child_rel) {
|
||||
if (observer)
|
||||
observer(observer_context, child_rel);
|
||||
char* escaped_path = output_escape(child_rel, log_get_8_bit_output());
|
||||
fprintf(stderr, " Deleted: %s\n", escaped_path ? escaped_path : "<allocation failed>");
|
||||
free(escaped_path);
|
||||
}
|
||||
free(child_rel);
|
||||
}
|
||||
}
|
||||
|
||||
/* Pass 3: kept subdirectories, ascending (rsync descends into these only
|
||||
after the parent's own extras have been handled). */
|
||||
for (size_t i = dir_count; i-- > 0;) {
|
||||
if (is_extra[i] || shielded[i])
|
||||
continue;
|
||||
char* child_rel = path_cat((char*)rel_path, entries[i].name);
|
||||
if (!child_rel) {
|
||||
operation_ok = false;
|
||||
continue;
|
||||
}
|
||||
int childfd = openat(dirfd, entries[i].name, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||
bool child_all_removed = false;
|
||||
if (childfd >= 0) {
|
||||
if (!delete_extras_fd(childfd, child_rel, keep, dirs, budget, skips, skip_count,
|
||||
protect_rules, deletable, &child_all_removed, observer,
|
||||
observer_context))
|
||||
operation_ok = false;
|
||||
close(childfd);
|
||||
} else if (errno != ENOENT) {
|
||||
operation_ok = false;
|
||||
}
|
||||
/* A kept/synchronized directory is never removed. */
|
||||
local_survives = true;
|
||||
free(child_rel);
|
||||
}
|
||||
closedir(dir);
|
||||
|
||||
free(shielded);
|
||||
free(is_extra);
|
||||
delete_dir_entries_free(entries, count);
|
||||
*all_removed = !local_survives;
|
||||
return operation_ok;
|
||||
}
|
||||
@@ -817,97 +988,147 @@ static bool list_extras_fd(int dirfd, const char* rel_path, const PathIndex* kee
|
||||
const DeleteSkipEntry* skips, int skip_count,
|
||||
const FilterRuleList* protect_rules, bool parent_deletable,
|
||||
bool* all_removed) {
|
||||
int scanfd = openat(dirfd, ".", O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||
if (scanfd < 0)
|
||||
DeleteDirEntry* entries = NULL;
|
||||
size_t count = 0;
|
||||
bool collect_ok = true;
|
||||
if (!delete_dir_entries_collect(dirfd, &entries, &count, &collect_ok))
|
||||
return false;
|
||||
DIR* dir = fdopendir(scanfd);
|
||||
if (!dir) {
|
||||
close(scanfd);
|
||||
bool operation_ok = collect_ok;
|
||||
bool local_survives = false;
|
||||
bool* shielded = calloc(count ? count : 1, sizeof(bool));
|
||||
bool* is_extra = calloc(count ? count : 1, sizeof(bool));
|
||||
if (!shielded || !is_extra) {
|
||||
free(shielded);
|
||||
free(is_extra);
|
||||
delete_dir_entries_free(entries, count);
|
||||
return false;
|
||||
}
|
||||
bool operation_ok = true;
|
||||
bool local_survives = false;
|
||||
bool deletable = parent_deletable || is_synced_dir(dirs, rel_path);
|
||||
const struct dirent* entry;
|
||||
while ((entry = readdir(dir)) != NULL) {
|
||||
if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
|
||||
continue;
|
||||
char* child_rel = path_cat((char*)rel_path, entry->d_name);
|
||||
bool at_root = rel_path[0] == '\0';
|
||||
|
||||
/* Mirror the delete walk's rsync order (extraneous subdirectories descending,
|
||||
then extraneous files descending, then kept subdirectories ascending). */
|
||||
if (count > 1)
|
||||
qsort(entries, count, sizeof(*entries), delete_dir_entry_cmp_desc);
|
||||
size_t dir_count = 0;
|
||||
while (dir_count < count && entries[dir_count].is_dir)
|
||||
dir_count++;
|
||||
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
char* child_rel = path_cat((char*)rel_path, entries[i].name);
|
||||
if (!child_rel) {
|
||||
operation_ok = false;
|
||||
continue;
|
||||
}
|
||||
if (path_under_skip_prefix(child_rel, rel_path[0] == '\0', skips, skip_count)) {
|
||||
if (path_under_skip_prefix(child_rel, at_root, skips, skip_count)) {
|
||||
shielded[i] = true;
|
||||
local_survives = true;
|
||||
free(child_rel);
|
||||
continue;
|
||||
}
|
||||
struct stat st;
|
||||
if (fstatat(dirfd, entry->d_name, &st, AT_SYMLINK_NOFOLLOW) != 0) {
|
||||
if (errno != ENOENT)
|
||||
operation_ok = false;
|
||||
free(child_rel);
|
||||
continue;
|
||||
}
|
||||
bool is_dir = S_ISDIR(st.st_mode);
|
||||
if (protect_rules && filter_rules_apply_side(protect_rules, child_rel, entry->d_name, is_dir,
|
||||
FILTER_SIDE_RECEIVER) == FILTER_ACTION_PROTECT) {
|
||||
} else if (protect_rules &&
|
||||
filter_rules_apply_side(protect_rules, child_rel, entries[i].name, entries[i].is_dir,
|
||||
FILTER_SIDE_RECEIVER) == FILTER_ACTION_PROTECT) {
|
||||
/* Mirror the delete walk: a protected entry is never reported as a
|
||||
would-delete and a protected directory's subtree is not enumerated. */
|
||||
shielded[i] = true;
|
||||
local_survives = true;
|
||||
free(child_rel);
|
||||
} else if (entries[i].is_dir) {
|
||||
bool child_synced = dirs && path_index_contains(dirs, child_rel);
|
||||
is_extra[i] = deletable && !child_synced && !keep_is_dir(keep, child_rel);
|
||||
if (!is_extra[i])
|
||||
local_survives = true;
|
||||
} else {
|
||||
is_extra[i] = deletable && !keep_is_file(keep, child_rel);
|
||||
if (!is_extra[i])
|
||||
local_survives = true;
|
||||
}
|
||||
free(child_rel);
|
||||
}
|
||||
|
||||
/* Pass 1: extraneous subdirectories, descending (recorded after contents). */
|
||||
for (size_t i = 0; i < dir_count; i++) {
|
||||
if (!is_extra[i])
|
||||
continue;
|
||||
char* child_rel = path_cat((char*)rel_path, entries[i].name);
|
||||
if (!child_rel) {
|
||||
operation_ok = false;
|
||||
continue;
|
||||
}
|
||||
if (is_dir) {
|
||||
int childfd = openat(dirfd, entry->d_name, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||
bool child_all_removed = false;
|
||||
if (childfd >= 0) {
|
||||
if (!list_extras_fd(childfd, child_rel, keep, dirs, out, recorded, skips, skip_count,
|
||||
protect_rules, deletable, &child_all_removed))
|
||||
operation_ok = false;
|
||||
close(childfd);
|
||||
} else if (errno != ENOENT) {
|
||||
int childfd = openat(dirfd, entries[i].name, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||
bool child_all_removed = false;
|
||||
if (childfd >= 0) {
|
||||
if (!list_extras_fd(childfd, child_rel, keep, dirs, out, recorded, skips, skip_count,
|
||||
protect_rules, deletable, &child_all_removed))
|
||||
operation_ok = false;
|
||||
close(childfd);
|
||||
} else if (errno != ENOENT) {
|
||||
operation_ok = false;
|
||||
}
|
||||
if (child_all_removed && deletable) {
|
||||
size_t len = strlen(child_rel);
|
||||
char* copy = malloc(len + 2);
|
||||
if (!copy) {
|
||||
operation_ok = false;
|
||||
}
|
||||
bool child_synced = dirs && path_index_contains(dirs, child_rel);
|
||||
if (child_synced || keep_is_dir(keep, child_rel)) {
|
||||
local_survives = true;
|
||||
} else if (child_all_removed && deletable) {
|
||||
size_t len = strlen(child_rel);
|
||||
char* copy = malloc(len + 2);
|
||||
if (!copy) {
|
||||
operation_ok = false;
|
||||
} else {
|
||||
memcpy(copy, child_rel, len);
|
||||
copy[len] = '/';
|
||||
copy[len + 1] = '\0';
|
||||
if (!array_list_add(out, copy)) {
|
||||
free(copy);
|
||||
operation_ok = false;
|
||||
} else {
|
||||
(*recorded)++;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
local_survives = true;
|
||||
}
|
||||
} else {
|
||||
bool found = keep_is_file(keep, child_rel);
|
||||
if (found || !deletable) {
|
||||
local_survives = true;
|
||||
} else {
|
||||
char* copy = str_dup(child_rel);
|
||||
if (!copy || !array_list_add(out, copy)) {
|
||||
memcpy(copy, child_rel, len);
|
||||
copy[len] = '/';
|
||||
copy[len + 1] = '\0';
|
||||
if (!array_list_add(out, copy)) {
|
||||
free(copy);
|
||||
operation_ok = false;
|
||||
} else {
|
||||
(*recorded)++;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
local_survives = true;
|
||||
}
|
||||
free(child_rel);
|
||||
}
|
||||
closedir(dir);
|
||||
|
||||
/* Pass 2: extraneous files, descending. */
|
||||
for (size_t i = dir_count; i < count; i++) {
|
||||
if (!is_extra[i])
|
||||
continue;
|
||||
char* child_rel = path_cat((char*)rel_path, entries[i].name);
|
||||
if (!child_rel) {
|
||||
operation_ok = false;
|
||||
continue;
|
||||
}
|
||||
char* copy = str_dup(child_rel);
|
||||
if (!copy || !array_list_add(out, copy)) {
|
||||
free(copy);
|
||||
operation_ok = false;
|
||||
} else {
|
||||
(*recorded)++;
|
||||
}
|
||||
free(child_rel);
|
||||
}
|
||||
|
||||
/* Pass 3: kept subdirectories, ascending. */
|
||||
for (size_t i = dir_count; i-- > 0;) {
|
||||
if (is_extra[i] || shielded[i])
|
||||
continue;
|
||||
char* child_rel = path_cat((char*)rel_path, entries[i].name);
|
||||
if (!child_rel) {
|
||||
operation_ok = false;
|
||||
continue;
|
||||
}
|
||||
int childfd = openat(dirfd, entries[i].name, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||
bool child_all_removed = false;
|
||||
if (childfd >= 0) {
|
||||
if (!list_extras_fd(childfd, child_rel, keep, dirs, out, recorded, skips, skip_count,
|
||||
protect_rules, deletable, &child_all_removed))
|
||||
operation_ok = false;
|
||||
close(childfd);
|
||||
} else if (errno != ENOENT) {
|
||||
operation_ok = false;
|
||||
}
|
||||
local_survives = true;
|
||||
free(child_rel);
|
||||
}
|
||||
|
||||
free(shielded);
|
||||
free(is_extra);
|
||||
delete_dir_entries_free(entries, count);
|
||||
*all_removed = !local_survives;
|
||||
return operation_ok;
|
||||
}
|
||||
|
||||
@@ -134,6 +134,28 @@ typedef struct {
|
||||
only DIRECT children of the destination root, i.e. child_rel has no '/'). */
|
||||
bool path_under_skip_prefix(const char* child_rel, bool at_root, const DeleteSkipEntry* skips,
|
||||
int skip_count);
|
||||
/* One destination-directory entry collected up front so the delete walkers can
|
||||
reproduce rsync's traversal order instead of readdir() order. rsync processes
|
||||
a directory's extraneous subdirectories first (descending name, depth-first),
|
||||
then its extraneous files (descending name), and only afterwards descends into
|
||||
its kept subdirectories (ascending name). */
|
||||
typedef struct {
|
||||
char* name;
|
||||
bool is_dir;
|
||||
} DeleteDirEntry;
|
||||
/* Collect the entries of the directory open on `dirfd` (excluding "." and ".."),
|
||||
stat'ing each with AT_SYMLINK_NOFOLLOW. On success *out is a malloc'd array of
|
||||
*count entries whose names the caller frees with delete_dir_entries_free().
|
||||
Returns false on an allocation/readdir failure; a vanished entry (ENOENT) is
|
||||
skipped, any other stat failure is reported through *operation_ok while the
|
||||
walk continues. */
|
||||
bool delete_dir_entries_collect(int dirfd, DeleteDirEntry** out, size_t* count, bool* operation_ok);
|
||||
void delete_dir_entries_free(DeleteDirEntry* entries, size_t count);
|
||||
/* Sort comparators: `_desc` orders subdirectories before files and each group by
|
||||
descending name (rsync's extraneous-entry order); `_asc` orders plain ascending
|
||||
name (rsync's kept-subdirectory order). */
|
||||
int delete_dir_entry_cmp_desc(const void* a, const void* b);
|
||||
int delete_dir_entry_cmp_asc(const void* a, const void* b);
|
||||
/* Remove files/dirs/symlinks under dest_root that are not listed in manifest
|
||||
without ever descending into a protected prefix (see DeleteSkipEntry). When
|
||||
`synced_dirs` is non-NULL, extras are only removed directly inside a directory
|
||||
@@ -203,6 +225,11 @@ void utils_set_authorized_root_fd(int fd);
|
||||
* threads spawn; see utils.c). */
|
||||
int utils_get_authorized_root_fd(void);
|
||||
const char* utils_get_authorized_root_path(void);
|
||||
/* Write a diagnostic message into a caller-supplied buffer, mirroring
|
||||
* vsnprintf. A NULL `err` or a zero `err_size` is a no-op, so a caller that
|
||||
* only needs the boolean status may safely pass NULL. Returns nothing; the
|
||||
* buffer is always NUL-terminated by vsnprintf when err_size > 0. */
|
||||
void utils_set_error(char* err, size_t err_size, const char* fmt, ...);
|
||||
/* True when `path` is `root` itself or lies directly beneath it: a lexical
|
||||
* prefix test requiring the byte after `root` to be '\0' or '/'. Both `root`
|
||||
* and `path` must be absolute canonical paths free of "."/".." components (the
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
OLDDEST
|
||||
@@ -1 +0,0 @@
|
||||
NEWCONTENT
|
||||
@@ -1 +0,0 @@
|
||||
NEWCONTENT
|
||||
+55
-12
@@ -20,6 +20,12 @@ CLIENT_CMD = [os.path.join(BUILD_DIR, "client")]
|
||||
_WORKER = os.environ.get("PYTEST_XDIST_WORKER")
|
||||
TEST_DATA_DIR = os.path.join(PROJECT_ROOT, f"test_data-{_WORKER}" if _WORKER else "test_data")
|
||||
|
||||
# Default wall-clock budget for a short-lived client invocation. Every client
|
||||
# is expected to finish well within this; the bound exists so a hung client
|
||||
# fails the test instead of stalling the whole CI run indefinitely. Callers
|
||||
# that legitimately need longer can pass an explicit ``timeout``.
|
||||
CLIENT_TIMEOUT = 180
|
||||
|
||||
|
||||
class ServerManager:
|
||||
"""Manages a long-lived server process. Reuses across test cases."""
|
||||
@@ -137,7 +143,40 @@ class CountingProxy:
|
||||
return result
|
||||
|
||||
|
||||
def run_client(source_dir, dest_dir, flags=None, port=None, extra_args=None):
|
||||
def _run_client_cmd(cmd, timeout):
|
||||
"""Run one client command, returning ``(result, duration)``.
|
||||
|
||||
On timeout the client is killed and a result-like ``CompletedProcess`` with
|
||||
a non-zero returncode is returned instead of raising, so callers keep the
|
||||
established ``(result, duration)`` contract and the failure carries the
|
||||
command plus whatever output was captured for diagnosis.
|
||||
"""
|
||||
start = time.monotonic()
|
||||
try:
|
||||
result = subprocess.run(cmd, text=True, capture_output=True, timeout=timeout)
|
||||
except subprocess.TimeoutExpired as exc:
|
||||
duration = time.monotonic() - start
|
||||
stdout = exc.stdout or ""
|
||||
stderr = exc.stderr or ""
|
||||
if isinstance(stdout, bytes):
|
||||
stdout = stdout.decode(errors="replace")
|
||||
if isinstance(stderr, bytes):
|
||||
stderr = stderr.decode(errors="replace")
|
||||
diagnostic = (
|
||||
f"client timed out after {timeout}s\n"
|
||||
f"command: {cmd!r}\n"
|
||||
f"--- captured stdout ---\n{stdout}\n"
|
||||
f"--- captured stderr ---\n{stderr}"
|
||||
)
|
||||
result = subprocess.CompletedProcess(cmd, returncode=-1,
|
||||
stdout=stdout, stderr=diagnostic)
|
||||
return result, duration
|
||||
duration = time.monotonic() - start
|
||||
return result, duration
|
||||
|
||||
|
||||
def run_client(source_dir, dest_dir, flags=None, port=None, extra_args=None,
|
||||
timeout=CLIENT_TIMEOUT):
|
||||
"""Run the client and return (result, duration)."""
|
||||
cmd = CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir, "--save-to-disk"]
|
||||
if port:
|
||||
@@ -146,23 +185,18 @@ def run_client(source_dir, dest_dir, flags=None, port=None, extra_args=None):
|
||||
cmd += flags
|
||||
if extra_args:
|
||||
cmd += extra_args
|
||||
start = time.monotonic()
|
||||
result = subprocess.run(cmd, text=True, capture_output=True)
|
||||
duration = time.monotonic() - start
|
||||
return result, duration
|
||||
return _run_client_cmd(cmd, timeout)
|
||||
|
||||
|
||||
def run_client_posix(source_dir, dest_dir, flags=None, port=None):
|
||||
def run_client_posix(source_dir, dest_dir, flags=None, port=None,
|
||||
timeout=CLIENT_TIMEOUT):
|
||||
"""Run the client with positional args (rsync-style)."""
|
||||
cmd = CLIENT_CMD + [source_dir, dest_dir, "--save-to-disk"]
|
||||
if port:
|
||||
cmd += ["--server-port", str(port)]
|
||||
if flags:
|
||||
cmd += flags
|
||||
start = time.monotonic()
|
||||
result = subprocess.run(cmd, text=True, capture_output=True)
|
||||
duration = time.monotonic() - start
|
||||
return result, duration
|
||||
return _run_client_cmd(cmd, timeout)
|
||||
|
||||
|
||||
def generate_test_files(source_dir, full=False):
|
||||
@@ -238,8 +272,17 @@ def make_result(name, success, duration=None, error=""):
|
||||
|
||||
|
||||
def get_dest_received_dir(dest_dir, source_dir):
|
||||
"""Get the path where received files land inside dest_dir."""
|
||||
return os.path.join(dest_dir, os.path.abspath(source_dir).lstrip(os.sep))
|
||||
"""Get the path where received files land inside dest_dir.
|
||||
|
||||
FastSync mirrors the absolute source path below the receive root with the
|
||||
leading root separator removed. Strip that separator explicitly rather
|
||||
than with ``str.lstrip(os.sep)``: ``lstrip`` removes a *set* of characters
|
||||
rather than a path prefix, which is not the same operation.
|
||||
"""
|
||||
abs_source = os.path.abspath(source_dir)
|
||||
if abs_source.startswith(os.sep):
|
||||
abs_source = abs_source[len(os.sep):]
|
||||
return os.path.join(dest_dir, abs_source)
|
||||
|
||||
|
||||
def _find_free_port():
|
||||
|
||||
@@ -63,6 +63,10 @@ DETACH_MODULE = os.path.join(MODULE_ROOT, "detach")
|
||||
DETACH_CONF = os.path.join(TEST_DATA_DIR, "fastsyncd_detach.conf")
|
||||
DETACH_PORT = None
|
||||
|
||||
# A dedicated config for the umask test: the daemon must be launched in the real
|
||||
# (double-fork) detach path, whose daemonize() applies umask(022).
|
||||
UMASK_CONF = os.path.join(TEST_DATA_DIR, "fastsyncd_umask.conf")
|
||||
|
||||
# Passwords are never sent as plaintext and never logged; these literals are
|
||||
# only hashed into the server credential file / client password file.
|
||||
ALICE_PASS = "alice-s3cret"
|
||||
@@ -332,6 +336,44 @@ class TestDaemonModuleSelection:
|
||||
assert not missing, f"missing: {missing[:5]}"
|
||||
assert not mismatches, f"mismatch: {mismatches[:5]}"
|
||||
|
||||
def test_daemon_new_dirs_not_world_writable(self):
|
||||
"""The daemon must not force umask 0: implied parent directories created
|
||||
without -p are the source default (0755 under the daemon's 022 umask),
|
||||
never world-writable 0777.
|
||||
|
||||
This drives the real double-fork detach path, where the umask(022) fix
|
||||
lives (daemonize()); the --no-detach path never calls it. The launcher
|
||||
is run with umask 0, so without the fix the daemon would inherit 0 and
|
||||
create a 0777 directory; with the fix the assertion below fails only if
|
||||
the fix regresses."""
|
||||
port = _find_free_port()
|
||||
with open(UMASK_CONF, "w") as f:
|
||||
f.write("port = %d\n\n[files]\npath = %s\n" % (port, FILES_MODULE))
|
||||
sub = os.path.join(FILES_MODULE, "umask_check")
|
||||
shutil.rmtree(sub, ignore_errors=True)
|
||||
os.makedirs(sub, exist_ok=True)
|
||||
log_path = os.path.join(TEST_DATA_DIR, "fastsyncd_umask.log")
|
||||
log = open(log_path, "w")
|
||||
cmd = SERVER_CMD + ["--daemon", "--config", UMASK_CONF, "--allow-unauthenticated"]
|
||||
proc = subprocess.Popen(cmd, stdout=log, stderr=log, stdin=subprocess.DEVNULL,
|
||||
preexec_fn=lambda: os.umask(0))
|
||||
try:
|
||||
_wait_for_port(port, timeout=15)
|
||||
result = _push("127.0.0.1::files/umask_check", port)
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
received = get_dest_received_dir(sub, SOURCE_DIR)
|
||||
nested = os.path.join(received, "nested")
|
||||
assert os.path.isdir(nested), f"nested dir missing under {received}"
|
||||
mode = stat.S_IMODE(os.stat(nested).st_mode)
|
||||
assert (mode & 0o022) == 0, f"implied directory is group/other writable: {oct(mode)}"
|
||||
finally:
|
||||
_kill_by_cmdline_marker(UMASK_CONF)
|
||||
log.close()
|
||||
try:
|
||||
proc.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.kill()
|
||||
|
||||
|
||||
class TestDaemonRejection:
|
||||
def _tree_files(self):
|
||||
|
||||
@@ -0,0 +1,291 @@
|
||||
"""Differential coverage for the delete-timing ABORT BOUNDARY (A9/A10).
|
||||
|
||||
rsync's generator runs ahead of its throttled sender, so on a mid-transfer abort
|
||||
it has already removed every extra it planned. FastSync now transmits the
|
||||
COMPLETE per-directory plan set before the first data frame, so an abort has the
|
||||
same effect. Before that change FastSync only removed the extras of the
|
||||
directories its (slower) data stream had reached, and ``-d/--dirs`` used an
|
||||
end-of-transfer commit that removed nothing on abort.
|
||||
|
||||
These tests abort both tools mid-transfer and assert the destination extras
|
||||
removed match real ``rsync 3.4.1``. The rsync side is driven locally with
|
||||
``--bwlimit`` and a small timing window (its generator's delete list is computed
|
||||
long before the throttled payload finishes); the FastSync side uses the
|
||||
byte-deterministic slicing proxy from ``test_delete_timing_parity``.
|
||||
"""
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
from common import ( # noqa: E402
|
||||
TEST_DATA_DIR,
|
||||
ServerManager,
|
||||
clean_dir,
|
||||
get_dest_received_dir,
|
||||
run_client,
|
||||
)
|
||||
from test_delete_timing_parity import _SlicingProxy # noqa: E402
|
||||
|
||||
RSYNC = shutil.which("rsync")
|
||||
requires_rsync = pytest.mark.skipif(RSYNC is None, reason="rsync 3.4.1 not installed")
|
||||
|
||||
# Exceeds the 10 MiB scanner chunk, so the next directory lands in a later chunk
|
||||
# (still unreached when the proxy cuts the stream).
|
||||
BIG_BYTES = 16 * 1024 * 1024
|
||||
# Cut well past the (small) config + delete-plan frames and into the big payload,
|
||||
# so the receiver has provably processed every plan before the abort.
|
||||
MID_TRANSFER_BYTES = 256 * 1024
|
||||
PROXY_THROTTLE = 0.001
|
||||
# Throttle rsync's sender so the generator has deleted long before the payload
|
||||
# finishes, then interrupt it mid-transfer.
|
||||
RSYNC_BWLIMIT = 512 # KiB/s -> ~32 s for 16 MiB
|
||||
RSYNC_ABORT_DELAY = 1.5
|
||||
|
||||
|
||||
def _write(path, content):
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
with open(path, "wb") as fh:
|
||||
fh.write(content)
|
||||
|
||||
|
||||
def _rsync_aborted(args, delay=RSYNC_ABORT_DELAY):
|
||||
"""Start rsync, let its generator run, then interrupt it mid-transfer."""
|
||||
env = dict(os.environ, LC_ALL="C")
|
||||
proc = subprocess.Popen([RSYNC] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
||||
text=True, env=env)
|
||||
time.sleep(delay)
|
||||
proc.terminate()
|
||||
try:
|
||||
proc.wait(timeout=10)
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.kill()
|
||||
proc.wait(timeout=5)
|
||||
return proc
|
||||
|
||||
|
||||
class TestDeleteDuringAbortBoundary:
|
||||
"""A9: on an abort, every planned removal has already been applied."""
|
||||
|
||||
def _seed_recursive(self, tag):
|
||||
source = os.path.join(TEST_DATA_DIR, f"dab_{tag}_src")
|
||||
clean_dir(source)
|
||||
# ``a/keep.bin`` sorts first, so the client streams it (and the proxy
|
||||
# cuts) before the data pass ever reaches ``z/deep``.
|
||||
_write(os.path.join(source, "a", "keep.bin"), b"B" * BIG_BYTES)
|
||||
_write(os.path.join(source, "z", "deep", "keep.txt"), b"keep\n")
|
||||
return source
|
||||
|
||||
@requires_rsync
|
||||
def test_recursive_abort_removes_all_planned_extras(self):
|
||||
# ---- FastSync: abort mid ``a/keep.bin``; ``z/deep`` is never reached.
|
||||
source = self._seed_recursive("rec_fs")
|
||||
dest = os.path.join(TEST_DATA_DIR, "dab_rec_fs_dst")
|
||||
clean_dir(dest)
|
||||
received = get_dest_received_dir(dest, source)
|
||||
os.makedirs(os.path.join(received, "a"), exist_ok=True)
|
||||
_write(os.path.join(received, "a", "a_extra"), b"stale\n")
|
||||
os.makedirs(os.path.join(received, "z", "deep"), exist_ok=True)
|
||||
_write(os.path.join(received, "z", "deep", "old_extra"), b"stale\n")
|
||||
|
||||
with ServerManager() as server:
|
||||
server.start(extra_args=["--allow-delete"])
|
||||
proxy = _SlicingProxy(server.port, forward_limit=MID_TRANSFER_BYTES,
|
||||
throttle=PROXY_THROTTLE)
|
||||
result, _ = run_client(source, dest, flags=["--delete-during"], port=proxy.port)
|
||||
proxy.finish()
|
||||
assert result.returncode != 0, "truncated transfer reported success"
|
||||
assert not os.path.exists(os.path.join(received, "a", "a_extra"))
|
||||
assert not os.path.exists(os.path.join(received, "z", "deep", "old_extra")), (
|
||||
"FastSync left an extra in a directory it never reached before the abort"
|
||||
)
|
||||
|
||||
# ---- rsync 3.4.1: same tree, same abort, same delete outcome.
|
||||
source = self._seed_recursive("rec_rs")
|
||||
rsync_dst = os.path.join(TEST_DATA_DIR, "dab_rec_rs_dst")
|
||||
clean_dir(rsync_dst)
|
||||
os.makedirs(os.path.join(rsync_dst, "a"), exist_ok=True)
|
||||
_write(os.path.join(rsync_dst, "a", "a_extra"), b"stale\n")
|
||||
os.makedirs(os.path.join(rsync_dst, "z", "deep"), exist_ok=True)
|
||||
_write(os.path.join(rsync_dst, "z", "deep", "old_extra"), b"stale\n")
|
||||
|
||||
proc = _rsync_aborted(["-a", "--delete-during", f"--bwlimit={RSYNC_BWLIMIT}",
|
||||
source + "/", rsync_dst + "/"])
|
||||
assert proc.returncode != 0, "rsync was not actually interrupted"
|
||||
assert not os.path.exists(os.path.join(rsync_dst, "a", "a_extra"))
|
||||
assert not os.path.exists(os.path.join(rsync_dst, "z", "deep", "old_extra")), (
|
||||
"rsync's generator did not delete ahead of its sender"
|
||||
)
|
||||
|
||||
|
||||
class TestDirsDeleteAbortBoundary:
|
||||
"""A10: ``-d/--dirs`` uses per-directory plans like rsync.
|
||||
|
||||
The listed directory's direct extras are removed by the up-front plan while
|
||||
a kept but untraversed subdirectory (and its destination content) is
|
||||
shielded.
|
||||
"""
|
||||
|
||||
def _seed_dirs(self, tag):
|
||||
source = os.path.join(TEST_DATA_DIR, f"ddb_{tag}_src")
|
||||
clean_dir(source)
|
||||
_write(os.path.join(source, "big.bin"), b"B" * BIG_BYTES)
|
||||
_write(os.path.join(source, "subdir", "keep.txt"), b"inner\n")
|
||||
return source
|
||||
|
||||
@pytest.mark.parametrize("fs_flag,rs_flag", [("--delete-during", "--delete-during"),
|
||||
("--delete", "--delete")])
|
||||
@requires_rsync
|
||||
def test_dirs_abort_removes_direct_extras_only(self, fs_flag, rs_flag):
|
||||
label = f"{fs_flag.lstrip('-')}_{rs_flag.lstrip('-')}"
|
||||
# ---- FastSync: ``-d`` lists the immediate children; big.bin streams and
|
||||
# the abort lands mid-payload.
|
||||
source = self._seed_dirs(f"dirs_{label}_fs")
|
||||
dest = os.path.join(TEST_DATA_DIR, f"ddb_{label}_fs_dst")
|
||||
clean_dir(dest)
|
||||
received = get_dest_received_dir(dest, source)
|
||||
_write(os.path.join(received, "old_extra"), b"stale\n")
|
||||
os.makedirs(os.path.join(received, "subdir"), exist_ok=True)
|
||||
_write(os.path.join(received, "subdir", "stale.txt"), b"stale inner\n")
|
||||
|
||||
with ServerManager() as server:
|
||||
server.start(extra_args=["--allow-delete"])
|
||||
proxy = _SlicingProxy(server.port, forward_limit=MID_TRANSFER_BYTES,
|
||||
throttle=PROXY_THROTTLE)
|
||||
result, _ = run_client(source + "/", dest, flags=["-d", fs_flag], port=proxy.port)
|
||||
proxy.finish()
|
||||
assert result.returncode != 0, f"{fs_flag}: truncated transfer reported success"
|
||||
assert not os.path.exists(os.path.join(received, "old_extra")), (
|
||||
f"{fs_flag}: the listed directory's direct extra survived the abort"
|
||||
)
|
||||
assert os.path.exists(os.path.join(received, "subdir", "stale.txt")), (
|
||||
f"{fs_flag}: descended into a kept, untraversed subdirectory"
|
||||
)
|
||||
|
||||
# ---- rsync 3.4.1: same shape and same abort.
|
||||
source = self._seed_dirs(f"dirs_{label}_rs")
|
||||
rsync_dst = os.path.join(TEST_DATA_DIR, f"ddb_{label}_rs_dst")
|
||||
clean_dir(rsync_dst)
|
||||
_write(os.path.join(rsync_dst, "old_extra"), b"stale\n")
|
||||
os.makedirs(os.path.join(rsync_dst, "subdir"), exist_ok=True)
|
||||
_write(os.path.join(rsync_dst, "subdir", "stale.txt"), b"stale inner\n")
|
||||
|
||||
proc = _rsync_aborted(["-d", rs_flag, f"--bwlimit={RSYNC_BWLIMIT}",
|
||||
source + "/", rsync_dst + "/"])
|
||||
assert proc.returncode != 0, "rsync was not actually interrupted"
|
||||
assert not os.path.exists(os.path.join(rsync_dst, "old_extra")), (
|
||||
f"rsync {rs_flag}: the listed directory's direct extra survived the abort"
|
||||
)
|
||||
assert os.path.exists(os.path.join(rsync_dst, "subdir", "stale.txt")), (
|
||||
f"rsync {rs_flag}: descended into a kept, untraversed subdirectory"
|
||||
)
|
||||
|
||||
|
||||
def _tree(root):
|
||||
out = []
|
||||
for dirpath, dirs, files in os.walk(root):
|
||||
for name in dirs:
|
||||
out.append(os.path.relpath(os.path.join(dirpath, name), root))
|
||||
for name in files:
|
||||
out.append(os.path.relpath(os.path.join(dirpath, name), root))
|
||||
return sorted(out)
|
||||
|
||||
|
||||
class TestDirsDeleteFinalStateParity:
|
||||
"""A10 completed run: ``-d DIR/ --delete`` (during default) and
|
||||
``--delete-during`` match rsync's final tree, including a kept but
|
||||
untraversed subdirectory whose destination content survives."""
|
||||
|
||||
@pytest.mark.parametrize("flag", ["--delete", "--delete-during"])
|
||||
@requires_rsync
|
||||
def test_dirs_final_state_matches_rsync(self, flag):
|
||||
source = os.path.join(TEST_DATA_DIR, f"ddf_{flag.lstrip('-')}_src")
|
||||
clean_dir(source)
|
||||
_write(os.path.join(source, "keep.txt"), b"new keep\n")
|
||||
_write(os.path.join(source, "subdir", "inner.txt"), b"inner\n")
|
||||
|
||||
def seed_dest(root):
|
||||
clean_dir(root)
|
||||
_write(os.path.join(root, "keep.txt"), b"old keep\n")
|
||||
_write(os.path.join(root, "extra.txt"), b"extra\n")
|
||||
_write(os.path.join(root, "extrasub", "ex.txt"), b"extra sub\n")
|
||||
_write(os.path.join(root, "subdir", "stale.txt"), b"stale inner\n")
|
||||
|
||||
rsync_dst = os.path.join(TEST_DATA_DIR, f"ddf_{flag.lstrip('-')}_rs_dst")
|
||||
seed_dest(rsync_dst)
|
||||
env = dict(os.environ, LC_ALL="C")
|
||||
rsync_result = subprocess.run(
|
||||
[RSYNC, "-d", flag, source + "/", rsync_dst + "/"],
|
||||
capture_output=True, text=True, env=env, timeout=120)
|
||||
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||
rsync_tree = _tree(rsync_dst)
|
||||
|
||||
dest = os.path.join(TEST_DATA_DIR, f"ddf_{flag.lstrip('-')}_fs_dst")
|
||||
clean_dir(dest)
|
||||
received = get_dest_received_dir(dest, source)
|
||||
seed_dest(received)
|
||||
with ServerManager() as server:
|
||||
server.start(extra_args=["--allow-delete"])
|
||||
result, _ = run_client(source + "/", dest, flags=["-d", flag], port=server.port)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||
fastsync_tree = _tree(received)
|
||||
assert fastsync_tree == rsync_tree, (
|
||||
f"-d {flag}: fastsync tree {fastsync_tree} != rsync tree {rsync_tree}")
|
||||
|
||||
|
||||
class TestOneFileSystemDeleteParity:
|
||||
"""A9 side effect: the per-directory plan is now emitted only for directories
|
||||
whose children were enumerated, so a ``-x`` mount-point directory that is
|
||||
emitted but never traversed is shielded -- its destination content survives,
|
||||
exactly as rsync keeps a non-descended mount point under ``--delete``."""
|
||||
|
||||
@requires_rsync
|
||||
def test_mountpoint_content_survives_delete(self):
|
||||
local = os.stat(".")
|
||||
shm = "/dev/shm"
|
||||
if not os.path.isdir(shm) or os.stat(shm).st_dev == local.st_dev:
|
||||
pytest.skip("no cross-device filesystem available")
|
||||
probe = os.path.join(shm, f"fastsync_dofs_{os.getpid()}")
|
||||
clean_dir(probe)
|
||||
_write(os.path.join(probe, "inside.txt"), b"cross\n")
|
||||
try:
|
||||
source = os.path.join(TEST_DATA_DIR, "dofs_src")
|
||||
clean_dir(source)
|
||||
_write(os.path.join(source, "keep.txt"), b"keep\n")
|
||||
os.symlink(probe, os.path.join(source, "nested_link"))
|
||||
|
||||
def seed_dest(root):
|
||||
clean_dir(root)
|
||||
_write(os.path.join(root, "keep.txt"), b"old\n")
|
||||
_write(os.path.join(root, "nested_link", "stale.txt"), b"stale\n")
|
||||
|
||||
rsync_dst = os.path.join(TEST_DATA_DIR, "dofs_rs_dst")
|
||||
seed_dest(rsync_dst)
|
||||
env = dict(os.environ, LC_ALL="C")
|
||||
rsync_result = subprocess.run(
|
||||
[RSYNC, "-a", "--copy-links", "-x", "--delete-during",
|
||||
source + "/", rsync_dst + "/"],
|
||||
capture_output=True, text=True, env=env, timeout=120)
|
||||
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||
assert os.path.exists(os.path.join(rsync_dst, "nested_link", "stale.txt")), (
|
||||
"rsync unexpectedly descended into the mount point")
|
||||
|
||||
dest = os.path.join(TEST_DATA_DIR, "dofs_fs_dst")
|
||||
clean_dir(dest)
|
||||
received = get_dest_received_dir(dest, source)
|
||||
seed_dest(received)
|
||||
with ServerManager() as server:
|
||||
server.start(extra_args=["--allow-delete"])
|
||||
result, _ = run_client(source, dest,
|
||||
flags=["-a", "--copy-links", "-x", "--delete-during"],
|
||||
port=server.port)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||
assert os.path.exists(os.path.join(received, "nested_link", "stale.txt")), (
|
||||
"FastSync descended into a non-traversed mount point under --delete")
|
||||
finally:
|
||||
clean_dir(probe)
|
||||
|
||||
@@ -2275,6 +2275,36 @@ class TestPartialDir:
|
||||
partial = os.path.join(dest, ".partial", os.path.relpath(source_file, os.path.sep))
|
||||
assert not os.path.exists(partial)
|
||||
|
||||
def test_partial_dir_alone_implies_partial(self, shared_server):
|
||||
"""--partial-dir=DIR with no --partial implies --partial, like rsync.
|
||||
|
||||
rsync 3.4.1 retains the staged partial when --partial-dir is given by
|
||||
itself; before the implication was added FastSync discarded it. The
|
||||
transfer is made to fail deterministically by placing a non-empty
|
||||
directory at the destination path, so the final partial-dir ->
|
||||
destination rename fails and whatever was staged under the partial dir
|
||||
stays on disk."""
|
||||
source = os.path.join(TEST_DATA_DIR, "partial_dir_implied_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "partial_dir_implied_dst")
|
||||
clean_dir(source)
|
||||
clean_dir(dest)
|
||||
source_file = os.path.join(source, "f.bin")
|
||||
with open(source_file, "wb") as f:
|
||||
f.write(b"partial payload")
|
||||
|
||||
received = get_dest_received_dir(dest, source)
|
||||
os.makedirs(os.path.join(received, "f.bin"))
|
||||
with open(os.path.join(received, "f.bin", "keep"), "wb") as f:
|
||||
f.write(b"keep")
|
||||
|
||||
result, _ = run_client(source, dest, flags=["--partial-dir=.partial"],
|
||||
port=shared_server.port)
|
||||
assert result.returncode != 0, "expected the blocked install to fail"
|
||||
|
||||
partial = os.path.join(dest, ".partial", os.path.relpath(source_file, os.path.sep))
|
||||
assert os.path.exists(partial), \
|
||||
"--partial-dir alone must imply --partial and retain the partial file"
|
||||
|
||||
|
||||
class TestLargeFile:
|
||||
def test_transfer_100mb_file(self, shared_server):
|
||||
@@ -2606,35 +2636,30 @@ class TestTimeoutAndAllocLimits:
|
||||
mismatches, missing = verify_transfer(source, received)
|
||||
assert not missing and not mismatches
|
||||
|
||||
def test_temp_dir_cross_filesystem_fallback(self, shared_server):
|
||||
"""A confined relative --temp-dir that resolves (via a symlink under the
|
||||
destination root) to another filesystem must fall back to a non-atomic
|
||||
copy instead of aborting (rsync parity). Skipped when no second
|
||||
filesystem is available."""
|
||||
shm = "/dev/shm"
|
||||
if not os.path.isdir(shm):
|
||||
pytest.skip("/dev/shm not available")
|
||||
if os.stat(shm).st_dev == os.stat(TEST_DATA_DIR).st_dev:
|
||||
pytest.skip("/dev/shm is on the same filesystem as the test data")
|
||||
scratch = os.path.join(shm, f"fastsync_tmp_{os.getpid()}")
|
||||
shutil.rmtree(scratch, ignore_errors=True)
|
||||
os.makedirs(scratch)
|
||||
def test_temp_dir_symlink_escape_rejected(self, shared_server):
|
||||
"""A symlink planted inside the destination root pointing outside it
|
||||
must not redirect receiver scratch files: --temp-dir=<that link> is
|
||||
refused and nothing is written at the link target. An in-root symlink
|
||||
(e.g. to a mount point that stays inside the authorized root) is still
|
||||
accepted, preserving the engine's EXDEV cross-filesystem fallback."""
|
||||
source, dest = self._seed("tempdir_escape_src")
|
||||
outside = "/tmp/fastsync_tempdir_escape_%d" % os.getpid()
|
||||
shutil.rmtree(outside, ignore_errors=True)
|
||||
os.makedirs(outside)
|
||||
link = os.path.join(dest, "escape_scratch")
|
||||
if os.path.lexists(link):
|
||||
os.unlink(link)
|
||||
os.symlink(outside, link)
|
||||
try:
|
||||
source, dest = self._seed("tempdir_xdev_src")
|
||||
# The receiver resolves a relative temp dir under the destination
|
||||
# root; a symlink there points the scratch at the second filesystem.
|
||||
link = os.path.join(dest, "xdev_scratch")
|
||||
os.symlink(scratch, link)
|
||||
result, _ = run_client(source, dest, flags=["--temp-dir", "xdev_scratch"],
|
||||
result, _ = run_client(source, dest, flags=["--temp-dir", "escape_scratch"],
|
||||
port=shared_server.port)
|
||||
assert result.returncode == 0, f"cross-fs temp-dir failed: {result.stderr[:300]}"
|
||||
assert result.returncode != 0, "an escaping --temp-dir symlink must be refused"
|
||||
received = get_dest_received_dir(dest, source)
|
||||
mismatches, missing = verify_transfer(source, received)
|
||||
assert not missing, f"Missing: {missing}"
|
||||
assert not mismatches, f"Mismatch: {mismatches}"
|
||||
assert os.listdir(scratch) == [], "temp files left behind in the cross-fs scratch"
|
||||
assert not os.path.exists(os.path.join(received, "f.txt")), \
|
||||
"the receiver must not fall back to writing the file"
|
||||
assert os.listdir(outside) == [], "receiver wrote outside the authorized root"
|
||||
finally:
|
||||
shutil.rmtree(scratch, ignore_errors=True)
|
||||
shutil.rmtree(outside, ignore_errors=True)
|
||||
|
||||
|
||||
class TestRemoteOptionTransport:
|
||||
@@ -5782,6 +5807,35 @@ class TestStandaloneSuperDefault:
|
||||
"standalone server accepted --copy-as without --allow-super"
|
||||
)
|
||||
|
||||
@pytest.mark.skipif(
|
||||
os.geteuid() != 0,
|
||||
reason="root triggers the SUPER_MODE_OFF default and can create setuid sources",
|
||||
)
|
||||
def test_special_bits_masked_without_allow_super(self):
|
||||
"""A root standalone server without --allow-super forces SUPER_MODE_OFF,
|
||||
so client-supplied setuid/setgid/sticky bits must be stripped even under
|
||||
-p (they are super-user activities just like device-node creation)."""
|
||||
source = os.path.join(TEST_DATA_DIR, "super_default_mode_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "super_default_mode_dst")
|
||||
clean_dir(source)
|
||||
clean_dir(dest)
|
||||
src_file = os.path.join(source, "priv.sh")
|
||||
with open(src_file, "wb") as f:
|
||||
f.write(b"#!/bin/sh\necho hi\n")
|
||||
os.chmod(src_file, 0o4755)
|
||||
server = ServerManager()
|
||||
server.start() # deliberately no --allow-super -> SUPER_MODE_OFF as root
|
||||
try:
|
||||
result, _ = run_client(source, dest, flags=["-p"], port=server.port)
|
||||
finally:
|
||||
server.stop()
|
||||
assert result.returncode == 0, f"exit {result.returncode}: {(result.stderr or '')[:200]}"
|
||||
received = get_dest_received_dir(dest, source)
|
||||
mode = stat.S_IMODE(os.stat(os.path.join(received, "priv.sh")).st_mode)
|
||||
assert (mode & (stat.S_ISUID | stat.S_ISGID | stat.S_ISVTX)) == 0, \
|
||||
f"--no-super receiver kept a privileged bit: {oct(mode)}"
|
||||
assert (mode & 0o777) == 0o755, f"ordinary permission bits lost: {oct(mode)}"
|
||||
|
||||
@pytest.mark.skipif(os.geteuid() != 0, reason="root can create the source device node")
|
||||
def test_devices_skipped_without_allow_super(self):
|
||||
"""Root standalone server without --allow-super must skip device-node
|
||||
|
||||
@@ -662,6 +662,46 @@ class TestWireStatsParity:
|
||||
assert re.match(r"Number of created files: 1 \(reg: 1\)$", r_created), r_created
|
||||
assert f_created == r_created, (r_created, f_created)
|
||||
|
||||
@requires_rsync
|
||||
@pytest.mark.ci
|
||||
@pytest.mark.parametrize("mt", [False, True])
|
||||
def test_stats_r_directory_breakdown_matches_rsync(self, shared_server, mt):
|
||||
"""A recursive `-r` scan (no -t/-p) exposes no directory metadata, but
|
||||
rsync still counts every directory in `Number of files`; the sender's
|
||||
lightweight directory counter must reproduce the `dir: N` category."""
|
||||
source = os.path.join(TEST_DATA_DIR, "wire_stdir_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "wire_stdir_dst")
|
||||
rdst = os.path.join(TEST_DATA_DIR, "wire_stdir_rdst")
|
||||
clean_dir(source)
|
||||
clean_dir(dest)
|
||||
clean_dir(rdst)
|
||||
os.makedirs(os.path.join(source, "sub", "deep"))
|
||||
os.makedirs(os.path.join(source, "empty"))
|
||||
for rel in ("a.txt", os.path.join("sub", "b.txt"), os.path.join("sub", "deep", "c.txt")):
|
||||
with open(os.path.join(source, rel), "wb") as fh:
|
||||
fh.write(b"x\n")
|
||||
os.makedirs(get_dest_received_dir(dest, source), exist_ok=True)
|
||||
|
||||
rsync_result = _rsync(["-r", "--stats", source + "/", rdst + "/"])
|
||||
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||
flags = ["-r", "--stats"] + (["--threads"] if mt else [])
|
||||
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
|
||||
assert result.returncode == 0, result.stderr[:300]
|
||||
|
||||
def stats_line(text, key):
|
||||
for line in text.splitlines():
|
||||
if line.startswith(key + ":"):
|
||||
return line
|
||||
return None
|
||||
|
||||
r_files = stats_line(rsync_result.stdout, "Number of files")
|
||||
f_files = stats_line(result.stdout, "Number of files")
|
||||
# 3 regular files, 4 directories (root, sub, sub/deep, empty).
|
||||
assert re.match(r"Number of files: 7 \(reg: 3, dir: 4\)$", r_files), r_files
|
||||
assert f_files == r_files, (r_files, f_files)
|
||||
assert (stats_line(result.stdout, "Number of regular files transferred") ==
|
||||
stats_line(rsync_result.stdout, "Number of regular files transferred"))
|
||||
|
||||
@requires_rsync
|
||||
@pytest.mark.ci
|
||||
@pytest.mark.parametrize("mt", [False, True])
|
||||
|
||||
@@ -0,0 +1,198 @@
|
||||
"""Differential rsync-parity coverage for two residuals closed on this branch.
|
||||
|
||||
* A4 -- ``--compare-dest``/``--copy-dest``/``--link-dest`` relative-DIR
|
||||
resolution: rsync resolves a relative DIR against the destination directory
|
||||
and appends the file's TRANSFER-RELATIVE name. FastSync's default transfer
|
||||
mirrors the absolute source path below its receive root, so a naive relative
|
||||
DIR used to probe a different tree. These tests seed the basis at rsync's
|
||||
spelling and assert FastSync finds it (byte-exact / hard-linked / sparse),
|
||||
matching real rsync 3.4.1.
|
||||
|
||||
* A5 -- ``-y``/``--fuzzy`` candidate eligibility: rsync's ``find_fuzzy`` has no
|
||||
delta-size gate, so it reuses an oversized (>10x) or sub-16-KiB sibling;
|
||||
FastSync used to decline both. These tests assert FastSync now uses the same
|
||||
sibling as rsync (observable as ``Matched data``) with a byte-exact result.
|
||||
|
||||
Every test skips cleanly when rsync is absent.
|
||||
"""
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
from common import ( # noqa: E402
|
||||
TEST_DATA_DIR,
|
||||
clean_dir,
|
||||
get_dest_received_dir,
|
||||
run_client,
|
||||
)
|
||||
|
||||
RSYNC = shutil.which("rsync")
|
||||
requires_rsync = pytest.mark.skipif(RSYNC is None, reason="rsync 3.4.1 not installed")
|
||||
|
||||
OLD_MTIME = 1_500_000_000
|
||||
|
||||
|
||||
def _write(path, content, mtime=None):
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
with open(path, "wb") as fh:
|
||||
fh.write(content)
|
||||
if mtime is not None:
|
||||
os.utime(path, (mtime, mtime))
|
||||
|
||||
|
||||
def _read(path):
|
||||
with open(path, "rb") as fh:
|
||||
return fh.read()
|
||||
|
||||
|
||||
def _rsync(args):
|
||||
env = dict(os.environ, LC_ALL="C")
|
||||
return subprocess.run([RSYNC] + args, capture_output=True, text=True, env=env, timeout=120)
|
||||
|
||||
|
||||
def _stat_bytes(text, label):
|
||||
for line in text.splitlines():
|
||||
if line.startswith(label + ":"):
|
||||
return int(line.split(":", 1)[1].strip().split()[0].replace(",", ""))
|
||||
return None
|
||||
|
||||
|
||||
class TestRelativeBasisDirResolution:
|
||||
"""A4: a relative basis DIR must resolve to the same tree as rsync's."""
|
||||
|
||||
_FILES = {
|
||||
"root.txt": b"root-basis-content\n",
|
||||
"sub/nested.txt": b"nested-basis-content\n",
|
||||
}
|
||||
|
||||
def _seed_source(self, source):
|
||||
clean_dir(source)
|
||||
for rel, data in self._FILES.items():
|
||||
_write(os.path.join(source, rel), data, OLD_MTIME)
|
||||
return self._FILES
|
||||
|
||||
@requires_rsync
|
||||
@pytest.mark.parametrize("flag", ["--compare-dest", "--link-dest"])
|
||||
def test_relative_dir_resolves_like_rsync(self, shared_server, flag):
|
||||
tag = flag.lstrip("-")
|
||||
source = os.path.join(TEST_DATA_DIR, f"relbasis_{tag}_src")
|
||||
rdst = os.path.join(TEST_DATA_DIR, f"relbasis_{tag}_rdst")
|
||||
fdst = os.path.join(TEST_DATA_DIR, f"relbasis_{tag}_fdst")
|
||||
self._seed_source(source)
|
||||
|
||||
# rsync: relative DIR -> dest/basis/<transfer-relative name>.
|
||||
clean_dir(rdst)
|
||||
for rel, data in self._FILES.items():
|
||||
_write(os.path.join(rdst, "basis", rel), data, OLD_MTIME)
|
||||
rs = _rsync(["-a", f"{flag}=basis", source + "/", rdst + "/"])
|
||||
assert rs.returncode == 0, rs.stderr
|
||||
|
||||
# FastSync: the SAME relative spelling seeded at the SAME
|
||||
# transfer-relative location under its destination root.
|
||||
clean_dir(fdst)
|
||||
for rel, data in self._FILES.items():
|
||||
_write(os.path.join(fdst, "basis", rel), data, OLD_MTIME)
|
||||
result, _ = run_client(source, fdst,
|
||||
flags=["-a", f"{flag}=basis", "--incremental"],
|
||||
port=shared_server.port)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||
received = get_dest_received_dir(fdst, source)
|
||||
|
||||
for rel, data in self._FILES.items():
|
||||
rfile = os.path.join(rdst, rel)
|
||||
ffile = os.path.join(received, rel)
|
||||
basis = os.path.join(fdst, "basis", rel)
|
||||
if flag == "--compare-dest":
|
||||
# compare-dest never copies: both destinations stay sparse.
|
||||
assert not os.path.exists(rfile), f"rsync copied {rel}"
|
||||
assert not os.path.exists(ffile), (
|
||||
f"FastSync did not resolve the relative basis DIR at {basis!r} "
|
||||
f"(expected {rel!r} to stay sparse like rsync)")
|
||||
else:
|
||||
# link-dest hard-links; a basis miss would transfer a new file.
|
||||
assert os.path.exists(ffile), f"FastSync lost {rel}"
|
||||
assert _read(ffile) == data
|
||||
assert os.stat(ffile).st_ino == os.stat(basis).st_ino, (
|
||||
f"FastSync did not hard-link {rel!r} to the relative basis at "
|
||||
f"{basis!r} (basis not resolved like rsync)")
|
||||
|
||||
@requires_rsync
|
||||
def test_relative_dir_copy_dest_content(self, shared_server):
|
||||
source = os.path.join(TEST_DATA_DIR, "relbasis_copy_src")
|
||||
fdst = os.path.join(TEST_DATA_DIR, "relbasis_copy_fdst")
|
||||
self._seed_source(source)
|
||||
clean_dir(fdst)
|
||||
for rel, data in self._FILES.items():
|
||||
_write(os.path.join(fdst, "basis", rel), data, OLD_MTIME)
|
||||
result, _ = run_client(source, fdst,
|
||||
flags=["-a", "--copy-dest=basis", "--incremental"],
|
||||
port=shared_server.port)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||
received = get_dest_received_dir(fdst, source)
|
||||
for rel, data in self._FILES.items():
|
||||
ffile = os.path.join(received, rel)
|
||||
assert os.path.exists(ffile), f"copy-dest did not materialize {rel}"
|
||||
assert _read(ffile) == data
|
||||
assert os.stat(ffile).st_ino != os.stat(os.path.join(fdst, "basis", rel)).st_ino
|
||||
|
||||
|
||||
class TestFuzzyEligibilityWindow:
|
||||
"""A5: --fuzzy candidate eligibility must match rsync's uncapped window."""
|
||||
|
||||
BASE = b"the quick brown fox jumps over the lazy dog\n" * 4000
|
||||
|
||||
def _run_pair(self, shared_server, tag, payload, sibling):
|
||||
source = os.path.join(TEST_DATA_DIR, f"fzw_{tag}_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, f"fzw_{tag}_dst")
|
||||
rdst = os.path.join(TEST_DATA_DIR, f"fzw_{tag}_rdst")
|
||||
clean_dir(source)
|
||||
clean_dir(dest)
|
||||
clean_dir(rdst)
|
||||
_write(os.path.join(source, "report_v2.txt"), payload)
|
||||
for root in (rdst, get_dest_received_dir(dest, source)):
|
||||
_write(os.path.join(root, "report_v1.txt"), sibling)
|
||||
|
||||
rs = _rsync(["-a", "--no-whole-file", "--fuzzy", "--stats",
|
||||
source + "/", rdst + "/"])
|
||||
assert rs.returncode == 0, rs.stderr
|
||||
result, _ = run_client(
|
||||
source, dest,
|
||||
flags=["-a", "--incremental", "--delta", "--fuzzy", "--stats"],
|
||||
port=shared_server.port)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||
|
||||
# The reconstructed file is byte-exact in every case.
|
||||
assert _read(os.path.join(get_dest_received_dir(dest, source),
|
||||
"report_v2.txt")) == payload
|
||||
return rs, result
|
||||
|
||||
@requires_rsync
|
||||
def test_oversized_sibling_eligible_like_rsync(self, shared_server):
|
||||
"""A sibling 20x the source is used by rsync; FastSync must too (its old
|
||||
10x delta-size gate declined it)."""
|
||||
n = 65536
|
||||
payload = (self.BASE * ((n // len(self.BASE)) + 1))[:n]
|
||||
sibling = (self.BASE * 200)[: n * 20]
|
||||
rs, result = self._run_pair(shared_server, "big", payload, sibling)
|
||||
assert _stat_bytes(rs.stdout, "Matched data") > 0, \
|
||||
"rsync should use a >10x fuzzy basis"
|
||||
assert _stat_bytes(result.stdout, "Matched data") > 0, (
|
||||
"FastSync's fuzzy eligibility must accept a >10x sibling like rsync "
|
||||
f"(Matched data={_stat_bytes(result.stdout, 'Matched data')})")
|
||||
|
||||
@requires_rsync
|
||||
def test_small_source_sibling_eligible_like_rsync(self, shared_server):
|
||||
"""A sub-16-KiB source with an identical sibling is used by rsync;
|
||||
FastSync's old 16 KiB delta minimum declined it."""
|
||||
n = 8192
|
||||
payload = (self.BASE * ((n // len(self.BASE)) + 1))[:n]
|
||||
rs, result = self._run_pair(shared_server, "small", payload, payload)
|
||||
assert _stat_bytes(rs.stdout, "Matched data") > 0, \
|
||||
"rsync applies --fuzzy below 16 KiB"
|
||||
assert _stat_bytes(result.stdout, "Matched data") > 0, (
|
||||
"FastSync's fuzzy eligibility must accept a sub-16-KiB source like "
|
||||
f"rsync (Matched data={_stat_bytes(result.stdout, 'Matched data')})")
|
||||
@@ -0,0 +1,105 @@
|
||||
"""`--debug=FLAGS` natural-event categories (no-wire).
|
||||
|
||||
FastSync maps the rsync `--debug` categories that correspond to a real event it
|
||||
already performs (``flist``, ``del``, ``hash``/``deltasum``, ``recv``,
|
||||
``filter`` and ``send``) onto debug output. A normal run prints none of it.
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
from common import TEST_DATA_DIR, run_client, clean_dir, get_dest_received_dir, ServerManager
|
||||
|
||||
|
||||
def _make_tree(root):
|
||||
clean_dir(root)
|
||||
os.makedirs(os.path.join(root, "sub"))
|
||||
with open(os.path.join(root, "a.txt"), "wb") as fh:
|
||||
fh.write(b"alpha\n")
|
||||
with open(os.path.join(root, "keep.log"), "wb") as fh:
|
||||
fh.write(b"log\n")
|
||||
with open(os.path.join(root, "sub", "b.txt"), "wb") as fh:
|
||||
fh.write(b"beta\n")
|
||||
|
||||
|
||||
@pytest.mark.ci
|
||||
def test_debug_flist_and_send_emit_output(shared_server):
|
||||
"""`--debug=flist,send` produces category-tagged debug output."""
|
||||
source = os.path.join(TEST_DATA_DIR, "dbg_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "dbg_dst")
|
||||
_make_tree(source)
|
||||
clean_dir(dest)
|
||||
result, _ = run_client(source, dest, flags=["-a", "--debug=flist,send"],
|
||||
port=shared_server.port)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||
assert "flist: scanning" in result.stdout, result.stdout
|
||||
assert "send: " in result.stdout, result.stdout
|
||||
|
||||
|
||||
@pytest.mark.ci
|
||||
def test_debug_filter_emits_excluded_entry(shared_server):
|
||||
source = os.path.join(TEST_DATA_DIR, "dbg_filter_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "dbg_filter_dst")
|
||||
_make_tree(source)
|
||||
clean_dir(dest)
|
||||
result, _ = run_client(source, dest,
|
||||
flags=["-a", "--debug=filter", "--exclude=*.log"],
|
||||
port=shared_server.port)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||
assert "filter: excluded keep.log" in result.stdout, result.stdout
|
||||
|
||||
|
||||
@pytest.mark.ci
|
||||
def test_debug_hash_and_recv_emit_on_incremental(shared_server):
|
||||
source = os.path.join(TEST_DATA_DIR, "dbg_hash_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "dbg_hash_dst")
|
||||
_make_tree(source)
|
||||
clean_dir(dest)
|
||||
result, _ = run_client(source, dest,
|
||||
flags=["-a", "--incremental", "--checksum",
|
||||
"--debug=hash,recv"],
|
||||
port=shared_server.port)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||
assert "hash: " in result.stdout, result.stdout
|
||||
assert "recv: " in result.stdout, result.stdout
|
||||
|
||||
|
||||
@pytest.mark.ci
|
||||
def test_debug_del_emits_deleted_path():
|
||||
"""`--debug=del` reports the paths the receiver actually removed.
|
||||
|
||||
A deletion-capable server is required (the shared fixture refuses
|
||||
client-requested deletion)."""
|
||||
source = os.path.join(TEST_DATA_DIR, "dbg_del_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "dbg_del_dst")
|
||||
_make_tree(source)
|
||||
clean_dir(dest)
|
||||
seeded = get_dest_received_dir(dest, source)
|
||||
os.makedirs(seeded)
|
||||
with open(os.path.join(seeded, "extra.tmp"), "wb") as fh:
|
||||
fh.write(b"stale\n")
|
||||
server = ServerManager()
|
||||
server.start(extra_args=["--allow-super", "--allow-delete"])
|
||||
try:
|
||||
result, _ = run_client(source, dest, flags=["-a", "--delete", "--debug=del"],
|
||||
port=server.port)
|
||||
finally:
|
||||
server.stop()
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||
assert "del: " in result.stdout and "extra.tmp" in result.stdout, result.stdout
|
||||
assert not os.path.exists(os.path.join(seeded, "extra.tmp"))
|
||||
|
||||
|
||||
@pytest.mark.ci
|
||||
def test_normal_run_has_no_debug_output(shared_server):
|
||||
source = os.path.join(TEST_DATA_DIR, "dbg_quiet_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "dbg_quiet_dst")
|
||||
_make_tree(source)
|
||||
clean_dir(dest)
|
||||
result, _ = run_client(source, dest, flags=["-a"], port=shared_server.port)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||
assert "[DEBUG]" not in result.stdout
|
||||
assert "flist: scanning" not in result.stdout
|
||||
assert "send: " not in result.stdout
|
||||
@@ -0,0 +1,174 @@
|
||||
"""Differential parity for `--info=mount` and `--info=stats` (no-wire).
|
||||
|
||||
Both behaviours are compared against real rsync 3.4.1:
|
||||
|
||||
* `--info=mount` prints rsync's ``[sender] skipping mount-point dir NAME`` line
|
||||
when ``-xx`` drops a mount-point directory. Plain ``-x`` keeps the empty
|
||||
directory and stays silent, exactly like rsync.
|
||||
* `--info=stats` requests the same transfer-statistics block as `--stats`
|
||||
(rsync spells the full block ``--info=stats2``/``--stats``).
|
||||
|
||||
The tests are skipped when rsync is unavailable.
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
from common import TEST_DATA_DIR, run_client, clean_dir, get_dest_received_dir
|
||||
|
||||
RSYNC = shutil.which("rsync")
|
||||
requires_rsync = pytest.mark.skipif(RSYNC is None, reason="rsync 3.4.1 not installed")
|
||||
|
||||
|
||||
def _rsync(args):
|
||||
env = dict(os.environ, LC_ALL="C")
|
||||
return subprocess.run([RSYNC] + args, capture_output=True, text=True, env=env, timeout=120)
|
||||
|
||||
|
||||
def _cross_device_mount_tree(source):
|
||||
"""Build a source whose ``nested_link`` is a symlink onto a tmpfs directory.
|
||||
|
||||
``--copy-links`` dereferences it so ``-x`` sees a mount-point directory on a
|
||||
different device. Returns the probe path to remove, or skips the test when
|
||||
no cross-device filesystem is available.
|
||||
"""
|
||||
local = os.stat(".")
|
||||
shm = "/dev/shm"
|
||||
try:
|
||||
shm_stat = os.stat(shm)
|
||||
except OSError:
|
||||
pytest.skip("/dev/shm not available")
|
||||
if shm_stat.st_dev == local.st_dev:
|
||||
pytest.skip("no cross-device filesystem available")
|
||||
|
||||
clean_dir(source)
|
||||
with open(os.path.join(source, "keep.txt"), "wb") as fh:
|
||||
fh.write(b"keep\n")
|
||||
probe = os.path.join(shm, f"fastsync_info_mount_{os.getpid()}")
|
||||
shutil.rmtree(probe, ignore_errors=True)
|
||||
os.makedirs(probe)
|
||||
with open(os.path.join(probe, "inside.txt"), "wb") as fh:
|
||||
fh.write(b"cross\n")
|
||||
try:
|
||||
os.symlink(probe, os.path.join(source, "nested_link"))
|
||||
except OSError:
|
||||
shutil.rmtree(probe, ignore_errors=True)
|
||||
pytest.skip("cannot create symlink")
|
||||
return probe
|
||||
|
||||
|
||||
@requires_rsync
|
||||
@pytest.mark.ci
|
||||
def test_info_mount_xx_matches_rsync(shared_server):
|
||||
"""`-xx --info=mount` drops the mount-point dir and prints rsync's line."""
|
||||
source = os.path.join(TEST_DATA_DIR, "info_mount_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "info_mount_dst")
|
||||
rdst = os.path.join(TEST_DATA_DIR, "info_mount_rdst")
|
||||
probe = _cross_device_mount_tree(source)
|
||||
clean_dir(dest)
|
||||
clean_dir(rdst)
|
||||
flags = ["-a", "--copy-links", "-xx", "--info=mount"]
|
||||
try:
|
||||
rsync_result = _rsync(flags + [source + "/", rdst + "/"])
|
||||
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||
|
||||
expected = "[sender] skipping mount-point dir nested_link"
|
||||
assert expected in rsync_result.stdout, rsync_result.stdout
|
||||
assert expected in result.stdout, (result.stdout, result.stderr)
|
||||
|
||||
received = get_dest_received_dir(dest, source)
|
||||
assert os.path.exists(os.path.join(received, "keep.txt"))
|
||||
# -xx omits the mount-point directory entirely.
|
||||
assert not os.path.exists(os.path.join(received, "nested_link"))
|
||||
assert not os.path.exists(os.path.join(rdst, "nested_link"))
|
||||
finally:
|
||||
shutil.rmtree(probe, ignore_errors=True)
|
||||
|
||||
|
||||
@requires_rsync
|
||||
@pytest.mark.ci
|
||||
def test_info_mount_single_x_is_silent(shared_server):
|
||||
"""Plain `-x` keeps the empty mount-point directory and prints no line."""
|
||||
source = os.path.join(TEST_DATA_DIR, "info_mount1_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "info_mount1_dst")
|
||||
rdst = os.path.join(TEST_DATA_DIR, "info_mount1_rdst")
|
||||
probe = _cross_device_mount_tree(source)
|
||||
clean_dir(dest)
|
||||
clean_dir(rdst)
|
||||
flags = ["-a", "--copy-links", "-x", "--info=mount"]
|
||||
try:
|
||||
rsync_result = _rsync(flags + [source + "/", rdst + "/"])
|
||||
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||
|
||||
assert "skipping mount-point dir" not in rsync_result.stdout
|
||||
assert "skipping mount-point dir" not in result.stdout
|
||||
|
||||
received = get_dest_received_dir(dest, source)
|
||||
assert os.path.isdir(os.path.join(received, "nested_link"))
|
||||
assert not os.path.exists(os.path.join(received, "nested_link", "inside.txt"))
|
||||
assert os.path.isdir(os.path.join(rdst, "nested_link"))
|
||||
assert not os.path.exists(os.path.join(rdst, "nested_link", "inside.txt"))
|
||||
finally:
|
||||
shutil.rmtree(probe, ignore_errors=True)
|
||||
|
||||
|
||||
def _make_stats_tree(root):
|
||||
clean_dir(root)
|
||||
os.makedirs(os.path.join(root, "sub"))
|
||||
with open(os.path.join(root, "a.txt"), "wb") as fh:
|
||||
fh.write(b"alpha\n")
|
||||
with open(os.path.join(root, "sub", "b.txt"), "wb") as fh:
|
||||
fh.write(b"beta\n")
|
||||
|
||||
|
||||
def _pick_stats(text):
|
||||
keys = ("Number of files", "Number of regular files transferred", "Total file size",
|
||||
"Total transferred file size", "Literal data", "Matched data")
|
||||
out = {}
|
||||
for line in text.splitlines():
|
||||
for key in keys:
|
||||
if line.startswith(key + ":"):
|
||||
out[key] = line
|
||||
return out
|
||||
|
||||
|
||||
@requires_rsync
|
||||
@pytest.mark.ci
|
||||
def test_info_stats_emits_full_stats_block(shared_server):
|
||||
"""`--info=stats` is the same full block as `--stats` and matches rsync."""
|
||||
source = os.path.join(TEST_DATA_DIR, "info_stats_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "info_stats_dst")
|
||||
rdst = os.path.join(TEST_DATA_DIR, "info_stats_rdst")
|
||||
dest2 = os.path.join(TEST_DATA_DIR, "info_stats_dst2")
|
||||
_make_stats_tree(source)
|
||||
for path in (dest, rdst, dest2):
|
||||
clean_dir(path)
|
||||
os.makedirs(get_dest_received_dir(path, source), exist_ok=True)
|
||||
|
||||
rsync_result = _rsync(["-a", "--stats", source + "/", rdst + "/"])
|
||||
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||
|
||||
info_result, _ = run_client(source, dest, flags=["-a", "--info=stats"],
|
||||
port=shared_server.port)
|
||||
assert info_result.returncode == 0, (info_result.stderr or info_result.stdout)[:300]
|
||||
stats_result, _ = run_client(source, dest2, flags=["-a", "--stats"],
|
||||
port=shared_server.port)
|
||||
assert stats_result.returncode == 0, (stats_result.stderr or stats_result.stdout)[:300]
|
||||
|
||||
# --info=stats must print the same block as --stats...
|
||||
assert _pick_stats(info_result.stdout) == _pick_stats(stats_result.stdout), (
|
||||
f"info={info_result.stdout} stats={stats_result.stdout}")
|
||||
# ...and the protocol-independent counters must match real rsync.
|
||||
assert _pick_stats(info_result.stdout) == _pick_stats(rsync_result.stdout), (
|
||||
f"rsync={_pick_stats(rsync_result.stdout)} fastsync={_pick_stats(info_result.stdout)}")
|
||||
assert re.search(r"^Number of files: \d+ \(reg: 2, dir: 2\)$", info_result.stdout,
|
||||
re.MULTILINE), info_result.stdout
|
||||
@@ -0,0 +1,185 @@
|
||||
"""Differential rsync-parity coverage for FastSync's transfer/delete ORDER.
|
||||
|
||||
rsync walks a source tree in its sorted flist order: within each directory the
|
||||
non-directories come first (ascending name), then the subdirectories (ascending
|
||||
name), each subdirectory immediately followed by its own subtree (depth-first).
|
||||
The sequential scanner now reproduces that order, which makes both the
|
||||
``--info=name`` stream and the ``--delete-during`` deletion sequence match real
|
||||
``rsync 3.4.1`` exactly. ``--threads`` has no rsync analogue and is unordered.
|
||||
|
||||
Every test skips cleanly when rsync is absent.
|
||||
"""
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
from common import ( # noqa: E402
|
||||
TEST_DATA_DIR,
|
||||
ServerManager,
|
||||
clean_dir,
|
||||
get_dest_received_dir,
|
||||
run_client,
|
||||
)
|
||||
|
||||
RSYNC = shutil.which("rsync")
|
||||
requires_rsync = pytest.mark.skipif(RSYNC is None, reason="rsync 3.4.1 not installed")
|
||||
|
||||
MTIME = 1_500_000_000
|
||||
|
||||
_TREE = {
|
||||
"a.txt": b"a\n",
|
||||
"b.txt": b"b\n",
|
||||
"z.txt": b"z\n",
|
||||
"a_dir/f.txt": b"f\n",
|
||||
"a_dir/deep/g.txt": b"g\n",
|
||||
"m_dir/h.txt": b"h\n",
|
||||
"Z_dir/i.txt": b"i\n",
|
||||
}
|
||||
|
||||
|
||||
def _write(path, data):
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
with open(path, "wb") as fh:
|
||||
fh.write(data)
|
||||
os.utime(path, (MTIME, MTIME))
|
||||
|
||||
|
||||
def _rsync(args):
|
||||
env = dict(os.environ, LC_ALL="C")
|
||||
return subprocess.run([RSYNC] + args, capture_output=True, text=True, env=env, timeout=120)
|
||||
|
||||
|
||||
def _deleting(text):
|
||||
out = []
|
||||
for line in text.splitlines():
|
||||
stripped = line.strip()
|
||||
if stripped.startswith("*deleting") or stripped.startswith("deleting"):
|
||||
out.append(stripped.split()[-1])
|
||||
return out
|
||||
|
||||
|
||||
class TestTransferOrderParity:
|
||||
@requires_rsync
|
||||
def test_info_name_file_order_matches_rsync(self, shared_server):
|
||||
source = os.path.join(TEST_DATA_DIR, "order_name_src")
|
||||
clean_dir(source)
|
||||
for rel, data in _TREE.items():
|
||||
_write(os.path.join(source, rel), data)
|
||||
|
||||
rdst = os.path.join(TEST_DATA_DIR, "order_name_rdst")
|
||||
clean_dir(rdst)
|
||||
rs = _rsync(["-a", "--info=name", source + "/", rdst + "/"])
|
||||
assert rs.returncode == 0, rs.stderr
|
||||
# rsync also names the directories (trailing '/'); FastSync names the
|
||||
# transferred entries. Compare the file/symlink sequence, which is what
|
||||
# the traversal order determines.
|
||||
rsync_files = [l for l in rs.stdout.splitlines() if l.strip() and not l.endswith("/")]
|
||||
|
||||
fdst = os.path.join(TEST_DATA_DIR, "order_name_fdst")
|
||||
clean_dir(fdst)
|
||||
result, _ = run_client(source, fdst, flags=["-a", "--info=name"],
|
||||
port=shared_server.port)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||
fsync_files = [
|
||||
l for l in result.stdout.splitlines()
|
||||
if l.strip() and l.strip() != "./" and not l.startswith("sending")
|
||||
]
|
||||
assert fsync_files == rsync_files, (
|
||||
f"transfer order differs\nrsync={rsync_files}\nfastsync={fsync_files}")
|
||||
|
||||
|
||||
class TestDeleteOrderParity:
|
||||
_EXTRA = {
|
||||
"a_extra.txt": b"a\n",
|
||||
"z_extra.txt": b"z\n",
|
||||
"a_extra_dir/f": b"f\n",
|
||||
"z_extra_dir/f": b"f\n",
|
||||
"a_extra_dir/sub/g": b"g\n",
|
||||
}
|
||||
|
||||
def _seed_source(self):
|
||||
source = os.path.join(TEST_DATA_DIR, "order_del_src")
|
||||
clean_dir(source)
|
||||
_write(os.path.join(source, "keep.txt"), b"k\n")
|
||||
_write(os.path.join(source, "keepdir", "x.txt"), b"x\n")
|
||||
_write(os.path.join(source, "keep2", "y.txt"), b"y\n")
|
||||
return source
|
||||
|
||||
def _assert_order(self, timing, dry_run=False):
|
||||
source = self._seed_source()
|
||||
rdst = os.path.join(TEST_DATA_DIR, f"order_{timing}_rdst")
|
||||
clean_dir(rdst)
|
||||
for rel, data in self._EXTRA.items():
|
||||
_write(os.path.join(rdst, rel), data)
|
||||
rs_flags = ["-a", "-n"] if dry_run else ["-a"]
|
||||
rs = _rsync(rs_flags + [timing, "--info=del", source + "/", rdst + "/"])
|
||||
assert rs.returncode == 0, rs.stderr
|
||||
|
||||
fdst = os.path.join(TEST_DATA_DIR, f"order_{timing}_fdst")
|
||||
clean_dir(fdst)
|
||||
received = get_dest_received_dir(fdst, source)
|
||||
for rel, data in self._EXTRA.items():
|
||||
_write(os.path.join(received, rel), data)
|
||||
fs_flags = ["-a", "-n"] if dry_run else ["-a"]
|
||||
with ServerManager() as server:
|
||||
server.start(extra_args=["--allow-delete"])
|
||||
result, _ = run_client(source, fdst, flags=fs_flags + [timing, "--info=del"],
|
||||
port=server.port)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||
|
||||
rsync_order = _deleting(rs.stdout)
|
||||
fsync_order = _deleting(result.stdout)
|
||||
assert sorted(fsync_order) == sorted(rsync_order), (
|
||||
f"{timing} deleted set differs\nrsync={rsync_order}\nfastsync={fsync_order}")
|
||||
assert fsync_order == rsync_order, (
|
||||
f"{timing} deletion order differs\nrsync={rsync_order}\nfastsync={fsync_order}")
|
||||
|
||||
@requires_rsync
|
||||
def test_delete_during_deletion_order_matches_rsync(self):
|
||||
self._assert_order("--delete-during")
|
||||
|
||||
@requires_rsync
|
||||
def test_delete_delay_deletion_order_matches_rsync(self):
|
||||
self._assert_order("--delete-delay")
|
||||
|
||||
@requires_rsync
|
||||
def test_dry_run_delete_order_matches_rsync(self):
|
||||
self._assert_order("--delete", dry_run=True)
|
||||
|
||||
@requires_rsync
|
||||
def test_partial_max_delete_survivor_order_matches_rsync(self):
|
||||
"""With the exact removal order matching rsync, a --max-delete cap stops
|
||||
after the same entries, so the survivor set is identical too."""
|
||||
source = os.path.join(TEST_DATA_DIR, "order_maxdel_src")
|
||||
clean_dir(source)
|
||||
_write(os.path.join(source, "keep.txt"), b"k\n")
|
||||
extra = {f"e{i}.txt": b"x\n" for i in range(6)}
|
||||
extra["ed/f"] = b"f\n"
|
||||
extra["ed/g"] = b"g\n"
|
||||
|
||||
rdst = os.path.join(TEST_DATA_DIR, "order_maxdel_rdst")
|
||||
clean_dir(rdst)
|
||||
for rel, data in extra.items():
|
||||
_write(os.path.join(rdst, rel), data)
|
||||
rs = _rsync(["-a", "--delete-during", "--max-delete=3", "--info=del",
|
||||
source + "/", rdst + "/"])
|
||||
assert rs.returncode in (0, 25), (rs.returncode, rs.stderr)
|
||||
|
||||
fdst = os.path.join(TEST_DATA_DIR, "order_maxdel_fdst")
|
||||
clean_dir(fdst)
|
||||
received = get_dest_received_dir(fdst, source)
|
||||
for rel, data in extra.items():
|
||||
_write(os.path.join(received, rel), data)
|
||||
with ServerManager() as server:
|
||||
server.start(extra_args=["--allow-delete"])
|
||||
result, _ = run_client(source, fdst,
|
||||
flags=["-a", "--delete-during", "--max-delete=3", "--info=del"],
|
||||
port=server.port)
|
||||
assert result.returncode in (0, 25), (result.returncode, result.stderr[:300])
|
||||
assert _deleting(result.stdout) == _deleting(rs.stdout), (
|
||||
f"partial --max-delete survivor order differs\n"
|
||||
f"rsync={_deleting(rs.stdout)}\nfastsync={_deleting(result.stdout)}")
|
||||
@@ -861,11 +861,11 @@ class TestFuzzy:
|
||||
byte-exact result. FastSync ports rsync 3.4.1's weighted-Levenshtein name
|
||||
heuristic, so where both delta engines admit the candidate the tools pick
|
||||
the same basis (the ``fuzzy_basis`` differential asserts the tree and the
|
||||
Matched/Literal counters match with the block size pinned). The residual is
|
||||
candidate ELIGIBILITY: FastSync's delta size gate (both files >= 16 KiB and
|
||||
a <= 10x size ratio) is narrower than rsync's, which empirically uses a
|
||||
fuzzy basis well beyond 10x and below 16 KiB. These tests pin the window
|
||||
boundary and prove the byte-exact fallback on both sides of it."""
|
||||
Matched/Literal counters match with the block size pinned). Candidate
|
||||
ELIGIBILITY is now rsync's too: the fuzzy search no longer inherits the
|
||||
ordinary delta engine's 16 KiB minimum or 10x size-ratio bound, so an
|
||||
oversized or sub-16-KiB sibling is reused exactly as rsync reuses it.
|
||||
These tests pin that window on both sides."""
|
||||
|
||||
_BASE = b"the quick brown fox jumps over the lazy dog\n" * 4000
|
||||
|
||||
@@ -900,9 +900,10 @@ class TestFuzzy:
|
||||
return rs, result
|
||||
|
||||
@requires_rsync
|
||||
def test_fuzzy_above_size_window_declines_but_tree_exact(self, shared_server):
|
||||
"""A sibling >10x the source is used by rsync but declined by FastSync's
|
||||
delta size-ratio gate; both destinations stay byte-identical."""
|
||||
def test_fuzzy_above_size_window_matches_rsync(self, shared_server):
|
||||
"""A sibling >10x the source is used by rsync and by FastSync: fuzzy
|
||||
eligibility is rsync's, not the ordinary delta size-ratio gate; both
|
||||
destinations stay byte-identical and both reuse the basis."""
|
||||
n = 65536
|
||||
payload = (self._BASE * ((n // len(self._BASE)) + 1))[:n]
|
||||
sibling = (self._BASE * 200)[: n * 20]
|
||||
@@ -911,15 +912,16 @@ class TestFuzzy:
|
||||
rs, result = self._run_both(shared_server, source, dest, rdst,
|
||||
payload, sibling)
|
||||
assert _stat_bytes(rs.stdout, "Matched data") > 0, \
|
||||
"rsync should still use a >10x fuzzy basis"
|
||||
assert _stat_bytes(result.stdout, "Matched data") == 0, \
|
||||
"FastSync's 10x delta size-ratio gate must decline the oversized basis"
|
||||
assert _stat_bytes(result.stdout, "Literal data") == n
|
||||
"rsync should use a >10x fuzzy basis"
|
||||
assert _stat_bytes(result.stdout, "Matched data") > 0, \
|
||||
"FastSync must accept a >10x fuzzy basis like rsync"
|
||||
assert _stat_bytes(result.stdout, "Literal data") < n
|
||||
|
||||
@requires_rsync
|
||||
def test_fuzzy_below_delta_minimum_declines_but_tree_exact(self, shared_server):
|
||||
"""A sibling below the 16 KiB delta minimum is used by rsync but never
|
||||
enters FastSync's delta/fuzzy path; both trees stay byte-identical."""
|
||||
def test_fuzzy_below_delta_minimum_matches_rsync(self, shared_server):
|
||||
"""A sibling below the 16 KiB delta minimum is used by rsync and by
|
||||
FastSync: fuzzy eligibility no longer inherits the delta engine's
|
||||
minimum; both trees stay byte-identical and both reuse the basis."""
|
||||
n = 8192
|
||||
payload = (self._BASE * ((n // len(self._BASE)) + 1))[:n]
|
||||
source, dest, rdst = (self._src("small"), self._dst("small"),
|
||||
@@ -928,9 +930,9 @@ class TestFuzzy:
|
||||
payload, payload)
|
||||
assert _stat_bytes(rs.stdout, "Matched data") > 0, \
|
||||
"rsync applies --fuzzy below 16 KiB"
|
||||
assert _stat_bytes(result.stdout, "Matched data") == 0, \
|
||||
"FastSync's 16 KiB delta minimum must bypass the fuzzy basis"
|
||||
assert _stat_bytes(result.stdout, "Literal data") == n
|
||||
assert _stat_bytes(result.stdout, "Matched data") > 0, \
|
||||
"FastSync must apply --fuzzy below 16 KiB like rsync"
|
||||
assert _stat_bytes(result.stdout, "Literal data") < n
|
||||
|
||||
|
||||
class TestIgnoreExistingShortCircuit:
|
||||
|
||||
@@ -1,23 +1,57 @@
|
||||
"""CLI validation and preflight checks."""
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import os
|
||||
import shutil
|
||||
import time
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
from common import BUILD_DIR, CLIENT_CMD, SERVER_CMD, TEST_DATA_DIR, run_client, verify_transfer
|
||||
from common import (
|
||||
BUILD_DIR,
|
||||
CLIENT_CMD,
|
||||
CLIENT_TIMEOUT,
|
||||
SERVER_CMD,
|
||||
TEST_DATA_DIR,
|
||||
get_dest_received_dir,
|
||||
run_client,
|
||||
verify_transfer,
|
||||
)
|
||||
|
||||
DEFAULT_PORT = 8080
|
||||
|
||||
|
||||
def _port_is_listening(host, port, timeout=0.3):
|
||||
"""True if something accepts a TCP connection on host:port right now."""
|
||||
try:
|
||||
with socket.create_connection((host, port), timeout=timeout):
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def _wait_for_listener(host, port, timeout=5.0):
|
||||
"""Poll host:port until a listener accepts, or the deadline passes."""
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if _port_is_listening(host, port):
|
||||
return True
|
||||
time.sleep(0.05)
|
||||
return False
|
||||
|
||||
|
||||
class TestHelp:
|
||||
def test_client_help(self):
|
||||
r = subprocess.run(CLIENT_CMD + ["--help"], capture_output=True, text=True)
|
||||
r = subprocess.run(CLIENT_CMD + ["--help"], capture_output=True,
|
||||
text=True, timeout=CLIENT_TIMEOUT)
|
||||
assert r.returncode == 0
|
||||
assert "Usage:" in r.stdout
|
||||
assert "SSH transport" in r.stdout
|
||||
|
||||
def test_server_help(self):
|
||||
r = subprocess.run(SERVER_CMD + ["--help"], capture_output=True, text=True)
|
||||
r = subprocess.run(SERVER_CMD + ["--help"], capture_output=True,
|
||||
text=True, timeout=CLIENT_TIMEOUT)
|
||||
assert r.returncode == 0
|
||||
assert "Usage:" in r.stdout
|
||||
|
||||
@@ -67,19 +101,24 @@ class TestServerPort:
|
||||
|
||||
def test_default_port(self):
|
||||
"""Server should start on default port 8080."""
|
||||
if _port_is_listening("127.0.0.1", DEFAULT_PORT):
|
||||
pytest.skip(f"port {DEFAULT_PORT} already in use by another process")
|
||||
|
||||
proc = subprocess.Popen(
|
||||
SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None,
|
||||
)
|
||||
try:
|
||||
import socket, time
|
||||
time.sleep(0.5)
|
||||
with socket.create_connection(("127.0.0.1", 8080), timeout=2):
|
||||
pass # Port is listening
|
||||
except (ConnectionRefusedError, OSError):
|
||||
pytest.fail("Server not listening on default port 8080")
|
||||
if not _wait_for_listener("127.0.0.1", DEFAULT_PORT, timeout=5.0):
|
||||
if proc.poll() is not None and _port_is_listening("127.0.0.1", DEFAULT_PORT):
|
||||
pytest.skip(f"port {DEFAULT_PORT} was taken by another process")
|
||||
pytest.fail(f"Server not listening on default port {DEFAULT_PORT}")
|
||||
finally:
|
||||
proc.terminate()
|
||||
proc.wait(timeout=5)
|
||||
try:
|
||||
proc.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.kill()
|
||||
proc.wait()
|
||||
|
||||
|
||||
def _seed_protocol_source(source):
|
||||
@@ -105,7 +144,7 @@ class TestProtocol:
|
||||
port=shared_server.port)
|
||||
assert result.returncode == 0, \
|
||||
f"--protocol current run failed: {(result.stderr or result.stdout)[:400]}"
|
||||
received = os.path.join(dest, os.path.abspath(source).lstrip(os.sep))
|
||||
received = get_dest_received_dir(dest, source)
|
||||
mismatches, missing = verify_transfer(source, received)
|
||||
assert not mismatches and not missing, \
|
||||
f"transfer mismatch: missing={missing} mismatches={mismatches}"
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
#include "test_file.h"
|
||||
#include "test_file_list.h"
|
||||
#include "test_file_sendfile.h"
|
||||
#include "test_filter.h"
|
||||
#include "test_format.h"
|
||||
#include "test_fuzz_smoke.h"
|
||||
#include "test_glob.h"
|
||||
@@ -80,6 +81,7 @@ int main() {
|
||||
RUN_TEST(test_receiver_timeout);
|
||||
RUN_TEST(test_metadata);
|
||||
RUN_TEST(test_glob);
|
||||
RUN_TEST(test_filter);
|
||||
RUN_TEST(test_iconv);
|
||||
RUN_TEST(test_file);
|
||||
RUN_TEST(test_file_list);
|
||||
|
||||
+167
-28
@@ -171,6 +171,38 @@ static void test_validate_config_unified_invariants() {
|
||||
config_delete(cfg);
|
||||
}
|
||||
|
||||
/* The receiver enforces MAX_FILTER_RULES on the protect-rule block and would
|
||||
otherwise fail the session with an opaque protocol error. The client must
|
||||
accept exactly the limit and reject one more up front, before any network
|
||||
I/O, with an actionable message. */
|
||||
static void test_validate_config_filter_rule_limit() {
|
||||
Config* cfg = valid_client_config();
|
||||
cfg->filters = array_list_create(free);
|
||||
EXPECT_NOT_NULL(cfg->filters);
|
||||
for (int i = 0; i < MAX_FILTER_RULES; i++)
|
||||
EXPECT_TRUE(array_list_add(cfg->filters, str_dup("- *.tmp")));
|
||||
EXPECT_TRUE(validate_config(cfg)); /* exactly the limit is accepted */
|
||||
|
||||
FILE* log_capture = tmpfile();
|
||||
EXPECT_NOT_NULL(log_capture);
|
||||
log_set_file(log_capture);
|
||||
EXPECT_TRUE(array_list_add(cfg->filters, str_dup("- *.bak")));
|
||||
EXPECT_FALSE(validate_config(cfg)); /* one over the limit is rejected */
|
||||
fflush(log_capture);
|
||||
rewind(log_capture);
|
||||
char line[512];
|
||||
bool saw_message = false;
|
||||
while (fgets(line, sizeof(line), log_capture) != NULL) {
|
||||
if (strstr(line, "too many filter rules") != NULL && strstr(line, "(maximum 1024)") != NULL)
|
||||
saw_message = true;
|
||||
}
|
||||
log_set_file(NULL);
|
||||
fclose(log_capture);
|
||||
EXPECT_TRUE(saw_message);
|
||||
|
||||
config_delete(cfg);
|
||||
}
|
||||
|
||||
/* Test main() with --help flag (early return path, no server connection needed) */
|
||||
static void test_cli_help() {
|
||||
/* We can't easily call main() because it calls send_files which needs a server.
|
||||
@@ -511,6 +543,66 @@ static void test_parse_args_ignore_existing() {
|
||||
config_delete(cfg);
|
||||
}
|
||||
|
||||
/* --partial-dir=DIR implies --partial, matching rsync 3.4.1. rsync resolves
|
||||
* this after option parsing, so the implication wins over an explicit
|
||||
* --no-partial in either order. It is skipped under --inplace, where partial
|
||||
* staging is bypassed and the destination is written in place. */
|
||||
static void test_parse_args_partial_dir_implies_partial() {
|
||||
{
|
||||
Config* cfg = config_create();
|
||||
char* argv[] = {"fastsync", "--partial-dir=.partial", "/src", "/dst"};
|
||||
int positional_args[2];
|
||||
int positional_count = 0;
|
||||
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
|
||||
EXPECT_TRUE(cfg->partial);
|
||||
config_delete(cfg);
|
||||
}
|
||||
{
|
||||
/* Explicit --no-partial before --partial-dir: --partial-dir still wins. */
|
||||
Config* cfg = config_create();
|
||||
char* argv[] = {"fastsync", "--no-partial", "--partial-dir=.partial", "/src", "/dst"};
|
||||
int positional_args[2];
|
||||
int positional_count = 0;
|
||||
EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0);
|
||||
EXPECT_TRUE(cfg->partial);
|
||||
config_delete(cfg);
|
||||
}
|
||||
{
|
||||
/* Reversed order must not change the precedence. */
|
||||
Config* cfg = config_create();
|
||||
char* argv[] = {"fastsync", "--partial-dir=.partial", "--no-partial", "/src", "/dst"};
|
||||
int positional_args[2];
|
||||
int positional_count = 0;
|
||||
EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0);
|
||||
EXPECT_TRUE(cfg->partial);
|
||||
config_delete(cfg);
|
||||
}
|
||||
{
|
||||
/* --inplace bypasses partial staging, so parse_args must not set the
|
||||
implied --partial; the combination itself is invalid (rsync parity:
|
||||
"--inplace cannot be used with --partial-dir"), so validation rejects. */
|
||||
Config* cfg = config_create();
|
||||
char* argv[] = {"fastsync", "--inplace", "--partial-dir=.partial", "/src", "/dst"};
|
||||
int positional_args[2];
|
||||
int positional_count = 0;
|
||||
EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0);
|
||||
EXPECT_FALSE(cfg->partial);
|
||||
cfg->send_directory = str_dup("/src");
|
||||
cfg->receive_root_directory = str_dup("/dst");
|
||||
EXPECT_FALSE(validate_config(cfg));
|
||||
config_delete(cfg);
|
||||
}
|
||||
{
|
||||
Config* cfg = config_create();
|
||||
char* argv[] = {"fastsync", "--no-partial", "/src", "/dst"};
|
||||
int positional_args[2];
|
||||
int positional_count = 0;
|
||||
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
|
||||
EXPECT_FALSE(cfg->partial);
|
||||
config_delete(cfg);
|
||||
}
|
||||
}
|
||||
|
||||
static void test_parse_args_executability() {
|
||||
Config* cfg = config_create();
|
||||
char* argv[] = {"fastsync", "-E", "/src", "/dst"};
|
||||
@@ -762,8 +854,9 @@ static void test_parse_args_debug_flags() {
|
||||
int positional_count = 0;
|
||||
|
||||
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
|
||||
EXPECT_EQ_INT(cfg->debug_level, LOG_DEBUG_ALL);
|
||||
EXPECT_EQ_INT(get_log_debug_flags(), LOG_DEBUG_ALL);
|
||||
EXPECT_EQ_INT(cfg->debug_level, LOG_DEBUG_IO | LOG_DEBUG_PROTO | LOG_DEBUG_PACK | LOG_DEBUG_UTIL);
|
||||
EXPECT_EQ_INT(get_log_debug_flags(),
|
||||
LOG_DEBUG_IO | LOG_DEBUG_PROTO | LOG_DEBUG_PACK | LOG_DEBUG_UTIL);
|
||||
config_delete(cfg);
|
||||
}
|
||||
|
||||
@@ -1241,35 +1334,78 @@ static void test_parse_args_delete_timing_without_delete_rejected() {
|
||||
config_delete(cfg);
|
||||
}
|
||||
|
||||
/* Parsed-but-unimplemented options must fail instead of being silently accepted. */
|
||||
static void test_parse_args_rejects_unimplemented_options() {
|
||||
static const char* const options[] = {"--silent",
|
||||
"--queue-size",
|
||||
"-A",
|
||||
"--acls",
|
||||
"-X",
|
||||
"--xattrs",
|
||||
"-D",
|
||||
"--devices",
|
||||
"--delete-excluded",
|
||||
"--max-delete",
|
||||
"--prune-empty-dirs",
|
||||
"--bind-address",
|
||||
"--daemon",
|
||||
"--config",
|
||||
"--server"};
|
||||
/* Truly-unknown options (including server-only spellings) must be rejected
|
||||
* through the unknown-option path instead of being silently accepted. */
|
||||
static void test_parse_args_rejects_unknown_options() {
|
||||
static const char* const options[] = {"--silent", "--queue-size", "--bind-address",
|
||||
"--daemon", "--config", "--server"};
|
||||
|
||||
for (size_t i = 0; i < sizeof(options) / sizeof(options[0]); i++) {
|
||||
Config* cfg = config_create();
|
||||
char* argv[] = {"fastsync", (char*)options[i], "dummy", "/src", "/dst"};
|
||||
char* argv[] = {"fastsync", (char*)options[i], "/src", "/dst"};
|
||||
int positional_args[2];
|
||||
int positional_count = 0;
|
||||
|
||||
EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), -1);
|
||||
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), -1);
|
||||
config_delete(cfg);
|
||||
}
|
||||
}
|
||||
|
||||
/* Options that are genuinely implemented must parse successfully and record
|
||||
* their effect, rather than being lumped in with the unknown-option set. */
|
||||
static void test_parse_args_accepts_implemented_metadata_options() {
|
||||
Config* cfg = config_create();
|
||||
char* argv_x[] = {"fastsync", "-X", "/src", "/dst"};
|
||||
int positional_args[2];
|
||||
int positional_count = 0;
|
||||
EXPECT_EQ_INT(parse_args(cfg, 4, argv_x, positional_args, &positional_count), 0);
|
||||
EXPECT_TRUE(cfg->preserve_xattrs);
|
||||
config_delete(cfg);
|
||||
|
||||
cfg = config_create();
|
||||
positional_count = 0;
|
||||
char* argv_acls[] = {"fastsync", "--acls", "/src", "/dst"};
|
||||
EXPECT_EQ_INT(parse_args(cfg, 4, argv_acls, positional_args, &positional_count), 0);
|
||||
EXPECT_TRUE(cfg->preserve_acls);
|
||||
config_delete(cfg);
|
||||
|
||||
cfg = config_create();
|
||||
positional_count = 0;
|
||||
char* argv_d[] = {"fastsync", "-D", "/src", "/dst"};
|
||||
EXPECT_EQ_INT(parse_args(cfg, 4, argv_d, positional_args, &positional_count), 0);
|
||||
EXPECT_TRUE(cfg->preserve_devices);
|
||||
EXPECT_TRUE(cfg->preserve_specials);
|
||||
config_delete(cfg);
|
||||
|
||||
cfg = config_create();
|
||||
positional_count = 0;
|
||||
char* argv_devices[] = {"fastsync", "--devices", "/src", "/dst"};
|
||||
EXPECT_EQ_INT(parse_args(cfg, 4, argv_devices, positional_args, &positional_count), 0);
|
||||
EXPECT_TRUE(cfg->preserve_devices);
|
||||
config_delete(cfg);
|
||||
|
||||
cfg = config_create();
|
||||
positional_count = 0;
|
||||
char* argv_delete_excluded[] = {"fastsync", "--delete-excluded", "/src", "/dst"};
|
||||
EXPECT_EQ_INT(parse_args(cfg, 4, argv_delete_excluded, positional_args, &positional_count), 0);
|
||||
EXPECT_TRUE(cfg->delete_excluded);
|
||||
config_delete(cfg);
|
||||
|
||||
cfg = config_create();
|
||||
positional_count = 0;
|
||||
char* argv_max_delete[] = {"fastsync", "--max-delete=5", "/src", "/dst"};
|
||||
EXPECT_EQ_INT(parse_args(cfg, 4, argv_max_delete, positional_args, &positional_count), 0);
|
||||
EXPECT_EQ_INT(cfg->max_delete, 5);
|
||||
config_delete(cfg);
|
||||
|
||||
cfg = config_create();
|
||||
positional_count = 0;
|
||||
char* argv_prune[] = {"fastsync", "--prune-empty-dirs", "/src", "/dst"};
|
||||
EXPECT_EQ_INT(parse_args(cfg, 4, argv_prune, positional_args, &positional_count), 0);
|
||||
EXPECT_TRUE(cfg->prune_empty_dirs);
|
||||
config_delete(cfg);
|
||||
}
|
||||
|
||||
/* Test both rsync-compatible quiet spellings and option ordering. */
|
||||
static void test_parse_args_quiet() {
|
||||
static const char* const options[][2] = {
|
||||
@@ -1421,10 +1557,9 @@ static void test_parse_args_info_name_and_help() {
|
||||
config_delete(cfg);
|
||||
}
|
||||
|
||||
/* rsync 3.4.1's full --info/--debug vocabulary parses. The info categories
|
||||
* with a FastSync event set their flag; the remaining rsync-only categories
|
||||
* (backup/mount/symsafe/syms) parse but stay silent. Every --debug category
|
||||
* listed here is FastSync-silent, so debug_level stays 0. */
|
||||
/* rsync 3.4.1's full --info/--debug vocabulary parses. The categories with a
|
||||
* FastSync event set their flag; the remaining rsync-only categories
|
||||
* (backup/symsafe/syms, acl/bind/chdir/...) parse but stay silent. */
|
||||
static void test_parse_args_rsync_flag_vocabulary_accepted() {
|
||||
Config* cfg = config_create();
|
||||
char* argv[] = {"fastsync", "--info=backup,del,flist,mount,nonreg,progress,remove,symsafe,syms",
|
||||
@@ -1436,9 +1571,10 @@ static void test_parse_args_rsync_flag_vocabulary_accepted() {
|
||||
int positional_count = 0;
|
||||
|
||||
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
|
||||
EXPECT_EQ_INT(cfg->info_level, LOG_INFO_DEL | LOG_INFO_FLIST | LOG_INFO_NONREG |
|
||||
EXPECT_EQ_INT(cfg->info_level, LOG_INFO_DEL | LOG_INFO_FLIST | LOG_INFO_MOUNT | LOG_INFO_NONREG |
|
||||
LOG_INFO_PROGRESS | LOG_INFO_REMOVE);
|
||||
EXPECT_EQ_INT(cfg->debug_level, 0);
|
||||
EXPECT_EQ_INT(cfg->debug_level, LOG_DEBUG_DEL | LOG_DEBUG_FLIST | LOG_DEBUG_HASH |
|
||||
LOG_DEBUG_RECV | LOG_DEBUG_FILTER | LOG_DEBUG_SEND);
|
||||
config_delete(cfg);
|
||||
}
|
||||
|
||||
@@ -4768,6 +4904,7 @@ void test_client_cli() {
|
||||
test_validate_config_credentials_require_tls_or_loopback();
|
||||
test_validate_config_delta_sendfile_constraints();
|
||||
test_validate_config_unified_invariants();
|
||||
test_validate_config_filter_rule_limit();
|
||||
test_cli_help();
|
||||
test_cli_archive_flags();
|
||||
test_cli_dry_run();
|
||||
@@ -4783,6 +4920,7 @@ void test_client_cli() {
|
||||
test_parse_args_valid_port();
|
||||
test_parse_args_size_only();
|
||||
test_parse_args_ignore_existing();
|
||||
test_parse_args_partial_dir_implies_partial();
|
||||
test_parse_args_executability();
|
||||
test_parse_args_chmod();
|
||||
test_parse_args_numeric_chmod();
|
||||
@@ -4818,7 +4956,8 @@ void test_client_cli() {
|
||||
test_parse_args_delete_default_timing_and_commit();
|
||||
test_parse_args_delete_timing_conflict_rejected();
|
||||
test_parse_args_delete_timing_without_delete_rejected();
|
||||
test_parse_args_rejects_unimplemented_options();
|
||||
test_parse_args_rejects_unknown_options();
|
||||
test_parse_args_accepts_implemented_metadata_options();
|
||||
test_parse_args_quiet();
|
||||
test_parse_args_human_readable();
|
||||
test_parse_args_hard_links();
|
||||
|
||||
@@ -3,7 +3,9 @@
|
||||
#include "compression.h"
|
||||
#include "data.h"
|
||||
#include "file.h"
|
||||
#include "protocol.h"
|
||||
#include "utils.h"
|
||||
#include <stdint.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/stat.h>
|
||||
@@ -216,6 +218,67 @@ static void test_data_decompress_unknown_size_frame() {
|
||||
data_destroy(frame);
|
||||
}
|
||||
|
||||
/* The receiver advertises MAX_RECEIVE_WHOLE_FILE_SIZE (256 MiB) and the sender
|
||||
* compresses whole files, so the decompressor's internal ceiling must match that
|
||||
* protocol bound. A 130 MiB payload -- above the old 100 MiB ceiling but below
|
||||
* the protocol bound -- must round-trip through
|
||||
* data_decompress_limited(..., MAX_RECEIVE_WHOLE_FILE_SIZE). The payload is all
|
||||
* zeros so it compresses to a tiny frame while still declaring its full size. */
|
||||
static void test_data_decompress_limited_whole_file_ceiling() {
|
||||
const size_t size = 130ULL * 1024 * 1024;
|
||||
Data* input = data_create_empty(size);
|
||||
EXPECT_NOT_NULL(input);
|
||||
memset(input->data, 0, size);
|
||||
input->size = size;
|
||||
|
||||
/* Threaded zstd stores the content size in the frame header, as the sender
|
||||
* does for whole files, so the decompressor sees the exact declared size. */
|
||||
Data* compressed = data_compress_codec(input, COMPRESSION_ALGO_ZSTD, 1, 2);
|
||||
EXPECT_NOT_NULL(compressed);
|
||||
/* Premise: the declared size sits between the old 100 MiB cap and the
|
||||
* protocol whole-file bound -- exactly the range that used to be rejected. */
|
||||
unsigned long long declared =
|
||||
ZSTD_getFrameContentSize((uint8_t*)compressed->data + 1, compressed->size - 1);
|
||||
EXPECT_TRUE(declared > (100ULL * 1024 * 1024));
|
||||
EXPECT_TRUE(declared <= MAX_RECEIVE_WHOLE_FILE_SIZE);
|
||||
|
||||
Data* out = data_decompress_limited(compressed, MAX_RECEIVE_WHOLE_FILE_SIZE);
|
||||
EXPECT_NOT_NULL(out);
|
||||
EXPECT_EQ_INT((int)out->size, (int)size);
|
||||
EXPECT_EQ_INT(memcmp(out->data, input->data, size), 0);
|
||||
|
||||
data_destroy(out);
|
||||
data_destroy(compressed);
|
||||
data_destroy(input);
|
||||
}
|
||||
|
||||
/* A frame declaring an uncompressed size above the hard ceiling is still
|
||||
* rejected before any allocation, even when the caller passes a limit higher
|
||||
* than the protocol bound. The 13-byte header is a valid zstd frame header with
|
||||
* an 8-byte content size and no blocks; rejection happens at the size check. */
|
||||
static void test_data_decompress_limited_rejects_over_ceiling() {
|
||||
const uint64_t declared = MAX_RECEIVE_WHOLE_FILE_SIZE + 1;
|
||||
Data* frame = data_create_empty(1 + 13);
|
||||
EXPECT_NOT_NULL(frame);
|
||||
uint8_t* p = (uint8_t*)frame->data;
|
||||
p[0] = (uint8_t)COMPRESSION_ALGO_ZSTD;
|
||||
p[1] = 0x28; /* zstd magic number, little-endian */
|
||||
p[2] = 0xB5;
|
||||
p[3] = 0x2F;
|
||||
p[4] = 0xFD;
|
||||
p[5] = 0xE0; /* Frame_Header_Descriptor: 8-byte content size, single segment */
|
||||
for (int i = 0; i < 8; i++)
|
||||
p[6 + i] = (uint8_t)((declared >> (8 * i)) & 0xff);
|
||||
frame->size = 1 + 13;
|
||||
/* Guard the premise: zstd reads back exactly the declared over-ceiling size. */
|
||||
EXPECT_EQ_INT((int)ZSTD_getFrameContentSize(p + 1, frame->size - 1), (int)declared);
|
||||
|
||||
EXPECT_NULL(data_decompress_limited(frame, MAX_RECEIVE_WHOLE_FILE_SIZE * 2));
|
||||
EXPECT_NULL(data_decompress_limited(frame, MAX_RECEIVE_WHOLE_FILE_SIZE));
|
||||
|
||||
data_destroy(frame);
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
int id;
|
||||
int iterations;
|
||||
@@ -467,6 +530,8 @@ void test_compression() {
|
||||
test_data_compress_decompress_roundtrip();
|
||||
test_data_compress_decompress_large();
|
||||
test_data_decompress_unknown_size_frame();
|
||||
test_data_decompress_limited_whole_file_ceiling();
|
||||
test_data_decompress_limited_rejects_over_ceiling();
|
||||
test_data_decompress_truncated_frame_fails();
|
||||
test_skip_compress_suffix_matching();
|
||||
test_data_compress_with_threads_roundtrip();
|
||||
|
||||
@@ -3,12 +3,14 @@
|
||||
#include "test_utils.h"
|
||||
#include "utils.h"
|
||||
#include <errno.h>
|
||||
#include <fcntl.h>
|
||||
#include <glob.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/stat.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/wait.h>
|
||||
#include <unistd.h>
|
||||
|
||||
/* Known-answer vector, independently recomputed with Python
|
||||
@@ -719,6 +721,238 @@ static void test_credentials_read_secret_file_bad() {
|
||||
EXPECT_EQ_INT(credentials_read_secret_file(missing, NULL, NULL, err, sizeof(err)), -1);
|
||||
}
|
||||
|
||||
/* A symlink planted at a password-file path is refused (O_NOFOLLOW) instead of
|
||||
* being followed before the owner/mode gate, even when it resolves to a valid
|
||||
* owner-only regular file. */
|
||||
static void test_credentials_read_secret_file_symlink_rejected() {
|
||||
char err[512];
|
||||
char* target = make_tmp_file("alice:correct horse battery staple\n");
|
||||
EXPECT_NOT_NULL(target);
|
||||
|
||||
char link[256];
|
||||
snprintf(link, sizeof(link), "/tmp/fs_cred_pwlink_%d_%d", (int)getpid(), g_file_counter++);
|
||||
unlink(link);
|
||||
EXPECT_EQ_INT(symlink(target, link), 0);
|
||||
|
||||
char* user = (char*)1;
|
||||
char* password = (char*)1;
|
||||
EXPECT_EQ_INT(credentials_read_secret_file(link, &user, &password, err, sizeof(err)), -1);
|
||||
EXPECT_NULL(user);
|
||||
EXPECT_NULL(password);
|
||||
EXPECT_TRUE(err[0] != '\0');
|
||||
|
||||
unlink(link); /* remove the symlink itself, not its target */
|
||||
rm_temp(target);
|
||||
free(target);
|
||||
}
|
||||
|
||||
/* A named FIFO with no writer must not hang in fgets (O_NONBLOCK): the read
|
||||
* fails cleanly with "no user:password line" instead of blocking forever. */
|
||||
static void test_credentials_read_secret_file_fifo_no_hang() {
|
||||
char err[512];
|
||||
char fifo[256];
|
||||
snprintf(fifo, sizeof(fifo), "/tmp/fs_cred_pwfifo_%d_%d", (int)getpid(), g_file_counter++);
|
||||
unlink(fifo);
|
||||
EXPECT_EQ_INT(mkfifo(fifo, 0600), 0);
|
||||
|
||||
char* user = (char*)1;
|
||||
char* password = (char*)1;
|
||||
EXPECT_EQ_INT(credentials_read_secret_file(fifo, &user, &password, err, sizeof(err)), -1);
|
||||
EXPECT_NULL(user);
|
||||
EXPECT_NULL(password);
|
||||
EXPECT_TRUE(strstr(err, "no 'user:password'") != NULL || err[0] != '\0');
|
||||
|
||||
unlink(fifo);
|
||||
}
|
||||
|
||||
/* Write `s` fully to `fd`, retrying EINTR. */
|
||||
static void write_all_fd(int fd, const char* s) {
|
||||
size_t total = strlen(s);
|
||||
size_t off = 0;
|
||||
while (off < total) {
|
||||
ssize_t w = write(fd, s + off, total - off);
|
||||
if (w < 0) {
|
||||
if (errno == EINTR)
|
||||
continue;
|
||||
return;
|
||||
}
|
||||
off += (size_t)w;
|
||||
}
|
||||
}
|
||||
|
||||
/* Deterministically model a slow process substitution (`--password-file
|
||||
* <(sleep N; ...)`): attach a writer to the FIFO (so the reader sees EAGAIN --
|
||||
* the empty/no-writer FIFO instead yields an immediate EOF), have it sleep
|
||||
* `delay_ms`, then write `first` and, after another `delay_ms`, `second` (NULL
|
||||
* for a single write). Splitting across the delay exercises reassembly of a
|
||||
* line delivered by several write()s.
|
||||
*
|
||||
* The parent keeps a spare read end open for the lifetime of the test so the
|
||||
* writer always has a reader; the caller must close(*hold_out), waitpid() the
|
||||
* returned pid and unlink the FIFO. Returns the child pid, or -1 on setup
|
||||
* failure. */
|
||||
static pid_t fifo_writer_sleep_then_write(const char* fifo, const char* first, unsigned delay_ms,
|
||||
const char* second, int* hold_out) {
|
||||
int sync[2];
|
||||
if (pipe(sync) != 0)
|
||||
return -1;
|
||||
pid_t pid = fork();
|
||||
if (pid < 0) {
|
||||
close(sync[0]);
|
||||
close(sync[1]);
|
||||
return -1;
|
||||
}
|
||||
if (pid == 0) {
|
||||
close(sync[0]);
|
||||
int wfd = open(fifo, O_WRONLY | O_CLOEXEC);
|
||||
char ready = wfd >= 0 ? 1 : 0;
|
||||
if (write(sync[1], &ready, 1) != 1)
|
||||
_exit(1);
|
||||
close(sync[1]);
|
||||
if (wfd >= 0) {
|
||||
usleep(delay_ms * 1000);
|
||||
write_all_fd(wfd, first);
|
||||
if (second) {
|
||||
usleep(delay_ms * 1000);
|
||||
write_all_fd(wfd, second);
|
||||
}
|
||||
close(wfd);
|
||||
}
|
||||
_exit(0);
|
||||
}
|
||||
close(sync[1]);
|
||||
int hold = open(fifo, O_RDONLY | O_NONBLOCK | O_CLOEXEC);
|
||||
char ready = 0;
|
||||
ssize_t got = read(sync[0], &ready, 1);
|
||||
close(sync[0]);
|
||||
if (got != 1 || ready != 1) {
|
||||
if (hold >= 0)
|
||||
close(hold);
|
||||
return -1;
|
||||
}
|
||||
*hold_out = hold;
|
||||
return pid;
|
||||
}
|
||||
|
||||
/* A FIFO writer that produces its data after a short delay must be read
|
||||
* successfully (the regression: O_NONBLOCK made fgets fail with EAGAIN before
|
||||
* the writer ran). */
|
||||
static void test_credentials_read_secret_file_fifo_delayed_writer() {
|
||||
char err[512];
|
||||
char fifo[256];
|
||||
snprintf(fifo, sizeof(fifo), "/tmp/fs_cred_pwfifo_slow_%d_%d", (int)getpid(), g_file_counter++);
|
||||
unlink(fifo);
|
||||
EXPECT_EQ_INT(mkfifo(fifo, 0600), 0);
|
||||
|
||||
int hold = -1;
|
||||
pid_t writer =
|
||||
fifo_writer_sleep_then_write(fifo, "alice:correct horse battery staple\n", 250, NULL, &hold);
|
||||
EXPECT_TRUE(writer > 0);
|
||||
|
||||
char* user = NULL;
|
||||
char* password = NULL;
|
||||
EXPECT_EQ_INT(credentials_read_secret_file(fifo, &user, &password, err, sizeof(err)), 0);
|
||||
EXPECT_EQ_STR(user, "alice");
|
||||
EXPECT_EQ_STR(password, "correct horse battery staple");
|
||||
free(user);
|
||||
free(password);
|
||||
|
||||
int status = 0;
|
||||
waitpid(writer, &status, 0);
|
||||
if (hold >= 0)
|
||||
close(hold);
|
||||
unlink(fifo);
|
||||
}
|
||||
|
||||
/* The same, but the line is written in two chunks separated by the delay: the
|
||||
* reader must reassemble one line rather than parse the first chunk as an
|
||||
* empty-password entry. */
|
||||
static void test_credentials_read_secret_file_fifo_split_write() {
|
||||
char err[512];
|
||||
char fifo[256];
|
||||
snprintf(fifo, sizeof(fifo), "/tmp/fs_cred_pwfifo_split_%d_%d", (int)getpid(), g_file_counter++);
|
||||
unlink(fifo);
|
||||
EXPECT_EQ_INT(mkfifo(fifo, 0600), 0);
|
||||
|
||||
int hold = -1;
|
||||
pid_t writer =
|
||||
fifo_writer_sleep_then_write(fifo, "alice:correct horse", 200, " battery staple\n", &hold);
|
||||
EXPECT_TRUE(writer > 0);
|
||||
|
||||
char* user = NULL;
|
||||
char* password = NULL;
|
||||
EXPECT_EQ_INT(credentials_read_secret_file(fifo, &user, &password, err, sizeof(err)), 0);
|
||||
EXPECT_EQ_STR(user, "alice");
|
||||
EXPECT_EQ_STR(password, "correct horse battery staple");
|
||||
free(user);
|
||||
free(password);
|
||||
|
||||
int status = 0;
|
||||
waitpid(writer, &status, 0);
|
||||
if (hold >= 0)
|
||||
close(hold);
|
||||
unlink(fifo);
|
||||
}
|
||||
|
||||
/* The server-side store loader (--password-file / --early-input) must also
|
||||
* accept a FIFO whose writer appears after a delay. */
|
||||
static void test_credentials_store_fifo_delayed_writer() {
|
||||
char err[512];
|
||||
char fifo[256];
|
||||
snprintf(fifo, sizeof(fifo), "/tmp/fs_cred_storefifo_%d_%d", (int)getpid(), g_file_counter++);
|
||||
unlink(fifo);
|
||||
EXPECT_EQ_INT(mkfifo(fifo, 0600), 0);
|
||||
|
||||
int hold = -1;
|
||||
pid_t writer = fifo_writer_sleep_then_write(fifo, KAT_STORE_LINE "\n", 250, NULL, &hold);
|
||||
EXPECT_TRUE(writer > 0);
|
||||
|
||||
CredentialStore* store = credentials_load(fifo, NULL, err, sizeof(err));
|
||||
EXPECT_NOT_NULL(store);
|
||||
EXPECT_EQ_INT(credentials_store_size(store), 1);
|
||||
credentials_free(store);
|
||||
|
||||
int status = 0;
|
||||
waitpid(writer, &status, 0);
|
||||
if (hold >= 0)
|
||||
close(hold);
|
||||
rm_temp(fifo);
|
||||
}
|
||||
|
||||
/* fd-backed store paths (bash process substitution `<(...)`, i.e. /dev/fd/N and
|
||||
* /proc/self/fd/N) are symlinks, so the ordinary O_NOFOLLOW rule would reject
|
||||
* them with ELOOP. They name the calling process's own descriptors, so they
|
||||
* are exempt: opening one that points at an owner-only regular file is
|
||||
* accepted, while a symlink at a NORMAL path is still rejected
|
||||
* (test_credentials_read_secret_file_symlink_rejected). */
|
||||
static void test_credentials_read_secret_file_fd_backed_accepted() {
|
||||
char err[512];
|
||||
char* path = make_tmp_file("alice:correct horse battery staple\n");
|
||||
EXPECT_NOT_NULL(path);
|
||||
|
||||
int fd = open(path, O_RDONLY | O_CLOEXEC);
|
||||
EXPECT_TRUE(fd >= 0);
|
||||
|
||||
const char* prefixes[] = {"/proc/self/fd/", "/dev/fd/"};
|
||||
for (size_t i = 0; i < sizeof(prefixes) / sizeof(prefixes[0]); i++) {
|
||||
if (i == 1 && access("/dev/fd", F_OK) != 0)
|
||||
continue; /* /dev/fd is not present on every system */
|
||||
char fd_path[64];
|
||||
snprintf(fd_path, sizeof(fd_path), "%s%d", prefixes[i], fd);
|
||||
char* user = (char*)1;
|
||||
char* password = (char*)1;
|
||||
EXPECT_EQ_INT(credentials_read_secret_file(fd_path, &user, &password, err, sizeof(err)), 0);
|
||||
EXPECT_EQ_STR(user, "alice");
|
||||
EXPECT_EQ_STR(password, "correct horse battery staple");
|
||||
free(user);
|
||||
free(password);
|
||||
}
|
||||
|
||||
close(fd);
|
||||
rm_temp(path);
|
||||
free(path);
|
||||
}
|
||||
|
||||
static void test_credentials_hash_file() {
|
||||
char* plaintext = make_tmp_file("# comment\n\n alice :" KAT_PASSWORD "\nbob:bob-s3cret\n");
|
||||
EXPECT_NOT_NULL(plaintext);
|
||||
@@ -1103,6 +1337,12 @@ void test_credentials(void) {
|
||||
test_credentials_early_input_merge();
|
||||
test_credentials_read_secret_file();
|
||||
test_credentials_read_secret_file_bad();
|
||||
test_credentials_read_secret_file_symlink_rejected();
|
||||
test_credentials_read_secret_file_fifo_no_hang();
|
||||
test_credentials_read_secret_file_fifo_delayed_writer();
|
||||
test_credentials_read_secret_file_fifo_split_write();
|
||||
test_credentials_store_fifo_delayed_writer();
|
||||
test_credentials_read_secret_file_fd_backed_accepted();
|
||||
test_credentials_hash_file();
|
||||
test_credentials_rejects_group_or_other_accessible();
|
||||
test_credentials_dummy_key_persisted();
|
||||
|
||||
+109
-16
@@ -377,6 +377,98 @@ static void test_file_save_to_disk_temp_dir_confined() {
|
||||
rmdir(outside);
|
||||
}
|
||||
|
||||
/* A client-planted symlink under the receive root must never redirect the
|
||||
* --temp-dir scratch directory outside the authorized root: the REAL path of
|
||||
* the opened dir is checked. An in-root symlink (the EXDEV cross-filesystem
|
||||
* case) must still be accepted. */
|
||||
static void test_file_open_temp_dir_symlink_confinement() {
|
||||
const char* root = "test_tempdir_link_root";
|
||||
const char* outside = "test_tempdir_link_outside";
|
||||
char root_abs[PATH_MAX];
|
||||
char outside_abs[PATH_MAX];
|
||||
unlink("test_tempdir_link_root/escape");
|
||||
unlink("test_tempdir_link_root/inside_link");
|
||||
rmdir("test_tempdir_link_root/scratch");
|
||||
rmdir(root);
|
||||
rmdir(outside);
|
||||
|
||||
int mkdir_root_ret = mkdir(root, 0755);
|
||||
int mkdir_outside_ret = mkdir(outside, 0755);
|
||||
bool root_resolved = realpath(root, root_abs) != NULL;
|
||||
bool outside_resolved = realpath(outside, outside_abs) != NULL;
|
||||
int root_fd = root_resolved ? open(root_abs, O_RDONLY | O_DIRECTORY | O_CLOEXEC) : -1;
|
||||
|
||||
bool root_set = false;
|
||||
bool scratch_ok = false;
|
||||
int scratch_fd = -1;
|
||||
bool escape_staged = false;
|
||||
int escape_fd = 0;
|
||||
bool inside_staged = false;
|
||||
int inside_fd = -1;
|
||||
char* scratch = NULL;
|
||||
char* escape = NULL;
|
||||
char* inside_link = NULL;
|
||||
|
||||
/* Only touch the global authorized root and the scratch fixtures once the
|
||||
setup succeeded; the teardown below always runs regardless. */
|
||||
if (root_fd >= 0 && outside_resolved) {
|
||||
root_set = utils_set_authorized_root(root_fd, root_abs);
|
||||
|
||||
/* An existing in-root scratch dir opens normally. */
|
||||
scratch = path_cat(root_abs, "scratch");
|
||||
if (scratch && mkdir(scratch, 0755) == 0) {
|
||||
scratch_ok = true;
|
||||
scratch_fd = file_open_temp_dir(scratch);
|
||||
if (scratch_fd >= 0)
|
||||
close(scratch_fd);
|
||||
}
|
||||
|
||||
/* A symlink whose target is outside the root is refused. */
|
||||
escape = path_cat(root_abs, "escape");
|
||||
if (escape && symlink(outside_abs, escape) == 0) {
|
||||
escape_staged = true;
|
||||
escape_fd = file_open_temp_dir(escape);
|
||||
}
|
||||
|
||||
/* A symlink that stays inside the root is accepted (EXDEV fallback). */
|
||||
inside_link = path_cat(root_abs, "inside_link");
|
||||
if (inside_link && scratch && symlink(scratch, inside_link) == 0) {
|
||||
inside_staged = true;
|
||||
inside_fd = file_open_temp_dir(inside_link);
|
||||
if (inside_fd >= 0)
|
||||
close(inside_fd);
|
||||
}
|
||||
}
|
||||
|
||||
/* Release the global authorized root and all fixtures BEFORE asserting:
|
||||
EXPECT_* returns early on failure, so a failed assertion must not be able
|
||||
to leave the process state poisoned or leak root_fd. */
|
||||
utils_set_authorized_root(-1, NULL);
|
||||
if (root_fd >= 0)
|
||||
close(root_fd);
|
||||
free(inside_link);
|
||||
free(escape);
|
||||
free(scratch);
|
||||
unlink("test_tempdir_link_root/escape");
|
||||
unlink("test_tempdir_link_root/inside_link");
|
||||
rmdir("test_tempdir_link_root/scratch");
|
||||
rmdir(root);
|
||||
rmdir(outside);
|
||||
|
||||
EXPECT_EQ_INT(mkdir_root_ret, 0);
|
||||
EXPECT_EQ_INT(mkdir_outside_ret, 0);
|
||||
EXPECT_TRUE(root_resolved);
|
||||
EXPECT_TRUE(outside_resolved);
|
||||
EXPECT_TRUE(root_fd >= 0);
|
||||
EXPECT_TRUE(root_set);
|
||||
EXPECT_TRUE(scratch_ok);
|
||||
EXPECT_TRUE(scratch_fd >= 0);
|
||||
EXPECT_TRUE(escape_staged);
|
||||
EXPECT_EQ_INT(escape_fd, -1);
|
||||
EXPECT_TRUE(inside_staged);
|
||||
EXPECT_TRUE(inside_fd >= 0);
|
||||
}
|
||||
|
||||
/* Issue #251: file_save_to_disk_full must distinguish receiver-side skips
|
||||
(--existing/--ignore-existing/--update) from real writes so the sender can
|
||||
decide whether --remove-source-files may unlink its source. */
|
||||
@@ -1040,8 +1132,8 @@ static void test_atomic_no_perms_preserves_destination_mode() {
|
||||
|
||||
/* No -p/-E: the pre-existing 0640 survives the atomic overwrite. */
|
||||
bool ok = file_to_disk_secure_attrs(path, "data", 4, false, false, false, &m,
|
||||
(FileAttrPolicy){false, false, false, false}, false, false,
|
||||
false, NULL, false, false, NULL);
|
||||
(FileAttrPolicy){false, false, false, false, true}, false,
|
||||
false, false, NULL, false, false, NULL);
|
||||
EXPECT_TRUE(ok);
|
||||
struct stat st;
|
||||
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||
@@ -1049,8 +1141,8 @@ static void test_atomic_no_perms_preserves_destination_mode() {
|
||||
|
||||
/* -p: the source mode wins. */
|
||||
ok = file_to_disk_secure_attrs(path, "data2", 5, false, false, false, &m,
|
||||
(FileAttrPolicy){true, true, false, false}, false, false, false,
|
||||
NULL, false, false, NULL);
|
||||
(FileAttrPolicy){true, true, false, false, true}, false, false,
|
||||
false, NULL, false, false, NULL);
|
||||
EXPECT_TRUE(ok);
|
||||
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||
EXPECT_EQ_INT((int)(st.st_mode & 0777), 0755);
|
||||
@@ -1060,8 +1152,8 @@ static void test_atomic_no_perms_preserves_destination_mode() {
|
||||
source 0755 gives 0750, not 0751 and not the scratch 0711. */
|
||||
EXPECT_EQ_INT(chmod(path, 0640), 0);
|
||||
ok = file_to_disk_secure_attrs(path, "data3", 6, false, false, false, &m,
|
||||
(FileAttrPolicy){false, false, false, true}, false, false, false,
|
||||
NULL, false, false, NULL);
|
||||
(FileAttrPolicy){false, false, false, true, true}, false, false,
|
||||
false, NULL, false, false, NULL);
|
||||
EXPECT_TRUE(ok);
|
||||
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||
EXPECT_EQ_INT((int)(st.st_mode & 0777), 0750);
|
||||
@@ -1073,8 +1165,8 @@ static void test_atomic_no_perms_preserves_destination_mode() {
|
||||
const char* fresh = "test_attr_split_fresh.txt";
|
||||
unlink(fresh);
|
||||
ok = file_to_disk_secure_attrs(fresh, "data", 4, false, false, false, &m,
|
||||
(FileAttrPolicy){false, false, false, false}, false, false, false,
|
||||
NULL, false, false, NULL);
|
||||
(FileAttrPolicy){false, false, false, false, true}, false, false,
|
||||
false, NULL, false, false, NULL);
|
||||
EXPECT_TRUE(ok);
|
||||
EXPECT_EQ_INT(stat(fresh, &st), 0);
|
||||
EXPECT_EQ_INT((int)(st.st_mode & 0777), (int)(m.mode & 0777 & ~(mode_t)file_process_umask()));
|
||||
@@ -1083,8 +1175,8 @@ static void test_atomic_no_perms_preserves_destination_mode() {
|
||||
/* Without any metadata the historical fixed 0644 default still applies. */
|
||||
unlink(fresh);
|
||||
ok = file_to_disk_secure_attrs(fresh, "data", 4, false, false, false, NULL,
|
||||
(FileAttrPolicy){false, false, false, false}, false, false, false,
|
||||
NULL, false, false, NULL);
|
||||
(FileAttrPolicy){false, false, false, false, true}, false, false,
|
||||
false, NULL, false, false, NULL);
|
||||
EXPECT_TRUE(ok);
|
||||
EXPECT_EQ_INT(stat(fresh, &st), 0);
|
||||
EXPECT_EQ_INT((int)(st.st_mode & 0777), 0644);
|
||||
@@ -1104,8 +1196,8 @@ static void test_new_file_mode_honors_source_and_umask() {
|
||||
m.gid = getegid();
|
||||
|
||||
bool ok = file_to_disk_secure_attrs(path, "x", 1, false, false, false, &m,
|
||||
(FileAttrPolicy){false, false, false, false}, false, false,
|
||||
false, NULL, false, false, NULL);
|
||||
(FileAttrPolicy){false, false, false, false, true}, false,
|
||||
false, false, NULL, false, false, NULL);
|
||||
EXPECT_TRUE(ok);
|
||||
struct stat st;
|
||||
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||
@@ -1663,8 +1755,8 @@ static void test_file_write_to_disk_partial_retention() {
|
||||
m.atime_valid = false;
|
||||
m.crtime_valid = false;
|
||||
bool ok = file_to_disk_secure_attrs(path, content, strlen(content), false, false, true, &m,
|
||||
(FileAttrPolicy){true, true, false, false}, false, false,
|
||||
false, NULL, false, true, NULL);
|
||||
(FileAttrPolicy){true, true, false, false, true}, false,
|
||||
false, false, NULL, false, true, NULL);
|
||||
EXPECT_FALSE(ok); /* the write itself succeeded, but metadata restore failed */
|
||||
/* Retained: the already-written temp now sits at the destination path. */
|
||||
int fd = open(path, O_RDONLY);
|
||||
@@ -1683,8 +1775,8 @@ static void test_file_write_to_disk_partial_retention() {
|
||||
|
||||
/* Same failure with keep_partial=false: temp is unlinked, nothing retained. */
|
||||
ok = file_to_disk_secure_attrs(path, content, strlen(content), false, false, true, &m,
|
||||
(FileAttrPolicy){true, true, false, false}, false, false, false,
|
||||
NULL, false, false, NULL);
|
||||
(FileAttrPolicy){true, true, false, false, true}, false, false,
|
||||
false, NULL, false, false, NULL);
|
||||
EXPECT_FALSE(ok);
|
||||
EXPECT_TRUE(access(path, F_OK) == -1);
|
||||
}
|
||||
@@ -2264,6 +2356,7 @@ void test_file() {
|
||||
test_file_save_to_disk_ignore_existing_entry_types();
|
||||
test_file_save_to_disk_partial_install();
|
||||
test_file_save_to_disk_temp_dir_confined();
|
||||
test_file_open_temp_dir_symlink_confinement();
|
||||
test_file_save_to_disk_reports_skips();
|
||||
test_file_write_to_disk_sparse_preserves_holes();
|
||||
test_file_write_to_disk_partial_retention();
|
||||
|
||||
@@ -0,0 +1,294 @@
|
||||
#include "test_filter.h"
|
||||
#include "filter.h"
|
||||
#include "test_utils.h"
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/stat.h>
|
||||
#include <unistd.h>
|
||||
|
||||
/* Every `-f`/`--filter` rule string is validated through the list parser, so
|
||||
* the list parser must reject the modifiers the standalone parser rejects
|
||||
* rather than silently folding them into a pattern. */
|
||||
|
||||
static void test_filter_list_rejects_xattr_modifier() {
|
||||
static const char* const rules[] = {
|
||||
"-x user.foo", /* short exclude + x */
|
||||
"exclude,x user.foo", /* long exclude + x */
|
||||
"+x user.foo", /* include + x */
|
||||
"hide,x *.tmp", /* hide + x */
|
||||
"dir-merge,x .rules", /* x must not be dropped on dir-merge */
|
||||
"merge,x /tmp/nonexistent" /* x must not be dropped on merge */
|
||||
};
|
||||
for (size_t i = 0; i < sizeof(rules) / sizeof(rules[0]); i++) {
|
||||
FilterRuleList* list = filter_rule_list_create();
|
||||
EXPECT_NOT_NULL(list);
|
||||
char err[256] = "";
|
||||
bool ok = filter_rule_list_parse_append(list, rules[i], NULL, NULL, err, sizeof(err));
|
||||
EXPECT_FALSE(ok);
|
||||
EXPECT_TRUE(strstr(err, "xattr") != NULL);
|
||||
filter_rule_list_free(list);
|
||||
}
|
||||
|
||||
/* A bare "x" is not a rule at all: rejected as generic bad syntax. */
|
||||
FilterRuleList* list = filter_rule_list_create();
|
||||
EXPECT_NOT_NULL(list);
|
||||
char err[256] = "";
|
||||
EXPECT_FALSE(filter_rule_list_parse_append(list, "x user.foo", NULL, NULL, err, sizeof(err)));
|
||||
EXPECT_TRUE(err[0] != '\0');
|
||||
filter_rule_list_free(list);
|
||||
}
|
||||
|
||||
static void test_filter_list_rejects_unsupported_modifiers() {
|
||||
/* The merge-file modifiers e/n/w/- are invalid on every non-merge rule; a
|
||||
token made up solely of modifier characters is a modifier run, so it must
|
||||
be rejected rather than folded into the pattern. */
|
||||
static const char* const rules[] = {
|
||||
"-e foo", /* e: merge-only in rsync */
|
||||
"-n foo", /* n: merge-only in rsync */
|
||||
"-w foo", /* w: merge-only in rsync */
|
||||
"-new", /* pure modifier letters (n/e/w) */
|
||||
"-press", /* pure modifier letters (p/r/e/s) */
|
||||
"exclude,w foo", "exclude,e foo", "exclude,n foo",
|
||||
"hide,w foo", "protect,n foo", "risk,e foo",
|
||||
};
|
||||
for (size_t i = 0; i < sizeof(rules) / sizeof(rules[0]); i++) {
|
||||
FilterRuleList* list = filter_rule_list_create();
|
||||
EXPECT_NOT_NULL(list);
|
||||
char err[256] = "";
|
||||
bool ok = filter_rule_list_parse_append(list, rules[i], NULL, NULL, err, sizeof(err));
|
||||
EXPECT_FALSE(ok);
|
||||
EXPECT_TRUE(strstr(err, "unsupported filter modifier") != NULL);
|
||||
filter_rule_list_free(list);
|
||||
}
|
||||
}
|
||||
|
||||
/* rsync accepts the merge-file modifiers e/n/w/- on merge and dir-merge rules.
|
||||
* They must be consumed so they never leak into the merge filename. */
|
||||
static void test_filter_list_accepts_merge_modifiers() {
|
||||
char tmpl[] = "/tmp/fastsync_filter_mmod_XXXXXX";
|
||||
EXPECT_TRUE(mkdtemp(tmpl) != NULL);
|
||||
char path[512];
|
||||
snprintf(path, sizeof(path), "%s/rules", tmpl);
|
||||
FILE* fp = fopen(path, "w");
|
||||
EXPECT_NOT_NULL(fp);
|
||||
fputs("- *.tmp\n", fp);
|
||||
fclose(fp);
|
||||
|
||||
/* merge with e/n/w/- consumes the modifiers and reads the right file. */
|
||||
static const char* const fmts[] = {
|
||||
"merge,e %s", "merge,n %s", "merge,w %s", "merge,- %s", ".e %s", ".- %s",
|
||||
};
|
||||
for (size_t i = 0; i < sizeof(fmts) / sizeof(fmts[0]); i++) {
|
||||
FilterRuleList* list = filter_rule_list_create();
|
||||
EXPECT_NOT_NULL(list);
|
||||
char rule[600];
|
||||
char err[256] = "";
|
||||
snprintf(rule, sizeof(rule), fmts[i], path);
|
||||
bool ok = filter_rule_list_parse_append(list, rule, NULL, NULL, err, sizeof(err));
|
||||
if (!ok)
|
||||
printf(" merge rule '%s' errored: %s\n", rule, err);
|
||||
EXPECT_TRUE(ok);
|
||||
EXPECT_EQ_INT(list->count, 1);
|
||||
EXPECT_EQ_STR(list->items[0]->pattern, "*.tmp");
|
||||
filter_rule_list_free(list);
|
||||
}
|
||||
|
||||
/* dir-merge with e/n/w/- registers the basename without the modifiers. */
|
||||
static const struct {
|
||||
const char* rule;
|
||||
const char* want;
|
||||
} drules[] = {
|
||||
{"dir-merge,e .rules", ".rules"}, {"dir-merge,n .rules", ".rules"},
|
||||
{"dir-merge,w .rules", ".rules"}, {"dir-merge,- .rules", ".rules"},
|
||||
{":e .rules", ".rules"}, {":- .rules", ".rules"},
|
||||
};
|
||||
for (size_t i = 0; i < sizeof(drules) / sizeof(drules[0]); i++) {
|
||||
FilterRuleList* list = filter_rule_list_create();
|
||||
EXPECT_NOT_NULL(list);
|
||||
char err[256] = "";
|
||||
bool ok = filter_rule_list_parse_append(list, drules[i].rule, NULL, NULL, err, sizeof(err));
|
||||
if (!ok)
|
||||
printf(" dir-merge rule '%s' errored: %s\n", drules[i].rule, err);
|
||||
EXPECT_TRUE(ok);
|
||||
EXPECT_EQ_INT(list->dir_merge_count, 1);
|
||||
EXPECT_EQ_STR(list->dir_merge_names[0], drules[i].want);
|
||||
filter_rule_list_free(list);
|
||||
}
|
||||
|
||||
unlink(path);
|
||||
rmdir(tmpl);
|
||||
}
|
||||
|
||||
static void test_filter_list_accepts_supported_rules_and_modifiers() {
|
||||
static const char* const rules[] = {
|
||||
"- *.tmp", "+ /a.txt", "-s foo", "-r foo", "-p foo",
|
||||
"-! *.o", "-/ foo", "hide *.tmp", "show *.txt", "protect *.bak",
|
||||
"risk *.o", "dir-merge .rules", "-C",
|
||||
};
|
||||
for (size_t i = 0; i < sizeof(rules) / sizeof(rules[0]); i++) {
|
||||
FilterRuleList* list = filter_rule_list_create();
|
||||
EXPECT_NOT_NULL(list);
|
||||
char err[256] = "";
|
||||
bool ok = filter_rule_list_parse_append(list, rules[i], NULL, NULL, err, sizeof(err));
|
||||
if (!ok)
|
||||
printf(" rule '%s' errored: %s\n", rules[i], err);
|
||||
EXPECT_TRUE(ok);
|
||||
filter_rule_list_free(list);
|
||||
}
|
||||
|
||||
/* A glued word is a pattern, not a modifier run (no separator). */
|
||||
{
|
||||
FilterRuleList* list = filter_rule_list_create();
|
||||
char err[128] = "";
|
||||
EXPECT_TRUE(filter_rule_list_parse_append(list, "-newfile", NULL, NULL, err, sizeof(err)));
|
||||
EXPECT_EQ_INT(list->count, 1);
|
||||
EXPECT_EQ_STR(list->items[0]->pattern, "newfile");
|
||||
filter_rule_list_free(list);
|
||||
|
||||
list = filter_rule_list_create();
|
||||
EXPECT_TRUE(filter_rule_list_parse_append(list, "-e2e", NULL, NULL, err, sizeof(err)));
|
||||
EXPECT_EQ_INT(list->count, 1);
|
||||
EXPECT_EQ_STR(list->items[0]->pattern, "e2e");
|
||||
filter_rule_list_free(list);
|
||||
|
||||
list = filter_rule_list_create();
|
||||
EXPECT_TRUE(filter_rule_list_parse_append(list, "-*.o", NULL, NULL, err, sizeof(err)));
|
||||
EXPECT_EQ_INT(list->count, 1);
|
||||
EXPECT_EQ_STR(list->items[0]->pattern, "*.o");
|
||||
filter_rule_list_free(list);
|
||||
|
||||
/* A comma with no modifier still treats the rest as the pattern. */
|
||||
list = filter_rule_list_create();
|
||||
EXPECT_TRUE(filter_rule_list_parse_append(list, "exclude,foo", NULL, NULL, err, sizeof(err)));
|
||||
EXPECT_EQ_INT(list->count, 1);
|
||||
EXPECT_EQ_STR(list->items[0]->pattern, "foo");
|
||||
filter_rule_list_free(list);
|
||||
}
|
||||
|
||||
/* `!` clears the list. */
|
||||
{
|
||||
FilterRuleList* list = filter_rule_list_create();
|
||||
char err[128] = "";
|
||||
EXPECT_TRUE(filter_rule_list_parse_append(list, "- *.tmp", NULL, NULL, err, sizeof(err)));
|
||||
EXPECT_EQ_INT(list->count, 1);
|
||||
EXPECT_TRUE(filter_rule_list_parse_append(list, "!", NULL, NULL, err, sizeof(err)));
|
||||
EXPECT_EQ_INT(list->count, 0);
|
||||
filter_rule_list_free(list);
|
||||
}
|
||||
|
||||
/* -C injects the CVS defaults. */
|
||||
{
|
||||
FilterRuleList* list = filter_rule_list_create();
|
||||
char err[128] = "";
|
||||
EXPECT_TRUE(filter_rule_list_parse_append(list, "-C", NULL, NULL, err, sizeof(err)));
|
||||
EXPECT_TRUE(list->count > 0);
|
||||
filter_rule_list_free(list);
|
||||
}
|
||||
}
|
||||
|
||||
static void test_filter_list_merge_file_still_supported() {
|
||||
char tmpl[] = "/tmp/fastsync_filter_XXXXXX";
|
||||
EXPECT_TRUE(mkdtemp(tmpl) != NULL);
|
||||
char path[512];
|
||||
snprintf(path, sizeof(path), "%s/rules", tmpl);
|
||||
FILE* fp = fopen(path, "w");
|
||||
EXPECT_NOT_NULL(fp);
|
||||
fputs("- *.tmp\n", fp);
|
||||
fclose(fp);
|
||||
|
||||
FilterRuleList* list = filter_rule_list_create();
|
||||
char err[256] = "";
|
||||
char rule[600];
|
||||
snprintf(rule, sizeof(rule), "merge %s", path);
|
||||
EXPECT_TRUE(filter_rule_list_parse_append(list, rule, NULL, NULL, err, sizeof(err)));
|
||||
EXPECT_EQ_INT(list->count, 1);
|
||||
filter_rule_list_free(list);
|
||||
|
||||
/* The same merge with the x modifier is rejected, not silently read. */
|
||||
list = filter_rule_list_create();
|
||||
snprintf(rule, sizeof(rule), "merge,x %s", path);
|
||||
EXPECT_FALSE(filter_rule_list_parse_append(list, rule, NULL, NULL, err, sizeof(err)));
|
||||
EXPECT_TRUE(strstr(err, "xattr") != NULL);
|
||||
filter_rule_list_free(list);
|
||||
|
||||
unlink(path);
|
||||
rmdir(tmpl);
|
||||
}
|
||||
|
||||
static void test_filter_rule_parse_rejects_unsupported_and_keeps_supported() {
|
||||
char err[256] = "";
|
||||
|
||||
EXPECT_NULL(filter_rule_parse("-x user.foo", NULL, err, sizeof(err)));
|
||||
EXPECT_TRUE(strstr(err, "xattr") != NULL);
|
||||
|
||||
EXPECT_NULL(filter_rule_parse("-e foo", NULL, err, sizeof(err)));
|
||||
EXPECT_TRUE(strstr(err, "unsupported filter modifier") != NULL);
|
||||
|
||||
FilterRule* rule = filter_rule_parse("- *.tmp", NULL, err, sizeof(err));
|
||||
EXPECT_NOT_NULL(rule);
|
||||
EXPECT_EQ_STR(rule->pattern, "*.tmp");
|
||||
filter_rule_free(rule);
|
||||
|
||||
rule = filter_rule_parse("-newfile", NULL, err, sizeof(err));
|
||||
EXPECT_NOT_NULL(rule);
|
||||
EXPECT_EQ_STR(rule->pattern, "newfile");
|
||||
filter_rule_free(rule);
|
||||
}
|
||||
|
||||
static void test_filter_rules_apply_supported_modifiers() {
|
||||
/* exclude */
|
||||
{
|
||||
const char* texts[] = {"- *.tmp"};
|
||||
FilterRuleList* list = filter_base_build(texts, 1, false, false, NULL, 0);
|
||||
EXPECT_NOT_NULL(list);
|
||||
EXPECT_EQ_INT(filter_rules_apply_side(list, "b.tmp", "b.tmp", false, FILTER_SIDE_SENDER),
|
||||
FILTER_ACTION_EXCLUDE);
|
||||
EXPECT_EQ_INT(filter_rules_apply_side(list, "a.txt", "a.txt", false, FILTER_SIDE_SENDER),
|
||||
FILTER_ACTION_NONE);
|
||||
filter_rule_list_free(list);
|
||||
}
|
||||
/* anchored include then exclude-all */
|
||||
{
|
||||
const char* texts[] = {"+ /a.txt", "- *"};
|
||||
FilterRuleList* list = filter_base_build(texts, 2, false, false, NULL, 0);
|
||||
EXPECT_NOT_NULL(list);
|
||||
EXPECT_EQ_INT(filter_rules_apply_side(list, "a.txt", "a.txt", false, FILTER_SIDE_SENDER),
|
||||
FILTER_ACTION_INCLUDE);
|
||||
EXPECT_EQ_INT(filter_rules_apply_side(list, "b.txt", "b.txt", false, FILTER_SIDE_SENDER),
|
||||
FILTER_ACTION_EXCLUDE);
|
||||
filter_rule_list_free(list);
|
||||
}
|
||||
/* negate */
|
||||
{
|
||||
const char* texts[] = {"-! *.o"};
|
||||
FilterRuleList* list = filter_base_build(texts, 1, false, false, NULL, 0);
|
||||
EXPECT_NOT_NULL(list);
|
||||
EXPECT_EQ_INT(filter_rules_apply_side(list, "foo.c", "foo.c", false, FILTER_SIDE_SENDER),
|
||||
FILTER_ACTION_EXCLUDE);
|
||||
EXPECT_EQ_INT(filter_rules_apply_side(list, "foo.o", "foo.o", false, FILTER_SIDE_SENDER),
|
||||
FILTER_ACTION_NONE);
|
||||
filter_rule_list_free(list);
|
||||
}
|
||||
/* dir-only trailing slash */
|
||||
{
|
||||
const char* texts[] = {"+ dir/", "- *"};
|
||||
FilterRuleList* list = filter_base_build(texts, 2, false, false, NULL, 0);
|
||||
EXPECT_NOT_NULL(list);
|
||||
EXPECT_EQ_INT(filter_rules_apply_side(list, "dir", "dir", true, FILTER_SIDE_SENDER),
|
||||
FILTER_ACTION_INCLUDE);
|
||||
EXPECT_EQ_INT(filter_rules_apply_side(list, "dir", "dir", false, FILTER_SIDE_SENDER),
|
||||
FILTER_ACTION_EXCLUDE);
|
||||
filter_rule_list_free(list);
|
||||
}
|
||||
}
|
||||
|
||||
void test_filter() {
|
||||
test_filter_list_rejects_xattr_modifier();
|
||||
test_filter_list_rejects_unsupported_modifiers();
|
||||
test_filter_list_accepts_merge_modifiers();
|
||||
test_filter_list_accepts_supported_rules_and_modifiers();
|
||||
test_filter_list_merge_file_still_supported();
|
||||
test_filter_rule_parse_rejects_unsupported_and_keeps_supported();
|
||||
test_filter_rules_apply_supported_modifiers();
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
#ifndef TEST_FILTER_H
|
||||
#define TEST_FILTER_H
|
||||
|
||||
void test_filter(void);
|
||||
|
||||
#endif
|
||||
+66
-32
@@ -339,7 +339,7 @@ static void test_file_restore_metadata_applies_atime() {
|
||||
m.crtime_sec = 0;
|
||||
m.crtime_nsec = 0;
|
||||
|
||||
file_restore_metadata(path, &m, (FileAttrPolicy){true, true, true, false});
|
||||
file_restore_metadata(path, &m, (FileAttrPolicy){true, true, true, false, true});
|
||||
|
||||
struct stat st;
|
||||
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||
@@ -378,7 +378,7 @@ static void test_file_restore_metadata() {
|
||||
.atime_valid = false,
|
||||
.crtime_valid = false};
|
||||
|
||||
file_restore_metadata(path, &m, (FileAttrPolicy){true, true, false, false});
|
||||
file_restore_metadata(path, &m, (FileAttrPolicy){true, true, false, false, true});
|
||||
|
||||
struct stat st;
|
||||
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||
@@ -395,7 +395,7 @@ static void test_file_restore_executability_only() {
|
||||
|
||||
FileMetadata m = {
|
||||
.mode = 0751, .uid = getuid(), .gid = getgid(), .mtime_sec = 0, .mtime_nsec = 0};
|
||||
file_restore_metadata(path, &m, (FileAttrPolicy){false, false, false, true});
|
||||
file_restore_metadata(path, &m, (FileAttrPolicy){false, false, false, true, true});
|
||||
|
||||
struct stat st;
|
||||
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||
@@ -409,7 +409,7 @@ static void test_directory_restore_executability_only() {
|
||||
|
||||
FileMetadata m = {
|
||||
.mode = 0755, .uid = getuid(), .gid = getgid(), .mtime_sec = 0, .mtime_nsec = 0};
|
||||
file_restore_metadata(path, &m, (FileAttrPolicy){false, false, false, true});
|
||||
file_restore_metadata(path, &m, (FileAttrPolicy){false, false, false, true, true});
|
||||
|
||||
struct stat st;
|
||||
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||
@@ -437,7 +437,7 @@ static void test_file_restore_executability_rsync_rule() {
|
||||
EXPECT_TRUE(file_write_to_disk(path, "x", 1, false, false));
|
||||
EXPECT_EQ_INT(chmod(path, cases[i].dest), 0);
|
||||
FileMetadata m = {.mode = cases[i].src, .uid = getuid(), .gid = getgid()};
|
||||
file_restore_metadata(path, &m, (FileAttrPolicy){false, false, false, true});
|
||||
file_restore_metadata(path, &m, (FileAttrPolicy){false, false, false, true, true});
|
||||
struct stat st;
|
||||
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||
EXPECT_EQ_INT(st.st_mode & 0777, cases[i].want);
|
||||
@@ -453,34 +453,60 @@ static void test_file_restore_executability_rsync_rule() {
|
||||
* bits from the destination and --perms wins when both are set. */
|
||||
static void test_metadata_mode_for_policy() {
|
||||
mode_t out = 0xdead;
|
||||
EXPECT_FALSE(
|
||||
metadata_mode_for_policy(0777, 0644, (FileAttrPolicy){false, false, false, false}, &out));
|
||||
EXPECT_FALSE(metadata_mode_for_policy(0777, 0644,
|
||||
(FileAttrPolicy){false, false, false, false, true}, &out));
|
||||
EXPECT_EQ_INT((int)out, 0xdead); /* untouched when no change is requested */
|
||||
|
||||
EXPECT_TRUE(
|
||||
metadata_mode_for_policy(0777, 0644, (FileAttrPolicy){true, false, false, false}, &out));
|
||||
EXPECT_TRUE(metadata_mode_for_policy(0777, 0644,
|
||||
(FileAttrPolicy){true, false, false, false, true}, &out));
|
||||
EXPECT_EQ_INT((int)(out & 0777), 0777); /* group/other write is preserved */
|
||||
|
||||
mode_t specials = (mode_t)(S_ISUID | S_ISGID | S_ISVTX | 0672);
|
||||
EXPECT_TRUE(
|
||||
metadata_mode_for_policy(specials, 0644, (FileAttrPolicy){true, false, false, false}, &out));
|
||||
EXPECT_TRUE(metadata_mode_for_policy(specials, 0644,
|
||||
(FileAttrPolicy){true, false, false, false, true}, &out));
|
||||
EXPECT_EQ_INT((int)(out & (S_ISUID | S_ISGID | S_ISVTX | 0777)),
|
||||
(int)(S_ISUID | S_ISGID | S_ISVTX | 0672));
|
||||
|
||||
/* SUPER_MODE_OFF: the special bits are stripped even under -p (this also
|
||||
* covers bits introduced by --chmod, whose result is fed in as source_mode),
|
||||
* while the ordinary permission bits are still copied. */
|
||||
EXPECT_TRUE(metadata_mode_for_policy(specials, 0644,
|
||||
(FileAttrPolicy){true, false, false, false, false}, &out));
|
||||
EXPECT_EQ_INT((int)(out & (S_ISUID | S_ISGID | S_ISVTX)), 0);
|
||||
EXPECT_EQ_INT((int)(out & 0777), 0672);
|
||||
EXPECT_TRUE(metadata_mode_for_policy(04755, 0644,
|
||||
(FileAttrPolicy){true, false, false, false, false}, &out));
|
||||
EXPECT_EQ_INT((int)(out & 07777), 0755);
|
||||
/* The same holds for a special bit introduced by --chmod=+s. */
|
||||
mode_t chmodded = 0;
|
||||
EXPECT_TRUE(chmod_apply(0755, "u+s", &chmodded));
|
||||
EXPECT_TRUE(metadata_mode_for_policy(chmodded, 0644,
|
||||
(FileAttrPolicy){true, false, false, false, false}, &out));
|
||||
EXPECT_EQ_INT((int)(out & 07777), 0755);
|
||||
|
||||
/* -E: a destination's own special bits survive unless super-user activities
|
||||
* are forbidden, in which case they are stripped from the derived base. */
|
||||
EXPECT_TRUE(metadata_mode_for_policy(0755, (mode_t)(S_ISUID | 0750),
|
||||
(FileAttrPolicy){false, false, false, true, true}, &out));
|
||||
EXPECT_EQ_INT((int)(out & (S_ISUID | 0777)), (int)(S_ISUID | 0750));
|
||||
EXPECT_TRUE(metadata_mode_for_policy(0755, (mode_t)(S_ISUID | 0750),
|
||||
(FileAttrPolicy){false, false, false, true, false}, &out));
|
||||
EXPECT_EQ_INT((int)(out & (S_ISUID | 0777)), 0750);
|
||||
|
||||
/* -E: exec bits derive from the DESTINATION's read bits. */
|
||||
EXPECT_TRUE(
|
||||
metadata_mode_for_policy(0755, 0644, (FileAttrPolicy){false, false, false, true}, &out));
|
||||
EXPECT_TRUE(metadata_mode_for_policy(0755, 0644,
|
||||
(FileAttrPolicy){false, false, false, true, true}, &out));
|
||||
EXPECT_EQ_INT((int)(out & 0777), 0755);
|
||||
EXPECT_TRUE(
|
||||
metadata_mode_for_policy(0644, 0755, (FileAttrPolicy){false, false, false, true}, &out));
|
||||
EXPECT_TRUE(metadata_mode_for_policy(0644, 0755,
|
||||
(FileAttrPolicy){false, false, false, true, true}, &out));
|
||||
EXPECT_EQ_INT((int)(out & 0777), 0644);
|
||||
EXPECT_TRUE(
|
||||
metadata_mode_for_policy(0755, 0600, (FileAttrPolicy){false, false, false, true}, &out));
|
||||
EXPECT_TRUE(metadata_mode_for_policy(0755, 0600,
|
||||
(FileAttrPolicy){false, false, false, true, true}, &out));
|
||||
EXPECT_EQ_INT((int)(out & 0777), 0700);
|
||||
|
||||
/* --perms wins over -E when both are set. */
|
||||
EXPECT_TRUE(
|
||||
metadata_mode_for_policy(0700, 0644, (FileAttrPolicy){true, false, false, true}, &out));
|
||||
metadata_mode_for_policy(0700, 0644, (FileAttrPolicy){true, false, false, true, true}, &out));
|
||||
EXPECT_EQ_INT((int)(out & 0777), 0700);
|
||||
}
|
||||
|
||||
@@ -493,8 +519,8 @@ static void test_new_file_mode_from_source_and_umask() {
|
||||
mode_t want = (mode_t)(0751 & 0777 & ~(mode_t)file_process_umask());
|
||||
|
||||
bool ok = file_to_disk_secure_attrs(path, "x", 1, false, false, false, &m,
|
||||
(FileAttrPolicy){false, false, false, false}, false, false,
|
||||
false, NULL, false, false, NULL);
|
||||
(FileAttrPolicy){false, false, false, false, true}, false,
|
||||
false, false, NULL, false, false, NULL);
|
||||
EXPECT_TRUE(ok);
|
||||
struct stat st;
|
||||
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||
@@ -503,8 +529,8 @@ static void test_new_file_mode_from_source_and_umask() {
|
||||
|
||||
/* -E on top of the source&~umask base (src 0751, umask 022 -> 0751). */
|
||||
ok = file_to_disk_secure_attrs(path, "x", 1, false, false, false, &m,
|
||||
(FileAttrPolicy){false, false, false, true}, false, false, false,
|
||||
NULL, false, false, NULL);
|
||||
(FileAttrPolicy){false, false, false, true, true}, false, false,
|
||||
false, NULL, false, false, NULL);
|
||||
EXPECT_TRUE(ok);
|
||||
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||
mode_t want_e =
|
||||
@@ -529,7 +555,7 @@ static void test_file_restore_attribute_split() {
|
||||
.crtime_valid = false};
|
||||
|
||||
/* times only: mtime changes, mode stays 0640. */
|
||||
file_restore_metadata(path, &m, (FileAttrPolicy){false, true, false, false});
|
||||
file_restore_metadata(path, &m, (FileAttrPolicy){false, true, false, false, true});
|
||||
struct stat st;
|
||||
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||
EXPECT_EQ_INT(st.st_mode & 0777, 0640);
|
||||
@@ -543,7 +569,7 @@ static void test_file_restore_attribute_split() {
|
||||
FileMetadata m2 = m;
|
||||
m2.mode = 0700;
|
||||
m2.mtime_sec = 1600000000;
|
||||
file_restore_metadata(path, &m2, (FileAttrPolicy){false, false, false, false});
|
||||
file_restore_metadata(path, &m2, (FileAttrPolicy){false, false, false, false, true});
|
||||
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||
EXPECT_EQ_INT(st.st_mode & 0777, 0640);
|
||||
EXPECT_EQ_INT((int)st.st_mtime, 1000000000);
|
||||
@@ -569,6 +595,11 @@ static void test_file_attr_policy_from_config() {
|
||||
EXPECT_TRUE(p.times);
|
||||
EXPECT_TRUE(p.atimes);
|
||||
EXPECT_TRUE(p.executability);
|
||||
/* Default super mode (AUTO) permits special bits. */
|
||||
EXPECT_TRUE(p.super_permitted);
|
||||
c->super_mode = SUPER_MODE_OFF;
|
||||
p = file_attr_policy_from_config(c);
|
||||
EXPECT_FALSE(p.super_permitted);
|
||||
config_delete(c);
|
||||
}
|
||||
|
||||
@@ -582,8 +613,8 @@ static void test_perms_preserves_special_bits() {
|
||||
.mode = (mode_t)(S_ISUID | S_ISGID | S_ISVTX | 0755), .uid = getuid(), .gid = getgid()};
|
||||
|
||||
bool ok = file_to_disk_secure_attrs(path, "x", 1, false, false, false, &m,
|
||||
(FileAttrPolicy){true, false, false, false}, false, false,
|
||||
false, NULL, false, false, NULL);
|
||||
(FileAttrPolicy){true, false, false, false, true}, false,
|
||||
false, false, NULL, false, false, NULL);
|
||||
EXPECT_TRUE(ok);
|
||||
struct stat st;
|
||||
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||
@@ -670,7 +701,8 @@ static void test_file_restore_symlink_metadata() {
|
||||
|
||||
/* Positive path: a non-omitted apply stamps the link's own mtime. */
|
||||
FileMetadata applied = {.mtime_sec = 1000000000, .mtime_nsec = 0};
|
||||
file_restore_symlink_metadata(link, &applied, (FileAttrPolicy){false, true, false, false}, false);
|
||||
file_restore_symlink_metadata(link, &applied, (FileAttrPolicy){false, true, false, false, true},
|
||||
false);
|
||||
struct stat st;
|
||||
EXPECT_EQ_INT(lstat(link, &st), 0);
|
||||
EXPECT_TRUE(S_ISLNK(st.st_mode));
|
||||
@@ -679,7 +711,8 @@ static void test_file_restore_symlink_metadata() {
|
||||
|
||||
/* -J: a different time must be left untouched. */
|
||||
FileMetadata newer = {.mtime_sec = 1234567890, .mtime_nsec = 0};
|
||||
file_restore_symlink_metadata(link, &newer, (FileAttrPolicy){false, true, false, false}, true);
|
||||
file_restore_symlink_metadata(link, &newer, (FileAttrPolicy){false, true, false, false, true},
|
||||
true);
|
||||
EXPECT_EQ_INT(lstat(link, &st), 0);
|
||||
EXPECT_EQ_INT((int)st.st_mtime, (int)t1);
|
||||
if (symlink_times_supported)
|
||||
@@ -723,14 +756,14 @@ static void test_file_restore_metadata_fd_attribute_split() {
|
||||
|
||||
/* perms-only: mode applied, mtime untouched. */
|
||||
EXPECT_EQ_INT(fstat(fd, &before), 0);
|
||||
EXPECT_TRUE(file_restore_metadata_fd(fd, &m, (FileAttrPolicy){true, false, false, false}));
|
||||
EXPECT_TRUE(file_restore_metadata_fd(fd, &m, (FileAttrPolicy){true, false, false, false, true}));
|
||||
EXPECT_EQ_INT(fstat(fd, &st), 0);
|
||||
EXPECT_EQ_INT(st.st_mode & 0777, 0755);
|
||||
EXPECT_EQ_INT((int)st.st_mtime, (int)before.st_mtime);
|
||||
|
||||
/* times-only: mtime applied, mode untouched. */
|
||||
EXPECT_EQ_INT(chmod(path, 0600), 0);
|
||||
EXPECT_TRUE(file_restore_metadata_fd(fd, &m, (FileAttrPolicy){false, true, false, false}));
|
||||
EXPECT_TRUE(file_restore_metadata_fd(fd, &m, (FileAttrPolicy){false, true, false, false, true}));
|
||||
EXPECT_EQ_INT(fstat(fd, &st), 0);
|
||||
EXPECT_EQ_INT(st.st_mode & 0777, 0600);
|
||||
EXPECT_EQ_INT((int)st.st_mtime, 1234567890);
|
||||
@@ -740,7 +773,7 @@ static void test_file_restore_metadata_fd_attribute_split() {
|
||||
{.tv_sec = 1000000000, .tv_nsec = 0}};
|
||||
EXPECT_EQ_INT(futimens(fd, reset), 0);
|
||||
EXPECT_EQ_INT(fstat(fd, &before), 0);
|
||||
EXPECT_TRUE(file_restore_metadata_fd(fd, &m, (FileAttrPolicy){false, false, true, false}));
|
||||
EXPECT_TRUE(file_restore_metadata_fd(fd, &m, (FileAttrPolicy){false, false, true, false, true}));
|
||||
EXPECT_EQ_INT(fstat(fd, &st), 0);
|
||||
EXPECT_EQ_INT((int)st.st_atime, 999999999);
|
||||
EXPECT_EQ_INT((int)st.st_mtime, (int)before.st_mtime);
|
||||
@@ -753,7 +786,8 @@ static void test_file_restore_metadata_fd_attribute_split() {
|
||||
m2.mode = 0700;
|
||||
m2.mtime_sec = 1600000000;
|
||||
m2.atime_sec = 1700000000;
|
||||
EXPECT_TRUE(file_restore_metadata_fd(fd, &m2, (FileAttrPolicy){false, false, false, false}));
|
||||
EXPECT_TRUE(
|
||||
file_restore_metadata_fd(fd, &m2, (FileAttrPolicy){false, false, false, false, true}));
|
||||
EXPECT_EQ_INT(fstat(fd, &st), 0);
|
||||
EXPECT_EQ_INT(st.st_mode & 0777, 0640);
|
||||
EXPECT_EQ_INT((int)st.st_mtime, 1000000000);
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
#include "test_utils.h"
|
||||
#include <limits.h>
|
||||
#include <string.h>
|
||||
#include <time.h>
|
||||
#include <unistd.h>
|
||||
#include <threads.h>
|
||||
|
||||
@@ -215,6 +216,38 @@ static void test_send_receive_status() {
|
||||
close(p[1]);
|
||||
}
|
||||
|
||||
/* An unknown wire status outside the enum range must be rejected as a protocol
|
||||
* error instead of being handed to the caller as an unexpected verdict. The
|
||||
* last known enumerator (STATUS_STATS) must still be accepted, proving the
|
||||
* validation does not reject legitimate statuses. */
|
||||
static void test_receive_status_rejects_unknown() {
|
||||
int p[2];
|
||||
EXPECT_EQ_INT(pipe(p), 0);
|
||||
ProtocolSession session;
|
||||
protocol_session_init(&session, p[0], p[1]);
|
||||
|
||||
Status bogus = (Status)(STATUS_STATS + 1);
|
||||
EXPECT_EQ_INT((int)write(p[1], &bogus, sizeof(bogus)), (int)sizeof(bogus));
|
||||
Status received = STATUS_OK;
|
||||
EXPECT_FALSE(protocol_receive_status(&session, &received));
|
||||
|
||||
Status negative = (Status)-1;
|
||||
EXPECT_EQ_INT((int)write(p[1], &negative, sizeof(negative)), (int)sizeof(negative));
|
||||
EXPECT_FALSE(protocol_receive_status(&session, &received));
|
||||
|
||||
Status top = STATUS_STATS;
|
||||
EXPECT_EQ_INT((int)write(p[1], &top, sizeof(top)), (int)sizeof(top));
|
||||
EXPECT_TRUE(protocol_receive_status(&session, &received));
|
||||
EXPECT_EQ_INT((int)received, (int)STATUS_STATS);
|
||||
|
||||
Status timed_bogus = (Status)(STATUS_STATS + 7);
|
||||
EXPECT_EQ_INT((int)write(p[1], &timed_bogus, sizeof(timed_bogus)), (int)sizeof(timed_bogus));
|
||||
EXPECT_FALSE(protocol_receive_status_timed(&session, &received, 5));
|
||||
|
||||
close(p[0]);
|
||||
close(p[1]);
|
||||
}
|
||||
|
||||
static void test_receive_n_data_truncated() {
|
||||
int p[2];
|
||||
EXPECT_EQ_INT(pipe(p), 0);
|
||||
@@ -669,6 +702,50 @@ static void test_receive_status_keepalive_emits() {
|
||||
close(to_peer[1]);
|
||||
}
|
||||
|
||||
/* protocol_throttle_bytes() must apply the same token-bucket pacing as the
|
||||
* buffered protocol send path, so the plaintext sendfile fast path honors
|
||||
* --bwlimit exactly like the TLS path. With bwlimit=1 MB/s the initial burst
|
||||
* is 100 KB (bwlimit/10); pacing 150 KB therefore owes ~50 KB of debt, i.e. a
|
||||
* ~50 ms sleep. */
|
||||
static void test_protocol_throttle_bytes_paces() {
|
||||
ProtocolSession session;
|
||||
protocol_session_init(&session, -1, -1);
|
||||
protocol_session_bind(&session);
|
||||
protocol_session_set_bwlimit(&session, 1000000ULL);
|
||||
|
||||
struct timespec start;
|
||||
clock_gettime(CLOCK_MONOTONIC, &start);
|
||||
protocol_throttle_bytes(150000);
|
||||
struct timespec now;
|
||||
clock_gettime(CLOCK_MONOTONIC, &now);
|
||||
long long elapsed_ms =
|
||||
(now.tv_sec - start.tv_sec) * 1000LL + (now.tv_nsec - start.tv_nsec) / 1000000LL;
|
||||
/* Allow for scheduler slack but require the bulk of the expected 50 ms. */
|
||||
EXPECT_TRUE(elapsed_ms >= 40);
|
||||
|
||||
protocol_session_unbind();
|
||||
}
|
||||
|
||||
/* With no bandwidth limit the primitive must not sleep, however many bytes it
|
||||
* is handed. */
|
||||
static void test_protocol_throttle_bytes_unlimited() {
|
||||
ProtocolSession session;
|
||||
protocol_session_init(&session, -1, -1);
|
||||
protocol_session_bind(&session);
|
||||
protocol_session_set_bwlimit(&session, 0);
|
||||
|
||||
struct timespec start;
|
||||
clock_gettime(CLOCK_MONOTONIC, &start);
|
||||
protocol_throttle_bytes(100000000ULL);
|
||||
struct timespec now;
|
||||
clock_gettime(CLOCK_MONOTONIC, &now);
|
||||
long long elapsed_ms =
|
||||
(now.tv_sec - start.tv_sec) * 1000LL + (now.tv_nsec - start.tv_nsec) / 1000000LL;
|
||||
EXPECT_TRUE(elapsed_ms < 2000);
|
||||
|
||||
protocol_session_unbind();
|
||||
}
|
||||
|
||||
void test_protocol() {
|
||||
test_send_receive_n_data();
|
||||
test_send_receive_n_data_zero();
|
||||
@@ -678,6 +755,7 @@ void test_protocol() {
|
||||
test_send_receive_data();
|
||||
test_send_receive_int();
|
||||
test_send_receive_status();
|
||||
test_receive_status_rejects_unknown();
|
||||
test_protocol_session_io_timeout();
|
||||
test_protocol_server_io_timeout_floor();
|
||||
test_send_receive_status_timed();
|
||||
@@ -698,4 +776,6 @@ void test_protocol() {
|
||||
test_protocol_accounting_release_does_not_underflow();
|
||||
test_receive_data_charge_follows_owning_session();
|
||||
test_data_create_starts_uncharged_and_unowned();
|
||||
test_protocol_throttle_bytes_paces();
|
||||
test_protocol_throttle_bytes_unlimited();
|
||||
}
|
||||
|
||||
+21
-3
@@ -458,6 +458,11 @@ static void test_incremental_check_size_mismatch_full_transfer() {
|
||||
File* file = receive_incremental_check(p[0], cfg, &skipped);
|
||||
bool ok = file != NULL && !skipped && file->path != NULL && strcmp(file->path, "file.txt") == 0;
|
||||
file_destroy(file);
|
||||
/* Stay alive until the parent closes its write end so its send_data can
|
||||
never race this exit into a spurious EPIPE. */
|
||||
char drain;
|
||||
while (read(p[0], &drain, 1) > 0) {
|
||||
}
|
||||
config_delete(cfg);
|
||||
close(p[0]);
|
||||
_exit(ok ? 0 : 1);
|
||||
@@ -487,9 +492,9 @@ static void test_incremental_check_size_mismatch_full_transfer() {
|
||||
EXPECT_TRUE(send_data(p[1], body));
|
||||
data_destroy(body);
|
||||
|
||||
close(p[1]);
|
||||
int status;
|
||||
waitpid(pid, &status, 0);
|
||||
close(p[1]);
|
||||
config_delete(cfg);
|
||||
unlink(path);
|
||||
rmdir(root);
|
||||
@@ -1025,6 +1030,11 @@ static void test_incremental_check_fifo_destination_does_not_hang() {
|
||||
File* file = receive_incremental_check(p[0], cfg, &skipped);
|
||||
bool ok = file != NULL && !skipped;
|
||||
file_destroy(file);
|
||||
/* Stay alive until the parent closes its write end so its send_data can
|
||||
never race this exit into a spurious EPIPE. */
|
||||
char drain;
|
||||
while (read(p[0], &drain, 1) > 0) {
|
||||
}
|
||||
config_delete(cfg);
|
||||
close(p[0]);
|
||||
_exit(ok ? 0 : 1);
|
||||
@@ -1051,9 +1061,9 @@ static void test_incremental_check_fifo_destination_does_not_hang() {
|
||||
EXPECT_TRUE(send_data(p[1], body));
|
||||
data_destroy(body);
|
||||
|
||||
close(p[1]);
|
||||
int status;
|
||||
waitpid(pid, &status, 0);
|
||||
close(p[1]);
|
||||
config_delete(cfg);
|
||||
unlink(path);
|
||||
rmdir(root);
|
||||
@@ -1191,6 +1201,12 @@ static void test_incremental_check_basis_fifo_does_not_hang() {
|
||||
File* file = receive_incremental_check(p[0], cfg, &skipped);
|
||||
bool ok = file != NULL && !skipped;
|
||||
file_destroy(file);
|
||||
/* The parent sends the data body after the check reply and only then closes
|
||||
its write end. Stay alive until that EOF so the parent's send_data can
|
||||
never race this exit into a spurious EPIPE. */
|
||||
char drain;
|
||||
while (read(p[0], &drain, 1) > 0) {
|
||||
}
|
||||
config_delete(cfg);
|
||||
close(p[0]);
|
||||
_exit(ok ? 0 : 1);
|
||||
@@ -1222,9 +1238,11 @@ static void test_incremental_check_basis_fifo_does_not_hang() {
|
||||
EXPECT_TRUE(send_data(p[1], body));
|
||||
data_destroy(body);
|
||||
|
||||
/* Close the write end before waiting: this hands the child EOF so it can
|
||||
exit, and guarantees the child was still alive for the data write. */
|
||||
close(p[1]);
|
||||
int status;
|
||||
waitpid(pid, &status, 0);
|
||||
close(p[1]);
|
||||
config_delete(cfg);
|
||||
unlink(basis_path);
|
||||
rmdir(basis_dir);
|
||||
|
||||
+16
-31
@@ -5,13 +5,13 @@
|
||||
static void test_ssh_connect_invalid_dest_no_colon() {
|
||||
/* cppcheck-suppress constVariablePointer */
|
||||
Client* client =
|
||||
client_connect_ssh("invalid-destination-no-colon", 22, NULL, false, NULL, false, NULL, 0);
|
||||
client_connect_ssh("invalid-destination-no-colon", 22, NULL, NULL, false, NULL, 0);
|
||||
EXPECT_NULL(client);
|
||||
}
|
||||
|
||||
static void test_ssh_connect_invalid_dest_empty() {
|
||||
/* cppcheck-suppress constVariablePointer */
|
||||
Client* client = client_connect_ssh("", 22, NULL, false, NULL, false, NULL, 0);
|
||||
Client* client = client_connect_ssh("", 22, NULL, NULL, false, NULL, 0);
|
||||
EXPECT_NULL(client);
|
||||
}
|
||||
|
||||
@@ -22,7 +22,7 @@ static void test_ssh_connect_malformed() {
|
||||
setenv("PATH", "", 1);
|
||||
|
||||
/* cppcheck-suppress constVariablePointer */
|
||||
Client* client = client_connect_ssh(":", 22, NULL, false, NULL, false, NULL, 0);
|
||||
Client* client = client_connect_ssh(":", 22, NULL, NULL, false, NULL, 0);
|
||||
|
||||
if (saved_path) {
|
||||
setenv("PATH", saved_path, 1);
|
||||
@@ -38,7 +38,7 @@ static void test_ssh_connect_malformed() {
|
||||
* The function launches ssh which will fail to connect, returns a Client. */
|
||||
static void test_ssh_connect_unreachable() {
|
||||
Client* client =
|
||||
client_connect_ssh("nonexistent.invalid:/remote/path", 22, NULL, false, NULL, false, NULL, 0);
|
||||
client_connect_ssh("nonexistent.invalid:/remote/path", 22, NULL, NULL, false, NULL, 0);
|
||||
if (client != NULL) {
|
||||
client_disconnect(client);
|
||||
client_delete(client);
|
||||
@@ -47,23 +47,13 @@ static void test_ssh_connect_unreachable() {
|
||||
}
|
||||
|
||||
static void test_ssh_remote_command_argument_modes() {
|
||||
char* command = ssh_build_remote_command("fast sync; touch /tmp/pwned", false, NULL, 0);
|
||||
char* command = ssh_build_remote_command("fast sync; touch /tmp/pwned", NULL, 0);
|
||||
EXPECT_EQ_STR(command, "'fast sync; touch /tmp/pwned' --stdio");
|
||||
free(command);
|
||||
|
||||
command = ssh_build_remote_command("fast'sync", false, NULL, 0);
|
||||
command = ssh_build_remote_command("fast'sync", NULL, 0);
|
||||
EXPECT_EQ_STR(command, "'fast'\\''sync' --stdio");
|
||||
free(command);
|
||||
|
||||
/* --old-args no longer disables injection-safe quoting: the path is still one
|
||||
single-quoted word, even when it carries shell metacharacters. */
|
||||
command = ssh_build_remote_command("fast sync; touch /tmp/pwned", true, NULL, 0);
|
||||
EXPECT_EQ_STR(command, "'fast sync; touch /tmp/pwned' --stdio");
|
||||
free(command);
|
||||
|
||||
command = ssh_build_remote_command("fast'sync; rm -rf /", true, NULL, 0);
|
||||
EXPECT_EQ_STR(command, "'fast'\\''sync; rm -rf /' --stdio");
|
||||
free(command);
|
||||
}
|
||||
|
||||
/* The build for a single-word argv is [prog, six -o args, "--", user, command]. */
|
||||
@@ -120,12 +110,12 @@ static void test_ssh_build_client_argv_whitespace_command_and_port() {
|
||||
* returned and no command can run. */
|
||||
static void test_ssh_connect_rejects_option_host() {
|
||||
/* cppcheck-suppress constVariablePointer */
|
||||
Client* client = client_connect_ssh("-oProxyCommand=touch /tmp/pwned:/remote", 22, NULL, false,
|
||||
NULL, false, NULL, 0);
|
||||
Client* client =
|
||||
client_connect_ssh("-oProxyCommand=touch /tmp/pwned:/remote", 22, NULL, NULL, false, NULL, 0);
|
||||
EXPECT_NULL(client);
|
||||
client = client_connect_ssh("-evil:/remote", 22, NULL, false, NULL, false, NULL, 0);
|
||||
client = client_connect_ssh("-evil:/remote", 22, NULL, NULL, false, NULL, 0);
|
||||
EXPECT_NULL(client);
|
||||
client = client_connect_ssh("user@:/remote", 22, NULL, false, NULL, false, NULL, 0);
|
||||
client = client_connect_ssh("user@:/remote", 22, NULL, NULL, false, NULL, 0);
|
||||
EXPECT_NULL(client);
|
||||
}
|
||||
|
||||
@@ -135,13 +125,13 @@ static void test_ssh_connect_rejects_option_host() {
|
||||
* ssh_build_remote_command safety boundary for the server path. */
|
||||
static void test_ssh_remote_command_with_remote_options() {
|
||||
char* noop[] = {"--allow-delete"};
|
||||
char* command = ssh_build_remote_command("fastsync-server", false, noop, 1);
|
||||
char* command = ssh_build_remote_command("fastsync-server", noop, 1);
|
||||
EXPECT_EQ_STR(command, "'fastsync-server' --stdio '--allow-delete'");
|
||||
free(command);
|
||||
|
||||
/* Multiple options append in order, each as its own quoted word. */
|
||||
char* multi[] = {"-v", "--allow-delete"};
|
||||
command = ssh_build_remote_command("srv", false, multi, 2);
|
||||
command = ssh_build_remote_command("srv", multi, 2);
|
||||
EXPECT_EQ_STR(command, "'srv' --stdio '-v' '--allow-delete'");
|
||||
free(command);
|
||||
|
||||
@@ -150,29 +140,24 @@ static void test_ssh_remote_command_with_remote_options() {
|
||||
break out into an arbitrary remote command. */
|
||||
char* val = strdup("--x=un'der; touch /tmp/pwned");
|
||||
char* dangerous[1] = {val};
|
||||
command = ssh_build_remote_command("srv", false, dangerous, 1);
|
||||
command = ssh_build_remote_command("srv", dangerous, 1);
|
||||
EXPECT_EQ_STR(command, "'srv' --stdio '--x=un'\\''der; touch /tmp/pwned'");
|
||||
free(command);
|
||||
free(val);
|
||||
|
||||
/* --old-args still quotes both the server path and the remote options. */
|
||||
command = ssh_build_remote_command("srv", true, multi, 2);
|
||||
EXPECT_EQ_STR(command, "'srv' --stdio '-v' '--allow-delete'");
|
||||
free(command);
|
||||
}
|
||||
|
||||
/* The remote command builder refuses to forward an empty or control-character
|
||||
* remote option (defense-in-depth independent of the CLI validation). */
|
||||
static void test_ssh_remote_command_rejects_bad_options() {
|
||||
char* empty[] = {""};
|
||||
EXPECT_NULL(ssh_build_remote_command("srv", false, empty, 1));
|
||||
EXPECT_NULL(ssh_build_remote_command("srv", empty, 1));
|
||||
|
||||
char nl = '\n';
|
||||
char* newline[] = {&nl};
|
||||
EXPECT_NULL(ssh_build_remote_command("srv", false, newline, 1));
|
||||
EXPECT_NULL(ssh_build_remote_command("srv", newline, 1));
|
||||
|
||||
char* with_null[] = {NULL};
|
||||
EXPECT_NULL(ssh_build_remote_command("srv", false, with_null, 1));
|
||||
EXPECT_NULL(ssh_build_remote_command("srv", with_null, 1));
|
||||
}
|
||||
|
||||
void test_transport_ssh() {
|
||||
|
||||
+3
-3
@@ -248,7 +248,7 @@ static void test_link_copy_fallback_preserves_xattrs() {
|
||||
m.crtime_valid = false;
|
||||
|
||||
bool ok = file_to_disk_secure_link_attrs(dest, basis_dir, "payload", 7, false, &m,
|
||||
(FileAttrPolicy){true, true, false, false}, false,
|
||||
(FileAttrPolicy){true, true, false, false, true}, false,
|
||||
xattrs, true, NULL);
|
||||
xattr_list_free(xattrs);
|
||||
EXPECT_TRUE(ok);
|
||||
@@ -369,7 +369,7 @@ static void test_fake_super_restore() {
|
||||
}
|
||||
|
||||
/* No xattr present yet: restore is a silent no-op (returns false, no crash). */
|
||||
FileAttrPolicy policy = {true, true, false, false};
|
||||
FileAttrPolicy policy = {true, true, false, false, true};
|
||||
EXPECT_FALSE(fake_super_restore_fd(fd, policy));
|
||||
|
||||
fake_super_store_fd(fd, 1001, 1002, 0751, 1700000000, 123456789);
|
||||
@@ -423,7 +423,7 @@ static void test_fake_super_no_real_chown() {
|
||||
fake_super_store_fd(fd, 12345, 12346, 0755, 1700000000, 0);
|
||||
|
||||
Config* c = config_create();
|
||||
FileAttrPolicy policy = {true, true, false, false};
|
||||
FileAttrPolicy policy = {true, true, false, false, true};
|
||||
EXPECT_NOT_NULL(c);
|
||||
/* The strongest ownership request available plus permitted super mode. */
|
||||
c->preserve_owner = true;
|
||||
|
||||
Reference in New Issue
Block a user